Wire up Beamformer in AudioProcessing

R=andrew@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/38449004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@7969 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/modules/audio_processing/audio_processing.gypi

@@ -180,6 +180,7 @@
           ],
         }],
         ['rtc_use_openmax_dl==1', {
+          'defines': ['WEBRTC_BEAMFORMER'],
           'sources': [
             'beamformer/beamformer.cc',
             'beamformer/beamformer.h',

webrtc/modules/audio_processing/audio_processing_impl.cc

@@ -16,8 +16,8 @@
 #include "webrtc/common_audio/include/audio_util.h"
 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
-#include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
+#include "webrtc/modules/audio_processing/beamformer/beamformer.h"
 #include "webrtc/modules/audio_processing/channel_buffer.h"
 #include "webrtc/modules/audio_processing/common.h"
 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
@@ -27,6 +27,7 @@
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
+#include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
 #include "webrtc/modules/interface/module_common_types.h"
 #include "webrtc/system_wrappers/interface/compile_assert.h"
@@ -183,7 +184,8 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config)
 #else
       use_new_agc_(config.Get<ExperimentalAgc>().enabled),
 #endif
-      transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled) {
+      transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled),
+      beamformer_enabled_(config.Get<Beamforming>().enabled) {
   echo_cancellation_ = new EchoCancellationImpl(this, crit_);
   component_list_.push_back(echo_cancellation_);
 
@@ -265,6 +267,9 @@ int AudioProcessingImpl::Initialize(int input_sample_rate_hz,
 }
 
 int AudioProcessingImpl::InitializeLocked() {
+  const int fwd_audio_buffer_channels = beamformer_enabled_ ?
+                                        fwd_in_format_.num_channels() :
+                                        fwd_out_format_.num_channels();
   render_audio_.reset(new AudioBuffer(rev_in_format_.samples_per_channel(),
                                       rev_in_format_.num_channels(),
                                       rev_proc_format_.samples_per_channel(),
@@ -273,7 +278,7 @@ int AudioProcessingImpl::InitializeLocked() {
   capture_audio_.reset(new AudioBuffer(fwd_in_format_.samples_per_channel(),
                                        fwd_in_format_.num_channels(),
                                        fwd_proc_format_.samples_per_channel(),
-                                       fwd_out_format_.num_channels(),
+                                       fwd_audio_buffer_channels,
                                        fwd_out_format_.samples_per_channel()));
 
   // Initialize all components.
@@ -295,6 +300,8 @@ int AudioProcessingImpl::InitializeLocked() {
     return err;
   }
 
+  InitializeBeamformer();
+
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     int err = WriteInitMessage();
@@ -392,7 +399,10 @@ int AudioProcessingImpl::MaybeInitializeLocked(int input_sample_rate_hz,
       num_reverse_channels == rev_in_format_.num_channels()) {
     return kNoError;
   }
-
+  if (beamformer_enabled_ &&
+      (num_input_channels < 2 || num_output_channels > 1)) {
+    return kBadNumberChannelsError;
+  }
   return InitializeLocked(input_sample_rate_hz,
                           output_sample_rate_hz,
                           reverse_sample_rate_hz,
@@ -593,6 +603,18 @@ int AudioProcessingImpl::ProcessStreamLocked() {
     ca->SplitIntoFrequencyBands();
   }
 
+#ifdef WEBRTC_BEAMFORMER
+  if (beamformer_enabled_) {
+    beamformer_->ProcessChunk(ca->split_channels_const_f(kBand0To8kHz),
+                              ca->split_channels_const_f(kBand8To16kHz),
+                              ca->num_channels(),
+                              ca->samples_per_split_channel(),
+                              ca->split_channels_f(kBand0To8kHz),
+                              ca->split_channels_f(kBand8To16kHz));
+    ca->set_num_channels(1);
+  }
+#endif
+
   RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca));
   RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca));
   RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca));
@@ -894,6 +916,10 @@ VoiceDetection* AudioProcessingImpl::voice_detection() const {
 }
 
 bool AudioProcessingImpl::is_data_processed() const {
+  if (beamformer_enabled_) {
+    return true;
+  }
+
   int enabled_count = 0;
   std::list<ProcessingComponent*>::const_iterator it;
   for (it = component_list_.begin(); it != component_list_.end(); it++) {
@@ -966,6 +992,20 @@ int AudioProcessingImpl::InitializeTransient() {
   return kNoError;
 }
 
+void AudioProcessingImpl::InitializeBeamformer() {
+  if (beamformer_enabled_) {
+#ifdef WEBRTC_BEAMFORMER
+    // TODO(aluebs): Don't use a hard-coded microphone spacing.
+    beamformer_.reset(new Beamformer(kChunkSizeMs,
+                                     split_rate_,
+                                     fwd_in_format_.num_channels(),
+                                     0.05f));
+#else
+    assert(false);
+#endif
+  }
+}
+
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
 int AudioProcessingImpl::WriteMessageToDebugFile() {
   int32_t size = event_msg_->ByteSize();

webrtc/modules/audio_processing/audio_processing_impl.h

@@ -23,6 +23,7 @@ namespace webrtc {
 
 class AgcManagerDirect;
 class AudioBuffer;
+class Beamformer;
 class CriticalSectionWrapper;
 class EchoCancellationImpl;
 class EchoControlMobileImpl;
@@ -168,6 +169,7 @@ class AudioProcessingImpl : public AudioProcessing {
   bool analysis_needed(bool is_data_processed) const;
   int InitializeExperimentalAgc() EXCLUSIVE_LOCKS_REQUIRED(crit_);
   int InitializeTransient() EXCLUSIVE_LOCKS_REQUIRED(crit_);
+  void InitializeBeamformer() EXCLUSIVE_LOCKS_REQUIRED(crit_);
 
   EchoCancellationImpl* echo_cancellation_;
   EchoControlMobileImpl* echo_control_mobile_;
@@ -215,6 +217,8 @@ class AudioProcessingImpl : public AudioProcessing {
 
   bool transient_suppressor_enabled_;
   scoped_ptr<TransientSuppressor> transient_suppressor_;
+  const bool beamformer_enabled_;
+  scoped_ptr<Beamformer> beamformer_;
 };
 
 }  // namespace webrtc

webrtc/modules/audio_processing/include/audio_processing.h

@@ -82,6 +82,14 @@ struct ExperimentalNs {
   bool enabled;
 };
 
+// Use to enable beamforming. Must be provided through the constructor. It will
+// have no impact if used with AudioProcessing::SetExtraOptions().
+struct Beamforming {
+  Beamforming() : enabled(false) {}
+  explicit Beamforming(bool enabled) : enabled(enabled) {}
+  bool enabled;
+};
+
 static const int kAudioProcMaxNativeSampleRateHz = 32000;
 
 // The Audio Processing Module (APM) provides a collection of voice processing