From ae643ce280876d3b1ebcf0cc628f27ba7116b534 Mon Sep 17 00:00:00 2001 From: "aluebs@webrtc.org" Date: Fri, 19 Dec 2014 19:57:34 +0000 Subject: [PATCH] Wire up Beamformer in AudioProcessing R=andrew@webrtc.org Review URL: https://webrtc-codereview.appspot.com/38449004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@7969 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../audio_processing/audio_processing.gypi | 1 + .../audio_processing/audio_processing_impl.cc | 48 +++++++++++++++++-- .../audio_processing/audio_processing_impl.h | 4 ++ .../include/audio_processing.h | 8 ++++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/webrtc/modules/audio_processing/audio_processing.gypi b/webrtc/modules/audio_processing/audio_processing.gypi index a2c26a3ed..af6a7ed4a 100644 --- a/webrtc/modules/audio_processing/audio_processing.gypi +++ b/webrtc/modules/audio_processing/audio_processing.gypi @@ -180,6 +180,7 @@ ], }], ['rtc_use_openmax_dl==1', { + 'defines': ['WEBRTC_BEAMFORMER'], 'sources': [ 'beamformer/beamformer.cc', 'beamformer/beamformer.h', diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index 3ce84fb0b..086380e40 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -16,8 +16,8 @@ #include "webrtc/common_audio/include/audio_util.h" #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" -#include "webrtc/modules/audio_processing/transient/transient_suppressor.h" #include "webrtc/modules/audio_processing/audio_buffer.h" +#include "webrtc/modules/audio_processing/beamformer/beamformer.h" #include "webrtc/modules/audio_processing/channel_buffer.h" #include "webrtc/modules/audio_processing/common.h" #include "webrtc/modules/audio_processing/echo_cancellation_impl.h" @@ -27,6 +27,7 @@ #include "webrtc/modules/audio_processing/level_estimator_impl.h" #include "webrtc/modules/audio_processing/noise_suppression_impl.h" #include "webrtc/modules/audio_processing/processing_component.h" +#include "webrtc/modules/audio_processing/transient/transient_suppressor.h" #include "webrtc/modules/audio_processing/voice_detection_impl.h" #include "webrtc/modules/interface/module_common_types.h" #include "webrtc/system_wrappers/interface/compile_assert.h" @@ -183,7 +184,8 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config) #else use_new_agc_(config.Get().enabled), #endif - transient_suppressor_enabled_(config.Get().enabled) { + transient_suppressor_enabled_(config.Get().enabled), + beamformer_enabled_(config.Get().enabled) { echo_cancellation_ = new EchoCancellationImpl(this, crit_); component_list_.push_back(echo_cancellation_); @@ -265,6 +267,9 @@ int AudioProcessingImpl::Initialize(int input_sample_rate_hz, } int AudioProcessingImpl::InitializeLocked() { + const int fwd_audio_buffer_channels = beamformer_enabled_ ? + fwd_in_format_.num_channels() : + fwd_out_format_.num_channels(); render_audio_.reset(new AudioBuffer(rev_in_format_.samples_per_channel(), rev_in_format_.num_channels(), rev_proc_format_.samples_per_channel(), @@ -273,7 +278,7 @@ int AudioProcessingImpl::InitializeLocked() { capture_audio_.reset(new AudioBuffer(fwd_in_format_.samples_per_channel(), fwd_in_format_.num_channels(), fwd_proc_format_.samples_per_channel(), - fwd_out_format_.num_channels(), + fwd_audio_buffer_channels, fwd_out_format_.samples_per_channel())); // Initialize all components. @@ -295,6 +300,8 @@ int AudioProcessingImpl::InitializeLocked() { return err; } + InitializeBeamformer(); + #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP if (debug_file_->Open()) { int err = WriteInitMessage(); @@ -392,7 +399,10 @@ int AudioProcessingImpl::MaybeInitializeLocked(int input_sample_rate_hz, num_reverse_channels == rev_in_format_.num_channels()) { return kNoError; } - + if (beamformer_enabled_ && + (num_input_channels < 2 || num_output_channels > 1)) { + return kBadNumberChannelsError; + } return InitializeLocked(input_sample_rate_hz, output_sample_rate_hz, reverse_sample_rate_hz, @@ -593,6 +603,18 @@ int AudioProcessingImpl::ProcessStreamLocked() { ca->SplitIntoFrequencyBands(); } +#ifdef WEBRTC_BEAMFORMER + if (beamformer_enabled_) { + beamformer_->ProcessChunk(ca->split_channels_const_f(kBand0To8kHz), + ca->split_channels_const_f(kBand8To16kHz), + ca->num_channels(), + ca->samples_per_split_channel(), + ca->split_channels_f(kBand0To8kHz), + ca->split_channels_f(kBand8To16kHz)); + ca->set_num_channels(1); + } +#endif + RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); @@ -894,6 +916,10 @@ VoiceDetection* AudioProcessingImpl::voice_detection() const { } bool AudioProcessingImpl::is_data_processed() const { + if (beamformer_enabled_) { + return true; + } + int enabled_count = 0; std::list::const_iterator it; for (it = component_list_.begin(); it != component_list_.end(); it++) { @@ -966,6 +992,20 @@ int AudioProcessingImpl::InitializeTransient() { return kNoError; } +void AudioProcessingImpl::InitializeBeamformer() { + if (beamformer_enabled_) { +#ifdef WEBRTC_BEAMFORMER + // TODO(aluebs): Don't use a hard-coded microphone spacing. + beamformer_.reset(new Beamformer(kChunkSizeMs, + split_rate_, + fwd_in_format_.num_channels(), + 0.05f)); +#else + assert(false); +#endif + } +} + #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP int AudioProcessingImpl::WriteMessageToDebugFile() { int32_t size = event_msg_->ByteSize(); diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index be70273e6..152d64cd9 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -23,6 +23,7 @@ namespace webrtc { class AgcManagerDirect; class AudioBuffer; +class Beamformer; class CriticalSectionWrapper; class EchoCancellationImpl; class EchoControlMobileImpl; @@ -168,6 +169,7 @@ class AudioProcessingImpl : public AudioProcessing { bool analysis_needed(bool is_data_processed) const; int InitializeExperimentalAgc() EXCLUSIVE_LOCKS_REQUIRED(crit_); int InitializeTransient() EXCLUSIVE_LOCKS_REQUIRED(crit_); + void InitializeBeamformer() EXCLUSIVE_LOCKS_REQUIRED(crit_); EchoCancellationImpl* echo_cancellation_; EchoControlMobileImpl* echo_control_mobile_; @@ -215,6 +217,8 @@ class AudioProcessingImpl : public AudioProcessing { bool transient_suppressor_enabled_; scoped_ptr transient_suppressor_; + const bool beamformer_enabled_; + scoped_ptr beamformer_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h index 1cca0edf5..79340aeee 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.h +++ b/webrtc/modules/audio_processing/include/audio_processing.h @@ -82,6 +82,14 @@ struct ExperimentalNs { bool enabled; }; +// Use to enable beamforming. Must be provided through the constructor. It will +// have no impact if used with AudioProcessing::SetExtraOptions(). +struct Beamforming { + Beamforming() : enabled(false) {} + explicit Beamforming(bool enabled) : enabled(enabled) {} + bool enabled; +}; + static const int kAudioProcMaxNativeSampleRateHz = 32000; // The Audio Processing Module (APM) provides a collection of voice processing