From 40ee3d07eda24b8e8214429d9885d9ad9a2c04f7 Mon Sep 17 00:00:00 2001 From: "andrew@webrtc.org" Date: Thu, 3 Apr 2014 21:56:01 +0000 Subject: [PATCH] Consolidate audio conversion from Channel and TransmitMixer. Replace the two versions with a single DownConvertToCodecFormat. As mentioned in comments, this could be further consolidated with RemixAndResample but we should write a full audio converter class in that case. Along the way: - Fix the bug present in Channel::Demultiplex with mono input and a stereo codec. - Remove the 32 kHz max from the OnDataAvailable path. This avoids a 48 -> 32 -> 48 conversion when VoE is passed 48 kHz audio; instead we get a straight pass-through to ACM. The 32 kHz conversion is still needed in the RecordedDataIsAvailable path until APM natively supports 48 kHz. - Merge resampler improvements from ACM1 to ACM2. This allows ACM to handle 44.1 kHz audio passed to VoE and was originally done here: https://webrtc-codereview.appspot.com/1590004 - Reuse the RemixAndResample unit tests for DownConvertToCodecFormat. - Remove unused functions from utility.cc. BUG=3155,3000,b/12867572 TESTED=voe_cmd_test using both the OnDataAvailable and RecordedDataIsAvailable paths, with a captured audio format of all combinations of {44.1,48} kHz and {1,2} channels, running through all codecs, and finally using both ACM1 and ACM2. R=henrika@webrtc.org, turaj@webrtc.org, xians@webrtc.org Review URL: https://webrtc-codereview.appspot.com/11019005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@5843 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../audio_coding/main/acm2/acm_resampler.cc | 42 +- .../audio_coding/main/acm2/acm_resampler.h | 10 +- .../main/acm2/audio_coding_module_impl.cc | 8 +- .../main/source/audio_coding_module_impl.cc | 8 +- .../include/audio_processing.h | 2 + webrtc/voice_engine/channel.cc | 79 +- webrtc/voice_engine/channel.h | 2 +- webrtc/voice_engine/output_mixer.cc | 14 +- webrtc/voice_engine/output_mixer_internal.cc | 70 - webrtc/voice_engine/output_mixer_internal.h | 33 - webrtc/voice_engine/output_mixer_unittest.cc | 66 +- webrtc/voice_engine/transmit_mixer.cc | 2797 ++++++++--------- webrtc/voice_engine/transmit_mixer.h | 14 +- webrtc/voice_engine/utility.cc | 228 +- webrtc/voice_engine/utility.h | 65 +- webrtc/voice_engine/voe_base_impl.cc | 10 +- webrtc/voice_engine/voice_engine.gyp | 2 - webrtc/voice_engine/voice_engine_defines.h | 4 + 18 files changed, 1663 insertions(+), 1791 deletions(-) delete mode 100644 webrtc/voice_engine/output_mixer_internal.cc delete mode 100644 webrtc/voice_engine/output_mixer_internal.h diff --git a/webrtc/modules/audio_coding/main/acm2/acm_resampler.cc b/webrtc/modules/audio_coding/main/acm2/acm_resampler.cc index 3abe4f1ec..294092258 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_resampler.cc +++ b/webrtc/modules/audio_coding/main/acm2/acm_resampler.cc @@ -13,20 +13,15 @@ #include #include "webrtc/common_audio/resampler/include/resampler.h" -#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" -#include "webrtc/system_wrappers/interface/critical_section_wrapper.h" -#include "webrtc/system_wrappers/interface/trace.h" +#include "webrtc/system_wrappers/interface/logging.h" namespace webrtc { - namespace acm2 { -ACMResampler::ACMResampler() - : resampler_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()) { +ACMResampler::ACMResampler() { } ACMResampler::~ACMResampler() { - delete resampler_crit_sect_; } int ACMResampler::Resample10Msec(const int16_t* in_audio, @@ -34,37 +29,28 @@ int ACMResampler::Resample10Msec(const int16_t* in_audio, int out_freq_hz, int num_audio_channels, int16_t* out_audio) { - CriticalSectionScoped cs(resampler_crit_sect_); - + int in_length = in_freq_hz * num_audio_channels / 100; + int out_length = out_freq_hz * num_audio_channels / 100; if (in_freq_hz == out_freq_hz) { - size_t length = static_cast(in_freq_hz * num_audio_channels / 100); - memcpy(out_audio, in_audio, length * sizeof(int16_t)); - return static_cast(in_freq_hz / 100); + memcpy(out_audio, in_audio, in_length * sizeof(int16_t)); + return in_length / num_audio_channels; } - // |maxLen| is maximum number of samples for 10ms at 48kHz. - int max_len = 480 * num_audio_channels; - int length_in = (in_freq_hz / 100) * num_audio_channels; - int out_len; - - ResamplerType type = (num_audio_channels == 1) ? kResamplerSynchronous : - kResamplerSynchronousStereo; - - if (resampler_.ResetIfNeeded(in_freq_hz, out_freq_hz, type) < 0) { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, 0, - "Error in reset of resampler"); + if (resampler_.InitializeIfNeeded(in_freq_hz, out_freq_hz, + num_audio_channels) != 0) { + LOG_FERR3(LS_ERROR, InitializeIfNeeded, in_freq_hz, out_freq_hz, + num_audio_channels); return -1; } - if (resampler_.Push(in_audio, length_in, out_audio, max_len, out_len) < 0) { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, 0, - "Error in resampler: resampler.Push"); + out_length = resampler_.Resample(in_audio, in_length, out_audio, out_length); + if (out_length == -1) { + LOG_FERR4(LS_ERROR, Resample, in_audio, in_length, out_audio, out_length); return -1; } - return out_len / num_audio_channels; + return out_length / num_audio_channels; } } // namespace acm2 - } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/acm2/acm_resampler.h b/webrtc/modules/audio_coding/main/acm2/acm_resampler.h index e992955f5..5d952e54b 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_resampler.h +++ b/webrtc/modules/audio_coding/main/acm2/acm_resampler.h @@ -11,13 +11,10 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_ACM_RESAMPLER_H_ #define WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_ACM_RESAMPLER_H_ -#include "webrtc/common_audio/resampler/include/resampler.h" +#include "webrtc/common_audio/resampler/include/push_resampler.h" #include "webrtc/typedefs.h" namespace webrtc { - -class CriticalSectionWrapper; - namespace acm2 { class ACMResampler { @@ -32,13 +29,10 @@ class ACMResampler { int16_t* out_audio); private: - // Use the Resampler class. - Resampler resampler_; - CriticalSectionWrapper* resampler_crit_sect_; + PushResampler resampler_; }; } // namespace acm2 - } // namespace webrtc #endif // WEBRTC_MODULES_AUDIO_CODING_MAIN_ACM2_ACM_RESAMPLER_H_ diff --git a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc index 4c64e07dd..92a69cf8f 100644 --- a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc +++ b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc @@ -1205,11 +1205,7 @@ int AudioCodingModuleImpl::Add10MsData( return -1; } - // Allow for 8, 16, 32 and 48kHz input audio. - if ((audio_frame.sample_rate_hz_ != 8000) - && (audio_frame.sample_rate_hz_ != 16000) - && (audio_frame.sample_rate_hz_ != 32000) - && (audio_frame.sample_rate_hz_ != 48000)) { + if (audio_frame.sample_rate_hz_ > 48000) { assert(false); WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, "Cannot Add 10 ms audio, input frequency not valid"); @@ -1371,7 +1367,7 @@ int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame, if (preprocess_frame_.samples_per_channel_ < 0) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, - "Cannot add 10 ms audio, resmapling failed"); + "Cannot add 10 ms audio, resampling failed"); return -1; } preprocess_frame_.sample_rate_hz_ = send_codec_inst_.plfreq; diff --git a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc index 556f530ec..b8060ce43 100644 --- a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc +++ b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc @@ -1273,11 +1273,7 @@ int32_t AudioCodingModuleImpl::Add10MsData( return -1; } - // Allow for 8, 16, 32 and 48kHz input audio. - if ((audio_frame.sample_rate_hz_ != 8000) - && (audio_frame.sample_rate_hz_ != 16000) - && (audio_frame.sample_rate_hz_ != 32000) - && (audio_frame.sample_rate_hz_ != 48000)) { + if (audio_frame.sample_rate_hz_ > 48000) { assert(false); WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, "Cannot Add 10 ms audio, input frequency not valid"); @@ -1444,7 +1440,7 @@ int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame, if (preprocess_frame_.samples_per_channel_ < 0) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, - "Cannot add 10 ms audio, resmapling failed"); + "Cannot add 10 ms audio, resampling failed"); return -1; } preprocess_frame_.sample_rate_hz_ = send_codec_inst_.plfreq; diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h index 957e6902f..096193cec 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.h +++ b/webrtc/modules/audio_processing/include/audio_processing.h @@ -61,6 +61,8 @@ struct ExperimentalAgc { bool enabled; }; +static const int kAudioProcMaxNativeSampleRateHz = 32000; + // The Audio Processing Module (APM) provides a collection of voice processing // components designed for real-time communications software. // diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc index bbea25e49..026b6b3ce 100644 --- a/webrtc/voice_engine/channel.cc +++ b/webrtc/voice_engine/channel.cc @@ -4150,61 +4150,26 @@ Channel::Demultiplex(const AudioFrame& audioFrame) return 0; } -// TODO(xians): This method borrows quite some code from -// TransmitMixer::GenerateAudioFrame(), refactor these two methods and reduce -// code duplication. void Channel::Demultiplex(const int16_t* audio_data, int sample_rate, int number_of_frames, int number_of_channels) { - // The highest sample rate that WebRTC supports for mono audio is 96kHz. - static const int kMaxNumberOfFrames = 960; - assert(number_of_frames <= kMaxNumberOfFrames); - - // Get the send codec information for doing resampling or downmixing later on. CodecInst codec; GetSendCodec(codec); - assert(codec.channels == 1 || codec.channels == 2); - int support_sample_rate = std::min(32000, - std::min(sample_rate, codec.plfreq)); - // Downmix the data to mono if needed. - const int16_t* audio_ptr = audio_data; - if (number_of_channels == 2 && codec.channels == 1) { - if (!mono_recording_audio_.get()) - mono_recording_audio_.reset(new int16_t[kMaxNumberOfFrames]); - - AudioFrameOperations::StereoToMono(audio_data, number_of_frames, - mono_recording_audio_.get()); - audio_ptr = mono_recording_audio_.get(); + if (!mono_recording_audio_.get()) { + // Temporary space for DownConvertToCodecFormat. + mono_recording_audio_.reset(new int16_t[kMaxMonoDataSizeSamples]); } - - // Resample the data to the sample rate that the codec is using. - if (input_resampler_.InitializeIfNeeded(sample_rate, - support_sample_rate, - codec.channels)) { - WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, -1), - "Channel::Demultiplex() unable to resample"); - return; - } - - int out_length = input_resampler_.Resample(audio_ptr, - number_of_frames * codec.channels, - _audioFrame.data_, - AudioFrame::kMaxDataSizeSamples); - if (out_length == -1) { - WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, -1), - "Channel::Demultiplex() resampling failed"); - return; - } - - _audioFrame.samples_per_channel_ = out_length / codec.channels; - _audioFrame.timestamp_ = -1; - _audioFrame.sample_rate_hz_ = support_sample_rate; - _audioFrame.speech_type_ = AudioFrame::kNormalSpeech; - _audioFrame.vad_activity_ = AudioFrame::kVadUnknown; - _audioFrame.num_channels_ = codec.channels; - _audioFrame.id_ = _channelId; + DownConvertToCodecFormat(audio_data, + number_of_frames, + number_of_channels, + sample_rate, + codec.channels, + codec.plfreq, + mono_recording_audio_.get(), + &input_resampler_, + &_audioFrame); } uint32_t @@ -4694,11 +4659,11 @@ Channel::MixOrReplaceAudioWithFile(int mixingFrequency) { // Currently file stream is always mono. // TODO(xians): Change the code when FilePlayer supports real stereo. - Utility::MixWithSat(_audioFrame.data_, - _audioFrame.num_channels_, - fileBuffer.get(), - 1, - fileSamples); + MixWithSat(_audioFrame.data_, + _audioFrame.num_channels_, + fileBuffer.get(), + 1, + fileSamples); } else { @@ -4754,11 +4719,11 @@ Channel::MixAudioWithFile(AudioFrame& audioFrame, { // Currently file stream is always mono. // TODO(xians): Change the code when FilePlayer supports real stereo. - Utility::MixWithSat(audioFrame.data_, - audioFrame.num_channels_, - fileBuffer.get(), - 1, - fileSamples); + MixWithSat(audioFrame.data_, + audioFrame.num_channels_, + fileBuffer.get(), + 1, + fileSamples); } else { diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h index 43cfe99ac..b490f4896 100644 --- a/webrtc/voice_engine/channel.h +++ b/webrtc/voice_engine/channel.h @@ -545,7 +545,7 @@ private: AudioLevel _outputAudioLevel; bool _externalTransport; AudioFrame _audioFrame; - scoped_array mono_recording_audio_; + scoped_ptr mono_recording_audio_; // Resampler is used when input data is stereo while codec is mono. PushResampler input_resampler_; uint8_t _audioLevel_dBov; diff --git a/webrtc/voice_engine/output_mixer.cc b/webrtc/voice_engine/output_mixer.cc index 5503361c5..1648a9d97 100644 --- a/webrtc/voice_engine/output_mixer.cc +++ b/webrtc/voice_engine/output_mixer.cc @@ -16,11 +16,10 @@ #include "webrtc/system_wrappers/interface/file_wrapper.h" #include "webrtc/system_wrappers/interface/trace.h" #include "webrtc/voice_engine/include/voe_external_media.h" -#include "webrtc/voice_engine/output_mixer_internal.h" #include "webrtc/voice_engine/statistics.h" +#include "webrtc/voice_engine/utility.h" namespace webrtc { - namespace voe { void @@ -528,7 +527,8 @@ int OutputMixer::GetMixedAudio(int sample_rate_hz, frame->sample_rate_hz_ = sample_rate_hz; // TODO(andrew): Ideally the downmixing would occur much earlier, in // AudioCodingModule. - return RemixAndResample(_audioFrame, &resampler_, frame); + RemixAndResample(_audioFrame, &resampler_, frame); + return 0; } int32_t @@ -565,7 +565,9 @@ OutputMixer::DoOperationsOnCombinedSignal() } // --- Far-end Voice Quality Enhancement (AudioProcessing Module) - + // TODO(ajm): Check with VoEBase if |need_audio_processing| is false. + // If so, we don't need to call this method and can avoid the subsequent + // resampling. See: https://code.google.com/p/webrtc/issues/detail?id=3147 APMAnalyzeReverseStream(); // --- External media processing @@ -603,8 +605,7 @@ void OutputMixer::APMAnalyzeReverseStream() { AudioFrame frame; frame.num_channels_ = 1; frame.sample_rate_hz_ = _audioProcessingModulePtr->sample_rate_hz(); - if (RemixAndResample(_audioFrame, &audioproc_resampler_, &frame) == -1) - return; + RemixAndResample(_audioFrame, &audioproc_resampler_, &frame); if (_audioProcessingModulePtr->AnalyzeReverseStream(&frame) == -1) { WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1), @@ -657,5 +658,4 @@ OutputMixer::InsertInbandDtmfTone() } } // namespace voe - } // namespace webrtc diff --git a/webrtc/voice_engine/output_mixer_internal.cc b/webrtc/voice_engine/output_mixer_internal.cc deleted file mode 100644 index 55eedb383..000000000 --- a/webrtc/voice_engine/output_mixer_internal.cc +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/voice_engine/output_mixer_internal.h" - -#include "webrtc/common_audio/resampler/include/push_resampler.h" -#include "webrtc/modules/interface/module_common_types.h" -#include "webrtc/modules/utility/interface/audio_frame_operations.h" -#include "webrtc/system_wrappers/interface/logging.h" -#include "webrtc/system_wrappers/interface/trace.h" - -namespace webrtc { -namespace voe { - -int RemixAndResample(const AudioFrame& src_frame, - PushResampler* resampler, - AudioFrame* dst_frame) { - const int16_t* audio_ptr = src_frame.data_; - int audio_ptr_num_channels = src_frame.num_channels_; - int16_t mono_audio[AudioFrame::kMaxDataSizeSamples]; - - // Downmix before resampling. - if (src_frame.num_channels_ == 2 && dst_frame->num_channels_ == 1) { - AudioFrameOperations::StereoToMono(src_frame.data_, - src_frame.samples_per_channel_, - mono_audio); - audio_ptr = mono_audio; - audio_ptr_num_channels = 1; - } - - if (resampler->InitializeIfNeeded(src_frame.sample_rate_hz_, - dst_frame->sample_rate_hz_, - audio_ptr_num_channels) == -1) { - dst_frame->CopyFrom(src_frame); - LOG_FERR3(LS_ERROR, InitializeIfNeeded, src_frame.sample_rate_hz_, - dst_frame->sample_rate_hz_, audio_ptr_num_channels); - return -1; - } - - const int src_length = src_frame.samples_per_channel_ * - audio_ptr_num_channels; - int out_length = resampler->Resample(audio_ptr, src_length, dst_frame->data_, - AudioFrame::kMaxDataSizeSamples); - if (out_length == -1) { - dst_frame->CopyFrom(src_frame); - LOG_FERR3(LS_ERROR, Resample, src_length, dst_frame->data_, - AudioFrame::kMaxDataSizeSamples); - return -1; - } - dst_frame->samples_per_channel_ = out_length / audio_ptr_num_channels; - - // Upmix after resampling. - if (src_frame.num_channels_ == 1 && dst_frame->num_channels_ == 2) { - // The audio in dst_frame really is mono at this point; MonoToStereo will - // set this back to stereo. - dst_frame->num_channels_ = 1; - AudioFrameOperations::MonoToStereo(dst_frame); - } - return 0; -} - -} // namespace voe -} // namespace webrtc diff --git a/webrtc/voice_engine/output_mixer_internal.h b/webrtc/voice_engine/output_mixer_internal.h deleted file mode 100644 index 88a3a5b2f..000000000 --- a/webrtc/voice_engine/output_mixer_internal.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_VOICE_ENGINE_OUTPUT_MIXER_INTERNAL_H_ -#define WEBRTC_VOICE_ENGINE_OUTPUT_MIXER_INTERNAL_H_ - -namespace webrtc { - -class AudioFrame; -class PushResampler; - -namespace voe { - -// Upmix or downmix and resample the audio in |src_frame| to |dst_frame|. -// Expects |dst_frame| to have its |num_channels_| and |sample_rate_hz_| set to -// the desired values. Updates |samples_per_channel_| accordingly. -// -// On failure, returns -1 and copies |src_frame| to |dst_frame|. -int RemixAndResample(const AudioFrame& src_frame, - PushResampler* resampler, - AudioFrame* dst_frame); - -} // namespace voe -} // namespace webrtc - -#endif // VOICE_ENGINE_OUTPUT_MIXER_INTERNAL_H_ diff --git a/webrtc/voice_engine/output_mixer_unittest.cc b/webrtc/voice_engine/output_mixer_unittest.cc index 006c45fa1..08412d611 100644 --- a/webrtc/voice_engine/output_mixer_unittest.cc +++ b/webrtc/voice_engine/output_mixer_unittest.cc @@ -11,13 +11,20 @@ #include #include "testing/gtest/include/gtest/gtest.h" -#include "webrtc/voice_engine/output_mixer.h" -#include "webrtc/voice_engine/output_mixer_internal.h" +#include "webrtc/common_audio/resampler/include/push_resampler.h" +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/voice_engine/utility.h" +#include "webrtc/voice_engine/voice_engine_defines.h" namespace webrtc { namespace voe { namespace { +enum FunctionToTest { + TestRemixAndResample, + TestDownConvertToCodecFormat +}; + class OutputMixerTest : public ::testing::Test { protected: OutputMixerTest() { @@ -29,7 +36,8 @@ class OutputMixerTest : public ::testing::Test { } void RunResampleTest(int src_channels, int src_sample_rate_hz, - int dst_channels, int dst_sample_rate_hz); + int dst_channels, int dst_sample_rate_hz, + FunctionToTest function); PushResampler resampler_; AudioFrame src_frame_; @@ -121,7 +129,8 @@ void VerifyFramesAreEqual(const AudioFrame& ref_frame, void OutputMixerTest::RunResampleTest(int src_channels, int src_sample_rate_hz, int dst_channels, - int dst_sample_rate_hz) { + int dst_sample_rate_hz, + FunctionToTest function) { PushResampler resampler; // Create a new one with every test. const int16_t kSrcLeft = 30; // Shouldn't overflow for any used sample rate. const int16_t kSrcRight = 15; @@ -157,7 +166,21 @@ void OutputMixerTest::RunResampleTest(int src_channels, / src_sample_rate_hz * kInputKernelDelaySamples * dst_channels * 2; printf("(%d, %d Hz) -> (%d, %d Hz) ", // SNR reported on the same line later. src_channels, src_sample_rate_hz, dst_channels, dst_sample_rate_hz); - EXPECT_EQ(0, RemixAndResample(src_frame_, &resampler, &dst_frame_)); + if (function == TestRemixAndResample) { + RemixAndResample(src_frame_, &resampler, &dst_frame_); + } else { + int16_t mono_buffer[kMaxMonoDataSizeSamples]; + DownConvertToCodecFormat(src_frame_.data_, + src_frame_.samples_per_channel_, + src_frame_.num_channels_, + src_frame_.sample_rate_hz_, + dst_frame_.num_channels_, + dst_frame_.sample_rate_hz_, + mono_buffer, + &resampler, + &dst_frame_); + } + if (src_sample_rate_hz == 96000 && dst_sample_rate_hz == 8000) { // The sinc resampler gives poor SNR at this extreme conversion, but we // expect to see this rarely in practice. @@ -171,13 +194,13 @@ TEST_F(OutputMixerTest, RemixAndResampleCopyFrameSucceeds) { // Stereo -> stereo. SetStereoFrame(&src_frame_, 10, 10); SetStereoFrame(&dst_frame_, 0, 0); - EXPECT_EQ(0, RemixAndResample(src_frame_, &resampler_, &dst_frame_)); + RemixAndResample(src_frame_, &resampler_, &dst_frame_); VerifyFramesAreEqual(src_frame_, dst_frame_); // Mono -> mono. SetMonoFrame(&src_frame_, 20); SetMonoFrame(&dst_frame_, 0); - EXPECT_EQ(0, RemixAndResample(src_frame_, &resampler_, &dst_frame_)); + RemixAndResample(src_frame_, &resampler_, &dst_frame_); VerifyFramesAreEqual(src_frame_, dst_frame_); } @@ -186,20 +209,18 @@ TEST_F(OutputMixerTest, RemixAndResampleMixingOnlySucceeds) { SetStereoFrame(&dst_frame_, 0, 0); SetMonoFrame(&src_frame_, 10); SetStereoFrame(&golden_frame_, 10, 10); - EXPECT_EQ(0, RemixAndResample(src_frame_, &resampler_, &dst_frame_)); + RemixAndResample(src_frame_, &resampler_, &dst_frame_); VerifyFramesAreEqual(dst_frame_, golden_frame_); // Mono -> stereo. SetMonoFrame(&dst_frame_, 0); SetStereoFrame(&src_frame_, 10, 20); SetMonoFrame(&golden_frame_, 15); - EXPECT_EQ(0, RemixAndResample(src_frame_, &resampler_, &dst_frame_)); + RemixAndResample(src_frame_, &resampler_, &dst_frame_); VerifyFramesAreEqual(golden_frame_, dst_frame_); } TEST_F(OutputMixerTest, RemixAndResampleSucceeds) { - // TODO(ajm): convert this to the parameterized TEST_P style used in - // sinc_resampler_unittest.cc. We can then easily add tighter SNR thresholds. const int kSampleRates[] = {8000, 16000, 32000, 44100, 48000, 96000}; const int kSampleRatesSize = sizeof(kSampleRates) / sizeof(*kSampleRates); const int kChannels[] = {1, 2}; @@ -209,7 +230,28 @@ TEST_F(OutputMixerTest, RemixAndResampleSucceeds) { for (int src_channel = 0; src_channel < kChannelsSize; src_channel++) { for (int dst_channel = 0; dst_channel < kChannelsSize; dst_channel++) { RunResampleTest(kChannels[src_channel], kSampleRates[src_rate], - kChannels[dst_channel], kSampleRates[dst_rate]); + kChannels[dst_channel], kSampleRates[dst_rate], + TestRemixAndResample); + } + } + } + } +} + +TEST_F(OutputMixerTest, ConvertToCodecFormatSucceeds) { + const int kSampleRates[] = {8000, 16000, 32000, 44100, 48000, 96000}; + const int kSampleRatesSize = sizeof(kSampleRates) / sizeof(*kSampleRates); + const int kChannels[] = {1, 2}; + const int kChannelsSize = sizeof(kChannels) / sizeof(*kChannels); + for (int src_rate = 0; src_rate < kSampleRatesSize; src_rate++) { + for (int dst_rate = 0; dst_rate < kSampleRatesSize; dst_rate++) { + for (int src_channel = 0; src_channel < kChannelsSize; src_channel++) { + for (int dst_channel = 0; dst_channel < kChannelsSize; dst_channel++) { + if (dst_rate <= src_rate && dst_channel <= src_channel) { + RunResampleTest(kChannels[src_channel], kSampleRates[src_rate], + kChannels[src_channel], kSampleRates[dst_rate], + TestDownConvertToCodecFormat); + } } } } diff --git a/webrtc/voice_engine/transmit_mixer.cc b/webrtc/voice_engine/transmit_mixer.cc index 56bf2fa09..3893da365 100644 --- a/webrtc/voice_engine/transmit_mixer.cc +++ b/webrtc/voice_engine/transmit_mixer.cc @@ -1,1418 +1,1379 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/voice_engine/transmit_mixer.h" - -#include "webrtc/modules/utility/interface/audio_frame_operations.h" -#include "webrtc/system_wrappers/interface/critical_section_wrapper.h" -#include "webrtc/system_wrappers/interface/event_wrapper.h" -#include "webrtc/system_wrappers/interface/logging.h" -#include "webrtc/system_wrappers/interface/trace.h" -#include "webrtc/voice_engine/channel.h" -#include "webrtc/voice_engine/channel_manager.h" -#include "webrtc/voice_engine/include/voe_external_media.h" -#include "webrtc/voice_engine/statistics.h" -#include "webrtc/voice_engine/utility.h" -#include "webrtc/voice_engine/voe_base_impl.h" - -#define WEBRTC_ABS(a) (((a) < 0) ? -(a) : (a)) - -namespace webrtc { - -namespace voe { - -// Used for downmixing before resampling. -// TODO(ajm): audio_device should advertise the maximum sample rate it can -// provide. -static const int kMaxMonoDeviceDataSizeSamples = 1920; // 10 ms, 192 kHz, mono. - -// TODO(ajm): The thread safety of this is dubious... -void -TransmitMixer::OnPeriodicProcess() -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::OnPeriodicProcess()"); - -#if defined(WEBRTC_VOICE_ENGINE_TYPING_DETECTION) - if (_typingNoiseWarningPending) - { - CriticalSectionScoped cs(&_callbackCritSect); - if (_voiceEngineObserverPtr) - { - if (_typingNoiseDetected) { - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::OnPeriodicProcess() => " - "CallbackOnError(VE_TYPING_NOISE_WARNING)"); - _voiceEngineObserverPtr->CallbackOnError( - -1, - VE_TYPING_NOISE_WARNING); - } else { - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::OnPeriodicProcess() => " - "CallbackOnError(VE_TYPING_NOISE_OFF_WARNING)"); - _voiceEngineObserverPtr->CallbackOnError( - -1, - VE_TYPING_NOISE_OFF_WARNING); - } - } - _typingNoiseWarningPending = false; - } -#endif - - bool saturationWarning = false; - { - // Modify |_saturationWarning| under lock to avoid conflict with write op - // in ProcessAudio and also ensure that we don't hold the lock during the - // callback. - CriticalSectionScoped cs(&_critSect); - saturationWarning = _saturationWarning; - if (_saturationWarning) - _saturationWarning = false; - } - - if (saturationWarning) - { - CriticalSectionScoped cs(&_callbackCritSect); - if (_voiceEngineObserverPtr) - { - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::OnPeriodicProcess() =>" - " CallbackOnError(VE_SATURATION_WARNING)"); - _voiceEngineObserverPtr->CallbackOnError(-1, VE_SATURATION_WARNING); - } - } -} - - -void TransmitMixer::PlayNotification(int32_t id, - uint32_t durationMs) -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::PlayNotification(id=%d, durationMs=%d)", - id, durationMs); - - // Not implement yet -} - -void TransmitMixer::RecordNotification(int32_t id, - uint32_t durationMs) -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1), - "TransmitMixer::RecordNotification(id=%d, durationMs=%d)", - id, durationMs); - - // Not implement yet -} - -void TransmitMixer::PlayFileEnded(int32_t id) -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::PlayFileEnded(id=%d)", id); - - assert(id == _filePlayerId); - - CriticalSectionScoped cs(&_critSect); - - _filePlaying = false; - WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::PlayFileEnded() =>" - "file player module is shutdown"); -} - -void -TransmitMixer::RecordFileEnded(int32_t id) -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::RecordFileEnded(id=%d)", id); - - if (id == _fileRecorderId) - { - CriticalSectionScoped cs(&_critSect); - _fileRecording = false; - WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::RecordFileEnded() => fileRecorder module" - "is shutdown"); - } else if (id == _fileCallRecorderId) - { - CriticalSectionScoped cs(&_critSect); - _fileCallRecording = false; - WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::RecordFileEnded() => fileCallRecorder" - "module is shutdown"); - } -} - -int32_t -TransmitMixer::Create(TransmitMixer*& mixer, uint32_t instanceId) -{ - WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, -1), - "TransmitMixer::Create(instanceId=%d)", instanceId); - mixer = new TransmitMixer(instanceId); - if (mixer == NULL) - { - WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, -1), - "TransmitMixer::Create() unable to allocate memory" - "for mixer"); - return -1; - } - return 0; -} - -void -TransmitMixer::Destroy(TransmitMixer*& mixer) -{ - if (mixer) - { - delete mixer; - mixer = NULL; - } -} - -TransmitMixer::TransmitMixer(uint32_t instanceId) : - _engineStatisticsPtr(NULL), - _channelManagerPtr(NULL), - audioproc_(NULL), - _voiceEngineObserverPtr(NULL), - _processThreadPtr(NULL), - _filePlayerPtr(NULL), - _fileRecorderPtr(NULL), - _fileCallRecorderPtr(NULL), - // Avoid conflict with other channels by adding 1024 - 1026, - // won't use as much as 1024 channels. - _filePlayerId(instanceId + 1024), - _fileRecorderId(instanceId + 1025), - _fileCallRecorderId(instanceId + 1026), - _filePlaying(false), - _fileRecording(false), - _fileCallRecording(false), - _audioLevel(), - _critSect(*CriticalSectionWrapper::CreateCriticalSection()), - _callbackCritSect(*CriticalSectionWrapper::CreateCriticalSection()), -#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION - _typingNoiseWarningPending(false), - _typingNoiseDetected(false), -#endif - _saturationWarning(false), - _instanceId(instanceId), - _mixFileWithMicrophone(false), - _captureLevel(0), - external_postproc_ptr_(NULL), - external_preproc_ptr_(NULL), - _mute(false), - _remainingMuteMicTimeMs(0), - stereo_codec_(false), - swap_stereo_channels_(false) -{ - WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::TransmitMixer() - ctor"); -} - -TransmitMixer::~TransmitMixer() -{ - WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::~TransmitMixer() - dtor"); - _monitorModule.DeRegisterObserver(); - if (_processThreadPtr) - { - _processThreadPtr->DeRegisterModule(&_monitorModule); - } - DeRegisterExternalMediaProcessing(kRecordingAllChannelsMixed); - DeRegisterExternalMediaProcessing(kRecordingPreprocessing); - { - CriticalSectionScoped cs(&_critSect); - if (_fileRecorderPtr) - { - _fileRecorderPtr->RegisterModuleFileCallback(NULL); - _fileRecorderPtr->StopRecording(); - FileRecorder::DestroyFileRecorder(_fileRecorderPtr); - _fileRecorderPtr = NULL; - } - if (_fileCallRecorderPtr) - { - _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); - _fileCallRecorderPtr->StopRecording(); - FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); - _fileCallRecorderPtr = NULL; - } - if (_filePlayerPtr) - { - _filePlayerPtr->RegisterModuleFileCallback(NULL); - _filePlayerPtr->StopPlayingFile(); - FilePlayer::DestroyFilePlayer(_filePlayerPtr); - _filePlayerPtr = NULL; - } - } - delete &_critSect; - delete &_callbackCritSect; -} - -int32_t -TransmitMixer::SetEngineInformation(ProcessThread& processThread, - Statistics& engineStatistics, - ChannelManager& channelManager) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::SetEngineInformation()"); - - _processThreadPtr = &processThread; - _engineStatisticsPtr = &engineStatistics; - _channelManagerPtr = &channelManager; - - if (_processThreadPtr->RegisterModule(&_monitorModule) == -1) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::SetEngineInformation() failed to" - "register the monitor module"); - } else - { - _monitorModule.RegisterObserver(*this); - } - - return 0; -} - -int32_t -TransmitMixer::RegisterVoiceEngineObserver(VoiceEngineObserver& observer) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::RegisterVoiceEngineObserver()"); - CriticalSectionScoped cs(&_callbackCritSect); - - if (_voiceEngineObserverPtr) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_OPERATION, kTraceError, - "RegisterVoiceEngineObserver() observer already enabled"); - return -1; - } - _voiceEngineObserverPtr = &observer; - return 0; -} - -int32_t -TransmitMixer::SetAudioProcessingModule(AudioProcessing* audioProcessingModule) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::SetAudioProcessingModule(" - "audioProcessingModule=0x%x)", - audioProcessingModule); - audioproc_ = audioProcessingModule; - return 0; -} - -void TransmitMixer::GetSendCodecInfo(int* max_sample_rate, int* max_channels) { - *max_sample_rate = 8000; - *max_channels = 1; - for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); - it.Increment()) { - Channel* channel = it.GetChannel(); - if (channel->Sending()) { - CodecInst codec; - channel->GetSendCodec(codec); - // TODO(tlegrand): Remove the 32 kHz restriction once we have full 48 kHz - // support in Audio Coding Module. - *max_sample_rate = std::min(32000, - std::max(*max_sample_rate, codec.plfreq)); - *max_channels = std::max(*max_channels, codec.channels); - } - } -} - -int32_t -TransmitMixer::PrepareDemux(const void* audioSamples, - uint32_t nSamples, - uint8_t nChannels, - uint32_t samplesPerSec, - uint16_t totalDelayMS, - int32_t clockDrift, - uint16_t currentMicLevel, - bool keyPressed) -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::PrepareDemux(nSamples=%u, nChannels=%u," - "samplesPerSec=%u, totalDelayMS=%u, clockDrift=%d," - "currentMicLevel=%u)", nSamples, nChannels, samplesPerSec, - totalDelayMS, clockDrift, currentMicLevel); - - // --- Resample input audio and create/store the initial audio frame - if (GenerateAudioFrame(static_cast(audioSamples), - nSamples, - nChannels, - samplesPerSec) == -1) - { - return -1; - } - - { - CriticalSectionScoped cs(&_callbackCritSect); - if (external_preproc_ptr_) { - external_preproc_ptr_->Process(-1, kRecordingPreprocessing, - _audioFrame.data_, - _audioFrame.samples_per_channel_, - _audioFrame.sample_rate_hz_, - _audioFrame.num_channels_ == 2); - } - } - - // --- Near-end audio processing. - ProcessAudio(totalDelayMS, clockDrift, currentMicLevel, keyPressed); - - if (swap_stereo_channels_ && stereo_codec_) - // Only bother swapping if we're using a stereo codec. - AudioFrameOperations::SwapStereoChannels(&_audioFrame); - - // --- Annoying typing detection (utilizes the APM/VAD decision) -#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION - TypingDetection(keyPressed); -#endif - - // --- Mute during DTMF tone if direct feedback is enabled - if (_remainingMuteMicTimeMs > 0) - { - AudioFrameOperations::Mute(_audioFrame); - _remainingMuteMicTimeMs -= 10; - if (_remainingMuteMicTimeMs < 0) - { - _remainingMuteMicTimeMs = 0; - } - } - - // --- Mute signal - if (_mute) - { - AudioFrameOperations::Mute(_audioFrame); - } - - // --- Mix with file (does not affect the mixing frequency) - if (_filePlaying) - { - MixOrReplaceAudioWithFile(_audioFrame.sample_rate_hz_); - } - - // --- Record to file - bool file_recording = false; - { - CriticalSectionScoped cs(&_critSect); - file_recording = _fileRecording; - } - if (file_recording) - { - RecordAudioToFile(_audioFrame.sample_rate_hz_); - } - - { - CriticalSectionScoped cs(&_callbackCritSect); - if (external_postproc_ptr_) { - external_postproc_ptr_->Process(-1, kRecordingAllChannelsMixed, - _audioFrame.data_, - _audioFrame.samples_per_channel_, - _audioFrame.sample_rate_hz_, - _audioFrame.num_channels_ == 2); - } - } - - // --- Measure audio level of speech after all processing. - _audioLevel.ComputeLevel(_audioFrame); - return 0; -} - -int32_t -TransmitMixer::DemuxAndMix() -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::DemuxAndMix()"); - - for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); - it.Increment()) - { - Channel* channelPtr = it.GetChannel(); - if (channelPtr->InputIsOnHold()) - { - channelPtr->UpdateLocalTimeStamp(); - } else if (channelPtr->Sending()) - { - // Demultiplex makes a copy of its input. - channelPtr->Demultiplex(_audioFrame); - channelPtr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_); - } - } - return 0; -} - -void TransmitMixer::DemuxAndMix(const int voe_channels[], - int number_of_voe_channels) { - for (int i = 0; i < number_of_voe_channels; ++i) { - voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]); - voe::Channel* channel_ptr = ch.channel(); - if (channel_ptr) { - if (channel_ptr->InputIsOnHold()) { - channel_ptr->UpdateLocalTimeStamp(); - } else if (channel_ptr->Sending()) { - // Demultiplex makes a copy of its input. - channel_ptr->Demultiplex(_audioFrame); - channel_ptr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_); - } - } - } -} - -int32_t -TransmitMixer::EncodeAndSend() -{ - WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::EncodeAndSend()"); - - for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); - it.Increment()) - { - Channel* channelPtr = it.GetChannel(); - if (channelPtr->Sending() && !channelPtr->InputIsOnHold()) - { - channelPtr->EncodeAndSend(); - } - } - return 0; -} - -void TransmitMixer::EncodeAndSend(const int voe_channels[], - int number_of_voe_channels) { - for (int i = 0; i < number_of_voe_channels; ++i) { - voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]); - voe::Channel* channel_ptr = ch.channel(); - if (channel_ptr && channel_ptr->Sending() && !channel_ptr->InputIsOnHold()) - channel_ptr->EncodeAndSend(); - } -} - -uint32_t TransmitMixer::CaptureLevel() const -{ - return _captureLevel; -} - -void -TransmitMixer::UpdateMuteMicrophoneTime(uint32_t lengthMs) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::UpdateMuteMicrophoneTime(lengthMs=%d)", - lengthMs); - _remainingMuteMicTimeMs = lengthMs; -} - -int32_t -TransmitMixer::StopSend() -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StopSend()"); - _audioLevel.Clear(); - return 0; -} - -int TransmitMixer::StartPlayingFileAsMicrophone(const char* fileName, - bool loop, - FileFormats format, - int startPosition, - float volumeScaling, - int stopPosition, - const CodecInst* codecInst) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StartPlayingFileAsMicrophone(" - "fileNameUTF8[]=%s,loop=%d, format=%d, volumeScaling=%5.3f," - " startPosition=%d, stopPosition=%d)", fileName, loop, - format, volumeScaling, startPosition, stopPosition); - - if (_filePlaying) - { - _engineStatisticsPtr->SetLastError( - VE_ALREADY_PLAYING, kTraceWarning, - "StartPlayingFileAsMicrophone() is already playing"); - return 0; - } - - CriticalSectionScoped cs(&_critSect); - - // Destroy the old instance - if (_filePlayerPtr) - { - _filePlayerPtr->RegisterModuleFileCallback(NULL); - FilePlayer::DestroyFilePlayer(_filePlayerPtr); - _filePlayerPtr = NULL; - } - - // Dynamically create the instance - _filePlayerPtr - = FilePlayer::CreateFilePlayer(_filePlayerId, - (const FileFormats) format); - - if (_filePlayerPtr == NULL) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_ARGUMENT, kTraceError, - "StartPlayingFileAsMicrophone() filePlayer format isnot correct"); - return -1; - } - - const uint32_t notificationTime(0); - - if (_filePlayerPtr->StartPlayingFile( - fileName, - loop, - startPosition, - volumeScaling, - notificationTime, - stopPosition, - (const CodecInst*) codecInst) != 0) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_FILE, kTraceError, - "StartPlayingFile() failed to start file playout"); - _filePlayerPtr->StopPlayingFile(); - FilePlayer::DestroyFilePlayer(_filePlayerPtr); - _filePlayerPtr = NULL; - return -1; - } - - _filePlayerPtr->RegisterModuleFileCallback(this); - _filePlaying = true; - - return 0; -} - -int TransmitMixer::StartPlayingFileAsMicrophone(InStream* stream, - FileFormats format, - int startPosition, - float volumeScaling, - int stopPosition, - const CodecInst* codecInst) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1), - "TransmitMixer::StartPlayingFileAsMicrophone(format=%d," - " volumeScaling=%5.3f, startPosition=%d, stopPosition=%d)", - format, volumeScaling, startPosition, stopPosition); - - if (stream == NULL) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_FILE, kTraceError, - "StartPlayingFileAsMicrophone() NULL as input stream"); - return -1; - } - - if (_filePlaying) - { - _engineStatisticsPtr->SetLastError( - VE_ALREADY_PLAYING, kTraceWarning, - "StartPlayingFileAsMicrophone() is already playing"); - return 0; - } - - CriticalSectionScoped cs(&_critSect); - - // Destroy the old instance - if (_filePlayerPtr) - { - _filePlayerPtr->RegisterModuleFileCallback(NULL); - FilePlayer::DestroyFilePlayer(_filePlayerPtr); - _filePlayerPtr = NULL; - } - - // Dynamically create the instance - _filePlayerPtr - = FilePlayer::CreateFilePlayer(_filePlayerId, - (const FileFormats) format); - - if (_filePlayerPtr == NULL) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_ARGUMENT, kTraceWarning, - "StartPlayingFileAsMicrophone() filePlayer format isnot correct"); - return -1; - } - - const uint32_t notificationTime(0); - - if (_filePlayerPtr->StartPlayingFile( - (InStream&) *stream, - startPosition, - volumeScaling, - notificationTime, - stopPosition, - (const CodecInst*) codecInst) != 0) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_FILE, kTraceError, - "StartPlayingFile() failed to start file playout"); - _filePlayerPtr->StopPlayingFile(); - FilePlayer::DestroyFilePlayer(_filePlayerPtr); - _filePlayerPtr = NULL; - return -1; - } - _filePlayerPtr->RegisterModuleFileCallback(this); - _filePlaying = true; - - return 0; -} - -int TransmitMixer::StopPlayingFileAsMicrophone() -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1), - "TransmitMixer::StopPlayingFileAsMicrophone()"); - - if (!_filePlaying) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_OPERATION, kTraceWarning, - "StopPlayingFileAsMicrophone() isnot playing"); - return 0; - } - - CriticalSectionScoped cs(&_critSect); - - if (_filePlayerPtr->StopPlayingFile() != 0) - { - _engineStatisticsPtr->SetLastError( - VE_CANNOT_STOP_PLAYOUT, kTraceError, - "StopPlayingFile() couldnot stop playing file"); - return -1; - } - - _filePlayerPtr->RegisterModuleFileCallback(NULL); - FilePlayer::DestroyFilePlayer(_filePlayerPtr); - _filePlayerPtr = NULL; - _filePlaying = false; - - return 0; -} - -int TransmitMixer::IsPlayingFileAsMicrophone() const -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::IsPlayingFileAsMicrophone()"); - return _filePlaying; -} - -int TransmitMixer::ScaleFileAsMicrophonePlayout(float scale) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::ScaleFileAsMicrophonePlayout(scale=%5.3f)", - scale); - - CriticalSectionScoped cs(&_critSect); - - if (!_filePlaying) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_OPERATION, kTraceError, - "ScaleFileAsMicrophonePlayout() isnot playing file"); - return -1; - } - - if ((_filePlayerPtr == NULL) || - (_filePlayerPtr->SetAudioScaling(scale) != 0)) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_ARGUMENT, kTraceError, - "SetAudioScaling() failed to scale playout"); - return -1; - } - - return 0; -} - -int TransmitMixer::StartRecordingMicrophone(const char* fileName, - const CodecInst* codecInst) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StartRecordingMicrophone(fileName=%s)", - fileName); - - CriticalSectionScoped cs(&_critSect); - - if (_fileRecording) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "StartRecordingMicrophone() is already recording"); - return 0; - } - - FileFormats format; - const uint32_t notificationTime(0); // Not supported in VoE - CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; - - if (codecInst != NULL && - (codecInst->channels < 0 || codecInst->channels > 2)) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_ARGUMENT, kTraceError, - "StartRecordingMicrophone() invalid compression"); - return (-1); - } - if (codecInst == NULL) - { - format = kFileFormatPcm16kHzFile; - codecInst = &dummyCodec; - } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) - { - format = kFileFormatWavFile; - } else - { - format = kFileFormatCompressedFile; - } - - // Destroy the old instance - if (_fileRecorderPtr) - { - _fileRecorderPtr->RegisterModuleFileCallback(NULL); - FileRecorder::DestroyFileRecorder(_fileRecorderPtr); - _fileRecorderPtr = NULL; - } - - _fileRecorderPtr = - FileRecorder::CreateFileRecorder(_fileRecorderId, - (const FileFormats) format); - if (_fileRecorderPtr == NULL) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_ARGUMENT, kTraceError, - "StartRecordingMicrophone() fileRecorder format isnot correct"); - return -1; - } - - if (_fileRecorderPtr->StartRecordingAudioFile( - fileName, - (const CodecInst&) *codecInst, - notificationTime) != 0) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_FILE, kTraceError, - "StartRecordingAudioFile() failed to start file recording"); - _fileRecorderPtr->StopRecording(); - FileRecorder::DestroyFileRecorder(_fileRecorderPtr); - _fileRecorderPtr = NULL; - return -1; - } - _fileRecorderPtr->RegisterModuleFileCallback(this); - _fileRecording = true; - - return 0; -} - -int TransmitMixer::StartRecordingMicrophone(OutStream* stream, - const CodecInst* codecInst) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StartRecordingMicrophone()"); - - CriticalSectionScoped cs(&_critSect); - - if (_fileRecording) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "StartRecordingMicrophone() is already recording"); - return 0; - } - - FileFormats format; - const uint32_t notificationTime(0); // Not supported in VoE - CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; - - if (codecInst != NULL && codecInst->channels != 1) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_ARGUMENT, kTraceError, - "StartRecordingMicrophone() invalid compression"); - return (-1); - } - if (codecInst == NULL) - { - format = kFileFormatPcm16kHzFile; - codecInst = &dummyCodec; - } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) - { - format = kFileFormatWavFile; - } else - { - format = kFileFormatCompressedFile; - } - - // Destroy the old instance - if (_fileRecorderPtr) - { - _fileRecorderPtr->RegisterModuleFileCallback(NULL); - FileRecorder::DestroyFileRecorder(_fileRecorderPtr); - _fileRecorderPtr = NULL; - } - - _fileRecorderPtr = - FileRecorder::CreateFileRecorder(_fileRecorderId, - (const FileFormats) format); - if (_fileRecorderPtr == NULL) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_ARGUMENT, kTraceError, - "StartRecordingMicrophone() fileRecorder format isnot correct"); - return -1; - } - - if (_fileRecorderPtr->StartRecordingAudioFile(*stream, - *codecInst, - notificationTime) != 0) - { - _engineStatisticsPtr->SetLastError(VE_BAD_FILE, kTraceError, - "StartRecordingAudioFile() failed to start file recording"); - _fileRecorderPtr->StopRecording(); - FileRecorder::DestroyFileRecorder(_fileRecorderPtr); - _fileRecorderPtr = NULL; - return -1; - } - - _fileRecorderPtr->RegisterModuleFileCallback(this); - _fileRecording = true; - - return 0; -} - - -int TransmitMixer::StopRecordingMicrophone() -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StopRecordingMicrophone()"); - - CriticalSectionScoped cs(&_critSect); - - if (!_fileRecording) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "StopRecordingMicrophone() isnot recording"); - return 0; - } - - if (_fileRecorderPtr->StopRecording() != 0) - { - _engineStatisticsPtr->SetLastError( - VE_STOP_RECORDING_FAILED, kTraceError, - "StopRecording(), could not stop recording"); - return -1; - } - _fileRecorderPtr->RegisterModuleFileCallback(NULL); - FileRecorder::DestroyFileRecorder(_fileRecorderPtr); - _fileRecorderPtr = NULL; - _fileRecording = false; - - return 0; -} - -int TransmitMixer::StartRecordingCall(const char* fileName, - const CodecInst* codecInst) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StartRecordingCall(fileName=%s)", fileName); - - if (_fileCallRecording) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "StartRecordingCall() is already recording"); - return 0; - } - - FileFormats format; - const uint32_t notificationTime(0); // Not supported in VoE - CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; - - if (codecInst != NULL && codecInst->channels != 1) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_ARGUMENT, kTraceError, - "StartRecordingCall() invalid compression"); - return (-1); - } - if (codecInst == NULL) - { - format = kFileFormatPcm16kHzFile; - codecInst = &dummyCodec; - } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) - { - format = kFileFormatWavFile; - } else - { - format = kFileFormatCompressedFile; - } - - CriticalSectionScoped cs(&_critSect); - - // Destroy the old instance - if (_fileCallRecorderPtr) - { - _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); - FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); - _fileCallRecorderPtr = NULL; - } - - _fileCallRecorderPtr - = FileRecorder::CreateFileRecorder(_fileCallRecorderId, - (const FileFormats) format); - if (_fileCallRecorderPtr == NULL) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_ARGUMENT, kTraceError, - "StartRecordingCall() fileRecorder format isnot correct"); - return -1; - } - - if (_fileCallRecorderPtr->StartRecordingAudioFile( - fileName, - (const CodecInst&) *codecInst, - notificationTime) != 0) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_FILE, kTraceError, - "StartRecordingAudioFile() failed to start file recording"); - _fileCallRecorderPtr->StopRecording(); - FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); - _fileCallRecorderPtr = NULL; - return -1; - } - _fileCallRecorderPtr->RegisterModuleFileCallback(this); - _fileCallRecording = true; - - return 0; -} - -int TransmitMixer::StartRecordingCall(OutStream* stream, - const CodecInst* codecInst) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StartRecordingCall()"); - - if (_fileCallRecording) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "StartRecordingCall() is already recording"); - return 0; - } - - FileFormats format; - const uint32_t notificationTime(0); // Not supported in VoE - CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; - - if (codecInst != NULL && codecInst->channels != 1) - { - _engineStatisticsPtr->SetLastError( - VE_BAD_ARGUMENT, kTraceError, - "StartRecordingCall() invalid compression"); - return (-1); - } - if (codecInst == NULL) - { - format = kFileFormatPcm16kHzFile; - codecInst = &dummyCodec; - } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || - (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) - { - format = kFileFormatWavFile; - } else - { - format = kFileFormatCompressedFile; - } - - CriticalSectionScoped cs(&_critSect); - - // Destroy the old instance - if (_fileCallRecorderPtr) - { - _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); - FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); - _fileCallRecorderPtr = NULL; - } - - _fileCallRecorderPtr = - FileRecorder::CreateFileRecorder(_fileCallRecorderId, - (const FileFormats) format); - if (_fileCallRecorderPtr == NULL) - { - _engineStatisticsPtr->SetLastError( - VE_INVALID_ARGUMENT, kTraceError, - "StartRecordingCall() fileRecorder format isnot correct"); - return -1; - } - - if (_fileCallRecorderPtr->StartRecordingAudioFile(*stream, - *codecInst, - notificationTime) != 0) - { - _engineStatisticsPtr->SetLastError(VE_BAD_FILE, kTraceError, - "StartRecordingAudioFile() failed to start file recording"); - _fileCallRecorderPtr->StopRecording(); - FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); - _fileCallRecorderPtr = NULL; - return -1; - } - - _fileCallRecorderPtr->RegisterModuleFileCallback(this); - _fileCallRecording = true; - - return 0; -} - -int TransmitMixer::StopRecordingCall() -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::StopRecordingCall()"); - - if (!_fileCallRecording) - { - WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, -1), - "StopRecordingCall() file isnot recording"); - return -1; - } - - CriticalSectionScoped cs(&_critSect); - - if (_fileCallRecorderPtr->StopRecording() != 0) - { - _engineStatisticsPtr->SetLastError( - VE_STOP_RECORDING_FAILED, kTraceError, - "StopRecording(), could not stop recording"); - return -1; - } - - _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); - FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); - _fileCallRecorderPtr = NULL; - _fileCallRecording = false; - - return 0; -} - -void -TransmitMixer::SetMixWithMicStatus(bool mix) -{ - _mixFileWithMicrophone = mix; -} - -int TransmitMixer::RegisterExternalMediaProcessing( - VoEMediaProcess* object, - ProcessingTypes type) { - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::RegisterExternalMediaProcessing()"); - - CriticalSectionScoped cs(&_callbackCritSect); - if (!object) { - return -1; - } - - // Store the callback object according to the processing type. - if (type == kRecordingAllChannelsMixed) { - external_postproc_ptr_ = object; - } else if (type == kRecordingPreprocessing) { - external_preproc_ptr_ = object; - } else { - return -1; - } - return 0; -} - -int TransmitMixer::DeRegisterExternalMediaProcessing(ProcessingTypes type) { - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::DeRegisterExternalMediaProcessing()"); - - CriticalSectionScoped cs(&_callbackCritSect); - if (type == kRecordingAllChannelsMixed) { - external_postproc_ptr_ = NULL; - } else if (type == kRecordingPreprocessing) { - external_preproc_ptr_ = NULL; - } else { - return -1; - } - return 0; -} - -int -TransmitMixer::SetMute(bool enable) -{ - WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::SetMute(enable=%d)", enable); - _mute = enable; - return 0; -} - -bool -TransmitMixer::Mute() const -{ - return _mute; -} - -int8_t TransmitMixer::AudioLevel() const -{ - // Speech + file level [0,9] - return _audioLevel.Level(); -} - -int16_t TransmitMixer::AudioLevelFullRange() const -{ - // Speech + file level [0,32767] - return _audioLevel.LevelFullRange(); -} - -bool TransmitMixer::IsRecordingCall() -{ - return _fileCallRecording; -} - -bool TransmitMixer::IsRecordingMic() -{ - CriticalSectionScoped cs(&_critSect); - return _fileRecording; -} - -// TODO(andrew): use RemixAndResample for this. -int TransmitMixer::GenerateAudioFrame(const int16_t audio[], - int samples_per_channel, - int num_channels, - int sample_rate_hz) { - int destination_rate; - int num_codec_channels; - GetSendCodecInfo(&destination_rate, &num_codec_channels); - - // Never upsample the capture signal here. This should be done at the - // end of the send chain. - destination_rate = std::min(destination_rate, sample_rate_hz); - stereo_codec_ = num_codec_channels == 2; - - const int16_t* audio_ptr = audio; - int16_t mono_audio[kMaxMonoDeviceDataSizeSamples]; - assert(samples_per_channel <= kMaxMonoDeviceDataSizeSamples); - // If no stereo codecs are in use, we downmix a stereo stream from the - // device early in the chain, before resampling. - if (num_channels == 2 && !stereo_codec_) { - AudioFrameOperations::StereoToMono(audio, samples_per_channel, - mono_audio); - audio_ptr = mono_audio; - num_channels = 1; - } - - if (resampler_.InitializeIfNeeded(sample_rate_hz, - destination_rate, - num_channels) != 0) { - WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::GenerateAudioFrame() unable to resample"); - return -1; - } - - int out_length = resampler_.Resample(audio_ptr, - samples_per_channel * num_channels, - _audioFrame.data_, - AudioFrame::kMaxDataSizeSamples); - if (out_length == -1) { - WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::GenerateAudioFrame() resampling failed"); - return -1; - } - - _audioFrame.samples_per_channel_ = out_length / num_channels; - _audioFrame.id_ = _instanceId; - _audioFrame.timestamp_ = -1; - _audioFrame.sample_rate_hz_ = destination_rate; - _audioFrame.speech_type_ = AudioFrame::kNormalSpeech; - _audioFrame.vad_activity_ = AudioFrame::kVadUnknown; - _audioFrame.num_channels_ = num_channels; - - return 0; -} - -int32_t TransmitMixer::RecordAudioToFile( - uint32_t mixingFrequency) -{ - CriticalSectionScoped cs(&_critSect); - if (_fileRecorderPtr == NULL) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::RecordAudioToFile() filerecorder doesnot" - "exist"); - return -1; - } - - if (_fileRecorderPtr->RecordAudioToFile(_audioFrame) != 0) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::RecordAudioToFile() file recording" - "failed"); - return -1; - } - - return 0; -} - -int32_t TransmitMixer::MixOrReplaceAudioWithFile( - int mixingFrequency) -{ - scoped_array fileBuffer(new int16_t[640]); - - int fileSamples(0); - { - CriticalSectionScoped cs(&_critSect); - if (_filePlayerPtr == NULL) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, - VoEId(_instanceId, -1), - "TransmitMixer::MixOrReplaceAudioWithFile()" - "fileplayer doesnot exist"); - return -1; - } - - if (_filePlayerPtr->Get10msAudioFromFile(fileBuffer.get(), - fileSamples, - mixingFrequency) == -1) - { - WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), - "TransmitMixer::MixOrReplaceAudioWithFile() file" - " mixing failed"); - return -1; - } - } - - assert(_audioFrame.samples_per_channel_ == fileSamples); - - if (_mixFileWithMicrophone) - { - // Currently file stream is always mono. - // TODO(xians): Change the code when FilePlayer supports real stereo. - Utility::MixWithSat(_audioFrame.data_, - _audioFrame.num_channels_, - fileBuffer.get(), - 1, - fileSamples); - } else - { - // Replace ACM audio with file. - // Currently file stream is always mono. - // TODO(xians): Change the code when FilePlayer supports real stereo. - _audioFrame.UpdateFrame(-1, - -1, - fileBuffer.get(), - fileSamples, - mixingFrequency, - AudioFrame::kNormalSpeech, - AudioFrame::kVadUnknown, - 1); - } - return 0; -} - -void TransmitMixer::ProcessAudio(int delay_ms, int clock_drift, - int current_mic_level, bool key_pressed) { - if (audioproc_->set_stream_delay_ms(delay_ms) != 0) { - // A redundant warning is reported in AudioDevice, which we've throttled - // to avoid flooding the logs. Relegate this one to LS_VERBOSE to avoid - // repeating the problem here. - LOG_FERR1(LS_VERBOSE, set_stream_delay_ms, delay_ms); - } - - GainControl* agc = audioproc_->gain_control(); - if (agc->set_stream_analog_level(current_mic_level) != 0) { - LOG_FERR1(LS_ERROR, set_stream_analog_level, current_mic_level); - assert(false); - } - - EchoCancellation* aec = audioproc_->echo_cancellation(); - if (aec->is_drift_compensation_enabled()) { - aec->set_stream_drift_samples(clock_drift); - } - - audioproc_->set_stream_key_pressed(key_pressed); - - int err = audioproc_->ProcessStream(&_audioFrame); - if (err != 0) { - LOG(LS_ERROR) << "ProcessStream() error: " << err; - assert(false); - } - - // Store new capture level. Only updated when analog AGC is enabled. - _captureLevel = agc->stream_analog_level(); - - CriticalSectionScoped cs(&_critSect); - // Triggers a callback in OnPeriodicProcess(). - _saturationWarning |= agc->stream_is_saturated(); -} - -#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION -void TransmitMixer::TypingDetection(bool keyPressed) -{ - // We let the VAD determine if we're using this feature or not. - if (_audioFrame.vad_activity_ == AudioFrame::kVadUnknown) { - return; - } - - bool vadActive = _audioFrame.vad_activity_ == AudioFrame::kVadActive; - if (_typingDetection.Process(keyPressed, vadActive)) { - _typingNoiseWarningPending = true; - _typingNoiseDetected = true; - } else { - // If there is already a warning pending, do not change the state. - // Otherwise set a warning pending if last callback was for noise detected. - if (!_typingNoiseWarningPending && _typingNoiseDetected) { - _typingNoiseWarningPending = true; - _typingNoiseDetected = false; - } - } -} -#endif - -int TransmitMixer::GetMixingFrequency() -{ - assert(_audioFrame.sample_rate_hz_ != 0); - return _audioFrame.sample_rate_hz_; -} - -#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION -int TransmitMixer::TimeSinceLastTyping(int &seconds) -{ - // We check in VoEAudioProcessingImpl that this is only called when - // typing detection is active. - seconds = _typingDetection.TimeSinceLastDetectionInSeconds(); - return 0; -} -#endif - -#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION -int TransmitMixer::SetTypingDetectionParameters(int timeWindow, - int costPerTyping, - int reportingThreshold, - int penaltyDecay, - int typeEventDelay) -{ - _typingDetection.SetParameters(timeWindow, - costPerTyping, - reportingThreshold, - penaltyDecay, - typeEventDelay, - 0); - return 0; -} -#endif - -void TransmitMixer::EnableStereoChannelSwapping(bool enable) { - swap_stereo_channels_ = enable; -} - -bool TransmitMixer::IsStereoChannelSwappingEnabled() { - return swap_stereo_channels_; -} - -} // namespace voe - -} // namespace webrtc +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/voice_engine/transmit_mixer.h" + +#include "webrtc/modules/utility/interface/audio_frame_operations.h" +#include "webrtc/system_wrappers/interface/critical_section_wrapper.h" +#include "webrtc/system_wrappers/interface/event_wrapper.h" +#include "webrtc/system_wrappers/interface/logging.h" +#include "webrtc/system_wrappers/interface/trace.h" +#include "webrtc/voice_engine/channel.h" +#include "webrtc/voice_engine/channel_manager.h" +#include "webrtc/voice_engine/include/voe_external_media.h" +#include "webrtc/voice_engine/statistics.h" +#include "webrtc/voice_engine/utility.h" +#include "webrtc/voice_engine/voe_base_impl.h" + +#define WEBRTC_ABS(a) (((a) < 0) ? -(a) : (a)) + +namespace webrtc { +namespace voe { + +// TODO(ajm): The thread safety of this is dubious... +void +TransmitMixer::OnPeriodicProcess() +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::OnPeriodicProcess()"); + +#if defined(WEBRTC_VOICE_ENGINE_TYPING_DETECTION) + if (_typingNoiseWarningPending) + { + CriticalSectionScoped cs(&_callbackCritSect); + if (_voiceEngineObserverPtr) + { + if (_typingNoiseDetected) { + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::OnPeriodicProcess() => " + "CallbackOnError(VE_TYPING_NOISE_WARNING)"); + _voiceEngineObserverPtr->CallbackOnError( + -1, + VE_TYPING_NOISE_WARNING); + } else { + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::OnPeriodicProcess() => " + "CallbackOnError(VE_TYPING_NOISE_OFF_WARNING)"); + _voiceEngineObserverPtr->CallbackOnError( + -1, + VE_TYPING_NOISE_OFF_WARNING); + } + } + _typingNoiseWarningPending = false; + } +#endif + + bool saturationWarning = false; + { + // Modify |_saturationWarning| under lock to avoid conflict with write op + // in ProcessAudio and also ensure that we don't hold the lock during the + // callback. + CriticalSectionScoped cs(&_critSect); + saturationWarning = _saturationWarning; + if (_saturationWarning) + _saturationWarning = false; + } + + if (saturationWarning) + { + CriticalSectionScoped cs(&_callbackCritSect); + if (_voiceEngineObserverPtr) + { + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::OnPeriodicProcess() =>" + " CallbackOnError(VE_SATURATION_WARNING)"); + _voiceEngineObserverPtr->CallbackOnError(-1, VE_SATURATION_WARNING); + } + } +} + + +void TransmitMixer::PlayNotification(int32_t id, + uint32_t durationMs) +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::PlayNotification(id=%d, durationMs=%d)", + id, durationMs); + + // Not implement yet +} + +void TransmitMixer::RecordNotification(int32_t id, + uint32_t durationMs) +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1), + "TransmitMixer::RecordNotification(id=%d, durationMs=%d)", + id, durationMs); + + // Not implement yet +} + +void TransmitMixer::PlayFileEnded(int32_t id) +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::PlayFileEnded(id=%d)", id); + + assert(id == _filePlayerId); + + CriticalSectionScoped cs(&_critSect); + + _filePlaying = false; + WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::PlayFileEnded() =>" + "file player module is shutdown"); +} + +void +TransmitMixer::RecordFileEnded(int32_t id) +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::RecordFileEnded(id=%d)", id); + + if (id == _fileRecorderId) + { + CriticalSectionScoped cs(&_critSect); + _fileRecording = false; + WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::RecordFileEnded() => fileRecorder module" + "is shutdown"); + } else if (id == _fileCallRecorderId) + { + CriticalSectionScoped cs(&_critSect); + _fileCallRecording = false; + WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::RecordFileEnded() => fileCallRecorder" + "module is shutdown"); + } +} + +int32_t +TransmitMixer::Create(TransmitMixer*& mixer, uint32_t instanceId) +{ + WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, -1), + "TransmitMixer::Create(instanceId=%d)", instanceId); + mixer = new TransmitMixer(instanceId); + if (mixer == NULL) + { + WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, -1), + "TransmitMixer::Create() unable to allocate memory" + "for mixer"); + return -1; + } + return 0; +} + +void +TransmitMixer::Destroy(TransmitMixer*& mixer) +{ + if (mixer) + { + delete mixer; + mixer = NULL; + } +} + +TransmitMixer::TransmitMixer(uint32_t instanceId) : + _engineStatisticsPtr(NULL), + _channelManagerPtr(NULL), + audioproc_(NULL), + _voiceEngineObserverPtr(NULL), + _processThreadPtr(NULL), + _filePlayerPtr(NULL), + _fileRecorderPtr(NULL), + _fileCallRecorderPtr(NULL), + // Avoid conflict with other channels by adding 1024 - 1026, + // won't use as much as 1024 channels. + _filePlayerId(instanceId + 1024), + _fileRecorderId(instanceId + 1025), + _fileCallRecorderId(instanceId + 1026), + _filePlaying(false), + _fileRecording(false), + _fileCallRecording(false), + _audioLevel(), + _critSect(*CriticalSectionWrapper::CreateCriticalSection()), + _callbackCritSect(*CriticalSectionWrapper::CreateCriticalSection()), +#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION + _typingNoiseWarningPending(false), + _typingNoiseDetected(false), +#endif + _saturationWarning(false), + _instanceId(instanceId), + _mixFileWithMicrophone(false), + _captureLevel(0), + external_postproc_ptr_(NULL), + external_preproc_ptr_(NULL), + _mute(false), + _remainingMuteMicTimeMs(0), + stereo_codec_(false), + swap_stereo_channels_(false) +{ + WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::TransmitMixer() - ctor"); +} + +TransmitMixer::~TransmitMixer() +{ + WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::~TransmitMixer() - dtor"); + _monitorModule.DeRegisterObserver(); + if (_processThreadPtr) + { + _processThreadPtr->DeRegisterModule(&_monitorModule); + } + DeRegisterExternalMediaProcessing(kRecordingAllChannelsMixed); + DeRegisterExternalMediaProcessing(kRecordingPreprocessing); + { + CriticalSectionScoped cs(&_critSect); + if (_fileRecorderPtr) + { + _fileRecorderPtr->RegisterModuleFileCallback(NULL); + _fileRecorderPtr->StopRecording(); + FileRecorder::DestroyFileRecorder(_fileRecorderPtr); + _fileRecorderPtr = NULL; + } + if (_fileCallRecorderPtr) + { + _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); + _fileCallRecorderPtr->StopRecording(); + FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); + _fileCallRecorderPtr = NULL; + } + if (_filePlayerPtr) + { + _filePlayerPtr->RegisterModuleFileCallback(NULL); + _filePlayerPtr->StopPlayingFile(); + FilePlayer::DestroyFilePlayer(_filePlayerPtr); + _filePlayerPtr = NULL; + } + } + delete &_critSect; + delete &_callbackCritSect; +} + +int32_t +TransmitMixer::SetEngineInformation(ProcessThread& processThread, + Statistics& engineStatistics, + ChannelManager& channelManager) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::SetEngineInformation()"); + + _processThreadPtr = &processThread; + _engineStatisticsPtr = &engineStatistics; + _channelManagerPtr = &channelManager; + + if (_processThreadPtr->RegisterModule(&_monitorModule) == -1) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::SetEngineInformation() failed to" + "register the monitor module"); + } else + { + _monitorModule.RegisterObserver(*this); + } + + return 0; +} + +int32_t +TransmitMixer::RegisterVoiceEngineObserver(VoiceEngineObserver& observer) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::RegisterVoiceEngineObserver()"); + CriticalSectionScoped cs(&_callbackCritSect); + + if (_voiceEngineObserverPtr) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_OPERATION, kTraceError, + "RegisterVoiceEngineObserver() observer already enabled"); + return -1; + } + _voiceEngineObserverPtr = &observer; + return 0; +} + +int32_t +TransmitMixer::SetAudioProcessingModule(AudioProcessing* audioProcessingModule) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::SetAudioProcessingModule(" + "audioProcessingModule=0x%x)", + audioProcessingModule); + audioproc_ = audioProcessingModule; + return 0; +} + +void TransmitMixer::GetSendCodecInfo(int* max_sample_rate, int* max_channels) { + *max_sample_rate = 8000; + *max_channels = 1; + for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); + it.Increment()) { + Channel* channel = it.GetChannel(); + if (channel->Sending()) { + CodecInst codec; + channel->GetSendCodec(codec); + *max_sample_rate = std::max(*max_sample_rate, codec.plfreq); + *max_channels = std::max(*max_channels, codec.channels); + } + } +} + +int32_t +TransmitMixer::PrepareDemux(const void* audioSamples, + uint32_t nSamples, + uint8_t nChannels, + uint32_t samplesPerSec, + uint16_t totalDelayMS, + int32_t clockDrift, + uint16_t currentMicLevel, + bool keyPressed) +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::PrepareDemux(nSamples=%u, nChannels=%u," + "samplesPerSec=%u, totalDelayMS=%u, clockDrift=%d," + "currentMicLevel=%u)", nSamples, nChannels, samplesPerSec, + totalDelayMS, clockDrift, currentMicLevel); + + // --- Resample input audio and create/store the initial audio frame + GenerateAudioFrame(static_cast(audioSamples), + nSamples, + nChannels, + samplesPerSec); + + { + CriticalSectionScoped cs(&_callbackCritSect); + if (external_preproc_ptr_) { + external_preproc_ptr_->Process(-1, kRecordingPreprocessing, + _audioFrame.data_, + _audioFrame.samples_per_channel_, + _audioFrame.sample_rate_hz_, + _audioFrame.num_channels_ == 2); + } + } + + // --- Near-end audio processing. + ProcessAudio(totalDelayMS, clockDrift, currentMicLevel, keyPressed); + + if (swap_stereo_channels_ && stereo_codec_) + // Only bother swapping if we're using a stereo codec. + AudioFrameOperations::SwapStereoChannels(&_audioFrame); + + // --- Annoying typing detection (utilizes the APM/VAD decision) +#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION + TypingDetection(keyPressed); +#endif + + // --- Mute during DTMF tone if direct feedback is enabled + if (_remainingMuteMicTimeMs > 0) + { + AudioFrameOperations::Mute(_audioFrame); + _remainingMuteMicTimeMs -= 10; + if (_remainingMuteMicTimeMs < 0) + { + _remainingMuteMicTimeMs = 0; + } + } + + // --- Mute signal + if (_mute) + { + AudioFrameOperations::Mute(_audioFrame); + } + + // --- Mix with file (does not affect the mixing frequency) + if (_filePlaying) + { + MixOrReplaceAudioWithFile(_audioFrame.sample_rate_hz_); + } + + // --- Record to file + bool file_recording = false; + { + CriticalSectionScoped cs(&_critSect); + file_recording = _fileRecording; + } + if (file_recording) + { + RecordAudioToFile(_audioFrame.sample_rate_hz_); + } + + { + CriticalSectionScoped cs(&_callbackCritSect); + if (external_postproc_ptr_) { + external_postproc_ptr_->Process(-1, kRecordingAllChannelsMixed, + _audioFrame.data_, + _audioFrame.samples_per_channel_, + _audioFrame.sample_rate_hz_, + _audioFrame.num_channels_ == 2); + } + } + + // --- Measure audio level of speech after all processing. + _audioLevel.ComputeLevel(_audioFrame); + return 0; +} + +int32_t +TransmitMixer::DemuxAndMix() +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::DemuxAndMix()"); + + for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); + it.Increment()) + { + Channel* channelPtr = it.GetChannel(); + if (channelPtr->InputIsOnHold()) + { + channelPtr->UpdateLocalTimeStamp(); + } else if (channelPtr->Sending()) + { + // Demultiplex makes a copy of its input. + channelPtr->Demultiplex(_audioFrame); + channelPtr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_); + } + } + return 0; +} + +void TransmitMixer::DemuxAndMix(const int voe_channels[], + int number_of_voe_channels) { + for (int i = 0; i < number_of_voe_channels; ++i) { + voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]); + voe::Channel* channel_ptr = ch.channel(); + if (channel_ptr) { + if (channel_ptr->InputIsOnHold()) { + channel_ptr->UpdateLocalTimeStamp(); + } else if (channel_ptr->Sending()) { + // Demultiplex makes a copy of its input. + channel_ptr->Demultiplex(_audioFrame); + channel_ptr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_); + } + } + } +} + +int32_t +TransmitMixer::EncodeAndSend() +{ + WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::EncodeAndSend()"); + + for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); + it.Increment()) + { + Channel* channelPtr = it.GetChannel(); + if (channelPtr->Sending() && !channelPtr->InputIsOnHold()) + { + channelPtr->EncodeAndSend(); + } + } + return 0; +} + +void TransmitMixer::EncodeAndSend(const int voe_channels[], + int number_of_voe_channels) { + for (int i = 0; i < number_of_voe_channels; ++i) { + voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]); + voe::Channel* channel_ptr = ch.channel(); + if (channel_ptr && channel_ptr->Sending() && !channel_ptr->InputIsOnHold()) + channel_ptr->EncodeAndSend(); + } +} + +uint32_t TransmitMixer::CaptureLevel() const +{ + return _captureLevel; +} + +void +TransmitMixer::UpdateMuteMicrophoneTime(uint32_t lengthMs) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::UpdateMuteMicrophoneTime(lengthMs=%d)", + lengthMs); + _remainingMuteMicTimeMs = lengthMs; +} + +int32_t +TransmitMixer::StopSend() +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StopSend()"); + _audioLevel.Clear(); + return 0; +} + +int TransmitMixer::StartPlayingFileAsMicrophone(const char* fileName, + bool loop, + FileFormats format, + int startPosition, + float volumeScaling, + int stopPosition, + const CodecInst* codecInst) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StartPlayingFileAsMicrophone(" + "fileNameUTF8[]=%s,loop=%d, format=%d, volumeScaling=%5.3f," + " startPosition=%d, stopPosition=%d)", fileName, loop, + format, volumeScaling, startPosition, stopPosition); + + if (_filePlaying) + { + _engineStatisticsPtr->SetLastError( + VE_ALREADY_PLAYING, kTraceWarning, + "StartPlayingFileAsMicrophone() is already playing"); + return 0; + } + + CriticalSectionScoped cs(&_critSect); + + // Destroy the old instance + if (_filePlayerPtr) + { + _filePlayerPtr->RegisterModuleFileCallback(NULL); + FilePlayer::DestroyFilePlayer(_filePlayerPtr); + _filePlayerPtr = NULL; + } + + // Dynamically create the instance + _filePlayerPtr + = FilePlayer::CreateFilePlayer(_filePlayerId, + (const FileFormats) format); + + if (_filePlayerPtr == NULL) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_ARGUMENT, kTraceError, + "StartPlayingFileAsMicrophone() filePlayer format isnot correct"); + return -1; + } + + const uint32_t notificationTime(0); + + if (_filePlayerPtr->StartPlayingFile( + fileName, + loop, + startPosition, + volumeScaling, + notificationTime, + stopPosition, + (const CodecInst*) codecInst) != 0) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_FILE, kTraceError, + "StartPlayingFile() failed to start file playout"); + _filePlayerPtr->StopPlayingFile(); + FilePlayer::DestroyFilePlayer(_filePlayerPtr); + _filePlayerPtr = NULL; + return -1; + } + + _filePlayerPtr->RegisterModuleFileCallback(this); + _filePlaying = true; + + return 0; +} + +int TransmitMixer::StartPlayingFileAsMicrophone(InStream* stream, + FileFormats format, + int startPosition, + float volumeScaling, + int stopPosition, + const CodecInst* codecInst) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1), + "TransmitMixer::StartPlayingFileAsMicrophone(format=%d," + " volumeScaling=%5.3f, startPosition=%d, stopPosition=%d)", + format, volumeScaling, startPosition, stopPosition); + + if (stream == NULL) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_FILE, kTraceError, + "StartPlayingFileAsMicrophone() NULL as input stream"); + return -1; + } + + if (_filePlaying) + { + _engineStatisticsPtr->SetLastError( + VE_ALREADY_PLAYING, kTraceWarning, + "StartPlayingFileAsMicrophone() is already playing"); + return 0; + } + + CriticalSectionScoped cs(&_critSect); + + // Destroy the old instance + if (_filePlayerPtr) + { + _filePlayerPtr->RegisterModuleFileCallback(NULL); + FilePlayer::DestroyFilePlayer(_filePlayerPtr); + _filePlayerPtr = NULL; + } + + // Dynamically create the instance + _filePlayerPtr + = FilePlayer::CreateFilePlayer(_filePlayerId, + (const FileFormats) format); + + if (_filePlayerPtr == NULL) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_ARGUMENT, kTraceWarning, + "StartPlayingFileAsMicrophone() filePlayer format isnot correct"); + return -1; + } + + const uint32_t notificationTime(0); + + if (_filePlayerPtr->StartPlayingFile( + (InStream&) *stream, + startPosition, + volumeScaling, + notificationTime, + stopPosition, + (const CodecInst*) codecInst) != 0) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_FILE, kTraceError, + "StartPlayingFile() failed to start file playout"); + _filePlayerPtr->StopPlayingFile(); + FilePlayer::DestroyFilePlayer(_filePlayerPtr); + _filePlayerPtr = NULL; + return -1; + } + _filePlayerPtr->RegisterModuleFileCallback(this); + _filePlaying = true; + + return 0; +} + +int TransmitMixer::StopPlayingFileAsMicrophone() +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1), + "TransmitMixer::StopPlayingFileAsMicrophone()"); + + if (!_filePlaying) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_OPERATION, kTraceWarning, + "StopPlayingFileAsMicrophone() isnot playing"); + return 0; + } + + CriticalSectionScoped cs(&_critSect); + + if (_filePlayerPtr->StopPlayingFile() != 0) + { + _engineStatisticsPtr->SetLastError( + VE_CANNOT_STOP_PLAYOUT, kTraceError, + "StopPlayingFile() couldnot stop playing file"); + return -1; + } + + _filePlayerPtr->RegisterModuleFileCallback(NULL); + FilePlayer::DestroyFilePlayer(_filePlayerPtr); + _filePlayerPtr = NULL; + _filePlaying = false; + + return 0; +} + +int TransmitMixer::IsPlayingFileAsMicrophone() const +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::IsPlayingFileAsMicrophone()"); + return _filePlaying; +} + +int TransmitMixer::ScaleFileAsMicrophonePlayout(float scale) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::ScaleFileAsMicrophonePlayout(scale=%5.3f)", + scale); + + CriticalSectionScoped cs(&_critSect); + + if (!_filePlaying) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_OPERATION, kTraceError, + "ScaleFileAsMicrophonePlayout() isnot playing file"); + return -1; + } + + if ((_filePlayerPtr == NULL) || + (_filePlayerPtr->SetAudioScaling(scale) != 0)) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_ARGUMENT, kTraceError, + "SetAudioScaling() failed to scale playout"); + return -1; + } + + return 0; +} + +int TransmitMixer::StartRecordingMicrophone(const char* fileName, + const CodecInst* codecInst) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StartRecordingMicrophone(fileName=%s)", + fileName); + + CriticalSectionScoped cs(&_critSect); + + if (_fileRecording) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "StartRecordingMicrophone() is already recording"); + return 0; + } + + FileFormats format; + const uint32_t notificationTime(0); // Not supported in VoE + CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; + + if (codecInst != NULL && + (codecInst->channels < 0 || codecInst->channels > 2)) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_ARGUMENT, kTraceError, + "StartRecordingMicrophone() invalid compression"); + return (-1); + } + if (codecInst == NULL) + { + format = kFileFormatPcm16kHzFile; + codecInst = &dummyCodec; + } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) + { + format = kFileFormatWavFile; + } else + { + format = kFileFormatCompressedFile; + } + + // Destroy the old instance + if (_fileRecorderPtr) + { + _fileRecorderPtr->RegisterModuleFileCallback(NULL); + FileRecorder::DestroyFileRecorder(_fileRecorderPtr); + _fileRecorderPtr = NULL; + } + + _fileRecorderPtr = + FileRecorder::CreateFileRecorder(_fileRecorderId, + (const FileFormats) format); + if (_fileRecorderPtr == NULL) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_ARGUMENT, kTraceError, + "StartRecordingMicrophone() fileRecorder format isnot correct"); + return -1; + } + + if (_fileRecorderPtr->StartRecordingAudioFile( + fileName, + (const CodecInst&) *codecInst, + notificationTime) != 0) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_FILE, kTraceError, + "StartRecordingAudioFile() failed to start file recording"); + _fileRecorderPtr->StopRecording(); + FileRecorder::DestroyFileRecorder(_fileRecorderPtr); + _fileRecorderPtr = NULL; + return -1; + } + _fileRecorderPtr->RegisterModuleFileCallback(this); + _fileRecording = true; + + return 0; +} + +int TransmitMixer::StartRecordingMicrophone(OutStream* stream, + const CodecInst* codecInst) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StartRecordingMicrophone()"); + + CriticalSectionScoped cs(&_critSect); + + if (_fileRecording) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "StartRecordingMicrophone() is already recording"); + return 0; + } + + FileFormats format; + const uint32_t notificationTime(0); // Not supported in VoE + CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; + + if (codecInst != NULL && codecInst->channels != 1) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_ARGUMENT, kTraceError, + "StartRecordingMicrophone() invalid compression"); + return (-1); + } + if (codecInst == NULL) + { + format = kFileFormatPcm16kHzFile; + codecInst = &dummyCodec; + } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) + { + format = kFileFormatWavFile; + } else + { + format = kFileFormatCompressedFile; + } + + // Destroy the old instance + if (_fileRecorderPtr) + { + _fileRecorderPtr->RegisterModuleFileCallback(NULL); + FileRecorder::DestroyFileRecorder(_fileRecorderPtr); + _fileRecorderPtr = NULL; + } + + _fileRecorderPtr = + FileRecorder::CreateFileRecorder(_fileRecorderId, + (const FileFormats) format); + if (_fileRecorderPtr == NULL) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_ARGUMENT, kTraceError, + "StartRecordingMicrophone() fileRecorder format isnot correct"); + return -1; + } + + if (_fileRecorderPtr->StartRecordingAudioFile(*stream, + *codecInst, + notificationTime) != 0) + { + _engineStatisticsPtr->SetLastError(VE_BAD_FILE, kTraceError, + "StartRecordingAudioFile() failed to start file recording"); + _fileRecorderPtr->StopRecording(); + FileRecorder::DestroyFileRecorder(_fileRecorderPtr); + _fileRecorderPtr = NULL; + return -1; + } + + _fileRecorderPtr->RegisterModuleFileCallback(this); + _fileRecording = true; + + return 0; +} + + +int TransmitMixer::StopRecordingMicrophone() +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StopRecordingMicrophone()"); + + CriticalSectionScoped cs(&_critSect); + + if (!_fileRecording) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "StopRecordingMicrophone() isnot recording"); + return 0; + } + + if (_fileRecorderPtr->StopRecording() != 0) + { + _engineStatisticsPtr->SetLastError( + VE_STOP_RECORDING_FAILED, kTraceError, + "StopRecording(), could not stop recording"); + return -1; + } + _fileRecorderPtr->RegisterModuleFileCallback(NULL); + FileRecorder::DestroyFileRecorder(_fileRecorderPtr); + _fileRecorderPtr = NULL; + _fileRecording = false; + + return 0; +} + +int TransmitMixer::StartRecordingCall(const char* fileName, + const CodecInst* codecInst) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StartRecordingCall(fileName=%s)", fileName); + + if (_fileCallRecording) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "StartRecordingCall() is already recording"); + return 0; + } + + FileFormats format; + const uint32_t notificationTime(0); // Not supported in VoE + CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; + + if (codecInst != NULL && codecInst->channels != 1) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_ARGUMENT, kTraceError, + "StartRecordingCall() invalid compression"); + return (-1); + } + if (codecInst == NULL) + { + format = kFileFormatPcm16kHzFile; + codecInst = &dummyCodec; + } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) + { + format = kFileFormatWavFile; + } else + { + format = kFileFormatCompressedFile; + } + + CriticalSectionScoped cs(&_critSect); + + // Destroy the old instance + if (_fileCallRecorderPtr) + { + _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); + FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); + _fileCallRecorderPtr = NULL; + } + + _fileCallRecorderPtr + = FileRecorder::CreateFileRecorder(_fileCallRecorderId, + (const FileFormats) format); + if (_fileCallRecorderPtr == NULL) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_ARGUMENT, kTraceError, + "StartRecordingCall() fileRecorder format isnot correct"); + return -1; + } + + if (_fileCallRecorderPtr->StartRecordingAudioFile( + fileName, + (const CodecInst&) *codecInst, + notificationTime) != 0) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_FILE, kTraceError, + "StartRecordingAudioFile() failed to start file recording"); + _fileCallRecorderPtr->StopRecording(); + FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); + _fileCallRecorderPtr = NULL; + return -1; + } + _fileCallRecorderPtr->RegisterModuleFileCallback(this); + _fileCallRecording = true; + + return 0; +} + +int TransmitMixer::StartRecordingCall(OutStream* stream, + const CodecInst* codecInst) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StartRecordingCall()"); + + if (_fileCallRecording) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "StartRecordingCall() is already recording"); + return 0; + } + + FileFormats format; + const uint32_t notificationTime(0); // Not supported in VoE + CodecInst dummyCodec = { 100, "L16", 16000, 320, 1, 320000 }; + + if (codecInst != NULL && codecInst->channels != 1) + { + _engineStatisticsPtr->SetLastError( + VE_BAD_ARGUMENT, kTraceError, + "StartRecordingCall() invalid compression"); + return (-1); + } + if (codecInst == NULL) + { + format = kFileFormatPcm16kHzFile; + codecInst = &dummyCodec; + } else if ((STR_CASE_CMP(codecInst->plname,"L16") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) || + (STR_CASE_CMP(codecInst->plname,"PCMA") == 0)) + { + format = kFileFormatWavFile; + } else + { + format = kFileFormatCompressedFile; + } + + CriticalSectionScoped cs(&_critSect); + + // Destroy the old instance + if (_fileCallRecorderPtr) + { + _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); + FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); + _fileCallRecorderPtr = NULL; + } + + _fileCallRecorderPtr = + FileRecorder::CreateFileRecorder(_fileCallRecorderId, + (const FileFormats) format); + if (_fileCallRecorderPtr == NULL) + { + _engineStatisticsPtr->SetLastError( + VE_INVALID_ARGUMENT, kTraceError, + "StartRecordingCall() fileRecorder format isnot correct"); + return -1; + } + + if (_fileCallRecorderPtr->StartRecordingAudioFile(*stream, + *codecInst, + notificationTime) != 0) + { + _engineStatisticsPtr->SetLastError(VE_BAD_FILE, kTraceError, + "StartRecordingAudioFile() failed to start file recording"); + _fileCallRecorderPtr->StopRecording(); + FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); + _fileCallRecorderPtr = NULL; + return -1; + } + + _fileCallRecorderPtr->RegisterModuleFileCallback(this); + _fileCallRecording = true; + + return 0; +} + +int TransmitMixer::StopRecordingCall() +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::StopRecordingCall()"); + + if (!_fileCallRecording) + { + WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, -1), + "StopRecordingCall() file isnot recording"); + return -1; + } + + CriticalSectionScoped cs(&_critSect); + + if (_fileCallRecorderPtr->StopRecording() != 0) + { + _engineStatisticsPtr->SetLastError( + VE_STOP_RECORDING_FAILED, kTraceError, + "StopRecording(), could not stop recording"); + return -1; + } + + _fileCallRecorderPtr->RegisterModuleFileCallback(NULL); + FileRecorder::DestroyFileRecorder(_fileCallRecorderPtr); + _fileCallRecorderPtr = NULL; + _fileCallRecording = false; + + return 0; +} + +void +TransmitMixer::SetMixWithMicStatus(bool mix) +{ + _mixFileWithMicrophone = mix; +} + +int TransmitMixer::RegisterExternalMediaProcessing( + VoEMediaProcess* object, + ProcessingTypes type) { + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::RegisterExternalMediaProcessing()"); + + CriticalSectionScoped cs(&_callbackCritSect); + if (!object) { + return -1; + } + + // Store the callback object according to the processing type. + if (type == kRecordingAllChannelsMixed) { + external_postproc_ptr_ = object; + } else if (type == kRecordingPreprocessing) { + external_preproc_ptr_ = object; + } else { + return -1; + } + return 0; +} + +int TransmitMixer::DeRegisterExternalMediaProcessing(ProcessingTypes type) { + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::DeRegisterExternalMediaProcessing()"); + + CriticalSectionScoped cs(&_callbackCritSect); + if (type == kRecordingAllChannelsMixed) { + external_postproc_ptr_ = NULL; + } else if (type == kRecordingPreprocessing) { + external_preproc_ptr_ = NULL; + } else { + return -1; + } + return 0; +} + +int +TransmitMixer::SetMute(bool enable) +{ + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::SetMute(enable=%d)", enable); + _mute = enable; + return 0; +} + +bool +TransmitMixer::Mute() const +{ + return _mute; +} + +int8_t TransmitMixer::AudioLevel() const +{ + // Speech + file level [0,9] + return _audioLevel.Level(); +} + +int16_t TransmitMixer::AudioLevelFullRange() const +{ + // Speech + file level [0,32767] + return _audioLevel.LevelFullRange(); +} + +bool TransmitMixer::IsRecordingCall() +{ + return _fileCallRecording; +} + +bool TransmitMixer::IsRecordingMic() +{ + CriticalSectionScoped cs(&_critSect); + return _fileRecording; +} + +void TransmitMixer::GenerateAudioFrame(const int16_t* audio, + int samples_per_channel, + int num_channels, + int sample_rate_hz) { + int codec_rate; + int num_codec_channels; + GetSendCodecInfo(&codec_rate, &num_codec_channels); + // TODO(ajm): This currently restricts the sample rate to 32 kHz. + // See: https://code.google.com/p/webrtc/issues/detail?id=3146 + // When 48 kHz is supported natively by AudioProcessing, this will have + // to be changed to handle 44.1 kHz. + codec_rate = std::min(codec_rate, kAudioProcMaxNativeSampleRateHz); + stereo_codec_ = num_codec_channels == 2; + + if (!mono_buffer_.get()) { + // Temporary space for DownConvertToCodecFormat. + mono_buffer_.reset(new int16_t[kMaxMonoDataSizeSamples]); + } + DownConvertToCodecFormat(audio, + samples_per_channel, + num_channels, + sample_rate_hz, + num_codec_channels, + codec_rate, + mono_buffer_.get(), + &resampler_, + &_audioFrame); +} + +int32_t TransmitMixer::RecordAudioToFile( + uint32_t mixingFrequency) +{ + CriticalSectionScoped cs(&_critSect); + if (_fileRecorderPtr == NULL) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::RecordAudioToFile() filerecorder doesnot" + "exist"); + return -1; + } + + if (_fileRecorderPtr->RecordAudioToFile(_audioFrame) != 0) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::RecordAudioToFile() file recording" + "failed"); + return -1; + } + + return 0; +} + +int32_t TransmitMixer::MixOrReplaceAudioWithFile( + int mixingFrequency) +{ + scoped_array fileBuffer(new int16_t[640]); + + int fileSamples(0); + { + CriticalSectionScoped cs(&_critSect); + if (_filePlayerPtr == NULL) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, + VoEId(_instanceId, -1), + "TransmitMixer::MixOrReplaceAudioWithFile()" + "fileplayer doesnot exist"); + return -1; + } + + if (_filePlayerPtr->Get10msAudioFromFile(fileBuffer.get(), + fileSamples, + mixingFrequency) == -1) + { + WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1), + "TransmitMixer::MixOrReplaceAudioWithFile() file" + " mixing failed"); + return -1; + } + } + + assert(_audioFrame.samples_per_channel_ == fileSamples); + + if (_mixFileWithMicrophone) + { + // Currently file stream is always mono. + // TODO(xians): Change the code when FilePlayer supports real stereo. + MixWithSat(_audioFrame.data_, + _audioFrame.num_channels_, + fileBuffer.get(), + 1, + fileSamples); + } else + { + // Replace ACM audio with file. + // Currently file stream is always mono. + // TODO(xians): Change the code when FilePlayer supports real stereo. + _audioFrame.UpdateFrame(-1, + -1, + fileBuffer.get(), + fileSamples, + mixingFrequency, + AudioFrame::kNormalSpeech, + AudioFrame::kVadUnknown, + 1); + } + return 0; +} + +void TransmitMixer::ProcessAudio(int delay_ms, int clock_drift, + int current_mic_level, bool key_pressed) { + if (audioproc_->set_stream_delay_ms(delay_ms) != 0) { + // A redundant warning is reported in AudioDevice, which we've throttled + // to avoid flooding the logs. Relegate this one to LS_VERBOSE to avoid + // repeating the problem here. + LOG_FERR1(LS_VERBOSE, set_stream_delay_ms, delay_ms); + } + + GainControl* agc = audioproc_->gain_control(); + if (agc->set_stream_analog_level(current_mic_level) != 0) { + LOG_FERR1(LS_ERROR, set_stream_analog_level, current_mic_level); + assert(false); + } + + EchoCancellation* aec = audioproc_->echo_cancellation(); + if (aec->is_drift_compensation_enabled()) { + aec->set_stream_drift_samples(clock_drift); + } + + audioproc_->set_stream_key_pressed(key_pressed); + + int err = audioproc_->ProcessStream(&_audioFrame); + if (err != 0) { + LOG(LS_ERROR) << "ProcessStream() error: " << err; + assert(false); + } + + // Store new capture level. Only updated when analog AGC is enabled. + _captureLevel = agc->stream_analog_level(); + + CriticalSectionScoped cs(&_critSect); + // Triggers a callback in OnPeriodicProcess(). + _saturationWarning |= agc->stream_is_saturated(); +} + +#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION +void TransmitMixer::TypingDetection(bool keyPressed) +{ + // We let the VAD determine if we're using this feature or not. + if (_audioFrame.vad_activity_ == AudioFrame::kVadUnknown) { + return; + } + + bool vadActive = _audioFrame.vad_activity_ == AudioFrame::kVadActive; + if (_typingDetection.Process(keyPressed, vadActive)) { + _typingNoiseWarningPending = true; + _typingNoiseDetected = true; + } else { + // If there is already a warning pending, do not change the state. + // Otherwise set a warning pending if last callback was for noise detected. + if (!_typingNoiseWarningPending && _typingNoiseDetected) { + _typingNoiseWarningPending = true; + _typingNoiseDetected = false; + } + } +} +#endif + +int TransmitMixer::GetMixingFrequency() +{ + assert(_audioFrame.sample_rate_hz_ != 0); + return _audioFrame.sample_rate_hz_; +} + +#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION +int TransmitMixer::TimeSinceLastTyping(int &seconds) +{ + // We check in VoEAudioProcessingImpl that this is only called when + // typing detection is active. + seconds = _typingDetection.TimeSinceLastDetectionInSeconds(); + return 0; +} +#endif + +#ifdef WEBRTC_VOICE_ENGINE_TYPING_DETECTION +int TransmitMixer::SetTypingDetectionParameters(int timeWindow, + int costPerTyping, + int reportingThreshold, + int penaltyDecay, + int typeEventDelay) +{ + _typingDetection.SetParameters(timeWindow, + costPerTyping, + reportingThreshold, + penaltyDecay, + typeEventDelay, + 0); + return 0; +} +#endif + +void TransmitMixer::EnableStereoChannelSwapping(bool enable) { + swap_stereo_channels_ = enable; +} + +bool TransmitMixer::IsStereoChannelSwappingEnabled() { + return swap_stereo_channels_; +} + +} // namespace voe +} // namespace webrtc diff --git a/webrtc/voice_engine/transmit_mixer.h b/webrtc/voice_engine/transmit_mixer.h index 58bc73de2..b408614ae 100644 --- a/webrtc/voice_engine/transmit_mixer.h +++ b/webrtc/voice_engine/transmit_mixer.h @@ -17,6 +17,7 @@ #include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/utility/interface/file_player.h" #include "webrtc/modules/utility/interface/file_recorder.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" #include "webrtc/voice_engine/include/voe_base.h" #include "webrtc/voice_engine/level_indicator.h" #include "webrtc/voice_engine/monitor_module.h" @@ -36,9 +37,7 @@ class MixedAudio; class Statistics; class TransmitMixer : public MonitorObserver, - public FileCallback - -{ + public FileCallback { public: static int32_t Create(TransmitMixer*& mixer, uint32_t instanceId); @@ -175,10 +174,10 @@ private: // sending codecs. void GetSendCodecInfo(int* max_sample_rate, int* max_channels); - int GenerateAudioFrame(const int16_t audioSamples[], - int nSamples, - int nChannels, - int samplesPerSec); + void GenerateAudioFrame(const int16_t audioSamples[], + int nSamples, + int nChannels, + int samplesPerSec); int32_t RecordAudioToFile(uint32_t mixingFrequency); int32_t MixOrReplaceAudioWithFile( @@ -232,6 +231,7 @@ private: int32_t _remainingMuteMicTimeMs; bool stereo_codec_; bool swap_stereo_channels_; + scoped_ptr mono_buffer_; }; } // namespace voe diff --git a/webrtc/voice_engine/utility.cc b/webrtc/voice_engine/utility.cc index 5b7ee8149..5058aa321 100644 --- a/webrtc/voice_engine/utility.cc +++ b/webrtc/voice_engine/utility.cc @@ -10,116 +10,150 @@ #include "webrtc/voice_engine/utility.h" +#include "webrtc/common_audio/resampler/include/push_resampler.h" #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" -#include "webrtc/modules/interface/module.h" -#include "webrtc/system_wrappers/interface/trace.h" +#include "webrtc/common_types.h" +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/modules/utility/interface/audio_frame_operations.h" +#include "webrtc/system_wrappers/interface/logging.h" +#include "webrtc/voice_engine/voice_engine_defines.h" -namespace webrtc -{ +namespace webrtc { +namespace voe { -namespace voe -{ -enum{kMaxTargetLen = 2*32*10}; // stereo 32KHz 10ms +// TODO(ajm): There is significant overlap between RemixAndResample and +// ConvertToCodecFormat, but if we're to consolidate we should probably make a +// real converter class. +void RemixAndResample(const AudioFrame& src_frame, + PushResampler* resampler, + AudioFrame* dst_frame) { + const int16_t* audio_ptr = src_frame.data_; + int audio_ptr_num_channels = src_frame.num_channels_; + int16_t mono_audio[AudioFrame::kMaxDataSizeSamples]; -void Utility::MixWithSat(int16_t target[], - int target_channel, - const int16_t source[], - int source_channel, - int source_len) -{ - assert((target_channel == 1) || (target_channel == 2)); - assert((source_channel == 1) || (source_channel == 2)); - assert(source_len <= kMaxTargetLen); + // Downmix before resampling. + if (src_frame.num_channels_ == 2 && dst_frame->num_channels_ == 1) { + AudioFrameOperations::StereoToMono(src_frame.data_, + src_frame.samples_per_channel_, + mono_audio); + audio_ptr = mono_audio; + audio_ptr_num_channels = 1; + } - if ((target_channel == 2) && (source_channel == 1)) - { - // Convert source from mono to stereo. - int32_t left = 0; - int32_t right = 0; - for (int i = 0; i < source_len; ++i) { - left = source[i] + target[i*2]; - right = source[i] + target[i*2 + 1]; - target[i*2] = WebRtcSpl_SatW32ToW16(left); - target[i*2 + 1] = WebRtcSpl_SatW32ToW16(right); - } - } - else if ((target_channel == 1) && (source_channel == 2)) - { - // Convert source from stereo to mono. - int32_t temp = 0; - for (int i = 0; i < source_len/2; ++i) { - temp = ((source[i*2] + source[i*2 + 1])>>1) + target[i]; - target[i] = WebRtcSpl_SatW32ToW16(temp); - } - } - else - { - int32_t temp = 0; - for (int i = 0; i < source_len; ++i) { - temp = source[i] + target[i]; - target[i] = WebRtcSpl_SatW32ToW16(temp); - } - } + if (resampler->InitializeIfNeeded(src_frame.sample_rate_hz_, + dst_frame->sample_rate_hz_, + audio_ptr_num_channels) == -1) { + dst_frame->CopyFrom(src_frame); + LOG_FERR3(LS_ERROR, InitializeIfNeeded, src_frame.sample_rate_hz_, + dst_frame->sample_rate_hz_, audio_ptr_num_channels); + assert(false); + } + + const int src_length = src_frame.samples_per_channel_ * + audio_ptr_num_channels; + int out_length = resampler->Resample(audio_ptr, src_length, dst_frame->data_, + AudioFrame::kMaxDataSizeSamples); + if (out_length == -1) { + dst_frame->CopyFrom(src_frame); + LOG_FERR3(LS_ERROR, Resample, audio_ptr, src_length, dst_frame->data_); + assert(false); + } + dst_frame->samples_per_channel_ = out_length / audio_ptr_num_channels; + + // Upmix after resampling. + if (src_frame.num_channels_ == 1 && dst_frame->num_channels_ == 2) { + // The audio in dst_frame really is mono at this point; MonoToStereo will + // set this back to stereo. + dst_frame->num_channels_ = 1; + AudioFrameOperations::MonoToStereo(dst_frame); + } } -void Utility::MixSubtractWithSat(int16_t target[], - const int16_t source[], - uint16_t len) -{ - int32_t temp(0); - for (int i = 0; i < len; i++) - { - temp = target[i] - source[i]; - if (temp > 32767) - target[i] = 32767; - else if (temp < -32768) - target[i] = -32768; - else - target[i] = (int16_t) temp; - } +void DownConvertToCodecFormat(const int16_t* src_data, + int samples_per_channel, + int num_channels, + int sample_rate_hz, + int codec_num_channels, + int codec_rate_hz, + int16_t* mono_buffer, + PushResampler* resampler, + AudioFrame* dst_af) { + assert(samples_per_channel <= kMaxMonoDataSizeSamples); + assert(num_channels == 1 || num_channels == 2); + assert(codec_num_channels == 1 || codec_num_channels == 2); + + // Never upsample the capture signal here. This should be done at the + // end of the send chain. + int destination_rate = std::min(codec_rate_hz, sample_rate_hz); + + // If no stereo codecs are in use, we downmix a stereo stream from the + // device early in the chain, before resampling. + if (num_channels == 2 && codec_num_channels == 1) { + AudioFrameOperations::StereoToMono(src_data, samples_per_channel, + mono_buffer); + src_data = mono_buffer; + num_channels = 1; + } + + if (resampler->InitializeIfNeeded( + sample_rate_hz, destination_rate, num_channels) != 0) { + LOG_FERR3(LS_ERROR, + InitializeIfNeeded, + sample_rate_hz, + destination_rate, + num_channels); + assert(false); + } + + const int in_length = samples_per_channel * num_channels; + int out_length = resampler->Resample( + src_data, in_length, dst_af->data_, AudioFrame::kMaxDataSizeSamples); + if (out_length == -1) { + LOG_FERR3(LS_ERROR, Resample, src_data, in_length, dst_af->data_); + assert(false); + } + + dst_af->samples_per_channel_ = out_length / num_channels; + dst_af->sample_rate_hz_ = destination_rate; + dst_af->num_channels_ = num_channels; + dst_af->timestamp_ = -1; + dst_af->speech_type_ = AudioFrame::kNormalSpeech; + dst_af->vad_activity_ = AudioFrame::kVadUnknown; } -void Utility::MixAndScaleWithSat(int16_t target[], - const int16_t source[], float scale, - uint16_t len) -{ - int32_t temp(0); - for (int i = 0; i < len; i++) - { - temp = (int32_t) (target[i] + scale * source[i]); - if (temp > 32767) - target[i] = 32767; - else if (temp < -32768) - target[i] = -32768; - else - target[i] = (int16_t) temp; - } -} +void MixWithSat(int16_t target[], + int target_channel, + const int16_t source[], + int source_channel, + int source_len) { + assert(target_channel == 1 || target_channel == 2); + assert(source_channel == 1 || source_channel == 2); -void Utility::Scale(int16_t vector[], float scale, uint16_t len) -{ - for (int i = 0; i < len; i++) - { - vector[i] = (int16_t) (scale * vector[i]); + if (target_channel == 2 && source_channel == 1) { + // Convert source from mono to stereo. + int32_t left = 0; + int32_t right = 0; + for (int i = 0; i < source_len; ++i) { + left = source[i] + target[i * 2]; + right = source[i] + target[i * 2 + 1]; + target[i * 2] = WebRtcSpl_SatW32ToW16(left); + target[i * 2 + 1] = WebRtcSpl_SatW32ToW16(right); } -} - -void Utility::ScaleWithSat(int16_t vector[], float scale, - uint16_t len) -{ - int32_t temp(0); - for (int i = 0; i < len; i++) - { - temp = (int32_t) (scale * vector[i]); - if (temp > 32767) - vector[i] = 32767; - else if (temp < -32768) - vector[i] = -32768; - else - vector[i] = (int16_t) temp; + } else if (target_channel == 1 && source_channel == 2) { + // Convert source from stereo to mono. + int32_t temp = 0; + for (int i = 0; i < source_len / 2; ++i) { + temp = ((source[i * 2] + source[i * 2 + 1]) >> 1) + target[i]; + target[i] = WebRtcSpl_SatW32ToW16(temp); } + } else { + int32_t temp = 0; + for (int i = 0; i < source_len; ++i) { + temp = source[i] + target[i]; + target[i] = WebRtcSpl_SatW32ToW16(temp); + } + } } } // namespace voe - } // namespace webrtc diff --git a/webrtc/voice_engine/utility.h b/webrtc/voice_engine/utility.h index fcb4462e5..f6fa35bf7 100644 --- a/webrtc/voice_engine/utility.h +++ b/webrtc/voice_engine/utility.h @@ -12,47 +12,48 @@ * Contains functions often used by different parts of VoiceEngine. */ -#ifndef WEBRTC_VOICE_ENGINE_UTILITY_H -#define WEBRTC_VOICE_ENGINE_UTILITY_H +#ifndef WEBRTC_VOICE_ENGINE_UTILITY_H_ +#define WEBRTC_VOICE_ENGINE_UTILITY_H_ #include "webrtc/typedefs.h" -#include "webrtc/voice_engine/voice_engine_defines.h" -namespace webrtc -{ +namespace webrtc { -class Module; +class AudioFrame; +class PushResampler; -namespace voe -{ +namespace voe { -class Utility -{ -public: - static void MixWithSat(int16_t target[], - int target_channel, - const int16_t source[], - int source_channel, - int source_len); +// Upmix or downmix and resample the audio in |src_frame| to |dst_frame|. +// Expects |dst_frame| to have its |num_channels_| and |sample_rate_hz_| set to +// the desired values. Updates |samples_per_channel_| accordingly. +// +// On failure, returns -1 and copies |src_frame| to |dst_frame|. +void RemixAndResample(const AudioFrame& src_frame, + PushResampler* resampler, + AudioFrame* dst_frame); - static void MixSubtractWithSat(int16_t target[], - const int16_t source[], - uint16_t len); +// Downmix and downsample the audio in |src_data| to |dst_af| as necessary, +// specified by |codec_num_channels| and |codec_rate_hz|. |mono_buffer| is +// temporary space and must be of sufficient size to hold the downmixed source +// audio (recommend using a size of kMaxMonoDataSizeSamples). +void DownConvertToCodecFormat(const int16_t* src_data, + int samples_per_channel, + int num_channels, + int sample_rate_hz, + int codec_num_channels, + int codec_rate_hz, + int16_t* mono_buffer, + PushResampler* resampler, + AudioFrame* dst_af); - static void MixAndScaleWithSat(int16_t target[], - const int16_t source[], - float scale, - uint16_t len); - - static void Scale(int16_t vector[], float scale, uint16_t len); - - static void ScaleWithSat(int16_t vector[], - float scale, - uint16_t len); -}; +void MixWithSat(int16_t target[], + int target_channel, + const int16_t source[], + int source_channel, + int source_len); } // namespace voe - } // namespace webrtc -#endif // WEBRTC_VOICE_ENGINE_UTILITY_H +#endif // WEBRTC_VOICE_ENGINE_UTILITY_H_ diff --git a/webrtc/voice_engine/voe_base_impl.cc b/webrtc/voice_engine/voe_base_impl.cc index 80f802068..1b6e14491 100644 --- a/webrtc/voice_engine/voe_base_impl.cc +++ b/webrtc/voice_engine/voe_base_impl.cc @@ -25,13 +25,6 @@ #include "webrtc/voice_engine/utility.h" #include "webrtc/voice_engine/voice_engine_impl.h" -#if (defined(_WIN32) && defined(_DLL) && (_MSC_VER == 1400)) -// Fix for VS 2005 MD/MDd link problem -#include -extern "C" - { FILE _iob[3] = { __iob_func()[0], __iob_func()[1], __iob_func()[2]}; } -#endif - namespace webrtc { @@ -223,6 +216,9 @@ int VoEBaseImpl::OnDataAvailable(const int voe_channels[], // No need to go through the APM, demultiplex the data to each VoE channel, // encode and send to the network. for (int i = 0; i < number_of_voe_channels; ++i) { + // TODO(ajm): In the case where multiple channels are using the same codec + // rate, this path needlessly does extra conversions. We should convert once + // and share between channels. OnData(voe_channels[i], audio_data, 16, sample_rate, number_of_channels, number_of_frames); } diff --git a/webrtc/voice_engine/voice_engine.gyp b/webrtc/voice_engine/voice_engine.gyp index 1ae4e0274..eae0ab89d 100644 --- a/webrtc/voice_engine/voice_engine.gyp +++ b/webrtc/voice_engine/voice_engine.gyp @@ -57,8 +57,6 @@ 'monitor_module.h', 'output_mixer.cc', 'output_mixer.h', - 'output_mixer_internal.cc', - 'output_mixer_internal.h', 'shared_data.cc', 'shared_data.h', 'statistics.cc', diff --git a/webrtc/voice_engine/voice_engine_defines.h b/webrtc/voice_engine/voice_engine_defines.h index d75f938b1..b3cba7c33 100644 --- a/webrtc/voice_engine/voice_engine_defines.h +++ b/webrtc/voice_engine/voice_engine_defines.h @@ -27,6 +27,10 @@ namespace webrtc { +// Internal buffer size required for mono audio, based on the highest sample +// rate voice engine supports (10 ms of audio at 192 kHz). +static const int kMaxMonoDataSizeSamples = 1920; + // VolumeControl enum { kMinVolumeLevel = 0 }; enum { kMaxVolumeLevel = 255 };