From 6aac93bd9c3da92e92b016d83c8f84c65aae65b6 Mon Sep 17 00:00:00 2001 From: "minyue@webrtc.org" Date: Tue, 12 Aug 2014 08:13:33 +0000 Subject: [PATCH] Adding SetOpusMaxBandwidth in VoE and ACM This is a step to solve https://code.google.com/p/webrtc/issues/detail?id=1906 In particular, we add an API in VoE and ACM to call Opus's API of setting maximum bandwidth. TEST = added a test in voe_cmd_test and listened to the result BUG= R=henrika@google.com, henrika@webrtc.org, turaj@webrtc.org Review URL: https://webrtc-codereview.appspot.com/21129004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@6869 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/acm2/acm_generic_codec.cc | 6 +++++ .../main/acm2/acm_generic_codec.h | 17 ++++++++++++ .../audio_coding/main/acm2/acm_opus.cc | 5 ++++ .../modules/audio_coding/main/acm2/acm_opus.h | 2 ++ .../main/acm2/audio_coding_module_impl.cc | 9 +++++++ .../main/acm2/audio_coding_module_impl.h | 4 +++ .../main/interface/audio_coding_module.h | 17 ++++++++++++ webrtc/voice_engine/channel.cc | 13 +++++++++ webrtc/voice_engine/channel.h | 1 + webrtc/voice_engine/include/voe_codec.h | 8 ++++++ .../test/auto_test/standard/codec_test.cc | 27 +++++++++++++++++++ .../test/cmd_test/voe_cmd_test.cc | 7 +++++ webrtc/voice_engine/voe_codec_impl.cc | 18 +++++++++++++ webrtc/voice_engine/voe_codec_impl.h | 2 ++ 14 files changed, 136 insertions(+) diff --git a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc index db776d2e8..565d291f2 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc +++ b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc @@ -1000,6 +1000,12 @@ int16_t ACMGenericCodec::REDPayloadISAC(const int32_t /* isac_rate */, return -1; } +int ACMGenericCodec::SetOpusMaxBandwidth(int /* max_bandwidth */) { + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, unique_id_, + "The send-codec is not Opus, failed to set maximum bandwidth."); + return -1; +} + } // namespace acm2 } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h index 80f239ac6..b88e28f0f 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h +++ b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h @@ -537,6 +537,23 @@ class ACMGenericCodec { uint8_t* payload, int16_t* payload_len_bytes); + /////////////////////////////////////////////////////////////////////////// + // int SetOpusMaxBandwidth() + // Sets maximum required encoding bandwidth for Opus. This is to tell Opus + // that it is enough to code the input audio up to a bandwidth. A use case of + // this is when the receiver cannot render the full band. Opus can take this + // information to optimize the bit rate and increase the computation + // efficiency. + // + // Input: + // -max_bandwidth : maximum required bandwidth. + // + // Return value: + // -1 if failed or on codecs other than Opus + // 0 if succeeded. + // + virtual int SetOpusMaxBandwidth(int /* max_bandwidth */); + /////////////////////////////////////////////////////////////////////////// // HasFrameToEncode() // Returns true if there is enough audio buffered for encoding, such that diff --git a/webrtc/modules/audio_coding/main/acm2/acm_opus.cc b/webrtc/modules/audio_coding/main/acm2/acm_opus.cc index f75a34881..c778982bc 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_opus.cc +++ b/webrtc/modules/audio_coding/main/acm2/acm_opus.cc @@ -261,6 +261,11 @@ int ACMOpus::SetPacketLossRate(int loss_rate) { return -1; } +int ACMOpus::SetOpusMaxBandwidth(int max_bandwidth) { + // Ask the encoder to change the maximum required bandwidth. + return WebRtcOpus_SetMaxBandwidth(encoder_inst_ptr_, max_bandwidth); +} + #endif // WEBRTC_CODEC_OPUS } // namespace acm2 diff --git a/webrtc/modules/audio_coding/main/acm2/acm_opus.h b/webrtc/modules/audio_coding/main/acm2/acm_opus.h index b94adc471..8c2882c0c 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_opus.h +++ b/webrtc/modules/audio_coding/main/acm2/acm_opus.h @@ -38,6 +38,8 @@ class ACMOpus : public ACMGenericCodec { virtual int SetPacketLossRate(int loss_rate) OVERRIDE; + virtual int SetOpusMaxBandwidth(int max_bandwidth) OVERRIDE; + protected: void DestructEncoderSafe(); diff --git a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc index 26f5b542f..164c0bbfd 100644 --- a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc +++ b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.cc @@ -1911,6 +1911,15 @@ int AudioCodingModuleImpl::ConfigISACBandwidthEstimator( frame_size_ms, rate_bit_per_sec, enforce_frame_size); } +// Informs Opus encoder about the maximum audio bandwidth needs to be encoded. +int AudioCodingModuleImpl::SetOpusMaxBandwidth(int bandwidth_hz) { + CriticalSectionScoped lock(acm_crit_sect_); + if (!HaveValidEncoder("SetOpusMaxBandwidth")) { + return -1; + } + return codecs_[current_send_codec_idx_]->SetOpusMaxBandwidth(bandwidth_hz); +} + int AudioCodingModuleImpl::PlayoutTimestamp(uint32_t* timestamp) { return receiver_.GetPlayoutTimestamp(timestamp) ? 0 : -1; } diff --git a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h index 02290da04..9e5cc370a 100644 --- a/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h +++ b/webrtc/modules/audio_coding/main/acm2/audio_coding_module_impl.h @@ -232,6 +232,10 @@ class AudioCodingModuleImpl : public AudioCodingModule { int rate_bit_per_sec, bool enforce_frame_size = false); + // If current send codec is Opus, informs it about the maximum audio + // bandwidth needs to be encoded. + int SetOpusMaxBandwidth(int bandwidth_hz); + int UnregisterReceiveCodec(uint8_t payload_type); int EnableNack(size_t max_nack_list_size); diff --git a/webrtc/modules/audio_coding/main/interface/audio_coding_module.h b/webrtc/modules/audio_coding/main/interface/audio_coding_module.h index 482700474..aae7ead71 100644 --- a/webrtc/modules/audio_coding/main/interface/audio_coding_module.h +++ b/webrtc/modules/audio_coding/main/interface/audio_coding_module.h @@ -915,6 +915,23 @@ class AudioCodingModule: public Module { int init_rate_bps, bool enforce_frame_size = false) = 0; + /////////////////////////////////////////////////////////////////////////// + // int SetOpusMaxBandwidth() + // If current send codec is Opus, informs it about maximum audio bandwidth + // needs to be encoded. A use case of this is when the receiver can only play + // audio up to frequency limit. Opus can use this information to optimize + // the bit rate and increase the computation efficiency. + // + // Input: + // -banbwidth_hz : maximum bandwidth in Hz. + // + // Return value: + // -1 if current send codec is not Opus or + // error occurred in setting the bandwidth, + // 0 maximum bandwidth is set successfully. + // + virtual int SetOpusMaxBandwidth(int banbwidth_hz) = 0; + /////////////////////////////////////////////////////////////////////////// // statistics // diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc index c9364efb1..3f13ba273 100644 --- a/webrtc/voice_engine/channel.cc +++ b/webrtc/voice_engine/channel.cc @@ -1749,6 +1749,19 @@ Channel::SetSendCNPayloadType(int type, PayloadFrequencies frequency) return 0; } +int Channel::SetOpusMaxBandwidth(int bandwidth_hz) { + WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId), + "Channel::SetOpusMaxBandwidth()"); + + if (audio_coding_->SetOpusMaxBandwidth(bandwidth_hz) != 0) { + _engineStatisticsPtr->SetLastError( + VE_AUDIO_CODING_MODULE_ERROR, kTraceError, + "SetOpusMaxBandwidth() failed to set maximum encoding bandwidth"); + return -1; + } + return 0; +} + int32_t Channel::RegisterExternalTransport(Transport& transport) { WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId), diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h index 8385ccc81..a4f7ecd4c 100644 --- a/webrtc/voice_engine/channel.h +++ b/webrtc/voice_engine/channel.h @@ -207,6 +207,7 @@ public: int32_t SetRecPayloadType(const CodecInst& codec); int32_t GetRecPayloadType(CodecInst& codec); int32_t SetSendCNPayloadType(int type, PayloadFrequencies frequency); + int SetOpusMaxBandwidth(int bandwidth_hz); // VoE dual-streaming. int SetSecondarySendCodec(const CodecInst& codec, int red_payload_type); diff --git a/webrtc/voice_engine/include/voe_codec.h b/webrtc/voice_engine/include/voe_codec.h index 34a223211..56835f70b 100644 --- a/webrtc/voice_engine/include/voe_codec.h +++ b/webrtc/voice_engine/include/voe_codec.h @@ -126,6 +126,14 @@ public: virtual int GetVADStatus(int channel, bool& enabled, VadModes& mode, bool& disabledDTX) = 0; + // Sets the maximum audio bandwidth needs to be encoded in Hz, + // |bandwidth_hz|, for the Opus encoder on a specific |channel|. + // TODO(minyue): Make SetOpusMaxBandwidth() pure virtual when + // fakewebrtcvoiceengine in talk is ready. + virtual int SetOpusMaxBandwidth(int channel, int bandwidth_hz) { + return -1; + } + // Don't use. To be removed. virtual int SetAMREncFormat(int channel, AmrMode mode) { return -1; } virtual int SetAMRDecFormat(int channel, AmrMode mode) { return -1; } diff --git a/webrtc/voice_engine/test/auto_test/standard/codec_test.cc b/webrtc/voice_engine/test/auto_test/standard/codec_test.cc index 2970ab34a..8ca432835 100644 --- a/webrtc/voice_engine/test/auto_test/standard/codec_test.cc +++ b/webrtc/voice_engine/test/auto_test/standard/codec_test.cc @@ -130,6 +130,33 @@ TEST_F(CodecTest, VoiceActivityDetectionCanBeTurnedOff) { EXPECT_EQ(webrtc::kVadConventional, vad_mode); } +TEST_F(CodecTest, OpusMaxBandwidthCanBeSet) { + for (int i = 0; i < voe_codec_->NumOfCodecs(); ++i) { + voe_codec_->GetCodec(i, codec_instance_); + if (_stricmp("opus", codec_instance_.plname)) { + continue; + } + voe_codec_->SetSendCodec(channel_, codec_instance_); + // SetOpusMaxBandwidth can handle any integer as the bandwidth. Following + // tests some most commonly used numbers. + EXPECT_EQ(0, voe_codec_->SetOpusMaxBandwidth(channel_, 24000)); + EXPECT_EQ(0, voe_codec_->SetOpusMaxBandwidth(channel_, 16000)); + EXPECT_EQ(0, voe_codec_->SetOpusMaxBandwidth(channel_, 8000)); + EXPECT_EQ(0, voe_codec_->SetOpusMaxBandwidth(channel_, 4000)); + } +} + +TEST_F(CodecTest, OpusMaxBandwidthCannotBeSetForNonOpus) { + for (int i = 0; i < voe_codec_->NumOfCodecs(); ++i) { + voe_codec_->GetCodec(i, codec_instance_); + if (!_stricmp("opus", codec_instance_.plname)) { + continue; + } + voe_codec_->SetSendCodec(channel_, codec_instance_); + EXPECT_EQ(-1, voe_codec_->SetOpusMaxBandwidth(channel_, 16000)); + } +} + // TODO(xians, phoglund): Re-enable when issue 372 is resolved. TEST_F(CodecTest, DISABLED_ManualVerifySendCodecsForAllPacketSizes) { for (int i = 0; i < voe_codec_->NumOfCodecs(); ++i) { diff --git a/webrtc/voice_engine/test/cmd_test/voe_cmd_test.cc b/webrtc/voice_engine/test/cmd_test/voe_cmd_test.cc index a30764ad9..5e636459b 100644 --- a/webrtc/voice_engine/test/cmd_test/voe_cmd_test.cc +++ b/webrtc/voice_engine/test/cmd_test/voe_cmd_test.cc @@ -445,6 +445,7 @@ void RunTest(std::string out_path) { printf("%i. Remove a file-playing channel \n", option_index++); printf("%i. Toggle Opus stereo (Opus must be selected again to apply " "the setting) \n", option_index++); + printf("%i. Set Opus maximum audio bandwidth \n", option_index++); printf("Select action or %i to stop the call: ", option_index); int option_selection; @@ -757,6 +758,12 @@ void RunTest(std::string out_path) { else printf("\n Opus mono enabled (select Opus again to apply the " "setting). \n"); + } else if (option_selection == option_index++) { + printf("\n Input bandwidth in Hz: "); + int max_playback_rate; + ASSERT_EQ(1, scanf("%i", &max_playback_rate)); + res = codec->SetOpusMaxBandwidth(chan, max_playback_rate); + VALIDATE; } else { break; } diff --git a/webrtc/voice_engine/voe_codec_impl.cc b/webrtc/voice_engine/voe_codec_impl.cc index 4aa0556ea..50994070f 100644 --- a/webrtc/voice_engine/voe_codec_impl.cc +++ b/webrtc/voice_engine/voe_codec_impl.cc @@ -418,6 +418,24 @@ int VoECodecImpl::GetVADStatus(int channel, bool& enabled, VadModes& mode, return 0; } +int VoECodecImpl::SetOpusMaxBandwidth(int channel, int bandwidth_hz) { + WEBRTC_TRACE(kTraceApiCall, kTraceVoice, VoEId(_shared->instance_id(), -1), + "SetOpusMaxBandwidth(channel=%d, bandwidth_hz=%d)", channel, + bandwidth_hz); + if (!_shared->statistics().Initialized()) { + _shared->SetLastError(VE_NOT_INITED, kTraceError); + return -1; + } + voe::ChannelOwner ch = _shared->channel_manager().GetChannel(channel); + voe::Channel* channelPtr = ch.channel(); + if (channelPtr == NULL) { + _shared->SetLastError(VE_CHANNEL_NOT_VALID, kTraceError, + "SetOpusMaxBandwidth failed to locate channel"); + return -1; + } + return channelPtr->SetOpusMaxBandwidth(bandwidth_hz); +} + void VoECodecImpl::ACMToExternalCodecRepresentation(CodecInst& toInst, const CodecInst& fromInst) { diff --git a/webrtc/voice_engine/voe_codec_impl.h b/webrtc/voice_engine/voe_codec_impl.h index 1b9f00e4e..498854d8d 100644 --- a/webrtc/voice_engine/voe_codec_impl.h +++ b/webrtc/voice_engine/voe_codec_impl.h @@ -54,6 +54,8 @@ public: VadModes& mode, bool& disabledDTX); + virtual int SetOpusMaxBandwidth(int channel, int bandwidth_hz); + // Dual-streaming virtual int SetSecondarySendCodec(int channel, const CodecInst& codec, int red_payload_type);