diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.cc b/webrtc/modules/audio_coding/codecs/audio_decoder.cc index 6114e70a9..bc6b9c59c 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.cc @@ -37,7 +37,8 @@ int AudioDecoder::IncomingPacket(const uint8_t* payload, int AudioDecoder::ErrorCode() { return 0; } -int AudioDecoder::PacketDuration(const uint8_t* encoded, size_t encoded_len) { +int AudioDecoder::PacketDuration(const uint8_t* encoded, + size_t encoded_len) const { return kNotImplemented; } diff --git a/webrtc/modules/audio_coding/codecs/audio_decoder.h b/webrtc/modules/audio_coding/codecs/audio_decoder.h index 0b4e20ca5..30fdc440c 100644 --- a/webrtc/modules/audio_coding/codecs/audio_decoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_decoder.h @@ -69,7 +69,7 @@ class AudioDecoder { // Returns the duration in samples of the payload in |encoded| which is // |encoded_len| bytes long. Returns kNotImplemented if no duration estimate // is available, or -1 in case of an error. - virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len); + virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len) const; // Returns the duration in samples of the redandant payload in |encoded| which // is |encoded_len| bytes long. Returns kNotImplemented if no duration diff --git a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc index 19c395669..47474f218 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc +++ b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.cc @@ -1134,7 +1134,7 @@ int AudioDecoderProxy::ErrorCode() { } int AudioDecoderProxy::PacketDuration(const uint8_t* encoded, - size_t encoded_len) { + size_t encoded_len) const { CriticalSectionScoped decoder_lock(decoder_lock_.get()); return decoder_->PacketDuration(encoded, encoded_len); } diff --git a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h index d2caea792..2154ebf93 100644 --- a/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h +++ b/webrtc/modules/audio_coding/main/acm2/acm_generic_codec.h @@ -1004,7 +1004,7 @@ class AudioDecoderProxy final : public AudioDecoder { uint32_t rtp_timestamp, uint32_t arrival_timestamp) override; int ErrorCode() override; - int PacketDuration(const uint8_t* encoded, size_t encoded_len) override; + int PacketDuration(const uint8_t* encoded, size_t encoded_len) const override; int PacketDurationRedundant(const uint8_t* encoded, size_t encoded_len) const override; bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const override; diff --git a/webrtc/modules/audio_coding/neteq/audio_decoder_impl.cc b/webrtc/modules/audio_coding/neteq/audio_decoder_impl.cc index 66eba12ba..882c8f352 100644 --- a/webrtc/modules/audio_coding/neteq/audio_decoder_impl.cc +++ b/webrtc/modules/audio_coding/neteq/audio_decoder_impl.cc @@ -48,7 +48,7 @@ int AudioDecoderPcmU::Decode(const uint8_t* encoded, size_t encoded_len, } int AudioDecoderPcmU::PacketDuration(const uint8_t* encoded, - size_t encoded_len) { + size_t encoded_len) const { // One encoded byte per sample per channel. return static_cast(encoded_len / channels_); } @@ -64,7 +64,7 @@ int AudioDecoderPcmA::Decode(const uint8_t* encoded, size_t encoded_len, } int AudioDecoderPcmA::PacketDuration(const uint8_t* encoded, - size_t encoded_len) { + size_t encoded_len) const { // One encoded byte per sample per channel. return static_cast(encoded_len / channels_); } @@ -82,7 +82,7 @@ int AudioDecoderPcm16B::Decode(const uint8_t* encoded, size_t encoded_len, } int AudioDecoderPcm16B::PacketDuration(const uint8_t* encoded, - size_t encoded_len) { + size_t encoded_len) const { // Two encoded byte per sample per channel. return static_cast(encoded_len / (2 * channels_)); } @@ -148,7 +148,7 @@ int AudioDecoderG722::Init() { } int AudioDecoderG722::PacketDuration(const uint8_t* encoded, - size_t encoded_len) { + size_t encoded_len) const { // 1/2 encoded byte per sample per channel. return static_cast(2 * encoded_len / channels_); } @@ -260,6 +260,11 @@ int AudioDecoderOpus::Decode(const uint8_t* encoded, size_t encoded_len, int AudioDecoderOpus::DecodeRedundant(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type) { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return Decode(encoded, encoded_len, decoded, speech_type); + } + int16_t temp_type = 1; // Default is speech. int16_t ret = WebRtcOpus_DecodeFec(dec_state_, encoded, static_cast(encoded_len), decoded, @@ -275,13 +280,18 @@ int AudioDecoderOpus::Init() { } int AudioDecoderOpus::PacketDuration(const uint8_t* encoded, - size_t encoded_len) { + size_t encoded_len) const { return WebRtcOpus_DurationEst(dec_state_, encoded, static_cast(encoded_len)); } int AudioDecoderOpus::PacketDurationRedundant(const uint8_t* encoded, size_t encoded_len) const { + if (!PacketHasFec(encoded, encoded_len)) { + // This packet is a RED packet. + return PacketDuration(encoded, encoded_len); + } + return WebRtcOpus_FecDurationEst(encoded, static_cast(encoded_len)); } diff --git a/webrtc/modules/audio_coding/neteq/audio_decoder_impl.h b/webrtc/modules/audio_coding/neteq/audio_decoder_impl.h index 8e1ba7e0e..57bd522ec 100644 --- a/webrtc/modules/audio_coding/neteq/audio_decoder_impl.h +++ b/webrtc/modules/audio_coding/neteq/audio_decoder_impl.h @@ -40,7 +40,7 @@ class AudioDecoderPcmU : public AudioDecoder { virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); virtual int Init() { return 0; } - virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len); + virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len) const; private: DISALLOW_COPY_AND_ASSIGN(AudioDecoderPcmU); @@ -52,7 +52,7 @@ class AudioDecoderPcmA : public AudioDecoder { virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); virtual int Init() { return 0; } - virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len); + virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len) const; private: DISALLOW_COPY_AND_ASSIGN(AudioDecoderPcmA); @@ -89,7 +89,7 @@ class AudioDecoderPcm16B : public AudioDecoder { virtual int Decode(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); virtual int Init() { return 0; } - virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len); + virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len) const; private: DISALLOW_COPY_AND_ASSIGN(AudioDecoderPcm16B); @@ -133,7 +133,7 @@ class AudioDecoderG722 : public AudioDecoder { int16_t* decoded, SpeechType* speech_type); virtual bool HasDecodePlc() const { return false; } virtual int Init(); - virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len); + virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len) const; private: G722DecInst* dec_state_; @@ -174,7 +174,7 @@ class AudioDecoderOpus : public AudioDecoder { virtual int DecodeRedundant(const uint8_t* encoded, size_t encoded_len, int16_t* decoded, SpeechType* speech_type); virtual int Init(); - virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len); + virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len) const; virtual int PacketDurationRedundant(const uint8_t* encoded, size_t encoded_len) const; virtual bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const; diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter.cc b/webrtc/modules/audio_coding/neteq/payload_splitter.cc index 118556b17..c19375b72 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter.cc +++ b/webrtc/modules/audio_coding/neteq/payload_splitter.cc @@ -151,8 +151,11 @@ int PayloadSplitter::SplitFec(PacketList* packet_list, switch (info->codec_type) { case kDecoderOpus: case kDecoderOpus_2ch: { - Packet* new_packet = new Packet; + // The main payload of this packet should be decoded as a primary + // payload, even if it comes as a secondary payload in a RED packet. + packet->primary = true; + Packet* new_packet = new Packet; new_packet->header = packet->header; int duration = decoder-> PacketDurationRedundant(packet->payload, packet->payload_length); diff --git a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc b/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc index d397a076b..085e76fb0 100644 --- a/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/payload_splitter_unittest.cc @@ -32,6 +32,28 @@ static const size_t kRedHeaderLength = 4; // 4 bytes RED header. static const uint16_t kSequenceNumber = 0; static const uint32_t kBaseTimestamp = 0x12345678; +// A possible Opus packet that contains FEC is the following. +// The frame is 20 ms in duration. +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |0|0|0|0|1|0|0|0|x|1|x|x|x|x|x|x|x| | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | +// | Compressed frame 1 (N-2 bytes)... : +// : | +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +void CreateOpusFecPayload(uint8_t* payload, size_t payload_length, + uint8_t payload_value) { + if (payload_length < 2) { + return; + } + payload[0] = 0x08; + payload[1] = 0x40; + memset(&payload[2], payload_value, payload_length - 2); +} + // RED headers (according to RFC 2198): // // 0 1 2 3 @@ -52,7 +74,8 @@ static const uint32_t kBaseTimestamp = 0x12345678; // "behind" the the previous payload. Packet* CreateRedPayload(size_t num_payloads, uint8_t* payload_types, - int timestamp_offset) { + int timestamp_offset, + bool embed_opus_fec = false) { Packet* packet = new Packet; packet->header.payloadType = kRedPayloadType; packet->header.timestamp = kBaseTimestamp; @@ -84,52 +107,34 @@ Packet* CreateRedPayload(size_t num_payloads, } for (size_t i = 0; i < num_payloads; ++i) { // Write |i| to all bytes in each payload. - memset(payload_ptr, static_cast(i), kPayloadLength); + if (embed_opus_fec) { + CreateOpusFecPayload(payload_ptr, kPayloadLength, + static_cast(i)); + } else { + memset(payload_ptr, static_cast(i), kPayloadLength); + } payload_ptr += kPayloadLength; } packet->payload = payload; return packet; } - -// A possible Opus packet that contains FEC is the following. -// The frame is 20 ms in duration. -// -// 0 1 2 3 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// |0|0|0|0|1|0|0|0|x|1|x|x|x|x|x|x|x| | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | -// | Compressed frame 1 (N-2 bytes)... : -// : | -// | | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -Packet* CreateOpusFecPacket(uint8_t payload_type, size_t payload_length, - uint8_t payload_value) { - Packet* packet = new Packet; - packet->header.payloadType = payload_type; - packet->header.timestamp = kBaseTimestamp; - packet->header.sequenceNumber = kSequenceNumber; - packet->payload_length = payload_length; - uint8_t* payload = new uint8_t[packet->payload_length]; - payload[0] = 0x08; - payload[1] = 0x40; - memset(&payload[2], payload_value, payload_length - 2); - packet->payload = payload; - return packet; -} - // Create a packet with all payload bytes set to |payload_value|. Packet* CreatePacket(uint8_t payload_type, size_t payload_length, - uint8_t payload_value) { + uint8_t payload_value, bool opus_fec = false) { Packet* packet = new Packet; packet->header.payloadType = payload_type; packet->header.timestamp = kBaseTimestamp; packet->header.sequenceNumber = kSequenceNumber; packet->payload_length = payload_length; uint8_t* payload = new uint8_t[packet->payload_length]; - memset(payload, payload_value, payload_length); packet->payload = payload; + if (opus_fec) { + CreateOpusFecPayload(packet->payload, packet->payload_length, + payload_value); + } else { + memset(payload, payload_value, payload_length); + } return packet; } @@ -726,7 +731,7 @@ TEST(FecPayloadSplitter, MixedPayload) { decoder_database.RegisterPayload(0, kDecoderOpus); decoder_database.RegisterPayload(1, kDecoderPCMu); - Packet* packet = CreateOpusFecPacket(0, 10, 0xFF); + Packet* packet = CreatePacket(0, 10, 0xFF, true); packet_list.push_back(packet); packet = CreatePacket(0, 10, 0); // Non-FEC Opus payload. @@ -774,4 +779,67 @@ TEST(FecPayloadSplitter, MixedPayload) { delete packet; } +TEST(FecPayloadSplitter, EmbedFecInRed) { + PacketList packet_list; + DecoderDatabase decoder_database; + + const int kTimestampOffset = 20 * 48; // 20 ms * 48 kHz. + uint8_t payload_types[] = {0, 0}; + decoder_database.RegisterPayload(0, kDecoderOpus); + Packet* packet = CreateRedPayload(2, payload_types, kTimestampOffset, true); + packet_list.push_back(packet); + + PayloadSplitter splitter; + EXPECT_EQ(PayloadSplitter::kOK, + splitter.SplitRed(&packet_list)); + EXPECT_EQ(PayloadSplitter::kOK, + splitter.SplitFec(&packet_list, &decoder_database)); + + EXPECT_EQ(4u, packet_list.size()); + + // Check first packet. FEC packet copied from primary payload in RED. + packet = packet_list.front(); + EXPECT_EQ(0, packet->header.payloadType); + EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp); + EXPECT_EQ(kPayloadLength, packet->payload_length); + EXPECT_FALSE(packet->primary); + EXPECT_EQ(packet->payload[3], 1); + delete [] packet->payload; + delete packet; + packet_list.pop_front(); + + // Check second packet. Normal packet copied from primary payload in RED. + packet = packet_list.front(); + EXPECT_EQ(0, packet->header.payloadType); + EXPECT_EQ(kBaseTimestamp, packet->header.timestamp); + EXPECT_EQ(kPayloadLength, packet->payload_length); + EXPECT_TRUE(packet->primary); + EXPECT_EQ(packet->payload[3], 1); + delete [] packet->payload; + delete packet; + packet_list.pop_front(); + + // Check third packet. FEC packet copied from secondary payload in RED. + packet = packet_list.front(); + EXPECT_EQ(0, packet->header.payloadType); + EXPECT_EQ(kBaseTimestamp - 2 * kTimestampOffset, packet->header.timestamp); + EXPECT_EQ(kPayloadLength, packet->payload_length); + EXPECT_FALSE(packet->primary); + EXPECT_EQ(packet->payload[3], 0); + delete [] packet->payload; + delete packet; + packet_list.pop_front(); + + // Check fourth packet. Normal packet copied from primary payload in RED. + packet = packet_list.front(); + EXPECT_EQ(0, packet->header.payloadType); + EXPECT_EQ(kBaseTimestamp - kTimestampOffset, packet->header.timestamp); + EXPECT_EQ(kPayloadLength, packet->payload_length); + EXPECT_TRUE(packet->primary); + EXPECT_EQ(packet->payload[3], 0); + delete [] packet->payload; + delete packet; + packet_list.pop_front(); +} + } // namespace webrtc