From 7b75ac6756a594d81ae88e8f798b0376886b69f5 Mon Sep 17 00:00:00 2001 From: "turaj@webrtc.org" Date: Thu, 26 Sep 2013 00:27:56 +0000 Subject: [PATCH] Sync-packet insertion into NetEq4. This is related to r3883 & r4052 for NetEq 3. BUG= R=henrik.lundin@webrtc.org Review URL: https://webrtc-codereview.appspot.com/2099004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@4850 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../audio_coding/neteq4/interface/neteq.h | 19 +- .../modules/audio_coding/neteq4/neteq_impl.cc | 83 +++++-- .../modules/audio_coding/neteq4/neteq_impl.h | 18 +- .../audio_coding/neteq4/neteq_unittest.cc | 234 ++++++++++++++++++ webrtc/modules/audio_coding/neteq4/packet.h | 35 ++- .../audio_coding/neteq4/packet_buffer.cc | 10 +- .../audio_coding/neteq4/payload_splitter.cc | 5 + 7 files changed, 365 insertions(+), 39 deletions(-) diff --git a/webrtc/modules/audio_coding/neteq4/interface/neteq.h b/webrtc/modules/audio_coding/neteq4/interface/neteq.h index 547360b2e..ded87f58b 100644 --- a/webrtc/modules/audio_coding/neteq4/interface/neteq.h +++ b/webrtc/modules/audio_coding/neteq4/interface/neteq.h @@ -105,7 +105,8 @@ class NetEq { kFrameSplitError, kRedundancySplitError, kPacketBufferCorruption, - kOversizePacket + kOversizePacket, + kSyncPacketNotAccepted }; static const int kMaxNumPacketsInBuffer = 240; // TODO(hlundin): Remove. @@ -127,6 +128,18 @@ class NetEq { int length_bytes, uint32_t receive_timestamp) = 0; + // Inserts a sync-packet into packet queue. Sync-packets are decoded to + // silence and are intended to keep AV-sync intact in an event of long packet + // losses when Video NACK is enabled but Audio NACK is not. Clients of NetEq + // might insert sync-packet when they observe that buffer level of NetEq is + // decreasing below a certain threshold, defined by the application. + // Sync-packets should have the same payload type as the last audio payload + // type, i.e. they cannot have DTMF or CNG payload type, nor a codec change + // can be implied by inserting a sync-packet. + // Returns kOk on success, kFail on failure. + virtual int InsertSyncPacket(const WebRtcRTPHeader& rtp_header, + uint32_t receive_timestamp) = 0; + // Instructs NetEq to deliver 10 ms of audio data. The data is written to // |output_audio|, which can hold (at least) |max_length| elements. // The number of channels that were written to the output is provided in @@ -244,10 +257,6 @@ class NetEq { virtual int DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const = 0; - // Not implemented. - virtual int InsertSyncPacket(const WebRtcRTPHeader& rtp_header, - uint32_t receive_timestamp) = 0; - // Sets the background noise mode. virtual void SetBackgroundNoiseMode(NetEqBackgroundNoiseMode mode) = 0; diff --git a/webrtc/modules/audio_coding/neteq4/neteq_impl.cc b/webrtc/modules/audio_coding/neteq4/neteq_impl.cc index 8e8ffe24c..289b67fa4 100644 --- a/webrtc/modules/audio_coding/neteq4/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq4/neteq_impl.cc @@ -130,7 +130,7 @@ int NetEqImpl::InsertPacket(const WebRtcRTPHeader& rtp_header, ", ssrc=" << rtp_header.header.ssrc << ", len=" << length_bytes; int error = InsertPacketInternal(rtp_header, payload, length_bytes, - receive_timestamp); + receive_timestamp, false); if (error != 0) { LOG_FERR1(LS_WARNING, InsertPacketInternal, error); error_code_ = error; @@ -139,6 +139,27 @@ int NetEqImpl::InsertPacket(const WebRtcRTPHeader& rtp_header, return kOK; } +int NetEqImpl::InsertSyncPacket(const WebRtcRTPHeader& rtp_header, + uint32_t receive_timestamp) { + CriticalSectionScoped lock(crit_sect_.get()); + LOG(LS_VERBOSE) << "InsertPacket-Sync: ts=" + << rtp_header.header.timestamp << + ", sn=" << rtp_header.header.sequenceNumber << + ", pt=" << static_cast(rtp_header.header.payloadType) << + ", ssrc=" << rtp_header.header.ssrc; + + const uint8_t kSyncPayload[] = { 's', 'y', 'n', 'c' }; + int error = InsertPacketInternal( + rtp_header, kSyncPayload, sizeof(kSyncPayload), receive_timestamp, true); + + if (error != 0) { + LOG_FERR1(LS_WARNING, InsertPacketInternal, error); + error_code_ = error; + return kFail; + } + return kOK; +} + int NetEqImpl::GetAudio(size_t max_length, int16_t* output_audio, int* samples_per_channel, int* num_channels, NetEqOutputType* type) { @@ -372,11 +393,6 @@ int NetEqImpl::DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const { return 0; } -int NetEqImpl::InsertSyncPacket(const WebRtcRTPHeader& /* rtp_header */, - uint32_t /* receive_timestamp */) { - return kNotImplemented; -} - void NetEqImpl::SetBackgroundNoiseMode(NetEqBackgroundNoiseMode mode) { CriticalSectionScoped lock(crit_sect_.get()); assert(background_noise_.get()); @@ -391,15 +407,34 @@ NetEqBackgroundNoiseMode NetEqImpl::BackgroundNoiseMode() const { // Methods below this line are private. - int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, const uint8_t* payload, int length_bytes, - uint32_t receive_timestamp) { + uint32_t receive_timestamp, + bool is_sync_packet) { if (!payload) { LOG_F(LS_ERROR) << "payload == NULL"; return kInvalidPointer; } + // Sanity checks for sync-packets. + if (is_sync_packet) { + if (decoder_database_->IsDtmf(rtp_header.header.payloadType) || + decoder_database_->IsRed(rtp_header.header.payloadType) || + decoder_database_->IsComfortNoise(rtp_header.header.payloadType)) { + LOG_F(LS_ERROR) << "Sync-packet with an unacceptable payload type " + << rtp_header.header.payloadType; + return kSyncPacketNotAccepted; + } + if (first_packet_ || + rtp_header.header.payloadType != current_rtp_payload_type_ || + rtp_header.header.ssrc != ssrc_) { + // Even if |current_rtp_payload_type_| is 0xFF, sync-packet isn't + // accepted. + LOG_F(LS_ERROR) << "Changing codec, SSRC or first packet " + "with sync-packet."; + return kSyncPacketNotAccepted; + } + } PacketList packet_list; RTPHeader main_header; { @@ -418,6 +453,7 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, packet->primary = true; packet->waiting_time = 0; packet->payload = new uint8_t[packet->payload_length]; + packet->sync_packet = is_sync_packet; if (!packet->payload) { LOG_F(LS_ERROR) << "Payload pointer is NULL."; } @@ -455,11 +491,13 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, timestamp_scaler_->Reset(); } - // Update RTCP statistics. - rtcp_.Update(main_header, receive_timestamp); + // Update RTCP statistics, only for regular packets. + if (!is_sync_packet) + rtcp_.Update(main_header, receive_timestamp); // Check for RED payload type, and separate payloads into several packets. if (decoder_database_->IsRed(main_header.payloadType)) { + assert(!is_sync_packet); // We had a sanity check for this. if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) { LOG_FERR1(LS_WARNING, SplitRed, packet_list.size()); PacketBuffer::DeleteAllPackets(&packet_list); @@ -492,6 +530,7 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, assert(current_packet); assert(current_packet->payload); if (decoder_database_->IsDtmf(current_packet->header.payloadType)) { + assert(!current_packet->sync_packet); // We had a sanity check for this. DtmfEvent event; int ret = DtmfBuffer::ParseEvent( current_packet->header.timestamp, @@ -519,7 +558,8 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, } // Split payloads into smaller chunks. This also verifies that all payloads - // are of a known payload type. + // are of a known payload type. SplitAudio() method is protected against + // sync-packets. int ret = payload_splitter_->SplitAudio(&packet_list, *decoder_database_); if (ret != PayloadSplitter::kOK) { LOG_FERR1(LS_WARNING, SplitAudio, packet_list.size()); @@ -534,8 +574,8 @@ int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header, } } - // Update bandwidth estimate. - if (!packet_list.empty()) { + // Update bandwidth estimate, if the packet is not sync-packet. + if (!packet_list.empty() && !packet_list.front()->sync_packet) { // The list can be empty here if we got nothing but DTMF payloads. AudioDecoder* decoder = decoder_database_->GetDecoder(main_header.payloadType); @@ -1160,7 +1200,18 @@ int NetEqImpl::DecodeLoop(PacketList* packet_list, Operations* operation, packet_list->pop_front(); int payload_length = packet->payload_length; int16_t decode_length; - if (!packet->primary) { + if (packet->sync_packet) { + // Decode to silence with the same frame size as the last decode. + LOG(LS_VERBOSE) << "Decoding sync-packet: " << + " ts=" << packet->header.timestamp << + ", sn=" << packet->header.sequenceNumber << + ", pt=" << static_cast(packet->header.payloadType) << + ", ssrc=" << packet->header.ssrc << + ", len=" << packet->payload_length; + memset(&decoded_buffer_[*decoded_length], 0, decoder_frame_length_ * + decoder->channels() * sizeof(decoded_buffer_[0])); + decode_length = decoder_frame_length_; + } else if (!packet->primary) { // This is a redundant payload; call the special decoder method. LOG(LS_VERBOSE) << "Decoding packet (redundant):" << " ts=" << packet->header.timestamp << @@ -1696,8 +1747,8 @@ int NetEqImpl::ExtractPackets(int required_samples, PacketList* packet_list) { AudioDecoder* decoder = decoder_database_->GetDecoder( packet->header.payloadType); if (decoder) { - packet_duration = decoder->PacketDuration(packet->payload, - packet->payload_length); + packet_duration = packet->sync_packet ? decoder_frame_length_ : + decoder->PacketDuration(packet->payload, packet->payload_length); } else { LOG_FERR1(LS_WARNING, GetDecoder, packet->header.payloadType) << "Could not find a decoder for a packet about to be extracted."; diff --git a/webrtc/modules/audio_coding/neteq4/neteq_impl.h b/webrtc/modules/audio_coding/neteq4/neteq_impl.h index c0416809c..5e387f26a 100644 --- a/webrtc/modules/audio_coding/neteq4/neteq_impl.h +++ b/webrtc/modules/audio_coding/neteq4/neteq_impl.h @@ -76,6 +76,18 @@ class NetEqImpl : public webrtc::NetEq { int length_bytes, uint32_t receive_timestamp); + // Inserts a sync-packet into packet queue. Sync-packets are decoded to + // silence and are intended to keep AV-sync intact in an event of long packet + // losses when Video NACK is enabled but Audio NACK is not. Clients of NetEq + // might insert sync-packet when they observe that buffer level of NetEq is + // decreasing below a certain threshold, defined by the application. + // Sync-packets should have the same payload type as the last audio payload + // type, i.e. they cannot have DTMF or CNG payload type, nor a codec change + // can be implied by inserting a sync-packet. + // Returns kOk on success, kFail on failure. + virtual int InsertSyncPacket(const WebRtcRTPHeader& rtp_header, + uint32_t receive_timestamp); + // Instructs NetEq to deliver 10 ms of audio data. The data is written to // |output_audio|, which can hold (at least) |max_length| elements. // The number of channels that were written to the output is provided in @@ -181,9 +193,6 @@ class NetEqImpl : public webrtc::NetEq { // Gets background noise mode. virtual NetEqBackgroundNoiseMode BackgroundNoiseMode() const; - virtual int InsertSyncPacket(const WebRtcRTPHeader& rtp_header, - uint32_t receive_timestamp); - private: static const int kOutputSizeMs = 10; static const int kMaxFrameSize = 2880; // 60 ms @ 48 kHz. @@ -196,7 +205,8 @@ class NetEqImpl : public webrtc::NetEq { int InsertPacketInternal(const WebRtcRTPHeader& rtp_header, const uint8_t* payload, int length_bytes, - uint32_t receive_timestamp); + uint32_t receive_timestamp, + bool is_sync_packet); // Delivers 10 ms of audio data. The data is written to |output|, which can diff --git a/webrtc/modules/audio_coding/neteq4/neteq_unittest.cc b/webrtc/modules/audio_coding/neteq4/neteq_unittest.cc index 129bfa9a0..cefa3bb94 100644 --- a/webrtc/modules/audio_coding/neteq4/neteq_unittest.cc +++ b/webrtc/modules/audio_coding/neteq4/neteq_unittest.cc @@ -30,6 +30,20 @@ namespace webrtc { +static bool IsAllZero(const int16_t* buf, int buf_length) { + bool all_zero = true; + for (int n = 0; n < buf_length && all_zero; ++n) + all_zero = buf[n] == 0; + return all_zero; +} + +static bool IsAllNonZero(const int16_t* buf, int buf_length) { + bool all_non_zero = true; + for (int n = 0; n < buf_length && all_non_zero; ++n) + all_non_zero = buf[n] != 0; + return all_non_zero; +} + class RefFiles { public: RefFiles(const std::string& input_file, const std::string& output_file); @@ -856,4 +870,224 @@ TEST_F(NetEqDecodingTest, DISABLED_ON_ANDROID(BackgroundNoise)) { CheckBgnOff(32000, kBgnFade); EXPECT_EQ(kBgnFade, neteq_->BackgroundNoiseMode()); } + +TEST_F(NetEqDecodingTest, DISABLED_ON_ANDROID(SyncPacketInsert)) { + WebRtcRTPHeader rtp_info; + uint32_t receive_timestamp = 0; + // For the readability use the following payloads instead of the defaults of + // this test. + uint8_t kPcm16WbPayloadType = 1; + uint8_t kCngNbPayloadType = 2; + uint8_t kCngWbPayloadType = 3; + uint8_t kCngSwb32PayloadType = 4; + uint8_t kCngSwb48PayloadType = 5; + uint8_t kAvtPayloadType = 6; + uint8_t kRedPayloadType = 7; + uint8_t kIsacPayloadType = 9; // Payload type 8 is already registered. + + // Register decoders. + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderPCM16Bwb, + kPcm16WbPayloadType)); + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderCNGnb, kCngNbPayloadType)); + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderCNGwb, kCngWbPayloadType)); + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderCNGswb32kHz, + kCngSwb32PayloadType)); + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderCNGswb48kHz, + kCngSwb48PayloadType)); + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderAVT, kAvtPayloadType)); + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderRED, kRedPayloadType)); + ASSERT_EQ(0, neteq_->RegisterPayloadType(kDecoderISAC, kIsacPayloadType)); + + PopulateRtpInfo(0, 0, &rtp_info); + rtp_info.header.payloadType = kPcm16WbPayloadType; + + // The first packet injected cannot be sync-packet. + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + // Payload length of 10 ms PCM16 16 kHz. + const int kPayloadBytes = kBlockSize16kHz * sizeof(int16_t); + uint8_t payload[kPayloadBytes] = {0}; + ASSERT_EQ(0, neteq_->InsertPacket( + rtp_info, payload, kPayloadBytes, receive_timestamp)); + + // Next packet. Last packet contained 10 ms audio. + rtp_info.header.sequenceNumber++; + rtp_info.header.timestamp += kBlockSize16kHz; + receive_timestamp += kBlockSize16kHz; + + // Unacceptable payload types CNG, AVT (DTMF), RED. + rtp_info.header.payloadType = kCngNbPayloadType; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + rtp_info.header.payloadType = kCngWbPayloadType; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + rtp_info.header.payloadType = kCngSwb32PayloadType; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + rtp_info.header.payloadType = kCngSwb48PayloadType; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + rtp_info.header.payloadType = kAvtPayloadType; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + rtp_info.header.payloadType = kRedPayloadType; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + // Change of codec cannot be initiated with a sync packet. + rtp_info.header.payloadType = kIsacPayloadType; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + // Change of SSRC is not allowed with a sync packet. + rtp_info.header.payloadType = kPcm16WbPayloadType; + ++rtp_info.header.ssrc; + EXPECT_EQ(-1, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + + --rtp_info.header.ssrc; + EXPECT_EQ(0, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); +} + +// First insert several noise like packets, then sync-packets. Decoding all +// packets should not produce error, statistics should not show any packet loss +// and sync-packets should decode to zero. +TEST_F(NetEqDecodingTest, DISABLED_ON_ANDROID(SyncPacketDecode)) { + WebRtcRTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + const int kPayloadBytes = kBlockSize16kHz * sizeof(int16_t); + uint8_t payload[kPayloadBytes]; + int16_t decoded[kBlockSize16kHz]; + for (int n = 0; n < kPayloadBytes; ++n) { + payload[n] = (rand() & 0xF0) + 1; // Non-zero random sequence. + } + // Insert some packets which decode to noise. We are not interested in + // actual decoded values. + NetEqOutputType output_type; + int num_channels; + int samples_per_channel; + uint32_t receive_timestamp = 0; + int delay_samples = 0; + for (int n = 0; n < 100; ++n) { + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, kPayloadBytes, + receive_timestamp)); + ASSERT_EQ(0, neteq_->GetAudio(kBlockSize16kHz, decoded, + &samples_per_channel, &num_channels, + &output_type)); + ASSERT_EQ(kBlockSize16kHz, samples_per_channel); + ASSERT_EQ(1, num_channels); + + // Even if there is RTP packet in NetEq's buffer, the first frame pulled + // from NetEq starts with few zero samples. Here we measure this delay. + if (n == 0) { + while(decoded[delay_samples] == 0) delay_samples++; + } + rtp_info.header.sequenceNumber++; + rtp_info.header.timestamp += kBlockSize16kHz; + receive_timestamp += kBlockSize16kHz; + } + const int kNumSyncPackets = 10; + // Insert sync-packets, the decoded sequence should be all-zero. + for (int n = 0; n < kNumSyncPackets; ++n) { + ASSERT_EQ(0, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + ASSERT_EQ(0, neteq_->GetAudio(kBlockSize16kHz, decoded, + &samples_per_channel, &num_channels, + &output_type)); + ASSERT_EQ(kBlockSize16kHz, samples_per_channel); + ASSERT_EQ(1, num_channels); + EXPECT_TRUE(IsAllZero(&decoded[delay_samples], + samples_per_channel * num_channels - delay_samples)); + delay_samples = 0; // Delay only matters in the first frame. + rtp_info.header.sequenceNumber++; + rtp_info.header.timestamp += kBlockSize16kHz; + receive_timestamp += kBlockSize16kHz; + } + // We insert a regular packet, if sync packet are not correctly buffered then + // network statistics would show some packet loss. + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, kPayloadBytes, + receive_timestamp)); + ASSERT_EQ(0, neteq_->GetAudio(kBlockSize16kHz, decoded, + &samples_per_channel, &num_channels, + &output_type)); + // Make sure the last inserted packet is decoded and there are non-zero + // samples. + EXPECT_FALSE(IsAllZero(decoded, samples_per_channel * num_channels)); + NetEqNetworkStatistics network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats)); + // Expecting a "clean" network. + EXPECT_EQ(0, network_stats.packet_loss_rate); + EXPECT_EQ(0, network_stats.expand_rate); + EXPECT_EQ(0, network_stats.accelerate_rate); + EXPECT_EQ(0, network_stats.preemptive_rate); +} + +// Test if the size of the packet buffer reported correctly when containing +// sync packets. Also, test if network packets override sync packets. That is to +// prefer decoding a network packet to a sync packet, if both have same sequence +// number and timestamp. +TEST_F(NetEqDecodingTest, + DISABLED_ON_ANDROID(SyncPacketBufferSizeAndOverridenByNetworkPackets)) { + WebRtcRTPHeader rtp_info; + PopulateRtpInfo(0, 0, &rtp_info); + const int kPayloadBytes = kBlockSize16kHz * sizeof(int16_t); + uint8_t payload[kPayloadBytes]; + int16_t decoded[kBlockSize16kHz]; + for (int n = 0; n < kPayloadBytes; ++n) { + payload[n] = (rand() & 0xF0) + 1; // Non-zero random sequence. + } + // Insert some packets which decode to noise. We are not interested in + // actual decoded values. + NetEqOutputType output_type; + int num_channels; + int samples_per_channel; + uint32_t receive_timestamp = 0; + for (int n = 0; n < 1; ++n) { + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, kPayloadBytes, + receive_timestamp)); + ASSERT_EQ(0, neteq_->GetAudio(kBlockSize16kHz, decoded, + &samples_per_channel, &num_channels, + &output_type)); + ASSERT_EQ(kBlockSize16kHz, samples_per_channel); + ASSERT_EQ(1, num_channels); + rtp_info.header.sequenceNumber++; + rtp_info.header.timestamp += kBlockSize16kHz; + receive_timestamp += kBlockSize16kHz; + } + const int kNumSyncPackets = 10; + + WebRtcRTPHeader first_sync_packet_rtp_info; + memcpy(&first_sync_packet_rtp_info, &rtp_info, sizeof(rtp_info)); + + // Insert sync-packets, but no decoding. + for (int n = 0; n < kNumSyncPackets; ++n) { + ASSERT_EQ(0, neteq_->InsertSyncPacket(rtp_info, receive_timestamp)); + rtp_info.header.sequenceNumber++; + rtp_info.header.timestamp += kBlockSize16kHz; + receive_timestamp += kBlockSize16kHz; + } + NetEqNetworkStatistics network_stats; + ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats)); + EXPECT_EQ(kNumSyncPackets * 10, network_stats.current_buffer_size_ms); + + // Rewind |rtp_info| to that of the first sync packet. + memcpy(&rtp_info, &first_sync_packet_rtp_info, sizeof(rtp_info)); + + // Insert. + for (int n = 0; n < kNumSyncPackets; ++n) { + ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, kPayloadBytes, + receive_timestamp)); + rtp_info.header.sequenceNumber++; + rtp_info.header.timestamp += kBlockSize16kHz; + receive_timestamp += kBlockSize16kHz; + } + + // Decode. + for (int n = 0; n < kNumSyncPackets; ++n) { + ASSERT_EQ(0, neteq_->GetAudio(kBlockSize16kHz, decoded, + &samples_per_channel, &num_channels, + &output_type)); + ASSERT_EQ(kBlockSize16kHz, samples_per_channel); + ASSERT_EQ(1, num_channels); + EXPECT_TRUE(IsAllNonZero(decoded, samples_per_channel * num_channels)); + } +} + } // namespace diff --git a/webrtc/modules/audio_coding/neteq4/packet.h b/webrtc/modules/audio_coding/neteq4/packet.h index 90994a9b7..4518f9138 100644 --- a/webrtc/modules/audio_coding/neteq4/packet.h +++ b/webrtc/modules/audio_coding/neteq4/packet.h @@ -25,32 +25,49 @@ struct Packet { int payload_length; bool primary; // Primary, i.e., not redundant payload. int waiting_time; + bool sync_packet; // Constructor. Packet() : payload(NULL), payload_length(0), primary(true), - waiting_time(0) { + waiting_time(0), + sync_packet(false) { } // Comparison operators. Establish a packet ordering based on (1) timestamp, - // (2) sequence number, and (3) redundancy. Timestamp and sequence numbers - // are compared taking wrap-around into account. If both timestamp and - // sequence numbers are identical, a primary payload is considered "smaller" - // than a secondary. + // (2) sequence number, (3) regular packet vs sync-packet and (4) redundancy. + // Timestamp and sequence numbers are compared taking wrap-around into + // account. If both timestamp and sequence numbers are identical and one of + // the packets is sync-packet, the regular packet is considered earlier. For + // two regular packets with the same sequence number and timestamp a primary + // payload is considered "smaller" than a secondary. bool operator==(const Packet& rhs) const { return (this->header.timestamp == rhs.header.timestamp && this->header.sequenceNumber == rhs.header.sequenceNumber && - this->primary == rhs.primary); + this->primary == rhs.primary && + this->sync_packet == rhs.sync_packet); } bool operator!=(const Packet& rhs) const { return !operator==(rhs); } bool operator<(const Packet& rhs) const { if (this->header.timestamp == rhs.header.timestamp) { if (this->header.sequenceNumber == rhs.header.sequenceNumber) { - // Timestamp and sequence numbers are identical. Deem left hand side - // to be "smaller" (i.e., "earlier") if it is primary, and right hand - // side is not. + // Timestamp and sequence numbers are identical. A sync packet should + // be recognized "larger" (i.e. "later") compared to a "network packet" + // (regular packet from network not sync-packet). If none of the packets + // are sync-packets, then deem the left hand side to be "smaller" + // (i.e., "earlier") if it is primary, and right hand side is not. + // + // The condition on sync packets to be larger than "network packets," + // given same RTP sequence number and timestamp, guarantees that a + // "network packet" to be inserted in an earlier position into + // |packet_buffer_| compared to a sync packet of same timestamp and + // sequence number. + if (rhs.sync_packet) + return true; + if (this->sync_packet) + return false; return (this->primary && !rhs.primary); } return (static_cast(rhs.header.sequenceNumber diff --git a/webrtc/modules/audio_coding/neteq4/packet_buffer.cc b/webrtc/modules/audio_coding/neteq4/packet_buffer.cc index 2b5f28e4a..d19abbaa8 100644 --- a/webrtc/modules/audio_coding/neteq4/packet_buffer.cc +++ b/webrtc/modules/audio_coding/neteq4/packet_buffer.cc @@ -234,19 +234,19 @@ int PacketBuffer::NumSamplesInBuffer(DecoderDatabase* decoder_database, int last_decoded_length) const { PacketList::const_iterator it; int num_samples = 0; + int last_duration = last_decoded_length; for (it = buffer_.begin(); it != buffer_.end(); ++it) { Packet* packet = (*it); AudioDecoder* decoder = decoder_database->GetDecoder(packet->header.payloadType); if (decoder) { - int duration = decoder->PacketDuration(packet->payload, - packet->payload_length); + int duration = packet->sync_packet ? last_duration : + decoder->PacketDuration(packet->payload, packet->payload_length); if (duration >= 0) { - num_samples += duration; - continue; // Go to next packet in loop. + last_duration = duration; // Save the most up-to-date (valid) duration. } } - num_samples += last_decoded_length; + num_samples += last_duration; } return num_samples; } diff --git a/webrtc/modules/audio_coding/neteq4/payload_splitter.cc b/webrtc/modules/audio_coding/neteq4/payload_splitter.cc index 62ed5dae7..e07b61f70 100644 --- a/webrtc/modules/audio_coding/neteq4/payload_splitter.cc +++ b/webrtc/modules/audio_coding/neteq4/payload_splitter.cc @@ -163,6 +163,11 @@ int PayloadSplitter::SplitAudio(PacketList* packet_list, if (!info) { return kUnknownPayloadType; } + // No splitting for a sync-packet. + if (packet->sync_packet) { + ++it; + continue; + } PacketList new_packets; switch (info->codec_type) { case kDecoderPCMu: