From a305e9612afb9feac6bdbced512beb3a2557c3e2 Mon Sep 17 00:00:00 2001 From: "turaj@webrtc.org" Date: Thu, 6 Jun 2013 19:00:09 +0000 Subject: [PATCH] Nack for audio. R=stefan@webrtc.org, tina.legrand@webrtc.org Review URL: https://webrtc-codereview.appspot.com/1507004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@4188 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/interface/audio_coding_module.h | 59 ++- .../interface/audio_coding_module_typedefs.h | 3 + .../audio_coding/main/source/acm_neteq.cc | 7 + .../audio_coding/main/source/acm_neteq.h | 5 + .../main/source/audio_coding_module.cc | 14 +- .../main/source/audio_coding_module.gypi | 21 +- .../main/source/audio_coding_module_impl.cc | 118 ++++- .../main/source/audio_coding_module_impl.h | 24 +- .../modules/audio_coding/main/source/nack.cc | 224 ++++++++ .../modules/audio_coding/main/source/nack.h | 209 ++++++++ .../audio_coding/main/source/nack_unittest.cc | 482 ++++++++++++++++++ .../modules/audio_coding/main/test/APITest.h | 11 +- .../modules/audio_coding/main/test/Channel.cc | 26 +- .../modules/audio_coding/main/test/Channel.h | 25 +- .../main/test/insert_packet_with_timing.cc | 316 ++++++++++++ .../neteq/interface/webrtc_neteq.h | 3 + webrtc/modules/audio_coding/neteq/mcu.h | 3 + .../modules/audio_coding/neteq/signal_mcu.c | 11 + .../modules/audio_coding/neteq/webrtc_neteq.c | 16 + 19 files changed, 1522 insertions(+), 55 deletions(-) create mode 100644 webrtc/modules/audio_coding/main/source/nack.cc create mode 100644 webrtc/modules/audio_coding/main/source/nack.h create mode 100644 webrtc/modules/audio_coding/main/source/nack_unittest.cc create mode 100644 webrtc/modules/audio_coding/main/test/insert_packet_with_timing.cc diff --git a/webrtc/modules/audio_coding/main/interface/audio_coding_module.h b/webrtc/modules/audio_coding/main/interface/audio_coding_module.h index c3bbc9b77..dbc4fe535 100644 --- a/webrtc/modules/audio_coding/main/interface/audio_coding_module.h +++ b/webrtc/modules/audio_coding/main/interface/audio_coding_module.h @@ -8,8 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_H -#define WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_H +#ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_H_ +#define WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_H_ + +#include #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h" @@ -23,8 +25,9 @@ struct CodecInst; struct WebRtcRTPHeader; class AudioFrame; class RTPFragmentationHeader; +class Clock; -#define WEBRTC_10MS_PCM_AUDIO 960 // 16 bits super wideband 48 kHz +#define WEBRTC_10MS_PCM_AUDIO 960 // 16 bits super wideband 48 kHz // Callback class used for sending data ready to be packetized class AudioPacketizationCallback { @@ -63,11 +66,11 @@ class ACMVQMonCallback { virtual ~ACMVQMonCallback() {} virtual int32_t NetEqStatistics( - const int32_t id, // current ACM id - const uint16_t MIUsValid, // valid voice duration in ms - const uint16_t MIUsReplaced, // concealed voice duration in ms - const uint8_t eventFlags, // concealed voice flags - const uint16_t delayMS) = 0; // average delay in ms + const int32_t id, // current ACM id + const uint16_t MIUsValid, // valid voice duration in ms + const uint16_t MIUsReplaced, // concealed voice duration in ms + const uint8_t eventFlags, // concealed voice flags + const uint16_t delayMS) = 0; // average delay in ms }; class AudioCodingModule: public Module { @@ -77,9 +80,14 @@ class AudioCodingModule: public Module { public: /////////////////////////////////////////////////////////////////////////// - // Creation and destruction of a ACM + // Creation and destruction of a ACM. + // + // The second method is used for testing where a simulated clock can be + // injected into ACM. ACM will take the owner ship of the object clock and + // delete it when destroyed. // static AudioCodingModule* Create(const int32_t id); + static AudioCodingModule* Create(const int32_t id, Clock* clock); static void Destroy(AudioCodingModule* module); @@ -814,8 +822,8 @@ class AudioCodingModule: public Module { // // Input: // -desired_freq_hz : the desired sampling frequency, in Hertz, of the - // output audio. If set to -1, the function returns the - // audio at the current sampling frequency. + // output audio. If set to -1, the function returns + // the audio at the current sampling frequency. // // Output: // -audio_frame : output audio frame which contains raw audio data @@ -966,8 +974,35 @@ class AudioCodingModule: public Module { // 0 if delay is set successfully. // virtual int SetInitialPlayoutDelay(int delay_ms) = 0; + + // + // Enable NACK and set the maximum size of the NACK list. If NACK is already + // enable then the maximum NACK list size is modified accordingly. + // + // If the sequence number of last received packet is N, the sequence numbers + // of NACK list are in the range of [N - |max_nack_list_size|, N). + // + // |max_nack_list_size| should be positive (none zero) and less than or + // equal to |Nack::kNackListSizeLimit|. Otherwise, No change is applied and -1 + // is returned. 0 is returned at success. + // + virtual int EnableNack(size_t max_nack_list_size) = 0; + + // Disable NACK. + virtual void DisableNack() = 0; + + // + // Get a list of packets to be retransmitted. |round_trip_time_ms| is an + // estimate of the round-trip-time (in milliseconds). Missing packets which + // will be playout in a shorter time than the round-trip-time (with respect + // to the time this API is called) will not be included in the list. + // + // Negative |round_trip_time_ms| results is an error message and empty list + // is returned. + // + virtual std::vector GetNackList(int round_trip_time_ms) const = 0; }; } // namespace webrtc -#endif // WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_H +#endif // WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_H_ diff --git a/webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h b/webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h index 03a7df829..199be963a 100644 --- a/webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h +++ b/webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h @@ -11,6 +11,9 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_TYPEDEFS_H_ #define WEBRTC_MODULES_AUDIO_CODING_MAIN_INTERFACE_AUDIO_CODING_MODULE_TYPEDEFS_H_ +#include + +#include "webrtc/modules/interface/module_common_types.h" #include "webrtc/typedefs.h" namespace webrtc { diff --git a/webrtc/modules/audio_coding/main/source/acm_neteq.cc b/webrtc/modules/audio_coding/main/source/acm_neteq.cc index f2eafd71b..6f28de75a 100644 --- a/webrtc/modules/audio_coding/main/source/acm_neteq.cc +++ b/webrtc/modules/audio_coding/main/source/acm_neteq.cc @@ -1117,4 +1117,11 @@ int ACMNetEQ::LeastRequiredDelayMs() const { return WebRtcNetEQ_GetRequiredDelayMs(inst_[0]); } +bool ACMNetEQ::DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const { + CriticalSectionScoped lock(neteq_crit_sect_); + if (WebRtcNetEQ_DecodedRtpInfo(inst_[0], sequence_number, timestamp) < 0) + return false; + return true; +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/source/acm_neteq.h b/webrtc/modules/audio_coding/main/source/acm_neteq.h index e70ac246d..0a04c294e 100644 --- a/webrtc/modules/audio_coding/main/source/acm_neteq.h +++ b/webrtc/modules/audio_coding/main/source/acm_neteq.h @@ -289,6 +289,11 @@ class ACMNetEQ { // void EnableAVSync(bool enable); + // + // Get sequence number and timestamp of the last decoded RTP. + // + bool DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const; + // // Set a minimum delay in NetEq. Unless channel condition dictates a longer // delay, the given delay is maintained by NetEq. diff --git a/webrtc/modules/audio_coding/main/source/audio_coding_module.cc b/webrtc/modules/audio_coding/main/source/audio_coding_module.cc index 159fbb849..570fec51c 100644 --- a/webrtc/modules/audio_coding/main/source/audio_coding_module.cc +++ b/webrtc/modules/audio_coding/main/source/audio_coding_module.cc @@ -14,13 +14,21 @@ #include "webrtc/modules/audio_coding/main/source/acm_codec_database.h" #include "webrtc/modules/audio_coding/main/source/acm_dtmf_detection.h" #include "webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h" +#include "webrtc/system_wrappers/interface/clock.h" #include "webrtc/system_wrappers/interface/trace.h" namespace webrtc { // Create module AudioCodingModule* AudioCodingModule::Create(const int32_t id) { - return new AudioCodingModuleImpl(id); + return new AudioCodingModuleImpl(id, Clock::GetRealTimeClock()); +} + +// Used for testing by inserting a simulated clock. ACM will not destroy the +// injected |clock| the client has to take care of that. +AudioCodingModule* AudioCodingModule::Create(const int32_t id, + Clock* clock) { + return new AudioCodingModuleImpl(id, clock); } // Destroy module @@ -49,7 +57,7 @@ int32_t AudioCodingModule::Codec(const char* payload_name, // Get the id of the codec from the database. codec_id = ACMCodecDB::CodecId(payload_name, sampling_freq_hz, channels); if (codec_id < 0) { - // We couldn't find a matching codec, set the parameterss to unacceptable + // We couldn't find a matching codec, set the parameters to unacceptable // values and return. codec->plname[0] = '\0'; codec->pltype = -1; @@ -63,7 +71,7 @@ int32_t AudioCodingModule::Codec(const char* payload_name, ACMCodecDB::Codec(codec_id, codec); // Keep the number of channels from the function call. For most codecs it - // will be the same value as in defaul codec settings, but not for all. + // will be the same value as in default codec settings, but not for all. codec->channels = channels; return 0; diff --git a/webrtc/modules/audio_coding/main/source/audio_coding_module.gypi b/webrtc/modules/audio_coding/main/source/audio_coding_module.gypi index e6ba5009b..464b35fbf 100644 --- a/webrtc/modules/audio_coding/main/source/audio_coding_module.gypi +++ b/webrtc/modules/audio_coding/main/source/audio_coding_module.gypi @@ -103,6 +103,8 @@ 'audio_coding_module.cc', 'audio_coding_module_impl.cc', 'audio_coding_module_impl.h', + 'nack.cc', + 'nack.h', ], }, ], @@ -164,6 +166,22 @@ '../test/PCMFile.cc', ], }, # delay_test + { + 'target_name': 'insert_packet_with_timing', + 'type': 'executable', + 'dependencies': [ + 'audio_coding_module', + '<(DEPTH)/testing/gtest.gyp:gtest', + '<(webrtc_root)/test/test.gyp:test_support_main', + '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', + '<(DEPTH)/third_party/google-gflags/google-gflags.gyp:google-gflags', + ], + 'sources': [ + '../test/insert_packet_with_timing.cc', + '../test/Channel.cc', + '../test/PCMFile.cc', + ], + }, # delay_test { 'target_name': 'audio_coding_unittests', 'type': 'executable', @@ -184,12 +202,13 @@ ], 'sources': [ 'acm_neteq_unittest.cc', + 'nack_unittest.cc', '../../codecs/cng/cng_unittest.cc', '../../codecs/isac/fix/source/filters_unittest.cc', '../../codecs/isac/fix/source/filterbanks_unittest.cc', '../../codecs/isac/fix/source/lpc_masking_model_unittest.cc', '../../codecs/isac/fix/source/transform_unittest.cc', - '../../codecs/isac/main/source/isac_unittest.cc', + '../../codecs/isac/main/source/isac_unittest.cc', '../../codecs/opus/opus_unittest.cc', # Test for NetEq 4. '../../neteq4/audio_multi_vector_unittest.cc', diff --git a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc index 260ec6937..ba15921bc 100644 --- a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc +++ b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc @@ -13,13 +13,18 @@ #include #include +#include // For std::max. + #include "webrtc/engine_configurations.h" #include "webrtc/modules/audio_coding/main/source/acm_codec_database.h" #include "webrtc/modules/audio_coding/main/source/acm_common_defs.h" #include "webrtc/modules/audio_coding/main/source/acm_dtmf_detection.h" #include "webrtc/modules/audio_coding/main/source/acm_generic_codec.h" #include "webrtc/modules/audio_coding/main/source/acm_resampler.h" +#include "webrtc/modules/audio_coding/main/source/nack.h" +#include "webrtc/system_wrappers/interface/clock.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" +#include "webrtc/system_wrappers/interface/logging.h" #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h" #include "webrtc/system_wrappers/interface/tick_util.h" #include "webrtc/system_wrappers/interface/trace.h" @@ -47,6 +52,10 @@ enum { static const uint32_t kMaskTimestamp = 0x03ffffff; static const int kDefaultTimestampDiff = 960; // 20 ms @ 48 kHz. +// If packet N is arrived all packets prior to N - |kNackThresholdPackets| which +// are not received are considered as lost, and appear in NACK list. +static const int kNackThresholdPackets = 2; + namespace { bool IsCodecRED(const CodecInst* codec) { @@ -104,24 +113,9 @@ int TimestampLessThan(uint32_t t1, uint32_t t2) { } } -// -// Return the timestamp of current time, computed according to sampling rate -// of the codec identified by |codec_id|. -// -uint32_t NowTimestamp(int codec_id) { - // Down-cast the time to (32-6)-bit since we only care about - // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. - // we masked 6 most significant bits of 32-bit so we don't loose resolution - // when do the following multiplication. - int sample_rate_khz = ACMCodecDB::database_[codec_id].plfreq / 1000; - const uint32_t now_in_ms = static_cast( - TickTime::MillisecondTimestamp() & kMaskTimestamp); - return static_cast(sample_rate_khz * now_in_ms); -} - } // namespace -AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id) +AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id, Clock* clock) : packetization_callback_(NULL), id_(id), last_timestamp_(0xD87F3F9F), @@ -171,7 +165,10 @@ AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id) last_timestamp_diff_(kDefaultTimestampDiff), last_sequence_number_(0), last_ssrc_(0), - last_packet_was_sync_(false) { + last_packet_was_sync_(false), + clock_(clock), + nack_(), + nack_enabled_(false) { // Nullify send codec memory, set payload type and set codec name to // invalid values. @@ -726,13 +723,14 @@ int AudioCodingModuleImpl::ProcessSingleStream() { if (fec_active) { // Callback with payload data, including redundant data (FEC/RED). packetization_callback_->SendData(frame_type, current_payload_type, - rtp_timestamp, stream, length_bytes, + rtp_timestamp, stream, + length_bytes, &my_fragmentation); } else { // Callback with payload data. packetization_callback_->SendData(frame_type, current_payload_type, - rtp_timestamp, stream, length_bytes, - NULL); + rtp_timestamp, stream, + length_bytes, NULL); } } @@ -1654,7 +1652,7 @@ int32_t AudioCodingModuleImpl::InitializeReceiverSafe() { num_packets_accumulated_ = 0; num_bytes_accumulated_ = 0; accumulated_audio_ms_ = 0; - first_payload_received_ = 0;; + first_payload_received_ = 0; last_incoming_send_timestamp_ = 0; track_neteq_buffer_ = false; playout_ts_ = 0; @@ -2060,12 +2058,26 @@ int32_t AudioCodingModuleImpl::IncomingPacket( if (track_neteq_buffer_ || av_sync_) { last_incoming_send_timestamp_ = rtp_info.header.timestamp; } + + if (nack_enabled_) { + assert(nack_.get()); + // Codec is changed, reset NACK and update sampling rate. + nack_->Reset(); + nack_->UpdateSampleRate( + ACMCodecDB::database_[current_receive_codec_idx_].plfreq); + } } last_recv_audio_codec_pltype_ = my_payload_type; } // Current timestamp based on the receiver sampling frequency. last_receive_timestamp_ = NowTimestamp(current_receive_codec_idx_); + + if (nack_enabled_) { + assert(nack_.get()); + nack_->UpdateLastReceivedPacket(rtp_header.header.sequenceNumber, + rtp_header.header.timestamp); + } } int per_neteq_payload_length = payload_length; @@ -2105,7 +2117,7 @@ int32_t AudioCodingModuleImpl::IncomingPacket( UpdateBufferingSafe(rtp_header, per_neteq_payload_length); if (av_sync_) { - if(rtp_info.header.sequenceNumber == last_sequence_number_ + 1) { + if (rtp_info.header.sequenceNumber == last_sequence_number_ + 1) { last_timestamp_diff_ = rtp_info.header.timestamp - last_incoming_send_timestamp_; } @@ -2309,6 +2321,10 @@ int32_t AudioCodingModuleImpl::PlayoutData10Ms( "PlayoutData failed, RecOut Failed"); return -1; } + int seq_num; + uint32_t timestamp; + bool update_nack = nack_enabled_ && // Update NACK only if it is enabled. + neteq_.DecodedRtpInfo(&seq_num, ×tamp); audio_frame->num_channels_ = audio_frame_.num_channels_; audio_frame->vad_activity_ = audio_frame_.vad_activity_; @@ -2328,6 +2344,11 @@ int32_t AudioCodingModuleImpl::PlayoutData10Ms( { CriticalSectionScoped lock(acm_crit_sect_); + if (update_nack) { + assert(nack_.get()); + nack_->UpdateLastDecodedPacket(seq_num, timestamp); + } + // If we are in AV-sync and have already received an audio packet, but the // latest packet is too late, then insert sync packet. if (av_sync_ && first_payload_received_ && @@ -3047,8 +3068,61 @@ void AudioCodingModuleImpl::UpdateBufferingSafe(const WebRtcRTPHeader& rtp_info, initial_delay_ms_ * in_sample_rate_khz)); } +uint32_t AudioCodingModuleImpl::NowTimestamp(int codec_id) { + // Down-cast the time to (32-6)-bit since we only care about + // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. + // we masked 6 most significant bits of 32-bit so we don't lose resolution + // when do the following multiplication. + int sample_rate_khz = ACMCodecDB::database_[codec_id].plfreq / 1000; + const uint32_t now_in_ms = static_cast( + clock_->TimeInMilliseconds() & kMaskTimestamp); + return static_cast(sample_rate_khz * now_in_ms); +} + +std::vector AudioCodingModuleImpl::GetNackList( + int round_trip_time_ms) const { + CriticalSectionScoped lock(acm_crit_sect_); + if (round_trip_time_ms < 0) { + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_, + "GetNackList: round trip time cannot be negative." + " round_trip_time_ms=%d", round_trip_time_ms); + } + if (nack_enabled_ && round_trip_time_ms >= 0) { + assert(nack_.get()); + return nack_->GetNackList(round_trip_time_ms); + } + std::vector empty_list; + return empty_list; +} + int AudioCodingModuleImpl::LeastRequiredDelayMs() const { return std::max(neteq_.LeastRequiredDelayMs(), initial_delay_ms_); } +int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) { + // Don't do anything if |max_nack_list_size| is out of range. + if (max_nack_list_size == 0 || max_nack_list_size > Nack::kNackListSizeLimit) + return -1; + + CriticalSectionScoped lock(acm_crit_sect_); + if (!nack_enabled_) { + nack_.reset(Nack::Create(kNackThresholdPackets)); + nack_enabled_ = true; + + // Sampling rate might need to be updated if we change from disable to + // enable. Do it if the receive codec is valid. + if (current_receive_codec_idx_ >= 0) { + nack_->UpdateSampleRate( + ACMCodecDB::database_[current_receive_codec_idx_].plfreq); + } + } + return nack_->SetMaxNackListSize(max_nack_list_size); +} + +void AudioCodingModuleImpl::DisableNack() { + CriticalSectionScoped lock(acm_crit_sect_); + nack_.reset(); // Memory is released. + nack_enabled_ = false; +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h index a0ae01434..a20159bbd 100644 --- a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h +++ b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_SOURCE_AUDIO_CODING_MODULE_IMPL_H_ #define WEBRTC_MODULES_AUDIO_CODING_MAIN_SOURCE_AUDIO_CODING_MODULE_IMPL_H_ +#include + #include "webrtc/common_types.h" #include "webrtc/engine_configurations.h" #include "webrtc/modules/audio_coding/main/source/acm_codec_database.h" @@ -24,11 +26,13 @@ class ACMDTMFDetection; class ACMGenericCodec; class CriticalSectionWrapper; class RWLockWrapper; +class Clock; +class Nack; class AudioCodingModuleImpl : public AudioCodingModule { public: // Constructor - explicit AudioCodingModuleImpl(const int32_t id); + AudioCodingModuleImpl(const int32_t id, Clock* clock); // Destructor ~AudioCodingModuleImpl(); @@ -243,6 +247,8 @@ class AudioCodingModuleImpl : public AudioCodingModule { int32_t UnregisterReceiveCodec(const int16_t payload_type); + std::vector GetNackList(int round_trip_time_ms) const; + protected: void UnregisterSendCodec(); @@ -299,6 +305,12 @@ class AudioCodingModuleImpl : public AudioCodingModule { // 0: if delay set successfully. int SetInitialPlayoutDelay(int delay_ms); + // Enable NACK and set the maximum size of the NACK list. + int EnableNack(size_t max_nack_list_size); + + // Disable NACK. + void DisableNack(); + private: // Change required states after starting to receive the codec corresponding // to |index|. @@ -333,6 +345,12 @@ class AudioCodingModuleImpl : public AudioCodingModule { void UpdateBufferingSafe(const WebRtcRTPHeader& rtp_info, int payload_len_bytes); + // + // Return the timestamp of current time, computed according to sampling rate + // of the codec identified by |codec_id|. + // + uint32_t NowTimestamp(int codec_id); + AudioPacketizationCallback* packetization_callback_; int32_t id_; uint32_t last_timestamp_; @@ -427,6 +445,10 @@ class AudioCodingModuleImpl : public AudioCodingModule { uint32_t last_ssrc_; bool last_packet_was_sync_; int64_t last_receive_timestamp_; + + Clock* clock_; + scoped_ptr nack_; + bool nack_enabled_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/source/nack.cc b/webrtc/modules/audio_coding/main/source/nack.cc new file mode 100644 index 000000000..f42e4d4eb --- /dev/null +++ b/webrtc/modules/audio_coding/main/source/nack.cc @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_coding/main/source/nack.h" + +#include // For std::max. +#include // For assert. + +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/system_wrappers/interface/logging.h" + +namespace webrtc { + +namespace { + +const int kDefaultSampleRateKhz = 48; +const int kDefaultPacketSizeMs = 20; + +} // namespace + +Nack::Nack(int nack_threshold_packets) + : nack_threshold_packets_(nack_threshold_packets), + sequence_num_last_received_rtp_(0), + timestamp_last_received_rtp_(0), + any_rtp_received_(false), + sequence_num_last_decoded_rtp_(0), + timestamp_last_decoded_rtp_(0), + any_rtp_decoded_(false), + sample_rate_khz_(kDefaultSampleRateKhz), + samples_per_packet_(sample_rate_khz_ * kDefaultPacketSizeMs), + max_nack_list_size_(kNackListSizeLimit) {} + +Nack* Nack::Create(int nack_threshold_packets) { + return new Nack(nack_threshold_packets); +} + +void Nack::UpdateSampleRate(int sample_rate_hz) { + assert(sample_rate_hz > 0); + sample_rate_khz_ = sample_rate_hz / 1000; +} + +void Nack::UpdateLastReceivedPacket(uint16_t sequence_number, + uint32_t timestamp) { + // Just record the value of sequence number and timestamp if this is the + // first packet. + if (!any_rtp_received_) { + sequence_num_last_received_rtp_ = sequence_number; + timestamp_last_received_rtp_ = timestamp; + any_rtp_received_ = true; + // If no packet is decoded, to have a reasonable estimate of time-to-play + // use the given values. + if (!any_rtp_decoded_) { + sequence_num_last_decoded_rtp_ = sequence_number; + timestamp_last_decoded_rtp_ = timestamp; + } + return; + } + + if (sequence_number == sequence_num_last_received_rtp_) + return; + + // Received RTP should not be in the list. + nack_list_.erase(sequence_number); + + // If this is an old sequence number, no more action is required, return. + if (IsNewerSequenceNumber(sequence_num_last_received_rtp_, sequence_number)) + return; + + UpdateSamplesPerPacket(sequence_number, timestamp); + + UpdateList(sequence_number); + + sequence_num_last_received_rtp_ = sequence_number; + timestamp_last_received_rtp_ = timestamp; + LimitNackListSize(); +} + +void Nack::UpdateSamplesPerPacket(uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp) { + uint32_t timestamp_increase = timestamp_current_received_rtp - + timestamp_last_received_rtp_; + uint16_t sequence_num_increase = sequence_number_current_received_rtp - + sequence_num_last_received_rtp_; + + samples_per_packet_ = timestamp_increase / sequence_num_increase; +} + +void Nack::UpdateList(uint16_t sequence_number_current_received_rtp) { + // Some of the packets which were considered late, now are considered missing. + ChangeFromLateToMissing(sequence_number_current_received_rtp); + + if (IsNewerSequenceNumber(sequence_number_current_received_rtp, + sequence_num_last_received_rtp_ + 1)) + AddToList(sequence_number_current_received_rtp); +} + +void Nack::ChangeFromLateToMissing( + uint16_t sequence_number_current_received_rtp) { + NackList::const_iterator lower_bound = nack_list_.lower_bound( + static_cast(sequence_number_current_received_rtp - + nack_threshold_packets_)); + + for (NackList::iterator it = nack_list_.begin(); it != lower_bound; ++it) + it->second.is_missing = true; +} + +uint32_t Nack::EstimateTimestamp(uint16_t sequence_num) { + uint16_t sequence_num_diff = sequence_num - sequence_num_last_received_rtp_; + return sequence_num_diff * samples_per_packet_ + timestamp_last_received_rtp_; +} + +void Nack::AddToList(uint16_t sequence_number_current_received_rtp) { + assert(!any_rtp_decoded_ || IsNewerSequenceNumber( + sequence_number_current_received_rtp, sequence_num_last_decoded_rtp_)); + + // Packets with sequence numbers older than |upper_bound_missing| are + // considered missing, and the rest are considered late. + uint16_t upper_bound_missing = sequence_number_current_received_rtp - + nack_threshold_packets_; + + for (uint16_t n = sequence_num_last_received_rtp_ + 1; + IsNewerSequenceNumber(sequence_number_current_received_rtp, n); ++n) { + bool is_missing = IsNewerSequenceNumber(upper_bound_missing, n); + uint32_t timestamp = EstimateTimestamp(n); + NackElement nack_element(TimeToPlay(timestamp), timestamp, is_missing); + nack_list_.insert(nack_list_.end(), std::make_pair(n, nack_element)); + } +} + +void Nack::UpdateEstimatedPlayoutTimeBy10ms() { + while (!nack_list_.empty() && + nack_list_.begin()->second.time_to_play_ms <= 10) + nack_list_.erase(nack_list_.begin()); + + for (NackList::iterator it = nack_list_.begin(); it != nack_list_.end(); ++it) + it->second.time_to_play_ms -= 10; +} + +void Nack::UpdateLastDecodedPacket(uint16_t sequence_number, + uint32_t timestamp) { + if (IsNewerSequenceNumber(sequence_number, sequence_num_last_decoded_rtp_) || + !any_rtp_decoded_) { + sequence_num_last_decoded_rtp_ = sequence_number; + timestamp_last_decoded_rtp_ = timestamp; + // Packets in the list with sequence numbers less than the + // sequence number of the decoded RTP should be removed from the lists. + // They will be discarded by the jitter buffer if they arrive. + nack_list_.erase(nack_list_.begin(), nack_list_.upper_bound( + sequence_num_last_decoded_rtp_)); + + // Update estimated time-to-play. + for (NackList::iterator it = nack_list_.begin(); it != nack_list_.end(); + ++it) + it->second.time_to_play_ms = TimeToPlay(it->second.estimated_timestamp); + } else { + assert(sequence_number == sequence_num_last_decoded_rtp_); + + // Same sequence number as before. 10 ms is elapsed, update estimations for + // time-to-play. + UpdateEstimatedPlayoutTimeBy10ms(); + + // Update timestamp for better estimate of time-to-play, for packets which + // are added to NACK list later on. + timestamp_last_decoded_rtp_ += sample_rate_khz_ * 10; + } + any_rtp_decoded_ = true; +} + +Nack::NackList Nack::GetNackList() const { + return nack_list_; +} + +void Nack::Reset() { + nack_list_.clear(); + + sequence_num_last_received_rtp_ = 0; + timestamp_last_received_rtp_ = 0; + any_rtp_received_ = false; + sequence_num_last_decoded_rtp_ = 0; + timestamp_last_decoded_rtp_ = 0; + any_rtp_decoded_ = false; + sample_rate_khz_ = kDefaultSampleRateKhz; + samples_per_packet_ = sample_rate_khz_ * kDefaultPacketSizeMs; +} + +int Nack::SetMaxNackListSize(size_t max_nack_list_size) { + if (max_nack_list_size == 0 || max_nack_list_size > kNackListSizeLimit) + return -1; + max_nack_list_size_ = max_nack_list_size; + LimitNackListSize(); + return 0; +} + +void Nack::LimitNackListSize() { + uint16_t limit = sequence_num_last_received_rtp_ - + static_cast(max_nack_list_size_) - 1; + nack_list_.erase(nack_list_.begin(), nack_list_.upper_bound(limit)); +} + +int Nack::TimeToPlay(uint32_t timestamp) const { + uint32_t timestamp_increase = timestamp - timestamp_last_decoded_rtp_; + return timestamp_increase / sample_rate_khz_; +} + +// We don't erase elements with time-to-play shorter than round-trip-time. +std::vector Nack::GetNackList(int round_trip_time_ms) const { + std::vector sequence_numbers; + for (NackList::const_iterator it = nack_list_.begin(); it != nack_list_.end(); + ++it) { + if (it->second.is_missing && + it->second.time_to_play_ms > round_trip_time_ms) + sequence_numbers.push_back(it->first); + } + return sequence_numbers; +} + +} // webrtc diff --git a/webrtc/modules/audio_coding/main/source/nack.h b/webrtc/modules/audio_coding/main/source/nack.h new file mode 100644 index 000000000..e047c2824 --- /dev/null +++ b/webrtc/modules/audio_coding/main/source/nack.h @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_CODING_MAIN_SOURCE_NACK_H_ +#define WEBRTC_MODULES_AUDIO_CODING_MAIN_SOURCE_NACK_H_ + +#include +#include + +#include "webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" +#include "webrtc/test/testsupport/gtest_prod_util.h" + +// +// The Nack class keeps track of the lost packets, an estimate of time-to-play +// for each packet is also given. +// +// Every time a packet is pushed into NetEq, LastReceivedPacket() has to be +// called to update the NACK list. +// +// Every time 10ms audio is pulled from NetEq LastDecodedPacket() should be +// called, and time-to-play is updated at that moment. +// +// If packet N is received, any packet prior to |N - NackThreshold| which is not +// arrived is considered lost, and should be labeled as "missing" (the size of +// the list might be limited and older packet eliminated from the list). Packets +// |N - NackThreshold|, |N - NackThreshold + 1|, ..., |N - 1| are considered +// "late." A "late" packet with sequence number K is changed to "missing" any +// time a packet with sequence number newer than |K + NackList| is arrived. +// +// The Nack class has to know about the sample rate of the packets to compute +// time-to-play. So sample rate should be set as soon as the first packet is +// received. If there is a change in the receive codec (sender changes codec) +// then Nack should be reset. This is because NetEQ would flush its buffer and +// re-transmission is meaning less for old packet. Therefore, in that case, +// after reset the sampling rate has to be updated. +// +// Thread Safety +// ============= +// Please note that this class in not thread safe. The class must be protected +// if different APIs are called from different threads. +// +namespace webrtc { + +class Nack { + public: + // A limit for the size of the NACK list. + static const size_t kNackListSizeLimit = 500; // 10 seconds for 20 ms frame + // packets. + // Factory method. + static Nack* Create(int nack_threshold_packets); + + ~Nack() {} + + // Set a maximum for the size of the NACK list. If the last received packet + // has sequence number of N, then NACK list will not contain any element + // with sequence number earlier than N - |max_nack_list_size|. + // + // The largest maximum size is defined by |kNackListSizeLimit| + int SetMaxNackListSize(size_t max_nack_list_size); + + // Set the sampling rate. + // + // If associated sampling rate of the received packets is changed, call this + // function to update sampling rate. Note that if there is any change in + // received codec then NetEq will flush its buffer and NACK has to be reset. + // After Reset() is called sampling rate has to be set. + void UpdateSampleRate(int sample_rate_hz); + + // Update the sequence number and the timestamp of the last decoded RTP. This + // API should be called every time 10 ms audio is pulled from NetEq. + void UpdateLastDecodedPacket(uint16_t sequence_number, uint32_t timestamp); + + // Update the sequence number and the timestamp of the last received RTP. This + // API should be called every time a packet pushed into ACM. + void UpdateLastReceivedPacket(uint16_t sequence_number, uint32_t timestamp); + + // Get a list of "missing" packets which have expected time-to-play larger + // than the given round-trip-time (in milliseconds). + // Note: Late packets are not included. + std::vector GetNackList(int round_trip_time_ms) const; + + // Reset to default values. The NACK list is cleared. + // |nack_threshold_packets_| & |max_nack_list_size_| preserve their values. + void Reset(); + + private: + // This test need to access the private method GetNackList(). + FRIEND_TEST_ALL_PREFIXES(NackTest, EstimateTimestampAndTimeToPlay); + + struct NackElement { + NackElement(int initial_time_to_play_ms, + uint32_t initial_timestamp, + bool missing) + : time_to_play_ms(initial_time_to_play_ms), + estimated_timestamp(initial_timestamp), + is_missing(missing) {} + + // Estimated time (ms) left for this packet to be decoded. This estimate is + // updated every time jitter buffer decodes a packet. + int time_to_play_ms; + + // A guess about the timestamp of the missing packet, it is used for + // estimation of |time_to_play_ms|. The estimate might be slightly wrong if + // there has been frame-size change since the last received packet and the + // missing packet. However, the risk of this is low, and in case of such + // errors, there will be a minor misestimation in time-to-play of missing + // packets. This will have a very minor effect on NACK performance. + uint32_t estimated_timestamp; + + // True if the packet is considered missing. Otherwise indicates packet is + // late. + bool is_missing; + }; + + class NackListCompare { + public: + bool operator() (uint16_t sequence_number_old, + uint16_t sequence_number_new) const { + return IsNewerSequenceNumber(sequence_number_new, sequence_number_old); + } + }; + + typedef std::map NackList; + + // Constructor. + explicit Nack(int nack_threshold_packets); + + // This API is used only for testing to assess whether time-to-play is + // computed correctly. + NackList GetNackList() const; + + // Given the |sequence_number_current_received_rtp| of currently received RTP, + // recognize packets which are not arrive and add to the list. + void AddToList(uint16_t sequence_number_current_received_rtp); + + // This function subtracts 10 ms of time-to-play for all packets in NACK list. + // This is called when 10 ms elapsed with no new RTP packet decoded. + void UpdateEstimatedPlayoutTimeBy10ms(); + + // Given the |sequence_number_current_received_rtp| and + // |timestamp_current_received_rtp| of currently received RTP update number + // of samples per packet. + void UpdateSamplesPerPacket(uint16_t sequence_number_current_received_rtp, + uint32_t timestamp_current_received_rtp); + + // Given the |sequence_number_current_received_rtp| of currently received RTP + // update the list. That is; some packets will change from late to missing, + // some packets are inserted as missing and some inserted as late. + void UpdateList(uint16_t sequence_number_current_received_rtp); + + // Packets which are considered late for too long (according to + // |nack_threshold_packets_|) are flagged as missing. + void ChangeFromLateToMissing(uint16_t sequence_number_current_received_rtp); + + // Packets which have sequence number older that + // |sequence_num_last_received_rtp_| - |max_nack_list_size_| are removed + // from the NACK list. + void LimitNackListSize(); + + // Estimate timestamp of a missing packet given its sequence number. + uint32_t EstimateTimestamp(uint16_t sequence_number); + + // Compute time-to-play given a timestamp. + int TimeToPlay(uint32_t timestamp) const; + + // If packet N is arrived, any packet prior to N - |nack_threshold_packets_| + // which is not arrived is considered missing, and should be in NACK list. + // Also any packet in the range of N-1 and N - |nack_threshold_packets_|, + // exclusive, which is not arrived is considered late, and should should be + // in the list of late packets. + const int nack_threshold_packets_; + + // Valid if a packet is received. + uint16_t sequence_num_last_received_rtp_; + uint32_t timestamp_last_received_rtp_; + bool any_rtp_received_; // If any packet received. + + // Valid if a packet is decoded. + uint16_t sequence_num_last_decoded_rtp_; + uint32_t timestamp_last_decoded_rtp_; + bool any_rtp_decoded_; // If any packet decoded. + + int sample_rate_khz_; // Sample rate in kHz. + + // Number of samples per packet. We update this every time we receive a + // packet, not only for consecutive packets. + int samples_per_packet_; + + // A list of missing packets to be retransmitted. Components of the list + // contain the sequence number of missing packets and the estimated time that + // each pack is going to be played out. + NackList nack_list_; + + // NACK list will not keep track of missing packets prior to + // |sequence_num_last_received_rtp_| - |max_nack_list_size_|. + size_t max_nack_list_size_; +}; + +} // webrtc + +#endif // WEBRTC_MODULES_AUDIO_CODING_MAIN_SOURCE_NACK_H_ diff --git a/webrtc/modules/audio_coding/main/source/nack_unittest.cc b/webrtc/modules/audio_coding/main/source/nack_unittest.cc new file mode 100644 index 000000000..ba92f0e76 --- /dev/null +++ b/webrtc/modules/audio_coding/main/source/nack_unittest.cc @@ -0,0 +1,482 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_coding/main/source/nack.h" + +#include + +#include + +#include "gtest/gtest.h" +#include "webrtc/typedefs.h" +#include "webrtc/modules/audio_coding/main/interface/audio_coding_module_typedefs.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" + +namespace webrtc { + +namespace { + +const int kNackThreshold = 3; +const int kSampleRateHz = 16000; +const int kPacketSizeMs = 30; +const uint32_t kTimestampIncrement = 480; // 30 ms. +const int kShortRoundTripTimeMs = 1; + +bool IsNackListCorrect(const std::vector& nack_list, + const uint16_t* lost_sequence_numbers, + size_t num_lost_packets) { + if (nack_list.size() != num_lost_packets) + return false; + + if (num_lost_packets == 0) + return true; + + for (size_t k = 0; k < nack_list.size(); ++k) { + int seq_num = nack_list[k]; + bool seq_num_matched = false; + for (size_t n = 0; n < num_lost_packets; ++n) { + if (seq_num == lost_sequence_numbers[n]) { + seq_num_matched = true; + break; + } + } + if (!seq_num_matched) + return false; + } + return true; +} + +} // namespace + +TEST(NackTest, EmptyListWhenNoPacketLoss) { + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + int seq_num = 1; + uint32_t timestamp = 0; + + std::vector nack_list; + for (int n = 0; n < 100; n++) { + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + seq_num++; + timestamp += kTimestampIncrement; + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } +} + +TEST(NackTest, NoNackIfReorderWithinNackThreshold) { + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + int seq_num = 1; + uint32_t timestamp = 0; + std::vector nack_list; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + int num_late_packets = kNackThreshold + 1; + + // Push in reverse order + while (num_late_packets > 0) { + nack->UpdateLastReceivedPacket(seq_num + num_late_packets, timestamp + + num_late_packets * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + num_late_packets--; + } +} + +TEST(NackTest, LatePacketsMovedToNackThenNackListDoesNotChange) { + const uint16_t kSequenceNumberLostPackets[] = { 2, 3, 4, 5, 6, 7, 8, 9 }; + static const int kNumAllLostPackets = sizeof(kSequenceNumberLostPackets) / + sizeof(kSequenceNumberLostPackets[0]); + + for (int k = 0; k < 2; k++) { // Two iteration with/without wrap around. + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + uint16_t sequence_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; n++) { + sequence_num_lost_packets[n] = kSequenceNumberLostPackets[n] + k * + 65531; // Have wrap around in sequence numbers for |k == 1|. + } + uint16_t seq_num = sequence_num_lost_packets[0] - 1; + + uint32_t timestamp = 0; + std::vector nack_list; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + seq_num = sequence_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp += kTimestampIncrement * (kNumAllLostPackets + 1); + int num_lost_packets = std::max(0, kNumAllLostPackets - kNackThreshold); + + for (int n = 0; n < kNackThreshold + 1; ++n) { + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect(nack_list, sequence_num_lost_packets, + num_lost_packets)); + seq_num++; + timestamp += kTimestampIncrement; + num_lost_packets++; + } + + for (int n = 0; n < 100; ++n) { + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect(nack_list, sequence_num_lost_packets, + kNumAllLostPackets)); + seq_num++; + timestamp += kTimestampIncrement; + } + } +} + +TEST(NackTest, ArrivedPacketsAreRemovedFromNackList) { + const uint16_t kSequenceNumberLostPackets[] = { 2, 3, 4, 5, 6, 7, 8, 9 }; + static const int kNumAllLostPackets = sizeof(kSequenceNumberLostPackets) / + sizeof(kSequenceNumberLostPackets[0]); + + for (int k = 0; k < 2; ++k) { // Two iteration with/without wrap around. + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + uint16_t sequence_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; ++n) { + sequence_num_lost_packets[n] = kSequenceNumberLostPackets[n] + k * + 65531; // Wrap around for |k == 1|. + } + + uint16_t seq_num = sequence_num_lost_packets[0] - 1; + uint32_t timestamp = 0; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + size_t index_retransmitted_rtp = 0; + uint32_t timestamp_retransmitted_rtp = timestamp + kTimestampIncrement; + + seq_num = sequence_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp += kTimestampIncrement * (kNumAllLostPackets + 1); + size_t num_lost_packets = std::max(0, kNumAllLostPackets - kNackThreshold); + for (int n = 0; n < kNumAllLostPackets; ++n) { + // Number of lost packets does not change for the first + // |kNackThreshold + 1| packets, one is added to the list and one is + // removed. Thereafter, the list shrinks every iteration. + if (n >= kNackThreshold + 1) + num_lost_packets--; + + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &sequence_num_lost_packets[index_retransmitted_rtp], + num_lost_packets)); + seq_num++; + timestamp += kTimestampIncrement; + + // Retransmission of a lost RTP. + nack->UpdateLastReceivedPacket( + sequence_num_lost_packets[index_retransmitted_rtp], + timestamp_retransmitted_rtp); + index_retransmitted_rtp++; + timestamp_retransmitted_rtp += kTimestampIncrement; + + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &sequence_num_lost_packets[index_retransmitted_rtp], + num_lost_packets - 1)); // One less lost packet in the list. + } + ASSERT_TRUE(nack_list.empty()); + } +} + +// Assess if estimation of timestamps and time-to-play is correct. Introduce all +// combinations that timestamps and sequence numbers might have wrap around. +TEST(NackTest, EstimateTimestampAndTimeToPlay) { + const uint16_t kLostPackets[] = { 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15 }; + static const int kNumAllLostPackets = sizeof(kLostPackets) / + sizeof(kLostPackets[0]); + + + for (int k = 0; k < 4; ++k) { + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + // Sequence number wrap around if |k| is 2 or 3; + int seq_num_offset = (k < 2) ? 0 : 65531; + + // Timestamp wrap around if |k| is 1 or 3. + uint32_t timestamp_offset = (k & 0x1) ? + static_cast(0xffffffff) - 6 : 0; + + uint32_t timestamp_lost_packets[kNumAllLostPackets]; + uint16_t seq_num_lost_packets[kNumAllLostPackets]; + for (int n = 0; n < kNumAllLostPackets; ++n) { + timestamp_lost_packets[n] = timestamp_offset + kLostPackets[n] * + kTimestampIncrement; + seq_num_lost_packets[n] = seq_num_offset + kLostPackets[n]; + } + + // We and to push two packets before lost burst starts. + uint16_t seq_num = seq_num_lost_packets[0] - 2; + uint32_t timestamp = timestamp_lost_packets[0] - 2 * kTimestampIncrement; + + const uint16_t first_seq_num = seq_num; + const uint32_t first_timestamp = timestamp; + + // Two consecutive packets to have a correct estimate of timestamp increase. + nack->UpdateLastReceivedPacket(seq_num, timestamp); + seq_num++; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // A packet after the last one which is supposed to be lost. + seq_num = seq_num_lost_packets[kNumAllLostPackets - 1] + 1; + timestamp = timestamp_lost_packets[kNumAllLostPackets - 1] + + kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + Nack::NackList nack_list = nack->GetNackList(); + EXPECT_EQ(static_cast(kNumAllLostPackets), nack_list.size()); + + // Pretend the first packet is decoded. + nack->UpdateLastDecodedPacket(first_seq_num, first_timestamp); + nack_list = nack->GetNackList(); + + Nack::NackList::iterator it = nack_list.begin(); + while (it != nack_list.end()) { + seq_num = it->first - seq_num_offset; + int index = seq_num - kLostPackets[0]; + EXPECT_EQ(timestamp_lost_packets[index], it->second.estimated_timestamp); + EXPECT_EQ((index + 2) * kPacketSizeMs, it->second.time_to_play_ms); + ++it; + } + + // Pretend 10 ms is passed, and we had pulled audio from NetEq, it still + // reports the same sequence number as decoded, time-to-play should be + // updated by 10 ms. + nack->UpdateLastDecodedPacket(first_seq_num, first_timestamp); + nack_list = nack->GetNackList(); + it = nack_list.begin(); + while (it != nack_list.end()) { + seq_num = it->first - seq_num_offset; + int index = seq_num - kLostPackets[0]; + EXPECT_EQ((index + 2) * kPacketSizeMs - 10, it->second.time_to_play_ms); + ++it; + } + } +} + +TEST(NackTest, MissingPacketsPriorToLastDecodedRtpShouldNotBeInNackList) { + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65531; // Wrap around if |m| is 1. + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + // Two consecutive packets to have a correct estimate of timestamp increase. + uint16_t seq_num = 0; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + seq_num++; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + + // Skip 10 packets (larger than NACK threshold). + const int kNumLostPackets = 10; + seq_num += kNumLostPackets + 1; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + + const size_t kExpectedListSize = kNumLostPackets - kNackThreshold; + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + + for (int k = 0; k < 2; ++k) { + // Decoding of the first and the second arrived packets. + for (int n = 0; n < kPacketSizeMs / 10; ++n) { + nack->UpdateLastDecodedPacket(seq_num_offset + k, + k * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + } + } + + // Decoding of the last received packet. + nack->UpdateLastDecodedPacket(seq_num + seq_num_offset, + seq_num * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + + // Make sure list of late packets is also empty. To check that, push few + // packets, if the late list is not empty its content will pop up in NACK + // list. + for (int n = 0; n < kNackThreshold + 10; ++n) { + seq_num++; + nack->UpdateLastReceivedPacket(seq_num_offset + seq_num, + seq_num * kTimestampIncrement); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } + } +} + +TEST(NackTest, Reset) { + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + // Two consecutive packets to have a correct estimate of timestamp increase. + uint16_t seq_num = 0; + nack->UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + seq_num++; + nack->UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + + // Skip 10 packets (larger than NACK threshold). + const int kNumLostPackets = 10; + seq_num += kNumLostPackets + 1; + nack->UpdateLastReceivedPacket(seq_num, seq_num * kTimestampIncrement); + + const size_t kExpectedListSize = kNumLostPackets - kNackThreshold; + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kExpectedListSize, nack_list.size()); + + nack->Reset(); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); +} + +TEST(NackTest, ListSizeAppliedFromBeginning) { + const size_t kNackListSize = 10; + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65525; // Wrap around if |m| is 1. + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + nack->SetMaxNackListSize(kNackListSize); + + uint16_t seq_num = seq_num_offset; + uint32_t timestamp = 0x12345678; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t num_lost_packets = kNackThreshold + kNackListSize + 5; + + seq_num += num_lost_packets + 1; + timestamp += (num_lost_packets + 1) * kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(kNackListSize - kNackThreshold, nack_list.size()); + } +} + +TEST(NackTest, ChangeOfListSizeAppliedAndOldElementsRemoved) { + const size_t kNackListSize = 10; + for (int m = 0; m < 2; ++m) { + uint16_t seq_num_offset = (m == 0) ? 0 : 65525; // Wrap around if |m| is 1. + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + + uint16_t seq_num = seq_num_offset; + uint32_t timestamp = 0x87654321; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t num_lost_packets = kNackThreshold + kNackListSize + 5; + + scoped_array seq_num_lost(new uint16_t[num_lost_packets]); + for (int n = 0; n < num_lost_packets; ++n) { + seq_num_lost[n] = ++seq_num; + } + + ++seq_num; + timestamp += (num_lost_packets + 1) * kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + size_t expected_size = num_lost_packets - kNackThreshold; + + std::vector nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_EQ(expected_size, nack_list.size()); + + nack->SetMaxNackListSize(kNackListSize); + expected_size = kNackListSize - kNackThreshold; + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize], + expected_size)); + + // NACK list does not change size but the content is changing. The oldest + // element is removed and one from late list is inserted. + size_t n; + for (n = 1; n <= static_cast(kNackThreshold); ++n) { + ++seq_num; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize + n], + expected_size)); + } + + // NACK list should shrink. + for (; n < kNackListSize; ++n) { + ++seq_num; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + --expected_size; + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(IsNackListCorrect( + nack_list, &seq_num_lost[num_lost_packets - kNackListSize + n], + expected_size)); + } + + // After this packet, NACK list should be empty. + ++seq_num; + timestamp += kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + nack_list = nack->GetNackList(kShortRoundTripTimeMs); + EXPECT_TRUE(nack_list.empty()); + } +} + +TEST(NackTest, RoudTripTimeIsApplied) { + const int kNackListSize = 200; + scoped_ptr nack(Nack::Create(kNackThreshold)); + nack->UpdateSampleRate(kSampleRateHz); + nack->SetMaxNackListSize(kNackListSize); + + uint16_t seq_num = 0; + uint32_t timestamp = 0x87654321; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Packet lost more than NACK-list size limit. + uint16_t kNumLostPackets = kNackThreshold + 5; + + seq_num += (1 + kNumLostPackets); + timestamp += (1 + kNumLostPackets) * kTimestampIncrement; + nack->UpdateLastReceivedPacket(seq_num, timestamp); + + // Expected time-to-play are: + // kPacketSizeMs - 10, 2*kPacketSizeMs - 10, 3*kPacketSizeMs - 10, ... + // + // sequence number: 1, 2, 3, 4, 5 + // time-to-play: 20, 50, 80, 110, 140 + // + std::vector nack_list = nack->GetNackList(100); + ASSERT_EQ(2u, nack_list.size()); + EXPECT_EQ(4, nack_list[0]); + EXPECT_EQ(5, nack_list[1]); +} + +} // webrtc diff --git a/webrtc/modules/audio_coding/main/test/APITest.h b/webrtc/modules/audio_coding/main/test/APITest.h index 056eca64f..f29abf458 100644 --- a/webrtc/modules/audio_coding/main/test/APITest.h +++ b/webrtc/modules/audio_coding/main/test/APITest.h @@ -11,11 +11,12 @@ #ifndef API_TEST_H #define API_TEST_H -#include "ACMTest.h" -#include "Channel.h" -#include "PCMFile.h" -#include "event_wrapper.h" -#include "utility.h" +#include "webrtc/modules/audio_coding/main/test/ACMTest.h" +#include "webrtc/modules/audio_coding/main/test/Channel.h" +#include "webrtc/modules/audio_coding/main/test/PCMFile.h" +#include "webrtc/modules/audio_coding/main/test/utility.h" +#include "webrtc/system_wrappers/interface/event_wrapper.h" +#include "webrtc/system_wrappers/interface/rw_lock_wrapper.h" namespace webrtc { diff --git a/webrtc/modules/audio_coding/main/test/Channel.cc b/webrtc/modules/audio_coding/main/test/Channel.cc index 2e097f74d..8296d2a7f 100644 --- a/webrtc/modules/audio_coding/main/test/Channel.cc +++ b/webrtc/modules/audio_coding/main/test/Channel.cc @@ -8,14 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "webrtc/modules/audio_coding/main/test/Channel.h" + #include #include -#include "audio_coding_module.h" -#include "Channel.h" -#include "tick_util.h" -#include "typedefs.h" -#include "common_types.h" +#include "webrtc/system_wrappers/interface/tick_util.h" +#include "webrtc/system_wrappers/interface/critical_section_wrapper.h" namespace webrtc { @@ -29,9 +28,12 @@ int32_t Channel::SendData(const FrameType frameType, const uint8_t payloadType, rtpInfo.header.markerBit = false; rtpInfo.header.ssrc = 0; - rtpInfo.header.sequenceNumber = _seqNo++; + rtpInfo.header.sequenceNumber = (external_sequence_number_ < 0) ? + _seqNo++ : static_cast(external_sequence_number_); rtpInfo.header.payloadType = payloadType; - rtpInfo.header.timestamp = timeStamp; + rtpInfo.header.timestamp = (external_send_timestamp_ < 0) ? timeStamp : + static_cast(external_send_timestamp_); + if (frameType == kAudioFrameCN) { rtpInfo.type.Audio.isCNG = true; } else { @@ -108,6 +110,11 @@ int32_t Channel::SendData(const FrameType frameType, const uint8_t payloadType, } } + if (num_packets_to_drop_ > 0) { + num_packets_to_drop_--; + return 0; + } + status = _receiverACM->IncomingPacket(_payloadData, payloadDataSize, rtpInfo); return status; @@ -208,7 +215,10 @@ Channel::Channel(int16_t chID) _packetLoss(0), _useFECTestWithPacketLoss(false), _beginTime(TickTime::MillisecondTimestamp()), - _totalBytes(0) { + _totalBytes(0), + external_send_timestamp_(-1), + external_sequence_number_(-1), + num_packets_to_drop_(0) { int n; int k; for (n = 0; n < MAX_NUM_PAYLOADS; n++) { diff --git a/webrtc/modules/audio_coding/main/test/Channel.h b/webrtc/modules/audio_coding/main/test/Channel.h index b4b51cf61..e53988bf8 100644 --- a/webrtc/modules/audio_coding/main/test/Channel.h +++ b/webrtc/modules/audio_coding/main/test/Channel.h @@ -13,13 +13,14 @@ #include -#include "audio_coding_module.h" -#include "critical_section_wrapper.h" -#include "rw_lock_wrapper.h" +#include "webrtc/modules/audio_coding/main/interface/audio_coding_module.h" #include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/typedefs.h" namespace webrtc { +class CriticalSectionWrapper; + #define MAX_NUM_PAYLOADS 50 #define MAX_NUM_FRAMESIZES 6 @@ -76,6 +77,18 @@ class Channel : public AudioPacketizationCallback { double BitRate(); + void set_send_timestamp(uint32_t new_send_ts) { + external_send_timestamp_ = new_send_ts; + } + + void set_sequence_number(uint16_t new_sequence_number) { + external_sequence_number_ = new_sequence_number; + } + + void set_num_packets_to_drop(int new_num_packets_to_drop) { + num_packets_to_drop_ = new_num_packets_to_drop; + } + private: void CalcStatistics(WebRtcRTPHeader& rtpInfo, uint16_t payloadSize); @@ -98,6 +111,12 @@ class Channel : public AudioPacketizationCallback { bool _useFECTestWithPacketLoss; uint64_t _beginTime; uint64_t _totalBytes; + + // External timing info, defaulted to -1. Only used if they are + // non-negative. + int64_t external_send_timestamp_; + int32_t external_sequence_number_; + int num_packets_to_drop_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/test/insert_packet_with_timing.cc b/webrtc/modules/audio_coding/main/test/insert_packet_with_timing.cc new file mode 100644 index 000000000..543d90186 --- /dev/null +++ b/webrtc/modules/audio_coding/main/test/insert_packet_with_timing.cc @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "gflags/gflags.h" +#include "gtest/gtest.h" +#include "webrtc/common_types.h" +#include "webrtc/modules/audio_coding/main/interface/audio_coding_module.h" +#include "webrtc/modules/audio_coding/main/test/Channel.h" +#include "webrtc/modules/audio_coding/main/test/PCMFile.h" +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/system_wrappers/interface/clock.h" +#include "webrtc/test/testsupport/fileutils.h" + +// Codec. +DEFINE_string(codec, "opus", "Codec Name"); +DEFINE_int32(codec_sample_rate_hz, 48000, "Sampling rate in Hertz."); +DEFINE_int32(codec_channels, 1, "Number of channels of the codec."); + +// PCM input/output. +DEFINE_string(input, "", "Input PCM file at 16 kHz."); +DEFINE_bool(input_stereo, false, "Input is stereo."); +DEFINE_int32(input_fs_hz, 32000, "Input sample rate Hz."); +DEFINE_string(output, "insert_rtp_with_timing_out.pcm", "OutputFile"); +DEFINE_int32(output_fs_hz, 32000, "Output sample rate Hz"); + +// Timing files +DEFINE_string(seq_num, "seq_num", "Sequence number file."); +DEFINE_string(send_ts, "send_timestamp", "Send timestamp file."); +DEFINE_string(receive_ts, "last_rec_timestamp", "Receive timestamp file"); + +// Delay logging +DEFINE_string(delay, "", "Log for delay."); + +// Other setups +DEFINE_int32(init_delay, 0, "Initial delay."); +DEFINE_bool(verbose, false, "Verbosity."); +DEFINE_double(loss_rate, 0, "Rate of packet loss < 1"); + +const int32_t kAudioPlayedOut = 0x00000001; +const int32_t kPacketPushedIn = 0x00000001 << 1; +const int kPlayoutPeriodMs = 10; + +namespace webrtc { + +class InsertPacketWithTiming { + public: + InsertPacketWithTiming() + : sender_clock_(new SimulatedClock(0)), + receiver_clock_(new SimulatedClock(0)), + send_acm_(AudioCodingModule::Create(0, sender_clock_)), + receive_acm_(AudioCodingModule::Create(0, receiver_clock_)), + channel_(new Channel), + seq_num_fid_(fopen(FLAGS_seq_num.c_str(), "rt")), + send_ts_fid_(fopen(FLAGS_send_ts.c_str(), "rt")), + receive_ts_fid_(fopen(FLAGS_receive_ts.c_str(), "rt")), + pcm_out_fid_(fopen(FLAGS_output.c_str(), "wb")), + samples_in_1ms_(48), + num_10ms_in_codec_frame_(2), // Typical 20 ms frames. + time_to_insert_packet_ms_(3), // An arbitrary offset on pushing packet. + next_receive_ts_(0), + time_to_playout_audio_ms_(kPlayoutPeriodMs), + loss_threshold_(0), + playout_timing_fid_(fopen("playout_timing.txt", "wt")) {} + + void SetUp() { + ASSERT_TRUE(sender_clock_ != NULL); + ASSERT_TRUE(receiver_clock_ != NULL); + + ASSERT_TRUE(send_acm_ != NULL); + ASSERT_TRUE(receive_acm_ != NULL); + ASSERT_TRUE(channel_ != NULL); + + ASSERT_TRUE(seq_num_fid_ != NULL); + ASSERT_TRUE(send_ts_fid_ != NULL); + ASSERT_TRUE(receive_ts_fid_ != NULL); + + ASSERT_TRUE(playout_timing_fid_ != NULL); + + next_receive_ts_ = ReceiveTimestamp(); + + CodecInst codec; + ASSERT_EQ(0, AudioCodingModule::Codec(FLAGS_codec.c_str(), &codec, + FLAGS_codec_sample_rate_hz, + FLAGS_codec_channels)); + ASSERT_EQ(0, receive_acm_->InitializeReceiver()); + ASSERT_EQ(0, send_acm_->RegisterSendCodec(codec)); + ASSERT_EQ(0, receive_acm_->RegisterReceiveCodec(codec)); + + // Set codec-dependent parameters. + samples_in_1ms_ = codec.plfreq / 1000; + num_10ms_in_codec_frame_ = codec.pacsize / (codec.plfreq / 100); + + channel_->RegisterReceiverACM(receive_acm_); + send_acm_->RegisterTransportCallback(channel_); + + if (FLAGS_input.size() == 0) { + std::string file_name = test::ResourcePath("audio_coding/testfile32kHz", + "pcm"); + pcm_in_fid_.Open(file_name, 32000, "r", true); // auto-rewind + std::cout << "Input file " << file_name << " 32 kHz mono." << std::endl; + } else { + pcm_in_fid_.Open(FLAGS_input, static_cast(FLAGS_input_fs_hz), + "r", true); // auto-rewind + std::cout << "Input file " << FLAGS_input << "at " << FLAGS_input_fs_hz + << " Hz in " << ((FLAGS_input_stereo) ? "stereo." : "mono.") + << std::endl; + pcm_in_fid_.ReadStereo(FLAGS_input_stereo); + } + + ASSERT_TRUE(pcm_out_fid_ != NULL); + std::cout << "Output file " << FLAGS_output << " at " << FLAGS_output_fs_hz + << " Hz." << std::endl; + + // Other setups + if (FLAGS_init_delay > 0) + EXPECT_EQ(0, receive_acm_->SetInitialPlayoutDelay(FLAGS_init_delay)); + + if (FLAGS_loss_rate > 0) + loss_threshold_ = RAND_MAX * FLAGS_loss_rate; + else + loss_threshold_ = 0; + } + + void TickOneMillisecond(uint32_t* action) { + // One millisecond passed. + time_to_insert_packet_ms_--; + time_to_playout_audio_ms_--; + sender_clock_->AdvanceTimeMilliseconds(1); + receiver_clock_->AdvanceTimeMilliseconds(1); + + // Reset action. + *action = 0; + + // Is it time to pull audio? + if (time_to_playout_audio_ms_ == 0) { + time_to_playout_audio_ms_ = kPlayoutPeriodMs; + receive_acm_->PlayoutData10Ms(static_cast(FLAGS_output_fs_hz), + &frame_); + fwrite(frame_.data_, sizeof(frame_.data_[0]), + frame_.samples_per_channel_ * frame_.num_channels_, pcm_out_fid_); + *action |= kAudioPlayedOut; + } + + // Is it time to push in next packet? + if (time_to_insert_packet_ms_ <= .5) { + *action |= kPacketPushedIn; + + // Update time-to-insert packet. + uint32_t t = next_receive_ts_; + next_receive_ts_ = ReceiveTimestamp(); + time_to_insert_packet_ms_ += static_cast(next_receive_ts_ - t) / + samples_in_1ms_; + + // Push in just enough audio. + for (int n = 0; n < num_10ms_in_codec_frame_; n++) { + pcm_in_fid_.Read10MsData(frame_); + EXPECT_EQ(0, send_acm_->Add10MsData(frame_)); + } + + // Set the parameters for the packet to be pushed in receiver ACM right + // now. + uint32_t ts = SendTimestamp(); + int seq_num = SequenceNumber(); + bool lost = false; + channel_->set_send_timestamp(ts); + channel_->set_sequence_number(seq_num); + if (loss_threshold_ > 0 && rand() < loss_threshold_) { + channel_->set_num_packets_to_drop(1); + lost = true; + } + + // Process audio in send ACM, this should result in generation of a + // packet. + EXPECT_GT(send_acm_->Process(), 0); + + if (FLAGS_verbose) { + if (!lost) { + std::cout << "\nInserting packet number " << seq_num + << " timestamp " << ts << std::endl; + } else { + std::cout << "\nLost packet number " << seq_num + << " timestamp " << ts << std::endl; + } + } + } + } + + void TearDown() { + AudioCodingModule::Destroy(send_acm_); + AudioCodingModule::Destroy(receive_acm_); + delete channel_; + + fclose(seq_num_fid_); + fclose(send_ts_fid_); + fclose(receive_ts_fid_); + fclose(pcm_out_fid_); + pcm_in_fid_.Close(); + } + + ~InsertPacketWithTiming() { + delete sender_clock_; + delete receiver_clock_; + } + + // Are there more info to simulate. + bool HasPackets() { + if (feof(seq_num_fid_) || feof(send_ts_fid_) || feof(receive_ts_fid_)) + return false; + return true; + } + + // Jitter buffer delay. + void Delay(int* optimal_delay, int* current_delay) { + ACMNetworkStatistics statistics; + receive_acm_->NetworkStatistics(&statistics); + *optimal_delay = statistics.preferredBufferSize; + *current_delay = statistics.currentBufferSize; + } + + private: + uint32_t SendTimestamp() { + uint32_t t; + EXPECT_EQ(1, fscanf(send_ts_fid_, "%u\n", &t)); + return t; + } + + uint32_t ReceiveTimestamp() { + uint32_t t; + EXPECT_EQ(1, fscanf(receive_ts_fid_, "%u\n", &t)); + return t; + } + + int SequenceNumber() { + int n; + EXPECT_EQ(1, fscanf(seq_num_fid_, "%d\n", &n)); + return n; + } + + // This class just creates these pointers, not deleting them. They are deleted + // by the associated ACM. + SimulatedClock* sender_clock_; + SimulatedClock* receiver_clock_; + + AudioCodingModule* send_acm_; + AudioCodingModule* receive_acm_; + Channel* channel_; + + FILE* seq_num_fid_; // Input (text), one sequence number per line. + FILE* send_ts_fid_; // Input (text), one send timestamp per line. + FILE* receive_ts_fid_; // Input (text), one receive timestamp per line. + FILE* pcm_out_fid_; // Output PCM16. + + PCMFile pcm_in_fid_; // Input PCM16. + + int samples_in_1ms_; + + // TODO(turajs): this can be computed from the send timestamp, but there is + // some complication to account for lost and reordered packets. + int num_10ms_in_codec_frame_; + + float time_to_insert_packet_ms_; + uint32_t next_receive_ts_; + uint32_t time_to_playout_audio_ms_; + + AudioFrame frame_; + + double loss_threshold_; + + // Output (text), sequence number, playout timestamp, time (ms) of playout, + // per line. + FILE* playout_timing_fid_; +}; + +} // webrtc + +int main(int argc, char* argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + webrtc::InsertPacketWithTiming test; + test.SetUp(); + + FILE* delay_log = NULL; + if (FLAGS_delay.size() > 0) { + delay_log = fopen(FLAGS_delay.c_str(), "wt"); + if (delay_log == NULL) { + std::cout << "Cannot open the file to log delay values." << std::endl; + exit(1); + } + } + + uint32_t action_taken; + int optimal_delay_ms; + int current_delay_ms; + while (test.HasPackets()) { + test.TickOneMillisecond(&action_taken); + + if (action_taken != 0) { + test.Delay(&optimal_delay_ms, ¤t_delay_ms); + if (delay_log != NULL) { + fprintf(delay_log, "%3d %3d\n", optimal_delay_ms, current_delay_ms); + } + } + } + std::cout << std::endl; + test.TearDown(); + if (delay_log != NULL) + fclose(delay_log); +} diff --git a/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq.h b/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq.h index d6c68fd08..c2a01340b 100644 --- a/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq.h +++ b/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq.h @@ -209,6 +209,9 @@ int WebRtcNetEQ_RecOut(void *inst, int16_t *pw16_outData, int16_t *pw16_len); int WebRtcNetEQ_GetRTCPStats(void *inst, WebRtcNetEQ_RTCPStat *RTCP_inst); int WebRtcNetEQ_GetRTCPStatsNoReset(void *inst, WebRtcNetEQ_RTCPStat *RTCP_inst); int WebRtcNetEQ_GetSpeechTimeStamp(void *inst, uint32_t *timestamp); +int WebRtcNetEQ_DecodedRtpInfo(const void* inst, + int* sequence_number, + uint32_t* timestamp); int WebRtcNetEQ_GetSpeechOutputType(void *inst, enum WebRtcNetEQOutputType *outputType); /* VQmon related functions */ diff --git a/webrtc/modules/audio_coding/neteq/mcu.h b/webrtc/modules/audio_coding/neteq/mcu.h index 1be81c201..931e6dcf5 100644 --- a/webrtc/modules/audio_coding/neteq/mcu.h +++ b/webrtc/modules/audio_coding/neteq/mcu.h @@ -99,6 +99,9 @@ typedef struct int usingStereo; #endif + /* The sequence number of the latest decoded RTP payload. */ + int decoded_packet_sequence_number; + uint32_t decoded_packet_timestamp; } MCUInst_t; /**************************************************************************** diff --git a/webrtc/modules/audio_coding/neteq/signal_mcu.c b/webrtc/modules/audio_coding/neteq/signal_mcu.c index b3791d1af..b795ec30e 100644 --- a/webrtc/modules/audio_coding/neteq/signal_mcu.c +++ b/webrtc/modules/audio_coding/neteq/signal_mcu.c @@ -663,6 +663,17 @@ int WebRtcNetEQ_SignalMcu(MCUInst_t *inst) prevTS = inst->PacketBuffer_inst.timeStamp[i_bufferpos]; oldPT = inst->PacketBuffer_inst.payloadType[i_bufferpos]; + /* These values are used by NACK module to estimate time-to-play of + * a missing packet. Occasionally, NetEq might decide to decode more + * than one packet. Therefore, these values store sequence number and + * timestamp of the first packet pulled from the packet buffer. In + * such cases, these values do not exactly represent the sequence number + * or timestamp associated with a 10ms audio pulled from NetEq. NACK + * module is designed to compensate for this. + */ + inst->decoded_packet_sequence_number = prevSeqNo; + inst->decoded_packet_timestamp = prevTS; + /* clear flag bits */ inst->pw16_writeAddress[0] = inst->pw16_writeAddress[0] & 0xFF3F; do diff --git a/webrtc/modules/audio_coding/neteq/webrtc_neteq.c b/webrtc/modules/audio_coding/neteq/webrtc_neteq.c index 83479252f..8f1e34cff 100644 --- a/webrtc/modules/audio_coding/neteq/webrtc_neteq.c +++ b/webrtc/modules/audio_coding/neteq/webrtc_neteq.c @@ -468,6 +468,10 @@ int WebRtcNetEQ_Init(void *inst, uint16_t fs) NetEqMainInst->masterSlave = 0; #endif + /* Set to an invalid value. */ + NetEqMainInst->MCUinst.decoded_packet_sequence_number = -1; + NetEqMainInst->MCUinst.decoded_packet_timestamp = 0; + return (ok); } @@ -1729,3 +1733,15 @@ int WebRtcNetEQ_GetRequiredDelayMs(const void* inst) { ((auto_mode->packetSpeechLenSamp * 1000) / NetEqMainInst->MCUinst.fs) + 128) >> 8; } + +int WebRtcNetEQ_DecodedRtpInfo(const void* inst, + int* sequence_number, + uint32_t* timestamp) { + const MainInst_t *NetEqMainInst = (inst == NULL) ? NULL : + (const MainInst_t*) inst; + if (NetEqMainInst->MCUinst.decoded_packet_sequence_number < 0) + return -1; + *sequence_number = NetEqMainInst->MCUinst.decoded_packet_sequence_number; + *timestamp = NetEqMainInst->MCUinst.decoded_packet_timestamp; + return 0; +}