diff --git a/webrtc/modules/audio_coding/main/source/acm_neteq.cc b/webrtc/modules/audio_coding/main/source/acm_neteq.cc index d9451783d..f6b64d710 100644 --- a/webrtc/modules/audio_coding/main/source/acm_neteq.cc +++ b/webrtc/modules/audio_coding/main/source/acm_neteq.cc @@ -21,7 +21,6 @@ #include "webrtc/modules/audio_coding/neteq/interface/webrtc_neteq_internal.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h" -#include "webrtc/system_wrappers/interface/tick_util.h" #include "webrtc/system_wrappers/interface/trace.h" #include "webrtc/system_wrappers/interface/trace_event.h" @@ -49,7 +48,8 @@ ACMNetEQ::ACMNetEQ() callback_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), min_of_max_num_packets_(0), min_of_buffer_size_bytes_(0), - per_packet_overhead_bytes_(0) { + per_packet_overhead_bytes_(0), + av_sync_(false) { for (int n = 0; n < MAX_NUM_SLAVE_NETEQ + 1; n++) { is_initialized_[n] = false; ptr_vadinst_[n] = NULL; @@ -436,12 +436,59 @@ int32_t ACMNetEQ::NetworkStatistics( return 0; } -int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload, - const int32_t length_payload, - const WebRtcRTPHeader& rtp_info) { - int16_t payload_length = static_cast(length_payload); +// Should only be called in AV-sync mode. +int ACMNetEQ::RecIn(const WebRtcRTPHeader& rtp_info, + uint32_t receive_timestamp) { + assert(av_sync_); - // translate to NetEq struct + // Translate to NetEq structure. + WebRtcNetEQ_RTPInfo neteq_rtpinfo; + neteq_rtpinfo.payloadType = rtp_info.header.payloadType; + neteq_rtpinfo.sequenceNumber = rtp_info.header.sequenceNumber; + neteq_rtpinfo.timeStamp = rtp_info.header.timestamp; + neteq_rtpinfo.SSRC = rtp_info.header.ssrc; + neteq_rtpinfo.markerBit = rtp_info.header.markerBit; + + CriticalSectionScoped lock(neteq_crit_sect_); + + // Master should be initialized. + assert(is_initialized_[0]); + + // Push into Master. + int status = WebRtcNetEQ_RecInSyncRTP(inst_[0], &neteq_rtpinfo, + receive_timestamp); + if (status < 0) { + LogError("RecInSyncRTP", 0); + WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, + "RecIn (sync): NetEq, error in pushing in Master"); + return -1; + } + + // If the received stream is stereo, insert a sync payload into slave. + if (rtp_info.type.Audio.channel == 2) { + // Slave should be initialized. + assert(is_initialized_[1]); + + // PUSH into Slave + status = WebRtcNetEQ_RecInSyncRTP(inst_[1], &neteq_rtpinfo, + receive_timestamp); + if (status < 0) { + LogError("RecInRTPStruct", 1); + WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, + "RecIn (sync): NetEq, error in pushing in Slave"); + return -1; + } + } + return status; +} + +int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload, + const int32_t length_payload, + const WebRtcRTPHeader& rtp_info, + uint32_t receive_timestamp) { + int16_t payload_length = static_cast(length_payload); + + // Translate to NetEq structure. WebRtcNetEQ_RTPInfo neteq_rtpinfo; neteq_rtpinfo.payloadType = rtp_info.header.payloadType; neteq_rtpinfo.sequenceNumber = rtp_info.header.sequenceNumber; @@ -450,15 +497,6 @@ int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload, neteq_rtpinfo.markerBit = rtp_info.header.markerBit; CriticalSectionScoped lock(neteq_crit_sect_); - // Down-cast the time to (32-6)-bit since we only care about - // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. - // we masked 6 most significant bits of 32-bit so we don't loose resolution - // when do the following multiplication. - const uint32_t now_in_ms = - static_cast( - TickTime::MillisecondTimestamp() & 0x03ffffff); - uint32_t recv_timestamp = static_cast( - current_samp_freq_khz_ * now_in_ms); int status; // In case of stereo payload, first half of the data should be pushed into @@ -473,10 +511,10 @@ int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload, "RecIn: NetEq is not initialized."); return -1; } - // PUSH into Master + // Push into Master. status = WebRtcNetEQ_RecInRTPStruct(inst_[0], &neteq_rtpinfo, incoming_payload, payload_length, - recv_timestamp); + receive_timestamp); if (status < 0) { LogError("RecInRTPStruct", 0); WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, @@ -491,10 +529,10 @@ int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload, "RecIn: NetEq is not initialized."); return -1; } - // PUSH into Slave + // Push into Slave. status = WebRtcNetEQ_RecInRTPStruct(inst_[1], &neteq_rtpinfo, &incoming_payload[payload_length], - payload_length, recv_timestamp); + payload_length, receive_timestamp); if (status < 0) { LogError("RecInRTPStruct", 1); WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, @@ -529,7 +567,6 @@ int32_t ACMNetEQ::RecOut(AudioFrame& audio_frame) { LogError("RecOut", 0); WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, "RecOut: NetEq, error in pulling out for mono case"); - // Check for errors that can be recovered from: // RECOUT_ERROR_SAMPLEUNDERRUN = 2003 int error_code = WebRtcNetEQ_GetErrorCode(inst_[0]); @@ -1056,6 +1093,8 @@ int16_t ACMNetEQ::AddSlave(const WebRtcNetEQDecoder* used_codecs, "AddSlave: AddSlave Failed, Could not Set Playout Mode."); return -1; } + // Set AV-sync for the slave. + WebRtcNetEQ_EnableAVSync(inst_[slave_idx], av_sync_ ? 1 : 0); } return 0; @@ -1071,4 +1110,13 @@ uint8_t ACMNetEQ::num_slaves() { return num_slaves_; } +void ACMNetEQ::EnableAVSync(bool enable) { + CriticalSectionScoped lock(neteq_crit_sect_); + av_sync_ = enable; + for (int i = 0; i < num_slaves_ + 1; ++i) { + assert(is_initialized_[i]); + WebRtcNetEQ_EnableAVSync(inst_[i], enable ? 1 : 0); + } +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/source/acm_neteq.h b/webrtc/modules/audio_coding/main/source/acm_neteq.h index ac91f9fb4..ed815443b 100644 --- a/webrtc/modules/audio_coding/main/source/acm_neteq.h +++ b/webrtc/modules/audio_coding/main/source/acm_neteq.h @@ -60,13 +60,31 @@ class ACMNetEQ { // - rtp_info : RTP header for the incoming payload containing // information about payload type, sequence number, // timestamp, SSRC and marker bit. + // - receive_timestamp : received timestamp. // // Return value : 0 if ok. // <0 if NetEQ returned an error. // int32_t RecIn(const uint8_t* incoming_payload, const int32_t length_payload, - const WebRtcRTPHeader& rtp_info); + const WebRtcRTPHeader& rtp_info, + uint32_t receive_timestamp); + + // + // RecIn() + // Insert a sync payload to NetEq. Should only be called if |av_sync_| is + // enabled; + // + // Input: + // - rtp_info : RTP header for the incoming payload containing + // information about payload type, sequence number, + // timestamp, SSRC and marker bit. + // - receive_timestamp : received timestamp. + // + // Return value : 0 if ok. + // <0 if NetEQ returned an error. + // + int RecIn(const WebRtcRTPHeader& rtp_info, uint32_t receive_timestamp); // // RecOut() @@ -278,6 +296,11 @@ class ACMNetEQ { overhead_bytes = per_packet_overhead_bytes_; } + // + // Set AV-sync mode. + // + void EnableAVSync(bool enable); + private: // // RTPPack() @@ -350,6 +373,9 @@ class ACMNetEQ { // Minimum of buffer-size among all NetEq instances. int min_of_buffer_size_bytes_; int per_packet_overhead_bytes_; + + // Keep track of AV-sync. Just used to set the slave when a slave is added. + bool av_sync_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/source/acm_neteq_unittest.cc b/webrtc/modules/audio_coding/main/source/acm_neteq_unittest.cc index 9cf280bb7..aef0acdde 100644 --- a/webrtc/modules/audio_coding/main/source/acm_neteq_unittest.cc +++ b/webrtc/modules/audio_coding/main/source/acm_neteq_unittest.cc @@ -68,8 +68,9 @@ void AcmNetEqTest::InsertZeroPacket(uint16_t sequence_number, rtp_header.header.payloadType = payload_type; rtp_header.header.markerBit = marker_bit; rtp_header.type.Audio.channel = 1; + // Receive timestamp can be set to send timestamp in this test. ASSERT_EQ(0, neteq_.RecIn(reinterpret_cast(payload), - len_payload_bytes, rtp_header)); + len_payload_bytes, rtp_header, timestamp)); } void AcmNetEqTest::PullData(int expected_num_samples) { diff --git a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc index 54cc6273e..6085181a7 100644 --- a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc +++ b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc @@ -21,6 +21,7 @@ #include "webrtc/modules/audio_coding/main/source/acm_resampler.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h" +#include "webrtc/system_wrappers/interface/tick_util.h" #include "webrtc/system_wrappers/interface/trace.h" #include "webrtc/system_wrappers/interface/trace_event.h" @@ -43,6 +44,9 @@ enum { kMaxNumFragmentationVectors = 3 }; +static const uint32_t kMaskTimestamp = 0x03ffffff; +static const int kDefaultTimestampDiff = 960; // 20 ms @ 48 kHz. + namespace { bool IsCodecRED(const CodecInst* codec) { @@ -85,7 +89,7 @@ int UpMix(const AudioFrame& frame, int length_out_buff, int16_t* out_buff) { // Return 1 if timestamp t1 is less than timestamp t2, while compensating for // wrap-around. -static int TimestampLessThan(uint32_t t1, uint32_t t2) { +int TimestampLessThan(uint32_t t1, uint32_t t2) { uint32_t kHalfFullRange = static_cast(0xFFFFFFFF) / 2; if (t1 == t2) { return 0; @@ -100,6 +104,21 @@ static int TimestampLessThan(uint32_t t1, uint32_t t2) { } } +// +// Return the timestamp of current time, computed according to sampling rate +// of the codec identified by |codec_id|. +// +uint32_t NowTimestamp(int codec_id) { + // Down-cast the time to (32-6)-bit since we only care about + // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms. + // we masked 6 most significant bits of 32-bit so we don't loose resolution + // when do the following multiplication. + int sample_rate_khz = ACMCodecDB::database_[codec_id].plfreq / 1000; + const uint32_t now_in_ms = static_cast( + TickTime::MillisecondTimestamp() & kMaskTimestamp); + return static_cast(sample_rate_khz * now_in_ms); +} + } // namespace AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id) @@ -147,7 +166,12 @@ AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id) first_payload_received_(false), last_incoming_send_timestamp_(0), track_neteq_buffer_(false), - playout_ts_(0) { + playout_ts_(0), + av_sync_(false), + last_timestamp_diff_(kDefaultTimestampDiff), + last_sequence_number_(0), + last_ssrc_(0), + last_packet_was_sync_(false) { // Nullify send codec memory, set payload type and set codec name to // invalid values. @@ -1574,8 +1598,8 @@ int AudioCodingModuleImpl::SetVADSafe(bool enable_dtx, // If a send codec is registered, set VAD/DTX for the codec. if (HaveValidEncoder("SetVAD")) { int16_t status = codecs_[current_send_codec_idx_]->SetVAD(enable_dtx, - enable_vad, - mode); + enable_vad, + mode); if (status == 1) { // Vad was enabled. vad_enabled_ = true; @@ -1981,6 +2005,29 @@ int32_t AudioCodingModuleImpl::IncomingPacket( // and "received frequency." CriticalSectionScoped lock(acm_crit_sect_); + // Check there are packets missed between the last injected packet, and the + // latest received packet. If so and we are in AV-sync mode then we would + // like to fill the gap. Shouldn't be the first payload. + if (av_sync_ && first_payload_received_ && + rtp_info.header.sequenceNumber > last_sequence_number_ + 1) { + // If the last packet pushed was sync-packet account for all missing + // packets. Otherwise leave some room for PLC. + if (last_packet_was_sync_) { + while (rtp_info.header.sequenceNumber > last_sequence_number_ + 2) { + PushSyncPacketSafe(); + } + } else { + // Leave two packet room for NetEq perform PLC. + if (rtp_info.header.sequenceNumber > last_sequence_number_ + 3) { + last_sequence_number_ += 2; + last_incoming_send_timestamp_ += last_timestamp_diff_ * 2; + last_receive_timestamp_ += 2 * last_timestamp_diff_; + while (rtp_info.header.sequenceNumber > last_sequence_number_ + 1) + PushSyncPacketSafe(); + } + } + } + uint8_t my_payload_type; // Check if this is an RED payload. @@ -2010,32 +2057,18 @@ int32_t AudioCodingModuleImpl::IncomingPacket( } // Codec is changed, there might be a jump in timestamp, therefore, // we have to reset some variables that track NetEq buffer. - if (track_neteq_buffer_) { + if (track_neteq_buffer_ || av_sync_) { last_incoming_send_timestamp_ = rtp_info.header.timestamp; } } last_recv_audio_codec_pltype_ = my_payload_type; } - if (track_neteq_buffer_) { - const int in_sample_rate_khz = - (ACMCodecDB::database_[current_receive_codec_idx_].plfreq / 1000); - if (first_payload_received_) { - if (rtp_info.header.timestamp > last_incoming_send_timestamp_) { - accumulated_audio_ms_ += (rtp_info.header.timestamp - - last_incoming_send_timestamp_) / in_sample_rate_khz; - } - } else { - first_payload_received_ = true; - } - num_packets_accumulated_++; - last_incoming_send_timestamp_ = rtp_info.header.timestamp; - playout_ts_ = static_cast( - rtp_info.header.timestamp - static_cast( - initial_delay_ms_ * in_sample_rate_khz)); - } + // Current timestamp based on the receiver sampling frequency. + last_receive_timestamp_ = NowTimestamp(current_receive_codec_idx_); } + int per_neteq_payload_length = payload_length; // Split the payload for stereo packets, so that first half of payload // vector holds left channel, and second half holds right channel. if (expected_channels_ == 2) { @@ -2047,24 +2080,46 @@ int32_t AudioCodingModuleImpl::IncomingPacket( memcpy(payload, incoming_payload, payload_length); codecs_[current_receive_codec_idx_]->SplitStereoPacket(payload, &length); rtp_header.type.Audio.channel = 2; - if (track_neteq_buffer_) - num_bytes_accumulated_ += length / 2; // Per neteq, half is inserted - // into master and half to slave. + per_neteq_payload_length = length / 2; // Insert packet into NetEQ. - return neteq_.RecIn(payload, length, rtp_header); + if (neteq_.RecIn(payload, length, rtp_header, + last_receive_timestamp_) < 0) + return -1; } else { - // If we receive a CNG packet while expecting stereo, we ignore the packet - // and continue. CNG is not supported for stereo. + // If we receive a CNG packet while expecting stereo, we ignore the + // packet and continue. CNG is not supported for stereo. return 0; } } else { - { - CriticalSectionScoped lock(acm_crit_sect_); - if (track_neteq_buffer_) - num_bytes_accumulated_ += payload_length; - } - return neteq_.RecIn(incoming_payload, payload_length, rtp_header); + if (neteq_.RecIn(incoming_payload, payload_length, rtp_header, + last_receive_timestamp_) < 0) + return -1; } + + { + CriticalSectionScoped lock(acm_crit_sect_); + + // Update buffering uses |last_incoming_send_timestamp_| so it should be + // before the next block. + if (track_neteq_buffer_) + UpdateBufferingSafe(rtp_header, per_neteq_payload_length); + + if (av_sync_) { + if(rtp_info.header.sequenceNumber == last_sequence_number_ + 1) { + last_timestamp_diff_ = rtp_info.header.timestamp - + last_incoming_send_timestamp_; + } + last_sequence_number_ = rtp_info.header.sequenceNumber; + last_ssrc_ = rtp_info.header.ssrc; + last_packet_was_sync_ = false; + } + + if (av_sync_ || track_neteq_buffer_) { + last_incoming_send_timestamp_ = rtp_info.header.timestamp; + first_payload_received_ = true; + } + } + return 0; } int AudioCodingModuleImpl::UpdateUponReceivingCodec(int index) { @@ -2257,9 +2312,9 @@ int32_t AudioCodingModuleImpl::PlayoutData10Ms( audio_frame->speech_type_ = audio_frame_.speech_type_; stereo_mode = (audio_frame_.num_channels_ > 1); + // For stereo playout: // Master and Slave samples are interleaved starting with Master. - const uint16_t receive_freq = static_cast(audio_frame_.sample_rate_hz_); bool tone_detected = false; @@ -2270,6 +2325,23 @@ int32_t AudioCodingModuleImpl::PlayoutData10Ms( { CriticalSectionScoped lock(acm_crit_sect_); + // If we are in AV-sync and number of packets is below a threshold or + // next packet is late then inject a sync packet. + if (av_sync_ && NowTimestamp(current_receive_codec_idx_) > 5 * + last_timestamp_diff_ + last_receive_timestamp_) { + if (!last_packet_was_sync_) { + // If the last packet inserted has been a regular packet Skip two + // packets to give room for PLC. + last_incoming_send_timestamp_ += 2 * last_timestamp_diff_; + last_sequence_number_ += 2; + last_receive_timestamp_ += 2 * last_timestamp_diff_; + } + + // One sync packet. + if (PushSyncPacketSafe() < 0) + return -1; + } + if ((receive_freq != desired_freq_hz) && (desired_freq_hz != -1)) { TRACE_EVENT_ASYNC_END2("webrtc", "ACM::PlayoutData10Ms", 0, "stereo", stereo_mode, "resample", true); @@ -2449,7 +2521,11 @@ int32_t AudioCodingModuleImpl::RegisterVADCallback( return 0; } +// TODO(turajs): Remove this API if it is not used. // TODO(tlegrand): Modify this function to work for stereo, and add tests. +// TODO(turajs): Receive timestamp in this method is incremented by frame-size +// and does not reflect the true receive frame-size. Therefore, subsequent +// jitter computations are not accurate. int32_t AudioCodingModuleImpl::IncomingPayload( const uint8_t* incoming_payload, const int32_t payload_length, const uint8_t payload_type, const uint32_t timestamp) { @@ -2512,8 +2588,10 @@ int32_t AudioCodingModuleImpl::IncomingPayload( // and "received frequency." last_recv_audio_codec_pltype_ = payload_type; + last_receive_timestamp_ += recv_pl_frame_size_smpls_; // Insert in NetEQ. - if (neteq_.RecIn(incoming_payload, payload_length, *dummy_rtp_header_) < 0) { + if (neteq_.RecIn(incoming_payload, payload_length, *dummy_rtp_header_, + last_receive_timestamp_) < 0) { return -1; } @@ -2836,6 +2914,7 @@ void AudioCodingModuleImpl::ResetFragmentation(int vector_size) { static_cast(vector_size); } +// TODO(turajs): Add second parameter to enable/disable AV-sync. int AudioCodingModuleImpl::SetInitialPlayoutDelay(int delay_ms) { if (delay_ms < 0 || delay_ms > 10000) { return -1; @@ -2854,13 +2933,19 @@ int AudioCodingModuleImpl::SetInitialPlayoutDelay(int delay_ms) { } initial_delay_ms_ = delay_ms; track_neteq_buffer_ = true; + av_sync_ = true; + neteq_.EnableAVSync(av_sync_); return neteq_.SetExtraDelay(delay_ms); } bool AudioCodingModuleImpl::GetSilence(int desired_sample_rate_hz, AudioFrame* frame) { CriticalSectionScoped lock(acm_crit_sect_); - if (initial_delay_ms_ == 0 || accumulated_audio_ms_ >= initial_delay_ms_) { + if (initial_delay_ms_ == 0 || !track_neteq_buffer_) { + return false; + } + + if (accumulated_audio_ms_ >= initial_delay_ms_) { track_neteq_buffer_ = false; return false; } @@ -2906,4 +2991,50 @@ bool AudioCodingModuleImpl::GetSilence(int desired_sample_rate_hz, return true; } +// Must be called within the scope of ACM critical section. +int AudioCodingModuleImpl::PushSyncPacketSafe() { + assert(av_sync_); + last_sequence_number_++; + last_incoming_send_timestamp_ += last_timestamp_diff_; + last_receive_timestamp_ += last_timestamp_diff_; + + WebRtcRTPHeader rtp_info; + rtp_info.header.payloadType = last_recv_audio_codec_pltype_; + rtp_info.header.ssrc = last_ssrc_; + rtp_info.header.markerBit = false; + rtp_info.header.sequenceNumber = last_sequence_number_; + rtp_info.header.timestamp = last_incoming_send_timestamp_; + rtp_info.type.Audio.channel = stereo_receive_ ? 2 : 1; + last_packet_was_sync_ = true; + int payload_len_bytes = neteq_.RecIn(rtp_info, last_receive_timestamp_); + + if (payload_len_bytes < 0) + return -1; + + // This is to account for sync packets inserted during the buffering phase. + if (track_neteq_buffer_) + UpdateBufferingSafe(rtp_info, payload_len_bytes); + + return 0; +} + +// Must be called within the scope of ACM critical section. +void AudioCodingModuleImpl::UpdateBufferingSafe(const WebRtcRTPHeader& rtp_info, + int payload_len_bytes) { + const int in_sample_rate_khz = + (ACMCodecDB::database_[current_receive_codec_idx_].plfreq / 1000); + if (first_payload_received_ && + rtp_info.header.timestamp > last_incoming_send_timestamp_) { + accumulated_audio_ms_ += (rtp_info.header.timestamp - + last_incoming_send_timestamp_) / in_sample_rate_khz; + } + + num_packets_accumulated_++; + num_bytes_accumulated_ += payload_len_bytes; + + playout_ts_ = static_cast( + rtp_info.header.timestamp - static_cast( + initial_delay_ms_ * in_sample_rate_khz)); +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h index 7ce32dc84..fe1564dd3 100644 --- a/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h +++ b/webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h @@ -312,6 +312,18 @@ class AudioCodingModuleImpl : public AudioCodingModule { bool GetSilence(int desired_sample_rate_hz, AudioFrame* frame); + // Push a synchronization packet into NetEq. Such packets result in a frame + // of zeros (not decoded by the corresponding decoder). The size of the frame + // is the same as last decoding. NetEq has a special payload for this. + // Call within the scope of ACM critical section. + int PushSyncPacketSafe(); + + // Update the parameters required in initial phase of buffering, when + // initial playout delay is requested. Call within the scope of ACM critical + // section. + void UpdateBufferingSafe(const WebRtcRTPHeader& rtp_info, + int payload_len_bytes); + AudioPacketizationCallback* packetization_callback_; int32_t id_; uint32_t last_timestamp_; @@ -395,6 +407,17 @@ class AudioCodingModuleImpl : public AudioCodingModule { uint32_t last_incoming_send_timestamp_; bool track_neteq_buffer_; uint32_t playout_ts_; + + // AV-sync is enabled. In AV-sync mode, sync packet pushed during long packet + // losses. + bool av_sync_; + + // Latest send timestamp difference of two consecutive packets. + uint32_t last_timestamp_diff_; + uint16_t last_sequence_number_; + uint32_t last_ssrc_; + bool last_packet_was_sync_; + int64_t last_receive_timestamp_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_coding/neteq/dsp.h b/webrtc/modules/audio_coding/neteq/dsp.h index d6e587efc..9371938d5 100644 --- a/webrtc/modules/audio_coding/neteq/dsp.h +++ b/webrtc/modules/audio_coding/neteq/dsp.h @@ -422,24 +422,25 @@ void WebRtcNetEQ_ClearActivityStats(DSPInst_t *inst); * This function asks NetEQ for more speech/audio data. * * Input: - * - inst : NetEQ instance, i.e. the user that requests more - * speech/audio data. - * - outdata : Pointer to a memory space where the output data - * should be stored. - * - BGNonly : If non-zero, RecOut will only produce background - * noise. It will still draw packets from the packet - * buffer, but they will never be decoded. + * - inst : NetEQ instance, i.e. the user that requests more + * speech/audio data. + * - outdata : Pointer to a memory space where the output data + * should be stored. + * - BGNonly : If non-zero, RecOut will only produce background + * noise. It will still draw packets from the packet + * buffer, but they will never be decoded. + * - av_sync : 1 if NetEQ is in AV-sync, 0 otherwise. * * Output: - * - inst : Updated user information - * - len : Number of samples that were outputted from NetEq + * - inst : Updated user information + * - len : Number of samples that were outputted from NetEq * - * Return value : 0 - Ok - * -1 - Error + * Return value : 0 - Ok + * -1 - Error */ -int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData, int16_t *pw16_len, - int16_t BGNonly); +int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData, + int16_t *pw16_len, int16_t BGNonly, int av_sync); /**************************************************************************** * WebRtcNetEQ_Normal(...) diff --git a/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq_internal.h b/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq_internal.h index 46be2d128..4eefce069 100644 --- a/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq_internal.h +++ b/webrtc/modules/audio_coding/neteq/interface/webrtc_neteq_internal.h @@ -271,6 +271,44 @@ int WebRtcNetEQ_RecOutNoDecode(void *inst, int16_t *pw16_outData, int WebRtcNetEQ_FlushBuffers(void *inst); +/***************************************************************************** + * void WebRtcNetEq_EnableAVSync(...) + * + * Enable AV-sync. If Enabled, NetEq will screen for sync payloads. For + * each sync payload a silence frame is generated. + * + * Input: + * - inst : NetEQ instance + * - enable : non-zero to enable, otherwise disabled. + * + * Output: + * - inst : Updated NetEQ instance + * + */ + +void WebRtcNetEQ_EnableAVSync(void* inst, int enable); + +/**************************************************************************** + * WebRtcNetEQ_RecInSyncRTP(...) + * + * Insert a sync packet with the given RTP specification. + * + * Input: + * - inst : NetEQ instance + * - rtpInfo : Pointer to RTP info + * - receive_timestamp : Receive time (in timestamps of the used codec) + * + * Output: + * - inst : Updated NetEQ instance + * + * Return value : if succeeded it returns the number of bytes pushed + * in, otherwise returns -1. + */ + +int WebRtcNetEQ_RecInSyncRTP(void* inst, + WebRtcNetEQ_RTPInfo* rtp_info, + uint32_t receive_timestamp); + #ifdef __cplusplus } #endif diff --git a/webrtc/modules/audio_coding/neteq/mcu.h b/webrtc/modules/audio_coding/neteq/mcu.h index 6994a36be..1be81c201 100644 --- a/webrtc/modules/audio_coding/neteq/mcu.h +++ b/webrtc/modules/audio_coding/neteq/mcu.h @@ -88,6 +88,13 @@ typedef struct int16_t TSscalingInitialized; enum TsScaling scalingFactor; + /* AV-sync enabled. In AV-sync NetEq screens packets for specific sync + * packets. Sync packets are not decoded by a decoder but generate all-zero + * signal with the same number of samples as previously decoded payload. + * Also in AV-sync mode the sample-size of a sync payload is reported as + * previous frame-size. */ + int av_sync; + #ifdef NETEQ_STEREO int usingStereo; #endif @@ -196,6 +203,7 @@ int WebRtcNetEQ_McuSetFs(MCUInst_t *inst, uint16_t fs_hz); * * Input: * - inst : MCU instance + * - av_sync : 1 if NetEQ is in AV-sync mode, otherwise 0. * * Return value : 0 - Ok * <0 - Error @@ -229,12 +237,17 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacket, * - MCU_inst : MCU instance * - RTPpacket : The RTP packet, parsed into NetEQ's internal RTP struct * - uw32_timeRec : Time stamp for the arrival of the packet (not RTP timestamp) + * - av_sync : indicates if AV-sync is enabled, 1 enabled, + * 0 disabled. * * Return value : 0 - Ok * -1 - Error */ -int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_inst, - SplitInfo_t *split_inst, int16_t *flushed); +int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t* packet, + PacketBuf_t* Buffer_inst, + SplitInfo_t* split_inst, + int16_t* flushed, + int av_sync); /**************************************************************************** * WebRtcNetEQ_GetTimestampScaling(...) diff --git a/webrtc/modules/audio_coding/neteq/mcu_dsp_common.c b/webrtc/modules/audio_coding/neteq/mcu_dsp_common.c index 744a13186..2c48ec7dd 100644 --- a/webrtc/modules/audio_coding/neteq/mcu_dsp_common.c +++ b/webrtc/modules/audio_coding/neteq/mcu_dsp_common.c @@ -35,3 +35,11 @@ int WebRtcNetEQ_DSP2MCUinterrupt(MainInst_t *inst, int16_t *pw16_shared_mem) inst->MCUinst.pw16_writeAddress = pw16_shared_mem; return WebRtcNetEQ_SignalMcu(&inst->MCUinst); } + +int WebRtcNetEQ_IsSyncPayload(const void* payload, int payload_len_bytes) { + if (payload_len_bytes != SYNC_PAYLOAD_LEN_BYTES || + memcmp(payload, kSyncPayload, SYNC_PAYLOAD_LEN_BYTES) != 0) { + return 0; + } + return 1; +} diff --git a/webrtc/modules/audio_coding/neteq/mcu_dsp_common.h b/webrtc/modules/audio_coding/neteq/mcu_dsp_common.h index badffa19d..b4ab514bc 100644 --- a/webrtc/modules/audio_coding/neteq/mcu_dsp_common.h +++ b/webrtc/modules/audio_coding/neteq/mcu_dsp_common.h @@ -31,6 +31,10 @@ #define SHARED_MEM_SIZE 640 #endif +#define SYNC_PAYLOAD_LEN_BYTES 7 +static const uint8_t kSyncPayload[SYNC_PAYLOAD_LEN_BYTES] = { + 'a', 'v', 's', 'y', 'n', 'c', '\0' }; + /* Struct to hold the NetEQ instance */ typedef struct { @@ -58,4 +62,8 @@ int WebRtcNetEQ_DSPinit(MainInst_t *inst); /* The DSP side will call this function to interrupt the MCU side */ int WebRtcNetEQ_DSP2MCUinterrupt(MainInst_t *inst, int16_t *pw16_shared_mem); +/* Returns 1 if the given payload matches |kSyncPayload| payload, otherwise + * 0 is returned. */ +int WebRtcNetEQ_IsSyncPayload(const void* payload, int payload_len_bytes); + #endif diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer.c b/webrtc/modules/audio_coding/neteq/packet_buffer.c index 39f40ef4a..83079d7bf 100644 --- a/webrtc/modules/audio_coding/neteq/packet_buffer.c +++ b/webrtc/modules/audio_coding/neteq/packet_buffer.c @@ -12,12 +12,15 @@ * Implementation of the actual packet buffer data structure. */ +#include #include "packet_buffer.h" #include /* to define NULL */ #include "signal_processing_library.h" +#include "mcu_dsp_common.h" + #include "neteq_error_codes.h" #ifdef NETEQ_DELAY_LOGGING @@ -140,7 +143,7 @@ int WebRtcNetEQ_PacketBufferFlush(PacketBuf_t *bufferInst) int WebRtcNetEQ_PacketBufferInsert(PacketBuf_t *bufferInst, const RTPPacket_t *RTPpacket, - int16_t *flushed) + int16_t *flushed, int av_sync) { int nextPos; int i; @@ -169,6 +172,43 @@ int WebRtcNetEQ_PacketBufferInsert(PacketBuf_t *bufferInst, const RTPPacket_t *R return (-1); } + /* If we are in AV-sync mode, there is a risk that we have inserted a sync + * packet but now received the real version of it. Or because of some timing + * we might be overwriting a true payload with sync (I'm not sure why this + * should happen in regular case, but in some FEC enabled case happens). + * Go through packets and delete the sync version of the packet in hand. Or + * if this is sync packet and the regular version of it exists in the buffer + * refrain from inserting. + * + * TODO(turajs): Could we get this for free if we had set the RCU-counter of + * the sync packet to a number larger than 2? + */ + if (av_sync) { + for (i = 0; i < bufferInst->maxInsertPositions; ++i) { + /* Check if sequence numbers match and the payload actually exists. */ + if (bufferInst->seqNumber[i] == RTPpacket->seqNumber && + bufferInst->payloadLengthBytes[i] > 0) { + if (WebRtcNetEQ_IsSyncPayload(RTPpacket->payload, + RTPpacket->payloadLen)) { + return 0; + } + + if (WebRtcNetEQ_IsSyncPayload(bufferInst->payloadLocation[i], + bufferInst->payloadLengthBytes[i])) { + /* Clear the position in the buffer. */ + bufferInst->payloadType[i] = -1; + bufferInst->payloadLengthBytes[i] = 0; + + /* Reduce packet counter by one. */ + bufferInst->numPacketsInBuffer--; + /* TODO(turajs) if this is the latest packet better we rewind + * insertPosition and related variables. */ + break; /* There should be only one match. */ + } + } + } + } + /* Find a position in the buffer for this packet */ if (bufferInst->numPacketsInBuffer != 0) { @@ -406,7 +446,6 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst, int32_t new_diff; int i; int16_t rcu_payload_cntr; - if (buffer_inst->startPayloadMemory == NULL) { /* Packet buffer has not been initialized. */ return PBUFFER_NOT_INITIALIZED; @@ -493,10 +532,19 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst, int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst, int buffer_pos, const CodecDbInst_t* codec_database, - int codec_pos, int last_duration) { + int codec_pos, int last_duration, + int av_sync) { if (codec_database->funcDurationEst[codec_pos] == NULL) { return last_duration; } + + if (av_sync != 0 && + WebRtcNetEQ_IsSyncPayload(buffer_inst->payloadLocation[buffer_pos], + buffer_inst->payloadLengthBytes[buffer_pos])) { + // In AV-sync and sync payload, report |last_duration| as current duration. + return last_duration; + } + return (*codec_database->funcDurationEst[codec_pos])( codec_database->codec_state[codec_pos], (const uint8_t *)buffer_inst->payloadLocation[buffer_pos], @@ -504,7 +552,8 @@ int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst, } int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst, - const CodecDbInst_t* codec_database) { + const CodecDbInst_t* codec_database, + int av_sync) { int i, count; int last_duration; int last_codec_pos; @@ -546,9 +595,12 @@ int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst, * last_duration to compute a changing duration, we would have to * iterate through the packets in chronological order by timestamp. */ - last_duration = WebRtcNetEQ_PacketBufferGetPacketSize( - buffer_inst, i, codec_database, codec_pos, - last_duration); + /* Check for error before setting. */ + int temp_last_duration = WebRtcNetEQ_PacketBufferGetPacketSize( + buffer_inst, i, codec_database, codec_pos, + last_duration, av_sync); + if (temp_last_duration >= 0) + last_duration = temp_last_duration; } /* Add in the size of this packet. */ size_samples += last_duration; @@ -560,7 +612,6 @@ int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst, if (size_samples < 0) { size_samples = 0; } - return size_samples; } diff --git a/webrtc/modules/audio_coding/neteq/packet_buffer.h b/webrtc/modules/audio_coding/neteq/packet_buffer.h index afd74dbdd..61ff2b970 100644 --- a/webrtc/modules/audio_coding/neteq/packet_buffer.h +++ b/webrtc/modules/audio_coding/neteq/packet_buffer.h @@ -51,7 +51,6 @@ typedef struct 2 for redundant payload */ int *waitingTime; - /* Statistics counter */ uint16_t discardedPackets; /* Number of discarded packets */ @@ -104,20 +103,21 @@ int WebRtcNetEQ_PacketBufferFlush(PacketBuf_t *bufferInst); * This function inserts an RTP packet into the packet buffer. * * Input: - * - bufferInst : Buffer instance - * - RTPpacket : An RTP packet struct (with payload, sequence - * number, etc.) + * - bufferInst : Buffer instance + * - RTPpacket : An RTP packet struct (with payload, sequence + * number, etc.) + * - av_sync : 1 indicates AV-sync enabled, 0 disabled. * * Output: - * - bufferInst : Updated buffer instance - * - flushed : 1 if buffer was flushed, 0 otherwise + * - bufferInst : Updated buffer instance + * - flushed : 1 if buffer was flushed, 0 otherwise * - * Return value : 0 - Ok - * -1 - Error + * Return value : 0 - Ok + * -1 - Error */ int WebRtcNetEQ_PacketBufferInsert(PacketBuf_t *bufferInst, const RTPPacket_t *RTPpacket, - int16_t *flushed); + int16_t *flushed, int av_sync); /**************************************************************************** * WebRtcNetEQ_PacketBufferExtract(...) @@ -183,6 +183,7 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst, * - codec_pos : The codec database entry associated with the payload * type of the specified buffer. * - last_duration : The duration of the previous frame. + * - av_sync : 1 indicates AV-sync enabled, 0 disabled. * * Return value : The buffer size in samples */ @@ -190,7 +191,8 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst, int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst, int buffer_pos, const CodecDbInst_t* codec_database, - int codec_pos, int last_duration); + int codec_pos, int last_duration, + int av_sync); /**************************************************************************** * WebRtcNetEQ_PacketBufferGetSize(...) @@ -204,12 +206,14 @@ int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst, * Input: * - buffer_inst : Buffer instance * - codec_database : Codec database instance + * - av_sync : 1 indicates AV-sync enabled, 0 disabled. * * Return value : The buffer size in samples */ int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst, - const CodecDbInst_t* codec_database); + const CodecDbInst_t* codec_database, + int av_sync); /**************************************************************************** * WebRtcNetEQ_IncrementWaitingTimes(...) diff --git a/webrtc/modules/audio_coding/neteq/recin.c b/webrtc/modules/audio_coding/neteq/recin.c index f0dd210ca..15d618e76 100644 --- a/webrtc/modules/audio_coding/neteq/recin.c +++ b/webrtc/modules/audio_coding/neteq/recin.c @@ -43,7 +43,8 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput, #endif temp_bufsize = WebRtcNetEQ_PacketBufferGetSize(&MCU_inst->PacketBuffer_inst, - &MCU_inst->codec_DB_inst); + &MCU_inst->codec_DB_inst, + MCU_inst->av_sync); /* * Copy from input RTP packet to local copy * (mainly to enable multiple payloads using RED) @@ -223,7 +224,7 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput, MCU_inst->current_Codec = -1; } i_ok = WebRtcNetEQ_PacketBufferInsert(&MCU_inst->PacketBuffer_inst, - &RTPpacket[i_k], &flushed); + &RTPpacket[i_k], &flushed, MCU_inst->av_sync); if (i_ok < 0) { return RECIN_CNG_ERROR; @@ -259,7 +260,8 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput, /* Parse the payload and insert it into the buffer */ i_ok = WebRtcNetEQ_SplitAndInsertPayload(&RTPpacket[i_k], - &MCU_inst->PacketBuffer_inst, &MCU_inst->PayloadSplit_inst, &flushed); + &MCU_inst->PacketBuffer_inst, &MCU_inst->PayloadSplit_inst, + &flushed, MCU_inst->av_sync); if (i_ok < 0) { return i_ok; @@ -311,8 +313,8 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput, { /* Calculate the total speech length carried in each packet */ temp_bufsize = WebRtcNetEQ_PacketBufferGetSize( - &MCU_inst->PacketBuffer_inst, &MCU_inst->codec_DB_inst) - - temp_bufsize; + &MCU_inst->PacketBuffer_inst, &MCU_inst->codec_DB_inst, + MCU_inst->av_sync) - temp_bufsize; if ((temp_bufsize > 0) && (MCU_inst->BufferStat_inst.Automode_inst.lastPackCNGorDTMF == 0) && (temp_bufsize diff --git a/webrtc/modules/audio_coding/neteq/recout.c b/webrtc/modules/audio_coding/neteq/recout.c index 63abbd1ef..8f6200731 100644 --- a/webrtc/modules/audio_coding/neteq/recout.c +++ b/webrtc/modules/audio_coding/neteq/recout.c @@ -96,7 +96,8 @@ extern uint32_t tot_received_packets; int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData, - int16_t *pw16_len, int16_t BGNonly) + int16_t *pw16_len, int16_t BGNonly, + int av_sync) { int16_t blockLen, payloadLen, len = 0, pos; @@ -413,25 +414,36 @@ int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData, int16_t dec_Len; if (!BGNonly) { + /* Check if this is a sync payload. */ + if (av_sync && WebRtcNetEQ_IsSyncPayload(blockPtr, + payloadLen)) { + /* Zero-stuffing with same size as the last frame. */ + dec_Len = inst->w16_frameLen; + memset(&pw16_decoded_buffer[len], 0, dec_Len * + sizeof(pw16_decoded_buffer[len])); + } else { /* Do decoding as normal * * blockPtr is pointing to payload, at this point, - * the most significant bit of *(blockPtr - 1) is a flag if set to 1 - * indicates that the following payload is the redundant payload. + * the most significant bit of *(blockPtr - 1) is a flag if + * set to 1 indicates that the following payload is the + * redundant payload. */ if (((*(blockPtr - 1) & DSP_CODEC_RED_FLAG) != 0) && (inst->codec_ptr_inst.funcDecodeRCU != NULL)) { - dec_Len = inst->codec_ptr_inst.funcDecodeRCU( - inst->codec_ptr_inst.codec_state, blockPtr, payloadLen, - &pw16_decoded_buffer[len], &speechType); + dec_Len = inst->codec_ptr_inst.funcDecodeRCU( + inst->codec_ptr_inst.codec_state, blockPtr, + payloadLen, &pw16_decoded_buffer[len], &speechType); } else { - dec_Len = inst->codec_ptr_inst.funcDecode( - inst->codec_ptr_inst.codec_state, blockPtr, payloadLen, - &pw16_decoded_buffer[len], &speechType); + /* Regular decoding. */ + dec_Len = inst->codec_ptr_inst.funcDecode( + inst->codec_ptr_inst.codec_state, blockPtr, + payloadLen, &pw16_decoded_buffer[len], &speechType); } + } } else { diff --git a/webrtc/modules/audio_coding/neteq/signal_mcu.c b/webrtc/modules/audio_coding/neteq/signal_mcu.c index ebe035dcb..b3791d1af 100644 --- a/webrtc/modules/audio_coding/neteq/signal_mcu.c +++ b/webrtc/modules/audio_coding/neteq/signal_mcu.c @@ -43,9 +43,12 @@ static int WebRtcNetEQ_UpdatePackSizeSamples(MCUInst_t* inst, int buffer_pos, if (codec_pos >= 0) { codec_pos = inst->codec_DB_inst.position[codec_pos]; if (codec_pos >= 0) { - return WebRtcNetEQ_PacketBufferGetPacketSize( - &inst->PacketBuffer_inst, buffer_pos, - &inst->codec_DB_inst, codec_pos, pack_size_samples); + int temp_packet_size_samples = WebRtcNetEQ_PacketBufferGetPacketSize( + &inst->PacketBuffer_inst, buffer_pos, &inst->codec_DB_inst, + codec_pos, pack_size_samples, inst->av_sync); + if (temp_packet_size_samples > 0) + return temp_packet_size_samples; + return pack_size_samples; } } } @@ -245,7 +248,7 @@ int WebRtcNetEQ_SignalMcu(MCUInst_t *inst) /* Check packet buffer */ w32_bufsize = WebRtcNetEQ_PacketBufferGetSize(&inst->PacketBuffer_inst, - &inst->codec_DB_inst); + &inst->codec_DB_inst, inst->av_sync); if (dspInfo.lastMode == MODE_SUCCESS_ACCELERATE || dspInfo.lastMode == MODE_LOWEN_ACCELERATE || dspInfo.lastMode == MODE_SUCCESS_PREEMPTIVE diff --git a/webrtc/modules/audio_coding/neteq/split_and_insert.c b/webrtc/modules/audio_coding/neteq/split_and_insert.c index ce2e8217f..252d7139e 100644 --- a/webrtc/modules/audio_coding/neteq/split_and_insert.c +++ b/webrtc/modules/audio_coding/neteq/split_and_insert.c @@ -20,8 +20,11 @@ #include "neteq_error_codes.h" -int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_inst, - SplitInfo_t *split_inst, int16_t *flushed) +int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t* packet, + PacketBuf_t* Buffer_inst, + SplitInfo_t* split_inst, + int16_t* flushed, + int av_sync) { int i_ok; @@ -41,7 +44,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i if (split_inst->deltaBytes == NO_SPLIT) { /* Not splittable codec */ - i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, packet, &localFlushed); + i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, packet, + &localFlushed, av_sync); *flushed |= localFlushed; if (i_ok < 0) { @@ -76,7 +80,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i while (len >= (2 * split_size)) { /* insert every chunk */ - i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed); + i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, + &localFlushed, av_sync); *flushed |= localFlushed; temp_packet.timeStamp += ((2 * split_size) >> split_inst->deltaTime); i++; @@ -92,7 +97,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i /* Insert the rest */ temp_packet.payloadLen = len; - i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed); + i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, + &localFlushed, av_sync); *flushed |= localFlushed; if (i_ok < 0) { @@ -108,7 +114,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i { temp_packet.payloadLen = split_inst->deltaBytes; - i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed); + i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, + &localFlushed, av_sync); *flushed |= localFlushed; i++; temp_packet.payload = &(pw16_startPayload[(i * split_inst->deltaBytes) >> 1]); @@ -127,7 +134,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i { /* Must be a either an error or a SID frame at the end of the packet. */ temp_packet.payloadLen = len; - i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed); + i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, + &localFlushed, av_sync); *flushed |= localFlushed; if (i_ok < 0) { diff --git a/webrtc/modules/audio_coding/neteq/webrtc_neteq.c b/webrtc/modules/audio_coding/neteq/webrtc_neteq.c index 1be013302..31940c890 100644 --- a/webrtc/modules/audio_coding/neteq/webrtc_neteq.c +++ b/webrtc/modules/audio_coding/neteq/webrtc_neteq.c @@ -440,6 +440,9 @@ int WebRtcNetEQ_Init(void *inst, uint16_t fs) NetEqMainInst->MCUinst.NoOfExpandCalls = 0; NetEqMainInst->MCUinst.fs = fs; + /* Not in AV-sync by default. */ + NetEqMainInst->MCUinst.av_sync = 0; + #ifdef NETEQ_ATEVENT_DECODE /* init DTMF decoder */ ok = WebRtcNetEQ_DtmfDecoderInit(&(NetEqMainInst->MCUinst.DTMF_inst),fs,560); @@ -806,7 +809,7 @@ int WebRtcNetEQ_RecOut(void *inst, int16_t *pw16_outData, int16_t *pw16_len) #endif ok = WebRtcNetEQ_RecOutInternal(&NetEqMainInst->DSPinst, pw16_outData, - pw16_len, 0 /* not BGN only */); + pw16_len, 0 /* not BGN only */, NetEqMainInst->MCUinst.av_sync); if (ok != 0) { NetEqMainInst->ErrorCode = -ok; @@ -887,7 +890,7 @@ int WebRtcNetEQ_RecOutMasterSlave(void *inst, int16_t *pw16_outData, } ok = WebRtcNetEQ_RecOutInternal(&NetEqMainInst->DSPinst, pw16_outData, - pw16_len, 0 /* not BGN only */); + pw16_len, 0 /* not BGN only */, NetEqMainInst->MCUinst.av_sync); if (ok != 0) { NetEqMainInst->ErrorCode = -ok; @@ -958,7 +961,7 @@ int WebRtcNetEQ_RecOutNoDecode(void *inst, int16_t *pw16_outData, #endif ok = WebRtcNetEQ_RecOutInternal(&NetEqMainInst->DSPinst, pw16_outData, - pw16_len, 1 /* BGN only */); + pw16_len, 1 /* BGN only */, NetEqMainInst->MCUinst.av_sync); if (ok != 0) { NetEqMainInst->ErrorCode = -ok; @@ -1186,7 +1189,8 @@ int WebRtcNetEQ_GetNetworkStatistics(void *inst, WebRtcNetEQ_NetworkStatistics * /* Query packet buffer for number of samples. */ temp32 = WebRtcNetEQ_PacketBufferGetSize( &NetEqMainInst->MCUinst.PacketBuffer_inst, - &NetEqMainInst->MCUinst.codec_DB_inst); + &NetEqMainInst->MCUinst.codec_DB_inst, + NetEqMainInst->MCUinst.av_sync); /* Divide by sample rate. * Calculate temp32 * 1000 / fs to get result in ms. */ @@ -1671,3 +1675,21 @@ void WebRtcNetEQ_GetProcessingActivity(void *inst, WebRtcNetEQ_ClearActivityStats(&NetEqMainInst->DSPinst); } + +void WebRtcNetEQ_EnableAVSync(void* inst, int enable) { + MainInst_t *NetEqMainInst = (MainInst_t*) inst; + NetEqMainInst->MCUinst.av_sync = (enable != 0) ? 1 : 0; +} + +int WebRtcNetEQ_RecInSyncRTP(void* inst, WebRtcNetEQ_RTPInfo* rtp_info, + uint32_t receive_timestamp) { + MainInst_t *NetEqMainInst = (MainInst_t*) inst; + if (NetEqMainInst->MCUinst.av_sync == 0) + return -1; + if (WebRtcNetEQ_RecInRTPStruct(inst, rtp_info, kSyncPayload, + SYNC_PAYLOAD_LEN_BYTES, + receive_timestamp) < 0) { + return -1; + } + return SYNC_PAYLOAD_LEN_BYTES; +}