Improve AV-sync when initial delay is set and NetEq has long buffer.

Review URL: https://webrtc-codereview.appspot.com/1324006

git-svn-id: http://webrtc.googlecode.com/svn/trunk@3883 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/modules/audio_coding/main/source/acm_neteq.cc

@@ -21,7 +21,6 @@
 #include "webrtc/modules/audio_coding/neteq/interface/webrtc_neteq_internal.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h"
-#include "webrtc/system_wrappers/interface/tick_util.h"
 #include "webrtc/system_wrappers/interface/trace.h"
 #include "webrtc/system_wrappers/interface/trace_event.h"
 
@@ -49,7 +48,8 @@ ACMNetEQ::ACMNetEQ()
       callback_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
       min_of_max_num_packets_(0),
       min_of_buffer_size_bytes_(0),
-      per_packet_overhead_bytes_(0) {
+      per_packet_overhead_bytes_(0),
+      av_sync_(false) {
   for (int n = 0; n < MAX_NUM_SLAVE_NETEQ + 1; n++) {
     is_initialized_[n] = false;
     ptr_vadinst_[n] = NULL;
@@ -436,12 +436,59 @@ int32_t ACMNetEQ::NetworkStatistics(
   return 0;
 }
 
-int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload,
+// Should only be called in AV-sync mode.
-                        const int32_t length_payload,
+int ACMNetEQ::RecIn(const WebRtcRTPHeader& rtp_info,
-                        const WebRtcRTPHeader& rtp_info) {
+                    uint32_t receive_timestamp) {
-  int16_t payload_length = static_cast<int16_t>(length_payload);
+  assert(av_sync_);
 
-  // translate to NetEq struct
+  // Translate to NetEq structure.
+  WebRtcNetEQ_RTPInfo neteq_rtpinfo;
+  neteq_rtpinfo.payloadType = rtp_info.header.payloadType;
+  neteq_rtpinfo.sequenceNumber = rtp_info.header.sequenceNumber;
+  neteq_rtpinfo.timeStamp = rtp_info.header.timestamp;
+  neteq_rtpinfo.SSRC = rtp_info.header.ssrc;
+  neteq_rtpinfo.markerBit = rtp_info.header.markerBit;
+
+  CriticalSectionScoped lock(neteq_crit_sect_);
+
+  // Master should be initialized.
+  assert(is_initialized_[0]);
+
+  // Push into Master.
+  int status = WebRtcNetEQ_RecInSyncRTP(inst_[0], &neteq_rtpinfo,
+                                        receive_timestamp);
+  if (status < 0) {
+    LogError("RecInSyncRTP", 0);
+    WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
+                 "RecIn (sync): NetEq, error in pushing in Master");
+    return -1;
+  }
+
+  // If the received stream is stereo, insert a sync payload into slave.
+  if (rtp_info.type.Audio.channel == 2) {
+    // Slave should be initialized.
+    assert(is_initialized_[1]);
+
+    // PUSH into Slave
+    status = WebRtcNetEQ_RecInSyncRTP(inst_[1], &neteq_rtpinfo,
+                                      receive_timestamp);
+    if (status < 0) {
+      LogError("RecInRTPStruct", 1);
+      WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
+                   "RecIn (sync): NetEq, error in pushing in Slave");
+      return -1;
+    }
+  }
+  return status;
+}
+
+int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload,
+                        const int32_t length_payload,
+                        const WebRtcRTPHeader& rtp_info,
+                        uint32_t receive_timestamp) {
+  int16_t payload_length = static_cast<int16_t>(length_payload);
+
+  // Translate to NetEq structure.
   WebRtcNetEQ_RTPInfo neteq_rtpinfo;
   neteq_rtpinfo.payloadType = rtp_info.header.payloadType;
   neteq_rtpinfo.sequenceNumber = rtp_info.header.sequenceNumber;
@@ -450,15 +497,6 @@ int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload,
   neteq_rtpinfo.markerBit = rtp_info.header.markerBit;
 
   CriticalSectionScoped lock(neteq_crit_sect_);
-  // Down-cast the time to (32-6)-bit since we only care about
-  // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms.
-  // we masked 6 most significant bits of 32-bit so we don't loose resolution
-  // when do the following multiplication.
-  const uint32_t now_in_ms =
-      static_cast<uint32_t>(
-          TickTime::MillisecondTimestamp() & 0x03ffffff);
-  uint32_t recv_timestamp = static_cast<uint32_t>(
-      current_samp_freq_khz_ * now_in_ms);
 
   int status;
   // In case of stereo payload, first half of the data should be pushed into
@@ -473,10 +511,10 @@ int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload,
                  "RecIn: NetEq is not initialized.");
     return -1;
   }
-  // PUSH into Master
+  // Push into Master.
   status = WebRtcNetEQ_RecInRTPStruct(inst_[0], &neteq_rtpinfo,
                                       incoming_payload, payload_length,
-                                      recv_timestamp);
+                                      receive_timestamp);
   if (status < 0) {
     LogError("RecInRTPStruct", 0);
     WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
@@ -491,10 +529,10 @@ int32_t ACMNetEQ::RecIn(const uint8_t* incoming_payload,
                    "RecIn: NetEq is not initialized.");
       return -1;
     }
-    // PUSH into Slave
+    // Push into Slave.
     status = WebRtcNetEQ_RecInRTPStruct(inst_[1], &neteq_rtpinfo,
                                         &incoming_payload[payload_length],
-                                        payload_length, recv_timestamp);
+                                        payload_length, receive_timestamp);
     if (status < 0) {
       LogError("RecInRTPStruct", 1);
       WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
@@ -529,7 +567,6 @@ int32_t ACMNetEQ::RecOut(AudioFrame& audio_frame) {
         LogError("RecOut", 0);
         WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
                      "RecOut: NetEq, error in pulling out for mono case");
-
         // Check for errors that can be recovered from:
         // RECOUT_ERROR_SAMPLEUNDERRUN = 2003
         int error_code = WebRtcNetEQ_GetErrorCode(inst_[0]);
@@ -1056,6 +1093,8 @@ int16_t ACMNetEQ::AddSlave(const WebRtcNetEQDecoder* used_codecs,
                    "AddSlave: AddSlave Failed, Could not Set Playout Mode.");
       return -1;
     }
+    // Set AV-sync for the slave.
+    WebRtcNetEQ_EnableAVSync(inst_[slave_idx], av_sync_ ? 1 : 0);
   }
 
   return 0;
@@ -1071,4 +1110,13 @@ uint8_t ACMNetEQ::num_slaves() {
   return num_slaves_;
 }
 
+void ACMNetEQ::EnableAVSync(bool enable) {
+  CriticalSectionScoped lock(neteq_crit_sect_);
+  av_sync_ = enable;
+  for (int i = 0; i < num_slaves_ + 1; ++i) {
+    assert(is_initialized_[i]);
+    WebRtcNetEQ_EnableAVSync(inst_[i], enable ? 1 : 0);
+  }
+}
+
 }  // namespace webrtc

webrtc/modules/audio_coding/main/source/acm_neteq.h

@@ -60,13 +60,31 @@ class ACMNetEQ {
   //   - rtp_info             : RTP header for the incoming payload containing
   //                            information about payload type, sequence number,
   //                            timestamp, SSRC and marker bit.
+  //   - receive_timestamp    : received timestamp.
   //
   // Return value             : 0 if ok.
   //                           <0 if NetEQ returned an error.
   //
   int32_t RecIn(const uint8_t* incoming_payload,
                 const int32_t length_payload,
-                const WebRtcRTPHeader& rtp_info);
+                const WebRtcRTPHeader& rtp_info,
+                uint32_t receive_timestamp);
+
+  //
+  // RecIn()
+  // Insert a sync payload to NetEq. Should only be called if |av_sync_| is
+  // enabled;
+  //
+  // Input:
+  //   - rtp_info             : RTP header for the incoming payload containing
+  //                            information about payload type, sequence number,
+  //                            timestamp, SSRC and marker bit.
+  //   - receive_timestamp    : received timestamp.
+  //
+  // Return value             : 0 if ok.
+  //                           <0 if NetEQ returned an error.
+  //
+  int RecIn(const WebRtcRTPHeader& rtp_info, uint32_t receive_timestamp);
 
   //
   // RecOut()
@@ -278,6 +296,11 @@ class ACMNetEQ {
     overhead_bytes = per_packet_overhead_bytes_;
   }
 
+  //
+  // Set AV-sync mode.
+  //
+  void EnableAVSync(bool enable);
+
  private:
   //
   // RTPPack()
@@ -350,6 +373,9 @@ class ACMNetEQ {
   // Minimum of buffer-size among all NetEq instances.
   int min_of_buffer_size_bytes_;
   int per_packet_overhead_bytes_;
+
+  // Keep track of AV-sync. Just used to set the slave when a slave is added.
+  bool av_sync_;
 };
 
 }  // namespace webrtc

webrtc/modules/audio_coding/main/source/acm_neteq_unittest.cc

@@ -68,8 +68,9 @@ void AcmNetEqTest::InsertZeroPacket(uint16_t sequence_number,
   rtp_header.header.payloadType = payload_type;
   rtp_header.header.markerBit = marker_bit;
   rtp_header.type.Audio.channel = 1;
+  // Receive timestamp can be set to send timestamp in this test.
   ASSERT_EQ(0, neteq_.RecIn(reinterpret_cast<uint8_t*>(payload),
-                            len_payload_bytes, rtp_header));
+                            len_payload_bytes, rtp_header, timestamp));
 }
 
 void AcmNetEqTest::PullData(int expected_num_samples) {

webrtc/modules/audio_coding/main/source/audio_coding_module_impl.cc

@@ -21,6 +21,7 @@
 #include "webrtc/modules/audio_coding/main/source/acm_resampler.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h"
+#include "webrtc/system_wrappers/interface/tick_util.h"
 #include "webrtc/system_wrappers/interface/trace.h"
 #include "webrtc/system_wrappers/interface/trace_event.h"
 
@@ -43,6 +44,9 @@ enum {
   kMaxNumFragmentationVectors = 3
 };
 
+static const uint32_t kMaskTimestamp = 0x03ffffff;
+static const int kDefaultTimestampDiff = 960;  // 20 ms @ 48 kHz.
+
 namespace {
 
 bool IsCodecRED(const CodecInst* codec) {
@@ -85,7 +89,7 @@ int UpMix(const AudioFrame& frame, int length_out_buff, int16_t* out_buff) {
 
 // Return 1 if timestamp t1 is less than timestamp t2, while compensating for
 // wrap-around.
-static int TimestampLessThan(uint32_t t1, uint32_t t2) {
+int TimestampLessThan(uint32_t t1, uint32_t t2) {
   uint32_t kHalfFullRange = static_cast<uint32_t>(0xFFFFFFFF) / 2;
   if (t1 == t2) {
     return 0;
@@ -100,6 +104,21 @@ static int TimestampLessThan(uint32_t t1, uint32_t t2) {
   }
 }
 
+//
+// Return the timestamp of current time, computed according to sampling rate
+// of the codec identified by |codec_id|.
+//
+uint32_t NowTimestamp(int codec_id) {
+  // Down-cast the time to (32-6)-bit since we only care about
+  // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms.
+  // we masked 6 most significant bits of 32-bit so we don't loose resolution
+  // when do the following multiplication.
+  int sample_rate_khz = ACMCodecDB::database_[codec_id].plfreq / 1000;
+  const uint32_t now_in_ms = static_cast<uint32_t>(
+      TickTime::MillisecondTimestamp() & kMaskTimestamp);
+  return static_cast<uint32_t>(sample_rate_khz * now_in_ms);
+}
+
 }  // namespace
 
 AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id)
@@ -147,7 +166,12 @@ AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id)
       first_payload_received_(false),
       last_incoming_send_timestamp_(0),
       track_neteq_buffer_(false),
-      playout_ts_(0) {
+      playout_ts_(0),
+      av_sync_(false),
+      last_timestamp_diff_(kDefaultTimestampDiff),
+      last_sequence_number_(0),
+      last_ssrc_(0),
+      last_packet_was_sync_(false) {
 
   // Nullify send codec memory, set payload type and set codec name to
   // invalid values.
@@ -1574,8 +1598,8 @@ int AudioCodingModuleImpl::SetVADSafe(bool enable_dtx,
   // If a send codec is registered, set VAD/DTX for the codec.
   if (HaveValidEncoder("SetVAD")) {
     int16_t status = codecs_[current_send_codec_idx_]->SetVAD(enable_dtx,
-                                                                    enable_vad,
+                                                              enable_vad,
-                                                                    mode);
+                                                              mode);
     if (status == 1) {
       // Vad was enabled.
       vad_enabled_ = true;
@@ -1981,6 +2005,29 @@ int32_t AudioCodingModuleImpl::IncomingPacket(
     // and "received frequency."
     CriticalSectionScoped lock(acm_crit_sect_);
 
+    // Check there are packets missed between the last injected packet, and the
+    // latest received packet. If so and we are in AV-sync mode then we would
+    // like to fill the gap. Shouldn't be the first payload.
+    if (av_sync_ && first_payload_received_ &&
+        rtp_info.header.sequenceNumber > last_sequence_number_ + 1) {
+      // If the last packet pushed was sync-packet account for all missing
+      // packets. Otherwise leave some room for PLC.
+      if (last_packet_was_sync_) {
+        while (rtp_info.header.sequenceNumber > last_sequence_number_ + 2) {
+          PushSyncPacketSafe();
+        }
+      } else {
+        // Leave two packet room for NetEq perform PLC.
+        if (rtp_info.header.sequenceNumber > last_sequence_number_ + 3) {
+          last_sequence_number_ += 2;
+          last_incoming_send_timestamp_ += last_timestamp_diff_ * 2;
+          last_receive_timestamp_ += 2 * last_timestamp_diff_;
+          while (rtp_info.header.sequenceNumber > last_sequence_number_ + 1)
+            PushSyncPacketSafe();
+        }
+      }
+    }
+
     uint8_t my_payload_type;
 
     // Check if this is an RED payload.
@@ -2010,32 +2057,18 @@ int32_t AudioCodingModuleImpl::IncomingPacket(
         }
         // Codec is changed, there might be a jump in timestamp, therefore,
         // we have to reset some variables that track NetEq buffer.
-        if (track_neteq_buffer_) {
+        if (track_neteq_buffer_ || av_sync_) {
           last_incoming_send_timestamp_ = rtp_info.header.timestamp;
         }
       }
       last_recv_audio_codec_pltype_ = my_payload_type;
     }
 
-    if (track_neteq_buffer_) {
+    // Current timestamp based on the receiver sampling frequency.
-      const int in_sample_rate_khz =
+    last_receive_timestamp_ = NowTimestamp(current_receive_codec_idx_);
-          (ACMCodecDB::database_[current_receive_codec_idx_].plfreq / 1000);
-      if (first_payload_received_) {
-        if (rtp_info.header.timestamp > last_incoming_send_timestamp_) {
-          accumulated_audio_ms_ += (rtp_info.header.timestamp -
-              last_incoming_send_timestamp_) / in_sample_rate_khz;
-        }
-      } else {
-        first_payload_received_ = true;
-      }
-      num_packets_accumulated_++;
-      last_incoming_send_timestamp_ = rtp_info.header.timestamp;
-      playout_ts_ = static_cast<uint32_t>(
-          rtp_info.header.timestamp - static_cast<uint32_t>(
-              initial_delay_ms_ * in_sample_rate_khz));
-    }
   }
 
+  int per_neteq_payload_length = payload_length;
   // Split the payload for stereo packets, so that first half of payload
   // vector holds left channel, and second half holds right channel.
   if (expected_channels_ == 2) {
@@ -2047,24 +2080,46 @@ int32_t AudioCodingModuleImpl::IncomingPacket(
       memcpy(payload, incoming_payload, payload_length);
       codecs_[current_receive_codec_idx_]->SplitStereoPacket(payload, &length);
       rtp_header.type.Audio.channel = 2;
-      if (track_neteq_buffer_)
+      per_neteq_payload_length = length / 2;
-        num_bytes_accumulated_ += length / 2;  // Per neteq, half is inserted
-                                               // into master and half to slave.
       // Insert packet into NetEQ.
-      return neteq_.RecIn(payload, length, rtp_header);
+      if (neteq_.RecIn(payload, length, rtp_header,
+                       last_receive_timestamp_) < 0)
+        return -1;
     } else {
-      // If we receive a CNG packet while expecting stereo, we ignore the packet
+      // If we receive a CNG packet while expecting stereo, we ignore the
-      // and continue. CNG is not supported for stereo.
+      // packet and continue. CNG is not supported for stereo.
       return 0;
     }
   } else {
-    {
+    if (neteq_.RecIn(incoming_payload, payload_length, rtp_header,
-      CriticalSectionScoped lock(acm_crit_sect_);
+                     last_receive_timestamp_) < 0)
-      if (track_neteq_buffer_)
+      return -1;
-        num_bytes_accumulated_ += payload_length;
-    }
-    return neteq_.RecIn(incoming_payload, payload_length, rtp_header);
   }
+
+  {
+    CriticalSectionScoped lock(acm_crit_sect_);
+
+    // Update buffering uses |last_incoming_send_timestamp_| so it should be
+    // before the next block.
+    if (track_neteq_buffer_)
+      UpdateBufferingSafe(rtp_header, per_neteq_payload_length);
+
+    if (av_sync_) {
+      if(rtp_info.header.sequenceNumber == last_sequence_number_ + 1) {
+        last_timestamp_diff_ = rtp_info.header.timestamp -
+            last_incoming_send_timestamp_;
+      }
+      last_sequence_number_ = rtp_info.header.sequenceNumber;
+      last_ssrc_ = rtp_info.header.ssrc;
+      last_packet_was_sync_ = false;
+    }
+
+    if (av_sync_ || track_neteq_buffer_) {
+      last_incoming_send_timestamp_ = rtp_info.header.timestamp;
+      first_payload_received_ = true;
+    }
+  }
+  return 0;
 }
 
 int AudioCodingModuleImpl::UpdateUponReceivingCodec(int index) {
@@ -2257,9 +2312,9 @@ int32_t AudioCodingModuleImpl::PlayoutData10Ms(
   audio_frame->speech_type_ = audio_frame_.speech_type_;
 
   stereo_mode = (audio_frame_.num_channels_ > 1);
+
   // For stereo playout:
   // Master and Slave samples are interleaved starting with Master.
-
   const uint16_t receive_freq =
       static_cast<uint16_t>(audio_frame_.sample_rate_hz_);
   bool tone_detected = false;
@@ -2270,6 +2325,23 @@ int32_t AudioCodingModuleImpl::PlayoutData10Ms(
   {
     CriticalSectionScoped lock(acm_crit_sect_);
 
+    // If we are in AV-sync and number of packets is below a threshold or
+    // next packet is late then inject a sync packet.
+    if (av_sync_ && NowTimestamp(current_receive_codec_idx_) > 5 *
+        last_timestamp_diff_ + last_receive_timestamp_) {
+      if (!last_packet_was_sync_) {
+        // If the last packet inserted has been a regular packet Skip two
+        // packets to give room for PLC.
+        last_incoming_send_timestamp_ += 2 * last_timestamp_diff_;
+        last_sequence_number_ += 2;
+        last_receive_timestamp_ += 2 * last_timestamp_diff_;
+      }
+
+      // One sync packet.
+      if (PushSyncPacketSafe() < 0)
+        return -1;
+    }
+
     if ((receive_freq != desired_freq_hz) && (desired_freq_hz != -1)) {
       TRACE_EVENT_ASYNC_END2("webrtc", "ACM::PlayoutData10Ms", 0,
                              "stereo", stereo_mode, "resample", true);
@@ -2449,7 +2521,11 @@ int32_t AudioCodingModuleImpl::RegisterVADCallback(
   return 0;
 }
 
+// TODO(turajs): Remove this API if it is not used.
 // TODO(tlegrand): Modify this function to work for stereo, and add tests.
+// TODO(turajs): Receive timestamp in this method is incremented by frame-size
+// and does not reflect the true receive frame-size. Therefore, subsequent
+// jitter computations are not accurate.
 int32_t AudioCodingModuleImpl::IncomingPayload(
     const uint8_t* incoming_payload, const int32_t payload_length,
     const uint8_t payload_type, const uint32_t timestamp) {
@@ -2512,8 +2588,10 @@ int32_t AudioCodingModuleImpl::IncomingPayload(
   // and "received frequency."
   last_recv_audio_codec_pltype_ = payload_type;
 
+  last_receive_timestamp_ += recv_pl_frame_size_smpls_;
   // Insert in NetEQ.
-  if (neteq_.RecIn(incoming_payload, payload_length, *dummy_rtp_header_) < 0) {
+  if (neteq_.RecIn(incoming_payload, payload_length, *dummy_rtp_header_,
+                   last_receive_timestamp_) < 0) {
     return -1;
   }
 
@@ -2836,6 +2914,7 @@ void AudioCodingModuleImpl::ResetFragmentation(int vector_size) {
       static_cast<uint16_t>(vector_size);
 }
 
+// TODO(turajs): Add second parameter to enable/disable AV-sync.
 int AudioCodingModuleImpl::SetInitialPlayoutDelay(int delay_ms) {
   if (delay_ms < 0 || delay_ms > 10000) {
     return -1;
@@ -2854,13 +2933,19 @@ int AudioCodingModuleImpl::SetInitialPlayoutDelay(int delay_ms) {
   }
   initial_delay_ms_ = delay_ms;
   track_neteq_buffer_ = true;
+  av_sync_ = true;
+  neteq_.EnableAVSync(av_sync_);
   return neteq_.SetExtraDelay(delay_ms);
 }
 
 bool AudioCodingModuleImpl::GetSilence(int desired_sample_rate_hz,
                                        AudioFrame* frame) {
   CriticalSectionScoped lock(acm_crit_sect_);
-  if (initial_delay_ms_ == 0 || accumulated_audio_ms_ >= initial_delay_ms_) {
+  if (initial_delay_ms_ == 0 || !track_neteq_buffer_) {
+    return false;
+  }
+
+  if (accumulated_audio_ms_ >= initial_delay_ms_) {
     track_neteq_buffer_ = false;
     return false;
   }
@@ -2906,4 +2991,50 @@ bool AudioCodingModuleImpl::GetSilence(int desired_sample_rate_hz,
   return true;
 }
 
+// Must be called within the scope of ACM critical section.
+int AudioCodingModuleImpl::PushSyncPacketSafe() {
+  assert(av_sync_);
+  last_sequence_number_++;
+  last_incoming_send_timestamp_ += last_timestamp_diff_;
+  last_receive_timestamp_ += last_timestamp_diff_;
+
+  WebRtcRTPHeader rtp_info;
+  rtp_info.header.payloadType = last_recv_audio_codec_pltype_;
+  rtp_info.header.ssrc = last_ssrc_;
+  rtp_info.header.markerBit = false;
+  rtp_info.header.sequenceNumber = last_sequence_number_;
+  rtp_info.header.timestamp = last_incoming_send_timestamp_;
+  rtp_info.type.Audio.channel = stereo_receive_ ? 2 : 1;
+  last_packet_was_sync_ = true;
+  int payload_len_bytes = neteq_.RecIn(rtp_info, last_receive_timestamp_);
+
+  if (payload_len_bytes < 0)
+    return -1;
+
+  // This is to account for sync packets inserted during the buffering phase.
+  if (track_neteq_buffer_)
+    UpdateBufferingSafe(rtp_info, payload_len_bytes);
+
+  return 0;
+}
+
+// Must be called within the scope of ACM critical section.
+void AudioCodingModuleImpl::UpdateBufferingSafe(const WebRtcRTPHeader& rtp_info,
+                                                int payload_len_bytes) {
+  const int in_sample_rate_khz =
+      (ACMCodecDB::database_[current_receive_codec_idx_].plfreq / 1000);
+  if (first_payload_received_ &&
+      rtp_info.header.timestamp > last_incoming_send_timestamp_) {
+      accumulated_audio_ms_ += (rtp_info.header.timestamp -
+          last_incoming_send_timestamp_) / in_sample_rate_khz;
+  }
+
+  num_packets_accumulated_++;
+  num_bytes_accumulated_ += payload_len_bytes;
+
+  playout_ts_ = static_cast<uint32_t>(
+      rtp_info.header.timestamp - static_cast<uint32_t>(
+          initial_delay_ms_ * in_sample_rate_khz));
+}
+
 }  // namespace webrtc

webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h

@@ -312,6 +312,18 @@ class AudioCodingModuleImpl : public AudioCodingModule {
 
   bool GetSilence(int desired_sample_rate_hz, AudioFrame* frame);
 
+  // Push a synchronization packet into NetEq. Such packets result in a frame
+  // of zeros (not decoded by the corresponding decoder). The size of the frame
+  // is the same as last decoding. NetEq has a special payload for this.
+  // Call within the scope of ACM critical section.
+  int PushSyncPacketSafe();
+
+  // Update the parameters required in initial phase of buffering, when
+  // initial playout delay is requested. Call within the scope of ACM critical
+  // section.
+  void UpdateBufferingSafe(const WebRtcRTPHeader& rtp_info,
+                           int payload_len_bytes);
+
   AudioPacketizationCallback* packetization_callback_;
   int32_t id_;
   uint32_t last_timestamp_;
@@ -395,6 +407,17 @@ class AudioCodingModuleImpl : public AudioCodingModule {
   uint32_t last_incoming_send_timestamp_;
   bool track_neteq_buffer_;
   uint32_t playout_ts_;
+
+  // AV-sync is enabled. In AV-sync mode, sync packet pushed during long packet
+  // losses.
+  bool av_sync_;
+
+  // Latest send timestamp difference of two consecutive packets.
+  uint32_t last_timestamp_diff_;
+  uint16_t last_sequence_number_;
+  uint32_t last_ssrc_;
+  bool last_packet_was_sync_;
+  int64_t last_receive_timestamp_;
 };
 
 }  // namespace webrtc

webrtc/modules/audio_coding/neteq/dsp.h

@@ -422,24 +422,25 @@ void WebRtcNetEQ_ClearActivityStats(DSPInst_t *inst);
  * This function asks NetEQ for more speech/audio data.
  *
  * Input:
- *		- inst			: NetEQ instance, i.e. the user that requests more 
+ *    - inst      : NetEQ instance, i.e. the user that requests more
- *						  speech/audio data.
+ *                  speech/audio data.
- *		- outdata		: Pointer to a memory space where the output data
+ *    - outdata   : Pointer to a memory space where the output data
- *						  should be stored.
+ *                  should be stored.
- *      - BGNonly       : If non-zero, RecOut will only produce background
+ *    - BGNonly   : If non-zero, RecOut will only produce background
- *                        noise. It will still draw packets from the packet
+ *                  noise. It will still draw packets from the packet
- *                        buffer, but they will never be decoded.
+ *                  buffer, but they will never be decoded.
+ *    - av_sync   : 1 if NetEQ is in AV-sync, 0 otherwise.
  *
  * Output:
- *		- inst			: Updated user information
+ *    - inst      : Updated user information
- *		- len			: Number of samples that were outputted from NetEq
+ *    - len       : Number of samples that were outputted from NetEq
  *
- * Return value			:  0 - Ok
+ * Return value   : 0 - Ok
- *						  -1 - Error
+ *                 -1 - Error
  */
 
-int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData, int16_t *pw16_len,
+int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData,
-                       int16_t BGNonly);
+                               int16_t *pw16_len, int16_t BGNonly, int av_sync);
 
 /****************************************************************************
  * WebRtcNetEQ_Normal(...)

webrtc/modules/audio_coding/neteq/interface/webrtc_neteq_internal.h

@@ -271,6 +271,44 @@ int WebRtcNetEQ_RecOutNoDecode(void *inst, int16_t *pw16_outData,
 
 int WebRtcNetEQ_FlushBuffers(void *inst);
 
+/*****************************************************************************
+ * void WebRtcNetEq_EnableAVSync(...)
+ *
+ * Enable AV-sync. If Enabled, NetEq will screen for sync payloads. For
+ * each sync payload a silence frame is generated.
+ *
+ * Input:
+ *    - inst          : NetEQ instance
+ *    - enable        : non-zero to enable, otherwise disabled.
+ *
+ * Output:
+ *    - inst          : Updated NetEQ instance
+ *
+ */
+
+void WebRtcNetEQ_EnableAVSync(void* inst, int enable);
+
+/****************************************************************************
+ * WebRtcNetEQ_RecInSyncRTP(...)
+ *
+ * Insert a sync packet with the given RTP specification.
+ *
+ * Input:
+ *    - inst              : NetEQ instance
+ *    - rtpInfo           : Pointer to RTP info
+ *    - receive_timestamp : Receive time (in timestamps of the used codec)
+ *
+ * Output:
+ *    - inst              : Updated NetEQ instance
+ *
+ * Return value           : if succeeded it returns the number of bytes pushed
+ *                          in, otherwise returns -1.
+ */
+
+int WebRtcNetEQ_RecInSyncRTP(void* inst,
+                             WebRtcNetEQ_RTPInfo* rtp_info,
+                             uint32_t receive_timestamp);
+
 #ifdef __cplusplus
 }
 #endif

webrtc/modules/audio_coding/neteq/mcu.h

@@ -88,6 +88,13 @@ typedef struct
     int16_t TSscalingInitialized;
     enum TsScaling scalingFactor;
 
+    /* AV-sync enabled. In AV-sync NetEq screens packets for specific sync
+     * packets. Sync packets are not decoded by a decoder but generate all-zero
+     * signal with the same number of samples as previously decoded payload.
+     * Also in AV-sync mode the sample-size of a sync payload is reported as
+     * previous frame-size. */
+    int av_sync;
+
 #ifdef NETEQ_STEREO
     int usingStereo;
 #endif
@@ -196,6 +203,7 @@ int WebRtcNetEQ_McuSetFs(MCUInst_t *inst, uint16_t fs_hz);
  *
  * Input:
  *      - inst          : MCU instance
+ *      - av_sync       : 1 if NetEQ is in AV-sync mode, otherwise 0.
  *
  * Return value         :  0 - Ok
  *                        <0 - Error
@@ -229,12 +237,17 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacket,
  *      - MCU_inst      : MCU instance
  *      - RTPpacket     : The RTP packet, parsed into NetEQ's internal RTP struct
  *      - uw32_timeRec  : Time stamp for the arrival of the packet (not RTP timestamp)
+ *      - av_sync       : indicates if AV-sync is enabled, 1 enabled,
+ *                        0 disabled.
  *
  * Return value         :  0 - Ok
  *                        -1 - Error
  */
-int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_inst,
+int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t* packet,
-                                      SplitInfo_t *split_inst, int16_t *flushed);
+                                      PacketBuf_t* Buffer_inst,
+                                      SplitInfo_t* split_inst,
+                                      int16_t* flushed,
+                                      int av_sync);
 
 /****************************************************************************
  * WebRtcNetEQ_GetTimestampScaling(...)

webrtc/modules/audio_coding/neteq/mcu_dsp_common.c

@@ -35,3 +35,11 @@ int WebRtcNetEQ_DSP2MCUinterrupt(MainInst_t *inst, int16_t *pw16_shared_mem)
     inst->MCUinst.pw16_writeAddress = pw16_shared_mem;
     return WebRtcNetEQ_SignalMcu(&inst->MCUinst);
 }
+
+int WebRtcNetEQ_IsSyncPayload(const void* payload, int payload_len_bytes) {
+  if (payload_len_bytes != SYNC_PAYLOAD_LEN_BYTES ||
+      memcmp(payload, kSyncPayload, SYNC_PAYLOAD_LEN_BYTES) != 0) {
+    return 0;
+  }
+  return 1;
+}

webrtc/modules/audio_coding/neteq/mcu_dsp_common.h

@@ -31,6 +31,10 @@
     #define SHARED_MEM_SIZE 640
 #endif
 
+#define  SYNC_PAYLOAD_LEN_BYTES  7
+static const uint8_t kSyncPayload[SYNC_PAYLOAD_LEN_BYTES] = {
+    'a', 'v', 's', 'y', 'n', 'c', '\0' };
+
 /* Struct to hold the NetEQ instance */
 typedef struct
 {
@@ -58,4 +62,8 @@ int WebRtcNetEQ_DSPinit(MainInst_t *inst);
 /* The DSP side will call this function to interrupt the MCU side */
 int WebRtcNetEQ_DSP2MCUinterrupt(MainInst_t *inst, int16_t *pw16_shared_mem);
 
+/* Returns 1 if the given payload matches |kSyncPayload| payload, otherwise
+ * 0 is returned. */
+int WebRtcNetEQ_IsSyncPayload(const void* payload, int payload_len_bytes);
+
 #endif

webrtc/modules/audio_coding/neteq/packet_buffer.c

@@ -12,12 +12,15 @@
  * Implementation of the actual packet buffer data structure.
  */
 
+#include <assert.h>
 #include "packet_buffer.h"
 
 #include <string.h> /* to define NULL */
 
 #include "signal_processing_library.h"
 
+#include "mcu_dsp_common.h"
+
 #include "neteq_error_codes.h"
 
 #ifdef NETEQ_DELAY_LOGGING
@@ -140,7 +143,7 @@ int WebRtcNetEQ_PacketBufferFlush(PacketBuf_t *bufferInst)
 
 
 int WebRtcNetEQ_PacketBufferInsert(PacketBuf_t *bufferInst, const RTPPacket_t *RTPpacket,
-                                   int16_t *flushed)
+                                   int16_t *flushed, int av_sync)
 {
     int nextPos;
     int i;
@@ -169,6 +172,43 @@ int WebRtcNetEQ_PacketBufferInsert(PacketBuf_t *bufferInst, const RTPPacket_t *R
         return (-1);
     }
 
+    /* If we are in AV-sync mode, there is a risk that we have inserted a sync
+     * packet but now received the real version of it. Or because of some timing
+     * we might be overwriting a true payload with sync (I'm not sure why this
+     * should happen in regular case, but in some FEC enabled case happens).
+     * Go through packets and delete the sync version of the packet in hand. Or
+     * if this is sync packet and the regular version of it exists in the buffer
+     * refrain from inserting.
+     *
+     * TODO(turajs): Could we get this for free if we had set the RCU-counter of
+     * the sync packet to a number larger than 2?
+     */
+    if (av_sync) {
+      for (i = 0; i < bufferInst->maxInsertPositions; ++i) {
+        /* Check if sequence numbers match and the payload actually exists. */
+        if (bufferInst->seqNumber[i] == RTPpacket->seqNumber &&
+            bufferInst->payloadLengthBytes[i] > 0) {
+          if (WebRtcNetEQ_IsSyncPayload(RTPpacket->payload,
+                                        RTPpacket->payloadLen)) {
+            return 0;
+          }
+
+          if (WebRtcNetEQ_IsSyncPayload(bufferInst->payloadLocation[i],
+                                        bufferInst->payloadLengthBytes[i])) {
+            /* Clear the position in the buffer. */
+            bufferInst->payloadType[i] = -1;
+            bufferInst->payloadLengthBytes[i] = 0;
+
+            /* Reduce packet counter by one. */
+            bufferInst->numPacketsInBuffer--;
+            /* TODO(turajs) if this is the latest packet better we rewind
+             * insertPosition and related variables. */
+            break;  /* There should be only one match. */
+          }
+        }
+      }
+    }
+
     /* Find a position in the buffer for this packet */
     if (bufferInst->numPacketsInBuffer != 0)
     {
@@ -406,7 +446,6 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst,
   int32_t new_diff;
   int i;
   int16_t rcu_payload_cntr;
-
   if (buffer_inst->startPayloadMemory == NULL) {
     /* Packet buffer has not been initialized. */
     return PBUFFER_NOT_INITIALIZED;
@@ -493,10 +532,19 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst,
 int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst,
                                           int buffer_pos,
                                           const CodecDbInst_t* codec_database,
-                                          int codec_pos, int last_duration) {
+                                          int codec_pos, int last_duration,
+                                          int av_sync) {
   if (codec_database->funcDurationEst[codec_pos] == NULL) {
     return last_duration;
   }
+
+  if (av_sync != 0 &&
+      WebRtcNetEQ_IsSyncPayload(buffer_inst->payloadLocation[buffer_pos],
+                                buffer_inst->payloadLengthBytes[buffer_pos])) {
+    // In AV-sync and sync payload, report |last_duration| as current duration.
+    return last_duration;
+  }
+
   return (*codec_database->funcDurationEst[codec_pos])(
     codec_database->codec_state[codec_pos],
     (const uint8_t *)buffer_inst->payloadLocation[buffer_pos],
@@ -504,7 +552,8 @@ int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst,
 }
 
 int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst,
-                                        const CodecDbInst_t* codec_database) {
+                                        const CodecDbInst_t* codec_database,
+                                        int av_sync) {
   int i, count;
   int last_duration;
   int last_codec_pos;
@@ -546,9 +595,12 @@ int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst,
          * last_duration to compute a changing duration, we would have to
          * iterate through the packets in chronological order by timestamp.
          */
-         last_duration = WebRtcNetEQ_PacketBufferGetPacketSize(
+        /* Check for error before setting. */
-           buffer_inst, i, codec_database, codec_pos,
+        int temp_last_duration = WebRtcNetEQ_PacketBufferGetPacketSize(
-           last_duration);
+            buffer_inst, i, codec_database, codec_pos,
+            last_duration, av_sync);
+        if (temp_last_duration >= 0)
+          last_duration = temp_last_duration;
       }
       /* Add in the size of this packet. */
       size_samples += last_duration;
@@ -560,7 +612,6 @@ int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst,
   if (size_samples < 0) {
     size_samples = 0;
   }
-
   return size_samples;
 }
 

webrtc/modules/audio_coding/neteq/packet_buffer.h

@@ -51,7 +51,6 @@ typedef struct
      2 for redundant payload */
     int *waitingTime;
 
-
     /* Statistics counter */
     uint16_t discardedPackets; /* Number of discarded packets */
 
@@ -104,20 +103,21 @@ int WebRtcNetEQ_PacketBufferFlush(PacketBuf_t *bufferInst);
  * This function inserts an RTP packet into the packet buffer.
  *
  * Input:
- *		- bufferInst	: Buffer instance
+ *    - bufferInst  : Buffer instance
- *		- RTPpacket		: An RTP packet struct (with payload, sequence
+ *    - RTPpacket   : An RTP packet struct (with payload, sequence
- *						  number, etc.)
+ *                    number, etc.)
+ *    - av_sync     : 1 indicates AV-sync enabled, 0 disabled.
  *
  * Output:
- *      - bufferInst    : Updated buffer instance
+ *    - bufferInst  : Updated buffer instance
- *		- flushed		: 1 if buffer was flushed, 0 otherwise
+ *    - flushed     : 1 if buffer was flushed, 0 otherwise
  *
- * Return value			:  0 - Ok
+ * Return value     : 0 - Ok
- *						  -1 - Error
+ *                   -1 - Error
  */
 
 int WebRtcNetEQ_PacketBufferInsert(PacketBuf_t *bufferInst, const RTPPacket_t *RTPpacket,
-                                   int16_t *flushed);
+                                   int16_t *flushed, int av_sync);
 
 /****************************************************************************
  * WebRtcNetEQ_PacketBufferExtract(...)
@@ -183,6 +183,7 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst,
  *    - codec_pos       : The codec database entry associated with the payload
  *                        type of the specified buffer.
  *    - last_duration   : The duration of the previous frame.
+ *    - av_sync         : 1 indicates AV-sync enabled, 0 disabled.
  *
  * Return value         : The buffer size in samples
  */
@@ -190,7 +191,8 @@ int WebRtcNetEQ_PacketBufferFindLowestTimestamp(PacketBuf_t* buffer_inst,
 int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst,
                                           int buffer_pos,
                                           const CodecDbInst_t* codec_database,
-                                          int codec_pos, int last_duration);
+                                          int codec_pos, int last_duration,
+                                          int av_sync);
 
 /****************************************************************************
  * WebRtcNetEQ_PacketBufferGetSize(...)
@@ -204,12 +206,14 @@ int WebRtcNetEQ_PacketBufferGetPacketSize(const PacketBuf_t* buffer_inst,
  * Input:
  *    - buffer_inst     : Buffer instance
  *    - codec_database  : Codec database instance
+ *    - av_sync         : 1 indicates AV-sync enabled, 0 disabled.
  *
  * Return value         : The buffer size in samples
  */
 
 int32_t WebRtcNetEQ_PacketBufferGetSize(const PacketBuf_t* buffer_inst,
-                                        const CodecDbInst_t* codec_database);
+                                        const CodecDbInst_t* codec_database,
+                                        int av_sync);
 
 /****************************************************************************
  * WebRtcNetEQ_IncrementWaitingTimes(...)

webrtc/modules/audio_coding/neteq/recin.c

@@ -43,7 +43,8 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput,
 #endif
 
     temp_bufsize = WebRtcNetEQ_PacketBufferGetSize(&MCU_inst->PacketBuffer_inst,
-                                                   &MCU_inst->codec_DB_inst);
+                                                   &MCU_inst->codec_DB_inst,
+                                                   MCU_inst->av_sync);
     /*
      * Copy from input RTP packet to local copy
      * (mainly to enable multiple payloads using RED)
@@ -223,7 +224,7 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput,
                 MCU_inst->current_Codec = -1;
             }
             i_ok = WebRtcNetEQ_PacketBufferInsert(&MCU_inst->PacketBuffer_inst,
-                &RTPpacket[i_k], &flushed);
+                &RTPpacket[i_k], &flushed, MCU_inst->av_sync);
             if (i_ok < 0)
             {
                 return RECIN_CNG_ERROR;
@@ -259,7 +260,8 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput,
 
             /* Parse the payload and insert it into the buffer */
             i_ok = WebRtcNetEQ_SplitAndInsertPayload(&RTPpacket[i_k],
-                &MCU_inst->PacketBuffer_inst, &MCU_inst->PayloadSplit_inst, &flushed);
+                &MCU_inst->PacketBuffer_inst, &MCU_inst->PayloadSplit_inst,
+                &flushed, MCU_inst->av_sync);
             if (i_ok < 0)
             {
                 return i_ok;
@@ -311,8 +313,8 @@ int WebRtcNetEQ_RecInInternal(MCUInst_t *MCU_inst, RTPPacket_t *RTPpacketInput,
     {
         /* Calculate the total speech length carried in each packet */
         temp_bufsize = WebRtcNetEQ_PacketBufferGetSize(
-            &MCU_inst->PacketBuffer_inst, &MCU_inst->codec_DB_inst)
+            &MCU_inst->PacketBuffer_inst, &MCU_inst->codec_DB_inst,
-            - temp_bufsize;
+            MCU_inst->av_sync) - temp_bufsize;
 
         if ((temp_bufsize > 0) && (MCU_inst->BufferStat_inst.Automode_inst.lastPackCNGorDTMF
             == 0) && (temp_bufsize

webrtc/modules/audio_coding/neteq/recout.c

@@ -96,7 +96,8 @@ extern uint32_t tot_received_packets;
 
 
 int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData,
-                               int16_t *pw16_len, int16_t BGNonly)
+                               int16_t *pw16_len, int16_t BGNonly,
+                               int av_sync)
 {
 
     int16_t blockLen, payloadLen, len = 0, pos;
@@ -413,25 +414,36 @@ int WebRtcNetEQ_RecOutInternal(DSPInst_t *inst, int16_t *pw16_outData,
                 int16_t dec_Len;
                 if (!BGNonly)
                 {
+                  /* Check if this is a sync payload. */
+                  if (av_sync && WebRtcNetEQ_IsSyncPayload(blockPtr,
+                                                           payloadLen)) {
+                    /* Zero-stuffing with same size as the last frame. */
+                    dec_Len = inst->w16_frameLen;
+                    memset(&pw16_decoded_buffer[len], 0, dec_Len *
+                           sizeof(pw16_decoded_buffer[len]));
+                  } else {
                     /* Do decoding as normal
                      *
                      * blockPtr is pointing to payload, at this point,
-                     * the most significant bit of *(blockPtr - 1) is a flag if set to 1
+                     * the most significant bit of *(blockPtr - 1) is a flag if
-                     * indicates that the following payload is the redundant payload.
+                     * set to 1 indicates that the following payload is the
+                     * redundant payload.
                      */
                     if (((*(blockPtr - 1) & DSP_CODEC_RED_FLAG) != 0)
                         && (inst->codec_ptr_inst.funcDecodeRCU != NULL))
                     {
-                        dec_Len = inst->codec_ptr_inst.funcDecodeRCU(
+                      dec_Len = inst->codec_ptr_inst.funcDecodeRCU(
-                            inst->codec_ptr_inst.codec_state, blockPtr, payloadLen,
+                          inst->codec_ptr_inst.codec_state, blockPtr,
-                            &pw16_decoded_buffer[len], &speechType);
+                          payloadLen, &pw16_decoded_buffer[len], &speechType);
                     }
                     else
                     {
-                        dec_Len = inst->codec_ptr_inst.funcDecode(
+                      /* Regular decoding. */
-                            inst->codec_ptr_inst.codec_state, blockPtr, payloadLen,
+                      dec_Len = inst->codec_ptr_inst.funcDecode(
-                            &pw16_decoded_buffer[len], &speechType);
+                          inst->codec_ptr_inst.codec_state, blockPtr,
+                          payloadLen, &pw16_decoded_buffer[len], &speechType);
                     }
+                  }
                 }
                 else
                 {

webrtc/modules/audio_coding/neteq/signal_mcu.c

@@ -43,9 +43,12 @@ static int WebRtcNetEQ_UpdatePackSizeSamples(MCUInst_t* inst, int buffer_pos,
     if (codec_pos >= 0) {
       codec_pos = inst->codec_DB_inst.position[codec_pos];
       if (codec_pos >= 0) {
-        return WebRtcNetEQ_PacketBufferGetPacketSize(
+        int temp_packet_size_samples = WebRtcNetEQ_PacketBufferGetPacketSize(
-          &inst->PacketBuffer_inst, buffer_pos,
+            &inst->PacketBuffer_inst, buffer_pos, &inst->codec_DB_inst,
-          &inst->codec_DB_inst, codec_pos, pack_size_samples);
+            codec_pos, pack_size_samples, inst->av_sync);
+        if (temp_packet_size_samples > 0)
+          return temp_packet_size_samples;
+        return pack_size_samples;
       }
     }
   }
@@ -245,7 +248,7 @@ int WebRtcNetEQ_SignalMcu(MCUInst_t *inst)
 
     /* Check packet buffer */
     w32_bufsize = WebRtcNetEQ_PacketBufferGetSize(&inst->PacketBuffer_inst,
-        &inst->codec_DB_inst);
+        &inst->codec_DB_inst, inst->av_sync);
 
     if (dspInfo.lastMode == MODE_SUCCESS_ACCELERATE || dspInfo.lastMode
         == MODE_LOWEN_ACCELERATE || dspInfo.lastMode == MODE_SUCCESS_PREEMPTIVE

webrtc/modules/audio_coding/neteq/split_and_insert.c

@@ -20,8 +20,11 @@
 
 #include "neteq_error_codes.h"
 
-int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_inst,
+int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t* packet,
-                                      SplitInfo_t *split_inst, int16_t *flushed)
+                                      PacketBuf_t* Buffer_inst,
+                                      SplitInfo_t* split_inst,
+                                      int16_t* flushed,
+                                      int av_sync)
 {
 
     int i_ok;
@@ -41,7 +44,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i
     if (split_inst->deltaBytes == NO_SPLIT)
     {
         /* Not splittable codec */
-        i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, packet, &localFlushed);
+        i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, packet,
+                                              &localFlushed, av_sync);
         *flushed |= localFlushed;
         if (i_ok < 0)
         {
@@ -76,7 +80,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i
         while (len >= (2 * split_size))
         {
             /* insert every chunk */
-            i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed);
+            i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet,
+                                                  &localFlushed, av_sync);
             *flushed |= localFlushed;
             temp_packet.timeStamp += ((2 * split_size) >> split_inst->deltaTime);
             i++;
@@ -92,7 +97,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i
 
         /* Insert the rest */
         temp_packet.payloadLen = len;
-        i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed);
+        i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet,
+                                              &localFlushed, av_sync);
         *flushed |= localFlushed;
         if (i_ok < 0)
         {
@@ -108,7 +114,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i
         {
 
             temp_packet.payloadLen = split_inst->deltaBytes;
-            i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed);
+            i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet,
+                                                  &localFlushed, av_sync);
             *flushed |= localFlushed;
             i++;
             temp_packet.payload = &(pw16_startPayload[(i * split_inst->deltaBytes) >> 1]);
@@ -127,7 +134,8 @@ int WebRtcNetEQ_SplitAndInsertPayload(RTPPacket_t *packet, PacketBuf_t *Buffer_i
         {
             /* Must be a either an error or a SID frame at the end of the packet. */
             temp_packet.payloadLen = len;
-            i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet, &localFlushed);
+            i_ok = WebRtcNetEQ_PacketBufferInsert(Buffer_inst, &temp_packet,
+                                                  &localFlushed, av_sync);
             *flushed |= localFlushed;
             if (i_ok < 0)
             {

webrtc/modules/audio_coding/neteq/webrtc_neteq.c

@@ -440,6 +440,9 @@ int WebRtcNetEQ_Init(void *inst, uint16_t fs)
     NetEqMainInst->MCUinst.NoOfExpandCalls = 0;
     NetEqMainInst->MCUinst.fs = fs;
 
+    /* Not in AV-sync by default. */
+    NetEqMainInst->MCUinst.av_sync = 0;
+
 #ifdef NETEQ_ATEVENT_DECODE
     /* init DTMF decoder */
     ok = WebRtcNetEQ_DtmfDecoderInit(&(NetEqMainInst->MCUinst.DTMF_inst),fs,560);
@@ -806,7 +809,7 @@ int WebRtcNetEQ_RecOut(void *inst, int16_t *pw16_outData, int16_t *pw16_len)
 #endif
 
     ok = WebRtcNetEQ_RecOutInternal(&NetEqMainInst->DSPinst, pw16_outData,
-        pw16_len, 0 /* not BGN only */);
+        pw16_len, 0 /* not BGN only */, NetEqMainInst->MCUinst.av_sync);
     if (ok != 0)
     {
         NetEqMainInst->ErrorCode = -ok;
@@ -887,7 +890,7 @@ int WebRtcNetEQ_RecOutMasterSlave(void *inst, int16_t *pw16_outData,
     }
 
     ok  = WebRtcNetEQ_RecOutInternal(&NetEqMainInst->DSPinst, pw16_outData,
-        pw16_len, 0 /* not BGN only */);
+        pw16_len, 0 /* not BGN only */, NetEqMainInst->MCUinst.av_sync);
     if (ok != 0)
     {
         NetEqMainInst->ErrorCode = -ok;
@@ -958,7 +961,7 @@ int WebRtcNetEQ_RecOutNoDecode(void *inst, int16_t *pw16_outData,
 #endif
 
     ok = WebRtcNetEQ_RecOutInternal(&NetEqMainInst->DSPinst, pw16_outData,
-        pw16_len, 1 /* BGN only */);
+        pw16_len, 1 /* BGN only */, NetEqMainInst->MCUinst.av_sync);
     if (ok != 0)
     {
         NetEqMainInst->ErrorCode = -ok;
@@ -1186,7 +1189,8 @@ int WebRtcNetEQ_GetNetworkStatistics(void *inst, WebRtcNetEQ_NetworkStatistics *
         /* Query packet buffer for number of samples. */
         temp32 = WebRtcNetEQ_PacketBufferGetSize(
             &NetEqMainInst->MCUinst.PacketBuffer_inst,
-            &NetEqMainInst->MCUinst.codec_DB_inst);
+            &NetEqMainInst->MCUinst.codec_DB_inst,
+            NetEqMainInst->MCUinst.av_sync);
 
         /* Divide by sample rate.
          * Calculate temp32 * 1000 / fs to get result in ms. */
@@ -1671,3 +1675,21 @@ void WebRtcNetEQ_GetProcessingActivity(void *inst,
 
   WebRtcNetEQ_ClearActivityStats(&NetEqMainInst->DSPinst);
 }
+
+void WebRtcNetEQ_EnableAVSync(void* inst, int enable) {
+  MainInst_t *NetEqMainInst = (MainInst_t*) inst;
+  NetEqMainInst->MCUinst.av_sync = (enable != 0) ? 1 : 0;
+}
+
+int WebRtcNetEQ_RecInSyncRTP(void* inst, WebRtcNetEQ_RTPInfo* rtp_info,
+                             uint32_t receive_timestamp) {
+  MainInst_t *NetEqMainInst = (MainInst_t*) inst;
+  if (NetEqMainInst->MCUinst.av_sync == 0)
+    return -1;
+  if (WebRtcNetEQ_RecInRTPStruct(inst, rtp_info, kSyncPayload,
+                                 SYNC_PAYLOAD_LEN_BYTES,
+                                 receive_timestamp) < 0) {
+    return -1;
+  }
+  return SYNC_PAYLOAD_LEN_BYTES;
+}