webrtc/src/video_engine/vie_sync_module.cc

/*
 *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "vie_sync_module.h"

#include "critical_section_wrapper.h"
#include "rtp_rtcp.h"
#include "trace.h"
#include "video_coding.h"
#include "voe_video_sync.h"

namespace webrtc {

enum { kSyncInterval = 1000};
enum { kMaxVideoDiffMs = 80 };
enum { kMaxAudioDiffMs = 80 };
enum { kMaxDelay = 1500 };

ViESyncModule::ViESyncModule(int id, VideoCodingModule& vcm,
                             RtpRtcp& rtcp_module)
    : data_critsect_(*CriticalSectionWrapper::CreateCriticalSection()),
      id_(id),
      vcm_(vcm),
      rtcp_module_(rtcp_module),
      voe_channel_id_(-1),
      voe_sync_interface_(NULL),
      last_sync_time_(TickTime::Now()) {
}

ViESyncModule::~ViESyncModule() {
  delete &data_critsect_;
}

int ViESyncModule::SetVoiceChannel(int voe_channel_id,
                                   VoEVideoSync* voe_sync_interface) {
  CriticalSectionScoped cs(data_critsect_);
  voe_channel_id_ = voe_channel_id;
  voe_sync_interface_ = voe_sync_interface;
  rtcp_module_.DeRegisterSyncModule();

  if (!voe_sync_interface) {
    voe_channel_id_ = -1;
    if (voe_channel_id >= 0) {
      // Trying to set a voice channel but no interface exist.
      return -1;
    }
    return 0;
  }
  RtpRtcp* voe_rtp_rtcp = NULL;
  voe_sync_interface->GetRtpRtcp(voe_channel_id_, voe_rtp_rtcp);
  return rtcp_module_.RegisterSyncModule(voe_rtp_rtcp);
}

int ViESyncModule::VoiceChannel() {
  return voe_channel_id_;
}

void ViESyncModule::SetNetworkDelay(int network_delay) {
  channel_delay_.network_delay = network_delay;
}

WebRtc_Word32 ViESyncModule::Version(WebRtc_Word8* version,
                                     WebRtc_UWord32& remaining_buffer_in_bytes,
                                     WebRtc_UWord32& position) const {
  if (version == NULL) {
    WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideo, -1,
                 "Invalid in argument to ViESyncModule Version()");
    return -1;
  }
  WebRtc_Word8 our_version[] = "ViESyncModule 1.1.0";
  WebRtc_UWord32 our_length = (WebRtc_UWord32) strlen(our_version);
  if (remaining_buffer_in_bytes < our_length + 1) {
    return -1;
  }
  memcpy(version, our_version, our_length);
  version[our_length] = '\0';
  remaining_buffer_in_bytes -= (our_length + 1);
  position += (our_length + 1);
  return 0;
}

WebRtc_Word32 ViESyncModule::ChangeUniqueId(const WebRtc_Word32 id) {
  id_ = id;
  return 0;
}

WebRtc_Word32 ViESyncModule::TimeUntilNextProcess() {
  return (WebRtc_Word32)(kSyncInterval -
                         (TickTime::Now() - last_sync_time_).Milliseconds());
}

WebRtc_Word32 ViESyncModule::Process() {
  CriticalSectionScoped cs(data_critsect_);
  last_sync_time_ = TickTime::Now();

  int total_video_delay_target_ms = vcm_.Delay();
  WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, id_,
               "Video delay (JB + decoder) is %d ms",
               total_video_delay_target_ms);

  if (voe_channel_id_ == -1) {
    return 0;
  }

  int current_audio_delay_ms = 0;
  if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_,
                                            current_audio_delay_ms) != 0) {
    // Could not get VoE delay value, probably not a valid channel Id.
    WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, id_,
                 "%s: VE_GetDelayEstimate error for voice_channel %d",
                 __FUNCTION__, total_video_delay_target_ms, voe_channel_id_);
    return 0;
  }

  int current_diff_ms = 0;
  // Total video delay.
  int video_delay_ms = 0;
  // VoiceEngine report delay estimates even when not started, ignore if the
  // reported value is lower than 40 ms.
  if (current_audio_delay_ms < 40) {
    WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, id_,
                 "A/V Sync: Audio delay < 40, skipping.");
    return 0;
  }

  WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, id_,
               "Audio delay is: %d for voice channel: %d",
               current_audio_delay_ms, voe_channel_id_);
  WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, id_,
               "Network delay diff is: %d for voice channel: %d",
               channel_delay_.network_delay, voe_channel_id_);
  // Calculate the difference between the lowest possible video delay and
  // the current audio delay.
  current_diff_ms = total_video_delay_target_ms - current_audio_delay_ms +
      channel_delay_.network_delay;
  WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, id_,
               "Current diff is: %d for audio channel: %d",
               current_diff_ms, voe_channel_id_);

  if (current_diff_ms > 0) {
    // The minimum video delay is longer than the current audio delay.
    // We need to decrease extra video delay, if we have added extra delay
    // earlier, or add extra audio delay.
    if (channel_delay_.extra_video_delay_ms > 0) {
      // We have extra delay added to ViE. Reduce this delay before adding
      // extra delay to VoE.

      // This is the desired delay, we can't reduce more than this.
      video_delay_ms = total_video_delay_target_ms;

      // Check that we don't reduce the delay more than what is allowed.
      if (video_delay_ms <
          channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) {
        video_delay_ms =
            channel_delay_.last_video_delay_ms - kMaxVideoDiffMs;
        channel_delay_.extra_video_delay_ms =
            video_delay_ms - total_video_delay_target_ms;
      } else {
        channel_delay_.extra_video_delay_ms = 0;
      }
      channel_delay_.last_video_delay_ms = video_delay_ms;
      channel_delay_.last_sync_delay = -1;
      channel_delay_.extra_audio_delay_ms = 0;
    } else {  // channel_delay_.extra_video_delay_ms > 0
      // We have no extra video delay to remove, increase the audio delay.
      if (channel_delay_.last_sync_delay >= 0) {
        // We have increased the audio delay earlier, increase it even more.
        int audio_diff_ms = current_diff_ms / 2;
        if (audio_diff_ms > kMaxAudioDiffMs) {
          // We only allow a maximum change of KMaxAudioDiffMS for audio
          // due to NetEQ maximum changes.
          audio_diff_ms = kMaxAudioDiffMs;
        }
        // Increase the audio delay
        channel_delay_.extra_audio_delay_ms += audio_diff_ms;

        // Don't set a too high delay.
        if (channel_delay_.extra_audio_delay_ms > kMaxDelay) {
          channel_delay_.extra_audio_delay_ms = kMaxDelay;
        }

        // Don't add any extra video delay.
        video_delay_ms = total_video_delay_target_ms;
        channel_delay_.extra_video_delay_ms = 0;
        channel_delay_.last_video_delay_ms = video_delay_ms;
        channel_delay_.last_sync_delay = 1;
      } else {  // channel_delay_.last_sync_delay >= 0
        // First time after a delay change, don't add any extra delay.
        // This is to not toggle back and forth too much.
        channel_delay_.extra_audio_delay_ms = 0;
        // Set minimum video delay
        video_delay_ms = total_video_delay_target_ms;
        channel_delay_.extra_video_delay_ms = 0;
        channel_delay_.last_video_delay_ms = video_delay_ms;
        channel_delay_.last_sync_delay = 0;
      }
    }
  } else {  // if (current_diffMS > 0)
    // The minimum video delay is lower than the current audio delay.
    // We need to decrease possible extra audio delay, or
    // add extra video delay.

    if (channel_delay_.extra_audio_delay_ms > 0) {
      // We have extra delay in VoiceEngine
      // Start with decreasing the voice delay
      int audio_diff_ms = current_diff_ms / 2;
      if (audio_diff_ms < -1 * kMaxAudioDiffMs) {
        // Don't change the delay too much at once.
        audio_diff_ms = -1 * kMaxAudioDiffMs;
      }
      // Add the negative difference.
      channel_delay_.extra_audio_delay_ms += audio_diff_ms;

      if (channel_delay_.extra_audio_delay_ms < 0) {
        // Negative values not allowed.
        channel_delay_.extra_audio_delay_ms = 0;
        channel_delay_.last_sync_delay = 0;
      } else {
        // There is more audio delay to use for the next round.
        channel_delay_.last_sync_delay = 1;
      }

      // Keep the video delay at the minimum values.
      video_delay_ms = total_video_delay_target_ms;
      channel_delay_.extra_video_delay_ms = 0;
      channel_delay_.last_video_delay_ms = video_delay_ms;
    } else {  // channel_delay_.extra_audio_delay_ms > 0
      // We have no extra delay in VoiceEngine, increase the video delay.
      channel_delay_.extra_audio_delay_ms = 0;

      // Make the difference positive.
      int video_diff_ms = -1 * current_diff_ms;

      // This is the desired delay.
      video_delay_ms = total_video_delay_target_ms + video_diff_ms;
      if (video_delay_ms > channel_delay_.last_video_delay_ms) {
        if (video_delay_ms >
            channel_delay_.last_video_delay_ms + kMaxVideoDiffMs) {
          // Don't increase the delay too much at once
          video_delay_ms =
              channel_delay_.last_video_delay_ms + kMaxVideoDiffMs;
        }
        // Verify we don't go above the maximum allowed delay
        if (video_delay_ms > kMaxDelay) {
          video_delay_ms = kMaxDelay;
        }
      } else {
        if (video_delay_ms <
            channel_delay_.last_video_delay_ms - kMaxVideoDiffMs) {
          // Don't decrease the delay too much at once
          video_delay_ms =
              channel_delay_.last_video_delay_ms - kMaxVideoDiffMs;
        }
        // Verify we don't go below the minimum delay
        if (video_delay_ms < total_video_delay_target_ms) {
          video_delay_ms = total_video_delay_target_ms;
        }
      }
      // Store the values
      channel_delay_.extra_video_delay_ms =
          video_delay_ms - total_video_delay_target_ms;
      channel_delay_.last_video_delay_ms = video_delay_ms;
      channel_delay_.last_sync_delay = -1;
    }
  }

  WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, id_,
      "Sync video delay %d ms for video channel and audio delay %d for audio "
      "channel %d",
      video_delay_ms, channel_delay_.extra_audio_delay_ms, voe_channel_id_);

  // Set the extra audio delay.synchronization
  if (voe_sync_interface_->SetMinimumPlayoutDelay(
      voe_channel_id_, channel_delay_.extra_audio_delay_ms) == -1) {
    WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideo, id_,
                 "Error setting voice delay");
  }

  if (video_delay_ms < 0) {
    video_delay_ms = 0;
  }
  total_video_delay_target_ms =
      (total_video_delay_target_ms  >  video_delay_ms) ?
      total_video_delay_target_ms : video_delay_ms;
  vcm_.SetMinimumPlayoutDelay(total_video_delay_target_ms);
  WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, id_,
               "New Video delay target is: %d", total_video_delay_target_ms);
  return 0;
}

}  // namespace webrtc