333 lines
13 KiB
C++
333 lines
13 KiB
C++
/*
|
|
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "vie_sync_module.h"
|
|
#include "critical_section_wrapper.h"
|
|
#include "voe_video_sync.h"
|
|
#include "rtp_rtcp.h"
|
|
#include "trace.h"
|
|
#include "video_coding.h"
|
|
|
|
namespace webrtc {
|
|
|
|
ViESyncModule::ViESyncModule(int id, VideoCodingModule& vcm,
|
|
RtpRtcp& rtcpModule)
|
|
: _dataCritsect(*CriticalSectionWrapper::CreateCriticalSection()), _id(id),
|
|
_vcm(vcm), _rtcpModule(rtcpModule), _voiceChannelId(-1),
|
|
_voiceSyncInterface(NULL), _lastSyncTime(TickTime::Now())
|
|
{
|
|
}
|
|
|
|
ViESyncModule::~ViESyncModule()
|
|
{
|
|
delete &_dataCritsect;
|
|
}
|
|
|
|
int ViESyncModule::SetVoiceChannel(int voiceChannelId,
|
|
VoEVideoSync* veSyncInterface)
|
|
{
|
|
CriticalSectionScoped cs(_dataCritsect);
|
|
_voiceChannelId = voiceChannelId;
|
|
_voiceSyncInterface = veSyncInterface;
|
|
_rtcpModule.DeRegisterSyncModule();
|
|
|
|
if (!veSyncInterface)
|
|
{
|
|
_voiceChannelId = -1;
|
|
if (voiceChannelId >= 0) // trying to set a voice channel but no interface exist
|
|
{
|
|
return -1;
|
|
}
|
|
return 0;
|
|
}
|
|
RtpRtcp* voiceRTPRTCP = NULL;
|
|
veSyncInterface->GetRtpRtcp(_voiceChannelId, voiceRTPRTCP);
|
|
return _rtcpModule.RegisterSyncModule(voiceRTPRTCP);
|
|
}
|
|
|
|
int ViESyncModule::VoiceChannel()
|
|
{
|
|
return _voiceChannelId;
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// SetNetworkDelay
|
|
//
|
|
// Set how long time in ms voice is ahead of video when received on the network.
|
|
// Positive means audio is ahead of video.
|
|
// ----------------------------------------------------------------------------
|
|
void ViESyncModule::SetNetworkDelay(int networkDelay)
|
|
{
|
|
_channelDelay.networkDelay = networkDelay;
|
|
}
|
|
|
|
// Implements Module
|
|
WebRtc_Word32 ViESyncModule::Version(WebRtc_Word8* version,
|
|
WebRtc_UWord32& remainingBufferInBytes,
|
|
WebRtc_UWord32& position) const
|
|
{
|
|
if (version == NULL)
|
|
{
|
|
WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideo, -1,
|
|
"Invalid in argument to ViESyncModule Version()");
|
|
return -1;
|
|
}
|
|
WebRtc_Word8 ourVersion[] = "ViESyncModule 1.1.0";
|
|
WebRtc_UWord32 ourLength = (WebRtc_UWord32) strlen(ourVersion);
|
|
if (remainingBufferInBytes < ourLength + 1)
|
|
{
|
|
return -1;
|
|
}
|
|
memcpy(version, ourVersion, ourLength);
|
|
version[ourLength] = '\0'; // null terminaion
|
|
remainingBufferInBytes -= (ourLength + 1);
|
|
position += (ourLength + 1);
|
|
return 0;
|
|
}
|
|
|
|
WebRtc_Word32 ViESyncModule::ChangeUniqueId(const WebRtc_Word32 id)
|
|
{
|
|
_id = id;
|
|
return 0;
|
|
}
|
|
|
|
WebRtc_Word32 ViESyncModule::TimeUntilNextProcess()
|
|
{
|
|
return (WebRtc_Word32) (kSyncInterval - (TickTime::Now()
|
|
- _lastSyncTime).Milliseconds());
|
|
}
|
|
|
|
// Do the lip sync.
|
|
WebRtc_Word32 ViESyncModule::Process()
|
|
{
|
|
CriticalSectionScoped cs(_dataCritsect);
|
|
_lastSyncTime = TickTime::Now();
|
|
|
|
int totalVideoDelayTargetMS = _vcm.Delay();
|
|
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, _id,
|
|
"Video delay (JB + decoder) is %d ms", totalVideoDelayTargetMS);
|
|
|
|
if (_voiceChannelId != -1)
|
|
{
|
|
// Get //Sync start
|
|
int currentAudioDelayMS = 0;
|
|
if (_voiceSyncInterface->GetDelayEstimate(_voiceChannelId,
|
|
currentAudioDelayMS) != 0)
|
|
{
|
|
// Could not get VoE delay value, probably not a valid channel Id.
|
|
WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, _id,
|
|
"%s: VE_GetDelayEstimate error for voiceChannel %d",
|
|
__FUNCTION__, totalVideoDelayTargetMS, _voiceChannelId);
|
|
return 0;
|
|
}
|
|
int currentDiffMS = 0;
|
|
int videoDelayMS = 0; // Total video delay
|
|
if (currentAudioDelayMS > 40) // Voice Engine report delay estimates even when not started. Ignore
|
|
{
|
|
|
|
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, _id,
|
|
"Audio delay is: %d for voice channel: %d",
|
|
currentAudioDelayMS, _voiceChannelId);
|
|
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, _id,
|
|
"Network delay diff is: %d for voice channel: %d",
|
|
_channelDelay.networkDelay, _voiceChannelId);
|
|
// Calculate the diff between the lowest possible
|
|
// video delay and the current audio delay
|
|
currentDiffMS = totalVideoDelayTargetMS - currentAudioDelayMS
|
|
+ _channelDelay.networkDelay;
|
|
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, _id,
|
|
"Current diff is: %d for audio channel: %d",
|
|
currentDiffMS, _voiceChannelId);
|
|
|
|
if (currentDiffMS > 0)
|
|
{
|
|
// The minimum video delay is longer than the current audio delay.
|
|
// We need to decrease extra video delay, if we have added extra delay
|
|
// earlier, or add extra audio delay.
|
|
if (_channelDelay.extraVideoDelayMS > 0)
|
|
{
|
|
// We have extra delay added to ViE.
|
|
// Reduce this delay before adding delay to VE.
|
|
|
|
// This is the desired delay, we can't reduce more than this.
|
|
videoDelayMS = totalVideoDelayTargetMS;
|
|
|
|
// Check we don't reduce the delay too much
|
|
if (videoDelayMS < _channelDelay.lastVideoDelayMS
|
|
- kMaxVideoDiffMS)
|
|
{
|
|
// Too large step...
|
|
videoDelayMS = _channelDelay.lastVideoDelayMS
|
|
- kMaxVideoDiffMS;
|
|
_channelDelay.extraVideoDelayMS = videoDelayMS
|
|
- totalVideoDelayTargetMS;
|
|
} else
|
|
{
|
|
_channelDelay.extraVideoDelayMS = 0;
|
|
}
|
|
_channelDelay.lastVideoDelayMS = videoDelayMS;
|
|
_channelDelay.lastSyncDelay = -1;
|
|
_channelDelay.extraAudioDelayMS = 0;
|
|
} else
|
|
{
|
|
// We have no extra video delay to remove.
|
|
// Increase the audio delay
|
|
if (_channelDelay.lastSyncDelay >= 0)
|
|
{
|
|
// We have increased the audio delay earlier,
|
|
// increase it even more.
|
|
int audioDiffMS = currentDiffMS / 2;
|
|
if (audioDiffMS > kMaxAudioDiffMS)
|
|
{
|
|
// We only allow a maximum change of KMaxAudioDiffMS for audio
|
|
// due to NetEQ maximum changes.
|
|
audioDiffMS = kMaxAudioDiffMS;
|
|
}
|
|
// Increase the audio delay
|
|
_channelDelay.extraAudioDelayMS += audioDiffMS;
|
|
|
|
// Don't set a too high delay.
|
|
if (_channelDelay.extraAudioDelayMS > kMaxDelay)
|
|
{
|
|
_channelDelay.extraAudioDelayMS = kMaxDelay;
|
|
}
|
|
|
|
// Don't add any extra video delay.
|
|
videoDelayMS = totalVideoDelayTargetMS;
|
|
_channelDelay.extraVideoDelayMS = 0;
|
|
_channelDelay.lastVideoDelayMS = videoDelayMS;
|
|
|
|
_channelDelay.lastSyncDelay = 1;
|
|
} else // lastSyncDelay < 0
|
|
{
|
|
// First time after a delay change, don't add any extra delay.
|
|
// This is to not toggle back and forth too much.
|
|
_channelDelay.extraAudioDelayMS = 0;
|
|
// Set minimum video delay
|
|
videoDelayMS = totalVideoDelayTargetMS;
|
|
_channelDelay.extraVideoDelayMS = 0;
|
|
_channelDelay.lastVideoDelayMS = videoDelayMS;
|
|
_channelDelay.lastSyncDelay = 0;
|
|
}
|
|
}
|
|
} else // if (currentDiffMS > 0)
|
|
{
|
|
// The minimum video delay is lower than the current audio delay.
|
|
// We need to decrease possible extra audio delay, or
|
|
// add extra video delay.
|
|
|
|
if (_channelDelay.extraAudioDelayMS > 0)
|
|
{
|
|
// We have extra delay in VoiceEngine
|
|
// Start with decreasing the voice delay
|
|
int audioDiffMS = currentDiffMS / 2; // This is a negative value
|
|
if (audioDiffMS < -1 * kMaxAudioDiffMS)
|
|
{
|
|
// Don't change the delay too much at once.
|
|
audioDiffMS = -1 * kMaxAudioDiffMS;
|
|
}
|
|
_channelDelay.extraAudioDelayMS += audioDiffMS; // Add the negative change...
|
|
|
|
if (_channelDelay.extraAudioDelayMS < 0)
|
|
{
|
|
// Negative values not allowed
|
|
_channelDelay.extraAudioDelayMS = 0;
|
|
_channelDelay.lastSyncDelay = 0;
|
|
} else
|
|
{
|
|
// There is more audio delay to use for the next round.
|
|
_channelDelay.lastSyncDelay = 1;
|
|
}
|
|
|
|
// Keep the video delay at the minimum values.
|
|
videoDelayMS = totalVideoDelayTargetMS;
|
|
_channelDelay.extraVideoDelayMS = 0;
|
|
_channelDelay.lastVideoDelayMS = videoDelayMS;
|
|
} else
|
|
{
|
|
// We have no extra delay in VoiceEngine
|
|
// Increase the video delay
|
|
_channelDelay.extraAudioDelayMS = 0;
|
|
|
|
// Make the diff positive
|
|
int videoDiffMS = -1 * currentDiffMS;
|
|
|
|
// This is the desired delay we want
|
|
videoDelayMS = totalVideoDelayTargetMS + videoDiffMS;
|
|
if (videoDelayMS > _channelDelay.lastVideoDelayMS)
|
|
{
|
|
if (videoDelayMS > _channelDelay.lastVideoDelayMS
|
|
+ kMaxVideoDiffMS)
|
|
{
|
|
// Don't increase the delay too much at once
|
|
videoDelayMS = _channelDelay.lastVideoDelayMS
|
|
+ kMaxVideoDiffMS;
|
|
}
|
|
// Verify we don't go above the maximum allowed delay
|
|
if (videoDelayMS > kMaxDelay)
|
|
{
|
|
videoDelayMS = kMaxDelay;
|
|
}
|
|
} else
|
|
{
|
|
if (videoDelayMS < _channelDelay.lastVideoDelayMS
|
|
- kMaxVideoDiffMS)
|
|
{
|
|
// Don't decrease the delay too much at once
|
|
videoDelayMS = _channelDelay.lastVideoDelayMS
|
|
- kMaxVideoDiffMS;
|
|
}
|
|
// Verify we don't go below the minimum delay
|
|
if (videoDelayMS < totalVideoDelayTargetMS)
|
|
{
|
|
videoDelayMS = totalVideoDelayTargetMS;
|
|
}
|
|
}
|
|
// Store the values
|
|
_channelDelay.extraVideoDelayMS = videoDelayMS
|
|
- totalVideoDelayTargetMS;
|
|
_channelDelay.lastVideoDelayMS = videoDelayMS;
|
|
_channelDelay.lastSyncDelay = -1;
|
|
}
|
|
}
|
|
}
|
|
|
|
WEBRTC_TRACE(
|
|
webrtc::kTraceInfo,
|
|
webrtc::kTraceVideo,
|
|
_id,
|
|
"Sync video delay %d ms for video channel and audio delay %d for audio channel %d",
|
|
videoDelayMS, _channelDelay.extraAudioDelayMS,
|
|
_voiceChannelId);
|
|
|
|
// Set the extra audio delay
|
|
if (_voiceSyncInterface->SetMinimumPlayoutDelay(_voiceChannelId,
|
|
_channelDelay.extraAudioDelayMS) == -1)
|
|
{
|
|
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideo, _id,
|
|
"Error setting voice delay");
|
|
}
|
|
|
|
// sanity
|
|
// negative not valid
|
|
if (videoDelayMS < 0)
|
|
{
|
|
videoDelayMS = 0;
|
|
}
|
|
totalVideoDelayTargetMS = (totalVideoDelayTargetMS > videoDelayMS) ?
|
|
totalVideoDelayTargetMS : videoDelayMS;
|
|
_vcm.SetMinimumPlayoutDelay(totalVideoDelayTargetMS);
|
|
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, _id,
|
|
"New Video delay target is: %d", totalVideoDelayTargetMS);
|
|
}
|
|
return 0;
|
|
}
|
|
} // namespace webrtc
|