generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Updated the sync module with a slow moving filter

Browse Source 
Review URL: https://webrtc-codereview.appspot.com/1326008

git-svn-id: http://webrtc.googlecode.com/svn/trunk@3884 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
pwestin@webrtc.org 2013-04-22 18:57:14 +00:00
parent 28d54ab18f
commit 63117339dc
3 changed files with 82 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
webrtc/video_engine/stream_synchronization.cc
View File

@ -18,9 +18,11 @@
namespace webrtc {
const int kMaxVideoDiffMs = 80;
const int kMaxAudioDiffMs = 80;
const int kMaxDeltaDelayMs = 1500;
static const int kMaxChangeMs = 80;
static const int kMaxDeltaDelayMs = 10000;
static const int kFilterLength = 4;
// Minimum difference between audio and video to warrant a change.
static const int kMinDeltaMs = 30;
struct ViESyncDelay {
ViESyncDelay() {
@ -43,7 +45,8 @@ StreamSynchronization::StreamSynchronization(int audio_channel_id,
: channel_delay_(new ViESyncDelay),
audio_channel_id_(audio_channel_id),
video_channel_id_(video_channel_id),
base_target_delay_ms_(0) {}
base_target_delay_ms_(0),
avg_diff_ms_(0) {}
StreamSynchronization::~StreamSynchronization() {
delete channel_delay_;
@ -100,11 +103,24 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Current diff is: %d for audio channel: %d",
relative_delay_ms, audio_channel_id_);
int current_diff_ms = *total_video_delay_target_ms - current_audio_delay_ms +
relative_delay_ms;
avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
current_diff_ms) / kFilterLength;
if (abs(avg_diff_ms_) < kMinDeltaMs) {
// Don't adjust if the diff is within our margin.
return false;
}
// Make sure we don't move too fast.
int diff_ms = avg_diff_ms_ / 2;
diff_ms = std::min(diff_ms, kMaxChangeMs);
diff_ms = std::max(diff_ms, -kMaxChangeMs);
int video_delay_ms = base_target_delay_ms_;
if (current_diff_ms > 0) {
if (diff_ms > 0) {
// The minimum video delay is longer than the current audio delay.
// We need to decrease extra video delay, if we have added extra delay
// earlier, or add extra audio delay.
@ -116,10 +132,8 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
video_delay_ms = *total_video_delay_target_ms;
// Check that we don't reduce the delay more than what is allowed.
if (video_delay_ms <
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
video_delay_ms =
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
if (video_delay_ms < channel_delay_->last_video_delay_ms - diff_ms) {
video_delay_ms = channel_delay_->last_video_delay_ms - diff_ms;
channel_delay_->extra_video_delay_ms =
video_delay_ms - *total_video_delay_target_ms;
} else {
@ -132,21 +146,13 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
// We have no extra video delay to remove, increase the audio delay.
if (channel_delay_->last_sync_delay >= 0) {
// We have increased the audio delay earlier, increase it even more.
int audio_diff_ms = current_diff_ms / 2;
if (audio_diff_ms > kMaxAudioDiffMs) {
// We only allow a maximum change of KMaxAudioDiffMS for audio
// due to NetEQ maximum changes.
audio_diff_ms = kMaxAudioDiffMs;
}
// Increase the audio delay.
channel_delay_->extra_audio_delay_ms += audio_diff_ms;
channel_delay_->extra_audio_delay_ms += diff_ms;
// Don't set a too high delay.
if (channel_delay_->extra_audio_delay_ms >
base_target_delay_ms_ + kMaxDeltaDelayMs) {
channel_delay_->extra_audio_delay_ms =
base_target_delay_ms_ + kMaxDeltaDelayMs;
}
channel_delay_->extra_audio_delay_ms = std::min(
channel_delay_->extra_audio_delay_ms,
base_target_delay_ms_ + kMaxDeltaDelayMs);
// Don't add any extra video delay.
video_delay_ms = *total_video_delay_target_ms;
@ -164,20 +170,15 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
channel_delay_->last_sync_delay = 0;
}
}
} else { // if (current_diff_ms > 0)
} else { // if (diff_ms > 0)
// The minimum video delay is lower than the current audio delay.
// We need to decrease possible extra audio delay, or
// add extra video delay.
if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
// We have extra delay in VoiceEngine.
// Start with decreasing the voice delay.
int audio_diff_ms = current_diff_ms / 2;
if (audio_diff_ms < -1 * kMaxAudioDiffMs) {
// Don't change the delay too much at once.
audio_diff_ms = -1 * kMaxAudioDiffMs;
}
// Add the negative difference.
channel_delay_->extra_audio_delay_ms += audio_diff_ms;
// Note: diff_ms is negative; add the negative difference.
channel_delay_->extra_audio_delay_ms += diff_ms;
if (channel_delay_->extra_audio_delay_ms < 0) {
// Negative values not allowed.
@ -196,34 +197,15 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
// We have no extra delay in VoiceEngine, increase the video delay.
channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
// Make the difference positive.
int video_diff_ms = -1 * current_diff_ms;
// This is the desired delay.
video_delay_ms = *total_video_delay_target_ms + video_diff_ms;
if (video_delay_ms > channel_delay_->last_video_delay_ms) {
if (video_delay_ms >
channel_delay_->last_video_delay_ms + kMaxVideoDiffMs) {
// Don't increase the delay too much at once.
video_delay_ms =
channel_delay_->last_video_delay_ms + kMaxVideoDiffMs;
}
// Verify we don't go above the maximum allowed delay.
if (video_delay_ms > base_target_delay_ms_ + kMaxDeltaDelayMs) {
video_delay_ms = base_target_delay_ms_ + kMaxDeltaDelayMs;
}
} else {
if (video_delay_ms <
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs) {
// Don't decrease the delay too much at once.
video_delay_ms =
channel_delay_->last_video_delay_ms - kMaxVideoDiffMs;
}
// Verify we don't go below the minimum delay.
if (video_delay_ms < *total_video_delay_target_ms) {
video_delay_ms = *total_video_delay_target_ms;
}
}
// Note: diff_ms is negative.
video_delay_ms = std::max(*total_video_delay_target_ms,
channel_delay_->last_video_delay_ms) - diff_ms;
// Verify we don't go above the maximum allowed delay.
video_delay_ms = std::min(video_delay_ms,
base_target_delay_ms_ + kMaxDeltaDelayMs);
// Store the values.
channel_delay_->extra_video_delay_ms =
video_delay_ms - *total_video_delay_target_ms;
@ -231,20 +213,16 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
channel_delay_->last_sync_delay = -1;
}
}
avg_diff_ms_ = 0;
WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
"Sync video delay %d ms for video channel and audio delay %d for audio "
"channel %d",
video_delay_ms, channel_delay_->extra_audio_delay_ms, audio_channel_id_);
*extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
if (video_delay_ms < 0) {
video_delay_ms = 0;
}
*total_video_delay_target_ms =
(*total_video_delay_target_ms > video_delay_ms) ?
*total_video_delay_target_ms : video_delay_ms;
video_delay_ms = std::max(video_delay_ms, 0);
*total_video_delay_target_ms = std::max(*total_video_delay_target_ms,
video_delay_ms);
return true;
}

1
webrtc/video_engine/stream_synchronization.h
View File

@ -52,6 +52,7 @@ class StreamSynchronization {
int audio_channel_id_;
int video_channel_id_;
int base_target_delay_ms_;
int avg_diff_ms_;
};
} // namespace webrtc

75
webrtc/video_engine/stream_synchronization_unittest.cc
View File

@ -25,6 +25,7 @@ enum { kMaxDelay = 1500 };
enum { kDefaultAudioFrequency = 8000 };
enum { kDefaultVideoFrequency = 90000 };
const double kNtpFracPerMs = 4.294967296E6;
static const int kSmoothingFilter = 4 * 2;
class Time {
public:
@ -160,13 +161,14 @@ class StreamSynchronizationTest : public ::testing::Test {
int video_delay_ms = base_target_delay + 100;
int extra_audio_delay_ms = 0;
int total_video_delay_ms = base_target_delay;
int filtered_move = (audio_delay_ms - video_delay_ms) / kSmoothingFilter;
EXPECT_TRUE(DelayedStreams(audio_delay_ms,
video_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(base_target_delay + kMaxVideoDiffMs, total_video_delay_ms);
EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
@ -180,7 +182,7 @@ class StreamSynchronizationTest : public ::testing::Test {
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(base_target_delay + 2 * kMaxVideoDiffMs, total_video_delay_ms);
EXPECT_EQ(base_target_delay + 2 * filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
@ -194,8 +196,7 @@ class StreamSynchronizationTest : public ::testing::Test {
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(base_target_delay + audio_delay_ms - video_delay_ms,
total_video_delay_ms);
EXPECT_EQ(base_target_delay + 3 * filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
// Simulate that NetEQ introduces some audio delay.
@ -210,8 +211,9 @@ class StreamSynchronizationTest : public ::testing::Test {
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
total_video_delay_ms);
filtered_move = 3 * filtered_move +
(50 + audio_delay_ms - video_delay_ms) / kSmoothingFilter;
EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
// Simulate that NetEQ reduces its delay.
@ -226,8 +228,11 @@ class StreamSynchronizationTest : public ::testing::Test {
current_audio_delay_ms,
&extra_audio_delay_ms,
&total_video_delay_ms));
EXPECT_EQ(audio_delay_ms - video_delay_ms + current_audio_delay_ms,
total_video_delay_ms);
filtered_move = filtered_move +
(10 + audio_delay_ms - video_delay_ms) / kSmoothingFilter;
EXPECT_EQ(base_target_delay + filtered_move, total_video_delay_ms);
EXPECT_EQ(base_target_delay, extra_audio_delay_ms);
}
@ -245,7 +250,7 @@ class StreamSynchronizationTest : public ::testing::Test {
&total_video_delay_ms));
EXPECT_EQ(base_target_delay, total_video_delay_ms);
// The audio delay is not allowed to change more than this in 1 second.
EXPECT_EQ(base_target_delay + kMaxAudioDiffMs, extra_audio_delay_ms);
EXPECT_GE(base_target_delay + kMaxAudioDiffMs, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
int current_extra_delay_ms = extra_audio_delay_ms;
@ -283,7 +288,7 @@ class StreamSynchronizationTest : public ::testing::Test {
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason reduced the delay.
current_audio_delay_ms = base_target_delay + 170;
current_audio_delay_ms = base_target_delay + 10;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(audio_delay_ms,
@ -302,7 +307,7 @@ class StreamSynchronizationTest : public ::testing::Test {
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason significantly increased the delay.
current_audio_delay_ms = base_target_delay + 250;
current_audio_delay_ms = base_target_delay + 350;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(audio_delay_ms,
@ -320,12 +325,13 @@ class StreamSynchronizationTest : public ::testing::Test {
}
int MaxAudioDelayIncrease(int current_audio_delay_ms, int delay_ms) {
return std::min((delay_ms - current_audio_delay_ms) / 2,
return std::min((delay_ms - current_audio_delay_ms) / kSmoothingFilter,
static_cast<int>(kMaxAudioDiffMs));
}
int MaxAudioDelayDecrease(int current_audio_delay_ms, int delay_ms) {
return std::max((delay_ms - current_audio_delay_ms) / 2, -kMaxAudioDiffMs);
return std::max((delay_ms - current_audio_delay_ms) / kSmoothingFilter,
-kMaxAudioDiffMs);
}
enum { kSendTimeOffsetMs = 98765 };
@ -343,8 +349,8 @@ TEST_F(StreamSynchronizationTest, NoDelay) {
int extra_audio_delay_ms = 0;
int total_video_delay_ms = 0;
EXPECT_TRUE(DelayedStreams(0, 0, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_FALSE(DelayedStreams(0, 0, current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
EXPECT_EQ(0, total_video_delay_ms);
}
@ -359,7 +365,7 @@ TEST_F(StreamSynchronizationTest, VideoDelay) {
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
// The video delay is not allowed to change more than this in 1 second.
EXPECT_EQ(kMaxVideoDiffMs, total_video_delay_ms);
EXPECT_EQ(delay_ms / kSmoothingFilter, total_video_delay_ms);
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
@ -369,7 +375,7 @@ TEST_F(StreamSynchronizationTest, VideoDelay) {
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, extra_audio_delay_ms);
// The video delay is not allowed to change more than this in 1 second.
EXPECT_EQ(2*kMaxVideoDiffMs, total_video_delay_ms);
EXPECT_EQ(2 * delay_ms / kSmoothingFilter, total_video_delay_ms);
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
@ -380,7 +386,7 @@ TEST_F(StreamSynchronizationTest, VideoDelay) {
EXPECT_EQ(0, extra_audio_delay_ms);
// Enough time should have elapsed for the requested total video delay to be
// equal to the relative delay between audio and video, i.e., we are in sync.
EXPECT_EQ(delay_ms, total_video_delay_ms);
EXPECT_EQ(3 * delay_ms / kSmoothingFilter, total_video_delay_ms);
}
TEST_F(StreamSynchronizationTest, AudioDelay) {
@ -393,7 +399,7 @@ TEST_F(StreamSynchronizationTest, AudioDelay) {
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(0, total_video_delay_ms);
// The audio delay is not allowed to change more than this in 1 second.
EXPECT_EQ(kMaxAudioDiffMs, extra_audio_delay_ms);
EXPECT_EQ(delay_ms / kSmoothingFilter, extra_audio_delay_ms);
current_audio_delay_ms = extra_audio_delay_ms;
int current_extra_delay_ms = extra_audio_delay_ms;
@ -423,7 +429,7 @@ TEST_F(StreamSynchronizationTest, AudioDelay) {
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason reduced the delay.
current_audio_delay_ms = 170;
current_audio_delay_ms = 10;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
@ -438,7 +444,7 @@ TEST_F(StreamSynchronizationTest, AudioDelay) {
current_extra_delay_ms = extra_audio_delay_ms;
// Simulate that NetEQ for some reason significantly increased the delay.
current_audio_delay_ms = 250;
current_audio_delay_ms = 350;
send_time_->IncreaseTimeMs(1000);
receive_time_->IncreaseTimeMs(800);
EXPECT_TRUE(DelayedStreams(0, delay_ms, current_audio_delay_ms,
@ -485,32 +491,29 @@ TEST_F(StreamSynchronizationTest, BaseDelay) {
int extra_audio_delay_ms = 0;
int total_video_delay_ms = base_target_delay_ms;
sync_->SetTargetBufferingDelay(base_target_delay_ms);
EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms);
EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
// We are in sync don't change.
EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
// Triggering another call with the same values. Delay should not be modified.
base_target_delay_ms = 2000;
current_audio_delay_ms = base_target_delay_ms;
total_video_delay_ms = base_target_delay_ms;
sync_->SetTargetBufferingDelay(base_target_delay_ms);
EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms);
EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
// We are in sync don't change.
EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
// Changing delay value - intended to test this module only. In practice it
// would take VoE time to adapt.
base_target_delay_ms = 5000;
current_audio_delay_ms = base_target_delay_ms;
total_video_delay_ms = base_target_delay_ms;
sync_->SetTargetBufferingDelay(base_target_delay_ms);
EXPECT_TRUE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
EXPECT_EQ(base_target_delay_ms, extra_audio_delay_ms);
EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
// We are in sync don't change.
EXPECT_FALSE(DelayedStreams(base_target_delay_ms, base_target_delay_ms,
current_audio_delay_ms,
&extra_audio_delay_ms, &total_video_delay_ms));
}
TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {