From ef14488d030b1b9f5794ee2fad54d1389b060bea Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Tue, 7 May 2013 19:16:33 +0000 Subject: [PATCH] Trigger a PLI if the duration of non-decodable frames exceeds a threshold. BUG=1663 R=mikhal@webrtc.org, ronghuawu@chromium.org Review URL: https://webrtc-codereview.appspot.com/1359004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@3975 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/interface/video_coding.h | 7 +- .../main/source/decoding_state.cc | 20 -- .../video_coding/main/source/decoding_state.h | 2 - .../main/source/decoding_state_unittest.cc | 32 --- .../video_coding/main/source/jitter_buffer.cc | 186 ++++++++++++----- .../video_coding/main/source/jitter_buffer.h | 16 +- .../main/source/jitter_buffer_unittest.cc | 26 ++- .../video_coding/main/source/receiver.cc | 14 +- .../video_coding/main/source/receiver.h | 3 +- .../main/source/receiver_unittest.cc | 187 +++++++++++++++++- .../main/source/stream_generator.h | 2 +- .../main/source/video_coding_impl.cc | 12 +- .../main/source/video_coding_impl.h | 3 +- .../main/source/video_coding_impl_unittest.cc | 2 +- .../video_coding_robustness_unittest.cc | 2 +- .../main/test/vcm_payload_sink_factory.cc | 2 +- webrtc/video_engine/vie_channel.cc | 11 +- 17 files changed, 392 insertions(+), 135 deletions(-) diff --git a/webrtc/modules/video_coding/main/interface/video_coding.h b/webrtc/modules/video_coding/main/interface/video_coding.h index 9f5ddefb6..a92d80b6b 100644 --- a/webrtc/modules/video_coding/main/interface/video_coding.h +++ b/webrtc/modules/video_coding/main/interface/video_coding.h @@ -571,9 +571,12 @@ public: // Sets the maximum number of sequence numbers that we are allowed to NACK // and the oldest sequence number that we will consider to NACK. If a // sequence number older than |max_packet_age_to_nack| is missing - // a key frame will be requested. + // a key frame will be requested. A key frame will also be requested if the + // time of incomplete or non-continuous frames in the jitter buffer is above + // |max_incomplete_time_ms|. virtual void SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack) = 0; + int max_packet_age_to_nack, + int max_incomplete_time_ms) = 0; // Setting a desired delay to the VCM receiver. Video rendering will be // delayed by at least desired_delay_ms. diff --git a/webrtc/modules/video_coding/main/source/decoding_state.cc b/webrtc/modules/video_coding/main/source/decoding_state.cc index 0974584d7..1196b8866 100644 --- a/webrtc/modules/video_coding/main/source/decoding_state.cc +++ b/webrtc/modules/video_coding/main/source/decoding_state.cc @@ -82,26 +82,6 @@ void VCMDecodingState::CopyFrom(const VCMDecodingState& state) { in_initial_state_ = state.in_initial_state_; } -void VCMDecodingState::SetStateOneBack(const VCMFrameBuffer* frame) { - assert(frame != NULL && frame->GetHighSeqNum() >= 0); - sequence_num_ = static_cast(frame->GetHighSeqNum()) - 1u; - time_stamp_ = frame->TimeStamp() - 1u; - temporal_id_ = frame->TemporalId(); - if (frame->PictureId() != kNoPictureId) { - if (frame->PictureId() == 0) - picture_id_ = 0x7FFF; - else - picture_id_ = frame->PictureId() - 1; - } - if (frame->Tl0PicId() != kNoTl0PicIdx) { - if (frame->Tl0PicId() == 0) - tl0_pic_id_ = 0x00FF; - else - tl0_pic_id_ = frame->Tl0PicId() - 1; - } - in_initial_state_ = false; -} - void VCMDecodingState::UpdateEmptyFrame(const VCMFrameBuffer* frame) { if (ContinuousFrame(frame)) { time_stamp_ = frame->TimeStamp(); diff --git a/webrtc/modules/video_coding/main/source/decoding_state.h b/webrtc/modules/video_coding/main/source/decoding_state.h index f0019660e..b579cba5d 100644 --- a/webrtc/modules/video_coding/main/source/decoding_state.h +++ b/webrtc/modules/video_coding/main/source/decoding_state.h @@ -32,8 +32,6 @@ class VCMDecodingState { bool ContinuousFrame(const VCMFrameBuffer* frame) const; void SetState(const VCMFrameBuffer* frame); void CopyFrom(const VCMDecodingState& state); - // Set the decoding state one frame back. - void SetStateOneBack(const VCMFrameBuffer* frame); void UpdateEmptyFrame(const VCMFrameBuffer* frame); // Update the sequence number if the timestamp matches current state and the // sequence number is higher than the current one. This accounts for packets diff --git a/webrtc/modules/video_coding/main/source/decoding_state_unittest.cc b/webrtc/modules/video_coding/main/source/decoding_state_unittest.cc index e3eef0479..5f01edbc3 100644 --- a/webrtc/modules/video_coding/main/source/decoding_state_unittest.cc +++ b/webrtc/modules/video_coding/main/source/decoding_state_unittest.cc @@ -168,38 +168,6 @@ TEST(TestDecodingState, FrameContinuity) { delete packet; } -TEST(TestDecodingState, SetStateOneBack) { - VCMDecodingState dec_state; - VCMFrameBuffer frame; - frame.SetState(kStateEmpty); - VCMPacket* packet = new VCMPacket(); - // Based on PictureId. - packet->frameType = kVideoFrameDelta; - packet->codecSpecificHeader.codec = kRTPVideoVP8; - packet->timestamp = 0; - packet->seqNum = 0; - packet->codecSpecificHeader.codecHeader.VP8.pictureId = 0; - packet->frameType = kVideoFrameDelta; - frame.InsertPacket(*packet, 0, false, 0); - dec_state.SetStateOneBack(&frame); - EXPECT_EQ(dec_state.sequence_num(), 0xFFFF); - // Check continuity. - EXPECT_TRUE(dec_state.ContinuousFrame(&frame)); - - // Based on Temporal layers. - packet->timestamp = 0; - packet->seqNum = 0; - packet->codecSpecificHeader.codecHeader.VP8.pictureId = kNoPictureId; - packet->frameType = kVideoFrameDelta; - packet->codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0; - packet->codecSpecificHeader.codecHeader.VP8.temporalIdx = 0; - frame.InsertPacket(*packet, 0, false, 0); - dec_state.SetStateOneBack(&frame); - // Check continuity - EXPECT_TRUE(dec_state.ContinuousFrame(&frame)); - delete packet; -} - TEST(TestDecodingState, UpdateOldPacket) { VCMDecodingState dec_state; // Update only if zero size and newer than previous. diff --git a/webrtc/modules/video_coding/main/source/jitter_buffer.cc b/webrtc/modules/video_coding/main/source/jitter_buffer.cc index 200338361..da91b6817 100644 --- a/webrtc/modules/video_coding/main/source/jitter_buffer.cc +++ b/webrtc/modules/video_coding/main/source/jitter_buffer.cc @@ -29,7 +29,7 @@ namespace webrtc { // Use this rtt if no value has been reported. -static uint32_t kDefaultRtt = 200; +static const uint32_t kDefaultRtt = 200; // Predicates used when searching for frames in the frame buffer list class FrameSmallerTimestamp { @@ -54,6 +54,13 @@ class FrameEqualTimestamp { uint32_t timestamp_; }; +class KeyFrameCriteria { + public: + bool operator()(VCMFrameBuffer* frame) { + return frame->FrameType() == kVideoFrameKey; + } +}; + class CompleteKeyFrameCriteria { public: bool operator()(VCMFrameBuffer* frame) { @@ -105,6 +112,7 @@ VCMJitterBuffer::VCMJitterBuffer(Clock* clock, nack_seq_nums_(), max_nack_list_size_(0), max_packet_age_to_nack_(0), + max_incomplete_time_ms_(0), decode_with_errors_(false) { memset(frame_buffers_, 0, sizeof(frame_buffers_)); memset(receive_statistics_, 0, sizeof(receive_statistics_)); @@ -152,6 +160,7 @@ void VCMJitterBuffer::CopyFrom(const VCMJitterBuffer& rhs) { decode_with_errors_ = rhs.decode_with_errors_; assert(max_nack_list_size_ == rhs.max_nack_list_size_); assert(max_packet_age_to_nack_ == rhs.max_packet_age_to_nack_); + assert(max_incomplete_time_ms_ == rhs.max_incomplete_time_ms_); memcpy(receive_statistics_, rhs.receive_statistics_, sizeof(receive_statistics_)); nack_seq_nums_.resize(rhs.nack_seq_nums_.size()); @@ -391,7 +400,8 @@ bool VCMJitterBuffer::NextCompleteTimestamp( } CleanUpOldOrEmptyFrames(); - FrameList::iterator it = FindOldestCompleteContinuousFrame(); + FrameList::iterator it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); if (it == frame_list_.end()) { const int64_t end_wait_time_ms = clock_->TimeInMilliseconds() + max_wait_time_ms; @@ -410,7 +420,8 @@ bool VCMJitterBuffer::NextCompleteTimestamp( // Finding oldest frame ready for decoder, but check // sequence number and size CleanUpOldOrEmptyFrames(); - it = FindOldestCompleteContinuousFrame(); + it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); if (it == frame_list_.end()) { wait_time_ms = end_wait_time_ms - clock_->TimeInMilliseconds(); } else { @@ -800,14 +811,14 @@ void VCMJitterBuffer::SetNackMode(VCMNackMode mode, } void VCMJitterBuffer::SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack) { + int max_packet_age_to_nack, + int max_incomplete_time_ms) { CriticalSectionScoped cs(crit_sect_); assert(max_packet_age_to_nack >= 0); - if (max_packet_age_to_nack <= 0) { - return; - } + assert(max_incomplete_time_ms_ >= 0); max_nack_list_size_ = max_nack_list_size; max_packet_age_to_nack_ = max_packet_age_to_nack; + max_incomplete_time_ms_ = max_incomplete_time_ms; nack_seq_nums_.resize(max_nack_list_size_); } @@ -816,6 +827,27 @@ VCMNackMode VCMJitterBuffer::nack_mode() const { return nack_mode_; } +int VCMJitterBuffer::NonContinuousOrIncompleteDuration() { + if (frame_list_.empty()) { + return 0; + } + FrameList::iterator start_it; + FrameList::iterator end_it; + RenderBuffer(&start_it, &end_it); + if (end_it == frame_list_.end()) + end_it = frame_list_.begin(); + return frame_list_.back()->TimeStamp() - + (*end_it)->TimeStamp(); +} + +uint16_t VCMJitterBuffer::EstimatedLowSequenceNumber( + const VCMFrameBuffer& frame) const { + assert(frame.GetLowSeqNum() >= 0); + if (frame.HaveFirstPacket()) + return frame.GetLowSeqNum(); + return frame.GetLowSeqNum() - 1; +} + uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size, bool* request_key_frame) { CriticalSectionScoped cs(crit_sect_); @@ -844,6 +876,28 @@ uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size, "size", missing_sequence_numbers_.size()); *request_key_frame = !HandleTooLargeNackList(); } + if (max_incomplete_time_ms_ > 0) { + int non_continuous_incomplete_duration = + NonContinuousOrIncompleteDuration(); + if (non_continuous_incomplete_duration > 90 * max_incomplete_time_ms_) { + TRACE_EVENT_INSTANT1("webrtc", "JB::NonContinuousOrIncompleteDuration", + "duration", non_continuous_incomplete_duration); + FrameList::reverse_iterator rit = find_if(frame_list_.rbegin(), + frame_list_.rend(), + KeyFrameCriteria()); + if (rit == frame_list_.rend()) { + // Request a key frame if we don't have one already. + *request_key_frame = true; + *nack_list_size = 0; + return NULL; + } else { + // Skip to the last key frame. If it's incomplete we will start + // NACKing it. + last_decoded_state_.Reset(); + DropPacketsFromNackList(EstimatedLowSequenceNumber(**rit)); + } + } + } unsigned int i = 0; SequenceNumberSet::iterator it = missing_sequence_numbers_.begin(); for (; it != missing_sequence_numbers_.end(); ++it, ++i) { @@ -939,47 +993,78 @@ int64_t VCMJitterBuffer::LastDecodedTimestamp() const { return last_decoded_state_.time_stamp(); } -void VCMJitterBuffer::RenderBufferSize( - uint32_t* timestamp_start, uint32_t* timestamp_end) { - CriticalSectionScoped cs(crit_sect_); - CleanUpOldOrEmptyFrames(); - *timestamp_start = 0u; - *timestamp_end = 0u; - - if (frame_list_.empty()) { - return; - } - FrameList::iterator frame_it = frame_list_.begin(); - VCMFrameBuffer* current_frame = *frame_it; +FrameList::iterator VCMJitterBuffer::FindLastContinuousAndComplete( + FrameList::iterator start_it) { // Search for a complete and continuous sequence (starting from the last // decoded state or current frame if in initial state). VCMDecodingState previous_state; if (last_decoded_state_.in_initial_state()) { - // Start with a key frame. - frame_it = find_if(frame_list_.begin(), frame_list_.end(), - CompleteKeyFrameCriteria()); - if (frame_it == frame_list_.end()) { - return; - } - *timestamp_start = last_decoded_state_.time_stamp(); - current_frame = *frame_it; - previous_state.SetState(current_frame); - ++frame_it; + previous_state.SetState(*start_it); } else { previous_state.CopyFrom(last_decoded_state_); } bool continuous_complete = true; - while (frame_it != frame_list_.end() && continuous_complete) { - current_frame = *frame_it; - continuous_complete = current_frame->IsSessionComplete() && - previous_state.ContinuousFrame(current_frame); - previous_state.SetState(current_frame); - ++frame_it; + FrameList::iterator previous_it = start_it; + ++start_it; + while (start_it != frame_list_.end() && continuous_complete) { + start_it = FindOldestCompleteContinuousFrame(start_it, &previous_state); + if (start_it == frame_list_.end()) + break; + previous_state.SetState(*start_it); + previous_it = start_it; + ++start_it; } // Desired frame is the previous one. - --frame_it; - current_frame = *frame_it; - *timestamp_end = current_frame->TimeStamp(); + return previous_it; +} + +void VCMJitterBuffer::RenderBuffer(FrameList::iterator* start_it, + FrameList::iterator* end_it) { + *start_it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); + if (!decode_with_errors_ && *start_it == frame_list_.end()) { + // No complete continuous frame found. + // Look for a complete key frame if we're not decoding with errors. + *start_it = find_if(frame_list_.begin(), frame_list_.end(), + CompleteKeyFrameCriteria()); + } + if (*start_it == frame_list_.end()) { + *end_it = *start_it; + } else { + *end_it = *start_it; + // Look for the last complete key frame and use that as the end of the + // render buffer it's later than the last complete continuous frame. + FrameList::reverse_iterator rend(*end_it); + FrameList::reverse_iterator rit = find_if(frame_list_.rbegin(), + rend, + CompleteKeyFrameCriteria()); + if (rit != rend) { + // A key frame was found. The reverse iterator base points to the + // frame after it, so subtracting 1. + *end_it = rit.base(); + --*end_it; + } + *end_it = FindLastContinuousAndComplete(*end_it); + } +} + +void VCMJitterBuffer::RenderBufferSize(uint32_t* timestamp_start, + uint32_t* timestamp_end) { + CriticalSectionScoped cs(crit_sect_); + CleanUpOldOrEmptyFrames(); + *timestamp_start = 0; + *timestamp_end = 0; + if (frame_list_.empty()) { + return; + } + FrameList::iterator start_it; + FrameList::iterator end_it; + RenderBuffer(&start_it, &end_it); + if (start_it == frame_list_.end()) { + return; + } + *timestamp_start = (*start_it)->TimeStamp(); + *timestamp_end = (*end_it)->TimeStamp(); } // Set the frame state to free and remove it from the sorted @@ -1031,7 +1116,7 @@ VCMFrameBuffer* VCMJitterBuffer::GetEmptyFrame() { // full. bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() { // Remove up to oldest key frame - while (frame_list_.size() > 0) { + while (!frame_list_.empty()) { // Throw at least one frame. drop_count_++; FrameList::iterator it = frame_list_.begin(); @@ -1043,9 +1128,10 @@ bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() { ReleaseFrameIfNotDecoding(*it); it = frame_list_.erase(it); if (it != frame_list_.end() && (*it)->FrameType() == kVideoFrameKey) { - // Fake the last_decoded_state to match this key frame. - last_decoded_state_.SetStateOneBack(*it); - DropPacketsFromNackList(last_decoded_state_.sequence_num()); + // Reset last decoded state to make sure the next frame decoded is a key + // frame, and start NACKing from here. + last_decoded_state_.Reset(); + DropPacketsFromNackList(EstimatedLowSequenceNumber(**it)); return true; } } @@ -1147,7 +1233,8 @@ VCMFrameBufferEnum VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { assert(false); } } - const FrameList::iterator it = FindOldestCompleteContinuousFrame(); + const FrameList::iterator it = FindOldestCompleteContinuousFrame( + frame_list_.begin(), &last_decoded_state_); VCMFrameBuffer* old_frame = NULL; if (it != frame_list_.end()) { old_frame = *it; @@ -1163,22 +1250,23 @@ VCMFrameBufferEnum VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { // Find oldest complete frame used for getting next frame to decode // Must be called under critical section -FrameList::iterator VCMJitterBuffer::FindOldestCompleteContinuousFrame() { +FrameList::iterator VCMJitterBuffer::FindOldestCompleteContinuousFrame( + FrameList::iterator start_it, + const VCMDecodingState* decoding_state) { // If we have more than one frame done since last time, pick oldest. VCMFrameBuffer* oldest_frame = NULL; - FrameList::iterator it = frame_list_.begin(); // When temporal layers are available, we search for a complete or decodable // frame until we hit one of the following: // 1. Continuous base or sync layer. // 2. The end of the list was reached. - for (; it != frame_list_.end(); ++it) { - oldest_frame = *it; + for (; start_it != frame_list_.end(); ++start_it) { + oldest_frame = *start_it; VCMFrameBufferStateEnum state = oldest_frame->GetState(); // Is this frame complete or decodable and continuous? if ((state == kStateComplete || (decode_with_errors_ && state == kStateDecodable)) && - last_decoded_state_.ContinuousFrame(oldest_frame)) { + decoding_state->ContinuousFrame(oldest_frame)) { break; } else { int temporal_id = oldest_frame->TemporalId(); @@ -1197,7 +1285,7 @@ FrameList::iterator VCMJitterBuffer::FindOldestCompleteContinuousFrame() { } // We have a complete continuous frame. - return it; + return start_it; } // Must be called under the critical section |crit_sect_|. diff --git a/webrtc/modules/video_coding/main/source/jitter_buffer.h b/webrtc/modules/video_coding/main/source/jitter_buffer.h index f4723a957..8cd161618 100644 --- a/webrtc/modules/video_coding/main/source/jitter_buffer.h +++ b/webrtc/modules/video_coding/main/source/jitter_buffer.h @@ -146,7 +146,8 @@ class VCMJitterBuffer { int high_rtt_nack_threshold_ms); void SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack); + int max_packet_age_to_nack, + int max_incomplete_time_ms); // Returns the current NACK mode. VCMNackMode nack_mode() const; @@ -206,7 +207,13 @@ class VCMJitterBuffer { // Finds the oldest complete frame, used for getting next frame to decode. // Can return a decodable, incomplete frame when enabled. - FrameList::iterator FindOldestCompleteContinuousFrame(); + FrameList::iterator FindOldestCompleteContinuousFrame( + FrameList::iterator start_it, + const VCMDecodingState* decoding_state); + FrameList::iterator FindLastContinuousAndComplete( + FrameList::iterator start_it); + void RenderBuffer(FrameList::iterator* start_it, + FrameList::iterator* end_it); // Cleans the frame list in the JB from old/empty frames. // Should only be called prior to actual use. @@ -236,6 +243,10 @@ class VCMJitterBuffer { // Returns true if we should wait for retransmissions, false otherwise. bool WaitForRetransmissions(); + int NonContinuousOrIncompleteDuration(); + + uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const; + int vcm_id_; int receiver_id_; Clock* clock_; @@ -291,6 +302,7 @@ class VCMJitterBuffer { std::vector nack_seq_nums_; size_t max_nack_list_size_; int max_packet_age_to_nack_; // Measured in sequence numbers. + int max_incomplete_time_ms_; bool decode_with_errors_; DISALLOW_COPY_AND_ASSIGN(VCMJitterBuffer); diff --git a/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc b/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc index 2b24597a7..ec3e59d25 100644 --- a/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc +++ b/webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc @@ -131,7 +131,7 @@ class TestRunningJitterBuffer : public ::testing::Test { stream_generator_ = new StreamGenerator(0, 0, clock_->TimeInMilliseconds()); jitter_buffer_->Start(); jitter_buffer_->SetNackSettings(max_nack_list_size_, - oldest_packet_to_nack_); + oldest_packet_to_nack_, 0); memset(data_buffer_, 0, kDataBufferSize); } @@ -1514,6 +1514,30 @@ TEST_F(TestRunningJitterBuffer, SkipToKeyFrame) { EXPECT_TRUE(DecodeCompleteFrame()); } +TEST_F(TestRunningJitterBuffer, DontSkipToKeyFrameIfDecodable) { + InsertFrame(kVideoFrameKey); + EXPECT_TRUE(DecodeCompleteFrame()); + const int kNumDeltaFrames = 5; + EXPECT_GE(InsertFrames(kNumDeltaFrames, kVideoFrameDelta), kNoError); + InsertFrame(kVideoFrameKey); + for (int i = 0; i < kNumDeltaFrames + 1; ++i) { + EXPECT_TRUE(DecodeCompleteFrame()); + } +} + +TEST_F(TestRunningJitterBuffer, KeyDeltaKeyDelta) { + InsertFrame(kVideoFrameKey); + EXPECT_TRUE(DecodeCompleteFrame()); + const int kNumDeltaFrames = 5; + EXPECT_GE(InsertFrames(kNumDeltaFrames, kVideoFrameDelta), kNoError); + InsertFrame(kVideoFrameKey); + EXPECT_GE(InsertFrames(kNumDeltaFrames, kVideoFrameDelta), kNoError); + InsertFrame(kVideoFrameKey); + for (int i = 0; i < 2 * (kNumDeltaFrames + 1); ++i) { + EXPECT_TRUE(DecodeCompleteFrame()); + } +} + TEST_F(TestJitterBufferNack, EmptyPackets) { // Make sure empty packets doesn't clog the jitter buffer. jitter_buffer_->SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); diff --git a/webrtc/modules/video_coding/main/source/receiver.cc b/webrtc/modules/video_coding/main/source/receiver.cc index 38c5db680..c3da7aaeb 100644 --- a/webrtc/modules/video_coding/main/source/receiver.cc +++ b/webrtc/modules/video_coding/main/source/receiver.cc @@ -282,9 +282,11 @@ void VCMReceiver::SetNackMode(VCMNackMode nackMode, } void VCMReceiver::SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack) { + int max_packet_age_to_nack, + int max_incomplete_time_ms) { jitter_buffer_.SetNackSettings(max_nack_list_size, - max_packet_age_to_nack); + max_packet_age_to_nack, + max_incomplete_time_ms); } VCMNackMode VCMReceiver::NackMode() const { @@ -298,16 +300,16 @@ VCMNackStatus VCMReceiver::NackList(uint16_t* nack_list, bool request_key_frame = false; uint16_t* internal_nack_list = jitter_buffer_.GetNackList( nack_list_length, &request_key_frame); - if (request_key_frame) { - // This combination is used to trigger key frame requests. - return kNackKeyFrameRequest; - } if (*nack_list_length > size) { + *nack_list_length = 0; return kNackNeedMoreMemory; } if (internal_nack_list != NULL && *nack_list_length > 0) { memcpy(nack_list, internal_nack_list, *nack_list_length * sizeof(uint16_t)); } + if (request_key_frame) { + return kNackKeyFrameRequest; + } return kNackOk; } diff --git a/webrtc/modules/video_coding/main/source/receiver.h b/webrtc/modules/video_coding/main/source/receiver.h index b478f49d3..6bf76f43f 100644 --- a/webrtc/modules/video_coding/main/source/receiver.h +++ b/webrtc/modules/video_coding/main/source/receiver.h @@ -63,7 +63,8 @@ class VCMReceiver { int low_rtt_nack_threshold_ms, int high_rtt_nack_threshold_ms); void SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack); + int max_packet_age_to_nack, + int max_incomplete_time_ms); VCMNackMode NackMode() const; VCMNackStatus NackList(uint16_t* nackList, uint16_t size, uint16_t* nack_list_length); diff --git a/webrtc/modules/video_coding/main/source/receiver_unittest.cc b/webrtc/modules/video_coding/main/source/receiver_unittest.cc index 92f578c1e..40d0de1f4 100644 --- a/webrtc/modules/video_coding/main/source/receiver_unittest.cc +++ b/webrtc/modules/video_coding/main/source/receiver_unittest.cc @@ -70,10 +70,25 @@ class TestVCMReceiver : public ::testing::Test { (frame_type == kFrameEmpty) ? 1 : 0, clock_->TimeInMilliseconds()); int32_t ret = InsertPacketAndPop(0); + if (!complete) { + // Drop the second packet. + VCMPacket packet; + stream_generator_->PopPacket(&packet, 0); + } clock_->AdvanceTimeMilliseconds(kDefaultFramePeriodMs); return ret; } + bool DecodeNextFrame() { + int64_t render_time_ms = 0; + VCMEncodedFrame* frame = receiver_.FrameForDecoding(0, render_time_ms, + false, NULL); + if (!frame) + return false; + receiver_.ReleaseFrame(frame); + return true; + } + scoped_ptr clock_; VCMTiming timing_; NullEventFactory event_factory_; @@ -85,28 +100,43 @@ class TestVCMReceiver : public ::testing::Test { TEST_F(TestVCMReceiver, RenderBufferSize_AllComplete) { EXPECT_EQ(0, receiver_.RenderBufferSizeMs()); EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); - size_t num_of_frames = 10; - for (size_t i = 0; i < num_of_frames; ++i) { + int num_of_frames = 10; + for (int i = 0; i < num_of_frames; ++i) { EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); } - EXPECT_EQ(static_cast(num_of_frames * kDefaultFramePeriodMs), + EXPECT_EQ(num_of_frames * kDefaultFramePeriodMs, receiver_.RenderBufferSizeMs()); } +TEST_F(TestVCMReceiver, RenderBufferSize_SkipToKeyFrame) { + EXPECT_EQ(0, receiver_.RenderBufferSizeMs()); + const int kNumOfNonDecodableFrames = 2; + for (int i = 0; i < kNumOfNonDecodableFrames; ++i) { + EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); + } + const int kNumOfFrames = 10; + EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); + for (int i = 0; i < kNumOfFrames - 1; ++i) { + EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); + } + EXPECT_EQ((kNumOfFrames - 1) * kDefaultFramePeriodMs, + receiver_.RenderBufferSizeMs()); +} + TEST_F(TestVCMReceiver, RenderBufferSize_NotAllComplete) { EXPECT_EQ(0, receiver_.RenderBufferSizeMs()); EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); - size_t num_of_frames = 10; - for (size_t i = 0; i < num_of_frames; ++i) { + int num_of_frames = 10; + for (int i = 0; i < num_of_frames; ++i) { EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); } num_of_frames++; EXPECT_GE(InsertFrame(kVideoFrameDelta, false), kNoError); - for (size_t i = 0; i < num_of_frames; ++i) { + for (int i = 0; i < num_of_frames; ++i) { EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); } - EXPECT_EQ(static_cast(num_of_frames * kDefaultFramePeriodMs), - receiver_.RenderBufferSizeMs()); + EXPECT_EQ((num_of_frames - 1) * kDefaultFramePeriodMs, + receiver_.RenderBufferSizeMs()); } TEST_F(TestVCMReceiver, RenderBufferSize_NoKeyFrame) { @@ -126,4 +156,145 @@ TEST_F(TestVCMReceiver, RenderBufferSize_NoKeyFrame) { EXPECT_EQ(0, receiver_.RenderBufferSizeMs()); } +TEST_F(TestVCMReceiver, NonDecodableDuration_Empty) { + // Enable NACK and with no RTT thresholds for disabling retransmission delay. + receiver_.SetNackMode(kNack, -1, -1); + const size_t kMaxNackListSize = 1000; + const int kMaxPacketAgeToNack = 1000; + const int kMaxNonDecodableDuration = 500; + const int kMinDelayMs = 500; + receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, + kMaxNonDecodableDuration); + EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); + // Advance time until it's time to decode the key frame. + clock_->AdvanceTimeMilliseconds(kMinDelayMs); + EXPECT_TRUE(DecodeNextFrame()); + uint16_t nack_list[kMaxNackListSize]; + uint16_t nack_list_length = 0; + VCMNackStatus ret = receiver_.NackList(nack_list, kMaxNackListSize, + &nack_list_length); + EXPECT_EQ(kNackOk, ret); +} + +TEST_F(TestVCMReceiver, NonDecodableDuration_NoKeyFrame) { + // Enable NACK and with no RTT thresholds for disabling retransmission delay. + receiver_.SetNackMode(kNack, -1, -1); + const size_t kMaxNackListSize = 1000; + const int kMaxPacketAgeToNack = 1000; + const int kMaxNonDecodableDuration = 500; + receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, + kMaxNonDecodableDuration); + const int kNumFrames = kDefaultFrameRate * kMaxNonDecodableDuration / 1000; + for (int i = 0; i < kNumFrames; ++i) { + EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); + } + uint16_t nack_list[kMaxNackListSize]; + uint16_t nack_list_length = 0; + VCMNackStatus ret = receiver_.NackList(nack_list, kMaxNackListSize, + &nack_list_length); + EXPECT_EQ(kNackKeyFrameRequest, ret); +} + +TEST_F(TestVCMReceiver, NonDecodableDuration_OneIncomplete) { + // Enable NACK and with no RTT thresholds for disabling retransmission delay. + receiver_.SetNackMode(kNack, -1, -1); + const size_t kMaxNackListSize = 1000; + const int kMaxPacketAgeToNack = 1000; + const int kMaxNonDecodableDuration = 500; + const int kMaxNonDecodableDurationFrames = (kDefaultFrameRate * + kMaxNonDecodableDuration + 500) / 1000; + const int kMinDelayMs = 500; + receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, + kMaxNonDecodableDuration); + receiver_.SetMinReceiverDelay(kMinDelayMs); + int64_t key_frame_inserted = clock_->TimeInMilliseconds(); + EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); + // Insert an incomplete frame. + EXPECT_GE(InsertFrame(kVideoFrameDelta, false), kNoError); + // Insert enough frames to have too long non-decodable sequence. + for (int i = 0; i < kMaxNonDecodableDurationFrames; + ++i) { + EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); + } + // Advance time until it's time to decode the key frame. + clock_->AdvanceTimeMilliseconds(kMinDelayMs - clock_->TimeInMilliseconds() - + key_frame_inserted); + EXPECT_TRUE(DecodeNextFrame()); + // Make sure we get a key frame request. + uint16_t nack_list[kMaxNackListSize]; + uint16_t nack_list_length = 0; + VCMNackStatus ret = receiver_.NackList(nack_list, kMaxNackListSize, + &nack_list_length); + EXPECT_EQ(kNackKeyFrameRequest, ret); +} + +TEST_F(TestVCMReceiver, NonDecodableDuration_NoTrigger) { + // Enable NACK and with no RTT thresholds for disabling retransmission delay. + receiver_.SetNackMode(kNack, -1, -1); + const size_t kMaxNackListSize = 1000; + const int kMaxPacketAgeToNack = 1000; + const int kMaxNonDecodableDuration = 500; + const int kMaxNonDecodableDurationFrames = (kDefaultFrameRate * + kMaxNonDecodableDuration + 500) / 1000; + const int kMinDelayMs = 500; + receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, + kMaxNonDecodableDuration); + receiver_.SetMinReceiverDelay(kMinDelayMs); + int64_t key_frame_inserted = clock_->TimeInMilliseconds(); + EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); + // Insert an incomplete frame. + EXPECT_GE(InsertFrame(kVideoFrameDelta, false), kNoError); + // Insert all but one frame to not trigger a key frame request due to + // too long duration of non-decodable frames. + for (int i = 0; i < kMaxNonDecodableDurationFrames - 1; + ++i) { + EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); + } + // Advance time until it's time to decode the key frame. + clock_->AdvanceTimeMilliseconds(kMinDelayMs - clock_->TimeInMilliseconds() - + key_frame_inserted); + EXPECT_TRUE(DecodeNextFrame()); + // Make sure we don't get a key frame request since we haven't generated + // enough frames. + uint16_t nack_list[kMaxNackListSize]; + uint16_t nack_list_length = 0; + VCMNackStatus ret = receiver_.NackList(nack_list, kMaxNackListSize, + &nack_list_length); + EXPECT_EQ(kNackOk, ret); +} + +TEST_F(TestVCMReceiver, NonDecodableDuration_KeyFrameAfterIncompleteFrames) { + // Enable NACK and with no RTT thresholds for disabling retransmission delay. + receiver_.SetNackMode(kNack, -1, -1); + const size_t kMaxNackListSize = 1000; + const int kMaxPacketAgeToNack = 1000; + const int kMaxNonDecodableDuration = 500; + const int kMaxNonDecodableDurationFrames = (kDefaultFrameRate * + kMaxNonDecodableDuration + 500) / 1000; + const int kMinDelayMs = 500; + receiver_.SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, + kMaxNonDecodableDuration); + receiver_.SetMinReceiverDelay(kMinDelayMs); + int64_t key_frame_inserted = clock_->TimeInMilliseconds(); + EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); + // Insert an incomplete frame. + EXPECT_GE(InsertFrame(kVideoFrameDelta, false), kNoError); + // Insert enough frames to have too long non-decodable sequence. + for (int i = 0; i < kMaxNonDecodableDurationFrames; + ++i) { + EXPECT_GE(InsertFrame(kVideoFrameDelta, true), kNoError); + } + EXPECT_GE(InsertFrame(kVideoFrameKey, true), kNoError); + // Advance time until it's time to decode the key frame. + clock_->AdvanceTimeMilliseconds(kMinDelayMs - clock_->TimeInMilliseconds() - + key_frame_inserted); + EXPECT_TRUE(DecodeNextFrame()); + // Make sure we don't get a key frame request since we have a key frame + // in the list. + uint16_t nack_list[kMaxNackListSize]; + uint16_t nack_list_length = 0; + VCMNackStatus ret = receiver_.NackList(nack_list, kMaxNackListSize, + &nack_list_length); + EXPECT_EQ(kNackOk, ret); +} } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/stream_generator.h b/webrtc/modules/video_coding/main/source/stream_generator.h index f500252b8..24eeccc9d 100644 --- a/webrtc/modules/video_coding/main/source/stream_generator.h +++ b/webrtc/modules/video_coding/main/source/stream_generator.h @@ -24,7 +24,7 @@ const unsigned int kDefaultFrameRate = 25; const unsigned int kMaxPacketSize = 1500; const unsigned int kFrameSize = (kDefaultBitrateKbps + kDefaultFrameRate * 4) / (kDefaultFrameRate * 8); -const unsigned int kDefaultFramePeriodMs = 1000 / kDefaultFrameRate; +const int kDefaultFramePeriodMs = 1000 / kDefaultFrameRate; diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.cc b/webrtc/modules/video_coding/main/source/video_coding_impl.cc index aa7cd49b6..881a5db9f 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.cc +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.cc @@ -1452,15 +1452,17 @@ int VideoCodingModuleImpl::SetReceiverRobustnessMode( return VCM_OK; } -void VideoCodingModuleImpl::SetNackSettings( - size_t max_nack_list_size, int max_packet_age_to_nack) { +void VideoCodingModuleImpl::SetNackSettings(size_t max_nack_list_size, + int max_packet_age_to_nack, + int max_incomplete_time_ms) { if (max_nack_list_size != 0) { CriticalSectionScoped cs(_receiveCritSect); max_nack_list_size_ = max_nack_list_size; } - _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack); - _dualReceiver.SetNackSettings(max_nack_list_size, - max_packet_age_to_nack); + _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, + max_incomplete_time_ms); + _dualReceiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, + max_incomplete_time_ms); } int VideoCodingModuleImpl::SetMinReceiverDelay(int desired_delay_ms) { diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.h b/webrtc/modules/video_coding/main/source/video_coding_impl.h index 84972737e..0ffc14f06 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.h +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.h @@ -265,7 +265,8 @@ public: DecodeErrors errorMode); virtual void SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack); + int max_packet_age_to_nack, + int max_incomplete_time_ms); // Set the video delay for the receiver (default = 0). virtual int SetMinReceiverDelay(int desired_delay_ms); diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl_unittest.cc b/webrtc/modules/video_coding/main/source/video_coding_impl_unittest.cc index 7244b3b61..d4fde8ad4 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl_unittest.cc +++ b/webrtc/modules/video_coding/main/source/video_coding_impl_unittest.cc @@ -48,7 +48,7 @@ class TestVideoCodingModule : public ::testing::Test { true)); const size_t kMaxNackListSize = 250; const int kMaxPacketAgeToNack = 450; - vcm_->SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack); + vcm_->SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, 0); memset(&settings_, 0, sizeof(settings_)); EXPECT_EQ(0, vcm_->Codec(kVideoCodecVP8, &settings_)); settings_.numberOfSimulcastStreams = kNumberOfStreams; diff --git a/webrtc/modules/video_coding/main/source/video_coding_robustness_unittest.cc b/webrtc/modules/video_coding/main/source/video_coding_robustness_unittest.cc index 7c99c0868..a259aeebc 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_robustness_unittest.cc +++ b/webrtc/modules/video_coding/main/source/video_coding_robustness_unittest.cc @@ -40,7 +40,7 @@ class VCMRobustnessTest : public ::testing::Test { ASSERT_EQ(0, vcm_->InitializeReceiver()); const size_t kMaxNackListSize = 250; const int kMaxPacketAgeToNack = 450; - vcm_->SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack); + vcm_->SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, 0); ASSERT_EQ(0, vcm_->RegisterFrameTypeCallback(&frame_type_callback_)); ASSERT_EQ(0, vcm_->RegisterPacketRequestCallback(&request_callback_)); ASSERT_EQ(VCM_OK, vcm_->Codec(kVideoCodecVP8, &video_codec_)); diff --git a/webrtc/modules/video_coding/main/test/vcm_payload_sink_factory.cc b/webrtc/modules/video_coding/main/test/vcm_payload_sink_factory.cc index a84561ad7..ccc74e9e8 100644 --- a/webrtc/modules/video_coding/main/test/vcm_payload_sink_factory.cc +++ b/webrtc/modules/video_coding/main/test/vcm_payload_sink_factory.cc @@ -195,7 +195,7 @@ PayloadSinkInterface* VcmPayloadSinkFactory::Create( vcm->SetVideoProtection(protection_method_, protection_enabled_); vcm->SetRenderDelay(render_delay_ms_); vcm->SetMinimumPlayoutDelay(min_playout_delay_ms_); - vcm->SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack); + vcm->SetNackSettings(kMaxNackListSize, kMaxPacketAgeToNack, 0); scoped_ptr frame_receiver( new FileOutputFrameReceiver(base_out_filename_, stream->ssrc())); diff --git a/webrtc/video_engine/vie_channel.cc b/webrtc/video_engine/vie_channel.cc index 7aeb6a14d..679607016 100644 --- a/webrtc/video_engine/vie_channel.cc +++ b/webrtc/video_engine/vie_channel.cc @@ -35,6 +35,7 @@ namespace webrtc { const int kMaxDecodeWaitTimeMs = 50; const int kInvalidRtpExtensionId = 0; static const int kMaxTargetDelayMs = 10000; +static const float kMaxIncompleteTimeMultiplier = 3.5f; // Helper class receiving statistics callbacks. class ChannelStatsObserver : public CallStatsObserver { @@ -121,7 +122,7 @@ ViEChannel::ViEChannel(int32_t channel_id, rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(configuration)); vie_receiver_.SetRtpRtcpModule(rtp_rtcp_.get()); - vcm_.SetNackSettings(kMaxNackListSize, max_nack_reordering_threshold_); + vcm_.SetNackSettings(kMaxNackListSize, max_nack_reordering_threshold_, 0); } int32_t ViEChannel::Init() { @@ -779,15 +780,21 @@ int ViEChannel::SetReceiverBufferingMode(int target_delay_ms) { return -1; } int max_nack_list_size; + int max_incomplete_time_ms; if (target_delay_ms == 0) { // Real-time mode - restore default settings. max_nack_reordering_threshold_ = kMaxPacketAgeToNack; max_nack_list_size = kMaxNackListSize; + max_incomplete_time_ms = 0; } else { max_nack_list_size = 3 * GetRequiredNackListSize(target_delay_ms) / 4; max_nack_reordering_threshold_ = max_nack_list_size; + // Calculate the max incomplete time and round to int. + max_incomplete_time_ms = static_cast(kMaxIncompleteTimeMultiplier * + target_delay_ms + 0.5f); } - vcm_.SetNackSettings(max_nack_list_size, max_nack_reordering_threshold_); + vcm_.SetNackSettings(max_nack_list_size, max_nack_reordering_threshold_, + max_incomplete_time_ms); vcm_.SetMinReceiverDelay(target_delay_ms); if (vie_sync_.SetTargetBufferingDelay(target_delay_ms) < 0) return -1;