generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Revert r4146 "Revert 4104 "Refactor jitter buffer to use separate lists for de...""

Browse Source 
TBR=tnakamura@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/1677004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4232 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
stefan@webrtc.org 2013-06-17 07:13:16 +00:00
parent 7262ad1385
commit c8b29a2feb
7 changed files with 434 additions and 382 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
webrtc/modules/video_coding/main/source/decoding_state.cc
View File

@ -82,11 +82,21 @@ void VCMDecodingState::CopyFrom(const VCMDecodingState& state) {
in_initial_state_ = state.in_initial_state_; in_initial_state_ = state.in_initial_state_;
} }
void VCMDecodingState::UpdateEmptyFrame(const VCMFrameBuffer* frame) { bool VCMDecodingState::UpdateEmptyFrame(const VCMFrameBuffer* frame) {
if (ContinuousFrame(frame)) { bool empty_packet = frame->GetHighSeqNum() == frame->GetLowSeqNum();
time_stamp_ = frame->TimeStamp(); if (in_initial_state_ && empty_packet) {
sequence_num_ = frame->GetHighSeqNum(); // Drop empty packets as long as we are in the initial state.
return true;
} }
if ((empty_packet && ContinuousSeqNum(frame->GetHighSeqNum())) ||
ContinuousFrame(frame)) {
// Continuous empty packets or continuous frames can be dropped if we
// advance the sequence number.
sequence_num_ = frame->GetHighSeqNum();
time_stamp_ = frame->TimeStamp();
return true;
}
return false;
} }
void VCMDecodingState::UpdateOldPacket(const VCMPacket* packet) { void VCMDecodingState::UpdateOldPacket(const VCMPacket* packet) {
@ -139,11 +149,14 @@ bool VCMDecodingState::ContinuousFrame(const VCMFrameBuffer* frame) const {
// Return true when in initial state. // Return true when in initial state.
// Note that when a method is not applicable it will return false. // Note that when a method is not applicable it will return false.
assert(frame != NULL); assert(frame != NULL);
if (in_initial_state_) { // A key frame is always considered continuous as it doesn't refer to any
// Always start with a key frame. // frames and therefore won't introduce any errors even if prior frames are
if (frame->FrameType() == kVideoFrameKey) return true; // missing.
if (frame->FrameType() == kVideoFrameKey)
return true;
// When in the initial state we always require a key frame to start decoding.
if (in_initial_state_)
return false; return false;
}
if (!ContinuousLayer(frame->TemporalId(), frame->Tl0PicId())) { if (!ContinuousLayer(frame->TemporalId(), frame->Tl0PicId())) {
// Base layers are not continuous or temporal layers are inactive. // Base layers are not continuous or temporal layers are inactive.

2
webrtc/modules/video_coding/main/source/decoding_state.h
View File

@ -32,7 +32,7 @@ class VCMDecodingState {
bool ContinuousFrame(const VCMFrameBuffer* frame) const; bool ContinuousFrame(const VCMFrameBuffer* frame) const;
void SetState(const VCMFrameBuffer* frame); void SetState(const VCMFrameBuffer* frame);
void CopyFrom(const VCMDecodingState& state); void CopyFrom(const VCMDecodingState& state);
void UpdateEmptyFrame(const VCMFrameBuffer* frame); bool UpdateEmptyFrame(const VCMFrameBuffer* frame);
// Update the sequence number if the timestamp matches current state and the // Update the sequence number if the timestamp matches current state and the
// sequence number is higher than the current one. This accounts for packets // sequence number is higher than the current one. This accounts for packets
// arriving late. // arriving late.

588
webrtc/modules/video_coding/main/source/jitter_buffer.cc
View File

@ -73,6 +73,82 @@ bool HasNonEmptyState(VCMFrameBuffer* frame) {
return frame->GetState() != kStateEmpty; return frame->GetState() != kStateEmpty;
} }
void FrameList::InsertFrame(VCMFrameBuffer* frame) {
reverse_iterator rit = std::find_if(
rbegin(), rend(), FrameSmallerTimestamp(frame->TimeStamp()));
insert(rit.base(), frame);
}
VCMFrameBuffer* FrameList::FindFrame(uint32_t timestamp) const {
FrameList::const_iterator it = std::find_if(begin(), end(),
FrameEqualTimestamp(timestamp));
if (it == end())
return NULL;
return *it;
}
VCMFrameBuffer* FrameList::PopFrame(uint32_t timestamp) {
FrameList::iterator it = std::find_if(begin(), end(),
FrameEqualTimestamp(timestamp));
if (it == end())
return NULL;
VCMFrameBuffer* frame = *it;
erase(it);
return frame;
}
int FrameList::RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it) {
int drop_count = 0;
*key_frame_it = begin();
while (!empty()) {
// Throw at least one frame.
WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, -1,
"Recycling: type=%s, low seqnum=%u",
(**key_frame_it)->FrameType() == kVideoFrameKey ?
"key" : "delta", (**key_frame_it)->GetLowSeqNum());
if ((**key_frame_it)->GetState() != kStateDecoding) {
(**key_frame_it)->SetState(kStateFree);
}
*key_frame_it = erase(*key_frame_it);
++drop_count;
if (*key_frame_it != end() &&
(**key_frame_it)->FrameType() == kVideoFrameKey) {
return drop_count;
}
}
return drop_count;
}
int FrameList::CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state) {
int drop_count = 0;
while (!empty()) {
VCMFrameBuffer* oldest_frame = front();
bool remove_frame = false;
if (oldest_frame->GetState() == kStateEmpty && size() > 1) {
// This frame is empty, try to update the last decoded state and drop it
// if successful.
remove_frame = decoding_state->UpdateEmptyFrame(oldest_frame);
} else {
remove_frame = decoding_state->IsOldFrame(oldest_frame);
}
if (!remove_frame) {
break;
}
if (front()->GetState() != kStateDecoding) {
front()->SetState(kStateFree);
}
++drop_count;
TRACE_EVENT_INSTANT1("webrtc", "JB::OldOrEmptyFrameDropped", "timestamp",
oldest_frame->TimeStamp());
erase(begin());
}
if (empty()) {
TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied",
"type", "CleanUpOldOrEmptyFrames");
}
return drop_count;
}
VCMJitterBuffer::VCMJitterBuffer(Clock* clock, VCMJitterBuffer::VCMJitterBuffer(Clock* clock,
EventFactory* event_factory, EventFactory* event_factory,
int vcm_id, int vcm_id,
@ -88,7 +164,8 @@ VCMJitterBuffer::VCMJitterBuffer(Clock* clock,
packet_event_(event_factory->CreateEvent()), packet_event_(event_factory->CreateEvent()),
max_number_of_frames_(kStartNumberOfFrames), max_number_of_frames_(kStartNumberOfFrames),
frame_buffers_(), frame_buffers_(),
frame_list_(), decodable_frames_(),
incomplete_frames_(),
last_decoded_state_(), last_decoded_state_(),
first_packet_since_reset_(true), first_packet_since_reset_(true),
num_not_decodable_packets_(0), num_not_decodable_packets_(0),
@ -172,15 +249,18 @@ void VCMJitterBuffer::CopyFrom(const VCMJitterBuffer& rhs) {
frame_buffers_[i] = NULL; frame_buffers_[i] = NULL;
} }
} }
frame_list_.clear(); decodable_frames_.clear();
for (int i = 0; i < max_number_of_frames_; i++) { incomplete_frames_.clear();
frame_buffers_[i] = new VCMFrameBuffer(*(rhs.frame_buffers_[i])); int i = 0;
if (frame_buffers_[i]->Length() > 0) { for (FrameList::const_iterator it = rhs.decodable_frames_.begin();
FrameList::reverse_iterator rit = std::find_if( it != rhs.decodable_frames_.end(); ++it, ++i) {
frame_list_.rbegin(), frame_list_.rend(), frame_buffers_[i] = new VCMFrameBuffer(**it);
FrameSmallerTimestamp(frame_buffers_[i]->TimeStamp())); decodable_frames_.push_back(frame_buffers_[i]);
frame_list_.insert(rit.base(), frame_buffers_[i]);
} }
for (FrameList::const_iterator it = rhs.incomplete_frames_.begin();
it != rhs.incomplete_frames_.end(); ++it, ++i) {
frame_buffers_[i] = new VCMFrameBuffer(**it);
incomplete_frames_.push_back(frame_buffers_[i]);
} }
rhs.crit_sect_->Leave(); rhs.crit_sect_->Leave();
crit_sect_->Leave(); crit_sect_->Leave();
@ -221,7 +301,8 @@ void VCMJitterBuffer::Stop() {
crit_sect_->Enter(); crit_sect_->Enter();
running_ = false; running_ = false;
last_decoded_state_.Reset(); last_decoded_state_.Reset();
frame_list_.clear(); decodable_frames_.clear();
incomplete_frames_.clear();
TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", "type", "Stop"); TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", "type", "Stop");
for (int i = 0; i < kMaxNumberOfFrames; i++) { for (int i = 0; i < kMaxNumberOfFrames; i++) {
if (frame_buffers_[i] != NULL) { if (frame_buffers_[i] != NULL) {
@ -246,7 +327,8 @@ bool VCMJitterBuffer::Running() const {
void VCMJitterBuffer::Flush() { void VCMJitterBuffer::Flush() {
CriticalSectionScoped cs(crit_sect_); CriticalSectionScoped cs(crit_sect_);
// Erase all frames from the sorted list and set their state to free. // Erase all frames from the sorted list and set their state to free.
frame_list_.clear(); decodable_frames_.clear();
incomplete_frames_.clear();
TRACE_EVENT_INSTANT2("webrtc", "JB::FrameListEmptied", "type", "Flush", TRACE_EVENT_INSTANT2("webrtc", "JB::FrameListEmptied", "type", "Flush",
"frames", max_number_of_frames_); "frames", max_number_of_frames_);
for (int i = 0; i < max_number_of_frames_; i++) { for (int i = 0; i < max_number_of_frames_; i++) {
@ -361,32 +443,13 @@ bool VCMJitterBuffer::CompleteSequenceWithNextFrame() {
CriticalSectionScoped cs(crit_sect_); CriticalSectionScoped cs(crit_sect_);
// Finding oldest frame ready for decoder, check sequence number and size // Finding oldest frame ready for decoder, check sequence number and size
CleanUpOldOrEmptyFrames(); CleanUpOldOrEmptyFrames();
if (!decodable_frames_.empty())
if (frame_list_.empty())
return true; return true;
if (incomplete_frames_.size() <= 1) {
VCMFrameBuffer* oldest_frame = frame_list_.front();
if (frame_list_.size() <= 1 &&
oldest_frame->GetState() != kStateComplete) {
// Frame not ready to be decoded. // Frame not ready to be decoded.
return true; return true;
} }
if (oldest_frame->GetState() != kStateComplete) {
return false; return false;
}
// See if we have lost a frame before this one.
if (last_decoded_state_.in_initial_state()) {
// Following start, reset or flush -> check for key frame.
if (oldest_frame->FrameType() != kVideoFrameKey) {
return false;
}
} else if (oldest_frame->GetLowSeqNum() == -1) {
return false;
} else if (!last_decoded_state_.ContinuousFrame(oldest_frame)) {
return false;
}
return true;
} }
// Returns immediately or a |max_wait_time_ms| ms event hang waiting for a // Returns immediately or a |max_wait_time_ms| ms event hang waiting for a
@ -396,13 +459,12 @@ bool VCMJitterBuffer::NextCompleteTimestamp(
TRACE_EVENT0("webrtc", "JB::NextCompleteTimestamp"); TRACE_EVENT0("webrtc", "JB::NextCompleteTimestamp");
crit_sect_->Enter(); crit_sect_->Enter();
if (!running_) { if (!running_) {
crit_sect_->Leave();
return false; return false;
} }
CleanUpOldOrEmptyFrames(); CleanUpOldOrEmptyFrames();
FrameList::iterator it = FindOldestCompleteContinuousFrame( if (decodable_frames_.empty()) {
frame_list_.begin(), &last_decoded_state_);
if (it == frame_list_.end()) {
const int64_t end_wait_time_ms = clock_->TimeInMilliseconds() + const int64_t end_wait_time_ms = clock_->TimeInMilliseconds() +
max_wait_time_ms; max_wait_time_ms;
int64_t wait_time_ms = max_wait_time_ms; int64_t wait_time_ms = max_wait_time_ms;
@ -412,7 +474,7 @@ bool VCMJitterBuffer::NextCompleteTimestamp(
frame_event_->Wait(static_cast<uint32_t>(wait_time_ms)); frame_event_->Wait(static_cast<uint32_t>(wait_time_ms));
crit_sect_->Enter(); crit_sect_->Enter();
if (ret == kEventSignaled) { if (ret == kEventSignaled) {
// Are we closing down the Jitter buffer? // Are we shutting down the jitter buffer?
if (!running_) { if (!running_) {
crit_sect_->Leave(); crit_sect_->Leave();
return false; return false;
@ -420,9 +482,7 @@ bool VCMJitterBuffer::NextCompleteTimestamp(
// Finding oldest frame ready for decoder, but check // Finding oldest frame ready for decoder, but check
// sequence number and size // sequence number and size
CleanUpOldOrEmptyFrames(); CleanUpOldOrEmptyFrames();
it = FindOldestCompleteContinuousFrame( if (decodable_frames_.empty()) {
frame_list_.begin(), &last_decoded_state_);
if (it == frame_list_.end()) {
wait_time_ms = end_wait_time_ms - clock_->TimeInMilliseconds(); wait_time_ms = end_wait_time_ms - clock_->TimeInMilliseconds();
} else { } else {
break; break;
@ -433,31 +493,19 @@ bool VCMJitterBuffer::NextCompleteTimestamp(
} }
// Inside |crit_sect_|. // Inside |crit_sect_|.
} else { } else {
// We already have a frame reset the event. // We already have a frame, reset the event.
frame_event_->Reset(); frame_event_->Reset();
} }
if (decodable_frames_.empty()) {
if (!decode_with_errors_ && it == frame_list_.end()) {
// We're still missing a complete continuous frame.
// Look for a complete key frame if we're not decoding with errors.
it = find_if(frame_list_.begin(), frame_list_.end(),
CompleteKeyFrameCriteria());
}
if (it == frame_list_.end()) {
crit_sect_->Leave(); crit_sect_->Leave();
return false; return false;
} }
*timestamp = decodable_frames_.front()->TimeStamp();
VCMFrameBuffer* oldest_frame = *it;
*timestamp = oldest_frame->TimeStamp();
crit_sect_->Leave(); crit_sect_->Leave();
return true; return true;
} }
bool VCMJitterBuffer::NextMaybeIncompleteTimestamp( bool VCMJitterBuffer::NextMaybeIncompleteTimestamp(uint32_t* timestamp) {
uint32_t* timestamp) {
TRACE_EVENT0("webrtc", "JB::NextMaybeIncompleteTimestamp"); TRACE_EVENT0("webrtc", "JB::NextMaybeIncompleteTimestamp");
CriticalSectionScoped cs(crit_sect_); CriticalSectionScoped cs(crit_sect_);
if (!running_) { if (!running_) {
@ -470,17 +518,17 @@ bool VCMJitterBuffer::NextMaybeIncompleteTimestamp(
CleanUpOldOrEmptyFrames(); CleanUpOldOrEmptyFrames();
if (frame_list_.empty()) { VCMFrameBuffer* oldest_frame = NextFrame();
if (!oldest_frame) {
return false; return false;
} }
if (decodable_frames_.empty() && incomplete_frames_.size() <= 1 &&
VCMFrameBuffer* oldest_frame = frame_list_.front(); oldest_frame->GetState() == kStateIncomplete) {
// If we have only one frame in the buffer, release it only if it is complete. // If we have only one frame in the buffer, release it only if it is
if (frame_list_.size() <= 1 && oldest_frame->GetState() != kStateComplete) { // complete.
return false; return false;
} }
// Always start with a complete key frame.
// Always start with a key frame.
if (last_decoded_state_.in_initial_state() && if (last_decoded_state_.in_initial_state() &&
oldest_frame->FrameType() != kVideoFrameKey) { oldest_frame->FrameType() != kVideoFrameKey) {
return false; return false;
@ -498,18 +546,13 @@ VCMEncodedFrame* VCMJitterBuffer::ExtractAndSetDecode(uint32_t timestamp) {
return NULL; return NULL;
} }
// Extract the frame with the desired timestamp. // Extract the frame with the desired timestamp.
FrameList::iterator it = std::find_if( VCMFrameBuffer* frame = decodable_frames_.PopFrame(timestamp);
frame_list_.begin(), if (!frame) {
frame_list_.end(), frame = incomplete_frames_.PopFrame(timestamp);
FrameEqualTimestamp(timestamp)); if (!frame)
if (it == frame_list_.end()) {
return NULL; return NULL;
} }
// We got the frame. if (!NextFrame()) {
VCMFrameBuffer* frame = *it;
frame_list_.erase(it);
if (frame_list_.empty()) {
TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied",
"type", "ExtractAndSetDecode"); "type", "ExtractAndSetDecode");
} }
@ -585,16 +628,14 @@ VCMFrameBufferEnum VCMJitterBuffer::GetFrame(const VCMPacket& packet,
} }
num_consecutive_old_packets_ = 0; num_consecutive_old_packets_ = 0;
FrameList::iterator it = std::find_if( *frame = incomplete_frames_.FindFrame(packet.timestamp);
frame_list_.begin(), if (*frame) {
frame_list_.end(), return kNoError;
FrameEqualTimestamp(packet.timestamp)); }
*frame = decodable_frames_.FindFrame(packet.timestamp);
if (it != frame_list_.end()) { if (*frame) {
*frame = *it;
return kNoError; return kNoError;
} }
// No match, return empty frame. // No match, return empty frame.
*frame = GetEmptyFrame(); *frame = GetEmptyFrame();
if (*frame != NULL) { if (*frame != NULL) {
@ -640,6 +681,23 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
// reset the delay estimate. // reset the delay estimate.
inter_frame_delay_.Reset(clock_->TimeInMilliseconds()); inter_frame_delay_.Reset(clock_->TimeInMilliseconds());
} }
if (last_decoded_state_.IsOldPacket(&packet)) {
// This packet belongs to an old, already decoded frame, we want to update
// the last decoded sequence number.
last_decoded_state_.UpdateOldPacket(&packet);
frame->SetState(kStateFree);
TRACE_EVENT_INSTANT1("webrtc", "JB::DropLateFrame",
"timestamp", frame->TimeStamp());
drop_count_++;
// Flush() if this happens consistently.
num_consecutive_old_frames_++;
if (num_consecutive_old_frames_ > kMaxConsecutiveOldFrames) {
Flush();
return kFlushIndicator;
}
return kNoError;
}
num_consecutive_old_frames_ = 0;
// Empty packets may bias the jitter estimate (lacking size component), // Empty packets may bias the jitter estimate (lacking size component),
// therefore don't let empty packet trigger the following updates: // therefore don't let empty packet trigger the following updates:
@ -659,7 +717,7 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
} }
} }
VCMFrameBufferStateEnum state = frame->GetState(); VCMFrameBufferStateEnum previous_state = frame->GetState();
// Insert packet. // Insert packet.
// Check for first packet. High sequence number will be -1 if neither an empty // Check for first packet. High sequence number will be -1 if neither an empty
// packet nor a media packet has been inserted. // packet nor a media packet has been inserted.
@ -674,17 +732,6 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
ret = buffer_return; ret = buffer_return;
if (buffer_return > 0) { if (buffer_return > 0) {
incoming_bit_count_ += packet.sizeBytes << 3; incoming_bit_count_ += packet.sizeBytes << 3;
// Insert each frame once on the arrival of the first packet
// belonging to that frame (media or empty).
if (state == kStateEmpty && first) {
ret = kFirstPacket;
FrameList::reverse_iterator rit = std::find_if(
frame_list_.rbegin(),
frame_list_.rend(),
FrameSmallerTimestamp(frame->TimeStamp()));
frame_list_.insert(rit.base(), frame);
}
if (first_packet_since_reset_) { if (first_packet_since_reset_) {
latest_received_sequence_number_ = packet.seqNum; latest_received_sequence_number_ = packet.seqNum;
first_packet_since_reset_ = false; first_packet_since_reset_ = false;
@ -705,9 +752,8 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
case kTimeStampError: case kTimeStampError:
case kSizeError: { case kSizeError: {
if (frame != NULL) { if (frame != NULL) {
// Will be released when it gets old.
frame->Reset(); frame->Reset();
frame->SetState(kStateEmpty); frame->SetState(kStateFree);
} }
break; break;
} }
@ -715,17 +761,36 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
// Don't let the first packet be overridden by a complete session. // Don't let the first packet be overridden by a complete session.
ret = kCompleteSession; ret = kCompleteSession;
// Only update return value for a JB flush indicator. // Only update return value for a JB flush indicator.
if (UpdateFrameState(frame) == kFlushIndicator) UpdateFrameState(frame);
ret = kFlushIndicator;
*retransmitted = (frame->GetNackCount() > 0); *retransmitted = (frame->GetNackCount() > 0);
if (IsContinuous(*frame) && previous_state != kStateComplete) {
if (!first) {
incomplete_frames_.PopFrame(packet.timestamp);
}
decodable_frames_.InsertFrame(frame);
FindAndInsertContinuousFrames(*frame);
// Signal that we have a decodable frame.
frame_event_->Set();
} else if (first) {
incomplete_frames_.InsertFrame(frame);
}
// Signal that we have a received packet. // Signal that we have a received packet.
packet_event_->Set(); packet_event_->Set();
break; break;
} }
case kDecodableSession: case kDecodableSession:
case kIncomplete: { case kIncomplete: {
// Signal that we have a received packet. // No point in storing empty continuous frames.
if (frame->GetState() == kStateEmpty &&
last_decoded_state_.UpdateEmptyFrame(frame)) {
frame->SetState(kStateFree);
ret = kNoError;
} else if (first) {
ret = kFirstPacket;
incomplete_frames_.InsertFrame(frame);
// Signal that we have received a packet.
packet_event_->Set(); packet_event_->Set();
}
break; break;
} }
case kNoError: case kNoError:
@ -742,6 +807,66 @@ VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
return ret; return ret;
} }
bool VCMJitterBuffer::IsContinuousInState(const VCMFrameBuffer& frame,
const VCMDecodingState& decoding_state) const {
// Is this frame complete or decodable and continuous?
if ((frame.GetState() == kStateComplete ||
(decode_with_errors_ && frame.GetState() == kStateDecodable)) &&
decoding_state.ContinuousFrame(&frame)) {
return true;
} else {
return false;
}
}
bool VCMJitterBuffer::IsContinuous(const VCMFrameBuffer& frame) const {
if (IsContinuousInState(frame, last_decoded_state_)) {
return true;
}
VCMDecodingState decoding_state;
decoding_state.CopyFrom(last_decoded_state_);
for (FrameList::const_iterator it = decodable_frames_.begin();
it != decodable_frames_.end(); ++it) {
VCMFrameBuffer* decodable_frame = *it;
if (IsNewerTimestamp(decodable_frame->TimeStamp(), frame.TimeStamp())) {
break;
}
decoding_state.SetState(decodable_frame);
if (IsContinuousInState(frame, decoding_state)) {
return true;
}
}
return false;
}
void VCMJitterBuffer::FindAndInsertContinuousFrames(
const VCMFrameBuffer& new_frame) {
VCMDecodingState decoding_state;
decoding_state.CopyFrom(last_decoded_state_);
decoding_state.SetState(&new_frame);
// When temporal layers are available, we search for a complete or decodable
// frame until we hit one of the following:
// 1. Continuous base or sync layer.
// 2. The end of the list was reached.
for (FrameList::iterator it = incomplete_frames_.begin();
it != incomplete_frames_.end();) {
VCMFrameBuffer* frame = *it;
if (IsNewerTimestamp(new_frame.TimeStamp(), frame->TimeStamp())) {
++it;
continue;
}
if (IsContinuousInState(*frame, decoding_state)) {
decodable_frames_.InsertFrame(frame);
it = incomplete_frames_.erase(it);
decoding_state.SetState(frame);
} else if (frame->TemporalId() <= 0) {
break;
} else {
++it;
}
}
}
void VCMJitterBuffer::SetMaxJitterEstimate(bool enable) { void VCMJitterBuffer::SetMaxJitterEstimate(bool enable) {
CriticalSectionScoped cs(crit_sect_); CriticalSectionScoped cs(crit_sect_);
jitter_estimate_.SetMaxJitterEstimate(enable); jitter_estimate_.SetMaxJitterEstimate(enable);
@ -809,16 +934,14 @@ VCMNackMode VCMJitterBuffer::nack_mode() const {
} }
int VCMJitterBuffer::NonContinuousOrIncompleteDuration() { int VCMJitterBuffer::NonContinuousOrIncompleteDuration() {
if (frame_list_.empty()) { if (incomplete_frames_.empty()) {
return 0; return 0;
} }
FrameList::iterator start_it; uint32_t start_timestamp = incomplete_frames_.front()->TimeStamp();
FrameList::iterator end_it; if (!decodable_frames_.empty()) {
RenderBuffer(&start_it, &end_it); start_timestamp = decodable_frames_.back()->TimeStamp();
if (end_it == frame_list_.end()) }
end_it = frame_list_.begin(); return incomplete_frames_.back()->TimeStamp() - start_timestamp;
return frame_list_.back()->TimeStamp() -
(*end_it)->TimeStamp();
} }
uint16_t VCMJitterBuffer::EstimatedLowSequenceNumber( uint16_t VCMJitterBuffer::EstimatedLowSequenceNumber(
@ -841,12 +964,18 @@ uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size,
return NULL; return NULL;
} }
if (last_decoded_state_.in_initial_state()) { if (last_decoded_state_.in_initial_state()) {
bool first_frame_is_key = !frame_list_.empty() && const bool first_frame_is_key = NextFrame() &&
frame_list_.front()->FrameType() == kVideoFrameKey && NextFrame()->FrameType() == kVideoFrameKey &&
frame_list_.front()->HaveFirstPacket(); NextFrame()->HaveFirstPacket();
if (!first_frame_is_key) { if (!first_frame_is_key) {
const bool have_non_empty_frame = frame_list_.end() != find_if( bool have_non_empty_frame = decodable_frames_.end() != find_if(
frame_list_.begin(), frame_list_.end(), HasNonEmptyState); decodable_frames_.begin(), decodable_frames_.end(),
HasNonEmptyState);
if (!have_non_empty_frame) {
have_non_empty_frame = incomplete_frames_.end() != find_if(
incomplete_frames_.begin(), incomplete_frames_.end(),
HasNonEmptyState);
}
if (have_non_empty_frame) { if (have_non_empty_frame) {
LOG_F(LS_INFO) << "First frame is not key; Recycling."; LOG_F(LS_INFO) << "First frame is not key; Recycling.";
} }
@ -872,10 +1001,9 @@ uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size,
LOG_F(LS_INFO) << "Too long non-decodable duration: " << LOG_F(LS_INFO) << "Too long non-decodable duration: " <<
non_continuous_incomplete_duration << " > " << non_continuous_incomplete_duration << " > " <<
90 * max_incomplete_time_ms_; 90 * max_incomplete_time_ms_;
FrameList::reverse_iterator rit = find_if(frame_list_.rbegin(), FrameList::reverse_iterator rit = find_if(incomplete_frames_.rbegin(),
frame_list_.rend(), incomplete_frames_.rend(), KeyFrameCriteria());
KeyFrameCriteria()); if (rit == incomplete_frames_.rend()) {
if (rit == frame_list_.rend()) {
// Request a key frame if we don't have one already. // Request a key frame if we don't have one already.
*request_key_frame = true; *request_key_frame = true;
*nack_list_size = 0; *nack_list_size = 0;
@ -899,6 +1027,14 @@ uint16_t* VCMJitterBuffer::GetNackList(uint16_t* nack_list_size,
return &nack_seq_nums_[0]; return &nack_seq_nums_[0];
} }
VCMFrameBuffer* VCMJitterBuffer::NextFrame() const {
if (!decodable_frames_.empty())
return decodable_frames_.front();
if (!incomplete_frames_.empty())
return incomplete_frames_.front();
return NULL;
}
bool VCMJitterBuffer::UpdateNackList(uint16_t sequence_number) { bool VCMJitterBuffer::UpdateNackList(uint16_t sequence_number) {
if (nack_mode_ == kNoNack) { if (nack_mode_ == kNoNack) {
return true; return true;
@ -987,73 +1123,17 @@ int64_t VCMJitterBuffer::LastDecodedTimestamp() const {
return last_decoded_state_.time_stamp(); return last_decoded_state_.time_stamp();
} }
FrameList::iterator VCMJitterBuffer::FindLastContinuousAndComplete(
FrameList::iterator start_it) {
// Search for a complete and continuous sequence (starting from the last
// decoded state or current frame if in initial state).
VCMDecodingState previous_state;
previous_state.SetState(*start_it);
FrameList::iterator previous_it = start_it;
++start_it;
while (start_it != frame_list_.end()) {
start_it = FindOldestCompleteContinuousFrame(start_it, &previous_state);
if (start_it == frame_list_.end())
break;
previous_state.SetState(*start_it);
previous_it = start_it;
++start_it;
}
// Desired frame is the previous one.
return previous_it;
}
void VCMJitterBuffer::RenderBuffer(FrameList::iterator* start_it,
FrameList::iterator* end_it) {
*start_it = FindOldestCompleteContinuousFrame(
frame_list_.begin(), &last_decoded_state_);
if (!decode_with_errors_ && *start_it == frame_list_.end()) {
// No complete continuous frame found.
// Look for a complete key frame if we're not decoding with errors.
*start_it = find_if(frame_list_.begin(), frame_list_.end(),
CompleteKeyFrameCriteria());
}
if (*start_it == frame_list_.end()) {
*end_it = *start_it;
} else {
*end_it = *start_it;
// Look for the last complete key frame and use that as the end of the
// render buffer it's later than the last complete continuous frame.
FrameList::reverse_iterator rend(*end_it);
FrameList::reverse_iterator rit = find_if(frame_list_.rbegin(),
rend,
CompleteKeyFrameCriteria());
if (rit != rend) {
// A key frame was found. The reverse iterator base points to the
// frame after it, so subtracting 1.
*end_it = rit.base();
--*end_it;
}
*end_it = FindLastContinuousAndComplete(*end_it);
}
}
void VCMJitterBuffer::RenderBufferSize(uint32_t* timestamp_start, void VCMJitterBuffer::RenderBufferSize(uint32_t* timestamp_start,
uint32_t* timestamp_end) { uint32_t* timestamp_end) {
CriticalSectionScoped cs(crit_sect_); CriticalSectionScoped cs(crit_sect_);
CleanUpOldOrEmptyFrames(); CleanUpOldOrEmptyFrames();
*timestamp_start = 0; *timestamp_start = 0;
*timestamp_end = 0; *timestamp_end = 0;
if (frame_list_.empty()) { if (decodable_frames_.empty()) {
return; return;
} }
FrameList::iterator start_it; *timestamp_start = decodable_frames_.front()->TimeStamp();
FrameList::iterator end_it; *timestamp_end = decodable_frames_.back()->TimeStamp();
RenderBuffer(&start_it, &end_it);
if (start_it == frame_list_.end()) {
return;
}
*timestamp_start = (*start_it)->TimeStamp();
*timestamp_end = (*end_it)->TimeStamp();
} }
// Set the frame state to free and remove it from the sorted // Set the frame state to free and remove it from the sorted
@ -1095,48 +1175,42 @@ VCMFrameBuffer* VCMJitterBuffer::GetEmptyFrame() {
// Recycle oldest frames up to a key frame, used if jitter buffer is completely // Recycle oldest frames up to a key frame, used if jitter buffer is completely
// full. // full.
bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() { bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() {
// Remove up to oldest key frame // First release incomplete frames, and only release decodable frames if there
while (!frame_list_.empty()) { // are no incomplete ones.
// Throw at least one frame. FrameList::iterator key_frame_it;
drop_count_++; bool key_frame_found = false;
FrameList::iterator it = frame_list_.begin(); int dropped_frames = 0;
dropped_frames += incomplete_frames_.RecycleFramesUntilKeyFrame(
&key_frame_it);
key_frame_found = key_frame_it != incomplete_frames_.end();
if (dropped_frames == 0) {
dropped_frames += decodable_frames_.RecycleFramesUntilKeyFrame(
&key_frame_it);
key_frame_found = key_frame_it != decodable_frames_.end();
if (!key_frame_found) {
TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied", "type",
"RecycleFramesUntilKeyFrame");
}
}
drop_count_ += dropped_frames;
WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding,
VCMId(vcm_id_, receiver_id_), VCMId(vcm_id_, receiver_id_),
"Jitter buffer drop count:%d, low_seq %d, frame type: %s", "Jitter buffer drop count:%u", drop_count_);
drop_count_, (*it)->GetLowSeqNum(),
(*it)->FrameType() == kVideoFrameKey ? "key" : "delta");
TRACE_EVENT_INSTANT0("webrtc", "JB::RecycleFramesUntilKeyFrame"); TRACE_EVENT_INSTANT0("webrtc", "JB::RecycleFramesUntilKeyFrame");
ReleaseFrameIfNotDecoding(*it); if (key_frame_found) {
it = frame_list_.erase(it);
if (it != frame_list_.end() && (*it)->FrameType() == kVideoFrameKey) {
// Reset last decoded state to make sure the next frame decoded is a key // Reset last decoded state to make sure the next frame decoded is a key
// frame, and start NACKing from here. // frame, and start NACKing from here.
// Note that the estimated low sequence number is correct for VP8
// streams because only the first packet of a key frame is marked.
last_decoded_state_.Reset(); last_decoded_state_.Reset();
DropPacketsFromNackList(EstimatedLowSequenceNumber(**it)); DropPacketsFromNackList(EstimatedLowSequenceNumber(**key_frame_it));
return true; } else if (decodable_frames_.empty()) {
}
}
if (frame_list_.empty()) {
TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied",
"type", "RecycleFramesUntilKeyFrame");
}
last_decoded_state_.Reset(); // TODO(mikhal): No sync. last_decoded_state_.Reset(); // TODO(mikhal): No sync.
missing_sequence_numbers_.clear(); missing_sequence_numbers_.clear();
return false; }
return key_frame_found;
} }
// Must be called under the critical section |crit_sect_|. // Must be called under the critical section |crit_sect_|.
VCMFrameBufferEnum VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) {
if (frame == NULL) {
WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding,
VCMId(vcm_id_, receiver_id_), "JB(0x%x) FB(0x%x): "
"UpdateFrameState NULL frame pointer", this, frame);
return kNoError;
}
int length = frame->Length();
if (master_) { if (master_) {
// Only trace the primary jitter buffer to make it possible to parse // Only trace the primary jitter buffer to make it possible to parse
// and plot the trace file. // and plot the trace file.
@ -1144,43 +1218,17 @@ VCMFrameBufferEnum VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) {
VCMId(vcm_id_, receiver_id_), VCMId(vcm_id_, receiver_id_),
"JB(0x%x) FB(0x%x): Complete frame added to jitter buffer," "JB(0x%x) FB(0x%x): Complete frame added to jitter buffer,"
" size:%d type %d", " size:%d type %d",
this, frame, length, frame->FrameType()); this, frame, frame->Length(), frame->FrameType());
} }
bool frame_counted = false; bool frame_counted = false;
if (length != 0 && !frame->GetCountedFrame()) { if (!frame->GetCountedFrame()) {
// Ignore ACK frames. // Ignore ACK frames.
incoming_frame_count_++; incoming_frame_count_++;
frame->SetCountedFrame(true); frame->SetCountedFrame(true);
frame_counted = true; frame_counted = true;
} }
// Check if we should drop the frame. A complete frame can arrive too late.
if (last_decoded_state_.IsOldFrame(frame)) {
// Frame is older than the latest decoded frame, drop it. Will be
// released by CleanUpOldFrames later.
TRACE_EVENT_INSTANT1("webrtc", "JB::DropLateFrame",
"timestamp", frame->TimeStamp());
frame->Reset();
frame->SetState(kStateEmpty);
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding,
VCMId(vcm_id_, receiver_id_),
"JB(0x%x) FB(0x%x): Dropping old frame in Jitter buffer",
this, frame);
drop_count_++;
WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding,
VCMId(vcm_id_, receiver_id_),
"Jitter buffer drop count: %d, consecutive drops: %u",
drop_count_, num_consecutive_old_frames_);
// Flush() if this happens consistently.
num_consecutive_old_frames_++;
if (num_consecutive_old_frames_ > kMaxConsecutiveOldFrames) {
Flush();
return kFlushIndicator;
}
return kNoError;
}
num_consecutive_old_frames_ = 0;
frame->SetState(kStateComplete); frame->SetState(kStateComplete);
if (frame->FrameType() == kVideoFrameKey) { if (frame->FrameType() == kVideoFrameKey) {
TRACE_EVENT_INSTANT2("webrtc", "JB::AddKeyFrame", TRACE_EVENT_INSTANT2("webrtc", "JB::AddKeyFrame",
@ -1216,83 +1264,15 @@ VCMFrameBufferEnum VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) {
assert(false); assert(false);
} }
} }
const FrameList::iterator it = FindOldestCompleteContinuousFrame(
frame_list_.begin(), &last_decoded_state_);
VCMFrameBuffer* old_frame = NULL;
if (it != frame_list_.end()) {
old_frame = *it;
}
// Only signal if this is the oldest frame.
// Not necessarily the case due to packet reordering or NACK.
if (!WaitForRetransmissions() || (old_frame != NULL && old_frame == frame)) {
frame_event_->Set();
}
return kNoError;
}
// Find oldest complete frame used for getting next frame to decode
// Must be called under critical section
FrameList::iterator VCMJitterBuffer::FindOldestCompleteContinuousFrame(
FrameList::iterator start_it,
const VCMDecodingState* decoding_state) {
// If we have more than one frame done since last time, pick oldest.
VCMFrameBuffer* oldest_frame = NULL;
// When temporal layers are available, we search for a complete or decodable
// frame until we hit one of the following:
// 1. Continuous base or sync layer.
// 2. The end of the list was reached.
for (; start_it != frame_list_.end(); ++start_it) {
oldest_frame = *start_it;
VCMFrameBufferStateEnum state = oldest_frame->GetState();
// Is this frame complete or decodable and continuous?
if ((state == kStateComplete ||
(decode_with_errors_ && state == kStateDecodable)) &&
decoding_state->ContinuousFrame(oldest_frame)) {
break;
} else {
int temporal_id = oldest_frame->TemporalId();
oldest_frame = NULL;
if (temporal_id <= 0) {
// When temporal layers are disabled or we have hit a base layer
// we break (regardless of continuity and completeness).
break;
}
}
}
if (oldest_frame == NULL) {
// No complete frame no point to continue.
return frame_list_.end();
}
// We have a complete continuous frame.
return start_it;
} }
// Must be called under the critical section |crit_sect_|. // Must be called under the critical section |crit_sect_|.
void VCMJitterBuffer::CleanUpOldOrEmptyFrames() { void VCMJitterBuffer::CleanUpOldOrEmptyFrames() {
while (frame_list_.size() > 0) { drop_count_ +=
VCMFrameBuffer* oldest_frame = frame_list_.front(); decodable_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_);
if (oldest_frame->GetState() == kStateEmpty && frame_list_.size() > 1) { drop_count_ +=
// This frame is empty, mark it as decoded, thereby making it old. incomplete_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_);
last_decoded_state_.UpdateEmptyFrame(oldest_frame);
}
if (last_decoded_state_.IsOldFrame(oldest_frame)) {
ReleaseFrameIfNotDecoding(frame_list_.front());
TRACE_EVENT_INSTANT1("webrtc", "JB::OldFrameDropped",
"timestamp", oldest_frame->TimeStamp());
TRACE_COUNTER1("webrtc", "JBDroppedLateFrames", drop_count_); TRACE_COUNTER1("webrtc", "JBDroppedLateFrames", drop_count_);
frame_list_.erase(frame_list_.begin());
} else {
break;
}
}
if (frame_list_.empty()) {
TRACE_EVENT_INSTANT1("webrtc", "JB::FrameListEmptied",
"type", "CleanUpOldOrEmptyFrames");
}
if (!last_decoded_state_.in_initial_state()) { if (!last_decoded_state_.in_initial_state()) {
DropPacketsFromNackList(last_decoded_state_.sequence_num()); DropPacketsFromNackList(last_decoded_state_.sequence_num());
} }

43
webrtc/modules/video_coding/main/source/jitter_buffer.h
View File

@ -32,8 +32,6 @@ enum VCMNackMode {
kNoNack kNoNack
}; };
typedef std::list<VCMFrameBuffer*> FrameList;
// forward declarations // forward declarations
class Clock; class Clock;
class EventFactory; class EventFactory;
@ -49,6 +47,15 @@ struct VCMJitterSample {
int64_t latest_packet_time; int64_t latest_packet_time;
}; };
class FrameList : public std::list<VCMFrameBuffer*> {
public:
void InsertFrame(VCMFrameBuffer* frame);
VCMFrameBuffer* FindFrame(uint32_t timestamp) const;
VCMFrameBuffer* PopFrame(uint32_t timestamp);
int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it);
int CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state);
};
class VCMJitterBuffer { class VCMJitterBuffer {
public: public:
VCMJitterBuffer(Clock* clock, VCMJitterBuffer(Clock* clock,
@ -120,6 +127,8 @@ class VCMJitterBuffer {
bool* retransmitted) const; bool* retransmitted) const;
// Inserts a packet into a frame returned from GetFrame(). // Inserts a packet into a frame returned from GetFrame().
// If the return value is <= 0, |frame| is invalidated and the pointer must
// be dropped after this function returns.
VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, VCMFrameBufferEnum InsertPacket(const VCMPacket& packet,
bool* retransmitted); bool* retransmitted);
@ -175,6 +184,18 @@ class VCMJitterBuffer {
// existing frames if no free frames are available. Returns an error code if // existing frames if no free frames are available. Returns an error code if
// failing, or kNoError on success. // failing, or kNoError on success.
VCMFrameBufferEnum GetFrame(const VCMPacket& packet, VCMFrameBuffer** frame); VCMFrameBufferEnum GetFrame(const VCMPacket& packet, VCMFrameBuffer** frame);
// Returns true if |frame| is continuous in |decoding_state|, not taking
// decodable frames into account.
bool IsContinuousInState(const VCMFrameBuffer& frame,
const VCMDecodingState& decoding_state) const;
// Returns true if |frame| is continuous in the |last_decoded_state_|, taking
// all decodable frames into account.
bool IsContinuous(const VCMFrameBuffer& frame) const;
// Looks for frames in |incomplete_frames_| which are continuous in
// |last_decoded_state_| taking all decodable frames into account. Starts
// the search from |new_frame|.
void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame);
VCMFrameBuffer* NextFrame() const;
// Returns true if the NACK list was updated to cover sequence numbers up to // Returns true if the NACK list was updated to cover sequence numbers up to
// |sequence_number|. If false a key frame is needed to get into a state where // |sequence_number|. If false a key frame is needed to get into a state where
// we can continue decoding. // we can continue decoding.
@ -202,19 +223,8 @@ class VCMJitterBuffer {
bool RecycleFramesUntilKeyFrame(); bool RecycleFramesUntilKeyFrame();
// Sets the state of |frame| to complete if it's not too old to be decoded. // Sets the state of |frame| to complete if it's not too old to be decoded.
// Also updates the frame statistics. Signals the |frame_event| if this is // Also updates the frame statistics.
// the next frame to be decoded. void UpdateFrameState(VCMFrameBuffer* frame);
VCMFrameBufferEnum UpdateFrameState(VCMFrameBuffer* frame);
// Finds the oldest complete frame, used for getting next frame to decode.
// Can return a decodable, incomplete frame when enabled.
FrameList::iterator FindOldestCompleteContinuousFrame(
FrameList::iterator start_it,
const VCMDecodingState* decoding_state);
FrameList::iterator FindLastContinuousAndComplete(
FrameList::iterator start_it);
void RenderBuffer(FrameList::iterator* start_it,
FrameList::iterator* end_it);
// Cleans the frame list in the JB from old/empty frames. // Cleans the frame list in the JB from old/empty frames.
// Should only be called prior to actual use. // Should only be called prior to actual use.
@ -263,7 +273,8 @@ class VCMJitterBuffer {
int max_number_of_frames_; int max_number_of_frames_;
// Array of pointers to the frames in jitter buffer. // Array of pointers to the frames in jitter buffer.
VCMFrameBuffer* frame_buffers_[kMaxNumberOfFrames]; VCMFrameBuffer* frame_buffers_[kMaxNumberOfFrames];
FrameList frame_list_; FrameList decodable_frames_;
FrameList incomplete_frames_;
VCMDecodingState last_decoded_state_; VCMDecodingState last_decoded_state_;
bool first_packet_since_reset_; bool first_packet_since_reset_;

115
webrtc/modules/video_coding/main/source/jitter_buffer_unittest.cc
View File

@ -69,10 +69,10 @@ class TestBasicJitterBuffer : public ::testing::Test {
VCMEncodedFrame* frame = jitter_buffer_->ExtractAndSetDecode(timestamp); VCMEncodedFrame* frame = jitter_buffer_->ExtractAndSetDecode(timestamp);
return frame; return frame;
} }
int CheckOutFrame(VCMEncodedFrame* frame_out, void CheckOutFrame(VCMEncodedFrame* frame_out,
unsigned int size, unsigned int size,
bool startCode) { bool startCode) {
EXPECT_FALSE(frame_out == NULL); ASSERT_TRUE(frame_out);
const uint8_t* outData = frame_out->Buffer(); const uint8_t* outData = frame_out->Buffer();
unsigned int i = 0; unsigned int i = 0;
@ -104,7 +104,6 @@ class TestBasicJitterBuffer : public ::testing::Test {
} }
} }
} }
return 0;
} }
uint16_t seq_num_; uint16_t seq_num_;
@ -189,6 +188,8 @@ class TestRunningJitterBuffer : public ::testing::Test {
void DropFrame(int num_packets) { void DropFrame(int num_packets) {
stream_generator_->GenerateFrame(kVideoFrameDelta, num_packets, 0, stream_generator_->GenerateFrame(kVideoFrameDelta, num_packets, 0,
clock_->TimeInMilliseconds()); clock_->TimeInMilliseconds());
for (int i = 0; i < num_packets; ++i)
stream_generator_->DropLastPacket();
clock_->AdvanceTimeMilliseconds(kDefaultFramePeriodMs); clock_->AdvanceTimeMilliseconds(kDefaultFramePeriodMs);
} }
@ -261,7 +262,7 @@ TEST_F(TestBasicJitterBuffer, SinglePacketFrame) {
EXPECT_EQ(kCompleteSession, jitter_buffer_->InsertPacket(*packet_, EXPECT_EQ(kCompleteSession, jitter_buffer_->InsertPacket(*packet_,
&retransmitted)); &retransmitted));
VCMEncodedFrame* frame_out = DecodeCompleteFrame(); VCMEncodedFrame* frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); CheckOutFrame(frame_out, size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -286,7 +287,7 @@ TEST_F(TestBasicJitterBuffer, DualPacketFrame) {
&retransmitted)); &retransmitted));
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); CheckOutFrame(frame_out, 2 * size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -329,7 +330,7 @@ TEST_F(TestBasicJitterBuffer, 100PacketKeyFrame) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); CheckOutFrame(frame_out, 100 * size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -383,7 +384,7 @@ TEST_F(TestBasicJitterBuffer, 100PacketDeltaFrame) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); CheckOutFrame(frame_out, 100 * size_, false);
EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType()); EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType());
} }
@ -428,7 +429,7 @@ TEST_F(TestBasicJitterBuffer, PacketReorderingReverseOrder) {
frame_out = DecodeCompleteFrame();; frame_out = DecodeCompleteFrame();;
EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); CheckOutFrame(frame_out, 100 * size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -484,7 +485,7 @@ TEST_F(TestBasicJitterBuffer, FrameReordering2Frames2PacketsEach) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); CheckOutFrame(frame_out, 2 * size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
@ -492,7 +493,7 @@ TEST_F(TestBasicJitterBuffer, FrameReordering2Frames2PacketsEach) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); CheckOutFrame(frame_out, 2 * size_, false);
EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType()); EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType());
} }
@ -527,7 +528,7 @@ TEST_F(TestBasicJitterBuffer, DuplicatePackets) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); CheckOutFrame(frame_out, 2 * size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -559,7 +560,7 @@ TEST_F(TestBasicJitterBuffer, H264InsertStartCode) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, size_ * 2 + 4 * 2, true)); CheckOutFrame(frame_out, size_ * 2 + 4 * 2, true);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -631,7 +632,7 @@ TEST_F(TestBasicJitterBuffer, PacketLoss) {
// One of the packets has been discarded by the jitter buffer. // One of the packets has been discarded by the jitter buffer.
// Last frame can't be extracted yet. // Last frame can't be extracted yet.
if (i < 10) { if (i < 10) {
EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); CheckOutFrame(frame_out, size_, false);
if (i == 0) { if (i == 0) {
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
@ -716,7 +717,7 @@ TEST_F(TestBasicJitterBuffer, DeltaFrame100PacketsWithSeqNumWrap) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); CheckOutFrame(frame_out, 100 * size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -765,7 +766,7 @@ TEST_F(TestBasicJitterBuffer, PacketReorderingReverseWithNegSeqNumWrap) {
&retransmitted)); &retransmitted));
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 100 * size_, false)); CheckOutFrame(frame_out, 100 * size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -789,7 +790,7 @@ TEST_F(TestBasicJitterBuffer, TestInsertOldFrame) {
VCMEncodedFrame* frame_out = DecodeCompleteFrame(); VCMEncodedFrame* frame_out = DecodeCompleteFrame();
EXPECT_EQ(3000u, frame_out->TimeStamp()); EXPECT_EQ(3000u, frame_out->TimeStamp());
EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); CheckOutFrame(frame_out, size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
@ -828,7 +829,7 @@ TEST_F(TestBasicJitterBuffer, TestInsertOldFrameWithSeqNumWrap) {
VCMEncodedFrame* frame_out = DecodeCompleteFrame(); VCMEncodedFrame* frame_out = DecodeCompleteFrame();
EXPECT_EQ(timestamp_, frame_out->TimeStamp()); EXPECT_EQ(timestamp_, frame_out->TimeStamp());
EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); CheckOutFrame(frame_out, size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
@ -879,7 +880,7 @@ TEST_F(TestBasicJitterBuffer, TimestampWrap) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); CheckOutFrame(frame_out, 2 * size_, false);
jitter_buffer_->ReleaseFrame(frame_out); jitter_buffer_->ReleaseFrame(frame_out);
@ -908,7 +909,7 @@ TEST_F(TestBasicJitterBuffer, TimestampWrap) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, 2 * size_, false)); CheckOutFrame(frame_out, 2 * size_, false);
EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType()); EXPECT_EQ(kVideoFrameDelta, frame_out->FrameType());
} }
@ -945,14 +946,14 @@ TEST_F(TestBasicJitterBuffer, 2FrameWithTimestampWrap) {
VCMEncodedFrame* frame_out = DecodeCompleteFrame(); VCMEncodedFrame* frame_out = DecodeCompleteFrame();
EXPECT_EQ(0xffffff00, frame_out->TimeStamp()); EXPECT_EQ(0xffffff00, frame_out->TimeStamp());
EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); CheckOutFrame(frame_out, size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
VCMEncodedFrame* frame_out2 = DecodeCompleteFrame(); VCMEncodedFrame* frame_out2 = DecodeCompleteFrame();
EXPECT_EQ(2700u, frame_out2->TimeStamp()); EXPECT_EQ(2700u, frame_out2->TimeStamp());
EXPECT_EQ(0, CheckOutFrame(frame_out2, size_, false)); CheckOutFrame(frame_out2, size_, false);
EXPECT_EQ(kVideoFrameDelta, frame_out2->FrameType()); EXPECT_EQ(kVideoFrameDelta, frame_out2->FrameType());
} }
@ -990,14 +991,14 @@ TEST_F(TestBasicJitterBuffer, Insert2FramesReOrderedWithTimestampWrap) {
VCMEncodedFrame* frame_out = DecodeCompleteFrame(); VCMEncodedFrame* frame_out = DecodeCompleteFrame();
EXPECT_EQ(0xffffff00, frame_out->TimeStamp()); EXPECT_EQ(0xffffff00, frame_out->TimeStamp());
EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); CheckOutFrame(frame_out, size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
VCMEncodedFrame* frame_out2 = DecodeCompleteFrame(); VCMEncodedFrame* frame_out2 = DecodeCompleteFrame();
EXPECT_EQ(2700u, frame_out2->TimeStamp()); EXPECT_EQ(2700u, frame_out2->TimeStamp());
EXPECT_EQ(0, CheckOutFrame(frame_out2, size_, false)); CheckOutFrame(frame_out2, size_, false);
EXPECT_EQ(kVideoFrameDelta, frame_out2->FrameType()); EXPECT_EQ(kVideoFrameDelta, frame_out2->FrameType());
} }
@ -1090,7 +1091,7 @@ TEST_F(TestBasicJitterBuffer, ExceedNumOfFrameWithSeqNumWrap) {
VCMEncodedFrame* frame_out = DecodeCompleteFrame(); VCMEncodedFrame* frame_out = DecodeCompleteFrame();
EXPECT_EQ(first_key_frame_timestamp, frame_out->TimeStamp()); EXPECT_EQ(first_key_frame_timestamp, frame_out->TimeStamp());
EXPECT_EQ(0, CheckOutFrame(frame_out, size_, false)); CheckOutFrame(frame_out, size_, false);
EXPECT_EQ(kVideoFrameKey, frame_out->FrameType()); EXPECT_EQ(kVideoFrameKey, frame_out->FrameType());
} }
@ -1111,7 +1112,7 @@ TEST_F(TestBasicJitterBuffer, EmptyLastFrame) {
packet_->timestamp = timestamp_; packet_->timestamp = timestamp_;
packet_->frameType = kFrameEmpty; packet_->frameType = kFrameEmpty;
EXPECT_EQ(kFirstPacket, jitter_buffer_->InsertPacket(*packet_, EXPECT_EQ(kNoError, jitter_buffer_->InsertPacket(*packet_,
&retransmitted)); &retransmitted));
VCMEncodedFrame* testFrame = DecodeIncompleteFrame(); VCMEncodedFrame* testFrame = DecodeIncompleteFrame();
// Timestamp should never be the last TS inserted. // Timestamp should never be the last TS inserted.
@ -1184,7 +1185,7 @@ TEST_F(TestBasicJitterBuffer, H264IncompleteNalu) {
// We can decode everything from a NALU until a packet has been lost. // We can decode everything from a NALU until a packet has been lost.
// Thus we can decode the first packet of the first NALU and the second NALU // Thus we can decode the first packet of the first NALU and the second NALU
// which consists of one packet. // which consists of one packet.
EXPECT_EQ(0, CheckOutFrame(frame_out, packet_->sizeBytes * 2, false)); CheckOutFrame(frame_out, packet_->sizeBytes * 2, false);
jitter_buffer_->ReleaseFrame(frame_out); jitter_buffer_->ReleaseFrame(frame_out);
// Test reordered start frame + 1 lost. // Test reordered start frame + 1 lost.
@ -1251,7 +1252,7 @@ TEST_F(TestBasicJitterBuffer, H264IncompleteNalu) {
// is the last. // is the last.
frame_out = DecodeIncompleteFrame(); frame_out = DecodeIncompleteFrame();
// Only last NALU is complete. // Only last NALU is complete.
EXPECT_EQ(0, CheckOutFrame(frame_out, insertedLength, false)); CheckOutFrame(frame_out, insertedLength, false);
jitter_buffer_->ReleaseFrame(frame_out); jitter_buffer_->ReleaseFrame(frame_out);
@ -1302,7 +1303,7 @@ TEST_F(TestBasicJitterBuffer, H264IncompleteNalu) {
frame_out = DecodeCompleteFrame(); frame_out = DecodeCompleteFrame();
// Only last NALU is complete // Only last NALU is complete
EXPECT_EQ(0, CheckOutFrame(frame_out, packet_->sizeBytes, false)); CheckOutFrame(frame_out, packet_->sizeBytes, false);
} }
TEST_F(TestBasicJitterBuffer, NextFrameWhenIncomplete) { TEST_F(TestBasicJitterBuffer, NextFrameWhenIncomplete) {
@ -1344,7 +1345,7 @@ TEST_F(TestBasicJitterBuffer, NextFrameWhenIncomplete) {
frame_out = DecodeIncompleteFrame(); frame_out = DecodeIncompleteFrame();
EXPECT_EQ(0, CheckOutFrame(frame_out, packet_->sizeBytes, false)); CheckOutFrame(frame_out, packet_->sizeBytes, false);
} }
TEST_F(TestRunningJitterBuffer, Full) { TEST_F(TestRunningJitterBuffer, Full) {
@ -1368,14 +1369,17 @@ TEST_F(TestRunningJitterBuffer, EmptyPackets) {
stream_generator_->GenerateFrame(kVideoFrameKey, 3, 3, stream_generator_->GenerateFrame(kVideoFrameKey, 3, 3,
clock_->TimeInMilliseconds()); clock_->TimeInMilliseconds());
bool request_key_frame = false; bool request_key_frame = false;
EXPECT_EQ(kFirstPacket, InsertPacketAndPop(4)); // Insert empty packet.
EXPECT_EQ(kNoError, InsertPacketAndPop(4));
EXPECT_FALSE(request_key_frame); EXPECT_FALSE(request_key_frame);
EXPECT_EQ(kIncomplete, InsertPacketAndPop(4)); // Insert 3 media packets.
EXPECT_EQ(kFirstPacket, InsertPacketAndPop(0));
EXPECT_FALSE(request_key_frame); EXPECT_FALSE(request_key_frame);
EXPECT_EQ(kIncomplete, InsertPacketAndPop(0)); EXPECT_EQ(kIncomplete, InsertPacketAndPop(0));
EXPECT_FALSE(request_key_frame); EXPECT_FALSE(request_key_frame);
EXPECT_EQ(kIncomplete, InsertPacketAndPop(0)); EXPECT_EQ(kCompleteSession, InsertPacketAndPop(0));
EXPECT_FALSE(request_key_frame); EXPECT_FALSE(request_key_frame);
// Insert empty packet.
EXPECT_EQ(kCompleteSession, InsertPacketAndPop(0)); EXPECT_EQ(kCompleteSession, InsertPacketAndPop(0));
EXPECT_FALSE(request_key_frame); EXPECT_FALSE(request_key_frame);
} }
@ -1482,6 +1486,22 @@ TEST_F(TestRunningJitterBuffer, KeyDeltaKeyDelta) {
} }
} }
TEST_F(TestRunningJitterBuffer, TwoPacketsNonContinuous) {
InsertFrame(kVideoFrameKey);
EXPECT_TRUE(DecodeCompleteFrame());
stream_generator_->GenerateFrame(kVideoFrameDelta, 1, 0,
clock_->TimeInMilliseconds());
clock_->AdvanceTimeMilliseconds(kDefaultFramePeriodMs);
stream_generator_->GenerateFrame(kVideoFrameDelta, 2, 0,
clock_->TimeInMilliseconds());
EXPECT_EQ(kFirstPacket, InsertPacketAndPop(1));
EXPECT_EQ(kCompleteSession, InsertPacketAndPop(1));
EXPECT_FALSE(DecodeCompleteFrame());
EXPECT_EQ(kCompleteSession, InsertPacketAndPop(0));
EXPECT_TRUE(DecodeCompleteFrame());
EXPECT_TRUE(DecodeCompleteFrame());
}
TEST_F(TestJitterBufferNack, EmptyPackets) { TEST_F(TestJitterBufferNack, EmptyPackets) {
// Make sure empty packets doesn't clog the jitter buffer. // Make sure empty packets doesn't clog the jitter buffer.
jitter_buffer_->SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); jitter_buffer_->SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
@ -1499,7 +1519,7 @@ TEST_F(TestJitterBufferNack, NackTooOldPackets) {
// old packet. // old packet.
DropFrame(1); DropFrame(1);
// Insert a frame which should trigger a recycle until the next key frame. // Insert a frame which should trigger a recycle until the next key frame.
EXPECT_EQ(kFlushIndicator, InsertFrames(oldest_packet_to_nack_, EXPECT_EQ(kFlushIndicator, InsertFrames(oldest_packet_to_nack_ + 1,
kVideoFrameDelta)); kVideoFrameDelta));
EXPECT_FALSE(DecodeCompleteFrame()); EXPECT_FALSE(DecodeCompleteFrame());
@ -1555,7 +1575,7 @@ TEST_F(TestJitterBufferNack, NackListFull) {
EXPECT_TRUE(DecodeCompleteFrame()); EXPECT_TRUE(DecodeCompleteFrame());
// Generate and drop |kNackHistoryLength| packets to fill the NACK list. // Generate and drop |kNackHistoryLength| packets to fill the NACK list.
DropFrame(max_nack_list_size_); DropFrame(max_nack_list_size_ + 1);
// Insert a frame which should trigger a recycle until the next key frame. // Insert a frame which should trigger a recycle until the next key frame.
EXPECT_EQ(kFlushIndicator, InsertFrame(kVideoFrameDelta)); EXPECT_EQ(kFlushIndicator, InsertFrame(kVideoFrameDelta));
EXPECT_FALSE(DecodeCompleteFrame()); EXPECT_FALSE(DecodeCompleteFrame());
@ -1612,6 +1632,33 @@ TEST_F(TestJitterBufferNack, NackListBuiltBeforeFirstDecode) {
EXPECT_TRUE(list != NULL); EXPECT_TRUE(list != NULL);
} }
TEST_F(TestJitterBufferNack, VerifyRetransmittedFlag) {
stream_generator_->Init(0, 0, clock_->TimeInMilliseconds());
stream_generator_->GenerateFrame(kVideoFrameKey, 3, 0,
clock_->TimeInMilliseconds());
VCMPacket packet;
stream_generator_->PopPacket(&packet, 0);
bool retransmitted = false;
EXPECT_EQ(kFirstPacket, jitter_buffer_->InsertPacket(packet, &retransmitted));
EXPECT_FALSE(retransmitted);
// Drop second packet.
stream_generator_->PopPacket(&packet, 1);
EXPECT_EQ(kIncomplete, jitter_buffer_->InsertPacket(packet, &retransmitted));
EXPECT_FALSE(retransmitted);
EXPECT_FALSE(DecodeCompleteFrame());
uint16_t nack_list_size = 0;
bool extended = false;
uint16_t* list = jitter_buffer_->GetNackList(&nack_list_size, &extended);
EXPECT_EQ(1, nack_list_size);
ASSERT_TRUE(list != NULL);
stream_generator_->PopPacket(&packet, 0);
EXPECT_EQ(packet.seqNum, list[0]);
EXPECT_EQ(kCompleteSession, jitter_buffer_->InsertPacket(packet,
&retransmitted));
EXPECT_TRUE(retransmitted);
EXPECT_TRUE(DecodeCompleteFrame());
}
TEST_F(TestJitterBufferNack, UseNackToRecoverFirstKeyFrame) { TEST_F(TestJitterBufferNack, UseNackToRecoverFirstKeyFrame) {
stream_generator_->Init(0, 0, clock_->TimeInMilliseconds()); stream_generator_->Init(0, 0, clock_->TimeInMilliseconds());
stream_generator_->GenerateFrame(kVideoFrameKey, 3, 0, stream_generator_->GenerateFrame(kVideoFrameKey, 3, 0,

8
webrtc/modules/video_coding/main/source/stream_generator.cc
View File

@ -44,10 +44,6 @@ void StreamGenerator::GenerateFrame(FrameType type,
int num_empty_packets, int num_empty_packets,
int64_t current_time) { int64_t current_time) {
timestamp_ = 90 * (current_time - start_time_); timestamp_ = 90 * (current_time - start_time_);
// Move the sequence number counter if all packets from the previous frame
// wasn't collected.
sequence_number_ += packets_.size();
packets_.clear();
for (int i = 0; i < num_media_packets; ++i) { for (int i = 0; i < num_media_packets; ++i) {
const int packet_size = (kFrameSize + num_media_packets / 2) / const int packet_size = (kFrameSize + num_media_packets / 2) /
num_media_packets; num_media_packets;
@ -123,6 +119,10 @@ bool StreamGenerator::NextPacket(VCMPacket* packet) {
return true; return true;
} }
void StreamGenerator::DropLastPacket() {
packets_.pop_back();
}
uint16_t StreamGenerator::NextSequenceNumber() const { uint16_t StreamGenerator::NextSequenceNumber() const {
if (packets_.empty()) if (packets_.empty())
return sequence_number_; return sequence_number_;

1
webrtc/modules/video_coding/main/source/stream_generator.h
View File

@ -46,6 +46,7 @@ class StreamGenerator {
FrameType type); FrameType type);
bool PopPacket(VCMPacket* packet, int index); bool PopPacket(VCMPacket* packet, int index);
void DropLastPacket();
bool GetPacket(VCMPacket* packet, int index); bool GetPacket(VCMPacket* packet, int index);