generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

video-coding: Adding a decoded state to the JB logic (JB refactor).

Browse Source 
This new class stores the last decoded info, including temporal info. 
Review URL: http://webrtc-codereview.appspot.com/300005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1174 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
mikhal@webrtc.org 2011-12-13 21:15:05 +00:00
parent 65573f2922
commit 832cacacff
5 changed files with 414 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
src/modules/video_coding/main/source/decoding_state.cc Normal file
View File

@ -0,0 +1,203 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/main/source/decoding_state.h"
#include "modules/video_coding/main/source/frame_buffer.h"
#include "modules/video_coding/main/source/jitter_buffer_common.h"
#include "modules/video_coding/main/source/packet.h"
#include "modules/interface/module_common_types.h"
namespace webrtc {
VCMDecodingState::VCMDecodingState()
: sequence_num_(0),
time_stamp_(0),
picture_id_(kNoPictureId),
temporal_id_(kNoTemporalIdx),
tl0_pic_id_(kNoTl0PicIdx),
full_sync_(true),
init_(true) {}
VCMDecodingState::~VCMDecodingState() {}
void VCMDecodingState::Reset() {
// TODO(mikhal): Verify - not always would want to reset the sync
sequence_num_ = 0;
time_stamp_ = 0;
picture_id_ = kNoPictureId;
temporal_id_ = kNoTemporalIdx;
tl0_pic_id_ = kNoTl0PicIdx;
full_sync_ = true;
init_ = true;
}
uint32_t VCMDecodingState::time_stamp() const {
return time_stamp_;
}
uint16_t VCMDecodingState::sequence_num() const {
return sequence_num_;
}
bool VCMDecodingState::IsOldFrame(const VCMFrameBuffer* frame) const {
assert(frame != NULL);
if (init_)
return false;
return (LatestTimestamp(time_stamp_, frame->TimeStamp(), NULL)
== time_stamp_);
}
bool VCMDecodingState::IsOldPacket(const VCMPacket* packet) const {
assert(packet != NULL);
if (init_)
return false;
return (LatestTimestamp(time_stamp_, packet->timestamp, NULL)
== time_stamp_);
}
void VCMDecodingState::SetState(const VCMFrameBuffer* frame) {
assert(frame != NULL && frame->GetHighSeqNum() >= 0);
UpdateSyncState(frame);
sequence_num_ = static_cast<uint16_t>(frame->GetHighSeqNum());
time_stamp_ = frame->TimeStamp();
picture_id_ = frame->PictureId();
temporal_id_ = frame->TemporalId();
tl0_pic_id_ = frame->Tl0PicId();
init_ = false;
}
void VCMDecodingState::SetStateOneBack(const VCMFrameBuffer* frame) {
assert(frame != NULL && frame->GetHighSeqNum() >= 0);
sequence_num_ = static_cast <uint16_t>(frame->GetHighSeqNum()) - 1u;
time_stamp_ = frame->TimeStamp() - 1u;
temporal_id_ = frame->TemporalId();
if (frame->PictureId() != kNoPictureId) {
if (frame->PictureId() == 0)
picture_id_ = 0x7FFF;
else
picture_id_ = frame->PictureId() - 1;
}
if (frame->Tl0PicId() != kNoTl0PicIdx) {
if (frame->Tl0PicId() == 0)
tl0_pic_id_ = 0x00FF;
else
tl0_pic_id_ = frame->Tl0PicId() - 1;
}
init_ = false;
}
void VCMDecodingState::UpdateEmptyPacket(const VCMPacket* packet) {
assert(packet != NULL);
if (packet->sizeBytes == 0 && packet->timestamp == time_stamp_) {
// Empty packet (sizeBytes = 0), make sure we update the last
// decoded sequence number.
sequence_num_ = LatestSequenceNumber(packet->seqNum, sequence_num_, NULL);
}
}
void VCMDecodingState::SetSeqNum(uint16_t new_seq_num) {
sequence_num_ = new_seq_num;
}
bool VCMDecodingState::init() const {
return init_;
}
bool VCMDecodingState::full_sync() const {
return full_sync_;
}
void VCMDecodingState::UpdateSyncState(const VCMFrameBuffer* frame) {
if (init_)
return;
if (frame->TemporalId() == kNoTemporalIdx ||
frame->Tl0PicId() == kNoTl0PicIdx) {
full_sync_ = true;
} else if (frame->FrameType() == kVideoFrameKey ||
frame->NonReference()) {
full_sync_ = true;
} else if (full_sync_) {
// Verify that we are still in sync.
// Sync will be broken if continuity is true for layers but not for the
// other methods (PictureId and SeqNum).
if (!ContinuousPictureId(frame->PictureId()) &&
!ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum()))) {
// Non-layered methods have failed.
full_sync_ = false;
}
}
}
bool VCMDecodingState::ContinuousFrame(const VCMFrameBuffer* frame) const {
// Check continuity based on the following hierarchy:
// - Temporal layers (stop here if out of sync).
// - Picture Id when available.
// - Sequence numbers.
// Return true when in initial state.
// Note that when a method is not applicable it will return false.
assert(frame != NULL);
if (init_)
return true;
if (!ContinuousLayer(frame->TemporalId(), frame->Tl0PicId())) {
// Base layers are not continuous or temporal layers are inactive.
if (!full_sync_)
return false;
else if (!ContinuousPictureId(frame->PictureId()))
return ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum()));
}
return true;
}
bool VCMDecodingState::ContinuousPictureId(int picture_id) const {
// First, check if applicable.
if (picture_id == kNoPictureId || picture_id_ == kNoPictureId)
return false;
uint16_t local_pic_id = static_cast<uint16_t>(picture_id_);
uint16_t new_pic_id = static_cast<uint16_t>(picture_id);
if (new_pic_id < local_pic_id) {
// Wrap
if (local_pic_id >= (1 << 8)) {
// 15 bits used for picture id
return (((local_pic_id + 1) % 0x7FFF) == new_pic_id);
} else {
// 7 bits used for picture id
return (((local_pic_id + 1) % 0x7F) == new_pic_id);
}
}
// No wrap
return (local_pic_id + 1 == new_pic_id);
}
bool VCMDecodingState::ContinuousSeqNum(uint16_t seq_num) const {
return (seq_num == static_cast<uint16_t>(sequence_num_ + 1));
}
bool VCMDecodingState::ContinuousLayer(int temporal_id,
int tl0_pic_id) const {
// First, check if applicable.
if (temporal_id == kNoTemporalIdx || tl0_pic_id == kNoTl0PicIdx)
return false;
// If this is the first frame to use temporal layers, make sure we start
// from base.
else if (tl0_pic_id_ == kNoTl0PicIdx && temporal_id_ == kNoTemporalIdx &&
temporal_id == 0)
return true;
// Current implementation: applicable for base layer only.
if (temporal_id != 0)
return false;
return (static_cast<uint8_t>(tl0_pic_id_ + 1) ==
static_cast<uint8_t>(tl0_pic_id));
}
} // namespace webrtc

68
src/modules/video_coding/main/source/decoding_state.h Normal file
View File

@ -0,0 +1,68 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_
#define WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_
#include "typedefs.h"
namespace webrtc {
// Forward declarations
class VCMFrameBuffer;
class VCMPacket;
class VCMDecodingState {
public:
VCMDecodingState();
~VCMDecodingState();
// Check for old frame
bool IsOldFrame(const VCMFrameBuffer* frame) const;
// Check for old packet
bool IsOldPacket(const VCMPacket* packet) const;
// Check for frame continuity based on current decoded state. Use best method
// possible, i.e. temporal info, picture ID or sequence number.
bool ContinuousFrame(const VCMFrameBuffer* frame) const;
void SetState(const VCMFrameBuffer* frame);
// Set the decoding state one frame back.
void SetStateOneBack(const VCMFrameBuffer* frame);
// Update the sequence number if the timestamp matches current state and the
// packet is empty.
void UpdateEmptyPacket(const VCMPacket* packet);
void SetSeqNum(uint16_t new_seq_num);
void Reset();
uint32_t time_stamp() const;
uint16_t sequence_num() const;
// Return true if at initial state.
bool init() const;
// Return true when sync is on - decode all layers.
bool full_sync() const;
private:
void UpdateSyncState(const VCMFrameBuffer* frame);
// Designated continuity functions
bool ContinuousPictureId(int picture_id) const;
bool ContinuousSeqNum(uint16_t seq_num) const;
bool ContinuousLayer(int temporal_id, int tl0_pic_id) const;
// Keep state of last decoded frame.
// TODO(mikhal/stefan): create designated classes to handle these types.
uint16_t sequence_num_;
uint32_t time_stamp_;
int picture_id_;
int temporal_id_;
int tl0_pic_id_;
bool full_sync_; // Sync flag when temporal layers are used.
bool init_;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_

449
src/modules/video_coding/main/source/jitter_buffer.cc
View File

@ -69,9 +69,7 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId,
_maxNumberOfFrames(kStartNumberOfFrames),
_frameBuffers(),
_frameBuffersTSOrder(),
_lastDecodedSeqNum(-1),
_lastDecodedTimeStamp(-1),
_lastDecodedPictureId(-1),
_lastDecodedState(),
_packetsNotDecodable(0),
_receiveStatistics(),
_incomingFrameRate(0),
@ -143,9 +141,7 @@ VCMJitterBuffer::CopyFrom(const VCMJitterBuffer& rhs)
_NACKSeqNumLength = rhs._NACKSeqNumLength;
_waitingForKeyFrame = rhs._waitingForKeyFrame;
_firstPacket = rhs._firstPacket;
_lastDecodedSeqNum = rhs._lastDecodedSeqNum;
_lastDecodedTimeStamp = rhs._lastDecodedTimeStamp;
_lastDecodedPictureId = rhs._lastDecodedPictureId;
_lastDecodedState = rhs._lastDecodedState;
_packetsNotDecodable = rhs._packetsNotDecodable;
memcpy(_receiveStatistics, rhs._receiveStatistics,
sizeof(_receiveStatistics));
@ -213,9 +209,7 @@ VCMJitterBuffer::Stop()
{
_critSect->Enter();
_running = false;
_lastDecodedTimeStamp = -1;
_lastDecodedSeqNum = -1;
_lastDecodedPictureId = -1;
_lastDecodedState.Reset();
_frameBuffersTSOrder.Flush();
for (int i = 0; i < kMaxNumberOfFrames; i++)
{
@ -257,9 +251,7 @@ VCMJitterBuffer::FlushInternal()
{
ReleaseFrameInternal(_frameBuffers[i]);
}
_lastDecodedSeqNum = -1;
_lastDecodedTimeStamp = -1;
_lastDecodedPictureId = -1;
_lastDecodedState.Reset(); // TODO (mikhal): sync reset
_packetsNotDecodable = 0;
_frameEvent.Reset();
@ -331,9 +323,7 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
// Check if we should drop frame
// an old complete frame can arrive too late
if (_lastDecodedTimeStamp > 0 &&
LatestTimestamp(static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp),
frame->TimeStamp(), NULL) == _lastDecodedTimeStamp)
if (_lastDecodedState.IsOldFrame(frame))
{
// Frame is older than the latest decoded frame, drop it.
// This will trigger a release in CleanUpSizeZeroFrames later.
@ -442,14 +432,15 @@ VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMEncodedFrame*& frame)
}
_critSect->Enter();
// Make sure that old empty packets are inserted.
if (LatestTimestamp(static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp),
packet.timestamp, NULL) == _lastDecodedTimeStamp
&& packet.sizeBytes > 0)
// Does this packet belong to an old frame?
if (_lastDecodedState.IsOldPacket(&packet))
{
_discardedPackets++; // Only counts discarded media packets
// Trying to get an old frame.
_numConsecutiveOldPackets++;
// Account only for media packets
if (packet.sizeBytes > 0)
{
_discardedPackets++;
_numConsecutiveOldPackets++;
}
if (_numConsecutiveOldPackets > kMaxConsecutiveOldPackets)
{
FlushInternal();
@ -544,51 +535,6 @@ VCMJitterBuffer::GetEmptyFrame()
return NULL;
}
// Must be called under the critical section _critSect.
VCMFrameListItem*
VCMJitterBuffer::FindOldestSequenceNum() const
{
WebRtc_UWord16 currentLow = 0xffff;
WebRtc_UWord16 sequenceNumber = 0;
bool first = true;
VCMFrameListItem* frameListItem = _frameBuffersTSOrder.First();
VCMFrameListItem* oldestFrameListItem = NULL;
while (frameListItem != NULL)
{
// if we have more than one frame done since last time,
// pick oldest
VCMFrameBuffer* ptrFrame = NULL;
ptrFrame = frameListItem->GetItem();
sequenceNumber = static_cast<WebRtc_UWord16>(ptrFrame->GetLowSeqNum());
// Find the oldest, hence lowest, using sequence numbers
if (first)
{
currentLow = sequenceNumber;
oldestFrameListItem = frameListItem;
first = false;
}
else if ((currentLow < 0x0fff) && (sequenceNumber > 0xf000))
{
// We have a wrap and this one is older
currentLow = sequenceNumber;
oldestFrameListItem = frameListItem;
}
else if ((sequenceNumber < 0x0fff) && (currentLow > 0xf000))
{
// This one is after a wrap, leave as is
}
else if (currentLow > sequenceNumber)
{
// Normal case, this one is lower.
currentLow = sequenceNumber;
oldestFrameListItem = frameListItem;
}
frameListItem = _frameBuffersTSOrder.Next(frameListItem);
}
return oldestFrameListItem;
}
// Find oldest complete frame used for getting next frame to decode
// Must be called under critical section
@ -598,7 +544,6 @@ VCMFrameListItem*
VCMJitterBuffer::FindOldestCompleteContinuousFrame(bool enableDecodable) {
// If we have more than one frame done since last time, pick oldest.
VCMFrameBuffer* oldestFrame = NULL;
int currentLow = -1;
VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First();
if (oldestFrameItem != NULL) {
@ -626,23 +571,11 @@ VCMJitterBuffer::FindOldestCompleteContinuousFrame(bool enableDecodable) {
if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey) {
return NULL;
}
// Is this the first frame to be decoded?
// Otherwise, establish continuity - use PictureId when available.
if (_lastDecodedSeqNum == -1) {
// We have a complete frame - establish continuity.
if (_lastDecodedState.init()) {
return oldestFrameItem;
} else if (oldestFrame->PictureId() == kNoPictureId) {
currentLow = oldestFrame->GetLowSeqNum();
WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
// We could have received the first packet of the last frame before a long
// period if drop, that case is handled by GetNackList (when applicable).
if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) {
// Wait since we want a complete continuous frame.
} else if (!_lastDecodedState.ContinuousFrame(oldestFrame)) {
return NULL;
}
} else if (!ContinuousPictureId(oldestFrame->PictureId())) {
// We don't have a continuous frame.
return NULL;
}
return oldestFrameItem;
}
@ -748,7 +681,7 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
CleanUpOldFrames();
CleanUpSizeZeroFrames();
if (_lastDecodedSeqNum == -1 && WaitForNack()) {
if (_lastDecodedState.init() && WaitForNack()) {
_waitingForKeyFrame = true;
}
VCMFrameListItem*
@ -850,10 +783,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
_critSect->Leave();
// We have a frame - store seqnum & timestamp
_lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
_lastDecodedTimeStamp = oldestFrame->TimeStamp();
_lastDecodedPictureId = oldestFrame->PictureId();
// We have a frame - update decoded state with frame info.
_lastDecodedState.SetState(oldestFrame);
return oldestFrame;
}
@ -990,9 +921,9 @@ VCMJitterBuffer::CompleteSequenceWithNextFrame()
}
// See if we have lost a frame before this one.
if (_lastDecodedSeqNum == -1)
if (_lastDecodedState.init())
{
// The sequence is not complete since we haven't yet.
// Following start, reset or flush -> check for key frame.
if (oldestFrame->FrameType() != kVideoFrameKey)
{
return false;
@ -1002,18 +933,7 @@ VCMJitterBuffer::CompleteSequenceWithNextFrame()
{
return false;
}
// We can't use sequence numbers to detect frame loss when FEC is enabled.
// Assume FEC is only enabled for VP8 with picture ids, and use picture ids
// to detect frame loss in that situation.
else if (oldestFrame->PictureId() == kNoPictureId)
{
if (oldestFrame->GetLowSeqNum() !=
(_lastDecodedSeqNum + 1) % 0x00010000)
{
return false;
}
}
else if (!ContinuousPictureId(oldestFrame->PictureId()))
else if (!_lastDecodedState.ContinuousFrame(oldestFrame))
{
return false;
}
@ -1098,10 +1018,8 @@ VCMJitterBuffer::GetFrameForDecoding()
_packetsNotDecodable += oldestFrame->NotDecodablePackets();
// Store current seqnum & time
_lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
_lastDecodedTimeStamp = oldestFrame->TimeStamp();
_lastDecodedPictureId = oldestFrame->PictureId();
// We have a frame - update decoded state with frame info.
_lastDecodedState.SetState(oldestFrame);
return oldestFrame;
}
@ -1119,8 +1037,8 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
// First look for a complete _continuous_ frame.
// When waiting for nack, wait for a key frame, if a continuous frame cannot
// be determined (i.e. lastDecodedSeqNum = -1)
if (_lastDecodedSeqNum == -1) {
// be determined (i.e. initial decoding state).
if (_lastDecodedState.init()) {
_waitingForKeyFrame = true;
}
@ -1178,11 +1096,8 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
_waitingForKeyFrame = false;
}
// We have a complete/decodable continuous frame, decode it.
// Store seqnum & timestamp
_lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
_lastDecodedTimeStamp = oldestFrame->TimeStamp();
_lastDecodedPictureId = oldestFrame->PictureId();
// We have a frame - update decoded state with frame info.
_lastDecodedState.SetState(oldestFrame);
return oldestFrame;
}
@ -1299,11 +1214,12 @@ WebRtc_Word32
VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum,
WebRtc_Word32& highSeqNum) const
{
// TODO (mikhal/stefan): refactor to use lastDecodedState
WebRtc_Word32 i = 0;
WebRtc_Word32 seqNum = -1;
highSeqNum = -1;
lowSeqNum = _lastDecodedSeqNum;
lowSeqNum = _lastDecodedState.sequence_num();
// find highest seq numbers
for (i = 0; i < _maxNumberOfFrames; ++i)
@ -1329,6 +1245,7 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum,
WebRtc_UWord16*
VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
{
// TODO (mikhal/stefan): Refactor to use lastDecodedState.
CriticalSectionScoped cs(_critSect);
int i = 0;
WebRtc_Word32 lowSeqNum = -1;
@ -1388,13 +1305,13 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
"from seq: %d. Lowest seq in jb %d", highSeqNum,lowSeqNum);
// This nack size will trigger a key request...
bool foundIFrame = false;
bool foundKeyFrame = false;
while (numberOfSeqNum > kNackHistoryLength)
{
foundIFrame = RecycleFramesUntilKeyFrame();
foundKeyFrame = RecycleFramesUntilKeyFrame();
if (!foundIFrame)
if (!foundKeyFrame)
{
break;
}
@ -1428,20 +1345,20 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
} // end while
if (!foundIFrame)
if (!foundKeyFrame)
{
// No I frame in JB.
// No key frame in JB.
// Set the last decoded sequence number to current high.
// This is to not get a large nack list again right away
_lastDecodedSeqNum = highSeqNum;
_lastDecodedState.SetSeqNum(static_cast<uint16_t>(highSeqNum));
_waitingForKeyFrame = true;
// Set to trigger key frame signal
nackSize = 0xffff;
listExtended = true;
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1,
"\tNo key frame found, request one. _lastDecodedSeqNum[0] "
"%d", _lastDecodedSeqNum);
"%d", _lastDecodedState.sequence_num());
}
else
{
@ -1449,7 +1366,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
// The function itself has set last decoded seq.
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1,
"\tKey frame found. _lastDecodedSeqNum[0] %d",
_lastDecodedSeqNum);
_lastDecodedState.sequence_num());
nackSize = 0;
}
@ -1598,7 +1515,7 @@ WebRtc_Word64
VCMJitterBuffer::LastDecodedTimestamp() const
{
CriticalSectionScoped cs(_critSect);
return _lastDecodedTimeStamp;
return _lastDecodedState.time_stamp();
}
// Insert packet
@ -1647,14 +1564,7 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
if (frame != NULL)
{
VCMFrameBufferStateEnum state = frame->GetState();
if (packet.sizeBytes == 0 && packet.timestamp == _lastDecodedTimeStamp)
{
// Empty packet (sizeBytes = 0), make sure we update the last
// decoded sequence number
_lastDecodedSeqNum = LatestSequenceNumber(packet.seqNum,
_lastDecodedSeqNum, NULL);
}
_lastDecodedState.UpdateEmptyPacket(&packet);
// Insert packet
// Check for first packet
// High sequence number will be -1 if neither an empty packet nor
@ -1811,8 +1721,8 @@ VCMJitterBuffer::RecycleFramesUntilKeyFrame()
}
// Remove up to oldest key frame
bool foundIFrame = false;
while (oldestFrameListItem != NULL && !foundIFrame)
bool foundKeyFrame = false;
while (oldestFrameListItem != NULL && !foundKeyFrame)
{
// Throw at least one frame.
_dropCount++;
@ -1831,209 +1741,126 @@ VCMJitterBuffer::RecycleFramesUntilKeyFrame()
if (oldestFrame != NULL)
{
foundIFrame = foundIFrame ||
(oldestFrame->FrameType() != kVideoFrameDelta);
if (foundIFrame)
foundKeyFrame = foundKeyFrame ||
(oldestFrame->FrameType() != kVideoFrameDelta);
if (foundKeyFrame)
{
// fake the last played out to match the start of this key frame
_lastDecodedSeqNum = (WebRtc_UWord16)((WebRtc_UWord16)
(oldestFrame->GetLowSeqNum()) - 1);
_lastDecodedTimeStamp = (WebRtc_UWord32)
(oldestFrame->TimeStamp() - 1);
// Fake the lastDecodedState to match this key frame.
_lastDecodedState.SetStateOneBack(oldestFrame);
break;
}
}
}
_lastDecodedSeqNum = -1;
return foundIFrame;
_lastDecodedState.Reset(); // TODO (mikhal): no sync
return foundKeyFrame;
}
// Must be called under the critical section _critSect.
void
VCMJitterBuffer::CleanUpOldFrames()
VCMFrameListItem*
VCMJitterBuffer::FindOldestSequenceNum() const
{
VCMFrameListItem* oldestFrameListItem = _frameBuffersTSOrder.First();
VCMFrameBuffer* oldestFrame = NULL;
if (_lastDecodedTimeStamp == -1)
{
return;
}
while (oldestFrameListItem != NULL)
{
oldestFrame = oldestFrameListItem->GetItem();
WebRtc_UWord32 frameTimeStamp = oldestFrame->TimeStamp();
// Release the frame if it's older than the last decoded frame.
if (_lastDecodedTimeStamp > -1 &&
LatestTimestamp(static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp),
frameTimeStamp, NULL) ==
static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp))
{
const WebRtc_Word32 frameLowSeqNum = oldestFrame->GetLowSeqNum();
const WebRtc_Word32 frameHighSeqNum = oldestFrame->GetHighSeqNum();
if (frameTimeStamp == _lastDecodedTimeStamp &&
((frameLowSeqNum == (_lastDecodedSeqNum + 1)) ||
((frameLowSeqNum == 0) &&
(_lastDecodedSeqNum == 0xffff))))
{
// Could happen when sending empty packets
// Empty packet (size = 0) belonging to last decoded frame.
// Frame: | packet | packet | packet M=1 |
// empty data (size = 0) | empty data (size = 0)| ...
// This frame follows the last decoded frame
_lastDecodedSeqNum = frameHighSeqNum;
}
_frameBuffersTSOrder.Erase(oldestFrameListItem);
ReleaseFrameInternal(oldestFrame);
oldestFrameListItem = _frameBuffersTSOrder.First();
}
else
{
break;
}
}
}
// This function has changed to use sequence numbers
// Using timestamp won't work since can get
// nack requests with a higher time stamp than
// the following encoded frame, but with a lower sequence number.
// Must be called under _critSect.
void
VCMJitterBuffer::CleanUpSizeZeroFrames()
{
VCMFrameListItem* frameListItem = FindOldestSequenceNum();
WebRtc_UWord16 currentLow = 0xffff;
WebRtc_UWord16 sequenceNumber = 0;
bool first = true;
VCMFrameListItem* frameListItem = _frameBuffersTSOrder.First();
VCMFrameListItem* oldestFrameListItem = NULL;
while (frameListItem != NULL)
{
VCMFrameBuffer* ptrTempBuffer = frameListItem->GetItem();
// if we have more than one frame done since last time,
// pick oldest
VCMFrameBuffer* ptrFrame = NULL;
ptrFrame = frameListItem->GetItem();
sequenceNumber = static_cast<WebRtc_UWord16>(ptrFrame->GetLowSeqNum());
// pop frame if its size zero but store seqnum
if (ptrTempBuffer->Length() == 0)
// Find the oldest, hence lowest, using sequence numbers
if (first)
{
WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum();
if (frameHighSeqNum == -1)
{
// This frame has been Reset for this function to clean it up
_frameBuffersTSOrder.Erase(frameListItem);
ReleaseFrameInternal(ptrTempBuffer);
frameListItem = FindOldestSequenceNum();
}
else
{
bool releaseFrame = false;
const WebRtc_Word32 frameHighSeqNum =
ptrTempBuffer->GetHighSeqNum();
const WebRtc_Word32 frameLowSeqNum =
ptrTempBuffer->GetLowSeqNum();
if ((frameLowSeqNum == (_lastDecodedSeqNum + 1)) ||
// Frame is next in line
((frameLowSeqNum == 0) && (_lastDecodedSeqNum== 0xffff)))
{
// This frame follows the last decoded frame, release it.
_lastDecodedSeqNum = frameHighSeqNum;
releaseFrame = true;
}
// If frameHighSeqNum < _lastDecodedSeqNum
// but need to take wrap into account.
else if (frameHighSeqNum < _lastDecodedSeqNum)
{
if (frameHighSeqNum < 0x0fff &&
_lastDecodedSeqNum> 0xf000)
{
// Wrap, we don't want release this one. It's newer...
}
else
{
// This frame has lower seq than last decoded,
// and we have no wrap -> it's older.
releaseFrame = true;
}
}
else if (frameHighSeqNum > _lastDecodedSeqNum &&
_lastDecodedSeqNum < 0x0fff &&
frameHighSeqNum > 0xf000)
{
// Higher seq than last decoded,
// but last decoded has recently wrapped.
releaseFrame = true;
}
if (releaseFrame)
{
_frameBuffersTSOrder.Erase(frameListItem);
ReleaseFrameInternal(ptrTempBuffer);
frameListItem = FindOldestSequenceNum();
}
else
{
// We couldn't release this one and we're using nack,
// stop trying...
frameListItem = NULL;
}
}
currentLow = sequenceNumber;
oldestFrameListItem = frameListItem;
first = false;
}
else
else if ((currentLow < 0x0fff) && (sequenceNumber > 0xf000))
{
// we have a length
break;
// We have a wrap and this one is older
currentLow = sequenceNumber;
oldestFrameListItem = frameListItem;
}
else if ((sequenceNumber < 0x0fff) && (currentLow > 0xf000))
{
// This one is after a wrap, leave as is
}
else if (currentLow > sequenceNumber)
{
// Normal case, this one is lower.
currentLow = sequenceNumber;
oldestFrameListItem = frameListItem;
}
frameListItem = _frameBuffersTSOrder.Next(frameListItem);
}
return oldestFrameListItem;
}
// Must be called under the critical section _critSect.
void VCMJitterBuffer::CleanUpOldFrames() {
if (_lastDecodedState.init())
return;
VCMFrameListItem* oldestFrameListItem = _frameBuffersTSOrder.First();
VCMFrameBuffer* oldestFrame = NULL;
while (oldestFrameListItem != NULL) {
oldestFrame = oldestFrameListItem->GetItem();
if (_lastDecodedState.IsOldFrame(oldestFrame)) {
_frameBuffersTSOrder.Erase(oldestFrameListItem);
ReleaseFrameInternal(oldestFrame);
oldestFrameListItem = _frameBuffersTSOrder.First();
} else {
break;
}
}
}
// TODO (mikhal):
// 1. Merge with previous function.
// 2. Modify to use timestamps and not seqnum.
// Must be called under _critSect.
void VCMJitterBuffer::CleanUpSizeZeroFrames() {
VCMFrameListItem* frameListItem = FindOldestSequenceNum();
while (frameListItem != NULL) {
VCMFrameBuffer* ptrTempBuffer = frameListItem->GetItem();
// Pop frame if its size zero but store seqnum.
if (ptrTempBuffer->Length() == 0) {
WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum();
if (frameHighSeqNum == -1) {
// This frame has been Reset for this function to clean it up
_frameBuffersTSOrder.Erase(frameListItem);
ReleaseFrameInternal(ptrTempBuffer);
frameListItem = FindOldestSequenceNum();
}
} else {
// we have a length
break;
}
}
}
// Used in GetFrameForDecoding
void
VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame)
{
frame.MakeSessionDecodable(); // make sure the session can be decoded.
if (frame.FrameType() == kVideoFrameKey)
return;
void VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame) {
frame.MakeSessionDecodable(); // Make sure the session can be decoded.
if (frame.FrameType() == kVideoFrameKey)
return;
if (_lastDecodedSeqNum == -1)
{
// First frame is not a key frame
frame.SetPreviousFrameLoss();
return;
}
// We can't use sequence numbers to detect frame loss when FEC is enabled.
// Assume FEC is only enabled for VP8 with picture ids, and use picture ids
// to detect frame loss in that situation.
if (frame.PictureId() == kNoPictureId)
{
WebRtc_UWord16 nextExpectedSeqNum =
static_cast<WebRtc_UWord16>(_lastDecodedSeqNum + 1);
if (static_cast<WebRtc_UWord16>(frame.GetLowSeqNum()) !=
nextExpectedSeqNum)
{
frame.SetPreviousFrameLoss();
}
}
else if (!ContinuousPictureId(frame.PictureId()))
{
frame.SetPreviousFrameLoss();
}
}
bool VCMJitterBuffer::ContinuousPictureId(int pictureId) const {
if (pictureId < _lastDecodedPictureId) {
// Wrap
if (_lastDecodedPictureId >= (1<<8)) {
// 15 bits used for picture id
return (((_lastDecodedPictureId + 1) % 0x7FFF) == pictureId);
} else {
// 7 bits used for picture id
return (((_lastDecodedPictureId + 1) % 0x7F) == pictureId);
}
if (_lastDecodedState.init() ||
!_lastDecodedState.ContinuousFrame(&frame)) {
frame.SetPreviousFrameLoss();
}
// No wrap
return (_lastDecodedPictureId + 1 == pictureId);
}
bool
VCMJitterBuffer::WaitForNack()
{

10
src/modules/video_coding/main/source/jitter_buffer.h
View File

@ -13,6 +13,7 @@
#include "typedefs.h"
#include "critical_section_wrapper.h"
#include "decoding_state.h"
#include "module_common_types.h"
#include "video_coding_defines.h"
#include "inter_frame_delay.h"
@ -144,7 +145,6 @@ private:
void ReleaseFrameInternal(VCMFrameBuffer* frame);
// Flush and reset the jitter buffer. Call under critical section.
void FlushInternal();
VCMFrameListItem* FindOldestSequenceNum() const;
// Help functions for insert packet
// Get empty frame, creates new (i.e. increases JB size) if necessary
@ -187,9 +187,9 @@ private:
const void* timestamp);
static bool CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame,
const void* notUsed);
bool ContinuousPictureId(int pictureId) const;
// Decide whether should wait for NACK (mainly relevant for hybrid mode)
bool WaitForNack();
VCMFrameListItem* FindOldestSequenceNum() const;
WebRtc_Word32 _vcmId;
WebRtc_Word32 _receiverId;
@ -208,11 +208,7 @@ private:
VCMFrameListTimestampOrderAsc _frameBuffersTSOrder;
// timing
// Sequence number of last frame that was given to decoder
WebRtc_Word32 _lastDecodedSeqNum;
// Timestamp of last frame that was given to decoder
WebRtc_Word64 _lastDecodedTimeStamp;
int _lastDecodedPictureId;
VCMDecodingState _lastDecodedState;
WebRtc_UWord32 _packetsNotDecodable;
// Statistics

2
src/modules/video_coding/main/source/video_coding.gypi
View File

@ -38,6 +38,7 @@
'codec_database.h',
'codec_timer.h',
'content_metrics_processing.h',
'decoding_state.h',
'encoded_frame.h',
'er_tables_xor.h',
'event.h',
@ -72,6 +73,7 @@
'codec_database.cc',
'codec_timer.cc',
'content_metrics_processing.cc',
'decoding_state.cc',
'encoded_frame.cc',
'exp_filter.cc',
'frame_buffer.cc',