generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

video_coding: Allowing for a decodable state independent of selective nacking

Browse Source 
Review URL: http://webrtc-codereview.appspot.com/263001

git-svn-id: http://webrtc.googlecode.com/svn/trunk@993 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
mikhal@webrtc.org
2011-11-22 22:48:20 +00:00
parent 828af1b4b9
commit 6b9a7f8704
7 changed files with 144 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/modules/video_coding/main/source/frame_buffer.cc
View File

@@ -81,7 +81,8 @@ VCMFrameBuffer::IsSessionComplete() const
// Insert packet
VCMFrameBufferEnum
VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs,
bool enableDecodableState, WebRtc_UWord32 rttMS)
{
if (_state == kStateDecoding)
{
@@ -159,7 +160,9 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
CopyCodecSpecific(&packet.codecSpecificHeader);
WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer);
WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer,
enableDecodableState,
rttMS);
if (retVal == -1)
{
return kSizeError;
@@ -173,19 +176,18 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
_latestPacketTimeMs = timeInMs;
if (_sessionInfo.IsSessionComplete())
{
return kCompleteSession;
}
else
{
// this layer is not complete
if (_state == kStateComplete)
{
// we already have a complete layer
// wait for all independent layers belonging to the same frame
_state = kStateIncomplete;
}
if (_sessionInfo.IsSessionComplete()) {
return kCompleteSession;
} else if (_sessionInfo.IsSessionDecodable()) {
SetState(kStateDecodable);
return kDecodableSession;
} else {
// this layer is not complete
if (_state == kStateComplete) {
// we already have a complete layer
// wait for all independent layers belonging to the same frame
_state = kStateIncomplete;
}
}
return kIncomplete;
}
@@ -215,9 +217,9 @@ VCMFrameBuffer::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num)
WebRtc_Word32
VCMFrameBuffer::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 num,
float rttScore)
WebRtc_UWord32 rttMs)
{
return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttScore);
return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttMs);
}
void
@@ -308,7 +310,7 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
break;
case kStateDecoding:
// A frame migth have received empty packets, or media packets might
// A frame might have received empty packets, or media packets might
// have been removed when making the frame decodable. The frame can
// still be set to decodable since it can be used to inform the
// decoder of a frame loss.
@@ -320,11 +322,6 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
break;
case kStateDecodable:
if (_state == kStateComplete)
{
// if complete, obviously decodable, keep as is.
return;
}
assert(_state == kStateEmpty ||
_state == kStateIncomplete);
break;

6
src/modules/video_coding/main/source/frame_buffer.h
View File

@@ -33,7 +33,9 @@ public:
virtual void Reset();
VCMFrameBufferEnum InsertPacket(const VCMPacket& packet,
WebRtc_Word64 timeInMs);
WebRtc_Word64 timeInMs,
bool enableDecodableState,
WebRtc_UWord32 rttMs);
// State
// Get current state of frame
@@ -66,7 +68,7 @@ public:
// Hybrid extension: only NACK important packets, discard FEC packets
WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 num,
float rttScore);
WebRtc_UWord32 rttMs);
void IncrementNackCount();
WebRtc_Word16 GetNackCount() const;

135
src/modules/video_coding/main/source/jitter_buffer.cc
View File

@@ -391,7 +391,8 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
}
}
const VCMFrameListItem* oldFrameListItem = FindOldestCompleteContinuousFrame();
const VCMFrameListItem*
oldFrameListItem = FindOldestCompleteContinuousFrame(false);
VCMFrameBuffer* oldFrame = NULL;
if (oldFrameListItem != NULL)
{
@@ -590,59 +591,56 @@ VCMJitterBuffer::FindOldestSequenceNum() const
// Based on sequence number
// Return NULL for lost packets
VCMFrameListItem*
VCMJitterBuffer::FindOldestCompleteContinuousFrame()
{
// if we have more than one frame done since last time, pick oldest
VCMFrameBuffer* oldestFrame = NULL;
int currentLow = -1;
VCMJitterBuffer::FindOldestCompleteContinuousFrame(bool enableDecodable) {
// If we have more than one frame done since last time, pick oldest.
VCMFrameBuffer* oldestFrame = NULL;
int currentLow = -1;
VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First();
if (oldestFrameItem != NULL)
{
oldestFrame = oldestFrameItem->GetItem();
}
// is the frame complete?
if (oldestFrame != NULL && kStateComplete != oldestFrame->GetState())
{
oldestFrame = NULL;
}
if (oldestFrame == NULL)
{
// no complete frame no point to continue
return NULL;
VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First();
if (oldestFrameItem != NULL) {
oldestFrame = oldestFrameItem->GetItem();
}
if (oldestFrame != NULL) {
// Check for a complete or decodable frame (when enabled).
VCMFrameBufferStateEnum state = oldestFrame->GetState();
bool decodable = enableDecodable && (state == kStateDecodable);
if ((state != kStateComplete) && !decodable) {
oldestFrame = NULL;
}
}
if (oldestFrame == NULL) {
// No complete frame no point to continue
return NULL;
}
// we have a complete frame
// check if it's continuous, otherwise we are missing a full frame
// We have a complete frame.
// Check if it's continuous, otherwise we are missing a full frame.
// Use pictureId if available. Otherwise use seqNum and not timestamps, as
// a complete frame might be lost
// First determine if we are waiting for a key frame.
if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey) {
return NULL;
}
// Is this the first frame to be decoded?
// Otherwise, verify continuity - use PictureId when available.
if (_lastDecodedSeqNum == -1) {
return oldestFrameItem;
} else if (oldestFrame->PictureId() == kNoPictureId) {
// It's not enough that we have a complete frame we need the layers to
// be continuous too.
currentLow = oldestFrame->GetLowSeqNum();
WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
// We could have received the first packet of the last frame before a
// long period if drop, that case is handled by GetNackList
if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) {
// Wait since we want a complete continuous frame
return NULL;
}
}
else if (!ContinuousPictureId(oldestFrame->PictureId())) {
// We don't have a continuous frame
return NULL;
}
// Use pictureId if available. Otherwise use seqNum and not timestamps, as
// a complete frame might be lost.
// First determine if we are waiting for a key frame.
if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey) {
return NULL;
}
// Is this the first frame to be decoded?
// Otherwise, establish continuity - use PictureId when available.
if (_lastDecodedSeqNum == -1) {
return oldestFrameItem;
} else if (oldestFrame->PictureId() == kNoPictureId) {
currentLow = oldestFrame->GetLowSeqNum();
WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
// We could have received the first packet of the last frame before a long
// period if drop, that case is handled by GetNackList (when applicable).
if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) {
// Wait since we want a complete continuous frame.
return NULL;
}
} else if (!ContinuousPictureId(oldestFrame->PictureId())) {
// We don't have a continuous frame.
return NULL;
}
return oldestFrameItem;
}
// Call from inside the critical section _critSect
@@ -731,7 +729,7 @@ VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate)
return 0;
}
// Returns immediately or a X ms event hang waiting for a decodable frame,
// Returns immediately or a X ms event hang waiting for a complete frame,
// X decided by caller
VCMEncodedFrame*
VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
@@ -749,7 +747,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
if (_lastDecodedSeqNum == -1 && WaitForNack()) {
_waitingForKeyFrame = true;
}
VCMFrameListItem* oldestFrameListItem = FindOldestCompleteContinuousFrame();
VCMFrameListItem*
oldestFrameListItem = FindOldestCompleteContinuousFrame(false);
VCMFrameBuffer* oldestFrame = NULL;
if (oldestFrameListItem != NULL)
{
@@ -785,7 +784,7 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
// sequence number and size
CleanUpOldFrames();
CleanUpSizeZeroFrames();
oldestFrameListItem = FindOldestCompleteContinuousFrame();
oldestFrameListItem = FindOldestCompleteContinuousFrame(false);
if (oldestFrameListItem != NULL)
{
oldestFrame = oldestFrameListItem->GetItem();
@@ -1117,7 +1116,11 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
if (_lastDecodedSeqNum == -1) {
_waitingForKeyFrame = true;
}
VCMFrameListItem* oldestFrameListItem = FindOldestCompleteContinuousFrame();
// Allow for a decodable frame when in Hybrid mode.
bool enableDecodable = _nackMode == kNackHybrid ? true : false;
VCMFrameListItem*
oldestFrameListItem = FindOldestCompleteContinuousFrame(enableDecodable);
VCMFrameBuffer* oldestFrame = NULL;
if (oldestFrameListItem != NULL)
{
@@ -1473,18 +1476,9 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
// not to add empty packets to the nack list
if (_nackMode == kNackHybrid)
{
// build external rttScore based on RTT value
float rttScore = 1.0f;
int ret =
_frameBuffers[i]->ZeroOutSeqNumHybrid(_NACKSeqNumInternal,
numberOfSeqNum,
rttScore);
if (_frameBuffers[i]->IsRetransmitted() == false && ret != -1)
{
// If no retransmission required,set the state to decodable
// meaning that we will not wait for NACK.
_frameBuffers[i]->SetState(kStateDecodable);
}
_rttMs);
}
else
{
@@ -1659,7 +1653,13 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
// High sequence number will be -1 if neither an empty packet nor
// a media packet has been inserted.
bool first = (frame->GetHighSeqNum() == -1);
bufferReturn = frame->InsertPacket(packet, nowMs);
// When in Hybrid mode, we allow for a decodable state
// Note: Under current version, a decodable frame will never be
// triggered, as the body of the function is empty.
// TODO (mikhal): Update when decodable is enabled.
bufferReturn = frame->InsertPacket(packet, nowMs,
_nackMode == kNackHybrid,
_rttMs);
ret = bufferReturn;
if (bufferReturn > 0)
@@ -1702,11 +1702,12 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
_packetEvent.Set();
break;
}
case kDecodableSession:
case kIncomplete:
{
// Signal that we have a received packet
_packetEvent.Set();
break;
// Signal that we have a received packet
_packetEvent.Set();
break;
}
case kNoError:
case kDuplicatePacket:

4
src/modules/video_coding/main/source/jitter_buffer.h
View File

@@ -89,6 +89,7 @@ public:
// or more packets?
bool CompleteSequenceWithNextFrame();
// TODO (mikhal/stefan): Merge all GetFrameForDecoding into one.
// Wait maxWaitTimeMS for a complete frame to arrive. After timeout NULL
// is returned.
VCMEncodedFrame* GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS);
@@ -148,7 +149,8 @@ private:
// Help functions for getting a frame
// Find oldest complete frame, used for getting next frame to decode
VCMFrameListItem* FindOldestCompleteContinuousFrame();
// When enabled, will return a decodable frame
VCMFrameListItem* FindOldestCompleteContinuousFrame(bool enableDecodable);
void CleanUpOldFrames();
void CleanUpSizeZeroFrames();

1
src/modules/video_coding/main/source/jitter_buffer_common.h
View File

@@ -39,6 +39,7 @@ enum VCMFrameBufferEnum
kIncomplete = 1, // Frame incomplete
kFirstPacket = 2,
kCompleteSession = 3, // at least one layer in the frame complete
kDecodableSession = 4, // Frame incomplete, but ready to be decoded
kDuplicatePacket = 5 // We're receiving a duplicate packet.
};

62
src/modules/video_coding/main/source/session_info.cc
View File

@@ -18,6 +18,7 @@ VCMSessionInfo::VCMSessionInfo():
_markerBit(false),
_sessionNACK(false),
_completeSession(false),
_decodableSession(false),
_frameType(kVideoFrameDelta),
_previousFrameLoss(false),
_lowSeqNum(-1),
@@ -70,6 +71,7 @@ VCMSessionInfo::Reset() {
_emptySeqNumHigh = -1;
_markerBit = false;
_completeSession = false;
_decodableSession = false;
_frameType = kVideoFrameDelta;
_previousFrameLoss = false;
_sessionNACK = false;
@@ -186,34 +188,37 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer,
_markerSeqNum = packet.seqNum;
}
UpdateCompleteSession();
return returnLength;
}
void
VCMSessionInfo::UpdateCompleteSession()
{
if (_packets[0].isFirstPacket && _markerBit)
{
// Do we have all the packets in this session?
bool completeSession = true;
void VCMSessionInfo::UpdateCompleteSession() {
if (_packets[0].isFirstPacket && _markerBit) {
// Do we have all the packets in this session?
bool completeSession = true;
for (int i = 0; i <= _highestPacketIndex; ++i)
{
if (_packets[i].completeNALU == kNaluUnset)
{
completeSession = false;
break;
}
}
_completeSession = completeSession;
for (int i = 0; i <= _highestPacketIndex; ++i) {
if (_packets[i].completeNALU == kNaluUnset) {
completeSession = false;
break;
}
}
_completeSession = completeSession;
}
}
bool VCMSessionInfo::IsSessionComplete() const
{
return _completeSession;
void VCMSessionInfo::UpdateDecodableSession(WebRtc_UWord32 rttMs) {
// Irrelevant if session is already complete or decodable
if (_completeSession || _decodableSession)
return;
// First iteration - do nothing
}
bool VCMSessionInfo::IsSessionComplete() const {
return _completeSession;
}
bool VCMSessionInfo::IsSessionDecodable() const {
return _decodableSession;
}
// Find the start and end index of packetIndex packet.
@@ -537,7 +542,7 @@ VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list,
WebRtc_Word32
VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum,
float rttScore)
WebRtc_UWord32 rttMs)
{
if (NULL == list || numberOfSeqNum < 1)
{
@@ -590,6 +595,9 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
_emptySeqNumLow - 1: _highSeqNum;
}
// Place holder
int rttScore = 1.0f;
while (list[index] <= highMediaPacket && index < numberOfSeqNum)
{
if (_packets[i].completeNALU != kNaluUnset)
@@ -659,7 +667,9 @@ VCMSessionInfo::IsRetransmitted() const
WebRtc_Word64
VCMSessionInfo::InsertPacket(const VCMPacket& packet,
WebRtc_UWord8* ptrStartOfLayer)
WebRtc_UWord8* ptrStartOfLayer,
bool enableDecodableState,
WebRtc_UWord32 rttMs)
{
// not allowed
assert(!packet.insertStartCode || !packet.bits);
@@ -765,7 +775,11 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet,
_highestPacketIndex = packetIndex > _highestPacketIndex ?
packetIndex :_highestPacketIndex;
return InsertBuffer(ptrStartOfLayer, packetIndex, packet);
int returnLength = InsertBuffer(ptrStartOfLayer, packetIndex, packet);
UpdateCompleteSession();
if (enableDecodableState)
UpdateDecodableSession(rttMs);
return returnLength;
}

13
src/modules/video_coding/main/source/session_info.h
View File

@@ -34,17 +34,20 @@ public:
WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum);
// Hybrid version: Zero out seq num for NACK list
// apply a score based on the packet location and the external rttScore
// Selectively NACK packets.
WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum,
float rttScore);
WebRtc_UWord32 rttMs);
virtual void Reset();
WebRtc_Word64 InsertPacket(const VCMPacket& packet,
WebRtc_UWord8* ptrStartOfLayer);
WebRtc_UWord8* ptrStartOfLayer,
bool enableDecodableState,
WebRtc_UWord32 rttMs);
WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum);
virtual bool IsSessionComplete() const;
virtual bool IsSessionDecodable() const;
// Builds fragmentation headers for VP8, each fragment being a decodable
// VP8 partition. Returns the total number of bytes which are decodable. Is
@@ -106,11 +109,15 @@ private:
WebRtc_Word32 startIndex,
WebRtc_Word32 endIndex);
void UpdateCompleteSession();
// When enabled, determine if session is decodable, i.e. incomplete but
// would be sent to the decoder.
void UpdateDecodableSession(WebRtc_UWord32 rttMs);
// If we have inserted a packet with markerbit into this frame
bool _markerBit;
// If this session has been NACKed by JB
bool _sessionNACK;
bool _completeSession;
bool _decodableSession;
webrtc::FrameType _frameType;
bool _previousFrameLoss;
// Lowest/Highest packet sequence number in a session