generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Review URL: http://webrtc-codereview.appspot.com/28004

Browse Source 
git-svn-id: http://webrtc.googlecode.com/svn/trunk@74 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
mikhal@google.com 2011-06-14 17:54:20 +00:00
parent 0c08ed1ef9
commit 17705a9c5a
16 changed files with 998 additions and 729 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
modules/video_coding/main/source/frame_buffer.cc
View File

@ -133,14 +133,19 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
if (kStateEmpty == _state) if (kStateEmpty == _state)
{ {
// This is the first packet inserted into this frame, // This is the first packet (empty and/or data) inserted into this frame.
// store some info and set some initial values. // store some info and set some initial values.
_timeStamp = packet.timestamp; _timeStamp = packet.timestamp;
_codec = packet.codec; _codec = packet.codec;
SetState(kStateIncomplete); // for the first media packet
if (packet.frameType != kFrameEmpty)
{
SetState(kStateIncomplete);
}
} }
WebRtc_UWord32 requiredSizeBytes = Length() + packet.sizeBytes + (packet.insertStartCode?kH264StartCodeLengthBytes:0); WebRtc_UWord32 requiredSizeBytes = Length() + packet.sizeBytes +
(packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
if (requiredSizeBytes >= _size) if (requiredSizeBytes >= _size)
{ {
const WebRtc_UWord32 increments = requiredSizeBytes / kBufferIncStepSizeBytes + const WebRtc_UWord32 increments = requiredSizeBytes / kBufferIncStepSizeBytes +
@ -156,7 +161,7 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
} }
} }
WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer); WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer);
if(retVal == -1) if (retVal == -1)
{ {
return kSizeError; return kSizeError;
} }
@ -201,6 +206,17 @@ WebRtc_Word32 VCMFrameBuffer::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
return 0; return 0;
} }
// Zero out all entries in list up to and including the (first) entry equal to
// _lowSeqNum. Hybrid mode: 1. Don't NACK FEC packets 2. Make a smart decision
// on whether to NACK or not
WebRtc_Word32 VCMFrameBuffer::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 num,
float rttScore)
{
return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttScore);
}
void VCMFrameBuffer::IncrementNackCount() void VCMFrameBuffer::IncrementNackCount()
{ {
_nackCount++; _nackCount++;
@ -227,7 +243,6 @@ void VCMFrameBuffer::Reset()
{ {
_length = 0; _length = 0;
_timeStamp = 0; _timeStamp = 0;
_sessionInfo.Reset(); _sessionInfo.Reset();
_frameCounted = false; _frameCounted = false;
_payloadType = 0; _payloadType = 0;
@ -237,7 +252,7 @@ void VCMFrameBuffer::Reset()
VCMEncodedFrame::Reset(); VCMEncodedFrame::Reset();
} }
// Makes sure the session contain a decodable stream. // Makes sure the session contains a decodable stream.
void void
VCMFrameBuffer::MakeSessionDecodable() VCMFrameBuffer::MakeSessionDecodable()
{ {
@ -275,7 +290,8 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
case kStateComplete: case kStateComplete:
assert(_state == kStateEmpty || assert(_state == kStateEmpty ||
_state == kStateIncomplete); _state == kStateIncomplete ||
_state == kStateDecodable);
break; break;
@ -286,11 +302,22 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
case kStateDecoding: case kStateDecoding:
// we can go to this state from state kStateComplete kStateIncomplete // we can go to this state from state kStateComplete kStateIncomplete
assert(_state == kStateComplete || _state == kStateIncomplete); assert(_state == kStateComplete || _state == kStateIncomplete ||
_state == kStateDecodable);
// Transfer frame information to EncodedFrame and create any codec specific information // Transfer frame information to EncodedFrame and create any codec specific information
RestructureFrameInformation(); RestructureFrameInformation();
break; break;
case kStateDecodable:
if (_state == kStateComplete)
{
// if complete, obviously decodable, keep as is.
return;
}
assert(_state == kStateEmpty ||
_state == kStateIncomplete);
break;
default: default:
// Should never happen // Should never happen
assert(!"FrameBuffer::SetState Incorrect frame buffer state as input"); assert(!"FrameBuffer::SetState Incorrect frame buffer state as input");

4
modules/video_coding/main/source/frame_buffer.h
View File

@ -64,6 +64,10 @@ public:
// NACK // NACK
// Zero out all entries in list up to and including the entry equal to _lowSeqNum // Zero out all entries in list up to and including the entry equal to _lowSeqNum
WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num); WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num);
// Hybrid extension: only NACK important packets, discard FEC packets
WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 num,
float rttScore);
void IncrementNackCount(); void IncrementNackCount();
WebRtc_Word16 GetNackCount() const; WebRtc_Word16 GetNackCount() const;

329
modules/video_coding/main/source/jitter_buffer.cc
View File

@ -16,6 +16,7 @@
#include "jitter_buffer.h" #include "jitter_buffer.h"
#include "jitter_buffer_common.h" #include "jitter_buffer_common.h"
#include "jitter_estimator.h" #include "jitter_estimator.h"
#include "media_optimization.h" // hybrid NACK/FEC thresholds.
#include "packet.h" #include "packet.h"
#include "event.h" #include "event.h"
@ -46,11 +47,14 @@ VCMJitterBuffer::FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestam
} }
bool bool
VCMJitterBuffer::CompleteKeyFrameCriteria(VCMFrameBuffer* frame, const void* /*notUsed*/) VCMJitterBuffer::CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame,
const void* /*notUsed*/)
{ {
const VCMFrameBufferStateEnum state = frame->GetState(); const VCMFrameBufferStateEnum state = frame->GetState();
// We can decode key frame or decodable/complete frames.
return (frame->FrameType() == kVideoFrameKey) && return (frame->FrameType() == kVideoFrameKey) &&
(state == kStateComplete); ((state == kStateComplete)
|| (state == kStateDecodable));
} }
// Constructor // Constructor
@ -76,7 +80,8 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId,
_numConsecutiveOldFrames(0), _numConsecutiveOldFrames(0),
_numConsecutiveOldPackets(0), _numConsecutiveOldPackets(0),
_jitterEstimate(vcmId, receiverId), _jitterEstimate(vcmId, receiverId),
_usingNACK(false), _rttMs(0),
_nackMode(kNoNack),
_NACKSeqNum(), _NACKSeqNum(),
_NACKSeqNumLength(0), _NACKSeqNumLength(0),
_missingMarkerBits(false), _missingMarkerBits(false),
@ -87,7 +92,7 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId,
_lastDecodedSeqNum = -1; _lastDecodedSeqNum = -1;
memset(_NACKSeqNumInternal, -1, sizeof(_NACKSeqNumInternal)); memset(_NACKSeqNumInternal, -1, sizeof(_NACKSeqNumInternal));
for (int i=0; i< kStartNumberOfFrames; i++) for (int i = 0; i< kStartNumberOfFrames; i++)
{ {
_frameBuffers[i] = new VCMFrameBuffer(); _frameBuffers[i] = new VCMFrameBuffer();
} }
@ -130,7 +135,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs)
_jitterEstimate = rhs._jitterEstimate; _jitterEstimate = rhs._jitterEstimate;
_delayEstimate = rhs._delayEstimate; _delayEstimate = rhs._delayEstimate;
_waitingForCompletion = rhs._waitingForCompletion; _waitingForCompletion = rhs._waitingForCompletion;
_usingNACK = rhs._usingNACK; _nackMode = rhs._nackMode;
_rttMs = rhs._rttMs;
_NACKSeqNumLength = rhs._NACKSeqNumLength; _NACKSeqNumLength = rhs._NACKSeqNumLength;
_missingMarkerBits = rhs._missingMarkerBits; _missingMarkerBits = rhs._missingMarkerBits;
_firstPacket = rhs._firstPacket; _firstPacket = rhs._firstPacket;
@ -138,7 +144,7 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs)
memcpy(_receiveStatistics, rhs._receiveStatistics, sizeof(_receiveStatistics)); memcpy(_receiveStatistics, rhs._receiveStatistics, sizeof(_receiveStatistics));
memcpy(_NACKSeqNumInternal, rhs._NACKSeqNumInternal, sizeof(_NACKSeqNumInternal)); memcpy(_NACKSeqNumInternal, rhs._NACKSeqNumInternal, sizeof(_NACKSeqNumInternal));
memcpy(_NACKSeqNum, rhs._NACKSeqNum, sizeof(_NACKSeqNum)); memcpy(_NACKSeqNum, rhs._NACKSeqNum, sizeof(_NACKSeqNum));
for (int i=0; i < kMaxNumberOfFrames; i++) for (int i = 0; i < kMaxNumberOfFrames; i++)
{ {
if (_frameBuffers[i] != NULL) if (_frameBuffers[i] != NULL)
{ {
@ -147,7 +153,7 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs)
} }
} }
while(_frameBuffersTSOrder.Erase(_frameBuffersTSOrder.First()) != -1) { } while(_frameBuffersTSOrder.Erase(_frameBuffersTSOrder.First()) != -1) { }
for (int i=0; i < _maxNumberOfFrames; i++) for (int i = 0; i < _maxNumberOfFrames; i++)
{ {
_frameBuffers[i] = new VCMFrameBuffer(*(rhs._frameBuffers[i])); _frameBuffers[i] = new VCMFrameBuffer(*(rhs._frameBuffers[i]));
if (_frameBuffers[i]->Length() > 0) if (_frameBuffers[i]->Length() > 0)
@ -162,7 +168,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs)
} }
WebRtc_UWord32 WebRtc_UWord32
VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, const WebRtc_UWord32 newTimestamp) VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp,
const WebRtc_UWord32 newTimestamp)
{ {
bool wrap = (newTimestamp < 0x0000ffff && existingTimestamp > 0xffff0000) || bool wrap = (newTimestamp < 0x0000ffff && existingTimestamp > 0xffff0000) ||
(newTimestamp > 0xffff0000 && existingTimestamp < 0x0000ffff); (newTimestamp > 0xffff0000 && existingTimestamp < 0x0000ffff);
@ -185,7 +192,8 @@ VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, const W
} }
// Start jitter buffer // Start jitter buffer
void VCMJitterBuffer::Start() void
VCMJitterBuffer::Start()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_running = true; _running = true;
@ -205,12 +213,14 @@ void VCMJitterBuffer::Start()
_waitingForCompletion.latestPacketTime = -1; _waitingForCompletion.latestPacketTime = -1;
_missingMarkerBits = false; _missingMarkerBits = false;
_firstPacket = true; _firstPacket = true;
_rttMs = 0;
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: start", this); WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: start", this);
} }
// Stop jitter buffer // Stop jitter buffer
void VCMJitterBuffer::Stop() void
VCMJitterBuffer::Stop()
{ {
_critSect.Enter(); _critSect.Enter();
_running = false; _running = false;
@ -231,21 +241,24 @@ void VCMJitterBuffer::Stop()
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: stop", this); WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: stop", this);
} }
bool VCMJitterBuffer::Running() const bool
VCMJitterBuffer::Running() const
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return _running; return _running;
} }
// Flush jitter buffer // Flush jitter buffer
void VCMJitterBuffer::Flush() void
VCMJitterBuffer::Flush()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
FlushInternal(); FlushInternal();
} }
// Must be called under the critical section _critSect // Must be called under the critical section _critSect
void VCMJitterBuffer::FlushInternal() void
VCMJitterBuffer::FlushInternal()
{ {
// Erase all frames from the sorted list and set their state to free. // Erase all frames from the sorted list and set their state to free.
_frameBuffersTSOrder.Flush(); _frameBuffersTSOrder.Flush();
@ -292,7 +305,8 @@ VCMJitterBuffer::ReleaseFrameInternal(VCMFrameBuffer* frame)
// Doing it here increases the degree of freedom for e.g. future // Doing it here increases the degree of freedom for e.g. future
// reconstructability of separate layers. Must be called under the // reconstructability of separate layers. Must be called under the
// critical section _critSect. // critical section _critSect.
void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) void
VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
{ {
if (frame == NULL) if (frame == NULL)
{ {
@ -385,8 +399,7 @@ void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
// Only signal if this is the oldest frame. // Only signal if this is the oldest frame.
// Not necessary the case due to packet reordering or NACK. // Not necessary the case due to packet reordering or NACK.
if(!_usingNACK || if (!WaitForNack() || (oldFrame != NULL && oldFrame == frame))
(oldFrame != NULL && oldFrame == frame))
{ {
_frameEvent.Set(); _frameEvent.Set();
} }
@ -481,7 +494,7 @@ VCMJitterBuffer::GetEmptyFrame()
_critSect.Enter(); _critSect.Enter();
for (int i=0; i<_maxNumberOfFrames; ++i) for (int i = 0; i <_maxNumberOfFrames; ++i)
{ {
if (kStateFree == _frameBuffers[i]->GetState()) if (kStateFree == _frameBuffers[i]->GetState())
{ {
@ -512,7 +525,8 @@ VCMJitterBuffer::GetEmptyFrame()
} }
// Must be called under the critical section _critSect. // Must be called under the critical section _critSect.
VCMFrameListItem* VCMJitterBuffer::FindOldestSequenceNum() const VCMFrameListItem*
VCMJitterBuffer::FindOldestSequenceNum() const
{ {
WebRtc_UWord16 currentLow = 0xffff; WebRtc_UWord16 currentLow = 0xffff;
VCMFrameBufferStateEnum state = kStateFree; VCMFrameBufferStateEnum state = kStateFree;
@ -562,7 +576,8 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestSequenceNum() const
// Must be called under critical section // Must be called under critical section
// Based on sequence number // Based on sequence number
// Return NULL for lost packets // Return NULL for lost packets
VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame() VCMFrameListItem*
VCMJitterBuffer::FindOldestCompleteContinuousFrame()
{ {
// if we have more than one frame done since last time, pick oldest // if we have more than one frame done since last time, pick oldest
VCMFrameBuffer* oldestFrame = NULL; VCMFrameBuffer* oldestFrame = NULL;
@ -578,7 +593,8 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame()
{ {
if (kStateComplete != oldestFrame->GetState()) if (kStateComplete != oldestFrame->GetState())
{ {
// Try to see if the frame is complete even though the state is not complete. Can happen if markerbit is not set. // Try to see if the frame is complete even though the state is not
// complete. Can happen if markerbit is not set.
if (!CheckForCompleteFrame(oldestFrameItem)) if (!CheckForCompleteFrame(oldestFrameItem))
{ {
oldestFrame = NULL; oldestFrame = NULL;
@ -590,7 +606,7 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame()
currentLow = oldestFrame->GetLowSeqNum(); currentLow = oldestFrame->GetLowSeqNum();
} }
} }
if(oldestFrame == NULL) if (oldestFrame == NULL)
{ {
// no complete frame no point to continue // no complete frame no point to continue
return NULL; return NULL;
@ -601,15 +617,16 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame()
// Use seqNum not timestamp since a full frame might be lost // Use seqNum not timestamp since a full frame might be lost
if (_lastDecodedSeqNum != -1) if (_lastDecodedSeqNum != -1)
{ {
// it's not enough that we have complete frame we need the seq numbers to be continuous too // it's not enough that we have complete frame we need the seq numbers
// for layers it's not enough that we have complete frame we need the layers to be continuous too // to be continuous too for layers it's not enough that we have complete
// frame we need the layers to be continuous too
currentLow = oldestFrame->GetLowSeqNum(); currentLow = oldestFrame->GetLowSeqNum();
WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum; WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
// we could have received the first packet of the last frame before a long period // we could have received the first packet of the last frame before a
// if drop, that case is handled by GetNackList // long period if drop, that case is handled by GetNackList
if ( ((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow)
{ {
// wait since we want a complete continuous frame // wait since we want a complete continuous frame
return NULL; return NULL;
@ -618,11 +635,12 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame()
return oldestFrameItem; return oldestFrameItem;
} }
// Check if the oldest frame is complete even though it is not in a complete state. // Check if the oldest frame is complete even though it isn't complete.
// This can happen when makerbit is not set // This can happen when makerbit is not set
// Must be called under the critical section _critSect. // Must be called under the critical section _critSect.
// Return false for lost packets // Return false for lost packets
bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) bool
VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem)
{ {
const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameItem); const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameItem);
VCMFrameBuffer* oldestFrame = NULL; VCMFrameBuffer* oldestFrame = NULL;
@ -630,7 +648,7 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem)
{ {
oldestFrame = oldestFrameItem->GetItem(); oldestFrame = oldestFrameItem->GetItem();
} }
if(nextFrameItem != NULL) if (nextFrameItem != NULL)
{ {
// We have received at least one packet from a later frame. // We have received at least one packet from a later frame.
if(!oldestFrame->HaveLastPacket()) // If we don't have the markerbit if(!oldestFrame->HaveLastPacket()) // If we don't have the markerbit
@ -638,7 +656,7 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem)
VCMFrameBuffer* nextFrame = nextFrameItem->GetItem(); VCMFrameBuffer* nextFrame = nextFrameItem->GetItem();
// Verify that we have received the first packet of the next frame. // Verify that we have received the first packet of the next frame.
// This is the only way we can be sure we're not missing the last packet. // This is the only way we can be sure we're not missing the last packet.
if(nextFrame != NULL && nextFrame->GetLowSeqNum() == if (nextFrame != NULL && nextFrame->GetLowSeqNum() ==
static_cast<WebRtc_UWord16>(oldestFrame->GetHighSeqNum()+1)) // Sequence number is only 16 bit static_cast<WebRtc_UWord16>(oldestFrame->GetHighSeqNum()+1)) // Sequence number is only 16 bit
{ {
_missingMarkerBits = true; _missingMarkerBits = true;
@ -648,7 +666,7 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem)
UpdateFrameState(oldestFrame); UpdateFrameState(oldestFrame);
} }
const VCMFrameBufferStateEnum state = oldestFrame->GetState(); const VCMFrameBufferStateEnum state = oldestFrame->GetState();
if(state == kStateComplete) if (state == kStateComplete)
{ {
if(oldestFrame->Length() > 0) if(oldestFrame->Length() > 0)
{ {
@ -663,7 +681,8 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem)
} }
// Call from inside the critical section _critSect // Call from inside the critical section _critSect
void VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame) void
VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame)
{ {
if (frame == NULL) if (frame == NULL)
{ {
@ -679,7 +698,8 @@ void VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame)
// Calculate frame and bit rates // Calculate frame and bit rates
WebRtc_Word32 VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) WebRtc_Word32
VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
const WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); const WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp();
@ -744,7 +764,8 @@ WebRtc_Word32 VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord
} }
// Returns immediately or a X ms event hang waiting for a decodable frame, X decided by caller // Returns immediately or a X ms event hang waiting for a decodable frame, X decided by caller
VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) VCMEncodedFrame*
VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
{ {
if (!_running) if (!_running)
{ {
@ -853,38 +874,54 @@ VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 max
return oldestFrame; return oldestFrame;
} }
WebRtc_UWord32 VCMJitterBuffer::GetEstimatedJitterMS() WebRtc_UWord32
VCMJitterBuffer::GetEstimatedJitterMS()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return GetEstimatedJitterMsInternal(); return GetEstimatedJitterMsInternal();
} }
WebRtc_UWord32 VCMJitterBuffer::GetEstimatedJitterMsInternal() WebRtc_UWord32
VCMJitterBuffer::GetEstimatedJitterMsInternal()
{ {
WebRtc_UWord32 estimate = VCMJitterEstimator::OPERATING_SYSTEM_JITTER; WebRtc_UWord32 estimate = VCMJitterEstimator::OPERATING_SYSTEM_JITTER;
estimate += static_cast<WebRtc_UWord32>(_jitterEstimate.GetJitterEstimate() + 0.5);
// compute RTT multiplier for estimation
double rttMult = 1.0f;
if (_nackMode == kNackHybrid && _rttMs > kLowRttNackMs)
{
// from here we count on FEC
rttMult = 0.0f;
}
estimate += static_cast<WebRtc_UWord32>
(_jitterEstimate.GetJitterEstimate(rttMult) + 0.5);
if (_missingMarkerBits) if (_missingMarkerBits)
{ {
// Since the incoming packets are all missing marker bits we have to wait until the first // Since the incoming packets are all missing marker bits we have to
// packet of the next frame arrives, before we can safely say that the frame is complete. // wait until the first packet of the next frame arrives, before we can
// Therefore we have to compensate the jitter buffer level with one frame period. // safely say that the frame is complete. Therefore we have to compensate
// the jitter buffer level with one frame period.
// TODO(holmer): The timestamp diff should probably be filtered (max filter) since // TODO(holmer): The timestamp diff should probably be filtered
// the diff can alternate between e.g. 3000 and 6000 if we have a frame rate between // (max filter) since the diff can alternate between e.g. 3000 and 6000
// 15 and 30 frames per seconds. // if we have a frame rate between 15 and 30 frames per seconds.
estimate += _delayEstimate.CurrentTimeStampDiffMs(); estimate += _delayEstimate.CurrentTimeStampDiffMs();
} }
return estimate; return estimate;
} }
void VCMJitterBuffer::UpdateRtt(WebRtc_UWord32 rttMs) void
VCMJitterBuffer::UpdateRtt(WebRtc_UWord32 rttMs)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_rttMs = rttMs;
_jitterEstimate.UpdateRtt(rttMs); _jitterEstimate.UpdateRtt(rttMs);
} }
// wait for the first packet in the next frame to arrive // wait for the first packet in the next frame to arrive
WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, FrameType& incomingFrameType, WebRtc_Word64& renderTimeMs) WebRtc_Word64
VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS,
FrameType& incomingFrameType,
WebRtc_Word64& renderTimeMs)
{ {
if (!_running) if (!_running)
{ {
@ -913,7 +950,6 @@ WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, Fr
CleanUpOldFrames(); CleanUpOldFrames();
CleanUpSizeZeroFrames(); CleanUpSizeZeroFrames();
oldestFrame = _frameBuffersTSOrder.FirstFrame(); oldestFrame = _frameBuffersTSOrder.FirstFrame();
}else }else
{ {
@ -946,7 +982,8 @@ WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, Fr
// Will the packet sequence be complete if the next frame is grabbed for decoding right now? // Will the packet sequence be complete if the next frame is grabbed for decoding right now?
// That is, have we lost a frame between the last decoded frame and the next, or is the next // That is, have we lost a frame between the last decoded frame and the next, or is the next
// frame missing one or more packets? // frame missing one or more packets?
bool VCMJitterBuffer::CompleteSequenceWithNextFrame() bool
VCMJitterBuffer::CompleteSequenceWithNextFrame()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
// Finding oldest frame ready for decoder, but check sequence number and size // Finding oldest frame ready for decoder, but check sequence number and size
@ -989,7 +1026,8 @@ bool VCMJitterBuffer::CompleteSequenceWithNextFrame()
} }
// Returns immediately // Returns immediately
VCMEncodedFrame* VCMJitterBuffer::GetFrameForDecoding() VCMEncodedFrame*
VCMJitterBuffer::GetFrameForDecoding()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
if (!_running) if (!_running)
@ -997,7 +1035,7 @@ VCMEncodedFrame* VCMJitterBuffer::GetFrameForDecoding()
return NULL; return NULL;
} }
if(_usingNACK) if (WaitForNack())
{ {
return GetFrameForDecodingNACK(); return GetFrameForDecodingNACK();
} }
@ -1061,6 +1099,7 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
{ {
// when we use NACK we don't release non complete frames // when we use NACK we don't release non complete frames
// unless we have a complete key frame. // unless we have a complete key frame.
// In hybrid mode, we may release decodable frames (non-complete)
// Clean up old frames and empty frames // Clean up old frames and empty frames
CleanUpOldFrames(); CleanUpOldFrames();
@ -1077,8 +1116,9 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
if (oldestFrame == NULL) if (oldestFrame == NULL)
{ {
continuous = false; continuous = false;
// If we didn't find one we're good with a complete key frame. // If we didn't find one we're good with a complete key/decodable frame.
oldestFrameListItem = _frameBuffersTSOrder.FindFrameListItem(CompleteKeyFrameCriteria); oldestFrameListItem = _frameBuffersTSOrder.FindFrameListItem(
CompleteDecodableKeyFrameCriteria);
if (oldestFrameListItem != NULL) if (oldestFrameListItem != NULL)
{ {
oldestFrame = oldestFrameListItem->GetItem(); oldestFrame = oldestFrameListItem->GetItem();
@ -1089,7 +1129,7 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
} }
} }
// We have a complete continuous frame, decode it. // We have a complete/decodable continuous frame, decode it.
// store seqnum // store seqnum
_lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); _lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
// store current time // store current time
@ -1181,7 +1221,10 @@ VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, bool incom
// Must be called under the critical section _critSect. Should never be called with // Must be called under the critical section _critSect. Should never be called with
// retransmitted frames, they must be filtered out before this function is called. // retransmitted frames, they must be filtered out before this function is called.
void void
VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs, WebRtc_UWord32 timestamp, WebRtc_UWord32 frameSize, bool incompleteFrame) VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs,
WebRtc_UWord32 timestamp,
WebRtc_UWord32 frameSize,
bool incompleteFrame)
{ {
if (latestPacketTimeMs == -1) if (latestPacketTimeMs == -1)
{ {
@ -1217,17 +1260,18 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word
highSeqNum = -1; highSeqNum = -1;
lowSeqNum = _lastDecodedSeqNum; lowSeqNum = _lastDecodedSeqNum;
// find higest seqnumbers // find highest seqnumbers
for (i=0; i<_maxNumberOfFrames; ++i) for (i = 0; i < _maxNumberOfFrames; ++i)
{ {
seqNum = _frameBuffers[i]->GetHighSeqNum(); seqNum = _frameBuffers[i]->GetHighSeqNum();
// Ignore free frames // Ignore free / empty frames
VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState(); VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState();
if((kStateFree != state) &&
if ((kStateFree != state) &&
(kStateEmpty != state) && (kStateEmpty != state) &&
(kStateDecoding != state) && (kStateDecoding != state) &&
seqNum != -1) seqNum != -1)
{ {
if (highSeqNum == -1) if (highSeqNum == -1)
{ {
@ -1260,9 +1304,10 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
int i = 0; int i = 0;
WebRtc_Word32 lowSeqNum = -1; WebRtc_Word32 lowSeqNum = -1;
WebRtc_Word32 highSeqNum = -1; WebRtc_Word32 highSeqNum = -1;
listExtended=false; listExtended = false;
if (!_usingNACK) // don't create list, if we won't wait for it
if (!WaitForNack())
{ {
nackSize = 0; nackSize = 0;
return NULL; return NULL;
@ -1277,11 +1322,10 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
// write a list of all seq num we have // write a list of all seq num we have
if (lowSeqNum == -1 || highSeqNum == -1) if (lowSeqNum == -1 || highSeqNum == -1)
{ {
//This happens if we lose the first packet, nothing is poped //This happens if we lose the first packet, nothing is popped
if (highSeqNum == -1) if (highSeqNum == -1)
{ {
nackSize = 0;// we have not received any packets yet nackSize = 0;// we have not received any packets yet
} }
else else
{ {
@ -1371,7 +1415,8 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
// We have cleaned up the jb and found a key frame // We have cleaned up the jb and found a key frame
// The function itself has set last decoded seq. // The function itself has set last decoded seq.
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1, WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1,
"\tKey frame found. _lastDecodedSeqNum[0] %d", _lastDecodedSeqNum); "\tKey frame found. _lastDecodedSeqNum[0] %d",
_lastDecodedSeqNum);
nackSize = 0; nackSize = 0;
} }
@ -1379,27 +1424,51 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
} }
WebRtc_UWord16 seqNumberIterator = (WebRtc_UWord16)(lowSeqNum + 1); WebRtc_UWord16 seqNumberIterator = (WebRtc_UWord16)(lowSeqNum + 1);
for (i=0; i < numberOfSeqNum; i++) for (i = 0; i < numberOfSeqNum; i++)
{ {
_NACKSeqNumInternal[i] = seqNumberIterator; _NACKSeqNumInternal[i] = seqNumberIterator;
seqNumberIterator++; seqNumberIterator++;
} }
// now we have a list of all seq numbers that could have been sent
// now we have a list of all sequence numbers that could have been sent
// zero out the ones we have received // zero out the ones we have received
for (i = 0; i < _maxNumberOfFrames; i++) for (i = 0; i < _maxNumberOfFrames; i++)
{ {
// loop all created frames // loop all created frames
// We dont need to check if frame is decoding since lowSeqNum is based on _lastDecodedSeqNum // We don't need to check if frame is decoding since lowSeqNum is based
// on _lastDecodedSeqNum
// Ignore free frames // Ignore free frames
VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState(); VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState();
if ((kStateFree != state) && if ((kStateFree != state) &&
(kStateEmpty != state) && (kStateEmpty != state) &&
(kStateDecoding != state)) (kStateDecoding != state))
{ {
_frameBuffers[i]->ZeroOutSeqNum(_NACKSeqNumInternal, numberOfSeqNum); // Reaching thus far means we are going to update the nack list
// used when the frame is being processed by the decoding thread // When in hybrid mode, we also need to check empty frames, so as not
// dont need to use that info in this loop // to add empty packets to the nack list
if (_nackMode == kNackHybrid)
{
// build external rttScore based on RTT value
float rttScore = 1.0f;
_frameBuffers[i]->ZeroOutSeqNumHybrid(_NACKSeqNumInternal,
numberOfSeqNum,
rttScore);
if (_frameBuffers[i]->IsRetransmitted() == false)
{
// if no retransmission required,set the state to decodable
// meaning that we will not wait for NACK
_frameBuffers[i]->SetState(kStateDecodable);
}
}
else
{
// used when the frame is being processed by the decoding thread
// don't need to use that info in this loop
_frameBuffers[i]->ZeroOutSeqNum(_NACKSeqNumInternal,
numberOfSeqNum);
}
} }
} }
@ -1407,7 +1476,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
int emptyIndex = -1; int emptyIndex = -1;
for (i = 0; i < numberOfSeqNum; i++) for (i = 0; i < numberOfSeqNum; i++)
{ {
if (_NACKSeqNumInternal[i] == -1) if (_NACKSeqNumInternal[i] == -1 || _NACKSeqNumInternal[i] == -2 )
{ {
// this is empty // this is empty
if (emptyIndex == -1) if (emptyIndex == -1)
@ -1442,36 +1511,39 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
nackSize = emptyIndex; nackSize = emptyIndex;
} }
// convert to unsigned short 16 bit and store in a list to be used externally. // convert to unsigned short 16 bit and store in a list to be used externally.
if(nackSize > _NACKSeqNumLength) if (nackSize > _NACKSeqNumLength)
{ {
listExtended=true; // Larger list means that the nack list has been extended since the last call. // Larger list means that the nack list was extended since the last call.
listExtended = true;
} }
for(WebRtc_UWord32 j = 0; j < nackSize; j++) for(WebRtc_UWord32 j = 0; j < nackSize; j++)
{ {
// Check if the list has been extended since it was last created. I.e, new items have been added // Check if the list has been extended since it was last created. I.e,
if(_NACKSeqNumLength > j && !listExtended) // new items have been added
if (_NACKSeqNumLength > j && !listExtended)
{ {
WebRtc_UWord32 k = 0; WebRtc_UWord32 k = 0;
for(k = j; k < _NACKSeqNumLength; k++) for (k = j; k < _NACKSeqNumLength; k++)
{ {
// Found the item in the last list. I.e, no new items found yet. // Found the item in the last list. I.e, no new items found yet.
if(_NACKSeqNum[k] == (WebRtc_UWord16)_NACKSeqNumInternal[j]) if (_NACKSeqNum[k] == (WebRtc_UWord16)_NACKSeqNumInternal[j])
{ {
break; break;
} }
} }
if(k == _NACKSeqNumLength) // New item not found in last list. if (k == _NACKSeqNumLength) // New item not found in last list.
{ {
listExtended=true; listExtended = true;
} }
} }
else else
{ {
listExtended=true; listExtended = true;
} }
_NACKSeqNum[j] = (WebRtc_UWord16)_NACKSeqNumInternal[j]; _NACKSeqNum[j] = (WebRtc_UWord16)_NACKSeqNumInternal[j];
} }
_NACKSeqNumLength = nackSize; _NACKSeqNumLength = nackSize;
return _NACKSeqNum; return _NACKSeqNum;
@ -1511,35 +1583,39 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
VCMFrameBufferEnum ret = kSizeError; VCMFrameBufferEnum ret = kSizeError;
VCMFrameBuffer* frame = static_cast<VCMFrameBuffer*>(buffer); VCMFrameBuffer* frame = static_cast<VCMFrameBuffer*>(buffer);
if (_firstPacket) // Empty packets may bias the jitter estimate (lacking size component),
// therefore don't let empty packet trigger the following updates:
if (packet.frameType != kFrameEmpty)
{ {
// Now it's time to start estimating jitter if (_firstPacket)
// reset the delay estimate. {
_delayEstimate.Reset(); // Now it's time to start estimating jitter
_firstPacket = false; // reset the delay estimate.
} _delayEstimate.Reset();
_firstPacket = false;
}
if (_waitingForCompletion.timestamp == packet.timestamp) if (_waitingForCompletion.timestamp == packet.timestamp)
{ {
// This can get bad if we have a lot of duplicate packets, // This can get bad if we have a lot of duplicate packets,
// we will then count some packet multiple times. // we will then count some packet multiple times.
_waitingForCompletion.frameSize += packet.sizeBytes; _waitingForCompletion.frameSize += packet.sizeBytes;
_waitingForCompletion.latestPacketTime = nowMs; _waitingForCompletion.latestPacketTime = nowMs;
} }
else if (_waitingForCompletion.latestPacketTime >= 0 && else if (_waitingForCompletion.latestPacketTime >= 0 &&
_waitingForCompletion.latestPacketTime + 2000 <= nowMs) _waitingForCompletion.latestPacketTime + 2000 <= nowMs)
{ {
// A packet should never be more than two seconds late // A packet should never be more than two seconds late
UpdateJitterAndDelayEstimates(_waitingForCompletion, true); UpdateJitterAndDelayEstimates(_waitingForCompletion, true);
_waitingForCompletion.latestPacketTime = -1; _waitingForCompletion.latestPacketTime = -1;
_waitingForCompletion.frameSize = 0; _waitingForCompletion.frameSize = 0;
_waitingForCompletion.timestamp = 0; _waitingForCompletion.timestamp = 0;
}
} }
if (frame != NULL) if (frame != NULL)
{ {
VCMFrameBufferStateEnum state = frame->GetState(); VCMFrameBufferStateEnum state = frame->GetState();
if (state == kStateDecoding && packet.sizeBytes == 0) if (state == kStateDecoding && packet.sizeBytes == 0)
{ {
// Filler packet, make sure we update the last decoded seq num // Filler packet, make sure we update the last decoded seq num
@ -1655,7 +1731,7 @@ VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const
{ {
if (_NACKSeqNum && _NACKSeqNumLength > 0) if (_NACKSeqNum && _NACKSeqNumLength > 0)
{ {
for (WebRtc_UWord16 i=0; i < _NACKSeqNumLength; i++) for (WebRtc_UWord16 i = 0; i < _NACKSeqNumLength; i++)
{ {
if (packet.seqNum == _NACKSeqNum[i]) if (packet.seqNum == _NACKSeqNum[i])
{ {
@ -1667,18 +1743,20 @@ VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const
} }
// Get nack status (enabled/disabled) // Get nack status (enabled/disabled)
bool VCMJitterBuffer::GetNackStatus() VCMNackMode
VCMJitterBuffer::GetNackMode() const
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return _usingNACK; return _nackMode;
} }
// Enable/disable nack // Set NACK mode
void VCMJitterBuffer::SetNackStatus(bool enable) void
VCMJitterBuffer::SetNackMode(VCMNackMode mode)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_usingNACK = enable; _nackMode = mode;
if (!_usingNACK) if (_nackMode == kNoNack)
{ {
_jitterEstimate.ResetNackCount(); _jitterEstimate.ResetNackCount();
} }
@ -1808,7 +1886,7 @@ VCMJitterBuffer::CleanUpSizeZeroFrames()
const WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum(); const WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum();
const WebRtc_Word32 frameLowSeqNum = ptrTempBuffer->GetLowSeqNum(); const WebRtc_Word32 frameLowSeqNum = ptrTempBuffer->GetLowSeqNum();
if ((frameLowSeqNum == (_lastDecodedSeqNum+ 1)) || // Frame is next in line if ((frameLowSeqNum == (_lastDecodedSeqNum + 1)) || // Frame is next in line
((frameLowSeqNum == 0) && (_lastDecodedSeqNum== 0xffff))) ((frameLowSeqNum == 0) && (_lastDecodedSeqNum== 0xffff)))
{ {
// This frame follows the last decoded frame, release it. // This frame follows the last decoded frame, release it.
@ -1878,4 +1956,33 @@ VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame)
} }
} }
bool
VCMJitterBuffer::WaitForNack()
{
// NACK disabled -> can't wait
if (_nackMode == kNoNack)
{
return false;
}
// NACK only -> always wait
else if (_nackMode == kNackInfinite)
{
return true;
}
// else: hybrid mode, evaluate
// RTT high, don't wait
if (_rttMs >= kHighRttNackMs)
{
return false;
}
// RTT low, we can afford the wait
else if (_rttMs <= kLowRttNackMs)
{
return true;
}
// interim values - hybrid mode
return true;
}
} }

35
modules/video_coding/main/source/jitter_buffer.h
View File

@ -24,6 +24,13 @@
namespace webrtc namespace webrtc
{ {
enum VCMNackMode
{
kNackInfinite,
kNackHybrid,
kNoNack
};
// forward declarations // forward declarations
class VCMFrameBuffer; class VCMFrameBuffer;
class VCMPacket; class VCMPacket;
@ -90,7 +97,7 @@ public:
WebRtc_Word32 GetFrame(const VCMPacket& packet, VCMEncodedFrame*&); WebRtc_Word32 GetFrame(const VCMPacket& packet, VCMEncodedFrame*&);
VCMEncodedFrame* GetFrame(const VCMPacket& packet); // deprecated VCMEncodedFrame* GetFrame(const VCMPacket& packet); // deprecated
// Returns the time in ms when the latest packet was insterted into the frame. // Returns the time in ms when the latest packet was inserted into the frame.
// Retransmitted is set to true if any of the packets belonging to the frame // Retransmitted is set to true if any of the packets belonging to the frame
// has been retransmitted. // has been retransmitted.
WebRtc_Word64 LastPacketTime(VCMEncodedFrame* frame, bool& retransmitted) const; WebRtc_Word64 LastPacketTime(VCMEncodedFrame* frame, bool& retransmitted) const;
@ -103,8 +110,8 @@ public:
void UpdateRtt(WebRtc_UWord32 rttMs); void UpdateRtt(WebRtc_UWord32 rttMs);
// NACK // NACK
void SetNackStatus(bool enable); // Enable/disable nack void SetNackMode(VCMNackMode mode); // Enable/disable nack
bool GetNackStatus(); // Get nack status (enabled/disabled) VCMNackMode GetNackMode() const; // Get nack mode
// Get list of missing sequence numbers (size in number of elements) // Get list of missing sequence numbers (size in number of elements)
WebRtc_UWord16* GetNackList(WebRtc_UWord16& nackSize, bool& listExtended); WebRtc_UWord16* GetNackList(WebRtc_UWord16& nackSize, bool& listExtended);
@ -162,18 +169,23 @@ protected:
private: private:
static bool FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp); static bool FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp);
static bool CompleteKeyFrameCriteria(VCMFrameBuffer* frame, const void* notUsed); static bool CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame,
const void* notUsed);
// Decide whether should wait for NACK (mainly relevant for hybrid mode)
bool WaitForNack();
WebRtc_Word32 _vcmId; WebRtc_Word32 _vcmId;
WebRtc_Word32 _receiverId; WebRtc_Word32 _receiverId;
bool _running; // If we are running (have started) or not // If we are running (have started) or not
CriticalSectionWrapper& _critSect; bool _running;
CriticalSectionWrapper& _critSect;
bool _master; bool _master;
// Event to signal when we have a frame ready for decoder // Event to signal when we have a frame ready for decoder
VCMEvent _frameEvent; VCMEvent _frameEvent;
// Event to signal when we have received a packet // Event to signal when we have received a packet
VCMEvent _packetEvent; VCMEvent _packetEvent;
WebRtc_Word32 _maxNumberOfFrames; // Number of allocated frames // Number of allocated frames
WebRtc_Word32 _maxNumberOfFrames;
// Array of pointers to the frames in JB // Array of pointers to the frames in JB
VCMFrameBuffer* _frameBuffers[kMaxNumberOfFrames]; VCMFrameBuffer* _frameBuffers[kMaxNumberOfFrames];
VCMFrameListTimestampOrderAsc _frameBuffersTSOrder; VCMFrameListTimestampOrderAsc _frameBuffersTSOrder;
@ -189,10 +201,12 @@ private:
WebRtc_UWord8 _receiveStatistics[4]; WebRtc_UWord8 _receiveStatistics[4];
// Latest calculated frame rates of incoming stream // Latest calculated frame rates of incoming stream
WebRtc_UWord8 _incomingFrameRate; WebRtc_UWord8 _incomingFrameRate;
WebRtc_UWord32 _incomingFrameCount; // Frame counter, reset in GetUpdate // Frame counter, reset in GetUpdate
WebRtc_UWord32 _incomingFrameCount;
// Real time for last _frameCount reset // Real time for last _frameCount reset
WebRtc_Word64 _timeLastIncomingFrameCount; WebRtc_Word64 _timeLastIncomingFrameCount;
WebRtc_UWord32 _incomingBitCount; // Received bits counter, reset in GetUpdate // Received bits counter, reset in GetUpdate
WebRtc_UWord32 _incomingBitCount;
WebRtc_UWord32 _incomingBitRate; WebRtc_UWord32 _incomingBitRate;
WebRtc_UWord32 _dropCount; // Frame drop counter WebRtc_UWord32 _dropCount; // Frame drop counter
// Number of frames in a row that have been too old // Number of frames in a row that have been too old
@ -204,9 +218,10 @@ private:
// Calculates network delays used for jitter calculations // Calculates network delays used for jitter calculations
VCMInterFrameDelay _delayEstimate; VCMInterFrameDelay _delayEstimate;
VCMJitterSample _waitingForCompletion; VCMJitterSample _waitingForCompletion;
WebRtc_UWord32 _rttMs;
// NACK // NACK
bool _usingNACK; // If we are using nack VCMNackMode _nackMode;
// Holds the internal nack list (the missing seqence numbers) // Holds the internal nack list (the missing seqence numbers)
WebRtc_Word32 _NACKSeqNumInternal[kNackHistoryLength]; WebRtc_Word32 _NACKSeqNumInternal[kNackHistoryLength];
WebRtc_UWord16 _NACKSeqNum[kNackHistoryLength]; WebRtc_UWord16 _NACKSeqNum[kNackHistoryLength];

3
modules/video_coding/main/source/jitter_buffer_common.h
View File

@ -46,7 +46,8 @@ enum VCMFrameBufferStateEnum
kStateEmpty, // frame popped by the RTP receiver kStateEmpty, // frame popped by the RTP receiver
kStateIncomplete, // frame that have one or more packet(s) stored kStateIncomplete, // frame that have one or more packet(s) stored
kStateComplete, // frame that have all packets kStateComplete, // frame that have all packets
kStateDecoding // frame popped by the decoding thread kStateDecoding, // frame popped by the decoding thread
kStateDecodable // Hybrid mode - frame can be decoded
}; };
enum { kH264StartCodeLengthBytes = 4}; enum { kH264StartCodeLengthBytes = 4};

4
modules/video_coding/main/source/jitter_estimator.cc
View File

@ -422,7 +422,7 @@ VCMJitterEstimator::UpdateMaxFrameSize(WebRtc_UWord32 frameSizeBytes)
// Returns the current filtered estimate if available, // Returns the current filtered estimate if available,
// otherwise tries to calculate an estimate. // otherwise tries to calculate an estimate.
double double
VCMJitterEstimator::GetJitterEstimate() VCMJitterEstimator::GetJitterEstimate(double rttMultiplier)
{ {
double jitterMS = CalculateEstimate(); double jitterMS = CalculateEstimate();
if (_filterJitterEstimate > jitterMS) if (_filterJitterEstimate > jitterMS)
@ -431,7 +431,7 @@ VCMJitterEstimator::GetJitterEstimate()
} }
if (_nackCount >= _nackLimit) if (_nackCount >= _nackLimit)
{ {
return jitterMS + _rttFilter.RttMs(); return jitterMS + _rttFilter.RttMs() * rttMultiplier;
} }
return jitterMS; return jitterMS;
} }

4
modules/video_coding/main/source/jitter_estimator.h
View File

@ -41,9 +41,11 @@ public:
// Returns the current jitter estimate in milliseconds and adds // Returns the current jitter estimate in milliseconds and adds
// also adds an RTT dependent term in cases of retransmission. // also adds an RTT dependent term in cases of retransmission.
// Input:
// - rttMultiplier : RTT param multiplier (when applicable).
// //
// Return value : Jitter estimate in milliseconds // Return value : Jitter estimate in milliseconds
double GetJitterEstimate(); double GetJitterEstimate(double rttMultiplier);
// Updates the nack counter/timer. // Updates the nack counter/timer.
// //

815
modules/video_coding/main/source/media_opt_util.cc
View File

File diff suppressed because it is too large Load Diff

17
modules/video_coding/main/source/media_opt_util.h
View File

@ -34,6 +34,13 @@ enum VCMFecTypes
kXORFec kXORFec
}; };
// Thresholds for hybrid NACK/FEC
// common to media optimization and the jitter buffer.
enum HybridNackTH {
kHighRttNackMs = 100,
kLowRttNackMs = 20
};
struct VCMProtectionParameters struct VCMProtectionParameters
{ {
VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), packetsPerFrame(0), VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), packetsPerFrame(0),
@ -134,16 +141,16 @@ public:
WebRtc_UWord8 _effectivePacketLoss; WebRtc_UWord8 _effectivePacketLoss;
WebRtc_UWord8 _protectionFactorK; WebRtc_UWord8 _protectionFactorK;
WebRtc_UWord8 _protectionFactorD; WebRtc_UWord8 _protectionFactorD;
float _residualPacketLoss; float _residualPacketLoss;
float _scaleProtKey; float _scaleProtKey;
WebRtc_Word32 _maxPayloadSize; WebRtc_Word32 _maxPayloadSize;
protected: protected:
float _efficiency; float _efficiency;
float _score; float _score;
private: private:
const enum VCMProtectionMethodEnum _type; const enum VCMProtectionMethodEnum _type;
}; };

85
modules/video_coding/main/source/media_optimization.cc
View File

@ -100,7 +100,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
VCMFecTypes fecType = kXORFec; // generic FEC VCMFecTypes fecType = kXORFec; // generic FEC
_lossProtLogic->UpdateFecType(fecType); _lossProtLogic->UpdateFecType(fecType);
//Get frame rate for encoder: this is the actual/sent frame rate // Get frame rate for encoder: this is the actual/sent frame rate
float actualFrameRate = SentFrameRate(); float actualFrameRate = SentFrameRate();
// sanity // sanity
@ -109,13 +109,16 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
actualFrameRate = 1.0; actualFrameRate = 1.0;
} }
// Update frame rate for the loss protection logic class: frame rate should be the actual/sent rate // Update frame rate for the loss protection logic class: frame rate should
// be the actual/sent rate
_lossProtLogic->UpdateFrameRate(actualFrameRate); _lossProtLogic->UpdateFrameRate(actualFrameRate);
_fractionLost = fractionLost; _fractionLost = fractionLost;
// The effective packet loss may be the received loss or filtered, i.e., average or max filter may be used. // The effective packet loss may be the received loss or filtered, i.e.,
//We should think about which filter is appropriate for low/high bit rates, low/high loss rates, etc. // average or max filter may be used.
// We should think about which filter is appropriate for low/high bit rates,
// low/high loss rates, etc.
WebRtc_UWord8 packetLossEnc = _lossProtLogic->FilteredLoss(); WebRtc_UWord8 packetLossEnc = _lossProtLogic->FilteredLoss();
//For now use the filtered loss for computing the robustness settings //For now use the filtered loss for computing the robustness settings
@ -124,46 +127,48 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
// Rate cost of the protection methods // Rate cost of the protection methods
_lossProtOverhead = 0; _lossProtOverhead = 0;
if(selectedMethod) if (selectedMethod && (selectedMethod->Type() == kFEC ||
selectedMethod->Type() == kNackFec ))
{ {
//Update method will compute the robustness settings for the given protection method and the overhead cost // Update method will compute the robustness settings for the given
//the protection method is set by the user via SetVideoProtection. // protection method and the overhead cost
//The robustness settings are: the effecitve packet loss for ER and the FEC protection settings // the protection method is set by the user via SetVideoProtection.
// The robustness settings are: the effective packet loss for ER and the
// FEC protection settings
_lossProtLogic->UpdateMethod(); _lossProtLogic->UpdateMethod();
//Get the code rate for Key frames // Get the code rate for Key frames
const WebRtc_UWord8 codeRateKeyRTP = selectedMethod->RequiredProtectionFactorK(); const WebRtc_UWord8 codeRateKeyRTP = selectedMethod->RequiredProtectionFactorK();
//Get the code rate for Delta frames // Get the code rate for Delta frames
const WebRtc_UWord8 codeRateDeltaRTP = selectedMethod->RequiredProtectionFactorD(); const WebRtc_UWord8 codeRateDeltaRTP = selectedMethod->RequiredProtectionFactorD();
//Get the effective packet loss for ER // Get the effective packet loss for ER
packetLossEnc = selectedMethod->RequiredPacketLossER(); packetLossEnc = selectedMethod->RequiredPacketLossER();
// Get the bit cost of protection method // NACK is on for NACK and NackFec protection method: off for FEC method
_lossProtOverhead = static_cast<WebRtc_UWord32>(_lossProtLogic->HighestOverhead() + 0.5f); bool nackStatus = (selectedMethod->Type() == kNackFec ||
selectedMethod->Type() == kNACK);
//NACK is on for NACK and NackFec protection method: off for FEC method
bool nackStatus = true;
if (selectedMethod->Type() == kFEC)
{
nackStatus = false;
}
if(_videoProtectionCallback) if(_videoProtectionCallback)
{ {
_videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP ,codeRateKeyRTP, nackStatus); _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP,
codeRateKeyRTP,
nackStatus);
} }
} }
// Get the bit cost of protection method
_lossProtOverhead = static_cast<WebRtc_UWord32>(_lossProtLogic->HighestOverhead() + 0.5f);
// Update effective packet loss for encoder: note: fractionLost was passed as reference // Update effective packet loss for encoder: note: fractionLost was passed as reference
fractionLost = packetLossEnc; fractionLost = packetLossEnc;
WebRtc_UWord32 nackBitRate=0; WebRtc_UWord32 nackBitRate=0;
if(selectedMethod && _lossProtLogic->FindMethod(kNACK) != NULL) if(selectedMethod && _lossProtLogic->FindMethod(kNACK) != NULL)
{ {
// TODO(mikhal): update frame dropper with bit rate including both nack and fec
// Make sure we don't over-use the channel momentarily. This is // Make sure we don't over-use the channel momentarily. This is
// necessary for NACK since it can be very bursty. // necessary for NACK since it can be very bursty.
nackBitRate = (_lastBitRate * fractionLost) / 255; nackBitRate = (_lastBitRate * fractionLost) / 255;
@ -178,7 +183,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
_frameDropper->SetRates(static_cast<float>(bitRate - _lossProtOverhead), 0); _frameDropper->SetRates(static_cast<float>(bitRate - _lossProtOverhead), 0);
} }
//This may be used for UpdateEncoderBitRate: lastBitRate is total rate, before compensation // This may be used for UpdateEncoderBitRate: lastBitRate is total rate,
// before compensation
_lastBitRate = _targetBitRate; _lastBitRate = _targetBitRate;
//Source coding rate: total rate - protection overhead //Source coding rate: total rate - protection overhead
@ -187,7 +193,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
if (_enableQm) if (_enableQm)
{ {
//Update QM with rates //Update QM with rates
_qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,_incomingFrameRate); _qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps, _incomingFrameRate);
//Check for QM selection //Check for QM selection
bool selectQM = checkStatusForQMchange(); bool selectQM = checkStatusForQMchange();
if (selectQM) if (selectQM)
@ -203,7 +209,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
bool bool
VCMMediaOptimization::DropFrame() VCMMediaOptimization::DropFrame()
{ {
_frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f)); // leak appropriate number of bytes // leak appropriate number of bytes
_frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f));
return _frameDropper->DropFrame(); return _frameDropper->DropFrame();
} }
@ -285,7 +292,7 @@ VCMMediaOptimization::EnableNack(bool enable)
bool bool
VCMMediaOptimization::IsNackEnabled() VCMMediaOptimization::IsNackEnabled()
{ {
return (_lossProtLogic->FindMethod(kFEC) != NULL); return (_lossProtLogic->FindMethod(kNACK) != NULL);
} }
void void
@ -490,7 +497,7 @@ WebRtc_Word32
VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback *videoQMSettings) VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback *videoQMSettings)
{ {
_videoQMSettingsCallback = videoQMSettings; _videoQMSettingsCallback = videoQMSettings;
//Callback setting controls QM // Callback setting controls QM
if (_videoQMSettingsCallback != NULL) if (_videoQMSettingsCallback != NULL)
{ {
_enableQm = true; _enableQm = true;
@ -535,7 +542,7 @@ VCMMediaOptimization::SelectQuality()
// Check for updates to spatial/temporal modes // Check for updates to spatial/temporal modes
QMUpdate(qm); QMUpdate(qm);
//Reset all the rate and related frame counters quantities // Reset all the rate and related frame counters quantities
_qms->ResetRates(); _qms->ResetRates();
// Reset counters // Reset counters
@ -558,8 +565,10 @@ VCMMediaOptimization::checkStatusForQMchange()
bool status = true; bool status = true;
// Check that we do not call QMSelect too often, and that we waited some time (to sample the metrics) from the event lastChangeTime // Check that we do not call QMSelect too often, and that we waited some time
// lastChangeTime is the time where user changed the size/rate/frame rate (via SetEncodingData) // (to sample the metrics) from the event lastChangeTime
// lastChangeTime is the time where user changed the size/rate/frame rate
// (via SetEncodingData)
WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp();
if ((now - _lastQMUpdateTime) < kQmMinIntervalMs || if ((now - _lastQMUpdateTime) < kQmMinIntervalMs ||
(now - _lastChangeTime) < kQmMinIntervalMs) (now - _lastChangeTime) < kQmMinIntervalMs)
@ -574,7 +583,7 @@ VCMMediaOptimization::checkStatusForQMchange()
bool bool
VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) VCMMediaOptimization::QMUpdate(VCMQualityMode* qm)
{ {
//Check for no change // Check for no change
if (qm->spatialHeightFact == 1 && if (qm->spatialHeightFact == 1 &&
qm->spatialWidthFact == 1 && qm->spatialWidthFact == 1 &&
qm->temporalFact == 1) qm->temporalFact == 1)
@ -582,26 +591,26 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm)
return false; return false;
} }
//Content metrics hold native values // Content metrics hold native values
VideoContentMetrics* cm = _content->Data(); VideoContentMetrics* cm = _content->Data();
//Temporal // Temporal
WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>(_incomingFrameRate + 0.5f); WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>(_incomingFrameRate + 0.5f);
//Check if go back up in temporal resolution // Check if go back up in temporal resolution
if (qm->temporalFact == 0) if (qm->temporalFact == 0)
{ {
frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate;
} }
//go down in temporal resolution // go down in temporal resolution
else else
{ {
frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1);
} }
//Spatial // Spatial
WebRtc_UWord32 height = _codecHeight; WebRtc_UWord32 height = _codecHeight;
WebRtc_UWord32 width = _codecWidth; WebRtc_UWord32 width = _codecWidth;
//Check if go back up in spatial resolution // Check if go back up in spatial resolution
if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0)
{ {
height = cm->nativeHeight; height = cm->nativeHeight;
@ -617,7 +626,7 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm)
"Quality Mode Update: W = %d, H = %d, FR = %f", "Quality Mode Update: W = %d, H = %d, FR = %f",
width, height, frameRate); width, height, frameRate);
//Update VPM with new target frame rate and size // Update VPM with new target frame rate and size
_videoQMSettingsCallback->SetVideoQMSettings(frameRate, width, height); _videoQMSettingsCallback->SetVideoQMSettings(frameRate, width, height);
return true; return true;

32
modules/video_coding/main/source/receiver.cc
View File

@ -31,7 +31,6 @@ _master(master),
_jitterBuffer(vcmId, receiverId, master), _jitterBuffer(vcmId, receiverId, master),
_timing(timing), _timing(timing),
_renderWaitEvent(*new VCMEvent()), _renderWaitEvent(*new VCMEvent()),
_nackMode(kNoNack),
_state(kPassive) _state(kPassive)
{ {
} }
@ -164,6 +163,7 @@ VCMReceiver::InsertPacket(const VCMPacket& packet,
} }
// Insert packet into jitter buffer // Insert packet into jitter buffer
// both data and empty packets
const VCMFrameBufferEnum ret = _jitterBuffer.InsertPacket(buffer, packet); const VCMFrameBufferEnum ret = _jitterBuffer.InsertPacket(buffer, packet);
if (ret < 0) if (ret < 0)
@ -178,7 +178,8 @@ VCMReceiver::InsertPacket(const VCMPacket& packet,
} }
VCMEncodedFrame* VCMEncodedFrame*
VCMReceiver::FrameForDecoding(WebRtc_UWord16 maxWaitTimeMs, WebRtc_Word64& nextRenderTimeMs, bool renderTiming, VCMReceiver* dualReceiver) VCMReceiver::FrameForDecoding(WebRtc_UWord16 maxWaitTimeMs, WebRtc_Word64& nextRenderTimeMs,
bool renderTiming, VCMReceiver* dualReceiver)
{ {
// No need to enter the critical section here since the jitter buffer // No need to enter the critical section here since the jitter buffer
// is thread-safe. // is thread-safe.
@ -348,20 +349,7 @@ void
VCMReceiver::SetNackMode(VCMNackMode nackMode) VCMReceiver::SetNackMode(VCMNackMode nackMode)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_nackMode = nackMode; _jitterBuffer.SetNackMode(nackMode);
switch (_nackMode)
{
case kNackInfinite:
{
_jitterBuffer.SetNackStatus(true);
break;
}
case kNoNack:
{
_jitterBuffer.SetNackStatus(false);
break;
}
}
if (!_master) if (!_master)
{ {
_state = kPassive; // The dual decoder defaults to passive _state = kPassive; // The dual decoder defaults to passive
@ -372,7 +360,7 @@ VCMNackMode
VCMReceiver::NackMode() const VCMReceiver::NackMode() const
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return _nackMode; return _jitterBuffer.GetNackMode();
} }
VCMNackStatus VCMNackStatus
@ -418,14 +406,6 @@ void
VCMReceiver::CopyJitterBufferStateFromReceiver(const VCMReceiver& receiver) VCMReceiver::CopyJitterBufferStateFromReceiver(const VCMReceiver& receiver)
{ {
_jitterBuffer = receiver._jitterBuffer; _jitterBuffer = receiver._jitterBuffer;
{
CriticalSectionScoped cs(_critSect);
if (_nackMode != kNoNack)
{
_jitterBuffer.SetNackStatus(true);
}
}
} }
VCMReceiverState VCMReceiverState
@ -447,7 +427,7 @@ VCMReceiver::UpdateState(VCMReceiverState newState)
void void
VCMReceiver::UpdateState(VCMEncodedFrame& frame) VCMReceiver::UpdateState(VCMEncodedFrame& frame)
{ {
if (_nackMode == kNoNack) if (_jitterBuffer.GetNackMode() == kNoNack)
{ {
// Dual decoder mode has not been enabled. // Dual decoder mode has not been enabled.
return; return;

6
modules/video_coding/main/source/receiver.h
View File

@ -28,11 +28,6 @@ enum VCMNackStatus
kNackKeyFrameRequest kNackKeyFrameRequest
}; };
enum VCMNackMode
{
kNackInfinite,
kNoNack
};
enum VCMReceiverState enum VCMReceiverState
{ {
@ -91,7 +86,6 @@ private:
VCMJitterBuffer _jitterBuffer; VCMJitterBuffer _jitterBuffer;
VCMTiming& _timing; VCMTiming& _timing;
VCMEvent& _renderWaitEvent; VCMEvent& _renderWaitEvent;
VCMNackMode _nackMode;
VCMReceiverState _state; VCMReceiverState _state;
static WebRtc_Word32 _receiverIdCounter; static WebRtc_Word32 _receiverIdCounter;

309
modules/video_coding/main/source/session_info.cc
View File

@ -25,7 +25,9 @@ VCMSessionInfo::VCMSessionInfo():
_previousFrameLoss(false), _previousFrameLoss(false),
_lowSeqNum(-1), _lowSeqNum(-1),
_highSeqNum(-1), _highSeqNum(-1),
_highestPacketIndex(0) _highestPacketIndex(0),
_emptySeqNumLow(-1),
_emptySeqNumHigh(-1)
{ {
memset(_packetSizeBytes, 0, sizeof(_packetSizeBytes)); memset(_packetSizeBytes, 0, sizeof(_packetSizeBytes));
memset(_naluCompleteness, kNaluUnset, sizeof(_naluCompleteness)); memset(_naluCompleteness, kNaluUnset, sizeof(_naluCompleteness));
@ -50,6 +52,8 @@ void VCMSessionInfo::Reset()
{ {
_lowSeqNum = -1; _lowSeqNum = -1;
_highSeqNum = -1; _highSeqNum = -1;
_emptySeqNumLow = -1;
_emptySeqNumHigh = -1;
_markerBit = false; _markerBit = false;
_haveFirstPacket = false; _haveFirstPacket = false;
_completeSession = false; _completeSession = false;
@ -65,7 +69,7 @@ void VCMSessionInfo::Reset()
WebRtc_UWord32 VCMSessionInfo::GetSessionLength() WebRtc_UWord32 VCMSessionInfo::GetSessionLength()
{ {
WebRtc_UWord32 length = 0; WebRtc_UWord32 length = 0;
for (WebRtc_Word32 i=0; i<=_highestPacketIndex; ++i) for (WebRtc_Word32 i = 0; i <= _highestPacketIndex; ++i)
{ {
length += _packetSizeBytes[i]; length += _packetSizeBytes[i];
} }
@ -89,7 +93,9 @@ VCMSessionInfo::HaveStartSeqNumber()
return true; return true;
} }
WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebRtc_Word32 packetIndex, const VCMPacket& packet) WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer,
WebRtc_Word32 packetIndex,
const VCMPacket& packet)
{ {
WebRtc_UWord32 moveLength = 0; WebRtc_UWord32 moveLength = 0;
WebRtc_UWord32 returnLength = 0; WebRtc_UWord32 returnLength = 0;
@ -99,30 +105,33 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR
WebRtc_UWord32 offset = 0; WebRtc_UWord32 offset = 0;
WebRtc_UWord32 packetSize = 0; WebRtc_UWord32 packetSize = 0;
// Store this packet length. Add length since we could have data present already (e.g. multicall case). // Store this packet length. Add length since we could have data present
// already (e.g. multicall case).
if (packet.bits) if (packet.bits)
{ {
packetSize = packet.sizeBytes; packetSize = packet.sizeBytes;
} }
else else
{ {
packetSize = packet.sizeBytes + (packet.insertStartCode?kH264StartCodeLengthBytes:0); packetSize = packet.sizeBytes +
(packet.insertStartCode?kH264StartCodeLengthBytes:0);
} }
_packetSizeBytes[packetIndex] += packetSize; _packetSizeBytes[packetIndex] += packetSize;
// count only the one in our layer // count only the one in our layer
for (i=0; i<packetIndex; ++i) for (i = 0; i < packetIndex; ++i)
{ {
offset += _packetSizeBytes[i]; offset += _packetSizeBytes[i];
} }
for (i=packetIndex+1; i<=_highestPacketIndex; ++i) for (i = packetIndex + 1; i <= _highestPacketIndex; ++i)
{ {
moveLength += _packetSizeBytes[i]; moveLength += _packetSizeBytes[i];
} }
if (moveLength > 0) if (moveLength > 0)
{ {
memmove((void*)(ptrStartOfLayer + offset + packetSize), ptrStartOfLayer + offset, moveLength); memmove((void*)(ptrStartOfLayer + offset + packetSize),
ptrStartOfLayer + offset, moveLength);
} }
if (packet.bits) if (packet.bits)
@ -145,7 +154,8 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR
const unsigned char startCode[] = {0, 0, 0, 1}; const unsigned char startCode[] = {0, 0, 0, 1};
if(packet.insertStartCode) if(packet.insertStartCode)
{ {
memcpy((void*)(ptrStartOfLayer + offset), startCode, kH264StartCodeLengthBytes); memcpy((void*)(ptrStartOfLayer + offset), startCode,
kH264StartCodeLengthBytes);
} }
memcpy((void*)(ptrStartOfLayer + offset memcpy((void*)(ptrStartOfLayer + offset
+ (packet.insertStartCode?kH264StartCodeLengthBytes:0)), + (packet.insertStartCode?kH264StartCodeLengthBytes:0)),
@ -158,6 +168,9 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR
if (packet.isFirstPacket) if (packet.isFirstPacket)
{ {
_haveFirstPacket = true; _haveFirstPacket = true;
//initializing FEC sequence numbers
_emptySeqNumHigh = -1;
_emptySeqNumLow = -1;
} }
if (packet.markerBit) if (packet.markerBit)
{ {
@ -177,7 +190,7 @@ void VCMSessionInfo::UpdateCompleteSession()
{ {
// do we have all packets in this session? // do we have all packets in this session?
bool completeSession = true; bool completeSession = true;
for (int i=0; i<= _highestPacketIndex; ++i) for (int i = 0; i<= _highestPacketIndex; ++i)
{ {
if (_naluCompleteness[i] == kNaluUnset) if (_naluCompleteness[i] == kNaluUnset)
{ {
@ -194,36 +207,41 @@ bool VCMSessionInfo::IsSessionComplete()
return _completeSession; return _completeSession;
} }
// Find the start and end index of packetIndex packet. // Find the start and end index of packetIndex packet.
// startIndex -1 if start not found endIndex=-1 if end index not found // startIndex -1 if start not found endIndex=-1 if end index not found
void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& startIndex, WebRtc_Word32& endIndex) void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,
WebRtc_Word32& startIndex,
WebRtc_Word32& endIndex)
{ {
if(_naluCompleteness[packetIndex]==kNaluStart || if(_naluCompleteness[packetIndex]==kNaluStart ||
_naluCompleteness[packetIndex]==kNaluComplete) _naluCompleteness[packetIndex]==kNaluComplete)
{ {
startIndex=packetIndex; startIndex = packetIndex;
} }
else // Need to find the start else // Need to find the start
{ {
for(startIndex=packetIndex-1;startIndex>=0;--startIndex) for(startIndex = packetIndex - 1; startIndex >= 0; --startIndex)
{ {
if( (_naluCompleteness[startIndex]==kNaluComplete && _packetSizeBytes[startIndex]>0) ||(_naluCompleteness[startIndex]==kNaluEnd && startIndex>0)) // Found previous NALU. if( (_naluCompleteness[startIndex] == kNaluComplete &&
_packetSizeBytes[startIndex] > 0) ||
// Found previous NALU.
(_naluCompleteness[startIndex] == kNaluEnd && startIndex>0))
{ {
startIndex++; startIndex++;
break; break;
} }
if( _naluCompleteness[startIndex]==kNaluStart) // This is where the NALU start. // This is where the NALU start.
if( _naluCompleteness[startIndex] == kNaluStart)
{ {
break; break;
} }
} }
} }
if(_naluCompleteness[packetIndex]==kNaluEnd || if(_naluCompleteness[packetIndex] == kNaluEnd ||
_naluCompleteness[packetIndex]==kNaluComplete) _naluCompleteness[packetIndex] == kNaluComplete)
{ {
endIndex=packetIndex; endIndex=packetIndex;
} }
@ -232,7 +250,9 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& sta
// Find the next NALU // Find the next NALU
for(endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex) for(endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex)
{ {
if((_naluCompleteness[endIndex]==kNaluComplete && _packetSizeBytes[endIndex]>0) || _naluCompleteness[endIndex]==kNaluStart) // Found next NALU. if((_naluCompleteness[endIndex]==kNaluComplete &&
_packetSizeBytes[endIndex]>0) ||
_naluCompleteness[endIndex]==kNaluStart) // Found next NALU.
{ {
endIndex--; endIndex--;
break; break;
@ -242,40 +262,43 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& sta
break; break;
} }
} }
if(endIndex>_highestPacketIndex) if(endIndex > _highestPacketIndex)
endIndex=-1; endIndex = -1;
} }
} }
// Deletes all packets between startIndex and endIndex // Deletes all packets between startIndex and endIndex
WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer,WebRtc_Word32 startIndex,WebRtc_Word32 endIndex) WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer,
WebRtc_Word32 startIndex,
WebRtc_Word32 endIndex)
{ {
//Get the number of bytes to delete. //Get the number of bytes to delete.
//Clear the size of these packets. //Clear the size of these packets.
WebRtc_UWord32 bytesToDelete=0; /// The number of bytes to delete. WebRtc_UWord32 bytesToDelete = 0; /// The number of bytes to delete.
for(int j=startIndex;j<=endIndex;++j) for(int j = startIndex;j <= endIndex; ++j)
{ {
bytesToDelete+=_packetSizeBytes[j]; bytesToDelete += _packetSizeBytes[j];
_packetSizeBytes[j]=0; _packetSizeBytes[j]=0;
} }
if (bytesToDelete > 0) if (bytesToDelete > 0)
{ {
// Get the offset we want to move to. // Get the offset we want to move to.
int destOffset=0; int destOffset = 0;
for(int j=0;j<startIndex;j++) for(int j = 0;j < startIndex;j++)
{ {
destOffset+=_packetSizeBytes[j]; destOffset += _packetSizeBytes[j];
} }
//Get the number of bytes to move //Get the number of bytes to move
WebRtc_UWord32 numberOfBytesToMove=0; WebRtc_UWord32 numberOfBytesToMove=0;
for (int j=endIndex+1; j<=_highestPacketIndex; ++j) for (int j = endIndex + 1; j <= _highestPacketIndex; ++j)
{ {
numberOfBytesToMove += _packetSizeBytes[j]; numberOfBytesToMove += _packetSizeBytes[j];
} }
memmove((void*)(ptrStartOfLayer + destOffset),(void*)(ptrStartOfLayer + destOffset+bytesToDelete), numberOfBytesToMove); memmove((void*)(ptrStartOfLayer + destOffset),(void*)(ptrStartOfLayer +
destOffset+bytesToDelete), numberOfBytesToMove);
} }
@ -286,30 +309,30 @@ WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer,WebR
// return the number of bytes deleted from the session. -1 if an error occurs // return the number of bytes deleted from the session. -1 if an error occurs
WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer) WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer)
{ {
if(_lowSeqNum<0) // No packets in this session if(_lowSeqNum < 0) // No packets in this session
return 0; return 0;
WebRtc_Word32 startIndex=0; WebRtc_Word32 startIndex = 0;
WebRtc_Word32 endIndex=0; WebRtc_Word32 endIndex = 0;
int packetIndex=0; int packetIndex = 0;
WebRtc_UWord32 returnLength=0; WebRtc_UWord32 returnLength = 0;
for (packetIndex=0; packetIndex<= _highestPacketIndex; ++packetIndex) for (packetIndex = 0; packetIndex <= _highestPacketIndex; ++packetIndex)
{ {
if (_naluCompleteness[packetIndex] == kNaluUnset) // Found a lost packet if (_naluCompleteness[packetIndex] == kNaluUnset) // Found a lost packet
{ {
FindNaluBorder(packetIndex,startIndex,endIndex); FindNaluBorder(packetIndex,startIndex,endIndex);
if(startIndex==-1) if(startIndex == -1)
startIndex=0; startIndex = 0;
if(endIndex==-1) if(endIndex == -1)
endIndex=_highestPacketIndex; endIndex = _highestPacketIndex;
returnLength+=DeletePackets(ptrStartOfLayer,packetIndex,endIndex); returnLength += DeletePackets(ptrStartOfLayer,packetIndex,endIndex);
packetIndex=endIndex; packetIndex = endIndex;
}// end lost packet }// end lost packet
} }
//Make sure the first packet is decodable (Either complete nalu or start of NALU) //Make sure the first packet is decodable (Either complete nalu or start of NALU)
if(_packetSizeBytes[0]>0) if(_packetSizeBytes[0] > 0)
{ {
switch(_naluCompleteness[0]) switch(_naluCompleteness[0])
{ {
@ -321,15 +344,16 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay
case kNaluIncomplete: //Packet is not beginning or end of NALU case kNaluIncomplete: //Packet is not beginning or end of NALU
//Need to find the end of this fua NALU and delete all packets. //Need to find the end of this fua NALU and delete all packets.
FindNaluBorder(0,startIndex,endIndex); FindNaluBorder(0,startIndex,endIndex);
if(endIndex==-1) // No end found. Delete if(endIndex == -1) // No end found. Delete
{ {
endIndex=_highestPacketIndex; endIndex = _highestPacketIndex;
} }
returnLength+=DeletePackets(ptrStartOfLayer,0,endIndex);//Delete this NALU. //Delete this NALU.
returnLength += DeletePackets(ptrStartOfLayer,0,endIndex);
break; break;
case kNaluEnd: // Packet is the end of a NALU case kNaluEnd: // Packet is the end of a NALU
//Need to delete this packet //Delete this NALU
returnLength+=DeletePackets(ptrStartOfLayer,0,0);//Delete this NALU. returnLength += DeletePackets(ptrStartOfLayer,0,0);
break; break;
default: default:
assert(false); assert(false);
@ -339,9 +363,10 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay
return returnLength; return returnLength;
} }
WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num) WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum)
{ {
if ((NULL == list) || (num < 1)) if ((NULL == list) || (numberOfSeqNum < 1))
{ {
return -1; return -1;
} }
@ -353,7 +378,7 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
// Find end point (index of entry that equals _lowSeqNum) // Find end point (index of entry that equals _lowSeqNum)
int index = 0; int index = 0;
for (; index <num; index++) for (; index < numberOfSeqNum; index++)
{ {
if (list[index] == _lowSeqNum) if (list[index] == _lowSeqNum)
{ {
@ -364,7 +389,7 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
// Zero out between first entry and end point // Zero out between first entry and end point
int i = 0; int i = 0;
while ( i <= _highestPacketIndex && index < num) while ( i <= _highestPacketIndex && index < numberOfSeqNum)
{ {
if (_naluCompleteness[i] != kNaluUnset) if (_naluCompleteness[i] != kNaluUnset)
{ {
@ -384,6 +409,105 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
return 0; return 0;
} }
WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum,
float rttScore)
{
if ((NULL == list) || (numberOfSeqNum < 1))
{
return -1;
}
if (_lowSeqNum == -1)
{
// no packets in this frame
return 0;
}
WebRtc_Word32 index = 0;
// Find end point (index of entry that equals _lowSeqNum)
for (; index < numberOfSeqNum; index++)
{
if (list[index] == _lowSeqNum)
{
list[index] = -1;
break;
}
}
// TODO(mikhal): 1. update score based on RTT value 2. add partition data
// use the previous available
bool isBaseAvailable = false;
if ((index > 0) && (list[index] == -1))
{
// found first packet, for now let's go only one back
if ((list[index - 1] == -1) || (list[index - 1] == -2))
{
// this is indeed the first packet, as previous packet was populated
isBaseAvailable = true;
}
}
bool allowNack = false;
if (!_haveFirstPacket || !isBaseAvailable)
{
allowNack = true;
}
// Zero out between first entry and end point
int i = 0;
// Score place holder - based on RTT and partition (when available).
const float nackScoreTh = 0.25f;
WebRtc_Word32 highMediaPacket = _emptySeqNumLow > _lowSeqNum ?
_emptySeqNumLow - 1: _highSeqNum;
while (list[index] <= highMediaPacket && index < numberOfSeqNum)
{
if (_naluCompleteness[i] != kNaluUnset)
{
list[index] = -1;
}
else
{
// compute score of the packet
float score = 1.0f;
// multiply internal score (importance) by external score (RTT)
score *= rttScore;
if (score > nackScoreTh)
{
allowNack = true;
}
else
{
list[index] = -1;
}
}
i++;
index++;
}
// Empty packets follow the data packets, and therefore have a higher
// sequence number. We do not want to NACK empty packets.
if ((_emptySeqNumLow != -1) && (_emptySeqNumHigh != -1) &&
(index < numberOfSeqNum))
{
// first make sure that we are at least at the minimum value
// (if not we are missing last packet(s))
while (list[index] < _emptySeqNumLow && index < numberOfSeqNum)
{
index++;
}
// mark empty packets
while (list[index] <= _emptySeqNumHigh && index < numberOfSeqNum)
{
list[index] = -2;
index++;
}
}
_sessionNACK = allowNack;
return 0;
}
WebRtc_Word32 VCMSessionInfo::GetHighestPacketIndex() WebRtc_Word32 VCMSessionInfo::GetHighestPacketIndex()
{ {
return _highestPacketIndex; return _highestPacketIndex;
@ -410,7 +534,7 @@ void VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32
// sanity // sanity
if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0)
{ {
//not allowed // not allowed
assert(!"SessionInfo::UpdatePacketSize Error: invalid packetIndex"); assert(!"SessionInfo::UpdatePacketSize Error: invalid packetIndex");
return; return;
} }
@ -420,15 +544,17 @@ void VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32
void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices) void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices)
{ {
// sanity // sanity
if((numberOfPacketIndices + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer) || numberOfPacketIndices < 0) if((numberOfPacketIndices + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer)
|| numberOfPacketIndices < 0)
{ {
//not allowed // not allowed
assert(!"SessionInfo::PrependPacketIndexes Error: invalid packetIndex"); assert(!"SessionInfo::PrependPacketIndexes Error: invalid packetIndex");
return; return;
} }
// Works if we have new packets before packetIndex = 0 // Works if we have new packets before packetIndex = 0
int numOfPacketsToMove = GetHighestPacketIndex()+1; int numOfPacketsToMove = GetHighestPacketIndex()+1;
memmove(&_packetSizeBytes[numberOfPacketIndices], &_packetSizeBytes[0], (numOfPacketsToMove)*sizeof(WebRtc_UWord16)); memmove(&_packetSizeBytes[numberOfPacketIndices], &_packetSizeBytes[0],
(numOfPacketsToMove)*sizeof(WebRtc_UWord16));
memset(&_packetSizeBytes[0], 0, numberOfPacketIndices*sizeof(WebRtc_UWord16)); memset(&_packetSizeBytes[0], 0, numberOfPacketIndices*sizeof(WebRtc_UWord16));
_highestPacketIndex += (WebRtc_UWord16)numberOfPacketIndices; _highestPacketIndex += (WebRtc_UWord16)numberOfPacketIndices;
@ -439,11 +565,11 @@ void VCMSessionInfo::ClearPacketSize(WebRtc_Word32 packetIndex)
// sanity // sanity
if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0)
{ {
//not allowed // not allowed
assert(!"SessionInfo::ClearPacketSize Error: invalid packetIndex"); assert(!"SessionInfo::ClearPacketSize Error: invalid packetIndex");
return; return;
} }
_packetSizeBytes[packetIndex] =0; _packetSizeBytes[packetIndex] = 0;
} }
WebRtc_UWord32 VCMSessionInfo::GetPacketSize(WebRtc_Word32 packetIndex) WebRtc_UWord32 VCMSessionInfo::GetPacketSize(WebRtc_Word32 packetIndex)
@ -464,20 +590,31 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL
//not allowed //not allowed
assert(!packet.insertStartCode || !packet.bits); assert(!packet.insertStartCode || !packet.bits);
if (packet.frameType == kFrameEmpty)
{
// update seq number as an empty packet
// empty packets will be counted twice: both empty and standard packets.
InformOfEmptyPacket(packet.seqNum);
}
// Check if this is first packet (only valid for some codecs) // Check if this is first packet (only valid for some codecs)
if (packet.isFirstPacket) if (packet.isFirstPacket)
{ {
// the first packet in the frame always signals the frametype // the first packet in the frame always signals the frametype
_frameType = packet.frameType; _frameType = packet.frameType;
} }
else if (_frameType == kFrameEmpty && packet.frameType != kFrameEmpty)
{
// in case an empty packet came in first, update the frame type
_frameType = packet.frameType;
}
// Check sequence number and update highest and lowest sequence numbers received. // Check sequence number and update highest and lowest sequence numbers received.
// Move data if this seq num is lower than previously lowest. // Move data if this seq num is lower than previously lowest.
if (packet.seqNum > _highSeqNum) if (packet.seqNum > _highSeqNum)
{ {
// This packet's seq num is higher than previously highest seq num; normal case // This packet's seq num is higher than previously highest seq num;
// if we have a wrap, only update with wrapped values // normal case if we have a wrap, only update with wrapped values
if (!(_highSeqNum < 0x00ff && packet.seqNum > 0xff00)) if (!(_highSeqNum < 0x00ff && packet.seqNum > 0xff00))
{ {
_highSeqNum = packet.seqNum; _highSeqNum = packet.seqNum;
@ -488,7 +625,7 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL
_highSeqNum = packet.seqNum; _highSeqNum = packet.seqNum;
} }
int packetIndex = packet.seqNum - (WebRtc_UWord16)_lowSeqNum; int packetIndex = packet.seqNum - (WebRtc_UWord16)_lowSeqNum;
if(_lowSeqNum < 0x00ff && packet.seqNum > 0xff00) if (_lowSeqNum < 0x00ff && packet.seqNum > 0xff00)
{ {
// negative wrap // negative wrap
packetIndex = packet.seqNum - 0x10000 - _lowSeqNum; packetIndex = packet.seqNum - 0x10000 - _lowSeqNum;
@ -498,7 +635,8 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL
if (_lowSeqNum > 0xff00 && packet.seqNum < 0x00ff) if (_lowSeqNum > 0xff00 && packet.seqNum < 0x00ff)
{ {
// we have a false detect due to the wrap // we have a false detect due to the wrap
packetIndex = (0xffff - (WebRtc_UWord16)_lowSeqNum) + packet.seqNum + (WebRtc_UWord16)1; packetIndex = (0xffff - (WebRtc_UWord16)_lowSeqNum) + packet.seqNum
+ (WebRtc_UWord16)1;
} else } else
{ {
// This packet's seq num is lower than previously lowest seq num, but no wrap // This packet's seq num is lower than previously lowest seq num, but no wrap
@ -562,7 +700,50 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL
return InsertBuffer(ptrStartOfLayer, packetIndex, packet); return InsertBuffer(ptrStartOfLayer, packetIndex, packet);
} }
WebRtc_UWord32 VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec)
WebRtc_Word32
VCMSessionInfo::InformOfEmptyPacket(const WebRtc_UWord16 seqNum)
{
// Empty packets may be FEC or filler packets. They are sequential and
// follow the data packets, therefore, we should only keep track of the high
// and low sequence numbers and may assume that the packets in between are
// empty packets belonging to the same frame (timestamp).
if (_emptySeqNumLow == -1 && _emptySeqNumHigh == -1)
{
_emptySeqNumLow = seqNum;
_emptySeqNumHigh = seqNum;
}
else
{
if (seqNum > _emptySeqNumHigh)
{
// This packet's seq num is higher than previously highest seq num;
// normal case if we have a wrap, only update with wrapped values
if (!(_emptySeqNumHigh < 0x00ff && seqNum > 0xff00))
{
_emptySeqNumHigh = seqNum;
}
}
else if (_emptySeqNumHigh > 0xff00 && seqNum < 0x00ff)
{
// wrap
_emptySeqNumHigh = seqNum;
}
if (_emptySeqNumLow < 0x00ff && seqNum > 0xff00)
{
// negative wrap
if (seqNum - 0x10000 - _emptySeqNumLow < 0)
{
_emptySeqNumLow = seqNum;
}
}
}
return 0;
}
WebRtc_UWord32
VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec)
{ {
WebRtc_UWord32 currentPacketOffset = 0; WebRtc_UWord32 currentPacketOffset = 0;
WebRtc_UWord32 length = GetSessionLength(); WebRtc_UWord32 length = GetSessionLength();
@ -573,7 +754,7 @@ WebRtc_UWord32 VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer,
return length; return length;
} }
bool previousLost = false; bool previousLost = false;
for (int i=0; i <= _highestPacketIndex; i++) for (int i = 0; i <= _highestPacketIndex; i++)
{ {
if (_ORwithPrevByte[i]) if (_ORwithPrevByte[i])
{ {

26
modules/video_coding/main/source/session_info.h
View File

@ -26,10 +26,16 @@ public:
VCMSessionInfo(const VCMSessionInfo& rhs); VCMSessionInfo(const VCMSessionInfo& rhs);
WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num); WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 numberOfSeqNum);
// Hybrid version: Zero out seq num for NACK list
// apply a score based on the packet location and the external rttScore
WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum,
float rttScore);
virtual void Reset(); virtual void Reset();
WebRtc_Word64 InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer); WebRtc_Word64 InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer);
WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum);
virtual bool IsSessionComplete(); virtual bool IsSessionComplete();
WebRtc_UWord32 MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer); WebRtc_UWord32 MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer);
@ -75,18 +81,20 @@ protected:
bool _sessionNACK; // If this session has been NACKed by JB bool _sessionNACK; // If this session has been NACKed by JB
bool _completeSession; bool _completeSession;
webrtc::FrameType _frameType; webrtc::FrameType _frameType;
bool _previousFrameLoss; bool _previousFrameLoss;
WebRtc_Word32 _lowSeqNum; // Lowest packet sequence number in a session WebRtc_Word32 _lowSeqNum; // Lowest packet sequence number in a session
WebRtc_Word32 _highSeqNum; // Highest packet sequence number in a session WebRtc_Word32 _highSeqNum; // Highest packet sequence number in a session
// Highest packet index in this frame // Highest packet index in this frame
WebRtc_UWord16 _highestPacketIndex; WebRtc_UWord16 _highestPacketIndex;
// Length of packet (used for reordering) // Length of packet (used for reordering)
WebRtc_UWord32 _packetSizeBytes[kMaxPacketsInJitterBuffer]; WebRtc_UWord32 _packetSizeBytes[kMaxPacketsInJitterBuffer];
// Completness of packets. Used for deciding if the frame is decodable. // Completeness of packets. Used for deciding if the frame is decodable.
WebRtc_UWord8 _naluCompleteness[kMaxPacketsInJitterBuffer]; WebRtc_UWord8 _naluCompleteness[kMaxPacketsInJitterBuffer];
bool _ORwithPrevByte[kMaxPacketsInJitterBuffer]; WebRtc_Word32 _emptySeqNumLow;
WebRtc_Word32 _emptySeqNumHigh;
bool _ORwithPrevByte[kMaxPacketsInJitterBuffer];
}; };
} // namespace webrtc } // namespace webrtc

9
modules/video_coding/main/source/video_coding_impl.cc
View File

@ -269,7 +269,7 @@ VideoCodingModuleImpl::TimeUntilNextProcess()
{ {
WebRtc_UWord32 timeUntilNextProcess = VCM_MIN(_receiveStatsTimer.TimeUntilProcess(), WebRtc_UWord32 timeUntilNextProcess = VCM_MIN(_receiveStatsTimer.TimeUntilProcess(),
_sendStatsTimer.TimeUntilProcess()); _sendStatsTimer.TimeUntilProcess());
if ((_receiver.NackMode() == kNackInfinite) || (_dualReceiver.State() != kPassive)) if ((_receiver.NackMode() != kNoNack) || (_dualReceiver.State() != kPassive))
{ {
// We need a Process call more often if we are relying on retransmissions // We need a Process call more often if we are relying on retransmissions
timeUntilNextProcess = VCM_MIN(timeUntilNextProcess, timeUntilNextProcess = VCM_MIN(timeUntilNextProcess,
@ -576,6 +576,7 @@ VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, bo
{ {
WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id),
"SetVideoProtection()"); "SetVideoProtection()");
switch (videoProtection) switch (videoProtection)
{ {
@ -664,16 +665,18 @@ VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, bo
case kProtectionNackFEC: case kProtectionNackFEC:
{ {
{ {
// Receive side
CriticalSectionScoped cs(_receiveCritSect); CriticalSectionScoped cs(_receiveCritSect);
if (enable) if (enable)
{ {
_receiver.SetNackMode(kNackInfinite); _receiver.SetNackMode(kNackHybrid);
} }
else else
{ {
_receiver.SetNackMode(kNoNack); _receiver.SetNackMode(kNoNack);
} }
} }
// Send Side
{ {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
_mediaOpt.EnableNackFEC(enable); _mediaOpt.EnableNackFEC(enable);
@ -1298,7 +1301,7 @@ VideoCodingModuleImpl::NackList(WebRtc_UWord16* nackList, WebRtc_UWord16& size)
// Collect sequence numbers from the default receiver // Collect sequence numbers from the default receiver
// if in normal nack mode. Otherwise collect them from // if in normal nack mode. Otherwise collect them from
// the dual receiver if the dual receiver is receiving. // the dual receiver if the dual receiver is receiving.
if (_receiver.NackMode() == kNackInfinite) if (_receiver.NackMode() != kNoNack)
{ {
nackStatus = _receiver.NackList(nackList, size); nackStatus = _receiver.NackList(nackList, size);
} }

6
modules/video_coding/main/test/jitter_buffer_test.cc
View File

@ -1461,9 +1461,9 @@ int JitterBufferTest(CmdArgs& args)
// --------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------
// | 3 | 4 | 5 | 6 | 7 | 9 | x | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | x | 21 |.....| 102 | // | 3 | 4 | 5 | 6 | 7 | 9 | x | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | x | 21 |.....| 102 |
// --------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------
jb.SetNackStatus(true); jb.SetNackMode(kNackInfinite);
TEST(jb.GetNackStatus()); TEST(jb.GetNackMode() == kNackInfinite);
// insert first packet // insert first packet
timeStamp += 33*90; timeStamp += 33*90;
@ -1880,7 +1880,7 @@ int JitterBufferTest(CmdArgs& args)
//Test incomplete NALU frames //Test incomplete NALU frames
jb.Flush(); jb.Flush();
jb.SetNackStatus(false); jb.SetNackMode(kNoNack);
seqNum ++; seqNum ++;
timeStamp += 33*90; timeStamp += 33*90;
int insertedLength=0; int insertedLength=0;