generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Review URL: http://webrtc-codereview.appspot.com/28004

Browse Source 
git-svn-id: http://webrtc.googlecode.com/svn/trunk@74 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
mikhal@google.com 2011-06-14 17:54:20 +00:00
parent 0c08ed1ef9
commit 17705a9c5a
16 changed files with 998 additions and 729 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
modules/video_coding/main/source/frame_buffer.cc
View File

@ -133,14 +133,19 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
if (kStateEmpty == _state) if (kStateEmpty == _state)
{ {
// This is the first packet inserted into this frame, // This is the first packet (empty and/or data) inserted into this frame.
// store some info and set some initial values. // store some info and set some initial values.
_timeStamp = packet.timestamp; _timeStamp = packet.timestamp;
_codec = packet.codec; _codec = packet.codec;
// for the first media packet
if (packet.frameType != kFrameEmpty)
{
SetState(kStateIncomplete); SetState(kStateIncomplete);
} }
}
WebRtc_UWord32 requiredSizeBytes = Length() + packet.sizeBytes + (packet.insertStartCode?kH264StartCodeLengthBytes:0); WebRtc_UWord32 requiredSizeBytes = Length() + packet.sizeBytes +
(packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
if (requiredSizeBytes >= _size) if (requiredSizeBytes >= _size)
{ {
const WebRtc_UWord32 increments = requiredSizeBytes / kBufferIncStepSizeBytes + const WebRtc_UWord32 increments = requiredSizeBytes / kBufferIncStepSizeBytes +
@ -201,6 +206,17 @@ WebRtc_Word32 VCMFrameBuffer::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
return 0; return 0;
} }
// Zero out all entries in list up to and including the (first) entry equal to
// _lowSeqNum. Hybrid mode: 1. Don't NACK FEC packets 2. Make a smart decision
// on whether to NACK or not
WebRtc_Word32 VCMFrameBuffer::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 num,
float rttScore)
{
return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttScore);
}
void VCMFrameBuffer::IncrementNackCount() void VCMFrameBuffer::IncrementNackCount()
{ {
_nackCount++; _nackCount++;
@ -227,7 +243,6 @@ void VCMFrameBuffer::Reset()
{ {
_length = 0; _length = 0;
_timeStamp = 0; _timeStamp = 0;
_sessionInfo.Reset(); _sessionInfo.Reset();
_frameCounted = false; _frameCounted = false;
_payloadType = 0; _payloadType = 0;
@ -237,7 +252,7 @@ void VCMFrameBuffer::Reset()
VCMEncodedFrame::Reset(); VCMEncodedFrame::Reset();
} }
// Makes sure the session contain a decodable stream. // Makes sure the session contains a decodable stream.
void void
VCMFrameBuffer::MakeSessionDecodable() VCMFrameBuffer::MakeSessionDecodable()
{ {
@ -275,7 +290,8 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
case kStateComplete: case kStateComplete:
assert(_state == kStateEmpty || assert(_state == kStateEmpty ||
_state == kStateIncomplete); _state == kStateIncomplete ||
_state == kStateDecodable);
break; break;
@ -286,11 +302,22 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
case kStateDecoding: case kStateDecoding:
// we can go to this state from state kStateComplete kStateIncomplete // we can go to this state from state kStateComplete kStateIncomplete
assert(_state == kStateComplete || _state == kStateIncomplete); assert(_state == kStateComplete || _state == kStateIncomplete ||
_state == kStateDecodable);
// Transfer frame information to EncodedFrame and create any codec specific information // Transfer frame information to EncodedFrame and create any codec specific information
RestructureFrameInformation(); RestructureFrameInformation();
break; break;
case kStateDecodable:
if (_state == kStateComplete)
{
// if complete, obviously decodable, keep as is.
return;
}
assert(_state == kStateEmpty ||
_state == kStateIncomplete);
break;
default: default:
// Should never happen // Should never happen
assert(!"FrameBuffer::SetState Incorrect frame buffer state as input"); assert(!"FrameBuffer::SetState Incorrect frame buffer state as input");

4
modules/video_coding/main/source/frame_buffer.h
View File

@ -64,6 +64,10 @@ public:
// NACK // NACK
// Zero out all entries in list up to and including the entry equal to _lowSeqNum // Zero out all entries in list up to and including the entry equal to _lowSeqNum
WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num); WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num);
// Hybrid extension: only NACK important packets, discard FEC packets
WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 num,
float rttScore);
void IncrementNackCount(); void IncrementNackCount();
WebRtc_Word16 GetNackCount() const; WebRtc_Word16 GetNackCount() const;

239
modules/video_coding/main/source/jitter_buffer.cc
View File

@ -16,6 +16,7 @@
#include "jitter_buffer.h" #include "jitter_buffer.h"
#include "jitter_buffer_common.h" #include "jitter_buffer_common.h"
#include "jitter_estimator.h" #include "jitter_estimator.h"
#include "media_optimization.h" // hybrid NACK/FEC thresholds.
#include "packet.h" #include "packet.h"
#include "event.h" #include "event.h"
@ -46,11 +47,14 @@ VCMJitterBuffer::FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestam
} }
bool bool
VCMJitterBuffer::CompleteKeyFrameCriteria(VCMFrameBuffer* frame, const void* /*notUsed*/) VCMJitterBuffer::CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame,
const void* /*notUsed*/)
{ {
const VCMFrameBufferStateEnum state = frame->GetState(); const VCMFrameBufferStateEnum state = frame->GetState();
// We can decode key frame or decodable/complete frames.
return (frame->FrameType() == kVideoFrameKey) && return (frame->FrameType() == kVideoFrameKey) &&
(state == kStateComplete); ((state == kStateComplete)
|| (state == kStateDecodable));
} }
// Constructor // Constructor
@ -76,7 +80,8 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId,
_numConsecutiveOldFrames(0), _numConsecutiveOldFrames(0),
_numConsecutiveOldPackets(0), _numConsecutiveOldPackets(0),
_jitterEstimate(vcmId, receiverId), _jitterEstimate(vcmId, receiverId),
_usingNACK(false), _rttMs(0),
_nackMode(kNoNack),
_NACKSeqNum(), _NACKSeqNum(),
_NACKSeqNumLength(0), _NACKSeqNumLength(0),
_missingMarkerBits(false), _missingMarkerBits(false),
@ -130,7 +135,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs)
_jitterEstimate = rhs._jitterEstimate; _jitterEstimate = rhs._jitterEstimate;
_delayEstimate = rhs._delayEstimate; _delayEstimate = rhs._delayEstimate;
_waitingForCompletion = rhs._waitingForCompletion; _waitingForCompletion = rhs._waitingForCompletion;
_usingNACK = rhs._usingNACK; _nackMode = rhs._nackMode;
_rttMs = rhs._rttMs;
_NACKSeqNumLength = rhs._NACKSeqNumLength; _NACKSeqNumLength = rhs._NACKSeqNumLength;
_missingMarkerBits = rhs._missingMarkerBits; _missingMarkerBits = rhs._missingMarkerBits;
_firstPacket = rhs._firstPacket; _firstPacket = rhs._firstPacket;
@ -162,7 +168,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs)
} }
WebRtc_UWord32 WebRtc_UWord32
VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, const WebRtc_UWord32 newTimestamp) VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp,
const WebRtc_UWord32 newTimestamp)
{ {
bool wrap = (newTimestamp < 0x0000ffff && existingTimestamp > 0xffff0000) || bool wrap = (newTimestamp < 0x0000ffff && existingTimestamp > 0xffff0000) ||
(newTimestamp > 0xffff0000 && existingTimestamp < 0x0000ffff); (newTimestamp > 0xffff0000 && existingTimestamp < 0x0000ffff);
@ -185,7 +192,8 @@ VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, const W
} }
// Start jitter buffer // Start jitter buffer
void VCMJitterBuffer::Start() void
VCMJitterBuffer::Start()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_running = true; _running = true;
@ -205,12 +213,14 @@ void VCMJitterBuffer::Start()
_waitingForCompletion.latestPacketTime = -1; _waitingForCompletion.latestPacketTime = -1;
_missingMarkerBits = false; _missingMarkerBits = false;
_firstPacket = true; _firstPacket = true;
_rttMs = 0;
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: start", this); WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: start", this);
} }
// Stop jitter buffer // Stop jitter buffer
void VCMJitterBuffer::Stop() void
VCMJitterBuffer::Stop()
{ {
_critSect.Enter(); _critSect.Enter();
_running = false; _running = false;
@ -231,21 +241,24 @@ void VCMJitterBuffer::Stop()
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: stop", this); WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: stop", this);
} }
bool VCMJitterBuffer::Running() const bool
VCMJitterBuffer::Running() const
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return _running; return _running;
} }
// Flush jitter buffer // Flush jitter buffer
void VCMJitterBuffer::Flush() void
VCMJitterBuffer::Flush()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
FlushInternal(); FlushInternal();
} }
// Must be called under the critical section _critSect // Must be called under the critical section _critSect
void VCMJitterBuffer::FlushInternal() void
VCMJitterBuffer::FlushInternal()
{ {
// Erase all frames from the sorted list and set their state to free. // Erase all frames from the sorted list and set their state to free.
_frameBuffersTSOrder.Flush(); _frameBuffersTSOrder.Flush();
@ -292,7 +305,8 @@ VCMJitterBuffer::ReleaseFrameInternal(VCMFrameBuffer* frame)
// Doing it here increases the degree of freedom for e.g. future // Doing it here increases the degree of freedom for e.g. future
// reconstructability of separate layers. Must be called under the // reconstructability of separate layers. Must be called under the
// critical section _critSect. // critical section _critSect.
void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) void
VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
{ {
if (frame == NULL) if (frame == NULL)
{ {
@ -385,8 +399,7 @@ void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
// Only signal if this is the oldest frame. // Only signal if this is the oldest frame.
// Not necessary the case due to packet reordering or NACK. // Not necessary the case due to packet reordering or NACK.
if(!_usingNACK || if (!WaitForNack() || (oldFrame != NULL && oldFrame == frame))
(oldFrame != NULL && oldFrame == frame))
{ {
_frameEvent.Set(); _frameEvent.Set();
} }
@ -512,7 +525,8 @@ VCMJitterBuffer::GetEmptyFrame()
} }
// Must be called under the critical section _critSect. // Must be called under the critical section _critSect.
VCMFrameListItem* VCMJitterBuffer::FindOldestSequenceNum() const VCMFrameListItem*
VCMJitterBuffer::FindOldestSequenceNum() const
{ {
WebRtc_UWord16 currentLow = 0xffff; WebRtc_UWord16 currentLow = 0xffff;
VCMFrameBufferStateEnum state = kStateFree; VCMFrameBufferStateEnum state = kStateFree;
@ -562,7 +576,8 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestSequenceNum() const
// Must be called under critical section // Must be called under critical section
// Based on sequence number // Based on sequence number
// Return NULL for lost packets // Return NULL for lost packets
VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame() VCMFrameListItem*
VCMJitterBuffer::FindOldestCompleteContinuousFrame()
{ {
// if we have more than one frame done since last time, pick oldest // if we have more than one frame done since last time, pick oldest
VCMFrameBuffer* oldestFrame = NULL; VCMFrameBuffer* oldestFrame = NULL;
@ -578,7 +593,8 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame()
{ {
if (kStateComplete != oldestFrame->GetState()) if (kStateComplete != oldestFrame->GetState())
{ {
// Try to see if the frame is complete even though the state is not complete. Can happen if markerbit is not set. // Try to see if the frame is complete even though the state is not
// complete. Can happen if markerbit is not set.
if (!CheckForCompleteFrame(oldestFrameItem)) if (!CheckForCompleteFrame(oldestFrameItem))
{ {
oldestFrame = NULL; oldestFrame = NULL;
@ -601,14 +617,15 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame()
// Use seqNum not timestamp since a full frame might be lost // Use seqNum not timestamp since a full frame might be lost
if (_lastDecodedSeqNum != -1) if (_lastDecodedSeqNum != -1)
{ {
// it's not enough that we have complete frame we need the seq numbers to be continuous too // it's not enough that we have complete frame we need the seq numbers
// for layers it's not enough that we have complete frame we need the layers to be continuous too // to be continuous too for layers it's not enough that we have complete
// frame we need the layers to be continuous too
currentLow = oldestFrame->GetLowSeqNum(); currentLow = oldestFrame->GetLowSeqNum();
WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum; WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
// we could have received the first packet of the last frame before a long period // we could have received the first packet of the last frame before a
// if drop, that case is handled by GetNackList // long period if drop, that case is handled by GetNackList
if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow)
{ {
// wait since we want a complete continuous frame // wait since we want a complete continuous frame
@ -618,11 +635,12 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame()
return oldestFrameItem; return oldestFrameItem;
} }
// Check if the oldest frame is complete even though it is not in a complete state. // Check if the oldest frame is complete even though it isn't complete.
// This can happen when makerbit is not set // This can happen when makerbit is not set
// Must be called under the critical section _critSect. // Must be called under the critical section _critSect.
// Return false for lost packets // Return false for lost packets
bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) bool
VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem)
{ {
const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameItem); const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameItem);
VCMFrameBuffer* oldestFrame = NULL; VCMFrameBuffer* oldestFrame = NULL;
@ -663,7 +681,8 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem)
} }
// Call from inside the critical section _critSect // Call from inside the critical section _critSect
void VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame) void
VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame)
{ {
if (frame == NULL) if (frame == NULL)
{ {
@ -679,7 +698,8 @@ void VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame)
// Calculate frame and bit rates // Calculate frame and bit rates
WebRtc_Word32 VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) WebRtc_Word32
VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
const WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); const WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp();
@ -744,7 +764,8 @@ WebRtc_Word32 VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord
} }
// Returns immediately or a X ms event hang waiting for a decodable frame, X decided by caller // Returns immediately or a X ms event hang waiting for a decodable frame, X decided by caller
VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) VCMEncodedFrame*
VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
{ {
if (!_running) if (!_running)
{ {
@ -853,38 +874,54 @@ VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 max
return oldestFrame; return oldestFrame;
} }
WebRtc_UWord32 VCMJitterBuffer::GetEstimatedJitterMS() WebRtc_UWord32
VCMJitterBuffer::GetEstimatedJitterMS()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return GetEstimatedJitterMsInternal(); return GetEstimatedJitterMsInternal();
} }
WebRtc_UWord32 VCMJitterBuffer::GetEstimatedJitterMsInternal() WebRtc_UWord32
VCMJitterBuffer::GetEstimatedJitterMsInternal()
{ {
WebRtc_UWord32 estimate = VCMJitterEstimator::OPERATING_SYSTEM_JITTER; WebRtc_UWord32 estimate = VCMJitterEstimator::OPERATING_SYSTEM_JITTER;
estimate += static_cast<WebRtc_UWord32>(_jitterEstimate.GetJitterEstimate() + 0.5);
// compute RTT multiplier for estimation
double rttMult = 1.0f;
if (_nackMode == kNackHybrid && _rttMs > kLowRttNackMs)
{
// from here we count on FEC
rttMult = 0.0f;
}
estimate += static_cast<WebRtc_UWord32>
(_jitterEstimate.GetJitterEstimate(rttMult) + 0.5);
if (_missingMarkerBits) if (_missingMarkerBits)
{ {
// Since the incoming packets are all missing marker bits we have to wait until the first // Since the incoming packets are all missing marker bits we have to
// packet of the next frame arrives, before we can safely say that the frame is complete. // wait until the first packet of the next frame arrives, before we can
// Therefore we have to compensate the jitter buffer level with one frame period. // safely say that the frame is complete. Therefore we have to compensate
// the jitter buffer level with one frame period.
// TODO(holmer): The timestamp diff should probably be filtered (max filter) since // TODO(holmer): The timestamp diff should probably be filtered
// the diff can alternate between e.g. 3000 and 6000 if we have a frame rate between // (max filter) since the diff can alternate between e.g. 3000 and 6000
// 15 and 30 frames per seconds. // if we have a frame rate between 15 and 30 frames per seconds.
estimate += _delayEstimate.CurrentTimeStampDiffMs(); estimate += _delayEstimate.CurrentTimeStampDiffMs();
} }
return estimate; return estimate;
} }
void VCMJitterBuffer::UpdateRtt(WebRtc_UWord32 rttMs) void
VCMJitterBuffer::UpdateRtt(WebRtc_UWord32 rttMs)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_rttMs = rttMs;
_jitterEstimate.UpdateRtt(rttMs); _jitterEstimate.UpdateRtt(rttMs);
} }
// wait for the first packet in the next frame to arrive // wait for the first packet in the next frame to arrive
WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, FrameType& incomingFrameType, WebRtc_Word64& renderTimeMs) WebRtc_Word64
VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS,
FrameType& incomingFrameType,
WebRtc_Word64& renderTimeMs)
{ {
if (!_running) if (!_running)
{ {
@ -913,7 +950,6 @@ WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, Fr
CleanUpOldFrames(); CleanUpOldFrames();
CleanUpSizeZeroFrames(); CleanUpSizeZeroFrames();
oldestFrame = _frameBuffersTSOrder.FirstFrame(); oldestFrame = _frameBuffersTSOrder.FirstFrame();
}else }else
{ {
@ -946,7 +982,8 @@ WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, Fr
// Will the packet sequence be complete if the next frame is grabbed for decoding right now? // Will the packet sequence be complete if the next frame is grabbed for decoding right now?
// That is, have we lost a frame between the last decoded frame and the next, or is the next // That is, have we lost a frame between the last decoded frame and the next, or is the next
// frame missing one or more packets? // frame missing one or more packets?
bool VCMJitterBuffer::CompleteSequenceWithNextFrame() bool
VCMJitterBuffer::CompleteSequenceWithNextFrame()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
// Finding oldest frame ready for decoder, but check sequence number and size // Finding oldest frame ready for decoder, but check sequence number and size
@ -989,7 +1026,8 @@ bool VCMJitterBuffer::CompleteSequenceWithNextFrame()
} }
// Returns immediately // Returns immediately
VCMEncodedFrame* VCMJitterBuffer::GetFrameForDecoding() VCMEncodedFrame*
VCMJitterBuffer::GetFrameForDecoding()
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
if (!_running) if (!_running)
@ -997,7 +1035,7 @@ VCMEncodedFrame* VCMJitterBuffer::GetFrameForDecoding()
return NULL; return NULL;
} }
if(_usingNACK) if (WaitForNack())
{ {
return GetFrameForDecodingNACK(); return GetFrameForDecodingNACK();
} }
@ -1061,6 +1099,7 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
{ {
// when we use NACK we don't release non complete frames // when we use NACK we don't release non complete frames
// unless we have a complete key frame. // unless we have a complete key frame.
// In hybrid mode, we may release decodable frames (non-complete)
// Clean up old frames and empty frames // Clean up old frames and empty frames
CleanUpOldFrames(); CleanUpOldFrames();
@ -1077,8 +1116,9 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
if (oldestFrame == NULL) if (oldestFrame == NULL)
{ {
continuous = false; continuous = false;
// If we didn't find one we're good with a complete key frame. // If we didn't find one we're good with a complete key/decodable frame.
oldestFrameListItem = _frameBuffersTSOrder.FindFrameListItem(CompleteKeyFrameCriteria); oldestFrameListItem = _frameBuffersTSOrder.FindFrameListItem(
CompleteDecodableKeyFrameCriteria);
if (oldestFrameListItem != NULL) if (oldestFrameListItem != NULL)
{ {
oldestFrame = oldestFrameListItem->GetItem(); oldestFrame = oldestFrameListItem->GetItem();
@ -1089,7 +1129,7 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
} }
} }
// We have a complete continuous frame, decode it. // We have a complete/decodable continuous frame, decode it.
// store seqnum // store seqnum
_lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); _lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
// store current time // store current time
@ -1181,7 +1221,10 @@ VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, bool incom
// Must be called under the critical section _critSect. Should never be called with // Must be called under the critical section _critSect. Should never be called with
// retransmitted frames, they must be filtered out before this function is called. // retransmitted frames, they must be filtered out before this function is called.
void void
VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs, WebRtc_UWord32 timestamp, WebRtc_UWord32 frameSize, bool incompleteFrame) VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs,
WebRtc_UWord32 timestamp,
WebRtc_UWord32 frameSize,
bool incompleteFrame)
{ {
if (latestPacketTimeMs == -1) if (latestPacketTimeMs == -1)
{ {
@ -1217,13 +1260,14 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word
highSeqNum = -1; highSeqNum = -1;
lowSeqNum = _lastDecodedSeqNum; lowSeqNum = _lastDecodedSeqNum;
// find higest seqnumbers // find highest seqnumbers
for (i = 0; i < _maxNumberOfFrames; ++i) for (i = 0; i < _maxNumberOfFrames; ++i)
{ {
seqNum = _frameBuffers[i]->GetHighSeqNum(); seqNum = _frameBuffers[i]->GetHighSeqNum();
// Ignore free frames // Ignore free / empty frames
VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState(); VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState();
if ((kStateFree != state) && if ((kStateFree != state) &&
(kStateEmpty != state) && (kStateEmpty != state) &&
(kStateDecoding != state) && (kStateDecoding != state) &&
@ -1262,7 +1306,8 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
WebRtc_Word32 highSeqNum = -1; WebRtc_Word32 highSeqNum = -1;
listExtended = false; listExtended = false;
if (!_usingNACK) // don't create list, if we won't wait for it
if (!WaitForNack())
{ {
nackSize = 0; nackSize = 0;
return NULL; return NULL;
@ -1277,11 +1322,10 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
// write a list of all seq num we have // write a list of all seq num we have
if (lowSeqNum == -1 || highSeqNum == -1) if (lowSeqNum == -1 || highSeqNum == -1)
{ {
//This happens if we lose the first packet, nothing is poped //This happens if we lose the first packet, nothing is popped
if (highSeqNum == -1) if (highSeqNum == -1)
{ {
nackSize = 0;// we have not received any packets yet nackSize = 0;// we have not received any packets yet
} }
else else
{ {
@ -1371,7 +1415,8 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
// We have cleaned up the jb and found a key frame // We have cleaned up the jb and found a key frame
// The function itself has set last decoded seq. // The function itself has set last decoded seq.
WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1, WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1,
"\tKey frame found. _lastDecodedSeqNum[0] %d", _lastDecodedSeqNum); "\tKey frame found. _lastDecodedSeqNum[0] %d",
_lastDecodedSeqNum);
nackSize = 0; nackSize = 0;
} }
@ -1384,22 +1429,46 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
_NACKSeqNumInternal[i] = seqNumberIterator; _NACKSeqNumInternal[i] = seqNumberIterator;
seqNumberIterator++; seqNumberIterator++;
} }
// now we have a list of all seq numbers that could have been sent
// now we have a list of all sequence numbers that could have been sent
// zero out the ones we have received // zero out the ones we have received
for (i = 0; i < _maxNumberOfFrames; i++) for (i = 0; i < _maxNumberOfFrames; i++)
{ {
// loop all created frames // loop all created frames
// We dont need to check if frame is decoding since lowSeqNum is based on _lastDecodedSeqNum // We don't need to check if frame is decoding since lowSeqNum is based
// on _lastDecodedSeqNum
// Ignore free frames // Ignore free frames
VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState(); VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState();
if ((kStateFree != state) && if ((kStateFree != state) &&
(kStateEmpty != state) && (kStateEmpty != state) &&
(kStateDecoding != state)) (kStateDecoding != state))
{ {
_frameBuffers[i]->ZeroOutSeqNum(_NACKSeqNumInternal, numberOfSeqNum); // Reaching thus far means we are going to update the nack list
// When in hybrid mode, we also need to check empty frames, so as not
// to add empty packets to the nack list
if (_nackMode == kNackHybrid)
{
// build external rttScore based on RTT value
float rttScore = 1.0f;
_frameBuffers[i]->ZeroOutSeqNumHybrid(_NACKSeqNumInternal,
numberOfSeqNum,
rttScore);
if (_frameBuffers[i]->IsRetransmitted() == false)
{
// if no retransmission required,set the state to decodable
// meaning that we will not wait for NACK
_frameBuffers[i]->SetState(kStateDecodable);
}
}
else
{
// used when the frame is being processed by the decoding thread // used when the frame is being processed by the decoding thread
// dont need to use that info in this loop // don't need to use that info in this loop
_frameBuffers[i]->ZeroOutSeqNum(_NACKSeqNumInternal,
numberOfSeqNum);
}
} }
} }
@ -1407,7 +1476,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
int emptyIndex = -1; int emptyIndex = -1;
for (i = 0; i < numberOfSeqNum; i++) for (i = 0; i < numberOfSeqNum; i++)
{ {
if (_NACKSeqNumInternal[i] == -1) if (_NACKSeqNumInternal[i] == -1 || _NACKSeqNumInternal[i] == -2 )
{ {
// this is empty // this is empty
if (emptyIndex == -1) if (emptyIndex == -1)
@ -1444,11 +1513,14 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
// convert to unsigned short 16 bit and store in a list to be used externally. // convert to unsigned short 16 bit and store in a list to be used externally.
if (nackSize > _NACKSeqNumLength) if (nackSize > _NACKSeqNumLength)
{ {
listExtended=true; // Larger list means that the nack list has been extended since the last call. // Larger list means that the nack list was extended since the last call.
listExtended = true;
} }
for(WebRtc_UWord32 j = 0; j < nackSize; j++) for(WebRtc_UWord32 j = 0; j < nackSize; j++)
{ {
// Check if the list has been extended since it was last created. I.e, new items have been added // Check if the list has been extended since it was last created. I.e,
// new items have been added
if (_NACKSeqNumLength > j && !listExtended) if (_NACKSeqNumLength > j && !listExtended)
{ {
WebRtc_UWord32 k = 0; WebRtc_UWord32 k = 0;
@ -1459,7 +1531,6 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
{ {
break; break;
} }
} }
if (k == _NACKSeqNumLength) // New item not found in last list. if (k == _NACKSeqNumLength) // New item not found in last list.
{ {
@ -1472,6 +1543,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
} }
_NACKSeqNum[j] = (WebRtc_UWord16)_NACKSeqNumInternal[j]; _NACKSeqNum[j] = (WebRtc_UWord16)_NACKSeqNumInternal[j];
} }
_NACKSeqNumLength = nackSize; _NACKSeqNumLength = nackSize;
return _NACKSeqNum; return _NACKSeqNum;
@ -1511,6 +1583,10 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
VCMFrameBufferEnum ret = kSizeError; VCMFrameBufferEnum ret = kSizeError;
VCMFrameBuffer* frame = static_cast<VCMFrameBuffer*>(buffer); VCMFrameBuffer* frame = static_cast<VCMFrameBuffer*>(buffer);
// Empty packets may bias the jitter estimate (lacking size component),
// therefore don't let empty packet trigger the following updates:
if (packet.frameType != kFrameEmpty)
{
if (_firstPacket) if (_firstPacket)
{ {
// Now it's time to start estimating jitter // Now it's time to start estimating jitter
@ -1535,11 +1611,11 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
_waitingForCompletion.frameSize = 0; _waitingForCompletion.frameSize = 0;
_waitingForCompletion.timestamp = 0; _waitingForCompletion.timestamp = 0;
} }
}
if (frame != NULL) if (frame != NULL)
{ {
VCMFrameBufferStateEnum state = frame->GetState(); VCMFrameBufferStateEnum state = frame->GetState();
if (state == kStateDecoding && packet.sizeBytes == 0) if (state == kStateDecoding && packet.sizeBytes == 0)
{ {
// Filler packet, make sure we update the last decoded seq num // Filler packet, make sure we update the last decoded seq num
@ -1667,18 +1743,20 @@ VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const
} }
// Get nack status (enabled/disabled) // Get nack status (enabled/disabled)
bool VCMJitterBuffer::GetNackStatus() VCMNackMode
VCMJitterBuffer::GetNackMode() const
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return _usingNACK; return _nackMode;
} }
// Enable/disable nack // Set NACK mode
void VCMJitterBuffer::SetNackStatus(bool enable) void
VCMJitterBuffer::SetNackMode(VCMNackMode mode)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_usingNACK = enable; _nackMode = mode;
if (!_usingNACK) if (_nackMode == kNoNack)
{ {
_jitterEstimate.ResetNackCount(); _jitterEstimate.ResetNackCount();
} }
@ -1878,4 +1956,33 @@ VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame)
} }
} }
bool
VCMJitterBuffer::WaitForNack()
{
// NACK disabled -> can't wait
if (_nackMode == kNoNack)
{
return false;
}
// NACK only -> always wait
else if (_nackMode == kNackInfinite)
{
return true;
}
// else: hybrid mode, evaluate
// RTT high, don't wait
if (_rttMs >= kHighRttNackMs)
{
return false;
}
// RTT low, we can afford the wait
else if (_rttMs <= kLowRttNackMs)
{
return true;
}
// interim values - hybrid mode
return true;
}
} }

33
modules/video_coding/main/source/jitter_buffer.h
View File

@ -24,6 +24,13 @@
namespace webrtc namespace webrtc
{ {
enum VCMNackMode
{
kNackInfinite,
kNackHybrid,
kNoNack
};
// forward declarations // forward declarations
class VCMFrameBuffer; class VCMFrameBuffer;
class VCMPacket; class VCMPacket;
@ -90,7 +97,7 @@ public:
WebRtc_Word32 GetFrame(const VCMPacket& packet, VCMEncodedFrame*&); WebRtc_Word32 GetFrame(const VCMPacket& packet, VCMEncodedFrame*&);
VCMEncodedFrame* GetFrame(const VCMPacket& packet); // deprecated VCMEncodedFrame* GetFrame(const VCMPacket& packet); // deprecated
// Returns the time in ms when the latest packet was insterted into the frame. // Returns the time in ms when the latest packet was inserted into the frame.
// Retransmitted is set to true if any of the packets belonging to the frame // Retransmitted is set to true if any of the packets belonging to the frame
// has been retransmitted. // has been retransmitted.
WebRtc_Word64 LastPacketTime(VCMEncodedFrame* frame, bool& retransmitted) const; WebRtc_Word64 LastPacketTime(VCMEncodedFrame* frame, bool& retransmitted) const;
@ -103,8 +110,8 @@ public:
void UpdateRtt(WebRtc_UWord32 rttMs); void UpdateRtt(WebRtc_UWord32 rttMs);
// NACK // NACK
void SetNackStatus(bool enable); // Enable/disable nack void SetNackMode(VCMNackMode mode); // Enable/disable nack
bool GetNackStatus(); // Get nack status (enabled/disabled) VCMNackMode GetNackMode() const; // Get nack mode
// Get list of missing sequence numbers (size in number of elements) // Get list of missing sequence numbers (size in number of elements)
WebRtc_UWord16* GetNackList(WebRtc_UWord16& nackSize, bool& listExtended); WebRtc_UWord16* GetNackList(WebRtc_UWord16& nackSize, bool& listExtended);
@ -162,18 +169,23 @@ protected:
private: private:
static bool FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp); static bool FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp);
static bool CompleteKeyFrameCriteria(VCMFrameBuffer* frame, const void* notUsed); static bool CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame,
const void* notUsed);
// Decide whether should wait for NACK (mainly relevant for hybrid mode)
bool WaitForNack();
WebRtc_Word32 _vcmId; WebRtc_Word32 _vcmId;
WebRtc_Word32 _receiverId; WebRtc_Word32 _receiverId;
bool _running; // If we are running (have started) or not // If we are running (have started) or not
bool _running;
CriticalSectionWrapper& _critSect; CriticalSectionWrapper& _critSect;
bool _master; bool _master;
// Event to signal when we have a frame ready for decoder // Event to signal when we have a frame ready for decoder
VCMEvent _frameEvent; VCMEvent _frameEvent;
// Event to signal when we have received a packet // Event to signal when we have received a packet
VCMEvent _packetEvent; VCMEvent _packetEvent;
WebRtc_Word32 _maxNumberOfFrames; // Number of allocated frames // Number of allocated frames
WebRtc_Word32 _maxNumberOfFrames;
// Array of pointers to the frames in JB // Array of pointers to the frames in JB
VCMFrameBuffer* _frameBuffers[kMaxNumberOfFrames]; VCMFrameBuffer* _frameBuffers[kMaxNumberOfFrames];
VCMFrameListTimestampOrderAsc _frameBuffersTSOrder; VCMFrameListTimestampOrderAsc _frameBuffersTSOrder;
@ -189,10 +201,12 @@ private:
WebRtc_UWord8 _receiveStatistics[4]; WebRtc_UWord8 _receiveStatistics[4];
// Latest calculated frame rates of incoming stream // Latest calculated frame rates of incoming stream
WebRtc_UWord8 _incomingFrameRate; WebRtc_UWord8 _incomingFrameRate;
WebRtc_UWord32 _incomingFrameCount; // Frame counter, reset in GetUpdate // Frame counter, reset in GetUpdate
WebRtc_UWord32 _incomingFrameCount;
// Real time for last _frameCount reset // Real time for last _frameCount reset
WebRtc_Word64 _timeLastIncomingFrameCount; WebRtc_Word64 _timeLastIncomingFrameCount;
WebRtc_UWord32 _incomingBitCount; // Received bits counter, reset in GetUpdate // Received bits counter, reset in GetUpdate
WebRtc_UWord32 _incomingBitCount;
WebRtc_UWord32 _incomingBitRate; WebRtc_UWord32 _incomingBitRate;
WebRtc_UWord32 _dropCount; // Frame drop counter WebRtc_UWord32 _dropCount; // Frame drop counter
// Number of frames in a row that have been too old // Number of frames in a row that have been too old
@ -204,9 +218,10 @@ private:
// Calculates network delays used for jitter calculations // Calculates network delays used for jitter calculations
VCMInterFrameDelay _delayEstimate; VCMInterFrameDelay _delayEstimate;
VCMJitterSample _waitingForCompletion; VCMJitterSample _waitingForCompletion;
WebRtc_UWord32 _rttMs;
// NACK // NACK
bool _usingNACK; // If we are using nack VCMNackMode _nackMode;
// Holds the internal nack list (the missing seqence numbers) // Holds the internal nack list (the missing seqence numbers)
WebRtc_Word32 _NACKSeqNumInternal[kNackHistoryLength]; WebRtc_Word32 _NACKSeqNumInternal[kNackHistoryLength];
WebRtc_UWord16 _NACKSeqNum[kNackHistoryLength]; WebRtc_UWord16 _NACKSeqNum[kNackHistoryLength];

3
modules/video_coding/main/source/jitter_buffer_common.h
View File

@ -46,7 +46,8 @@ enum VCMFrameBufferStateEnum
kStateEmpty, // frame popped by the RTP receiver kStateEmpty, // frame popped by the RTP receiver
kStateIncomplete, // frame that have one or more packet(s) stored kStateIncomplete, // frame that have one or more packet(s) stored
kStateComplete, // frame that have all packets kStateComplete, // frame that have all packets
kStateDecoding // frame popped by the decoding thread kStateDecoding, // frame popped by the decoding thread
kStateDecodable // Hybrid mode - frame can be decoded
}; };
enum { kH264StartCodeLengthBytes = 4}; enum { kH264StartCodeLengthBytes = 4};

4
modules/video_coding/main/source/jitter_estimator.cc
View File

@ -422,7 +422,7 @@ VCMJitterEstimator::UpdateMaxFrameSize(WebRtc_UWord32 frameSizeBytes)
// Returns the current filtered estimate if available, // Returns the current filtered estimate if available,
// otherwise tries to calculate an estimate. // otherwise tries to calculate an estimate.
double double
VCMJitterEstimator::GetJitterEstimate() VCMJitterEstimator::GetJitterEstimate(double rttMultiplier)
{ {
double jitterMS = CalculateEstimate(); double jitterMS = CalculateEstimate();
if (_filterJitterEstimate > jitterMS) if (_filterJitterEstimate > jitterMS)
@ -431,7 +431,7 @@ VCMJitterEstimator::GetJitterEstimate()
} }
if (_nackCount >= _nackLimit) if (_nackCount >= _nackLimit)
{ {
return jitterMS + _rttFilter.RttMs(); return jitterMS + _rttFilter.RttMs() * rttMultiplier;
} }
return jitterMS; return jitterMS;
} }

4
modules/video_coding/main/source/jitter_estimator.h
View File

@ -41,9 +41,11 @@ public:
// Returns the current jitter estimate in milliseconds and adds // Returns the current jitter estimate in milliseconds and adds
// also adds an RTT dependent term in cases of retransmission. // also adds an RTT dependent term in cases of retransmission.
// Input:
// - rttMultiplier : RTT param multiplier (when applicable).
// //
// Return value : Jitter estimate in milliseconds // Return value : Jitter estimate in milliseconds
double GetJitterEstimate(); double GetJitterEstimate(double rttMultiplier);
// Updates the nack counter/timer. // Updates the nack counter/timer.
// //

433
modules/video_coding/main/source/media_opt_util.cc
View File

@ -18,52 +18,59 @@
#include <math.h> #include <math.h>
#include <float.h> #include <float.h>
#include <limits.h> #include <limits.h>
#include <stdio.h>
namespace webrtc { namespace webrtc {
bool
VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm) bool VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm)
{
if (pm == NULL)
{ {
if (pm == NULL) {
return true; return true;
} }
return pm->_score > _score; return pm->_score > _score;
} }
bool bool VCMNackFecMethod::ProtectionFactor(const VCMProtectionParameters* /*parameters*/)
VCMNackFecMethod::ProtectionFactor(const VCMProtectionParameters* /*parameters*/)
{ {
// use FEC model with modification with RTT for now // use FEC model with modification with RTT for now
return true; return true;
} }
bool bool VCMNackFecMethod::EffectivePacketLoss(const VCMProtectionParameters* /*parameters*/)
VCMNackFecMethod::EffectivePacketLoss(const VCMProtectionParameters* /*parameters*/)
{ {
// use FEC model with modification with RTT for now // use FEC model with modification with RTT for now
return true; return true;
} }
bool VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
bool
VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
{ {
// Hybrid Nack FEC has three operational modes:
// 1. Low RTT - Nack only (Set FEC rates to zero)
// 2. High RTT - FEC Only
// 3. Medium RTT values - Hybrid ; in hybrid mode, we will only nack the residual
// following the decoding of the FEC (and not in all cases, refer to JB logic)
// Low RTT - NACK only mode
if (parameters->rtt < kLowRttNackMs)
{
// Set the FEC parameters to 0
_protectionFactorK = 0;
_protectionFactorD = 0;
// assume packets will be restored via NACK
// TODO: relax this assumption?
_effectivePacketLoss = 0;
_score = _efficiency;
return true;
}
// otherwise: we count on FEC; if the RTT is below a threshold, then we can
// nack the residual, based on a decision made in the JB.
// TODO(mikhal): adapt the FEC rate based on the RTT, i.e. the the
// level on which we will rely on NACK, e.g. less as we approach upper threshold.
VCMFecMethod fecMethod; VCMFecMethod fecMethod;
VCMNackMethod nackMethod;
const WebRtc_UWord8 plossMax = 129; const WebRtc_UWord8 plossMax = 129;
WebRtc_UWord16 rttMax = nackMethod.MaxRttNack();
// We should reduce the NACK threshold for NackFec protection method,
// with FEC and ER, we should only use NACK for small RTT, to avoid delay
//But this parameter change should be shared with RTP and JB
//rttMax = (WebRtc_UWord16) 0.5*rttMax;
// Compute the protection factor // Compute the protection factor
fecMethod.ProtectionFactor(parameters); fecMethod.ProtectionFactor(parameters);
@ -76,47 +83,58 @@ VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
WebRtc_UWord8 effPacketLoss = fecMethod._effectivePacketLoss; WebRtc_UWord8 effPacketLoss = fecMethod._effectivePacketLoss;
float resPacketLoss = fecMethod._residualPacketLoss; float resPacketLoss = fecMethod._residualPacketLoss;
// Correct FEC rates based on the RTT ( NACK effectiveness)
WebRtc_Word16 rttIndex= (WebRtc_UWord16) parameters->rtt; WebRtc_Word16 rttIndex= (WebRtc_UWord16) parameters->rtt;
float softnessRtt = 1.0; float softnessRtt = 1.0;
if (parameters->rtt < rttMax) if (parameters->rtt < kHighRttNackMs)
{ {
// TODO(mikhal): update table
softnessRtt = (float)VCMNackFecTable[rttIndex] / (float)4096.0; softnessRtt = (float)VCMNackFecTable[rttIndex] / (float)4096.0;
// soften ER with NACK on // soften ER with NACK on
//table depends on roundtrip time relative to rttMax (NACK Threshold) // table depends on RTT relative to rttMax (NACK Threshold)
_effectivePacketLoss = (WebRtc_UWord8)(effPacketLoss * softnessRtt); _effectivePacketLoss = (WebRtc_UWord8)(effPacketLoss * softnessRtt);
// soften FEC with NACK on // soften FEC with NACK on
//table depends on roundtrip time relative to rttMax (NACK Threshold) // table depends on RTT relative to rttMax (NACK Threshold)
_protectionFactorK = (WebRtc_UWord8) (protFactorK * softnessRtt); _protectionFactorK = (WebRtc_UWord8) (protFactorK * softnessRtt);
_protectionFactorD = (WebRtc_UWord8) (protFactorD * softnessRtt); _protectionFactorD = (WebRtc_UWord8) (protFactorD * softnessRtt);
} }
// else - NACK is disabled, rely on FEC only
//make sure I frame protection is at least larger than P frame protection, and at least as high as received loss // make sure I frame protection is at least larger than P frame protection,
// and at least as high as received loss
WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr); WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr);
_protectionFactorK = static_cast<WebRtc_UWord8>(VCM_MAX(packetLoss,VCM_MAX(_scaleProtKey*protFactorD,protFactorK))); _protectionFactorK = static_cast<WebRtc_UWord8> (VCM_MAX(packetLoss,
VCM_MAX(_scaleProtKey * protFactorD, protFactorK)));
// check limit on amount of protection for I frame: 50% is max // check limit on amount of protection for I frame: 50% is max
if (_protectionFactorK >= plossMax) _protectionFactorK = plossMax - 1; if (_protectionFactorK >= plossMax)
_protectionFactorK = plossMax - 1;
// Bit cost for NackFec // Bit cost for NackFec
// NACK cost: based on residual packet loss (since we should only NACK packet not recovered by FEC) // NACK cost: based on residual packet loss (since we should only NACK packets
// not recovered by FEC)
_efficiency = 0.0f; _efficiency = 0.0f;
if (parameters->rtt < rttMax) if (parameters->rtt < kHighRttNackMs)
{
_efficiency = parameters->bitRate * resPacketLoss / (1.0f + resPacketLoss); _efficiency = parameters->bitRate * resPacketLoss / (1.0f + resPacketLoss);
} else
{
// efficiency based on FEC only
// add FEC cost: ignore I frames for now // add FEC cost: ignore I frames for now
float fecRate = static_cast<float> (_protectionFactorD) / 255.0f; float fecRate = static_cast<float> (_protectionFactorD) / 255.0f;
if (fecRate >= 0.0f) if (fecRate >= 0.0f)
_efficiency += parameters->bitRate * fecRate; _efficiency += parameters->bitRate * fecRate;
}
_score = _efficiency; _score = _efficiency;
//Protection/fec rates obtained above is defined relative to total number of packets (total rate: source+fec) // Protection/fec rates obtained above are defined relative to total number of
//FEC in RTP module assumes protection factor is defined relative to source number of packets // packets (total rate: source + fec) FEC in RTP module assumes protection
//so we should convert the factor to reduce mismatch between mediaOpt suggested rate and the actual rate // factor is defined relative to source number of packets so we should convert
// the factor to reduce mismatch between mediaOpt's rate and the actual one
WebRtc_UWord8 codeRate = protFactorK; WebRtc_UWord8 codeRate = protFactorK;
_protectionFactorK = fecMethod.ConvertFECRate(codeRate); _protectionFactorK = fecMethod.ConvertFECRate(codeRate);
codeRate = protFactorD; codeRate = protFactorD;
@ -125,15 +143,13 @@ VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
return true; return true;
} }
bool VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, WebRtc_UWord16 rttTime)
bool
VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, WebRtc_UWord16 rttTime)
{ {
WebRtc_UWord16 rttMax = MaxRttNack(); WebRtc_UWord16 rttMax = MaxRttNack();
//For large RTT, we should rely on some Error Resilience, so we set packetLossEnc = 0 // For large RTT, we should rely on some Error Resilience, so we set
//for RTT less than the NACK threshold // packetLossEnc = 0 for RTT less than the NACK threshold
if (rttTime < rttMax) if (rttTime < rttMax)
effPacketLoss = 0; //may want a softer transition here effPacketLoss = 0; //may want a softer transition here
@ -142,15 +158,13 @@ VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, WebRtc_UWord16 r
return true; return true;
} }
bool
VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters)
{
bool VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters)
{
// Compute the effective packet loss for ER // Compute the effective packet loss for ER
WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8) (255 * parameters->lossPr); WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8) (255 * parameters->lossPr);
WebRtc_UWord16 rttTime = (WebRtc_UWord16) parameters->rtt; WebRtc_UWord16 rttTime = (WebRtc_UWord16) parameters->rtt;
EffectivePacketLoss(effPacketLoss, rttTime); EffectivePacketLoss(effPacketLoss, rttTime);
//
// Compute the NACK bit cost // Compute the NACK bit cost
_efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr); _efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr);
@ -164,11 +178,9 @@ VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters)
return true; return true;
} }
WebRtc_UWord8 VCMFecMethod::BoostCodeRateKey(WebRtc_UWord8 packetFrameDelta,
WebRtc_UWord8 WebRtc_UWord8 packetFrameKey) const
VCMFecMethod::BoostCodeRateKey(WebRtc_UWord8 packetFrameDelta, WebRtc_UWord8 packetFrameKey) const
{ {
WebRtc_UWord8 boostRateKey = 2; WebRtc_UWord8 boostRateKey = 2;
// default: ratio scales the FEC protection up for I frames // default: ratio scales the FEC protection up for I frames
WebRtc_UWord8 ratio = 1; WebRtc_UWord8 ratio = 1;
@ -182,61 +194,53 @@ VCMFecMethod::BoostCodeRateKey(WebRtc_UWord8 packetFrameDelta, WebRtc_UWord8 pac
} }
WebRtc_UWord8 WebRtc_UWord8 VCMFecMethod::ConvertFECRate(WebRtc_UWord8 codeRateRTP) const
VCMFecMethod::ConvertFECRate(WebRtc_UWord8 codeRateRTP) const
{ {
return static_cast<WebRtc_UWord8> (VCM_MIN(255,(0.5 + 255.0*codeRateRTP/(float)(255 - codeRateRTP)))); return static_cast<WebRtc_UWord8> (VCM_MIN(255,(0.5 + 255.0*codeRateRTP/(float)(255 - codeRateRTP))));
} }
// AvgRecoveryFEC: average recovery from FEC, assuming random packet loss model // AvgRecoveryFEC: average recovery from FEC, assuming random packet loss model
// Computed offline for a range of FEC code parameters and loss rates // Computed offline for a range of FEC code parameters and loss rates
float float VCMFecMethod::AvgRecoveryFEC(const VCMProtectionParameters* parameters) const
VCMFecMethod::AvgRecoveryFEC(const VCMProtectionParameters* parameters) const
{ {
// Total (avg) bits available per frame: total rate over actual/sent frame rate // Total (avg) bits available per frame: total rate over actual/sent frame rate
// units are kbits/frame // units are kbits/frame
const WebRtc_UWord16 bitRatePerFrame = static_cast<WebRtc_UWord16>(parameters->bitRate/(parameters->frameRate)); const WebRtc_UWord16 bitRatePerFrame = static_cast<WebRtc_UWord16> (parameters->bitRate
/ (parameters->frameRate));
// Total (avg) number of packets per frame (source and fec): // Total (avg) number of packets per frame (source and fec):
const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8)((float)bitRatePerFrame*1000.0/(float)(8.0*_maxPayloadSize) + 0.5); const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8) ((float) bitRatePerFrame * 1000.0
/ (float) (8.0 * _maxPayloadSize) + 0.5);
// parameters for tables // parameters for tables
const WebRtc_UWord8 codeSize = 24; const WebRtc_UWord8 codeSize = 24;
const WebRtc_UWord8 plossMax = 129; const WebRtc_UWord8 plossMax = 129;
const WebRtc_UWord16 maxErTableSize = 38700; const WebRtc_UWord16 maxErTableSize = 38700;
//
//
// Get index for table // Get index for table
const float protectionFactor = (float) _protectionFactorD / (float) 255; const float protectionFactor = (float) _protectionFactorD / (float) 255;
WebRtc_UWord8 fecPacketsPerFrame = (WebRtc_UWord8) (0.5 + protectionFactor * avgTotPackets); WebRtc_UWord8 fecPacketsPerFrame = (WebRtc_UWord8) (0.5 + protectionFactor * avgTotPackets);
WebRtc_UWord8 sourcePacketsPerFrame = avgTotPackets - fecPacketsPerFrame; WebRtc_UWord8 sourcePacketsPerFrame = avgTotPackets - fecPacketsPerFrame;
if (fecPacketsPerFrame == 0) if (fecPacketsPerFrame == 0) {
{
return 0.0; // no protection, so avg. recov from FEC == 0 return 0.0; // no protection, so avg. recov from FEC == 0
} }
// table defined up to codeSizexcodeSize code // table defined up to codeSizexcodeSize code
if (sourcePacketsPerFrame > codeSize) if (sourcePacketsPerFrame > codeSize) {
{
sourcePacketsPerFrame = codeSize; sourcePacketsPerFrame = codeSize;
} }
// check: protection factor is maxed at 50%, so this should never happen // check: protection factor is maxed at 50%, so this should never happen
if (sourcePacketsPerFrame < 1) if (sourcePacketsPerFrame < 1) {
{
assert("average number of source packets below 1\n"); assert("average number of source packets below 1\n");
} }
// index for ER tables: up to codeSizexcodeSize mask // index for ER tables: up to codeSizexcodeSize mask
WebRtc_UWord16 codeIndexTable[codeSize * codeSize]; WebRtc_UWord16 codeIndexTable[codeSize * codeSize];
WebRtc_UWord16 k = -1; WebRtc_UWord16 k = -1;
for(WebRtc_UWord8 i=1;i<=codeSize;i++) for (WebRtc_UWord8 i = 1; i <= codeSize; i++) {
{ for (WebRtc_UWord8 j = 1; j <= i; j++) {
for(WebRtc_UWord8 j=1;j<=i;j++)
{
k += 1; k += 1;
codeIndexTable[(j - 1) * codeSize + i - 1] = k; codeIndexTable[(j - 1) * codeSize + i - 1] = k;
} }
@ -244,20 +248,19 @@ VCMFecMethod::AvgRecoveryFEC(const VCMProtectionParameters* parameters) const
const WebRtc_UWord8 lossRate = (WebRtc_UWord8) (255.0 * parameters->lossPr + 0.5f); const WebRtc_UWord8 lossRate = (WebRtc_UWord8) (255.0 * parameters->lossPr + 0.5f);
const WebRtc_UWord16 codeIndex = (fecPacketsPerFrame - 1)*codeSize + (sourcePacketsPerFrame - 1); const WebRtc_UWord16 codeIndex = (fecPacketsPerFrame - 1) * codeSize
+ (sourcePacketsPerFrame - 1);
const WebRtc_UWord16 indexTable = codeIndexTable[codeIndex] * plossMax + lossRate; const WebRtc_UWord16 indexTable = codeIndexTable[codeIndex] * plossMax + lossRate;
const WebRtc_UWord16 codeIndex2 = (fecPacketsPerFrame) * codeSize + (sourcePacketsPerFrame); const WebRtc_UWord16 codeIndex2 = (fecPacketsPerFrame) * codeSize + (sourcePacketsPerFrame);
WebRtc_UWord16 indexTable2 = codeIndexTable[codeIndex2] * plossMax + lossRate; WebRtc_UWord16 indexTable2 = codeIndexTable[codeIndex2] * plossMax + lossRate;
// checks on table index // checks on table index
if (indexTable >= maxErTableSize) if (indexTable >= maxErTableSize) {
{
assert("ER table index too large\n"); assert("ER table index too large\n");
} }
if (indexTable2 >= maxErTableSize) if (indexTable2 >= maxErTableSize) {
{
indexTable2 = indexTable; indexTable2 = indexTable;
} }
// //
@ -268,23 +271,21 @@ VCMFecMethod::AvgRecoveryFEC(const VCMProtectionParameters* parameters) const
WebRtc_UWord8 avgFecRecov2 = 0; WebRtc_UWord8 avgFecRecov2 = 0;
float avgFecRecov = 0; float avgFecRecov = 0;
if (fecPacketsPerFrame > 0) if (fecPacketsPerFrame > 0) {
{
avgFecRecov1 = VCMAvgFECRecoveryXOR[indexTable]; avgFecRecov1 = VCMAvgFECRecoveryXOR[indexTable];
avgFecRecov2 = VCMAvgFECRecoveryXOR[indexTable2]; avgFecRecov2 = VCMAvgFECRecoveryXOR[indexTable2];
} }
// interpolate over two FEC codes // interpolate over two FEC codes
const float weightRpl = (float)(0.5 + protectionFactor*avgTotPackets) - (float)fecPacketsPerFrame; const float weightRpl = (float) (0.5 + protectionFactor * avgTotPackets)
avgFecRecov = (float)weightRpl * (float)avgFecRecov2 + (float)(1.0 - weightRpl) * (float)avgFecRecov1; - (float) fecPacketsPerFrame;
avgFecRecov = (float) weightRpl * (float) avgFecRecov2 + (float) (1.0 - weightRpl)
* (float) avgFecRecov1;
return avgFecRecov; return avgFecRecov;
} }
bool bool VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
{ {
// FEC PROTECTION SETTINGS: varies with packet loss and bitrate // FEC PROTECTION SETTINGS: varies with packet loss and bitrate
@ -292,7 +293,6 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
const float bitRate = parameters->bitRate; const float bitRate = parameters->bitRate;
WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr); WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr);
// Size of tables // Size of tables
const WebRtc_UWord16 maxFecTableSize = 6450; const WebRtc_UWord16 maxFecTableSize = 6450;
// Parameters for range of rate and packet loss for tables // Parameters for range of rate and packet loss for tables
@ -301,7 +301,8 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
const WebRtc_UWord8 plossMax = 129; const WebRtc_UWord8 plossMax = 129;
// //
//Just for testing: for the case where we randomly lose slices instead of RTP packets and use SingleMode packetization in RTP module // Just for testing: for the case where we randomly lose slices instead of
// RTP packets and use SingleMode packetization in RTP module
// const WebRtc_UWord16 slice_size = 3000/6; //corresponds to rate=1000k with 4 cores // const WebRtc_UWord16 slice_size = 3000/6; //corresponds to rate=1000k with 4 cores
//float slice_mtu = (float)_maxPayloadSize/(float)slice_size; //float slice_mtu = (float)_maxPayloadSize/(float)slice_size;
@ -310,23 +311,24 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
//Total (avg) bits available per frame: total rate over actual/sent frame rate //Total (avg) bits available per frame: total rate over actual/sent frame rate
//units are kbits/frame //units are kbits/frame
const WebRtc_UWord16 bitRatePerFrame = static_cast<WebRtc_UWord16>(slice_mtu*bitRate/(parameters->frameRate)); const WebRtc_UWord16 bitRatePerFrame = static_cast<WebRtc_UWord16> (slice_mtu * bitRate
/ (parameters->frameRate));
//Total (avg) number of packets per frame (source and fec): //Total (avg) number of packets per frame (source and fec):
const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8)((float)bitRatePerFrame*1000.0/(float)(8.0*_maxPayloadSize) + 0.5); const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8) ((float) bitRatePerFrame * 1000.0
/ (float) (8.0 * _maxPayloadSize) + 0.5);
// TODO(marpan): Tune model for FEC Protection. // TODO(marpan): Tune model for FEC Protection.
//Better modulation of protection with available bits/frame (or avgTotpackets) using weight factors // Better modulation of protection with available bits/frame
// (or avgTotpackets) using weight factors
// FEC Tables include this effect already, but need to tune model off-line // FEC Tables include this effect already, but need to tune model off-line
float weight1 = 0.5; float weight1 = 0.5;
float weight2 = 0.5; float weight2 = 0.5;
if (avgTotPackets > 4) if (avgTotPackets > 4) {
{
weight1 = 1.0; weight1 = 1.0;
weight2 = 0.; weight2 = 0.;
} }
if (avgTotPackets > 6) if (avgTotPackets > 6) {
{
weight1 = 1.5; weight1 = 1.5;
weight2 = 0.; weight2 = 0.;
} }
@ -336,57 +338,56 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
WebRtc_UWord8 codeRateDelta = 0; WebRtc_UWord8 codeRateDelta = 0;
WebRtc_UWord8 codeRateKey = 0; WebRtc_UWord8 codeRateKey = 0;
// Get index for new table: the FEC protection depends on the (average) available bits/frame
//Get index for new table: the FEC protection depends on the (avergare) available bits/frame
// the range on the rate index corresponds to rates (bps) from 200k to 8000k, for 30fps // the range on the rate index corresponds to rates (bps) from 200k to 8000k, for 30fps
WebRtc_UWord8 rateIndexTable = (WebRtc_UWord8) VCM_MAX(VCM_MIN((bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0); WebRtc_UWord8 rateIndexTable =
(WebRtc_UWord8) VCM_MAX(VCM_MIN((bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0);
// Restrict packet loss range to 50 for now%: current tables defined only up to 50% // Restrict packet loss range to 50 for now%: current tables defined only up to 50%
if (packetLoss >= plossMax) if (packetLoss >= plossMax) {
{
packetLoss = plossMax - 1; packetLoss = plossMax - 1;
} }
WebRtc_UWord16 indexTable = rateIndexTable * plossMax + packetLoss; WebRtc_UWord16 indexTable = rateIndexTable * plossMax + packetLoss;
// check on table index // check on table index
if (indexTable >= maxFecTableSize) if (indexTable >= maxFecTableSize) {
{
assert("FEC table index too large\n"); assert("FEC table index too large\n");
} }
// //
//For Key frame: effectively at a higher rate, so we scale/boost the rate index. // For Key frame: effectively at a higher rate, so we scale/boost the rate
//the boost factor may depend on several factors: ratio of packet number of I to P frames, how much protection placed on P frames, etc. // index. The boost factor may depend on several factors: ratio of packet
// number of I to P frames, how much protection placed on P frames, etc.
// default is 2 // default is 2
const WebRtc_UWord8 packetFrameDelta = (WebRtc_UWord8) (0.5 + parameters->packetsPerFrame); const WebRtc_UWord8 packetFrameDelta = (WebRtc_UWord8) (0.5 + parameters->packetsPerFrame);
const WebRtc_UWord8 packetFrameKey = (WebRtc_UWord8) (0.5 + parameters->packetsPerFrameKey); const WebRtc_UWord8 packetFrameKey = (WebRtc_UWord8) (0.5 + parameters->packetsPerFrameKey);
const WebRtc_UWord8 boostKey = BoostCodeRateKey(packetFrameDelta, packetFrameKey); const WebRtc_UWord8 boostKey = BoostCodeRateKey(packetFrameDelta, packetFrameKey);
rateIndexTable = (WebRtc_UWord8) VCM_MAX(VCM_MIN(1+(boostKey*bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0); rateIndexTable
= (WebRtc_UWord8) VCM_MAX(VCM_MIN(1+(boostKey*bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0);
WebRtc_UWord16 indexTableKey = rateIndexTable * plossMax + packetLoss; WebRtc_UWord16 indexTableKey = rateIndexTable * plossMax + packetLoss;
indexTableKey = VCM_MIN(indexTableKey, maxFecTableSize); indexTableKey = VCM_MIN(indexTableKey, maxFecTableSize);
codeRateDelta = VCMCodeRateXORTable[indexTable]; //protection factor for P fra codeRateDelta = VCMCodeRateXORTable[indexTable]; //protection factor for P frame
codeRateKey = VCMCodeRateXORTable[indexTableKey]; //protection factor for I frame codeRateKey = VCMCodeRateXORTable[indexTableKey]; //protection factor for I frame
//average with minimum protection level given by (average) total number of packets //average with minimum protection level given by (average) total number of packets
if (packetLoss > 0) if (packetLoss > 0) {
{ codeRateDelta = static_cast<WebRtc_UWord8> ((weight1 * (float) codeRateDelta + weight2 * 255.0
codeRateDelta = static_cast<WebRtc_UWord8>((weight1*(float)codeRateDelta + weight2*255.0/(float)avgTotPackets)); / (float) avgTotPackets));
} }
//check limit on amount of protection for P frame; 50% is max //check limit on amount of protection for P frame; 50% is max
if (codeRateDelta >= plossMax) if (codeRateDelta >= plossMax) {
{
codeRateDelta = plossMax - 1; codeRateDelta = plossMax - 1;
} }
//make sure I frame protection is at least larger than P frame protection, and at least as high as received loss //make sure I frame protection is at least larger than P frame protection, and at least as high as received loss
codeRateKey = static_cast<WebRtc_UWord8>(VCM_MAX(packetLoss,VCM_MAX(_scaleProtKey*codeRateDelta, codeRateKey))); codeRateKey
= static_cast<WebRtc_UWord8> (VCM_MAX(packetLoss,VCM_MAX(_scaleProtKey*codeRateDelta, codeRateKey)));
//check limit on amount of protection for I frame: 50% is max //check limit on amount of protection for I frame: 50% is max
if (codeRateKey >= plossMax) if (codeRateKey >= plossMax) {
{
codeRateKey = plossMax - 1; codeRateKey = plossMax - 1;
} }
@ -394,16 +395,11 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
_protectionFactorD = codeRateDelta; _protectionFactorD = codeRateDelta;
// DONE WITH FEC PROTECTION SETTINGS // DONE WITH FEC PROTECTION SETTINGS
return true; return true;
} }
bool VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters)
bool
VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters)
{ {
// ER SETTINGS: // ER SETTINGS:
//Effective packet loss to encoder is based on RPL (residual packet loss) //Effective packet loss to encoder is based on RPL (residual packet loss)
//this is a soft setting based on degree of FEC protection //this is a soft setting based on degree of FEC protection
@ -428,23 +424,20 @@ VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters)
//Residual Packet Loss: //Residual Packet Loss:
_residualPacketLoss = (float) (effPacketLoss - avgFecRecov) / (float) 255.0; _residualPacketLoss = (float) (effPacketLoss - avgFecRecov) / (float) 255.0;
//Effective Packet Loss for encoder: //Effective Packet Loss for encoder:
_effectivePacketLoss = 0; _effectivePacketLoss = 0;
if (effPacketLoss > 0) if (effPacketLoss > 0) {
{ _effectivePacketLoss = VCM_MAX((effPacketLoss -
_effectivePacketLoss = VCM_MAX((effPacketLoss - (WebRtc_UWord8)(scaleEr*avgFecRecov)),static_cast<WebRtc_UWord8>(minErLevel*255)); (WebRtc_UWord8)(scaleEr * avgFecRecov)),
static_cast<WebRtc_UWord8>(minErLevel * 255));
} }
// DONE WITH ER SETTING // DONE WITH ER SETTING
return true; return true;
} }
bool VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
bool
VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
{ {
// Compute the protection factor // Compute the protection factor
@ -453,12 +446,10 @@ VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
// Compute the effective packet loss // Compute the effective packet loss
EffectivePacketLoss(parameters); EffectivePacketLoss(parameters);
// Compute the bit cost // Compute the bit cost
// Ignore key frames for now. // Ignore key frames for now.
float fecRate = static_cast<float> (_protectionFactorD) / 255.0f; float fecRate = static_cast<float> (_protectionFactorD) / 255.0f;
if (fecRate >= 0.0f) if (fecRate >= 0.0f) {
{
// use this formula if the fecRate (protection factor) is defined relative to number of source packets // use this formula if the fecRate (protection factor) is defined relative to number of source packets
// this is the case for the previous tables: // this is the case for the previous tables:
// _efficiency = parameters->bitRate * ( 1.0 - 1.0 / (1.0 + fecRate)); // _efficiency = parameters->bitRate * ( 1.0 - 1.0 / (1.0 + fecRate));
@ -466,14 +457,11 @@ VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
// in the new tables, the fecRate is defined relative to total number of packets (total rate), // in the new tables, the fecRate is defined relative to total number of packets (total rate),
// so overhead cost is: // so overhead cost is:
_efficiency = parameters->bitRate * fecRate; _efficiency = parameters->bitRate * fecRate;
} } else {
else
{
_efficiency = 0.0f; _efficiency = 0.0f;
} }
_score = _efficiency; _score = _efficiency;
// Protection/fec rates obtained above is defined relative to total number of packets (total rate: source+fec) // Protection/fec rates obtained above is defined relative to total number of packets (total rate: source+fec)
// FEC in RTP module assumes protection factor is defined relative to source number of packets // FEC in RTP module assumes protection factor is defined relative to source number of packets
// so we should convert the factor to reduce mismatch between mediaOpt suggested rate and the actual rate // so we should convert the factor to reduce mismatch between mediaOpt suggested rate and the actual rate
@ -483,28 +471,24 @@ VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters)
return true; return true;
} }
bool bool VCMIntraReqMethod::UpdateParameters(const VCMProtectionParameters* parameters)
VCMIntraReqMethod::UpdateParameters(const VCMProtectionParameters* parameters)
{ {
float packetRate = parameters->packetsPerFrame * parameters->frameRate; float packetRate = parameters->packetsPerFrame * parameters->frameRate;
// Assume that all lost packets cohere to different frames // Assume that all lost packets cohere to different frames
float lossRate = parameters->lossPr * packetRate; float lossRate = parameters->lossPr * packetRate;
if (parameters->keyFrameSize <= 1e-3) if (parameters->keyFrameSize <= 1e-3) {
{
_score = FLT_MAX; _score = FLT_MAX;
return false; return false;
} }
_efficiency = lossRate * parameters->keyFrameSize; _efficiency = lossRate * parameters->keyFrameSize;
_score = _efficiency; _score = _efficiency;
if (parameters->lossPr >= 1.0f / parameters->keyFrameSize || parameters->rtt > _IREQ_MAX_RTT) if (parameters->lossPr >= 1.0f / parameters->keyFrameSize || parameters->rtt > _IREQ_MAX_RTT) {
{
return false; return false;
} }
return true; return true;
} }
bool bool VCMPeriodicIntraMethod::UpdateParameters(const VCMProtectionParameters* /*parameters*/)
VCMPeriodicIntraMethod::UpdateParameters(const VCMProtectionParameters* /*parameters*/)
{ {
// Periodic I-frames. The last thing we want to use. // Periodic I-frames. The last thing we want to use.
_efficiency = 0.0f; _efficiency = 0.0f;
@ -512,21 +496,18 @@ VCMPeriodicIntraMethod::UpdateParameters(const VCMProtectionParameters* /*parame
return true; return true;
} }
bool bool VCMMbIntraRefreshMethod::UpdateParameters(const VCMProtectionParameters* parameters)
VCMMbIntraRefreshMethod::UpdateParameters(const VCMProtectionParameters* parameters)
{ {
// Assume optimal for now. // Assume optimal for now.
_efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr); _efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr);
_score = _efficiency; _score = _efficiency;
if (parameters->bitRate < _MBREF_MIN_BITRATE) if (parameters->bitRate < _MBREF_MIN_BITRATE) {
{
return false; return false;
} }
return true; return true;
} }
WebRtc_UWord16 WebRtc_UWord16 VCMNackMethod::MaxRttNack() const
VCMNackMethod::MaxRttNack() const
{ {
return _NACK_MAX_RTT; return _NACK_MAX_RTT;
} }
@ -536,15 +517,12 @@ VCMLossProtectionLogic::~VCMLossProtectionLogic()
ClearLossProtections(); ClearLossProtections();
} }
void void VCMLossProtectionLogic::ClearLossProtections()
VCMLossProtectionLogic::ClearLossProtections()
{ {
ListItem *item; ListItem *item;
while ((item = _availableMethods.First()) != 0) while ((item = _availableMethods.First()) != 0) {
{
VCMProtectionMethod *method = static_cast<VCMProtectionMethod*> (item->GetItem()); VCMProtectionMethod *method = static_cast<VCMProtectionMethod*> (item->GetItem());
if (method != NULL) if (method != NULL) {
{
delete method; delete method;
} }
_availableMethods.PopFront(); _availableMethods.PopFront();
@ -552,20 +530,16 @@ VCMLossProtectionLogic::ClearLossProtections()
_selectedMethod = NULL; _selectedMethod = NULL;
} }
bool bool VCMLossProtectionLogic::AddMethod(VCMProtectionMethod *newMethod)
VCMLossProtectionLogic::AddMethod(VCMProtectionMethod *newMethod)
{ {
VCMProtectionMethod *method; VCMProtectionMethod *method;
ListItem *item; ListItem *item;
if (newMethod == NULL) if (newMethod == NULL) {
{
return false; return false;
} }
for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) {
{
method = static_cast<VCMProtectionMethod *> (item->GetItem()); method = static_cast<VCMProtectionMethod *> (item->GetItem());
if (method != NULL && method->Type() == newMethod->Type()) if (method != NULL && method->Type() == newMethod->Type()) {
{
return false; return false;
} }
} }
@ -573,19 +547,15 @@ VCMLossProtectionLogic::AddMethod(VCMProtectionMethod *newMethod)
return true; return true;
} }
bool bool VCMLossProtectionLogic::RemoveMethod(VCMProtectionMethodEnum methodType)
VCMLossProtectionLogic::RemoveMethod(VCMProtectionMethodEnum methodType)
{ {
VCMProtectionMethod *method; VCMProtectionMethod *method;
ListItem *item; ListItem *item;
bool foundAndRemoved = false; bool foundAndRemoved = false;
for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) {
{
method = static_cast<VCMProtectionMethod *> (item->GetItem()); method = static_cast<VCMProtectionMethod *> (item->GetItem());
if (method != NULL && method->Type() == methodType) if (method != NULL && method->Type() == methodType) {
{ if (_selectedMethod != NULL && _selectedMethod->Type() == method->Type()) {
if (_selectedMethod != NULL && _selectedMethod->Type() == method->Type())
{
_selectedMethod = NULL; _selectedMethod = NULL;
} }
_availableMethods.Erase(item); _availableMethods.Erase(item);
@ -602,54 +572,44 @@ VCMLossProtectionLogic::FindMethod(VCMProtectionMethodEnum methodType) const
{ {
VCMProtectionMethod *method; VCMProtectionMethod *method;
ListItem *item; ListItem *item;
for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) {
{
method = static_cast<VCMProtectionMethod *> (item->GetItem()); method = static_cast<VCMProtectionMethod *> (item->GetItem());
if (method != NULL && method->Type() == methodType) if (method != NULL && method->Type() == methodType) {
{
return method; return method;
} }
} }
return NULL; return NULL;
} }
float float VCMLossProtectionLogic::HighestOverhead() const
VCMLossProtectionLogic::HighestOverhead() const
{ {
VCMProtectionMethod *method; VCMProtectionMethod *method;
ListItem *item; ListItem *item;
float highestOverhead = 0.0f; float highestOverhead = 0.0f;
for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) {
{
method = static_cast<VCMProtectionMethod *> (item->GetItem()); method = static_cast<VCMProtectionMethod *> (item->GetItem());
if (method != NULL && method->RequiredBitRate() > highestOverhead) if (method != NULL && method->RequiredBitRate() > highestOverhead) {
{
highestOverhead = method->RequiredBitRate(); highestOverhead = method->RequiredBitRate();
} }
} }
return highestOverhead; return highestOverhead;
} }
void void VCMLossProtectionLogic::UpdateRtt(WebRtc_UWord32 rtt)
VCMLossProtectionLogic::UpdateRtt(WebRtc_UWord32 rtt)
{ {
_rtt = rtt; _rtt = rtt;
} }
void void VCMLossProtectionLogic::UpdateResidualPacketLoss(float residualPacketLoss) {
VCMLossProtectionLogic::UpdateResidualPacketLoss(float residualPacketLoss)
{
_residualPacketLoss = residualPacketLoss; _residualPacketLoss = residualPacketLoss;
} }
void void VCMLossProtectionLogic::UpdateFecType(VCMFecTypes fecType)
VCMLossProtectionLogic::UpdateFecType(VCMFecTypes fecType)
{ {
_fecType = fecType; _fecType = fecType;
} }
void void VCMLossProtectionLogic::UpdateLossPr(WebRtc_UWord8 lossPr255)
VCMLossProtectionLogic::UpdateLossPr(WebRtc_UWord8 lossPr255)
{ {
WebRtc_UWord32 now = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp()); WebRtc_UWord32 now = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp());
UpdateMaxLossHistory(lossPr255, now); UpdateMaxLossHistory(lossPr255, now);
@ -658,36 +618,25 @@ VCMLossProtectionLogic::UpdateLossPr(WebRtc_UWord8 lossPr255)
_lossPr = _lossPr255.Value() / 255.0f; _lossPr = _lossPr255.Value() / 255.0f;
} }
void void VCMLossProtectionLogic::UpdateMaxLossHistory(WebRtc_UWord8 lossPr255, WebRtc_Word64 now)
VCMLossProtectionLogic::UpdateMaxLossHistory(WebRtc_UWord8 lossPr255, WebRtc_Word64 now)
{
if (_lossPrHistory[0].timeMs >= 0 &&
now - _lossPrHistory[0].timeMs < kLossPrShortFilterWinMs)
{
if (lossPr255 > _shortMaxLossPr255)
{ {
if (_lossPrHistory[0].timeMs >= 0 && now - _lossPrHistory[0].timeMs < kLossPrShortFilterWinMs) {
if (lossPr255 > _shortMaxLossPr255) {
_shortMaxLossPr255 = lossPr255; _shortMaxLossPr255 = lossPr255;
} }
} } else {
else
{
// Only add a new value to the history once a second // Only add a new value to the history once a second
if(_lossPrHistory[0].timeMs == -1) if (_lossPrHistory[0].timeMs == -1) {
{
// First, no shift // First, no shift
_shortMaxLossPr255 = lossPr255; _shortMaxLossPr255 = lossPr255;
} } else {
else
{
// Shift // Shift
for(WebRtc_Word32 i = (kLossPrHistorySize - 2); i >= 0 ; i--) for (WebRtc_Word32 i = (kLossPrHistorySize - 2); i >= 0; i--) {
{
_lossPrHistory[i + 1].lossPr255 = _lossPrHistory[i].lossPr255; _lossPrHistory[i + 1].lossPr255 = _lossPrHistory[i].lossPr255;
_lossPrHistory[i + 1].timeMs = _lossPrHistory[i].timeMs; _lossPrHistory[i + 1].timeMs = _lossPrHistory[i].timeMs;
} }
} }
if (_shortMaxLossPr255 == 0) if (_shortMaxLossPr255 == 0) {
{
_shortMaxLossPr255 = lossPr255; _shortMaxLossPr255 = lossPr255;
} }
@ -698,27 +647,21 @@ VCMLossProtectionLogic::UpdateMaxLossHistory(WebRtc_UWord8 lossPr255, WebRtc_Wor
} }
} }
WebRtc_UWord8 WebRtc_UWord8 VCMLossProtectionLogic::MaxFilteredLossPr(WebRtc_Word64 nowMs) const
VCMLossProtectionLogic::MaxFilteredLossPr(WebRtc_Word64 nowMs) const
{ {
WebRtc_UWord8 maxFound = _shortMaxLossPr255; WebRtc_UWord8 maxFound = _shortMaxLossPr255;
if (_lossPrHistory[0].timeMs == -1) if (_lossPrHistory[0].timeMs == -1) {
{
return maxFound; return maxFound;
} }
for (WebRtc_Word32 i=0; i < kLossPrHistorySize; i++) for (WebRtc_Word32 i = 0; i < kLossPrHistorySize; i++) {
{ if (_lossPrHistory[i].timeMs == -1) {
if (_lossPrHistory[i].timeMs == -1)
{
break; break;
} }
if (nowMs - _lossPrHistory[i].timeMs > kLossPrHistorySize * kLossPrShortFilterWinMs) if (nowMs - _lossPrHistory[i].timeMs > kLossPrHistorySize * kLossPrShortFilterWinMs) {
{
// This sample (and all samples after this) is too old // This sample (and all samples after this) is too old
break; break;
} }
if (_lossPrHistory[i].lossPr255 > maxFound) if (_lossPrHistory[i].lossPr255 > maxFound) {
{
// This sample is the largest one this far into the history // This sample is the largest one this far into the history
maxFound = _lossPrHistory[i].lossPr255; maxFound = _lossPrHistory[i].lossPr255;
} }
@ -726,61 +669,52 @@ VCMLossProtectionLogic::MaxFilteredLossPr(WebRtc_Word64 nowMs) const
return maxFound; return maxFound;
} }
WebRtc_UWord8 WebRtc_UWord8 VCMLossProtectionLogic::FilteredLoss() const
VCMLossProtectionLogic::FilteredLoss() const
{ {
//take the average received loss //take the average received loss
//return static_cast<WebRtc_UWord8>(_lossPr255.Value() + 0.5f); //return static_cast<WebRtc_UWord8>(_lossPr255.Value() + 0.5f);
//TODO: Update for hybrid
//take the windowed max of the received loss //take the windowed max of the received loss
if (_selectedMethod != NULL && _selectedMethod->Type() == kFEC) if (_selectedMethod != NULL && _selectedMethod->Type() == kFEC) {
{
return MaxFilteredLossPr(static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp())); return MaxFilteredLossPr(static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp()));
} } else {
else
{
return static_cast<WebRtc_UWord8> (_lossPr255.Value() + 0.5); return static_cast<WebRtc_UWord8> (_lossPr255.Value() + 0.5);
} }
} }
void void VCMLossProtectionLogic::UpdateFilteredLossPr(WebRtc_UWord8 packetLossEnc)
VCMLossProtectionLogic::UpdateFilteredLossPr(WebRtc_UWord8 packetLossEnc)
{ {
_lossPr = (float) packetLossEnc / (float) 255.0; _lossPr = (float) packetLossEnc / (float) 255.0;
} }
void void VCMLossProtectionLogic::UpdateBitRate(float bitRate)
VCMLossProtectionLogic::UpdateBitRate(float bitRate)
{ {
_bitRate = bitRate; _bitRate = bitRate;
} }
void void VCMLossProtectionLogic::UpdatePacketsPerFrame(float nPackets)
VCMLossProtectionLogic::UpdatePacketsPerFrame(float nPackets)
{ {
WebRtc_UWord32 now = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp()); WebRtc_UWord32 now = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp());
_packetsPerFrame.Apply(static_cast<float> (now - _lastPacketPerFrameUpdateT), nPackets); _packetsPerFrame.Apply(static_cast<float> (now - _lastPacketPerFrameUpdateT), nPackets);
_lastPacketPerFrameUpdateT = now; _lastPacketPerFrameUpdateT = now;
} }
void void VCMLossProtectionLogic::UpdatePacketsPerFrameKey(float nPackets)
VCMLossProtectionLogic::UpdatePacketsPerFrameKey(float nPackets)
{ {
WebRtc_UWord32 now = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp()); WebRtc_UWord32 now = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp());
_packetsPerFrameKey.Apply(static_cast<float> (now - _lastPacketPerFrameUpdateTKey), nPackets); _packetsPerFrameKey.Apply(static_cast<float> (now - _lastPacketPerFrameUpdateTKey), nPackets);
_lastPacketPerFrameUpdateTKey = now; _lastPacketPerFrameUpdateTKey = now;
} }
void void VCMLossProtectionLogic::UpdateKeyFrameSize(float keyFrameSize)
VCMLossProtectionLogic::UpdateKeyFrameSize(float keyFrameSize)
{ {
_keyFrameSize = keyFrameSize; _keyFrameSize = keyFrameSize;
} }
bool bool VCMLossProtectionLogic::UpdateMethod(VCMProtectionMethod *newMethod /*=NULL */)
VCMLossProtectionLogic::UpdateMethod(VCMProtectionMethod *newMethod /*=NULL */)
{ {
_currentParameters.rtt = _rtt; _currentParameters.rtt = _rtt;
_currentParameters.lossPr = _lossPr; _currentParameters.lossPr = _lossPr;
@ -794,30 +728,29 @@ VCMLossProtectionLogic::UpdateMethod(VCMProtectionMethod *newMethod /*=NULL */)
_currentParameters.residualPacketLoss = _residualPacketLoss; _currentParameters.residualPacketLoss = _residualPacketLoss;
_currentParameters.fecType = _fecType; _currentParameters.fecType = _fecType;
if (newMethod == NULL) if (newMethod == NULL) {
{
//_selectedMethod = _bestNotOkMethod = NULL; //_selectedMethod = _bestNotOkMethod = NULL;
VCMProtectionMethod *method; VCMProtectionMethod *method;
ListItem *item; ListItem *item;
for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) {
{
method = static_cast<VCMProtectionMethod *> (item->GetItem()); method = static_cast<VCMProtectionMethod *> (item->GetItem());
if (method != NULL) if (method != NULL) {
{ if (method->Type() == kFEC) {
if (method->Type() == kFEC) _selectedMethod = method;
{ }
if (method->Type() == kNACK) {
_selectedMethod = method;
}
if (method->Type() == kNackFec) {
_selectedMethod = method; _selectedMethod = method;
} }
method->UpdateParameters(&_currentParameters); method->UpdateParameters(&_currentParameters);
} }
} }
if (_selectedMethod != NULL && _selectedMethod->Type() != kFEC) if (_selectedMethod != NULL && _selectedMethod->Type() != kFEC) {
{
_selectedMethod = method; _selectedMethod = method;
} }
} } else {
else
{
_selectedMethod = newMethod; _selectedMethod = newMethod;
_selectedMethod->UpdateParameters(&_currentParameters); _selectedMethod->UpdateParameters(&_currentParameters);
} }
@ -830,16 +763,14 @@ VCMLossProtectionLogic::SelectedMethod() const
return _selectedMethod; return _selectedMethod;
} }
void void VCMLossProtectionLogic::Reset()
VCMLossProtectionLogic::Reset()
{ {
_lastPrUpdateT = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp()); _lastPrUpdateT = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp());
_lastPacketPerFrameUpdateT = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp()); _lastPacketPerFrameUpdateT = static_cast<WebRtc_UWord32> (VCMTickTime::MillisecondTimestamp());
_lossPr255.Reset(0.9999f); _lossPr255.Reset(0.9999f);
_packetsPerFrame.Reset(0.9999f); _packetsPerFrame.Reset(0.9999f);
_fecRateDelta = _fecRateKey = 0; _fecRateDelta = _fecRateKey = 0;
for (WebRtc_Word32 i=0; i < kLossPrHistorySize; i++) for (WebRtc_Word32 i = 0; i < kLossPrHistorySize; i++) {
{
_lossPrHistory[i].lossPr255 = 0; _lossPrHistory[i].lossPr255 = 0;
_lossPrHistory[i].timeMs = -1; _lossPrHistory[i].timeMs = -1;
} }

7
modules/video_coding/main/source/media_opt_util.h
View File

@ -34,6 +34,13 @@ enum VCMFecTypes
kXORFec kXORFec
}; };
// Thresholds for hybrid NACK/FEC
// common to media optimization and the jitter buffer.
enum HybridNackTH {
kHighRttNackMs = 100,
kLowRttNackMs = 20
};
struct VCMProtectionParameters struct VCMProtectionParameters
{ {
VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), packetsPerFrame(0), VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), packetsPerFrame(0),

51
modules/video_coding/main/source/media_optimization.cc
View File

@ -109,13 +109,16 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
actualFrameRate = 1.0; actualFrameRate = 1.0;
} }
// Update frame rate for the loss protection logic class: frame rate should be the actual/sent rate // Update frame rate for the loss protection logic class: frame rate should
// be the actual/sent rate
_lossProtLogic->UpdateFrameRate(actualFrameRate); _lossProtLogic->UpdateFrameRate(actualFrameRate);
_fractionLost = fractionLost; _fractionLost = fractionLost;
// The effective packet loss may be the received loss or filtered, i.e., average or max filter may be used. // The effective packet loss may be the received loss or filtered, i.e.,
//We should think about which filter is appropriate for low/high bit rates, low/high loss rates, etc. // average or max filter may be used.
// We should think about which filter is appropriate for low/high bit rates,
// low/high loss rates, etc.
WebRtc_UWord8 packetLossEnc = _lossProtLogic->FilteredLoss(); WebRtc_UWord8 packetLossEnc = _lossProtLogic->FilteredLoss();
//For now use the filtered loss for computing the robustness settings //For now use the filtered loss for computing the robustness settings
@ -124,12 +127,15 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
// Rate cost of the protection methods // Rate cost of the protection methods
_lossProtOverhead = 0; _lossProtOverhead = 0;
if(selectedMethod) if (selectedMethod && (selectedMethod->Type() == kFEC ||
selectedMethod->Type() == kNackFec ))
{ {
//Update method will compute the robustness settings for the given protection method and the overhead cost // Update method will compute the robustness settings for the given
// protection method and the overhead cost
// the protection method is set by the user via SetVideoProtection. // the protection method is set by the user via SetVideoProtection.
//The robustness settings are: the effecitve packet loss for ER and the FEC protection settings // The robustness settings are: the effective packet loss for ER and the
// FEC protection settings
_lossProtLogic->UpdateMethod(); _lossProtLogic->UpdateMethod();
// Get the code rate for Key frames // Get the code rate for Key frames
@ -141,22 +147,20 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
// Get the effective packet loss for ER // Get the effective packet loss for ER
packetLossEnc = selectedMethod->RequiredPacketLossER(); packetLossEnc = selectedMethod->RequiredPacketLossER();
// Get the bit cost of protection method
_lossProtOverhead = static_cast<WebRtc_UWord32>(_lossProtLogic->HighestOverhead() + 0.5f);
// NACK is on for NACK and NackFec protection method: off for FEC method // NACK is on for NACK and NackFec protection method: off for FEC method
bool nackStatus = true; bool nackStatus = (selectedMethod->Type() == kNackFec ||
if (selectedMethod->Type() == kFEC) selectedMethod->Type() == kNACK);
{
nackStatus = false;
}
if(_videoProtectionCallback) if(_videoProtectionCallback)
{ {
_videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP ,codeRateKeyRTP, nackStatus); _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP,
codeRateKeyRTP,
nackStatus);
}
} }
} // Get the bit cost of protection method
_lossProtOverhead = static_cast<WebRtc_UWord32>(_lossProtLogic->HighestOverhead() + 0.5f);
// Update effective packet loss for encoder: note: fractionLost was passed as reference // Update effective packet loss for encoder: note: fractionLost was passed as reference
fractionLost = packetLossEnc; fractionLost = packetLossEnc;
@ -164,6 +168,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
WebRtc_UWord32 nackBitRate=0; WebRtc_UWord32 nackBitRate=0;
if(selectedMethod && _lossProtLogic->FindMethod(kNACK) != NULL) if(selectedMethod && _lossProtLogic->FindMethod(kNACK) != NULL)
{ {
// TODO(mikhal): update frame dropper with bit rate including both nack and fec
// Make sure we don't over-use the channel momentarily. This is // Make sure we don't over-use the channel momentarily. This is
// necessary for NACK since it can be very bursty. // necessary for NACK since it can be very bursty.
nackBitRate = (_lastBitRate * fractionLost) / 255; nackBitRate = (_lastBitRate * fractionLost) / 255;
@ -178,7 +183,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
_frameDropper->SetRates(static_cast<float>(bitRate - _lossProtOverhead), 0); _frameDropper->SetRates(static_cast<float>(bitRate - _lossProtOverhead), 0);
} }
//This may be used for UpdateEncoderBitRate: lastBitRate is total rate, before compensation // This may be used for UpdateEncoderBitRate: lastBitRate is total rate,
// before compensation
_lastBitRate = _targetBitRate; _lastBitRate = _targetBitRate;
//Source coding rate: total rate - protection overhead //Source coding rate: total rate - protection overhead
@ -203,7 +209,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
bool bool
VCMMediaOptimization::DropFrame() VCMMediaOptimization::DropFrame()
{ {
_frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f)); // leak appropriate number of bytes // leak appropriate number of bytes
_frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f));
return _frameDropper->DropFrame(); return _frameDropper->DropFrame();
} }
@ -285,7 +292,7 @@ VCMMediaOptimization::EnableNack(bool enable)
bool bool
VCMMediaOptimization::IsNackEnabled() VCMMediaOptimization::IsNackEnabled()
{ {
return (_lossProtLogic->FindMethod(kFEC) != NULL); return (_lossProtLogic->FindMethod(kNACK) != NULL);
} }
void void
@ -558,8 +565,10 @@ VCMMediaOptimization::checkStatusForQMchange()
bool status = true; bool status = true;
// Check that we do not call QMSelect too often, and that we waited some time (to sample the metrics) from the event lastChangeTime // Check that we do not call QMSelect too often, and that we waited some time
// lastChangeTime is the time where user changed the size/rate/frame rate (via SetEncodingData) // (to sample the metrics) from the event lastChangeTime
// lastChangeTime is the time where user changed the size/rate/frame rate
// (via SetEncodingData)
WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp();
if ((now - _lastQMUpdateTime) < kQmMinIntervalMs || if ((now - _lastQMUpdateTime) < kQmMinIntervalMs ||
(now - _lastChangeTime) < kQmMinIntervalMs) (now - _lastChangeTime) < kQmMinIntervalMs)

32
modules/video_coding/main/source/receiver.cc
View File

@ -31,7 +31,6 @@ _master(master),
_jitterBuffer(vcmId, receiverId, master), _jitterBuffer(vcmId, receiverId, master),
_timing(timing), _timing(timing),
_renderWaitEvent(*new VCMEvent()), _renderWaitEvent(*new VCMEvent()),
_nackMode(kNoNack),
_state(kPassive) _state(kPassive)
{ {
} }
@ -164,6 +163,7 @@ VCMReceiver::InsertPacket(const VCMPacket& packet,
} }
// Insert packet into jitter buffer // Insert packet into jitter buffer
// both data and empty packets
const VCMFrameBufferEnum ret = _jitterBuffer.InsertPacket(buffer, packet); const VCMFrameBufferEnum ret = _jitterBuffer.InsertPacket(buffer, packet);
if (ret < 0) if (ret < 0)
@ -178,7 +178,8 @@ VCMReceiver::InsertPacket(const VCMPacket& packet,
} }
VCMEncodedFrame* VCMEncodedFrame*
VCMReceiver::FrameForDecoding(WebRtc_UWord16 maxWaitTimeMs, WebRtc_Word64& nextRenderTimeMs, bool renderTiming, VCMReceiver* dualReceiver) VCMReceiver::FrameForDecoding(WebRtc_UWord16 maxWaitTimeMs, WebRtc_Word64& nextRenderTimeMs,
bool renderTiming, VCMReceiver* dualReceiver)
{ {
// No need to enter the critical section here since the jitter buffer // No need to enter the critical section here since the jitter buffer
// is thread-safe. // is thread-safe.
@ -348,20 +349,7 @@ void
VCMReceiver::SetNackMode(VCMNackMode nackMode) VCMReceiver::SetNackMode(VCMNackMode nackMode)
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
_nackMode = nackMode; _jitterBuffer.SetNackMode(nackMode);
switch (_nackMode)
{
case kNackInfinite:
{
_jitterBuffer.SetNackStatus(true);
break;
}
case kNoNack:
{
_jitterBuffer.SetNackStatus(false);
break;
}
}
if (!_master) if (!_master)
{ {
_state = kPassive; // The dual decoder defaults to passive _state = kPassive; // The dual decoder defaults to passive
@ -372,7 +360,7 @@ VCMNackMode
VCMReceiver::NackMode() const VCMReceiver::NackMode() const
{ {
CriticalSectionScoped cs(_critSect); CriticalSectionScoped cs(_critSect);
return _nackMode; return _jitterBuffer.GetNackMode();
} }
VCMNackStatus VCMNackStatus
@ -418,14 +406,6 @@ void
VCMReceiver::CopyJitterBufferStateFromReceiver(const VCMReceiver& receiver) VCMReceiver::CopyJitterBufferStateFromReceiver(const VCMReceiver& receiver)
{ {
_jitterBuffer = receiver._jitterBuffer; _jitterBuffer = receiver._jitterBuffer;
{
CriticalSectionScoped cs(_critSect);
if (_nackMode != kNoNack)
{
_jitterBuffer.SetNackStatus(true);
}
}
} }
VCMReceiverState VCMReceiverState
@ -447,7 +427,7 @@ VCMReceiver::UpdateState(VCMReceiverState newState)
void void
VCMReceiver::UpdateState(VCMEncodedFrame& frame) VCMReceiver::UpdateState(VCMEncodedFrame& frame)
{ {
if (_nackMode == kNoNack) if (_jitterBuffer.GetNackMode() == kNoNack)
{ {
// Dual decoder mode has not been enabled. // Dual decoder mode has not been enabled.
return; return;

6
modules/video_coding/main/source/receiver.h
View File

@ -28,11 +28,6 @@ enum VCMNackStatus
kNackKeyFrameRequest kNackKeyFrameRequest
}; };
enum VCMNackMode
{
kNackInfinite,
kNoNack
};
enum VCMReceiverState enum VCMReceiverState
{ {
@ -91,7 +86,6 @@ private:
VCMJitterBuffer _jitterBuffer; VCMJitterBuffer _jitterBuffer;
VCMTiming& _timing; VCMTiming& _timing;
VCMEvent& _renderWaitEvent; VCMEvent& _renderWaitEvent;
VCMNackMode _nackMode;
VCMReceiverState _state; VCMReceiverState _state;
static WebRtc_Word32 _receiverIdCounter; static WebRtc_Word32 _receiverIdCounter;

233
modules/video_coding/main/source/session_info.cc
View File

@ -25,7 +25,9 @@ VCMSessionInfo::VCMSessionInfo():
_previousFrameLoss(false), _previousFrameLoss(false),
_lowSeqNum(-1), _lowSeqNum(-1),
_highSeqNum(-1), _highSeqNum(-1),
_highestPacketIndex(0) _highestPacketIndex(0),
_emptySeqNumLow(-1),
_emptySeqNumHigh(-1)
{ {
memset(_packetSizeBytes, 0, sizeof(_packetSizeBytes)); memset(_packetSizeBytes, 0, sizeof(_packetSizeBytes));
memset(_naluCompleteness, kNaluUnset, sizeof(_naluCompleteness)); memset(_naluCompleteness, kNaluUnset, sizeof(_naluCompleteness));
@ -50,6 +52,8 @@ void VCMSessionInfo::Reset()
{ {
_lowSeqNum = -1; _lowSeqNum = -1;
_highSeqNum = -1; _highSeqNum = -1;
_emptySeqNumLow = -1;
_emptySeqNumHigh = -1;
_markerBit = false; _markerBit = false;
_haveFirstPacket = false; _haveFirstPacket = false;
_completeSession = false; _completeSession = false;
@ -89,7 +93,9 @@ VCMSessionInfo::HaveStartSeqNumber()
return true; return true;
} }
WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebRtc_Word32 packetIndex, const VCMPacket& packet) WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer,
WebRtc_Word32 packetIndex,
const VCMPacket& packet)
{ {
WebRtc_UWord32 moveLength = 0; WebRtc_UWord32 moveLength = 0;
WebRtc_UWord32 returnLength = 0; WebRtc_UWord32 returnLength = 0;
@ -99,14 +105,16 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR
WebRtc_UWord32 offset = 0; WebRtc_UWord32 offset = 0;
WebRtc_UWord32 packetSize = 0; WebRtc_UWord32 packetSize = 0;
// Store this packet length. Add length since we could have data present already (e.g. multicall case). // Store this packet length. Add length since we could have data present
// already (e.g. multicall case).
if (packet.bits) if (packet.bits)
{ {
packetSize = packet.sizeBytes; packetSize = packet.sizeBytes;
} }
else else
{ {
packetSize = packet.sizeBytes + (packet.insertStartCode?kH264StartCodeLengthBytes:0); packetSize = packet.sizeBytes +
(packet.insertStartCode?kH264StartCodeLengthBytes:0);
} }
_packetSizeBytes[packetIndex] += packetSize; _packetSizeBytes[packetIndex] += packetSize;
@ -122,7 +130,8 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR
} }
if (moveLength > 0) if (moveLength > 0)
{ {
memmove((void*)(ptrStartOfLayer + offset + packetSize), ptrStartOfLayer + offset, moveLength); memmove((void*)(ptrStartOfLayer + offset + packetSize),
ptrStartOfLayer + offset, moveLength);
} }
if (packet.bits) if (packet.bits)
@ -145,7 +154,8 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR
const unsigned char startCode[] = {0, 0, 0, 1}; const unsigned char startCode[] = {0, 0, 0, 1};
if(packet.insertStartCode) if(packet.insertStartCode)
{ {
memcpy((void*)(ptrStartOfLayer + offset), startCode, kH264StartCodeLengthBytes); memcpy((void*)(ptrStartOfLayer + offset), startCode,
kH264StartCodeLengthBytes);
} }
memcpy((void*)(ptrStartOfLayer + offset memcpy((void*)(ptrStartOfLayer + offset
+ (packet.insertStartCode?kH264StartCodeLengthBytes:0)), + (packet.insertStartCode?kH264StartCodeLengthBytes:0)),
@ -158,6 +168,9 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR
if (packet.isFirstPacket) if (packet.isFirstPacket)
{ {
_haveFirstPacket = true; _haveFirstPacket = true;
//initializing FEC sequence numbers
_emptySeqNumHigh = -1;
_emptySeqNumLow = -1;
} }
if (packet.markerBit) if (packet.markerBit)
{ {
@ -194,10 +207,11 @@ bool VCMSessionInfo::IsSessionComplete()
return _completeSession; return _completeSession;
} }
// Find the start and end index of packetIndex packet. // Find the start and end index of packetIndex packet.
// startIndex -1 if start not found endIndex=-1 if end index not found // startIndex -1 if start not found endIndex=-1 if end index not found
void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& startIndex, WebRtc_Word32& endIndex) void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,
WebRtc_Word32& startIndex,
WebRtc_Word32& endIndex)
{ {
if(_naluCompleteness[packetIndex]==kNaluStart || if(_naluCompleteness[packetIndex]==kNaluStart ||
@ -210,12 +224,16 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& sta
for(startIndex = packetIndex - 1; startIndex >= 0; --startIndex) for(startIndex = packetIndex - 1; startIndex >= 0; --startIndex)
{ {
if( (_naluCompleteness[startIndex]==kNaluComplete && _packetSizeBytes[startIndex]>0) ||(_naluCompleteness[startIndex]==kNaluEnd && startIndex>0)) // Found previous NALU. if( (_naluCompleteness[startIndex] == kNaluComplete &&
_packetSizeBytes[startIndex] > 0) ||
// Found previous NALU.
(_naluCompleteness[startIndex] == kNaluEnd && startIndex>0))
{ {
startIndex++; startIndex++;
break; break;
} }
if( _naluCompleteness[startIndex]==kNaluStart) // This is where the NALU start. // This is where the NALU start.
if( _naluCompleteness[startIndex] == kNaluStart)
{ {
break; break;
} }
@ -232,7 +250,9 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& sta
// Find the next NALU // Find the next NALU
for(endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex) for(endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex)
{ {
if((_naluCompleteness[endIndex]==kNaluComplete && _packetSizeBytes[endIndex]>0) || _naluCompleteness[endIndex]==kNaluStart) // Found next NALU. if((_naluCompleteness[endIndex]==kNaluComplete &&
_packetSizeBytes[endIndex]>0) ||
_naluCompleteness[endIndex]==kNaluStart) // Found next NALU.
{ {
endIndex--; endIndex--;
break; break;
@ -248,7 +268,9 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& sta
} }
// Deletes all packets between startIndex and endIndex // Deletes all packets between startIndex and endIndex
WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer,WebRtc_Word32 startIndex,WebRtc_Word32 endIndex) WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer,
WebRtc_Word32 startIndex,
WebRtc_Word32 endIndex)
{ {
//Get the number of bytes to delete. //Get the number of bytes to delete.
@ -275,7 +297,8 @@ WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer,WebR
numberOfBytesToMove += _packetSizeBytes[j]; numberOfBytesToMove += _packetSizeBytes[j];
} }
memmove((void*)(ptrStartOfLayer + destOffset),(void*)(ptrStartOfLayer + destOffset+bytesToDelete), numberOfBytesToMove); memmove((void*)(ptrStartOfLayer + destOffset),(void*)(ptrStartOfLayer +
destOffset+bytesToDelete), numberOfBytesToMove);
} }
@ -325,11 +348,12 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay
{ {
endIndex = _highestPacketIndex; endIndex = _highestPacketIndex;
} }
returnLength+=DeletePackets(ptrStartOfLayer,0,endIndex);//Delete this NALU. //Delete this NALU.
returnLength += DeletePackets(ptrStartOfLayer,0,endIndex);
break; break;
case kNaluEnd: // Packet is the end of a NALU case kNaluEnd: // Packet is the end of a NALU
//Need to delete this packet //Delete this NALU
returnLength+=DeletePackets(ptrStartOfLayer,0,0);//Delete this NALU. returnLength += DeletePackets(ptrStartOfLayer,0,0);
break; break;
default: default:
assert(false); assert(false);
@ -339,9 +363,10 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay
return returnLength; return returnLength;
} }
WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num) WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum)
{ {
if ((NULL == list) || (num < 1)) if ((NULL == list) || (numberOfSeqNum < 1))
{ {
return -1; return -1;
} }
@ -353,7 +378,7 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
// Find end point (index of entry that equals _lowSeqNum) // Find end point (index of entry that equals _lowSeqNum)
int index = 0; int index = 0;
for (; index <num; index++) for (; index < numberOfSeqNum; index++)
{ {
if (list[index] == _lowSeqNum) if (list[index] == _lowSeqNum)
{ {
@ -364,7 +389,7 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
// Zero out between first entry and end point // Zero out between first entry and end point
int i = 0; int i = 0;
while ( i <= _highestPacketIndex && index < num) while ( i <= _highestPacketIndex && index < numberOfSeqNum)
{ {
if (_naluCompleteness[i] != kNaluUnset) if (_naluCompleteness[i] != kNaluUnset)
{ {
@ -384,6 +409,105 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n
return 0; return 0;
} }
WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum,
float rttScore)
{
if ((NULL == list) || (numberOfSeqNum < 1))
{
return -1;
}
if (_lowSeqNum == -1)
{
// no packets in this frame
return 0;
}
WebRtc_Word32 index = 0;
// Find end point (index of entry that equals _lowSeqNum)
for (; index < numberOfSeqNum; index++)
{
if (list[index] == _lowSeqNum)
{
list[index] = -1;
break;
}
}
// TODO(mikhal): 1. update score based on RTT value 2. add partition data
// use the previous available
bool isBaseAvailable = false;
if ((index > 0) && (list[index] == -1))
{
// found first packet, for now let's go only one back
if ((list[index - 1] == -1) || (list[index - 1] == -2))
{
// this is indeed the first packet, as previous packet was populated
isBaseAvailable = true;
}
}
bool allowNack = false;
if (!_haveFirstPacket || !isBaseAvailable)
{
allowNack = true;
}
// Zero out between first entry and end point
int i = 0;
// Score place holder - based on RTT and partition (when available).
const float nackScoreTh = 0.25f;
WebRtc_Word32 highMediaPacket = _emptySeqNumLow > _lowSeqNum ?
_emptySeqNumLow - 1: _highSeqNum;
while (list[index] <= highMediaPacket && index < numberOfSeqNum)
{
if (_naluCompleteness[i] != kNaluUnset)
{
list[index] = -1;
}
else
{
// compute score of the packet
float score = 1.0f;
// multiply internal score (importance) by external score (RTT)
score *= rttScore;
if (score > nackScoreTh)
{
allowNack = true;
}
else
{
list[index] = -1;
}
}
i++;
index++;
}
// Empty packets follow the data packets, and therefore have a higher
// sequence number. We do not want to NACK empty packets.
if ((_emptySeqNumLow != -1) && (_emptySeqNumHigh != -1) &&
(index < numberOfSeqNum))
{
// first make sure that we are at least at the minimum value
// (if not we are missing last packet(s))
while (list[index] < _emptySeqNumLow && index < numberOfSeqNum)
{
index++;
}
// mark empty packets
while (list[index] <= _emptySeqNumHigh && index < numberOfSeqNum)
{
list[index] = -2;
index++;
}
}
_sessionNACK = allowNack;
return 0;
}
WebRtc_Word32 VCMSessionInfo::GetHighestPacketIndex() WebRtc_Word32 VCMSessionInfo::GetHighestPacketIndex()
{ {
return _highestPacketIndex; return _highestPacketIndex;
@ -420,7 +544,8 @@ void VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32
void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices) void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices)
{ {
// sanity // sanity
if((numberOfPacketIndices + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer) || numberOfPacketIndices < 0) if((numberOfPacketIndices + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer)
|| numberOfPacketIndices < 0)
{ {
// not allowed // not allowed
assert(!"SessionInfo::PrependPacketIndexes Error: invalid packetIndex"); assert(!"SessionInfo::PrependPacketIndexes Error: invalid packetIndex");
@ -428,7 +553,8 @@ void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices)
} }
// Works if we have new packets before packetIndex = 0 // Works if we have new packets before packetIndex = 0
int numOfPacketsToMove = GetHighestPacketIndex()+1; int numOfPacketsToMove = GetHighestPacketIndex()+1;
memmove(&_packetSizeBytes[numberOfPacketIndices], &_packetSizeBytes[0], (numOfPacketsToMove)*sizeof(WebRtc_UWord16)); memmove(&_packetSizeBytes[numberOfPacketIndices], &_packetSizeBytes[0],
(numOfPacketsToMove)*sizeof(WebRtc_UWord16));
memset(&_packetSizeBytes[0], 0, numberOfPacketIndices*sizeof(WebRtc_UWord16)); memset(&_packetSizeBytes[0], 0, numberOfPacketIndices*sizeof(WebRtc_UWord16));
_highestPacketIndex += (WebRtc_UWord16)numberOfPacketIndices; _highestPacketIndex += (WebRtc_UWord16)numberOfPacketIndices;
@ -464,20 +590,31 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL
//not allowed //not allowed
assert(!packet.insertStartCode || !packet.bits); assert(!packet.insertStartCode || !packet.bits);
if (packet.frameType == kFrameEmpty)
{
// update seq number as an empty packet
// empty packets will be counted twice: both empty and standard packets.
InformOfEmptyPacket(packet.seqNum);
}
// Check if this is first packet (only valid for some codecs) // Check if this is first packet (only valid for some codecs)
if (packet.isFirstPacket) if (packet.isFirstPacket)
{ {
// the first packet in the frame always signals the frametype // the first packet in the frame always signals the frametype
_frameType = packet.frameType; _frameType = packet.frameType;
} }
else if (_frameType == kFrameEmpty && packet.frameType != kFrameEmpty)
{
// in case an empty packet came in first, update the frame type
_frameType = packet.frameType;
}
// Check sequence number and update highest and lowest sequence numbers received. // Check sequence number and update highest and lowest sequence numbers received.
// Move data if this seq num is lower than previously lowest. // Move data if this seq num is lower than previously lowest.
if (packet.seqNum > _highSeqNum) if (packet.seqNum > _highSeqNum)
{ {
// This packet's seq num is higher than previously highest seq num; normal case // This packet's seq num is higher than previously highest seq num;
// if we have a wrap, only update with wrapped values // normal case if we have a wrap, only update with wrapped values
if (!(_highSeqNum < 0x00ff && packet.seqNum > 0xff00)) if (!(_highSeqNum < 0x00ff && packet.seqNum > 0xff00))
{ {
_highSeqNum = packet.seqNum; _highSeqNum = packet.seqNum;
@ -498,7 +635,8 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL
if (_lowSeqNum > 0xff00 && packet.seqNum < 0x00ff) if (_lowSeqNum > 0xff00 && packet.seqNum < 0x00ff)
{ {
// we have a false detect due to the wrap // we have a false detect due to the wrap
packetIndex = (0xffff - (WebRtc_UWord16)_lowSeqNum) + packet.seqNum + (WebRtc_UWord16)1; packetIndex = (0xffff - (WebRtc_UWord16)_lowSeqNum) + packet.seqNum
+ (WebRtc_UWord16)1;
} else } else
{ {
// This packet's seq num is lower than previously lowest seq num, but no wrap // This packet's seq num is lower than previously lowest seq num, but no wrap
@ -562,7 +700,50 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL
return InsertBuffer(ptrStartOfLayer, packetIndex, packet); return InsertBuffer(ptrStartOfLayer, packetIndex, packet);
} }
WebRtc_UWord32 VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec)
WebRtc_Word32
VCMSessionInfo::InformOfEmptyPacket(const WebRtc_UWord16 seqNum)
{
// Empty packets may be FEC or filler packets. They are sequential and
// follow the data packets, therefore, we should only keep track of the high
// and low sequence numbers and may assume that the packets in between are
// empty packets belonging to the same frame (timestamp).
if (_emptySeqNumLow == -1 && _emptySeqNumHigh == -1)
{
_emptySeqNumLow = seqNum;
_emptySeqNumHigh = seqNum;
}
else
{
if (seqNum > _emptySeqNumHigh)
{
// This packet's seq num is higher than previously highest seq num;
// normal case if we have a wrap, only update with wrapped values
if (!(_emptySeqNumHigh < 0x00ff && seqNum > 0xff00))
{
_emptySeqNumHigh = seqNum;
}
}
else if (_emptySeqNumHigh > 0xff00 && seqNum < 0x00ff)
{
// wrap
_emptySeqNumHigh = seqNum;
}
if (_emptySeqNumLow < 0x00ff && seqNum > 0xff00)
{
// negative wrap
if (seqNum - 0x10000 - _emptySeqNumLow < 0)
{
_emptySeqNumLow = seqNum;
}
}
}
return 0;
}
WebRtc_UWord32
VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec)
{ {
WebRtc_UWord32 currentPacketOffset = 0; WebRtc_UWord32 currentPacketOffset = 0;
WebRtc_UWord32 length = GetSessionLength(); WebRtc_UWord32 length = GetSessionLength();

12
modules/video_coding/main/source/session_info.h
View File

@ -26,10 +26,16 @@ public:
VCMSessionInfo(const VCMSessionInfo& rhs); VCMSessionInfo(const VCMSessionInfo& rhs);
WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num); WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 numberOfSeqNum);
// Hybrid version: Zero out seq num for NACK list
// apply a score based on the packet location and the external rttScore
WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
WebRtc_Word32 numberOfSeqNum,
float rttScore);
virtual void Reset(); virtual void Reset();
WebRtc_Word64 InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer); WebRtc_Word64 InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer);
WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum);
virtual bool IsSessionComplete(); virtual bool IsSessionComplete();
WebRtc_UWord32 MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer); WebRtc_UWord32 MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer);
@ -84,8 +90,10 @@ protected:
WebRtc_UWord16 _highestPacketIndex; WebRtc_UWord16 _highestPacketIndex;
// Length of packet (used for reordering) // Length of packet (used for reordering)
WebRtc_UWord32 _packetSizeBytes[kMaxPacketsInJitterBuffer]; WebRtc_UWord32 _packetSizeBytes[kMaxPacketsInJitterBuffer];
// Completness of packets. Used for deciding if the frame is decodable. // Completeness of packets. Used for deciding if the frame is decodable.
WebRtc_UWord8 _naluCompleteness[kMaxPacketsInJitterBuffer]; WebRtc_UWord8 _naluCompleteness[kMaxPacketsInJitterBuffer];
WebRtc_Word32 _emptySeqNumLow;
WebRtc_Word32 _emptySeqNumHigh;
bool _ORwithPrevByte[kMaxPacketsInJitterBuffer]; bool _ORwithPrevByte[kMaxPacketsInJitterBuffer];
}; };

9
modules/video_coding/main/source/video_coding_impl.cc
View File

@ -269,7 +269,7 @@ VideoCodingModuleImpl::TimeUntilNextProcess()
{ {
WebRtc_UWord32 timeUntilNextProcess = VCM_MIN(_receiveStatsTimer.TimeUntilProcess(), WebRtc_UWord32 timeUntilNextProcess = VCM_MIN(_receiveStatsTimer.TimeUntilProcess(),
_sendStatsTimer.TimeUntilProcess()); _sendStatsTimer.TimeUntilProcess());
if ((_receiver.NackMode() == kNackInfinite) || (_dualReceiver.State() != kPassive)) if ((_receiver.NackMode() != kNoNack) || (_dualReceiver.State() != kPassive))
{ {
// We need a Process call more often if we are relying on retransmissions // We need a Process call more often if we are relying on retransmissions
timeUntilNextProcess = VCM_MIN(timeUntilNextProcess, timeUntilNextProcess = VCM_MIN(timeUntilNextProcess,
@ -576,6 +576,7 @@ VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, bo
{ {
WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id),
"SetVideoProtection()"); "SetVideoProtection()");
switch (videoProtection) switch (videoProtection)
{ {
@ -664,16 +665,18 @@ VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, bo
case kProtectionNackFEC: case kProtectionNackFEC:
{ {
{ {
// Receive side
CriticalSectionScoped cs(_receiveCritSect); CriticalSectionScoped cs(_receiveCritSect);
if (enable) if (enable)
{ {
_receiver.SetNackMode(kNackInfinite); _receiver.SetNackMode(kNackHybrid);
} }
else else
{ {
_receiver.SetNackMode(kNoNack); _receiver.SetNackMode(kNoNack);
} }
} }
// Send Side
{ {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
_mediaOpt.EnableNackFEC(enable); _mediaOpt.EnableNackFEC(enable);
@ -1298,7 +1301,7 @@ VideoCodingModuleImpl::NackList(WebRtc_UWord16* nackList, WebRtc_UWord16& size)
// Collect sequence numbers from the default receiver // Collect sequence numbers from the default receiver
// if in normal nack mode. Otherwise collect them from // if in normal nack mode. Otherwise collect them from
// the dual receiver if the dual receiver is receiving. // the dual receiver if the dual receiver is receiving.
if (_receiver.NackMode() == kNackInfinite) if (_receiver.NackMode() != kNoNack)
{ {
nackStatus = _receiver.NackList(nackList, size); nackStatus = _receiver.NackList(nackList, size);
} }

6
modules/video_coding/main/test/jitter_buffer_test.cc
View File

@ -1461,9 +1461,9 @@ int JitterBufferTest(CmdArgs& args)
// --------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------
// | 3 | 4 | 5 | 6 | 7 | 9 | x | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | x | 21 |.....| 102 | // | 3 | 4 | 5 | 6 | 7 | 9 | x | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | x | 21 |.....| 102 |
// --------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------
jb.SetNackStatus(true); jb.SetNackMode(kNackInfinite);
TEST(jb.GetNackStatus()); TEST(jb.GetNackMode() == kNackInfinite);
// insert first packet // insert first packet
timeStamp += 33*90; timeStamp += 33*90;
@ -1880,7 +1880,7 @@ int JitterBufferTest(CmdArgs& args)
//Test incomplete NALU frames //Test incomplete NALU frames
jb.Flush(); jb.Flush();
jb.SetNackStatus(false); jb.SetNackMode(kNoNack);
seqNum ++; seqNum ++;
timeStamp += 33*90; timeStamp += 33*90;
int insertedLength=0; int insertedLength=0;