diff --git a/modules/video_coding/main/source/frame_buffer.cc b/modules/video_coding/main/source/frame_buffer.cc index 6ea1f4baa..bd8be3ef7 100644 --- a/modules/video_coding/main/source/frame_buffer.cc +++ b/modules/video_coding/main/source/frame_buffer.cc @@ -133,14 +133,19 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) if (kStateEmpty == _state) { - // This is the first packet inserted into this frame, + // This is the first packet (empty and/or data) inserted into this frame. // store some info and set some initial values. _timeStamp = packet.timestamp; _codec = packet.codec; - SetState(kStateIncomplete); + // for the first media packet + if (packet.frameType != kFrameEmpty) + { + SetState(kStateIncomplete); + } } - WebRtc_UWord32 requiredSizeBytes = Length() + packet.sizeBytes + (packet.insertStartCode?kH264StartCodeLengthBytes:0); + WebRtc_UWord32 requiredSizeBytes = Length() + packet.sizeBytes + + (packet.insertStartCode ? kH264StartCodeLengthBytes : 0); if (requiredSizeBytes >= _size) { const WebRtc_UWord32 increments = requiredSizeBytes / kBufferIncStepSizeBytes + @@ -156,7 +161,7 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) } } WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer); - if(retVal == -1) + if (retVal == -1) { return kSizeError; } @@ -201,6 +206,17 @@ WebRtc_Word32 VCMFrameBuffer::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n return 0; } +// Zero out all entries in list up to and including the (first) entry equal to +// _lowSeqNum. Hybrid mode: 1. Don't NACK FEC packets 2. Make a smart decision +// on whether to NACK or not + +WebRtc_Word32 VCMFrameBuffer::ZeroOutSeqNumHybrid(WebRtc_Word32* list, + WebRtc_Word32 num, + float rttScore) +{ + return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttScore); +} + void VCMFrameBuffer::IncrementNackCount() { _nackCount++; @@ -227,7 +243,6 @@ void VCMFrameBuffer::Reset() { _length = 0; _timeStamp = 0; - _sessionInfo.Reset(); _frameCounted = false; _payloadType = 0; @@ -237,7 +252,7 @@ void VCMFrameBuffer::Reset() VCMEncodedFrame::Reset(); } -// Makes sure the session contain a decodable stream. +// Makes sure the session contains a decodable stream. void VCMFrameBuffer::MakeSessionDecodable() { @@ -275,7 +290,8 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) case kStateComplete: assert(_state == kStateEmpty || - _state == kStateIncomplete); + _state == kStateIncomplete || + _state == kStateDecodable); break; @@ -286,11 +302,22 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) case kStateDecoding: // we can go to this state from state kStateComplete kStateIncomplete - assert(_state == kStateComplete || _state == kStateIncomplete); + assert(_state == kStateComplete || _state == kStateIncomplete || + _state == kStateDecodable); // Transfer frame information to EncodedFrame and create any codec specific information RestructureFrameInformation(); break; + case kStateDecodable: + if (_state == kStateComplete) + { + // if complete, obviously decodable, keep as is. + return; + } + assert(_state == kStateEmpty || + _state == kStateIncomplete); + break; + default: // Should never happen assert(!"FrameBuffer::SetState Incorrect frame buffer state as input"); diff --git a/modules/video_coding/main/source/frame_buffer.h b/modules/video_coding/main/source/frame_buffer.h index 15d8f8150..65c56a3dc 100644 --- a/modules/video_coding/main/source/frame_buffer.h +++ b/modules/video_coding/main/source/frame_buffer.h @@ -64,6 +64,10 @@ public: // NACK // Zero out all entries in list up to and including the entry equal to _lowSeqNum WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num); + // Hybrid extension: only NACK important packets, discard FEC packets + WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list, + WebRtc_Word32 num, + float rttScore); void IncrementNackCount(); WebRtc_Word16 GetNackCount() const; diff --git a/modules/video_coding/main/source/jitter_buffer.cc b/modules/video_coding/main/source/jitter_buffer.cc index f7c85ea4e..b74adb06b 100644 --- a/modules/video_coding/main/source/jitter_buffer.cc +++ b/modules/video_coding/main/source/jitter_buffer.cc @@ -16,6 +16,7 @@ #include "jitter_buffer.h" #include "jitter_buffer_common.h" #include "jitter_estimator.h" +#include "media_optimization.h" // hybrid NACK/FEC thresholds. #include "packet.h" #include "event.h" @@ -46,11 +47,14 @@ VCMJitterBuffer::FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestam } bool -VCMJitterBuffer::CompleteKeyFrameCriteria(VCMFrameBuffer* frame, const void* /*notUsed*/) +VCMJitterBuffer::CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame, + const void* /*notUsed*/) { const VCMFrameBufferStateEnum state = frame->GetState(); + // We can decode key frame or decodable/complete frames. return (frame->FrameType() == kVideoFrameKey) && - (state == kStateComplete); + ((state == kStateComplete) + || (state == kStateDecodable)); } // Constructor @@ -76,7 +80,8 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId, _numConsecutiveOldFrames(0), _numConsecutiveOldPackets(0), _jitterEstimate(vcmId, receiverId), - _usingNACK(false), + _rttMs(0), + _nackMode(kNoNack), _NACKSeqNum(), _NACKSeqNumLength(0), _missingMarkerBits(false), @@ -87,7 +92,7 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId, _lastDecodedSeqNum = -1; memset(_NACKSeqNumInternal, -1, sizeof(_NACKSeqNumInternal)); - for (int i=0; i< kStartNumberOfFrames; i++) + for (int i = 0; i< kStartNumberOfFrames; i++) { _frameBuffers[i] = new VCMFrameBuffer(); } @@ -130,7 +135,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) _jitterEstimate = rhs._jitterEstimate; _delayEstimate = rhs._delayEstimate; _waitingForCompletion = rhs._waitingForCompletion; - _usingNACK = rhs._usingNACK; + _nackMode = rhs._nackMode; + _rttMs = rhs._rttMs; _NACKSeqNumLength = rhs._NACKSeqNumLength; _missingMarkerBits = rhs._missingMarkerBits; _firstPacket = rhs._firstPacket; @@ -138,7 +144,7 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) memcpy(_receiveStatistics, rhs._receiveStatistics, sizeof(_receiveStatistics)); memcpy(_NACKSeqNumInternal, rhs._NACKSeqNumInternal, sizeof(_NACKSeqNumInternal)); memcpy(_NACKSeqNum, rhs._NACKSeqNum, sizeof(_NACKSeqNum)); - for (int i=0; i < kMaxNumberOfFrames; i++) + for (int i = 0; i < kMaxNumberOfFrames; i++) { if (_frameBuffers[i] != NULL) { @@ -147,7 +153,7 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) } } while(_frameBuffersTSOrder.Erase(_frameBuffersTSOrder.First()) != -1) { } - for (int i=0; i < _maxNumberOfFrames; i++) + for (int i = 0; i < _maxNumberOfFrames; i++) { _frameBuffers[i] = new VCMFrameBuffer(*(rhs._frameBuffers[i])); if (_frameBuffers[i]->Length() > 0) @@ -162,7 +168,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) } WebRtc_UWord32 -VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, const WebRtc_UWord32 newTimestamp) +VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, + const WebRtc_UWord32 newTimestamp) { bool wrap = (newTimestamp < 0x0000ffff && existingTimestamp > 0xffff0000) || (newTimestamp > 0xffff0000 && existingTimestamp < 0x0000ffff); @@ -185,7 +192,8 @@ VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, const W } // Start jitter buffer -void VCMJitterBuffer::Start() +void +VCMJitterBuffer::Start() { CriticalSectionScoped cs(_critSect); _running = true; @@ -205,12 +213,14 @@ void VCMJitterBuffer::Start() _waitingForCompletion.latestPacketTime = -1; _missingMarkerBits = false; _firstPacket = true; + _rttMs = 0; WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: start", this); } // Stop jitter buffer -void VCMJitterBuffer::Stop() +void +VCMJitterBuffer::Stop() { _critSect.Enter(); _running = false; @@ -231,21 +241,24 @@ void VCMJitterBuffer::Stop() WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: stop", this); } -bool VCMJitterBuffer::Running() const +bool +VCMJitterBuffer::Running() const { CriticalSectionScoped cs(_critSect); return _running; } // Flush jitter buffer -void VCMJitterBuffer::Flush() +void +VCMJitterBuffer::Flush() { CriticalSectionScoped cs(_critSect); FlushInternal(); } // Must be called under the critical section _critSect -void VCMJitterBuffer::FlushInternal() +void +VCMJitterBuffer::FlushInternal() { // Erase all frames from the sorted list and set their state to free. _frameBuffersTSOrder.Flush(); @@ -292,7 +305,8 @@ VCMJitterBuffer::ReleaseFrameInternal(VCMFrameBuffer* frame) // Doing it here increases the degree of freedom for e.g. future // reconstructability of separate layers. Must be called under the // critical section _critSect. -void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) +void +VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { if (frame == NULL) { @@ -385,8 +399,7 @@ void VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) // Only signal if this is the oldest frame. // Not necessary the case due to packet reordering or NACK. - if(!_usingNACK || - (oldFrame != NULL && oldFrame == frame)) + if (!WaitForNack() || (oldFrame != NULL && oldFrame == frame)) { _frameEvent.Set(); } @@ -481,7 +494,7 @@ VCMJitterBuffer::GetEmptyFrame() _critSect.Enter(); - for (int i=0; i<_maxNumberOfFrames; ++i) + for (int i = 0; i <_maxNumberOfFrames; ++i) { if (kStateFree == _frameBuffers[i]->GetState()) { @@ -512,7 +525,8 @@ VCMJitterBuffer::GetEmptyFrame() } // Must be called under the critical section _critSect. -VCMFrameListItem* VCMJitterBuffer::FindOldestSequenceNum() const +VCMFrameListItem* +VCMJitterBuffer::FindOldestSequenceNum() const { WebRtc_UWord16 currentLow = 0xffff; VCMFrameBufferStateEnum state = kStateFree; @@ -562,7 +576,8 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestSequenceNum() const // Must be called under critical section // Based on sequence number // Return NULL for lost packets -VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame() +VCMFrameListItem* +VCMJitterBuffer::FindOldestCompleteContinuousFrame() { // if we have more than one frame done since last time, pick oldest VCMFrameBuffer* oldestFrame = NULL; @@ -578,7 +593,8 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame() { if (kStateComplete != oldestFrame->GetState()) { - // Try to see if the frame is complete even though the state is not complete. Can happen if markerbit is not set. + // Try to see if the frame is complete even though the state is not + // complete. Can happen if markerbit is not set. if (!CheckForCompleteFrame(oldestFrameItem)) { oldestFrame = NULL; @@ -590,7 +606,7 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame() currentLow = oldestFrame->GetLowSeqNum(); } } - if(oldestFrame == NULL) + if (oldestFrame == NULL) { // no complete frame no point to continue return NULL; @@ -601,15 +617,16 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame() // Use seqNum not timestamp since a full frame might be lost if (_lastDecodedSeqNum != -1) { - // it's not enough that we have complete frame we need the seq numbers to be continuous too - // for layers it's not enough that we have complete frame we need the layers to be continuous too + // it's not enough that we have complete frame we need the seq numbers + // to be continuous too for layers it's not enough that we have complete + // frame we need the layers to be continuous too currentLow = oldestFrame->GetLowSeqNum(); WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum; - // we could have received the first packet of the last frame before a long period - // if drop, that case is handled by GetNackList - if ( ((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) + // we could have received the first packet of the last frame before a + // long period if drop, that case is handled by GetNackList + if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) { // wait since we want a complete continuous frame return NULL; @@ -618,11 +635,12 @@ VCMFrameListItem* VCMJitterBuffer::FindOldestCompleteContinuousFrame() return oldestFrameItem; } -// Check if the oldest frame is complete even though it is not in a complete state. +// Check if the oldest frame is complete even though it isn't complete. // This can happen when makerbit is not set // Must be called under the critical section _critSect. // Return false for lost packets -bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) +bool +VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) { const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameItem); VCMFrameBuffer* oldestFrame = NULL; @@ -630,7 +648,7 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) { oldestFrame = oldestFrameItem->GetItem(); } - if(nextFrameItem != NULL) + if (nextFrameItem != NULL) { // We have received at least one packet from a later frame. if(!oldestFrame->HaveLastPacket()) // If we don't have the markerbit @@ -638,7 +656,7 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) VCMFrameBuffer* nextFrame = nextFrameItem->GetItem(); // Verify that we have received the first packet of the next frame. // This is the only way we can be sure we're not missing the last packet. - if(nextFrame != NULL && nextFrame->GetLowSeqNum() == + if (nextFrame != NULL && nextFrame->GetLowSeqNum() == static_cast(oldestFrame->GetHighSeqNum()+1)) // Sequence number is only 16 bit { _missingMarkerBits = true; @@ -648,7 +666,7 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) UpdateFrameState(oldestFrame); } const VCMFrameBufferStateEnum state = oldestFrame->GetState(); - if(state == kStateComplete) + if (state == kStateComplete) { if(oldestFrame->Length() > 0) { @@ -663,7 +681,8 @@ bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) } // Call from inside the critical section _critSect -void VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame) +void +VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame) { if (frame == NULL) { @@ -679,7 +698,8 @@ void VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame) // Calculate frame and bit rates -WebRtc_Word32 VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) +WebRtc_Word32 +VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) { CriticalSectionScoped cs(_critSect); const WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); @@ -744,7 +764,8 @@ WebRtc_Word32 VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord } // Returns immediately or a X ms event hang waiting for a decodable frame, X decided by caller -VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) +VCMEncodedFrame* +VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) { if (!_running) { @@ -853,38 +874,54 @@ VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 max return oldestFrame; } -WebRtc_UWord32 VCMJitterBuffer::GetEstimatedJitterMS() +WebRtc_UWord32 +VCMJitterBuffer::GetEstimatedJitterMS() { CriticalSectionScoped cs(_critSect); return GetEstimatedJitterMsInternal(); } -WebRtc_UWord32 VCMJitterBuffer::GetEstimatedJitterMsInternal() +WebRtc_UWord32 +VCMJitterBuffer::GetEstimatedJitterMsInternal() { WebRtc_UWord32 estimate = VCMJitterEstimator::OPERATING_SYSTEM_JITTER; - estimate += static_cast(_jitterEstimate.GetJitterEstimate() + 0.5); + + // compute RTT multiplier for estimation + double rttMult = 1.0f; + if (_nackMode == kNackHybrid && _rttMs > kLowRttNackMs) + { + // from here we count on FEC + rttMult = 0.0f; + } + estimate += static_cast + (_jitterEstimate.GetJitterEstimate(rttMult) + 0.5); if (_missingMarkerBits) { - // Since the incoming packets are all missing marker bits we have to wait until the first - // packet of the next frame arrives, before we can safely say that the frame is complete. - // Therefore we have to compensate the jitter buffer level with one frame period. - - // TODO(holmer): The timestamp diff should probably be filtered (max filter) since - // the diff can alternate between e.g. 3000 and 6000 if we have a frame rate between - // 15 and 30 frames per seconds. + // Since the incoming packets are all missing marker bits we have to + // wait until the first packet of the next frame arrives, before we can + // safely say that the frame is complete. Therefore we have to compensate + // the jitter buffer level with one frame period. + // TODO(holmer): The timestamp diff should probably be filtered + // (max filter) since the diff can alternate between e.g. 3000 and 6000 + // if we have a frame rate between 15 and 30 frames per seconds. estimate += _delayEstimate.CurrentTimeStampDiffMs(); } return estimate; } -void VCMJitterBuffer::UpdateRtt(WebRtc_UWord32 rttMs) +void +VCMJitterBuffer::UpdateRtt(WebRtc_UWord32 rttMs) { CriticalSectionScoped cs(_critSect); + _rttMs = rttMs; _jitterEstimate.UpdateRtt(rttMs); } // wait for the first packet in the next frame to arrive -WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, FrameType& incomingFrameType, WebRtc_Word64& renderTimeMs) +WebRtc_Word64 +VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, + FrameType& incomingFrameType, + WebRtc_Word64& renderTimeMs) { if (!_running) { @@ -913,7 +950,6 @@ WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, Fr CleanUpOldFrames(); CleanUpSizeZeroFrames(); - oldestFrame = _frameBuffersTSOrder.FirstFrame(); }else { @@ -946,7 +982,8 @@ WebRtc_Word64 VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, Fr // Will the packet sequence be complete if the next frame is grabbed for decoding right now? // That is, have we lost a frame between the last decoded frame and the next, or is the next // frame missing one or more packets? -bool VCMJitterBuffer::CompleteSequenceWithNextFrame() +bool +VCMJitterBuffer::CompleteSequenceWithNextFrame() { CriticalSectionScoped cs(_critSect); // Finding oldest frame ready for decoder, but check sequence number and size @@ -989,7 +1026,8 @@ bool VCMJitterBuffer::CompleteSequenceWithNextFrame() } // Returns immediately -VCMEncodedFrame* VCMJitterBuffer::GetFrameForDecoding() +VCMEncodedFrame* +VCMJitterBuffer::GetFrameForDecoding() { CriticalSectionScoped cs(_critSect); if (!_running) @@ -997,7 +1035,7 @@ VCMEncodedFrame* VCMJitterBuffer::GetFrameForDecoding() return NULL; } - if(_usingNACK) + if (WaitForNack()) { return GetFrameForDecodingNACK(); } @@ -1061,6 +1099,7 @@ VCMJitterBuffer::GetFrameForDecodingNACK() { // when we use NACK we don't release non complete frames // unless we have a complete key frame. + // In hybrid mode, we may release decodable frames (non-complete) // Clean up old frames and empty frames CleanUpOldFrames(); @@ -1077,8 +1116,9 @@ VCMJitterBuffer::GetFrameForDecodingNACK() if (oldestFrame == NULL) { continuous = false; - // If we didn't find one we're good with a complete key frame. - oldestFrameListItem = _frameBuffersTSOrder.FindFrameListItem(CompleteKeyFrameCriteria); + // If we didn't find one we're good with a complete key/decodable frame. + oldestFrameListItem = _frameBuffersTSOrder.FindFrameListItem( + CompleteDecodableKeyFrameCriteria); if (oldestFrameListItem != NULL) { oldestFrame = oldestFrameListItem->GetItem(); @@ -1089,7 +1129,7 @@ VCMJitterBuffer::GetFrameForDecodingNACK() } } - // We have a complete continuous frame, decode it. + // We have a complete/decodable continuous frame, decode it. // store seqnum _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); // store current time @@ -1181,7 +1221,10 @@ VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, bool incom // Must be called under the critical section _critSect. Should never be called with // retransmitted frames, they must be filtered out before this function is called. void -VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs, WebRtc_UWord32 timestamp, WebRtc_UWord32 frameSize, bool incompleteFrame) +VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs, + WebRtc_UWord32 timestamp, + WebRtc_UWord32 frameSize, + bool incompleteFrame) { if (latestPacketTimeMs == -1) { @@ -1217,17 +1260,18 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word highSeqNum = -1; lowSeqNum = _lastDecodedSeqNum; - // find higest seqnumbers - for (i=0; i<_maxNumberOfFrames; ++i) + // find highest seqnumbers + for (i = 0; i < _maxNumberOfFrames; ++i) { seqNum = _frameBuffers[i]->GetHighSeqNum(); - // Ignore free frames + // Ignore free / empty frames VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState(); - if((kStateFree != state) && + + if ((kStateFree != state) && (kStateEmpty != state) && (kStateDecoding != state) && - seqNum != -1) + seqNum != -1) { if (highSeqNum == -1) { @@ -1260,9 +1304,10 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) int i = 0; WebRtc_Word32 lowSeqNum = -1; WebRtc_Word32 highSeqNum = -1; - listExtended=false; + listExtended = false; - if (!_usingNACK) + // don't create list, if we won't wait for it + if (!WaitForNack()) { nackSize = 0; return NULL; @@ -1277,11 +1322,10 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) // write a list of all seq num we have if (lowSeqNum == -1 || highSeqNum == -1) { - //This happens if we lose the first packet, nothing is poped + //This happens if we lose the first packet, nothing is popped if (highSeqNum == -1) { nackSize = 0;// we have not received any packets yet - } else { @@ -1371,7 +1415,8 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) // We have cleaned up the jb and found a key frame // The function itself has set last decoded seq. WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1, - "\tKey frame found. _lastDecodedSeqNum[0] %d", _lastDecodedSeqNum); + "\tKey frame found. _lastDecodedSeqNum[0] %d", + _lastDecodedSeqNum); nackSize = 0; } @@ -1379,27 +1424,51 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) } WebRtc_UWord16 seqNumberIterator = (WebRtc_UWord16)(lowSeqNum + 1); - for (i=0; i < numberOfSeqNum; i++) + for (i = 0; i < numberOfSeqNum; i++) { _NACKSeqNumInternal[i] = seqNumberIterator; seqNumberIterator++; } - // now we have a list of all seq numbers that could have been sent + + // now we have a list of all sequence numbers that could have been sent // zero out the ones we have received for (i = 0; i < _maxNumberOfFrames; i++) { // loop all created frames - // We dont need to check if frame is decoding since lowSeqNum is based on _lastDecodedSeqNum + // We don't need to check if frame is decoding since lowSeqNum is based + // on _lastDecodedSeqNum // Ignore free frames VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState(); + if ((kStateFree != state) && (kStateEmpty != state) && (kStateDecoding != state)) { - _frameBuffers[i]->ZeroOutSeqNum(_NACKSeqNumInternal, numberOfSeqNum); - // used when the frame is being processed by the decoding thread - // dont need to use that info in this loop + // Reaching thus far means we are going to update the nack list + // When in hybrid mode, we also need to check empty frames, so as not + // to add empty packets to the nack list + if (_nackMode == kNackHybrid) + { + // build external rttScore based on RTT value + float rttScore = 1.0f; + _frameBuffers[i]->ZeroOutSeqNumHybrid(_NACKSeqNumInternal, + numberOfSeqNum, + rttScore); + if (_frameBuffers[i]->IsRetransmitted() == false) + { + // if no retransmission required,set the state to decodable + // meaning that we will not wait for NACK + _frameBuffers[i]->SetState(kStateDecodable); + } + } + else + { + // used when the frame is being processed by the decoding thread + // don't need to use that info in this loop + _frameBuffers[i]->ZeroOutSeqNum(_NACKSeqNumInternal, + numberOfSeqNum); + } } } @@ -1407,7 +1476,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) int emptyIndex = -1; for (i = 0; i < numberOfSeqNum; i++) { - if (_NACKSeqNumInternal[i] == -1) + if (_NACKSeqNumInternal[i] == -1 || _NACKSeqNumInternal[i] == -2 ) { // this is empty if (emptyIndex == -1) @@ -1442,36 +1511,39 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) nackSize = emptyIndex; } // convert to unsigned short 16 bit and store in a list to be used externally. - if(nackSize > _NACKSeqNumLength) + if (nackSize > _NACKSeqNumLength) { - listExtended=true; // Larger list means that the nack list has been extended since the last call. + // Larger list means that the nack list was extended since the last call. + listExtended = true; } + for(WebRtc_UWord32 j = 0; j < nackSize; j++) { - // Check if the list has been extended since it was last created. I.e, new items have been added - if(_NACKSeqNumLength > j && !listExtended) + // Check if the list has been extended since it was last created. I.e, + // new items have been added + if (_NACKSeqNumLength > j && !listExtended) { WebRtc_UWord32 k = 0; - for(k = j; k < _NACKSeqNumLength; k++) + for (k = j; k < _NACKSeqNumLength; k++) { // Found the item in the last list. I.e, no new items found yet. - if(_NACKSeqNum[k] == (WebRtc_UWord16)_NACKSeqNumInternal[j]) + if (_NACKSeqNum[k] == (WebRtc_UWord16)_NACKSeqNumInternal[j]) { break; } - } - if(k == _NACKSeqNumLength) // New item not found in last list. + if (k == _NACKSeqNumLength) // New item not found in last list. { - listExtended=true; + listExtended = true; } } else { - listExtended=true; + listExtended = true; } _NACKSeqNum[j] = (WebRtc_UWord16)_NACKSeqNumInternal[j]; } + _NACKSeqNumLength = nackSize; return _NACKSeqNum; @@ -1511,35 +1583,39 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) VCMFrameBufferEnum ret = kSizeError; VCMFrameBuffer* frame = static_cast(buffer); - if (_firstPacket) + // Empty packets may bias the jitter estimate (lacking size component), + // therefore don't let empty packet trigger the following updates: + if (packet.frameType != kFrameEmpty) { - // Now it's time to start estimating jitter - // reset the delay estimate. - _delayEstimate.Reset(); - _firstPacket = false; - } + if (_firstPacket) + { + // Now it's time to start estimating jitter + // reset the delay estimate. + _delayEstimate.Reset(); + _firstPacket = false; + } - if (_waitingForCompletion.timestamp == packet.timestamp) - { - // This can get bad if we have a lot of duplicate packets, - // we will then count some packet multiple times. - _waitingForCompletion.frameSize += packet.sizeBytes; - _waitingForCompletion.latestPacketTime = nowMs; - } - else if (_waitingForCompletion.latestPacketTime >= 0 && - _waitingForCompletion.latestPacketTime + 2000 <= nowMs) - { - // A packet should never be more than two seconds late - UpdateJitterAndDelayEstimates(_waitingForCompletion, true); - _waitingForCompletion.latestPacketTime = -1; - _waitingForCompletion.frameSize = 0; - _waitingForCompletion.timestamp = 0; + if (_waitingForCompletion.timestamp == packet.timestamp) + { + // This can get bad if we have a lot of duplicate packets, + // we will then count some packet multiple times. + _waitingForCompletion.frameSize += packet.sizeBytes; + _waitingForCompletion.latestPacketTime = nowMs; + } + else if (_waitingForCompletion.latestPacketTime >= 0 && + _waitingForCompletion.latestPacketTime + 2000 <= nowMs) + { + // A packet should never be more than two seconds late + UpdateJitterAndDelayEstimates(_waitingForCompletion, true); + _waitingForCompletion.latestPacketTime = -1; + _waitingForCompletion.frameSize = 0; + _waitingForCompletion.timestamp = 0; + } } if (frame != NULL) { VCMFrameBufferStateEnum state = frame->GetState(); - if (state == kStateDecoding && packet.sizeBytes == 0) { // Filler packet, make sure we update the last decoded seq num @@ -1655,7 +1731,7 @@ VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const { if (_NACKSeqNum && _NACKSeqNumLength > 0) { - for (WebRtc_UWord16 i=0; i < _NACKSeqNumLength; i++) + for (WebRtc_UWord16 i = 0; i < _NACKSeqNumLength; i++) { if (packet.seqNum == _NACKSeqNum[i]) { @@ -1667,18 +1743,20 @@ VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const } // Get nack status (enabled/disabled) -bool VCMJitterBuffer::GetNackStatus() +VCMNackMode +VCMJitterBuffer::GetNackMode() const { CriticalSectionScoped cs(_critSect); - return _usingNACK; + return _nackMode; } -// Enable/disable nack -void VCMJitterBuffer::SetNackStatus(bool enable) +// Set NACK mode +void +VCMJitterBuffer::SetNackMode(VCMNackMode mode) { CriticalSectionScoped cs(_critSect); - _usingNACK = enable; - if (!_usingNACK) + _nackMode = mode; + if (_nackMode == kNoNack) { _jitterEstimate.ResetNackCount(); } @@ -1808,7 +1886,7 @@ VCMJitterBuffer::CleanUpSizeZeroFrames() const WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum(); const WebRtc_Word32 frameLowSeqNum = ptrTempBuffer->GetLowSeqNum(); - if ((frameLowSeqNum == (_lastDecodedSeqNum+ 1)) || // Frame is next in line + if ((frameLowSeqNum == (_lastDecodedSeqNum + 1)) || // Frame is next in line ((frameLowSeqNum == 0) && (_lastDecodedSeqNum== 0xffff))) { // This frame follows the last decoded frame, release it. @@ -1878,4 +1956,33 @@ VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame) } } +bool +VCMJitterBuffer::WaitForNack() +{ + // NACK disabled -> can't wait + if (_nackMode == kNoNack) + { + return false; + } + // NACK only -> always wait + else if (_nackMode == kNackInfinite) + { + return true; + } + // else: hybrid mode, evaluate + // RTT high, don't wait + if (_rttMs >= kHighRttNackMs) + { + return false; + } + // RTT low, we can afford the wait + else if (_rttMs <= kLowRttNackMs) + { + return true; + } + // interim values - hybrid mode + return true; +} + + } diff --git a/modules/video_coding/main/source/jitter_buffer.h b/modules/video_coding/main/source/jitter_buffer.h index 4ec19359e..ab88f38bc 100644 --- a/modules/video_coding/main/source/jitter_buffer.h +++ b/modules/video_coding/main/source/jitter_buffer.h @@ -24,6 +24,13 @@ namespace webrtc { +enum VCMNackMode +{ + kNackInfinite, + kNackHybrid, + kNoNack +}; + // forward declarations class VCMFrameBuffer; class VCMPacket; @@ -90,7 +97,7 @@ public: WebRtc_Word32 GetFrame(const VCMPacket& packet, VCMEncodedFrame*&); VCMEncodedFrame* GetFrame(const VCMPacket& packet); // deprecated - // Returns the time in ms when the latest packet was insterted into the frame. + // Returns the time in ms when the latest packet was inserted into the frame. // Retransmitted is set to true if any of the packets belonging to the frame // has been retransmitted. WebRtc_Word64 LastPacketTime(VCMEncodedFrame* frame, bool& retransmitted) const; @@ -103,8 +110,8 @@ public: void UpdateRtt(WebRtc_UWord32 rttMs); // NACK - void SetNackStatus(bool enable); // Enable/disable nack - bool GetNackStatus(); // Get nack status (enabled/disabled) + void SetNackMode(VCMNackMode mode); // Enable/disable nack + VCMNackMode GetNackMode() const; // Get nack mode // Get list of missing sequence numbers (size in number of elements) WebRtc_UWord16* GetNackList(WebRtc_UWord16& nackSize, bool& listExtended); @@ -162,18 +169,23 @@ protected: private: static bool FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp); - static bool CompleteKeyFrameCriteria(VCMFrameBuffer* frame, const void* notUsed); + static bool CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame, + const void* notUsed); + // Decide whether should wait for NACK (mainly relevant for hybrid mode) + bool WaitForNack(); WebRtc_Word32 _vcmId; WebRtc_Word32 _receiverId; - bool _running; // If we are running (have started) or not - CriticalSectionWrapper& _critSect; + // If we are running (have started) or not + bool _running; + CriticalSectionWrapper& _critSect; bool _master; // Event to signal when we have a frame ready for decoder VCMEvent _frameEvent; // Event to signal when we have received a packet VCMEvent _packetEvent; - WebRtc_Word32 _maxNumberOfFrames; // Number of allocated frames + // Number of allocated frames + WebRtc_Word32 _maxNumberOfFrames; // Array of pointers to the frames in JB VCMFrameBuffer* _frameBuffers[kMaxNumberOfFrames]; VCMFrameListTimestampOrderAsc _frameBuffersTSOrder; @@ -189,10 +201,12 @@ private: WebRtc_UWord8 _receiveStatistics[4]; // Latest calculated frame rates of incoming stream WebRtc_UWord8 _incomingFrameRate; - WebRtc_UWord32 _incomingFrameCount; // Frame counter, reset in GetUpdate + // Frame counter, reset in GetUpdate + WebRtc_UWord32 _incomingFrameCount; // Real time for last _frameCount reset WebRtc_Word64 _timeLastIncomingFrameCount; - WebRtc_UWord32 _incomingBitCount; // Received bits counter, reset in GetUpdate + // Received bits counter, reset in GetUpdate + WebRtc_UWord32 _incomingBitCount; WebRtc_UWord32 _incomingBitRate; WebRtc_UWord32 _dropCount; // Frame drop counter // Number of frames in a row that have been too old @@ -204,9 +218,10 @@ private: // Calculates network delays used for jitter calculations VCMInterFrameDelay _delayEstimate; VCMJitterSample _waitingForCompletion; + WebRtc_UWord32 _rttMs; // NACK - bool _usingNACK; // If we are using nack + VCMNackMode _nackMode; // Holds the internal nack list (the missing seqence numbers) WebRtc_Word32 _NACKSeqNumInternal[kNackHistoryLength]; WebRtc_UWord16 _NACKSeqNum[kNackHistoryLength]; diff --git a/modules/video_coding/main/source/jitter_buffer_common.h b/modules/video_coding/main/source/jitter_buffer_common.h index a90418fb5..6f7bcad6f 100644 --- a/modules/video_coding/main/source/jitter_buffer_common.h +++ b/modules/video_coding/main/source/jitter_buffer_common.h @@ -46,7 +46,8 @@ enum VCMFrameBufferStateEnum kStateEmpty, // frame popped by the RTP receiver kStateIncomplete, // frame that have one or more packet(s) stored kStateComplete, // frame that have all packets - kStateDecoding // frame popped by the decoding thread + kStateDecoding, // frame popped by the decoding thread + kStateDecodable // Hybrid mode - frame can be decoded }; enum { kH264StartCodeLengthBytes = 4}; diff --git a/modules/video_coding/main/source/jitter_estimator.cc b/modules/video_coding/main/source/jitter_estimator.cc index 00dfe1379..59c71f14b 100644 --- a/modules/video_coding/main/source/jitter_estimator.cc +++ b/modules/video_coding/main/source/jitter_estimator.cc @@ -422,7 +422,7 @@ VCMJitterEstimator::UpdateMaxFrameSize(WebRtc_UWord32 frameSizeBytes) // Returns the current filtered estimate if available, // otherwise tries to calculate an estimate. double -VCMJitterEstimator::GetJitterEstimate() +VCMJitterEstimator::GetJitterEstimate(double rttMultiplier) { double jitterMS = CalculateEstimate(); if (_filterJitterEstimate > jitterMS) @@ -431,7 +431,7 @@ VCMJitterEstimator::GetJitterEstimate() } if (_nackCount >= _nackLimit) { - return jitterMS + _rttFilter.RttMs(); + return jitterMS + _rttFilter.RttMs() * rttMultiplier; } return jitterMS; } diff --git a/modules/video_coding/main/source/jitter_estimator.h b/modules/video_coding/main/source/jitter_estimator.h index 61263f004..1c5b0715c 100644 --- a/modules/video_coding/main/source/jitter_estimator.h +++ b/modules/video_coding/main/source/jitter_estimator.h @@ -41,9 +41,11 @@ public: // Returns the current jitter estimate in milliseconds and adds // also adds an RTT dependent term in cases of retransmission. + // Input: + // - rttMultiplier : RTT param multiplier (when applicable). // // Return value : Jitter estimate in milliseconds - double GetJitterEstimate(); + double GetJitterEstimate(double rttMultiplier); // Updates the nack counter/timer. // diff --git a/modules/video_coding/main/source/media_opt_util.cc b/modules/video_coding/main/source/media_opt_util.cc index 181144d00..140479301 100644 --- a/modules/video_coding/main/source/media_opt_util.cc +++ b/modules/video_coding/main/source/media_opt_util.cc @@ -18,105 +18,123 @@ #include #include #include -#include namespace webrtc { -bool -VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm) + +bool VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm) { - if (pm == NULL) - { - return true; - } - return pm->_score > _score; + if (pm == NULL) { + return true; + } + return pm->_score > _score; } -bool -VCMNackFecMethod::ProtectionFactor(const VCMProtectionParameters* /*parameters*/) +bool VCMNackFecMethod::ProtectionFactor(const VCMProtectionParameters* /*parameters*/) { - - //use FEC model with modification with RTT for now - + // use FEC model with modification with RTT for now return true; } -bool -VCMNackFecMethod::EffectivePacketLoss(const VCMProtectionParameters* /*parameters*/) +bool VCMNackFecMethod::EffectivePacketLoss(const VCMProtectionParameters* /*parameters*/) { - //use FEC model with modification with RTT for now - - return true; + // use FEC model with modification with RTT for now + return true; } - -bool -VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) +bool VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) { - VCMFecMethod fecMethod; - VCMNackMethod nackMethod; + // Hybrid Nack FEC has three operational modes: + // 1. Low RTT - Nack only (Set FEC rates to zero) + // 2. High RTT - FEC Only + // 3. Medium RTT values - Hybrid ; in hybrid mode, we will only nack the residual + // following the decoding of the FEC (and not in all cases, refer to JB logic) + + // Low RTT - NACK only mode + if (parameters->rtt < kLowRttNackMs) + { + // Set the FEC parameters to 0 + _protectionFactorK = 0; + _protectionFactorD = 0; + + // assume packets will be restored via NACK + // TODO: relax this assumption? + _effectivePacketLoss = 0; + _score = _efficiency; + return true; + } + // otherwise: we count on FEC; if the RTT is below a threshold, then we can + // nack the residual, based on a decision made in the JB. + // TODO(mikhal): adapt the FEC rate based on the RTT, i.e. the the + // level on which we will rely on NACK, e.g. less as we approach upper threshold. + VCMFecMethod fecMethod; const WebRtc_UWord8 plossMax = 129; - WebRtc_UWord16 rttMax = nackMethod.MaxRttNack(); - // We should reduce the NACK threshold for NackFec protection method, - // with FEC and ER, we should only use NACK for small RTT, to avoid delay - //But this parameter change should be shared with RTP and JB - //rttMax = (WebRtc_UWord16) 0.5*rttMax; + // Compute the protection factor + fecMethod.ProtectionFactor(parameters); - //Compute the protection factor - fecMethod.ProtectionFactor(parameters); + // Compute the effective packet loss + fecMethod.EffectivePacketLoss(parameters); - //Compute the effective packet loss - fecMethod.EffectivePacketLoss(parameters); - - WebRtc_UWord8 protFactorK = fecMethod._protectionFactorK; - WebRtc_UWord8 protFactorD = fecMethod._protectionFactorD; - WebRtc_UWord8 effPacketLoss = fecMethod._effectivePacketLoss; - float resPacketLoss = fecMethod._residualPacketLoss; + WebRtc_UWord8 protFactorK = fecMethod._protectionFactorK; + WebRtc_UWord8 protFactorD = fecMethod._protectionFactorD; + WebRtc_UWord8 effPacketLoss = fecMethod._effectivePacketLoss; + float resPacketLoss = fecMethod._residualPacketLoss; + // Correct FEC rates based on the RTT ( NACK effectiveness) WebRtc_Word16 rttIndex= (WebRtc_UWord16) parameters->rtt; float softnessRtt = 1.0; - if (parameters->rtt < rttMax) + if (parameters->rtt < kHighRttNackMs) { - softnessRtt = (float)VCMNackFecTable[rttIndex]/(float)4096.0; + // TODO(mikhal): update table + softnessRtt = (float)VCMNackFecTable[rttIndex] / (float)4096.0; - //soften ER with NACK on - //table depends on roundtrip time relative to rttMax (NACK Threshold) - _effectivePacketLoss = (WebRtc_UWord8)(effPacketLoss*softnessRtt); + // soften ER with NACK on + // table depends on RTT relative to rttMax (NACK Threshold) + _effectivePacketLoss = (WebRtc_UWord8)(effPacketLoss * softnessRtt); - //soften FEC with NACK on - //table depends on roundtrip time relative to rttMax (NACK Threshold) + // soften FEC with NACK on + // table depends on RTT relative to rttMax (NACK Threshold) _protectionFactorK = (WebRtc_UWord8) (protFactorK * softnessRtt); _protectionFactorD = (WebRtc_UWord8) (protFactorD * softnessRtt); } + // else - NACK is disabled, rely on FEC only - //make sure I frame protection is at least larger than P frame protection, and at least as high as received loss - WebRtc_UWord8 packetLoss = (WebRtc_UWord8)(255* parameters->lossPr); - _protectionFactorK = static_cast(VCM_MAX(packetLoss,VCM_MAX(_scaleProtKey*protFactorD,protFactorK))); + // make sure I frame protection is at least larger than P frame protection, + // and at least as high as received loss + WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr); + _protectionFactorK = static_cast (VCM_MAX(packetLoss, + VCM_MAX(_scaleProtKey * protFactorD, protFactorK))); - //check limit on amount of protection for I frame: 50% is max - if (_protectionFactorK >= plossMax) _protectionFactorK = plossMax - 1; + // check limit on amount of protection for I frame: 50% is max + if (_protectionFactorK >= plossMax) + _protectionFactorK = plossMax - 1; - //Bit cost for NackFec + // Bit cost for NackFec - // NACK cost: based on residual packet loss (since we should only NACK packet not recovered by FEC) + // NACK cost: based on residual packet loss (since we should only NACK packets + // not recovered by FEC) _efficiency = 0.0f; - if (parameters->rtt < rttMax) + if (parameters->rtt < kHighRttNackMs) + { _efficiency = parameters->bitRate * resPacketLoss / (1.0f + resPacketLoss); - - //add FEC cost: ignore I frames for now - float fecRate = static_cast(_protectionFactorD) / 255.0f; - if (fecRate >= 0.0f) - _efficiency += parameters->bitRate * fecRate; - + } else + { + // efficiency based on FEC only + // add FEC cost: ignore I frames for now + float fecRate = static_cast (_protectionFactorD) / 255.0f; + if (fecRate >= 0.0f) + _efficiency += parameters->bitRate * fecRate; + } _score = _efficiency; - //Protection/fec rates obtained above is defined relative to total number of packets (total rate: source+fec) - //FEC in RTP module assumes protection factor is defined relative to source number of packets - //so we should convert the factor to reduce mismatch between mediaOpt suggested rate and the actual rate + // Protection/fec rates obtained above are defined relative to total number of + // packets (total rate: source + fec) FEC in RTP module assumes protection + // factor is defined relative to source number of packets so we should convert + // the factor to reduce mismatch between mediaOpt's rate and the actual one WebRtc_UWord8 codeRate = protFactorK; _protectionFactorK = fecMethod.ConvertFECRate(codeRate); codeRate = protFactorD; @@ -125,34 +143,30 @@ VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) return true; } - -bool -VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, WebRtc_UWord16 rttTime) +bool VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, WebRtc_UWord16 rttTime) { WebRtc_UWord16 rttMax = MaxRttNack(); - //For large RTT, we should rely on some Error Resilience, so we set packetLossEnc = 0 - //for RTT less than the NACK threshold - if (rttTime < rttMax ) - effPacketLoss = 0; //may want a softer transition here + // For large RTT, we should rely on some Error Resilience, so we set + // packetLossEnc = 0 for RTT less than the NACK threshold + if (rttTime < rttMax) + effPacketLoss = 0; //may want a softer transition here - _effectivePacketLoss = effPacketLoss; + _effectivePacketLoss = effPacketLoss; return true; } -bool -VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters) -{ - //Compute the effective packet loss for ER - WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8)(255* parameters->lossPr); +bool VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters) +{ + // Compute the effective packet loss for ER + WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8) (255 * parameters->lossPr); WebRtc_UWord16 rttTime = (WebRtc_UWord16) parameters->rtt; EffectivePacketLoss(effPacketLoss, rttTime); - // - //Compute the NACK bit cost + // Compute the NACK bit cost _efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr); _score = _efficiency; if (parameters->rtt > _NACK_MAX_RTT) @@ -164,145 +178,132 @@ VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters) return true; } - -WebRtc_UWord8 -VCMFecMethod::BoostCodeRateKey(WebRtc_UWord8 packetFrameDelta, WebRtc_UWord8 packetFrameKey) const +WebRtc_UWord8 VCMFecMethod::BoostCodeRateKey(WebRtc_UWord8 packetFrameDelta, + WebRtc_UWord8 packetFrameKey) const { + WebRtc_UWord8 boostRateKey = 2; + // default: ratio scales the FEC protection up for I frames + WebRtc_UWord8 ratio = 1; - WebRtc_UWord8 boostRateKey = 2; - //default: ratio scales the FEC protection up for I frames - WebRtc_UWord8 ratio = 1; + if (packetFrameDelta > 0) + ratio = (WebRtc_Word8) (packetFrameKey / packetFrameDelta); - if (packetFrameDelta > 0) - ratio = (WebRtc_Word8)( packetFrameKey / packetFrameDelta ); + ratio = VCM_MAX(boostRateKey, ratio); - ratio = VCM_MAX(boostRateKey, ratio); - - return ratio; + return ratio; } -WebRtc_UWord8 -VCMFecMethod::ConvertFECRate(WebRtc_UWord8 codeRateRTP) const +WebRtc_UWord8 VCMFecMethod::ConvertFECRate(WebRtc_UWord8 codeRateRTP) const { - return static_cast(VCM_MIN(255,(0.5 + 255.0*codeRateRTP/(float)(255 - codeRateRTP)))); + return static_cast (VCM_MIN(255,(0.5 + 255.0*codeRateRTP/(float)(255 - codeRateRTP)))); } -//AvgRecoveryFEC: average recovery from FEC, assuming random packet loss model -//Computed offline for a range of FEC code parameters and loss rates -float -VCMFecMethod::AvgRecoveryFEC(const VCMProtectionParameters* parameters) const +// AvgRecoveryFEC: average recovery from FEC, assuming random packet loss model +// Computed offline for a range of FEC code parameters and loss rates +float VCMFecMethod::AvgRecoveryFEC(const VCMProtectionParameters* parameters) const { + // Total (avg) bits available per frame: total rate over actual/sent frame rate + // units are kbits/frame + const WebRtc_UWord16 bitRatePerFrame = static_cast (parameters->bitRate + / (parameters->frameRate)); - //Total (avg) bits available per frame: total rate over actual/sent frame rate - //units are kbits/frame - const WebRtc_UWord16 bitRatePerFrame = static_cast(parameters->bitRate/(parameters->frameRate)); + // Total (avg) number of packets per frame (source and fec): + const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8) ((float) bitRatePerFrame * 1000.0 + / (float) (8.0 * _maxPayloadSize) + 0.5); - //Total (avg) number of packets per frame (source and fec): - const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8)((float)bitRatePerFrame*1000.0/(float)(8.0*_maxPayloadSize) + 0.5); - - //parameters for tables + // parameters for tables const WebRtc_UWord8 codeSize = 24; const WebRtc_UWord8 plossMax = 129; const WebRtc_UWord16 maxErTableSize = 38700; - // - // - //Get index for table - const float protectionFactor = (float)_protectionFactorD/(float)255; - WebRtc_UWord8 fecPacketsPerFrame = (WebRtc_UWord8)(0.5 + protectionFactor*avgTotPackets); + // Get index for table + const float protectionFactor = (float) _protectionFactorD / (float) 255; + WebRtc_UWord8 fecPacketsPerFrame = (WebRtc_UWord8) (0.5 + protectionFactor * avgTotPackets); WebRtc_UWord8 sourcePacketsPerFrame = avgTotPackets - fecPacketsPerFrame; - if (fecPacketsPerFrame == 0) - { - return 0.0; //no protection, so avg. recov from FEC == 0 + if (fecPacketsPerFrame == 0) { + return 0.0; // no protection, so avg. recov from FEC == 0 } - //table defined up to codeSizexcodeSize code - if (sourcePacketsPerFrame > codeSize) - { - sourcePacketsPerFrame = codeSize; + // table defined up to codeSizexcodeSize code + if (sourcePacketsPerFrame > codeSize) { + sourcePacketsPerFrame = codeSize; } - //check: protection factor is maxed at 50%, so this should never happen - if (sourcePacketsPerFrame < 1) - { - assert("average number of source packets below 1\n"); + // check: protection factor is maxed at 50%, so this should never happen + if (sourcePacketsPerFrame < 1) { + assert("average number of source packets below 1\n"); } - //index for ER tables: up to codeSizexcodeSize mask - WebRtc_UWord16 codeIndexTable[codeSize*codeSize]; + // index for ER tables: up to codeSizexcodeSize mask + WebRtc_UWord16 codeIndexTable[codeSize * codeSize]; WebRtc_UWord16 k = -1; - for(WebRtc_UWord8 i=1;i<=codeSize;i++) - { - for(WebRtc_UWord8 j=1;j<=i;j++) - { - k += 1; - codeIndexTable[(j-1)*codeSize + i - 1] = k; - } + for (WebRtc_UWord8 i = 1; i <= codeSize; i++) { + for (WebRtc_UWord8 j = 1; j <= i; j++) { + k += 1; + codeIndexTable[(j - 1) * codeSize + i - 1] = k; + } } - const WebRtc_UWord8 lossRate = (WebRtc_UWord8) (255.0*parameters->lossPr + 0.5f); + const WebRtc_UWord8 lossRate = (WebRtc_UWord8) (255.0 * parameters->lossPr + 0.5f); - const WebRtc_UWord16 codeIndex = (fecPacketsPerFrame - 1)*codeSize + (sourcePacketsPerFrame - 1); + const WebRtc_UWord16 codeIndex = (fecPacketsPerFrame - 1) * codeSize + + (sourcePacketsPerFrame - 1); const WebRtc_UWord16 indexTable = codeIndexTable[codeIndex] * plossMax + lossRate; - const WebRtc_UWord16 codeIndex2 = (fecPacketsPerFrame)*codeSize + (sourcePacketsPerFrame); - WebRtc_UWord16 indexTable2 = codeIndexTable[codeIndex2] * plossMax + lossRate; + const WebRtc_UWord16 codeIndex2 = (fecPacketsPerFrame) * codeSize + (sourcePacketsPerFrame); + WebRtc_UWord16 indexTable2 = codeIndexTable[codeIndex2] * plossMax + lossRate; - //checks on table index - if (indexTable >= maxErTableSize) - { - assert("ER table index too large\n"); + // checks on table index + if (indexTable >= maxErTableSize) { + assert("ER table index too large\n"); } - if (indexTable2 >= maxErTableSize) - { - indexTable2 = indexTable; + if (indexTable2 >= maxErTableSize) { + indexTable2 = indexTable; } // - //Get the average effective packet loss recovery from FEC - //this is from tables, computed using random loss model + // Get the average effective packet loss recovery from FEC + // this is from tables, computed using random loss model WebRtc_UWord8 avgFecRecov1 = 0; WebRtc_UWord8 avgFecRecov2 = 0; float avgFecRecov = 0; - if (fecPacketsPerFrame > 0) - { - avgFecRecov1 = VCMAvgFECRecoveryXOR[indexTable]; - avgFecRecov2 = VCMAvgFECRecoveryXOR[indexTable2]; + if (fecPacketsPerFrame > 0) { + avgFecRecov1 = VCMAvgFECRecoveryXOR[indexTable]; + avgFecRecov2 = VCMAvgFECRecoveryXOR[indexTable2]; } - //interpolate over two FEC codes - const float weightRpl = (float)(0.5 + protectionFactor*avgTotPackets) - (float)fecPacketsPerFrame; - avgFecRecov = (float)weightRpl * (float)avgFecRecov2 + (float)(1.0 - weightRpl) * (float)avgFecRecov1; - + // interpolate over two FEC codes + const float weightRpl = (float) (0.5 + protectionFactor * avgTotPackets) + - (float) fecPacketsPerFrame; + avgFecRecov = (float) weightRpl * (float) avgFecRecov2 + (float) (1.0 - weightRpl) + * (float) avgFecRecov1; return avgFecRecov; - } -bool -VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) +bool VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) { - //FEC PROTECTION SETTINGS: varies with packet loss and bitrate + // FEC PROTECTION SETTINGS: varies with packet loss and bitrate const float bitRate = parameters->bitRate; - WebRtc_UWord8 packetLoss = (WebRtc_UWord8)(255* parameters->lossPr); + WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr); - - //Size of tables - const WebRtc_UWord16 maxFecTableSize = 6450; - //Parameters for range of rate and packet loss for tables + // Size of tables + const WebRtc_UWord16 maxFecTableSize = 6450; + // Parameters for range of rate and packet loss for tables const WebRtc_UWord8 ratePar1 = 5; const WebRtc_UWord8 ratePar2 = 49; const WebRtc_UWord8 plossMax = 129; // - //Just for testing: for the case where we randomly lose slices instead of RTP packets and use SingleMode packetization in RTP module - //const WebRtc_UWord16 slice_size = 3000/6; //corresponds to rate=1000k with 4 cores + // Just for testing: for the case where we randomly lose slices instead of + // RTP packets and use SingleMode packetization in RTP module + // const WebRtc_UWord16 slice_size = 3000/6; //corresponds to rate=1000k with 4 cores //float slice_mtu = (float)_maxPayloadSize/(float)slice_size; const float slice_mtu = 1.0; @@ -310,100 +311,95 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) //Total (avg) bits available per frame: total rate over actual/sent frame rate //units are kbits/frame - const WebRtc_UWord16 bitRatePerFrame = static_cast(slice_mtu*bitRate/(parameters->frameRate)); + const WebRtc_UWord16 bitRatePerFrame = static_cast (slice_mtu * bitRate + / (parameters->frameRate)); //Total (avg) number of packets per frame (source and fec): - const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8)((float)bitRatePerFrame*1000.0/(float)(8.0*_maxPayloadSize) + 0.5); + const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8) ((float) bitRatePerFrame * 1000.0 + / (float) (8.0 * _maxPayloadSize) + 0.5); - //TODO(marpan): Tune model for FEC Protection. - //Better modulation of protection with available bits/frame (or avgTotpackets) using weight factors - //FEC Tables include this effect already, but need to tune model off-line + // TODO(marpan): Tune model for FEC Protection. + // Better modulation of protection with available bits/frame + // (or avgTotpackets) using weight factors + // FEC Tables include this effect already, but need to tune model off-line float weight1 = 0.5; float weight2 = 0.5; - if (avgTotPackets > 4) - { - weight1 = 1.0; - weight2 = 0.; + if (avgTotPackets > 4) { + weight1 = 1.0; + weight2 = 0.; } - if (avgTotPackets > 6) - { - weight1 = 1.5; - weight2 = 0.; - } - // - - //Fec rate parameters: for P and I frame - WebRtc_UWord8 codeRateDelta = 0; - WebRtc_UWord8 codeRateKey = 0; - - - //Get index for new table: the FEC protection depends on the (avergare) available bits/frame - //the range on the rate index corresponds to rates (bps) from 200k to 8000k, for 30fps - WebRtc_UWord8 rateIndexTable = (WebRtc_UWord8) VCM_MAX(VCM_MIN((bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0); - - // Restrict packet loss range to 50 for now%: current tables defined only up to 50% - if (packetLoss >= plossMax) - { - packetLoss = plossMax - 1; - } - WebRtc_UWord16 indexTable = rateIndexTable * plossMax + packetLoss; - - //check on table index - if (indexTable >= maxFecTableSize) - { - assert("FEC table index too large\n"); + if (avgTotPackets > 6) { + weight1 = 1.5; + weight2 = 0.; } // - //For Key frame: effectively at a higher rate, so we scale/boost the rate index. - //the boost factor may depend on several factors: ratio of packet number of I to P frames, how much protection placed on P frames, etc. - //default is 2 - const WebRtc_UWord8 packetFrameDelta = (WebRtc_UWord8)(0.5 + parameters->packetsPerFrame); + // Fec rate parameters: for P and I frame + WebRtc_UWord8 codeRateDelta = 0; + WebRtc_UWord8 codeRateKey = 0; + + // Get index for new table: the FEC protection depends on the (average) available bits/frame + // the range on the rate index corresponds to rates (bps) from 200k to 8000k, for 30fps + WebRtc_UWord8 rateIndexTable = + (WebRtc_UWord8) VCM_MAX(VCM_MIN((bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0); + + // Restrict packet loss range to 50 for now%: current tables defined only up to 50% + if (packetLoss >= plossMax) { + packetLoss = plossMax - 1; + } + WebRtc_UWord16 indexTable = rateIndexTable * plossMax + packetLoss; + + // check on table index + if (indexTable >= maxFecTableSize) { + assert("FEC table index too large\n"); + } + // + + // For Key frame: effectively at a higher rate, so we scale/boost the rate + // index. The boost factor may depend on several factors: ratio of packet + // number of I to P frames, how much protection placed on P frames, etc. + // default is 2 + const WebRtc_UWord8 packetFrameDelta = (WebRtc_UWord8) (0.5 + parameters->packetsPerFrame); const WebRtc_UWord8 packetFrameKey = (WebRtc_UWord8) (0.5 + parameters->packetsPerFrameKey); const WebRtc_UWord8 boostKey = BoostCodeRateKey(packetFrameDelta, packetFrameKey); - rateIndexTable = (WebRtc_UWord8) VCM_MAX(VCM_MIN(1+(boostKey*bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0); + rateIndexTable + = (WebRtc_UWord8) VCM_MAX(VCM_MIN(1+(boostKey*bitRatePerFrame-ratePar1)/ratePar1,ratePar2),0); WebRtc_UWord16 indexTableKey = rateIndexTable * plossMax + packetLoss; indexTableKey = VCM_MIN(indexTableKey, maxFecTableSize); - codeRateDelta = VCMCodeRateXORTable[indexTable]; //protection factor for P fra + codeRateDelta = VCMCodeRateXORTable[indexTable]; //protection factor for P frame codeRateKey = VCMCodeRateXORTable[indexTableKey]; //protection factor for I frame //average with minimum protection level given by (average) total number of packets - if (packetLoss > 0) - { - codeRateDelta = static_cast((weight1*(float)codeRateDelta + weight2*255.0/(float)avgTotPackets)); + if (packetLoss > 0) { + codeRateDelta = static_cast ((weight1 * (float) codeRateDelta + weight2 * 255.0 + / (float) avgTotPackets)); } //check limit on amount of protection for P frame; 50% is max - if (codeRateDelta >= plossMax) - { - codeRateDelta = plossMax - 1; + if (codeRateDelta >= plossMax) { + codeRateDelta = plossMax - 1; } //make sure I frame protection is at least larger than P frame protection, and at least as high as received loss - codeRateKey = static_cast(VCM_MAX(packetLoss,VCM_MAX(_scaleProtKey*codeRateDelta, codeRateKey))); + codeRateKey + = static_cast (VCM_MAX(packetLoss,VCM_MAX(_scaleProtKey*codeRateDelta, codeRateKey))); //check limit on amount of protection for I frame: 50% is max - if (codeRateKey >= plossMax) - { - codeRateKey = plossMax - 1; + if (codeRateKey >= plossMax) { + codeRateKey = plossMax - 1; } _protectionFactorK = codeRateKey; _protectionFactorD = codeRateDelta; // DONE WITH FEC PROTECTION SETTINGS - - return true; } - -bool -VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters) +bool VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters) { - // ER SETTINGS: //Effective packet loss to encoder is based on RPL (residual packet loss) //this is a soft setting based on degree of FEC protection @@ -411,7 +407,7 @@ VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters) //note: received/input packet loss may be filtered according to FilteredLoss //The input packet loss: - WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8)(255*parameters->lossPr); + WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8) (255 * parameters->lossPr); float scaleErRS = 0.5; float scaleErXOR = 0.5; @@ -426,25 +422,22 @@ VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters) avgFecRecov = AvgRecoveryFEC(parameters); //Residual Packet Loss: - _residualPacketLoss = (float)(effPacketLoss - avgFecRecov)/(float)255.0; - + _residualPacketLoss = (float) (effPacketLoss - avgFecRecov) / (float) 255.0; //Effective Packet Loss for encoder: _effectivePacketLoss = 0; - if (effPacketLoss > 0) - { - _effectivePacketLoss = VCM_MAX((effPacketLoss - (WebRtc_UWord8)(scaleEr*avgFecRecov)),static_cast(minErLevel*255)); + if (effPacketLoss > 0) { + _effectivePacketLoss = VCM_MAX((effPacketLoss - + (WebRtc_UWord8)(scaleEr * avgFecRecov)), + static_cast(minErLevel * 255)); } - // DONE WITH ER SETTING - return true; + return true; } - -bool -VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) +bool VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) { // Compute the protection factor @@ -453,27 +446,22 @@ VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) // Compute the effective packet loss EffectivePacketLoss(parameters); - // Compute the bit cost // Ignore key frames for now. - float fecRate = static_cast(_protectionFactorD) / 255.0f; - if (fecRate >= 0.0f) - { - // use this formula if the fecRate (protection factor) is defined relative to number of source packets - // this is the case for the previous tables: - // _efficiency = parameters->bitRate * ( 1.0 - 1.0 / (1.0 + fecRate)); + float fecRate = static_cast (_protectionFactorD) / 255.0f; + if (fecRate >= 0.0f) { + // use this formula if the fecRate (protection factor) is defined relative to number of source packets + // this is the case for the previous tables: + // _efficiency = parameters->bitRate * ( 1.0 - 1.0 / (1.0 + fecRate)); - // in the new tables, the fecRate is defined relative to total number of packets (total rate), - // so overhead cost is: - _efficiency = parameters->bitRate * fecRate; - } - else - { - _efficiency = 0.0f; + // in the new tables, the fecRate is defined relative to total number of packets (total rate), + // so overhead cost is: + _efficiency = parameters->bitRate * fecRate; + } else { + _efficiency = 0.0f; } _score = _efficiency; - // Protection/fec rates obtained above is defined relative to total number of packets (total rate: source+fec) // FEC in RTP module assumes protection factor is defined relative to source number of packets // so we should convert the factor to reduce mismatch between mediaOpt suggested rate and the actual rate @@ -483,28 +471,24 @@ VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) return true; } -bool -VCMIntraReqMethod::UpdateParameters(const VCMProtectionParameters* parameters) +bool VCMIntraReqMethod::UpdateParameters(const VCMProtectionParameters* parameters) { float packetRate = parameters->packetsPerFrame * parameters->frameRate; // Assume that all lost packets cohere to different frames float lossRate = parameters->lossPr * packetRate; - if (parameters->keyFrameSize <= 1e-3) - { - _score = FLT_MAX; - return false; + if (parameters->keyFrameSize <= 1e-3) { + _score = FLT_MAX; + return false; } _efficiency = lossRate * parameters->keyFrameSize; _score = _efficiency; - if (parameters->lossPr >= 1.0f / parameters->keyFrameSize || parameters->rtt > _IREQ_MAX_RTT) - { - return false; + if (parameters->lossPr >= 1.0f / parameters->keyFrameSize || parameters->rtt > _IREQ_MAX_RTT) { + return false; } return true; } -bool -VCMPeriodicIntraMethod::UpdateParameters(const VCMProtectionParameters* /*parameters*/) +bool VCMPeriodicIntraMethod::UpdateParameters(const VCMProtectionParameters* /*parameters*/) { // Periodic I-frames. The last thing we want to use. _efficiency = 0.0f; @@ -512,21 +496,18 @@ VCMPeriodicIntraMethod::UpdateParameters(const VCMProtectionParameters* /*parame return true; } -bool -VCMMbIntraRefreshMethod::UpdateParameters(const VCMProtectionParameters* parameters) +bool VCMMbIntraRefreshMethod::UpdateParameters(const VCMProtectionParameters* parameters) { // Assume optimal for now. _efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr); _score = _efficiency; - if (parameters->bitRate < _MBREF_MIN_BITRATE) - { - return false; + if (parameters->bitRate < _MBREF_MIN_BITRATE) { + return false; } return true; } -WebRtc_UWord16 -VCMNackMethod::MaxRttNack() const +WebRtc_UWord16 VCMNackMethod::MaxRttNack() const { return _NACK_MAX_RTT; } @@ -536,63 +517,52 @@ VCMLossProtectionLogic::~VCMLossProtectionLogic() ClearLossProtections(); } -void -VCMLossProtectionLogic::ClearLossProtections() +void VCMLossProtectionLogic::ClearLossProtections() { ListItem *item; - while ((item = _availableMethods.First()) != 0) - { - VCMProtectionMethod *method = static_cast(item->GetItem()); - if (method != NULL) - { - delete method; - } - _availableMethods.PopFront(); + while ((item = _availableMethods.First()) != 0) { + VCMProtectionMethod *method = static_cast (item->GetItem()); + if (method != NULL) { + delete method; + } + _availableMethods.PopFront(); } _selectedMethod = NULL; } - bool -VCMLossProtectionLogic::AddMethod(VCMProtectionMethod *newMethod) +bool VCMLossProtectionLogic::AddMethod(VCMProtectionMethod *newMethod) { VCMProtectionMethod *method; ListItem *item; - if (newMethod == NULL) - { - return false; + if (newMethod == NULL) { + return false; } - for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) - { - method = static_cast(item->GetItem()); - if (method != NULL && method->Type() == newMethod->Type()) - { - return false; - } + for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) { + method = static_cast (item->GetItem()); + if (method != NULL && method->Type() == newMethod->Type()) { + return false; + } } _availableMethods.PushBack(newMethod); return true; } -bool -VCMLossProtectionLogic::RemoveMethod(VCMProtectionMethodEnum methodType) +bool VCMLossProtectionLogic::RemoveMethod(VCMProtectionMethodEnum methodType) { VCMProtectionMethod *method; ListItem *item; bool foundAndRemoved = false; - for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) - { - method = static_cast(item->GetItem()); - if (method != NULL && method->Type() == methodType) - { - if (_selectedMethod != NULL && _selectedMethod->Type() == method->Type()) - { - _selectedMethod = NULL; - } - _availableMethods.Erase(item); - item = NULL; - delete method; - foundAndRemoved = true; + for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) { + method = static_cast (item->GetItem()); + if (method != NULL && method->Type() == methodType) { + if (_selectedMethod != NULL && _selectedMethod->Type() == method->Type()) { + _selectedMethod = NULL; } + _availableMethods.Erase(item); + item = NULL; + delete method; + foundAndRemoved = true; + } } return foundAndRemoved; } @@ -602,224 +572,187 @@ VCMLossProtectionLogic::FindMethod(VCMProtectionMethodEnum methodType) const { VCMProtectionMethod *method; ListItem *item; - for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) - { - method = static_cast(item->GetItem()); - if (method != NULL && method->Type() == methodType) - { - return method; - } + for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) { + method = static_cast (item->GetItem()); + if (method != NULL && method->Type() == methodType) { + return method; + } } return NULL; } -float -VCMLossProtectionLogic::HighestOverhead() const +float VCMLossProtectionLogic::HighestOverhead() const { VCMProtectionMethod *method; ListItem *item; float highestOverhead = 0.0f; - for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) - { - method = static_cast(item->GetItem()); - if (method != NULL && method->RequiredBitRate() > highestOverhead) - { - highestOverhead = method->RequiredBitRate(); - } + for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) { + method = static_cast (item->GetItem()); + if (method != NULL && method->RequiredBitRate() > highestOverhead) { + highestOverhead = method->RequiredBitRate(); + } } return highestOverhead; } -void -VCMLossProtectionLogic::UpdateRtt(WebRtc_UWord32 rtt) +void VCMLossProtectionLogic::UpdateRtt(WebRtc_UWord32 rtt) { _rtt = rtt; } -void -VCMLossProtectionLogic::UpdateResidualPacketLoss(float residualPacketLoss) -{ +void VCMLossProtectionLogic::UpdateResidualPacketLoss(float residualPacketLoss) { _residualPacketLoss = residualPacketLoss; } -void -VCMLossProtectionLogic::UpdateFecType(VCMFecTypes fecType) +void VCMLossProtectionLogic::UpdateFecType(VCMFecTypes fecType) { _fecType = fecType; } -void -VCMLossProtectionLogic::UpdateLossPr(WebRtc_UWord8 lossPr255) +void VCMLossProtectionLogic::UpdateLossPr(WebRtc_UWord8 lossPr255) { - WebRtc_UWord32 now = static_cast(VCMTickTime::MillisecondTimestamp()); + WebRtc_UWord32 now = static_cast (VCMTickTime::MillisecondTimestamp()); UpdateMaxLossHistory(lossPr255, now); - _lossPr255.Apply(static_cast(now - _lastPrUpdateT), static_cast(lossPr255)); + _lossPr255.Apply(static_cast (now - _lastPrUpdateT), static_cast (lossPr255)); _lastPrUpdateT = now; _lossPr = _lossPr255.Value() / 255.0f; } -void -VCMLossProtectionLogic::UpdateMaxLossHistory(WebRtc_UWord8 lossPr255, WebRtc_Word64 now) +void VCMLossProtectionLogic::UpdateMaxLossHistory(WebRtc_UWord8 lossPr255, WebRtc_Word64 now) { - if (_lossPrHistory[0].timeMs >= 0 && - now - _lossPrHistory[0].timeMs < kLossPrShortFilterWinMs) - { - if (lossPr255 > _shortMaxLossPr255) - { - _shortMaxLossPr255 = lossPr255; - } + if (_lossPrHistory[0].timeMs >= 0 && now - _lossPrHistory[0].timeMs < kLossPrShortFilterWinMs) { + if (lossPr255 > _shortMaxLossPr255) { + _shortMaxLossPr255 = lossPr255; } - else - { - // Only add a new value to the history once a second - if(_lossPrHistory[0].timeMs == -1) - { - // First, no shift - _shortMaxLossPr255 = lossPr255; - } - else - { - // Shift - for(WebRtc_Word32 i = (kLossPrHistorySize - 2); i >= 0 ; i--) - { - _lossPrHistory[i+1].lossPr255 = _lossPrHistory[i].lossPr255; - _lossPrHistory[i+1].timeMs = _lossPrHistory[i].timeMs; - } - } - if (_shortMaxLossPr255 == 0) - { - _shortMaxLossPr255 = lossPr255; - } - - _lossPrHistory[0].lossPr255 = _shortMaxLossPr255; - _lossPrHistory[0].timeMs = now; - _shortMaxLossPr255 = 0; - + } else { + // Only add a new value to the history once a second + if (_lossPrHistory[0].timeMs == -1) { + // First, no shift + _shortMaxLossPr255 = lossPr255; + } else { + // Shift + for (WebRtc_Word32 i = (kLossPrHistorySize - 2); i >= 0; i--) { + _lossPrHistory[i + 1].lossPr255 = _lossPrHistory[i].lossPr255; + _lossPrHistory[i + 1].timeMs = _lossPrHistory[i].timeMs; + } } + if (_shortMaxLossPr255 == 0) { + _shortMaxLossPr255 = lossPr255; + } + + _lossPrHistory[0].lossPr255 = _shortMaxLossPr255; + _lossPrHistory[0].timeMs = now; + _shortMaxLossPr255 = 0; + + } } -WebRtc_UWord8 -VCMLossProtectionLogic::MaxFilteredLossPr(WebRtc_Word64 nowMs) const +WebRtc_UWord8 VCMLossProtectionLogic::MaxFilteredLossPr(WebRtc_Word64 nowMs) const { WebRtc_UWord8 maxFound = _shortMaxLossPr255; - if (_lossPrHistory[0].timeMs == -1) - { - return maxFound; + if (_lossPrHistory[0].timeMs == -1) { + return maxFound; } - for (WebRtc_Word32 i=0; i < kLossPrHistorySize; i++) - { - if (_lossPrHistory[i].timeMs == -1) - { - break; - } - if (nowMs - _lossPrHistory[i].timeMs > kLossPrHistorySize * kLossPrShortFilterWinMs) - { - // This sample (and all samples after this) is too old - break; - } - if (_lossPrHistory[i].lossPr255 > maxFound) - { - // This sample is the largest one this far into the history - maxFound = _lossPrHistory[i].lossPr255; - } + for (WebRtc_Word32 i = 0; i < kLossPrHistorySize; i++) { + if (_lossPrHistory[i].timeMs == -1) { + break; + } + if (nowMs - _lossPrHistory[i].timeMs > kLossPrHistorySize * kLossPrShortFilterWinMs) { + // This sample (and all samples after this) is too old + break; + } + if (_lossPrHistory[i].lossPr255 > maxFound) { + // This sample is the largest one this far into the history + maxFound = _lossPrHistory[i].lossPr255; + } } return maxFound; } -WebRtc_UWord8 -VCMLossProtectionLogic::FilteredLoss() const +WebRtc_UWord8 VCMLossProtectionLogic::FilteredLoss() const { //take the average received loss //return static_cast(_lossPr255.Value() + 0.5f); + //TODO: Update for hybrid //take the windowed max of the received loss - if (_selectedMethod != NULL && _selectedMethod->Type() == kFEC) - { - return MaxFilteredLossPr(static_cast(VCMTickTime::MillisecondTimestamp())); - } - else - { - return static_cast(_lossPr255.Value() + 0.5); + if (_selectedMethod != NULL && _selectedMethod->Type() == kFEC) { + return MaxFilteredLossPr(static_cast (VCMTickTime::MillisecondTimestamp())); + } else { + return static_cast (_lossPr255.Value() + 0.5); } } -void -VCMLossProtectionLogic::UpdateFilteredLossPr(WebRtc_UWord8 packetLossEnc) +void VCMLossProtectionLogic::UpdateFilteredLossPr(WebRtc_UWord8 packetLossEnc) { - _lossPr = (float)packetLossEnc/(float)255.0; + _lossPr = (float) packetLossEnc / (float) 255.0; } -void -VCMLossProtectionLogic::UpdateBitRate(float bitRate) +void VCMLossProtectionLogic::UpdateBitRate(float bitRate) { _bitRate = bitRate; } -void -VCMLossProtectionLogic::UpdatePacketsPerFrame(float nPackets) +void VCMLossProtectionLogic::UpdatePacketsPerFrame(float nPackets) { - WebRtc_UWord32 now = static_cast(VCMTickTime::MillisecondTimestamp()); - _packetsPerFrame.Apply(static_cast(now - _lastPacketPerFrameUpdateT), nPackets); + WebRtc_UWord32 now = static_cast (VCMTickTime::MillisecondTimestamp()); + _packetsPerFrame.Apply(static_cast (now - _lastPacketPerFrameUpdateT), nPackets); _lastPacketPerFrameUpdateT = now; } -void -VCMLossProtectionLogic::UpdatePacketsPerFrameKey(float nPackets) +void VCMLossProtectionLogic::UpdatePacketsPerFrameKey(float nPackets) { - WebRtc_UWord32 now = static_cast(VCMTickTime::MillisecondTimestamp()); - _packetsPerFrameKey.Apply(static_cast(now - _lastPacketPerFrameUpdateTKey), nPackets); + WebRtc_UWord32 now = static_cast (VCMTickTime::MillisecondTimestamp()); + _packetsPerFrameKey.Apply(static_cast (now - _lastPacketPerFrameUpdateTKey), nPackets); _lastPacketPerFrameUpdateTKey = now; } -void -VCMLossProtectionLogic::UpdateKeyFrameSize(float keyFrameSize) +void VCMLossProtectionLogic::UpdateKeyFrameSize(float keyFrameSize) { _keyFrameSize = keyFrameSize; } -bool -VCMLossProtectionLogic::UpdateMethod(VCMProtectionMethod *newMethod /*=NULL */) +bool VCMLossProtectionLogic::UpdateMethod(VCMProtectionMethod *newMethod /*=NULL */) { - _currentParameters.rtt = _rtt; - _currentParameters.lossPr = _lossPr; + _currentParameters.rtt = _rtt; + _currentParameters.lossPr = _lossPr; _currentParameters.bitRate = _bitRate; - _currentParameters.frameRate = _frameRate; //should this be named actual frame rate? + _currentParameters.frameRate = _frameRate; //should this be named actual frame rate? _currentParameters.keyFrameSize = _keyFrameSize; _currentParameters.fecRateDelta = _fecRateDelta; - _currentParameters.fecRateKey = _fecRateKey; + _currentParameters.fecRateKey = _fecRateKey; _currentParameters.packetsPerFrame = _packetsPerFrame.Value(); _currentParameters.packetsPerFrameKey = _packetsPerFrameKey.Value(); _currentParameters.residualPacketLoss = _residualPacketLoss; _currentParameters.fecType = _fecType; - if (newMethod == NULL) - { - //_selectedMethod = _bestNotOkMethod = NULL; - VCMProtectionMethod *method; - ListItem *item; - for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) - { - method = static_cast(item->GetItem()); - if (method != NULL) - { - if (method->Type() == kFEC) - { - _selectedMethod = method; - } - method->UpdateParameters(&_currentParameters); - } - } - if (_selectedMethod != NULL && _selectedMethod->Type() != kFEC) - { + if (newMethod == NULL) { + //_selectedMethod = _bestNotOkMethod = NULL; + VCMProtectionMethod *method; + ListItem *item; + for (item = _availableMethods.First(); item != NULL; item = _availableMethods.Next(item)) { + method = static_cast (item->GetItem()); + if (method != NULL) { + if (method->Type() == kFEC) { _selectedMethod = method; + } + if (method->Type() == kNACK) { + _selectedMethod = method; + } + if (method->Type() == kNackFec) { + _selectedMethod = method; + } + method->UpdateParameters(&_currentParameters); } - } - else - { - _selectedMethod = newMethod; - _selectedMethod->UpdateParameters(&_currentParameters); + } + if (_selectedMethod != NULL && _selectedMethod->Type() != kFEC) { + _selectedMethod = method; + } + } else { + _selectedMethod = newMethod; + _selectedMethod->UpdateParameters(&_currentParameters); } return true; } @@ -830,18 +763,16 @@ VCMLossProtectionLogic::SelectedMethod() const return _selectedMethod; } -void -VCMLossProtectionLogic::Reset() +void VCMLossProtectionLogic::Reset() { - _lastPrUpdateT = static_cast(VCMTickTime::MillisecondTimestamp()); - _lastPacketPerFrameUpdateT = static_cast(VCMTickTime::MillisecondTimestamp()); + _lastPrUpdateT = static_cast (VCMTickTime::MillisecondTimestamp()); + _lastPacketPerFrameUpdateT = static_cast (VCMTickTime::MillisecondTimestamp()); _lossPr255.Reset(0.9999f); _packetsPerFrame.Reset(0.9999f); _fecRateDelta = _fecRateKey = 0; - for (WebRtc_Word32 i=0; i < kLossPrHistorySize; i++) - { - _lossPrHistory[i].lossPr255 = 0; - _lossPrHistory[i].timeMs = -1; + for (WebRtc_Word32 i = 0; i < kLossPrHistorySize; i++) { + _lossPrHistory[i].lossPr255 = 0; + _lossPrHistory[i].timeMs = -1; } _shortMaxLossPr255 = 0; ClearLossProtections(); diff --git a/modules/video_coding/main/source/media_opt_util.h b/modules/video_coding/main/source/media_opt_util.h index 1faa4eaf5..00f00c17a 100644 --- a/modules/video_coding/main/source/media_opt_util.h +++ b/modules/video_coding/main/source/media_opt_util.h @@ -34,6 +34,13 @@ enum VCMFecTypes kXORFec }; +// Thresholds for hybrid NACK/FEC +// common to media optimization and the jitter buffer. +enum HybridNackTH { + kHighRttNackMs = 100, + kLowRttNackMs = 20 +}; + struct VCMProtectionParameters { VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), packetsPerFrame(0), @@ -134,16 +141,16 @@ public: WebRtc_UWord8 _effectivePacketLoss; WebRtc_UWord8 _protectionFactorK; WebRtc_UWord8 _protectionFactorD; - float _residualPacketLoss; - float _scaleProtKey; + float _residualPacketLoss; + float _scaleProtKey; WebRtc_Word32 _maxPayloadSize; protected: - float _efficiency; - float _score; + float _efficiency; + float _score; private: - const enum VCMProtectionMethodEnum _type; + const enum VCMProtectionMethodEnum _type; }; diff --git a/modules/video_coding/main/source/media_optimization.cc b/modules/video_coding/main/source/media_optimization.cc index 0586e670a..b4ea203c7 100644 --- a/modules/video_coding/main/source/media_optimization.cc +++ b/modules/video_coding/main/source/media_optimization.cc @@ -100,7 +100,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, VCMFecTypes fecType = kXORFec; // generic FEC _lossProtLogic->UpdateFecType(fecType); - //Get frame rate for encoder: this is the actual/sent frame rate + // Get frame rate for encoder: this is the actual/sent frame rate float actualFrameRate = SentFrameRate(); // sanity @@ -109,13 +109,16 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, actualFrameRate = 1.0; } - // Update frame rate for the loss protection logic class: frame rate should be the actual/sent rate + // Update frame rate for the loss protection logic class: frame rate should + // be the actual/sent rate _lossProtLogic->UpdateFrameRate(actualFrameRate); _fractionLost = fractionLost; - // The effective packet loss may be the received loss or filtered, i.e., average or max filter may be used. - //We should think about which filter is appropriate for low/high bit rates, low/high loss rates, etc. + // The effective packet loss may be the received loss or filtered, i.e., + // average or max filter may be used. + // We should think about which filter is appropriate for low/high bit rates, + // low/high loss rates, etc. WebRtc_UWord8 packetLossEnc = _lossProtLogic->FilteredLoss(); //For now use the filtered loss for computing the robustness settings @@ -124,46 +127,48 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, // Rate cost of the protection methods _lossProtOverhead = 0; - if(selectedMethod) + if (selectedMethod && (selectedMethod->Type() == kFEC || + selectedMethod->Type() == kNackFec )) { - //Update method will compute the robustness settings for the given protection method and the overhead cost - //the protection method is set by the user via SetVideoProtection. - //The robustness settings are: the effecitve packet loss for ER and the FEC protection settings + // Update method will compute the robustness settings for the given + // protection method and the overhead cost + // the protection method is set by the user via SetVideoProtection. + // The robustness settings are: the effective packet loss for ER and the + // FEC protection settings _lossProtLogic->UpdateMethod(); - //Get the code rate for Key frames + // Get the code rate for Key frames const WebRtc_UWord8 codeRateKeyRTP = selectedMethod->RequiredProtectionFactorK(); - //Get the code rate for Delta frames + // Get the code rate for Delta frames const WebRtc_UWord8 codeRateDeltaRTP = selectedMethod->RequiredProtectionFactorD(); - //Get the effective packet loss for ER + // Get the effective packet loss for ER packetLossEnc = selectedMethod->RequiredPacketLossER(); - // Get the bit cost of protection method - _lossProtOverhead = static_cast(_lossProtLogic->HighestOverhead() + 0.5f); - - //NACK is on for NACK and NackFec protection method: off for FEC method - bool nackStatus = true; - if (selectedMethod->Type() == kFEC) - { - nackStatus = false; - } + // NACK is on for NACK and NackFec protection method: off for FEC method + bool nackStatus = (selectedMethod->Type() == kNackFec || + selectedMethod->Type() == kNACK); if(_videoProtectionCallback) { - _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP ,codeRateKeyRTP, nackStatus); + _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP, + codeRateKeyRTP, + nackStatus); } - } + // Get the bit cost of protection method + _lossProtOverhead = static_cast(_lossProtLogic->HighestOverhead() + 0.5f); + // Update effective packet loss for encoder: note: fractionLost was passed as reference fractionLost = packetLossEnc; WebRtc_UWord32 nackBitRate=0; if(selectedMethod && _lossProtLogic->FindMethod(kNACK) != NULL) { + // TODO(mikhal): update frame dropper with bit rate including both nack and fec // Make sure we don't over-use the channel momentarily. This is // necessary for NACK since it can be very bursty. nackBitRate = (_lastBitRate * fractionLost) / 255; @@ -178,7 +183,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, _frameDropper->SetRates(static_cast(bitRate - _lossProtOverhead), 0); } - //This may be used for UpdateEncoderBitRate: lastBitRate is total rate, before compensation + // This may be used for UpdateEncoderBitRate: lastBitRate is total rate, + // before compensation _lastBitRate = _targetBitRate; //Source coding rate: total rate - protection overhead @@ -187,7 +193,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, if (_enableQm) { //Update QM with rates - _qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,_incomingFrameRate); + _qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps, _incomingFrameRate); //Check for QM selection bool selectQM = checkStatusForQMchange(); if (selectQM) @@ -203,7 +209,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, bool VCMMediaOptimization::DropFrame() { - _frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f)); // leak appropriate number of bytes + // leak appropriate number of bytes + _frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f)); return _frameDropper->DropFrame(); } @@ -285,7 +292,7 @@ VCMMediaOptimization::EnableNack(bool enable) bool VCMMediaOptimization::IsNackEnabled() { - return (_lossProtLogic->FindMethod(kFEC) != NULL); + return (_lossProtLogic->FindMethod(kNACK) != NULL); } void @@ -490,7 +497,7 @@ WebRtc_Word32 VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback *videoQMSettings) { _videoQMSettingsCallback = videoQMSettings; - //Callback setting controls QM + // Callback setting controls QM if (_videoQMSettingsCallback != NULL) { _enableQm = true; @@ -535,7 +542,7 @@ VCMMediaOptimization::SelectQuality() // Check for updates to spatial/temporal modes QMUpdate(qm); - //Reset all the rate and related frame counters quantities + // Reset all the rate and related frame counters quantities _qms->ResetRates(); // Reset counters @@ -558,8 +565,10 @@ VCMMediaOptimization::checkStatusForQMchange() bool status = true; - // Check that we do not call QMSelect too often, and that we waited some time (to sample the metrics) from the event lastChangeTime - // lastChangeTime is the time where user changed the size/rate/frame rate (via SetEncodingData) + // Check that we do not call QMSelect too often, and that we waited some time + // (to sample the metrics) from the event lastChangeTime + // lastChangeTime is the time where user changed the size/rate/frame rate + // (via SetEncodingData) WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); if ((now - _lastQMUpdateTime) < kQmMinIntervalMs || (now - _lastChangeTime) < kQmMinIntervalMs) @@ -574,7 +583,7 @@ VCMMediaOptimization::checkStatusForQMchange() bool VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) { - //Check for no change + // Check for no change if (qm->spatialHeightFact == 1 && qm->spatialWidthFact == 1 && qm->temporalFact == 1) @@ -582,26 +591,26 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) return false; } - //Content metrics hold native values + // Content metrics hold native values VideoContentMetrics* cm = _content->Data(); - //Temporal + // Temporal WebRtc_UWord32 frameRate = static_cast(_incomingFrameRate + 0.5f); - //Check if go back up in temporal resolution + // Check if go back up in temporal resolution if (qm->temporalFact == 0) { frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; } - //go down in temporal resolution + // go down in temporal resolution else { frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); } - //Spatial + // Spatial WebRtc_UWord32 height = _codecHeight; WebRtc_UWord32 width = _codecWidth; - //Check if go back up in spatial resolution + // Check if go back up in spatial resolution if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) { height = cm->nativeHeight; @@ -617,7 +626,7 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) "Quality Mode Update: W = %d, H = %d, FR = %f", width, height, frameRate); - //Update VPM with new target frame rate and size + // Update VPM with new target frame rate and size _videoQMSettingsCallback->SetVideoQMSettings(frameRate, width, height); return true; diff --git a/modules/video_coding/main/source/receiver.cc b/modules/video_coding/main/source/receiver.cc index ae812d03a..113d878e5 100644 --- a/modules/video_coding/main/source/receiver.cc +++ b/modules/video_coding/main/source/receiver.cc @@ -31,7 +31,6 @@ _master(master), _jitterBuffer(vcmId, receiverId, master), _timing(timing), _renderWaitEvent(*new VCMEvent()), -_nackMode(kNoNack), _state(kPassive) { } @@ -164,6 +163,7 @@ VCMReceiver::InsertPacket(const VCMPacket& packet, } // Insert packet into jitter buffer + // both data and empty packets const VCMFrameBufferEnum ret = _jitterBuffer.InsertPacket(buffer, packet); if (ret < 0) @@ -178,7 +178,8 @@ VCMReceiver::InsertPacket(const VCMPacket& packet, } VCMEncodedFrame* -VCMReceiver::FrameForDecoding(WebRtc_UWord16 maxWaitTimeMs, WebRtc_Word64& nextRenderTimeMs, bool renderTiming, VCMReceiver* dualReceiver) +VCMReceiver::FrameForDecoding(WebRtc_UWord16 maxWaitTimeMs, WebRtc_Word64& nextRenderTimeMs, + bool renderTiming, VCMReceiver* dualReceiver) { // No need to enter the critical section here since the jitter buffer // is thread-safe. @@ -348,20 +349,7 @@ void VCMReceiver::SetNackMode(VCMNackMode nackMode) { CriticalSectionScoped cs(_critSect); - _nackMode = nackMode; - switch (_nackMode) - { - case kNackInfinite: - { - _jitterBuffer.SetNackStatus(true); - break; - } - case kNoNack: - { - _jitterBuffer.SetNackStatus(false); - break; - } - } + _jitterBuffer.SetNackMode(nackMode); if (!_master) { _state = kPassive; // The dual decoder defaults to passive @@ -372,7 +360,7 @@ VCMNackMode VCMReceiver::NackMode() const { CriticalSectionScoped cs(_critSect); - return _nackMode; + return _jitterBuffer.GetNackMode(); } VCMNackStatus @@ -418,14 +406,6 @@ void VCMReceiver::CopyJitterBufferStateFromReceiver(const VCMReceiver& receiver) { _jitterBuffer = receiver._jitterBuffer; - - { - CriticalSectionScoped cs(_critSect); - if (_nackMode != kNoNack) - { - _jitterBuffer.SetNackStatus(true); - } - } } VCMReceiverState @@ -447,7 +427,7 @@ VCMReceiver::UpdateState(VCMReceiverState newState) void VCMReceiver::UpdateState(VCMEncodedFrame& frame) { - if (_nackMode == kNoNack) + if (_jitterBuffer.GetNackMode() == kNoNack) { // Dual decoder mode has not been enabled. return; diff --git a/modules/video_coding/main/source/receiver.h b/modules/video_coding/main/source/receiver.h index b99dccf5c..0ca6994a1 100644 --- a/modules/video_coding/main/source/receiver.h +++ b/modules/video_coding/main/source/receiver.h @@ -28,11 +28,6 @@ enum VCMNackStatus kNackKeyFrameRequest }; -enum VCMNackMode -{ - kNackInfinite, - kNoNack -}; enum VCMReceiverState { @@ -91,7 +86,6 @@ private: VCMJitterBuffer _jitterBuffer; VCMTiming& _timing; VCMEvent& _renderWaitEvent; - VCMNackMode _nackMode; VCMReceiverState _state; static WebRtc_Word32 _receiverIdCounter; diff --git a/modules/video_coding/main/source/session_info.cc b/modules/video_coding/main/source/session_info.cc index fb33867f6..5a5617730 100644 --- a/modules/video_coding/main/source/session_info.cc +++ b/modules/video_coding/main/source/session_info.cc @@ -25,7 +25,9 @@ VCMSessionInfo::VCMSessionInfo(): _previousFrameLoss(false), _lowSeqNum(-1), _highSeqNum(-1), - _highestPacketIndex(0) + _highestPacketIndex(0), + _emptySeqNumLow(-1), + _emptySeqNumHigh(-1) { memset(_packetSizeBytes, 0, sizeof(_packetSizeBytes)); memset(_naluCompleteness, kNaluUnset, sizeof(_naluCompleteness)); @@ -50,6 +52,8 @@ void VCMSessionInfo::Reset() { _lowSeqNum = -1; _highSeqNum = -1; + _emptySeqNumLow = -1; + _emptySeqNumHigh = -1; _markerBit = false; _haveFirstPacket = false; _completeSession = false; @@ -65,7 +69,7 @@ void VCMSessionInfo::Reset() WebRtc_UWord32 VCMSessionInfo::GetSessionLength() { WebRtc_UWord32 length = 0; - for (WebRtc_Word32 i=0; i<=_highestPacketIndex; ++i) + for (WebRtc_Word32 i = 0; i <= _highestPacketIndex; ++i) { length += _packetSizeBytes[i]; } @@ -89,7 +93,9 @@ VCMSessionInfo::HaveStartSeqNumber() return true; } -WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebRtc_Word32 packetIndex, const VCMPacket& packet) +WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, + WebRtc_Word32 packetIndex, + const VCMPacket& packet) { WebRtc_UWord32 moveLength = 0; WebRtc_UWord32 returnLength = 0; @@ -99,30 +105,33 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR WebRtc_UWord32 offset = 0; WebRtc_UWord32 packetSize = 0; - // Store this packet length. Add length since we could have data present already (e.g. multicall case). + // Store this packet length. Add length since we could have data present + // already (e.g. multicall case). if (packet.bits) { packetSize = packet.sizeBytes; } else { - packetSize = packet.sizeBytes + (packet.insertStartCode?kH264StartCodeLengthBytes:0); + packetSize = packet.sizeBytes + + (packet.insertStartCode?kH264StartCodeLengthBytes:0); } _packetSizeBytes[packetIndex] += packetSize; // count only the one in our layer - for (i=0; i 0) { - memmove((void*)(ptrStartOfLayer + offset + packetSize), ptrStartOfLayer + offset, moveLength); + memmove((void*)(ptrStartOfLayer + offset + packetSize), + ptrStartOfLayer + offset, moveLength); } if (packet.bits) @@ -145,7 +154,8 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR const unsigned char startCode[] = {0, 0, 0, 1}; if(packet.insertStartCode) { - memcpy((void*)(ptrStartOfLayer + offset), startCode, kH264StartCodeLengthBytes); + memcpy((void*)(ptrStartOfLayer + offset), startCode, + kH264StartCodeLengthBytes); } memcpy((void*)(ptrStartOfLayer + offset + (packet.insertStartCode?kH264StartCodeLengthBytes:0)), @@ -158,6 +168,9 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, WebR if (packet.isFirstPacket) { _haveFirstPacket = true; + //initializing FEC sequence numbers + _emptySeqNumHigh = -1; + _emptySeqNumLow = -1; } if (packet.markerBit) { @@ -177,7 +190,7 @@ void VCMSessionInfo::UpdateCompleteSession() { // do we have all packets in this session? bool completeSession = true; - for (int i=0; i<= _highestPacketIndex; ++i) + for (int i = 0; i<= _highestPacketIndex; ++i) { if (_naluCompleteness[i] == kNaluUnset) { @@ -194,36 +207,41 @@ bool VCMSessionInfo::IsSessionComplete() return _completeSession; } - // Find the start and end index of packetIndex packet. // startIndex -1 if start not found endIndex=-1 if end index not found -void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& startIndex, WebRtc_Word32& endIndex) +void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, + WebRtc_Word32& startIndex, + WebRtc_Word32& endIndex) { if(_naluCompleteness[packetIndex]==kNaluStart || _naluCompleteness[packetIndex]==kNaluComplete) { - startIndex=packetIndex; + startIndex = packetIndex; } else // Need to find the start { - for(startIndex=packetIndex-1;startIndex>=0;--startIndex) + for(startIndex = packetIndex - 1; startIndex >= 0; --startIndex) { - if( (_naluCompleteness[startIndex]==kNaluComplete && _packetSizeBytes[startIndex]>0) ||(_naluCompleteness[startIndex]==kNaluEnd && startIndex>0)) // Found previous NALU. + if( (_naluCompleteness[startIndex] == kNaluComplete && + _packetSizeBytes[startIndex] > 0) || + // Found previous NALU. + (_naluCompleteness[startIndex] == kNaluEnd && startIndex>0)) { startIndex++; break; } - if( _naluCompleteness[startIndex]==kNaluStart) // This is where the NALU start. + // This is where the NALU start. + if( _naluCompleteness[startIndex] == kNaluStart) { break; } } } - if(_naluCompleteness[packetIndex]==kNaluEnd || - _naluCompleteness[packetIndex]==kNaluComplete) + if(_naluCompleteness[packetIndex] == kNaluEnd || + _naluCompleteness[packetIndex] == kNaluComplete) { endIndex=packetIndex; } @@ -232,7 +250,9 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& sta // Find the next NALU for(endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex) { - if((_naluCompleteness[endIndex]==kNaluComplete && _packetSizeBytes[endIndex]>0) || _naluCompleteness[endIndex]==kNaluStart) // Found next NALU. + if((_naluCompleteness[endIndex]==kNaluComplete && + _packetSizeBytes[endIndex]>0) || + _naluCompleteness[endIndex]==kNaluStart) // Found next NALU. { endIndex--; break; @@ -242,40 +262,43 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex,WebRtc_Word32& sta break; } } - if(endIndex>_highestPacketIndex) - endIndex=-1; + if(endIndex > _highestPacketIndex) + endIndex = -1; } } // Deletes all packets between startIndex and endIndex -WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer,WebRtc_Word32 startIndex,WebRtc_Word32 endIndex) +WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, + WebRtc_Word32 startIndex, + WebRtc_Word32 endIndex) { //Get the number of bytes to delete. //Clear the size of these packets. - WebRtc_UWord32 bytesToDelete=0; /// The number of bytes to delete. - for(int j=startIndex;j<=endIndex;++j) + WebRtc_UWord32 bytesToDelete = 0; /// The number of bytes to delete. + for(int j = startIndex;j <= endIndex; ++j) { - bytesToDelete+=_packetSizeBytes[j]; + bytesToDelete += _packetSizeBytes[j]; _packetSizeBytes[j]=0; } if (bytesToDelete > 0) { // Get the offset we want to move to. - int destOffset=0; - for(int j=0;j0) + if(_packetSizeBytes[0] > 0) { switch(_naluCompleteness[0]) { @@ -321,15 +344,16 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay case kNaluIncomplete: //Packet is not beginning or end of NALU //Need to find the end of this fua NALU and delete all packets. FindNaluBorder(0,startIndex,endIndex); - if(endIndex==-1) // No end found. Delete + if(endIndex == -1) // No end found. Delete { - endIndex=_highestPacketIndex; + endIndex = _highestPacketIndex; } - returnLength+=DeletePackets(ptrStartOfLayer,0,endIndex);//Delete this NALU. + //Delete this NALU. + returnLength += DeletePackets(ptrStartOfLayer,0,endIndex); break; case kNaluEnd: // Packet is the end of a NALU - //Need to delete this packet - returnLength+=DeletePackets(ptrStartOfLayer,0,0);//Delete this NALU. + //Delete this NALU + returnLength += DeletePackets(ptrStartOfLayer,0,0); break; default: assert(false); @@ -339,9 +363,10 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay return returnLength; } -WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num) +WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, + WebRtc_Word32 numberOfSeqNum) { - if ((NULL == list) || (num < 1)) + if ((NULL == list) || (numberOfSeqNum < 1)) { return -1; } @@ -353,7 +378,7 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 n // Find end point (index of entry that equals _lowSeqNum) int index = 0; - for (; index 0) && (list[index] == -1)) + { + // found first packet, for now let's go only one back + if ((list[index - 1] == -1) || (list[index - 1] == -2)) + { + // this is indeed the first packet, as previous packet was populated + isBaseAvailable = true; + } + } + bool allowNack = false; + if (!_haveFirstPacket || !isBaseAvailable) + { + allowNack = true; + } + + // Zero out between first entry and end point + int i = 0; + // Score place holder - based on RTT and partition (when available). + const float nackScoreTh = 0.25f; + WebRtc_Word32 highMediaPacket = _emptySeqNumLow > _lowSeqNum ? + _emptySeqNumLow - 1: _highSeqNum; + + while (list[index] <= highMediaPacket && index < numberOfSeqNum) + { + if (_naluCompleteness[i] != kNaluUnset) + { + list[index] = -1; + } + else + { + // compute score of the packet + float score = 1.0f; + // multiply internal score (importance) by external score (RTT) + score *= rttScore; + if (score > nackScoreTh) + { + allowNack = true; + } + else + { + list[index] = -1; + } + } + i++; + index++; + } + // Empty packets follow the data packets, and therefore have a higher + // sequence number. We do not want to NACK empty packets. + + if ((_emptySeqNumLow != -1) && (_emptySeqNumHigh != -1) && + (index < numberOfSeqNum)) + { + // first make sure that we are at least at the minimum value + // (if not we are missing last packet(s)) + while (list[index] < _emptySeqNumLow && index < numberOfSeqNum) + { + index++; + } + + // mark empty packets + while (list[index] <= _emptySeqNumHigh && index < numberOfSeqNum) + { + list[index] = -2; + index++; + } + } + + _sessionNACK = allowNack; + return 0; +} + WebRtc_Word32 VCMSessionInfo::GetHighestPacketIndex() { return _highestPacketIndex; @@ -410,7 +534,7 @@ void VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32 // sanity if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) { - //not allowed + // not allowed assert(!"SessionInfo::UpdatePacketSize Error: invalid packetIndex"); return; } @@ -420,15 +544,17 @@ void VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32 void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices) { // sanity - if((numberOfPacketIndices + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer) || numberOfPacketIndices < 0) + if((numberOfPacketIndices + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer) + || numberOfPacketIndices < 0) { - //not allowed + // not allowed assert(!"SessionInfo::PrependPacketIndexes Error: invalid packetIndex"); return; } // Works if we have new packets before packetIndex = 0 int numOfPacketsToMove = GetHighestPacketIndex()+1; - memmove(&_packetSizeBytes[numberOfPacketIndices], &_packetSizeBytes[0], (numOfPacketsToMove)*sizeof(WebRtc_UWord16)); + memmove(&_packetSizeBytes[numberOfPacketIndices], &_packetSizeBytes[0], + (numOfPacketsToMove)*sizeof(WebRtc_UWord16)); memset(&_packetSizeBytes[0], 0, numberOfPacketIndices*sizeof(WebRtc_UWord16)); _highestPacketIndex += (WebRtc_UWord16)numberOfPacketIndices; @@ -439,11 +565,11 @@ void VCMSessionInfo::ClearPacketSize(WebRtc_Word32 packetIndex) // sanity if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) { - //not allowed + // not allowed assert(!"SessionInfo::ClearPacketSize Error: invalid packetIndex"); return; } - _packetSizeBytes[packetIndex] =0; + _packetSizeBytes[packetIndex] = 0; } WebRtc_UWord32 VCMSessionInfo::GetPacketSize(WebRtc_Word32 packetIndex) @@ -464,20 +590,31 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL //not allowed assert(!packet.insertStartCode || !packet.bits); + if (packet.frameType == kFrameEmpty) + { + // update seq number as an empty packet + // empty packets will be counted twice: both empty and standard packets. + InformOfEmptyPacket(packet.seqNum); + } // Check if this is first packet (only valid for some codecs) if (packet.isFirstPacket) { // the first packet in the frame always signals the frametype _frameType = packet.frameType; } + else if (_frameType == kFrameEmpty && packet.frameType != kFrameEmpty) + { + // in case an empty packet came in first, update the frame type + _frameType = packet.frameType; + } // Check sequence number and update highest and lowest sequence numbers received. // Move data if this seq num is lower than previously lowest. if (packet.seqNum > _highSeqNum) { - // This packet's seq num is higher than previously highest seq num; normal case - // if we have a wrap, only update with wrapped values + // This packet's seq num is higher than previously highest seq num; + // normal case if we have a wrap, only update with wrapped values if (!(_highSeqNum < 0x00ff && packet.seqNum > 0xff00)) { _highSeqNum = packet.seqNum; @@ -488,7 +625,7 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL _highSeqNum = packet.seqNum; } int packetIndex = packet.seqNum - (WebRtc_UWord16)_lowSeqNum; - if(_lowSeqNum < 0x00ff && packet.seqNum > 0xff00) + if (_lowSeqNum < 0x00ff && packet.seqNum > 0xff00) { // negative wrap packetIndex = packet.seqNum - 0x10000 - _lowSeqNum; @@ -498,7 +635,8 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL if (_lowSeqNum > 0xff00 && packet.seqNum < 0x00ff) { // we have a false detect due to the wrap - packetIndex = (0xffff - (WebRtc_UWord16)_lowSeqNum) + packet.seqNum + (WebRtc_UWord16)1; + packetIndex = (0xffff - (WebRtc_UWord16)_lowSeqNum) + packet.seqNum + + (WebRtc_UWord16)1; } else { // This packet's seq num is lower than previously lowest seq num, but no wrap @@ -562,7 +700,50 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL return InsertBuffer(ptrStartOfLayer, packetIndex, packet); } -WebRtc_UWord32 VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec) + +WebRtc_Word32 +VCMSessionInfo::InformOfEmptyPacket(const WebRtc_UWord16 seqNum) +{ + // Empty packets may be FEC or filler packets. They are sequential and + // follow the data packets, therefore, we should only keep track of the high + // and low sequence numbers and may assume that the packets in between are + // empty packets belonging to the same frame (timestamp). + + if (_emptySeqNumLow == -1 && _emptySeqNumHigh == -1) + { + _emptySeqNumLow = seqNum; + _emptySeqNumHigh = seqNum; + } + else + { + if (seqNum > _emptySeqNumHigh) + { + // This packet's seq num is higher than previously highest seq num; + // normal case if we have a wrap, only update with wrapped values + if (!(_emptySeqNumHigh < 0x00ff && seqNum > 0xff00)) + { + _emptySeqNumHigh = seqNum; + } + } + else if (_emptySeqNumHigh > 0xff00 && seqNum < 0x00ff) + { + // wrap + _emptySeqNumHigh = seqNum; + } + if (_emptySeqNumLow < 0x00ff && seqNum > 0xff00) + { + // negative wrap + if (seqNum - 0x10000 - _emptySeqNumLow < 0) + { + _emptySeqNumLow = seqNum; + } + } + } + return 0; +} + +WebRtc_UWord32 +VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec) { WebRtc_UWord32 currentPacketOffset = 0; WebRtc_UWord32 length = GetSessionLength(); @@ -573,7 +754,7 @@ WebRtc_UWord32 VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, return length; } bool previousLost = false; - for (int i=0; i <= _highestPacketIndex; i++) + for (int i = 0; i <= _highestPacketIndex; i++) { if (_ORwithPrevByte[i]) { diff --git a/modules/video_coding/main/source/session_info.h b/modules/video_coding/main/source/session_info.h index 70205816b..adbcf43e8 100644 --- a/modules/video_coding/main/source/session_info.h +++ b/modules/video_coding/main/source/session_info.h @@ -26,10 +26,16 @@ public: VCMSessionInfo(const VCMSessionInfo& rhs); - WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num); + WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 numberOfSeqNum); + // Hybrid version: Zero out seq num for NACK list + // apply a score based on the packet location and the external rttScore + WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list, + WebRtc_Word32 numberOfSeqNum, + float rttScore); virtual void Reset(); WebRtc_Word64 InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer); + WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum); virtual bool IsSessionComplete(); WebRtc_UWord32 MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer); @@ -75,18 +81,20 @@ protected: bool _sessionNACK; // If this session has been NACKed by JB bool _completeSession; webrtc::FrameType _frameType; - bool _previousFrameLoss; + bool _previousFrameLoss; - WebRtc_Word32 _lowSeqNum; // Lowest packet sequence number in a session - WebRtc_Word32 _highSeqNum; // Highest packet sequence number in a session + WebRtc_Word32 _lowSeqNum; // Lowest packet sequence number in a session + WebRtc_Word32 _highSeqNum; // Highest packet sequence number in a session // Highest packet index in this frame - WebRtc_UWord16 _highestPacketIndex; + WebRtc_UWord16 _highestPacketIndex; // Length of packet (used for reordering) - WebRtc_UWord32 _packetSizeBytes[kMaxPacketsInJitterBuffer]; - // Completness of packets. Used for deciding if the frame is decodable. - WebRtc_UWord8 _naluCompleteness[kMaxPacketsInJitterBuffer]; - bool _ORwithPrevByte[kMaxPacketsInJitterBuffer]; + WebRtc_UWord32 _packetSizeBytes[kMaxPacketsInJitterBuffer]; + // Completeness of packets. Used for deciding if the frame is decodable. + WebRtc_UWord8 _naluCompleteness[kMaxPacketsInJitterBuffer]; + WebRtc_Word32 _emptySeqNumLow; + WebRtc_Word32 _emptySeqNumHigh; + bool _ORwithPrevByte[kMaxPacketsInJitterBuffer]; }; } // namespace webrtc diff --git a/modules/video_coding/main/source/video_coding_impl.cc b/modules/video_coding/main/source/video_coding_impl.cc index 57e15c5cc..b3412d72e 100644 --- a/modules/video_coding/main/source/video_coding_impl.cc +++ b/modules/video_coding/main/source/video_coding_impl.cc @@ -269,7 +269,7 @@ VideoCodingModuleImpl::TimeUntilNextProcess() { WebRtc_UWord32 timeUntilNextProcess = VCM_MIN(_receiveStatsTimer.TimeUntilProcess(), _sendStatsTimer.TimeUntilProcess()); - if ((_receiver.NackMode() == kNackInfinite) || (_dualReceiver.State() != kPassive)) + if ((_receiver.NackMode() != kNoNack) || (_dualReceiver.State() != kPassive)) { // We need a Process call more often if we are relying on retransmissions timeUntilNextProcess = VCM_MIN(timeUntilNextProcess, @@ -576,6 +576,7 @@ VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, bo { WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "SetVideoProtection()"); + switch (videoProtection) { @@ -664,16 +665,18 @@ VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, bo case kProtectionNackFEC: { { + // Receive side CriticalSectionScoped cs(_receiveCritSect); if (enable) { - _receiver.SetNackMode(kNackInfinite); + _receiver.SetNackMode(kNackHybrid); } else { _receiver.SetNackMode(kNoNack); } } + // Send Side { CriticalSectionScoped cs(_sendCritSect); _mediaOpt.EnableNackFEC(enable); @@ -1298,7 +1301,7 @@ VideoCodingModuleImpl::NackList(WebRtc_UWord16* nackList, WebRtc_UWord16& size) // Collect sequence numbers from the default receiver // if in normal nack mode. Otherwise collect them from // the dual receiver if the dual receiver is receiving. - if (_receiver.NackMode() == kNackInfinite) + if (_receiver.NackMode() != kNoNack) { nackStatus = _receiver.NackList(nackList, size); } diff --git a/modules/video_coding/main/test/jitter_buffer_test.cc b/modules/video_coding/main/test/jitter_buffer_test.cc index c534f6156..effc5b100 100644 --- a/modules/video_coding/main/test/jitter_buffer_test.cc +++ b/modules/video_coding/main/test/jitter_buffer_test.cc @@ -1461,9 +1461,9 @@ int JitterBufferTest(CmdArgs& args) // --------------------------------------------------------------------------------------------- // | 3 | 4 | 5 | 6 | 7 | 9 | x | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | x | 21 |.....| 102 | // --------------------------------------------------------------------------------------------- - jb.SetNackStatus(true); + jb.SetNackMode(kNackInfinite); - TEST(jb.GetNackStatus()); + TEST(jb.GetNackMode() == kNackInfinite); // insert first packet timeStamp += 33*90; @@ -1880,7 +1880,7 @@ int JitterBufferTest(CmdArgs& args) //Test incomplete NALU frames jb.Flush(); - jb.SetNackStatus(false); + jb.SetNackMode(kNoNack); seqNum ++; timeStamp += 33*90; int insertedLength=0;