diff --git a/src/modules/video_coding/main/source/jitter_buffer.cc b/src/modules/video_coding/main/source/jitter_buffer.cc index d4d7ab9aa..820c48705 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.cc +++ b/src/modules/video_coding/main/source/jitter_buffer.cc @@ -37,7 +37,8 @@ namespace webrtc { // Criteria used when searching for frames in the frame buffer list bool -VCMJitterBuffer::FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp) +VCMJitterBuffer::FrameEqualTimestamp(VCMFrameBuffer* frame, + const void* timestamp) { if (timestamp == NULL) { @@ -142,8 +143,10 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) _missingMarkerBits = rhs._missingMarkerBits; _firstPacket = rhs._firstPacket; _lastDecodedSeqNum = rhs._lastDecodedSeqNum; - memcpy(_receiveStatistics, rhs._receiveStatistics, sizeof(_receiveStatistics)); - memcpy(_NACKSeqNumInternal, rhs._NACKSeqNumInternal, sizeof(_NACKSeqNumInternal)); + memcpy(_receiveStatistics, rhs._receiveStatistics, + sizeof(_receiveStatistics)); + memcpy(_NACKSeqNumInternal, rhs._NACKSeqNumInternal, + sizeof(_NACKSeqNumInternal)); memcpy(_NACKSeqNum, rhs._NACKSeqNum, sizeof(_NACKSeqNum)); for (int i = 0; i < kMaxNumberOfFrames; i++) { @@ -153,7 +156,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) _frameBuffers[i] = NULL; } } - while(_frameBuffersTSOrder.Erase(_frameBuffersTSOrder.First()) != -1) { } + while(_frameBuffersTSOrder.Erase(_frameBuffersTSOrder.First()) != -1) + { } for (int i = 0; i < _maxNumberOfFrames; i++) { _frameBuffers[i] = new VCMFrameBuffer(*(rhs._frameBuffers[i])); @@ -267,7 +271,7 @@ VCMJitterBuffer::FlushInternal() { // Erase all frames from the sorted list and set their state to free. _frameBuffersTSOrder.Flush(); - for (int i = 0; i < _maxNumberOfFrames; i++) + for (WebRtc_Word32 i = 0; i < _maxNumberOfFrames; i++) { ReleaseFrameInternal(_frameBuffers[i]); } @@ -326,10 +330,13 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) int length = frame->Length(); if (_master) { - // Only trace the primary jitter buffer to make it possible to parse and plot the trace file. - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "JB(0x%x) FB(0x%x): Complete frame added to jitter buffer, size:%d type %d", - this, frame,length,frame->FrameType()); + // Only trace the primary jitter buffer to make it possible to parse + // and plot the trace file. + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "JB(0x%x) FB(0x%x): Complete frame added to jitter buffer," + " size:%d type %d", + this, frame,length,frame->FrameType()); } if (length != 0 && !frame->GetCountedFrame()) @@ -341,7 +348,7 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) // Check if we should drop frame // an old complete frame can arrive too late - if(_lastDecodedTimeStamp > 0 && + if (_lastDecodedTimeStamp > 0 && LatestTimestamp(static_cast(_lastDecodedTimeStamp), frame->TimeStamp()) == _lastDecodedTimeStamp) { @@ -350,11 +357,15 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) frame->Reset(); frame->SetState(kStateEmpty); - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "JB(0x%x) FB(0x%x): Dropping old frame in Jitter buffer", this, frame); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "JB(0x%x) FB(0x%x): Dropping old frame in Jitter buffer", + this, frame); _dropCount++; - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "Jitter buffer drop count: %d, consecutive drops: %u", _dropCount, _numConsecutiveOldFrames); + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "Jitter buffer drop count: %d, consecutive drops: %u", + _dropCount, _numConsecutiveOldFrames); // Flush() if this happens consistently. _numConsecutiveOldFrames++; if (_numConsecutiveOldFrames > kMaxConsecutiveOldFrames) @@ -437,9 +448,10 @@ VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMEncodedFrame*& frame) } _critSect.Enter(); - if (LatestTimestamp(static_cast(_lastDecodedTimeStamp), packet.timestamp) - == _lastDecodedTimeStamp - && packet.sizeBytes > 0) // Make sure that old filler packets are inserted. + if (LatestTimestamp(static_cast(_lastDecodedTimeStamp), + packet.timestamp) == _lastDecodedTimeStamp + && packet.sizeBytes > 0) + // Make sure that old Empty packets are inserted. { // Trying to get an old frame. _numConsecutiveOldPackets++; @@ -452,7 +464,9 @@ VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMEncodedFrame*& frame) } _numConsecutiveOldPackets = 0; - frame = _frameBuffersTSOrder.FindFrame(FrameEqualTimestamp, &packet.timestamp); + frame = _frameBuffersTSOrder.FindFrame(FrameEqualTimestamp, + &packet.timestamp); + _critSect.Leave(); if (frame != NULL) @@ -626,18 +640,17 @@ VCMJitterBuffer::FindOldestCompleteContinuousFrame() // Use seqNum not timestamp since a full frame might be lost if (_lastDecodedSeqNum != -1) { - // it's not enough that we have complete frame we need the seq numbers - // to be continuous too for layers it's not enough that we have complete - // frame we need the layers to be continuous too + // it's not enough that we have complete frame we need the layers to + // be continuous too currentLow = oldestFrame->GetLowSeqNum(); WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum; - // we could have received the first packet of the last frame before a + // We could have received the first packet of the last frame before a // long period if drop, that case is handled by GetNackList if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) { - // wait since we want a complete continuous frame + // Wait since we want a complete continuous frame return NULL; } } @@ -651,7 +664,8 @@ VCMJitterBuffer::FindOldestCompleteContinuousFrame() bool VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) { - const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameItem); + const VCMFrameListItem* + nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameItem); VCMFrameBuffer* oldestFrame = NULL; if (oldestFrameItem != NULL) { @@ -659,12 +673,12 @@ VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) } if (nextFrameItem != NULL) { - // We have received at least one packet from a later frame. - if(!oldestFrame->HaveLastPacket()) // If we don't have the markerbit + // We have received at least one packet from a later frame. + if(!oldestFrame->HaveLastPacket()) // If we don't have the markerbit { VCMFrameBuffer* nextFrame = nextFrameItem->GetItem(); - // Verify that we have received the first packet of the next frame. - // This is the only way we can be sure we're not missing the last packet. + // Verify that we have received the first packet of the next frame, + // so we're not missing the last packet. if (nextFrame != NULL && nextFrame->GetLowSeqNum() == static_cast(oldestFrame->GetHighSeqNum() + 1)) { @@ -677,7 +691,7 @@ VCMJitterBuffer::CheckForCompleteFrame(VCMFrameListItem* oldestFrameItem) const VCMFrameBufferStateEnum state = oldestFrame->GetState(); if (state == kStateComplete) { - if(oldestFrame->Length() > 0) + if (oldestFrame->Length() > 0) { UpdateJitterAndDelayEstimates(*oldestFrame, false); } @@ -698,9 +712,10 @@ VCMJitterBuffer::RecycleFrame(VCMFrameBuffer* frame) return; } - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "JB(0x%x) FB(0x%x): RecycleFrame, size:%d", - this, frame, frame->Length()); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "JB(0x%x) FB(0x%x): RecycleFrame, size:%d", + this, frame, frame->Length()); ReleaseFrameInternal(frame); } @@ -738,8 +753,10 @@ VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) // Calculate frame rate // Let r be rate. - // r(0) = 1000*framecount/delta_time. (I.e. frames per second since last calculation.) - // frameRate = r(0)/2 + r(-1)/2 (I.e. fr/s average this and the previous calculation.) + // r(0) = 1000*framecount/delta_time. + // (I.e. frames per second since last calculation.) + // frameRate = r(0)/2 + r(-1)/2 + // (I.e. fr/s average this and the previous calculation.) frameRate = (_incomingFrameRate + (WebRtc_Word32)rate) >> 1; _incomingFrameRate = (WebRtc_UWord8)rate; @@ -750,7 +767,8 @@ VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) } else { - bitRate = 10 * ((100 * _incomingBitCount) / static_cast(diff)); + bitRate = 10 * ((100 * _incomingBitCount) / + static_cast(diff)); } _incomingBitRate = bitRate; @@ -772,7 +790,8 @@ VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) return 0; } -// Returns immediately or a X ms event hang waiting for a decodable frame, X decided by caller +// Returns immediately or a X ms event hang waiting for a decodable frame, +// X decided by caller VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) { @@ -806,7 +825,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) while (waitTimeMs > 0) { _critSect.Leave(); - const EventTypeWrapper ret = _frameEvent.Wait(static_cast(waitTimeMs)); + const EventTypeWrapper ret = + _frameEvent.Wait(static_cast(waitTimeMs)); _critSect.Enter(); if (ret == kEventSignaled) { @@ -817,7 +837,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) return NULL; } - // Finding oldest frame ready for decoder, but check sequence number and size + // Finding oldest frame ready for decoder, but check + // sequence number and size CleanUpOldFrames(); CleanUpSizeZeroFrames(); oldestFrameListItem = FindOldestCompleteContinuousFrame(); @@ -827,7 +848,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) } if (oldestFrame == NULL) { - waitTimeMs = endWaitTimeMs - VCMTickTime::MillisecondTimestamp(); + waitTimeMs = endWaitTimeMs - + VCMTickTime::MillisecondTimestamp(); } else { @@ -850,17 +872,11 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) if (oldestFrame == NULL) { - // Even after signaling we're still missing a complete _continuous_ frame + // Even after signaling we're still missing a complete continuous frame _critSect.Leave(); return NULL; } - // we have a frame - // store seqnum - _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); - // store current timestamp - _lastDecodedTimeStamp = oldestFrame->TimeStamp(); - // Update jitter estimate const bool retransmitted = (oldestFrame->GetNackCount() > 0); if (retransmitted) @@ -884,6 +900,10 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) _critSect.Leave(); + // We have a frame - store seqnum & timestamp + _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); + _lastDecodedTimeStamp = oldestFrame->TimeStamp(); + return oldestFrame; } @@ -899,7 +919,7 @@ VCMJitterBuffer::GetEstimatedJitterMsInternal() { WebRtc_UWord32 estimate = VCMJitterEstimator::OPERATING_SYSTEM_JITTER; - // compute RTT multiplier for estimation + // Compute RTT multiplier for estimation double rttMult = 1.0f; if (_nackMode == kNackHybrid && _rttMs > kLowRttNackMs) { @@ -912,8 +932,8 @@ VCMJitterBuffer::GetEstimatedJitterMsInternal() { // Since the incoming packets are all missing marker bits we have to // wait until the first packet of the next frame arrives, before we can - // safely say that the frame is complete. Therefore we have to compensate - // the jitter buffer level with one frame period. + // safely say that the frame is complete. Therefore we have to + // compensate the jitter buffer level with one frame period. // TODO(holmer): The timestamp diff should probably be filtered // (max filter) since the diff can alternate between e.g. 3000 and 6000 // if we have a frame rate between 15 and 30 frames per seconds. @@ -943,7 +963,7 @@ VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, _critSect.Enter(); - // Finding oldest frame ready for decoder, but check sequence number and size + // Finding oldest frame ready for decoder, check sequence number and size CleanUpOldFrames(); CleanUpSizeZeroFrames(); @@ -952,7 +972,7 @@ VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, if (oldestFrame == NULL) { _critSect.Leave(); - if(_packetEvent.Wait(maxWaitTimeMS) == kEventSignaled) + if (_packetEvent.Wait(maxWaitTimeMS) == kEventSignaled) { // are we closing down the Jitter buffer if (!_running) @@ -964,7 +984,8 @@ VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, CleanUpOldFrames(); CleanUpSizeZeroFrames(); oldestFrame = _frameBuffersTSOrder.FirstFrame(); - }else + } + else { _critSect.Enter(); } @@ -979,7 +1000,8 @@ VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, // we have a frame // return frame type - incomingFrameType = oldestFrame->FrameType(); // All layers are assumed to have the same type + // All layers are assumed to have the same type + incomingFrameType = oldestFrame->FrameType(); renderTimeMs = oldestFrame->RenderTimeMs(); @@ -992,14 +1014,15 @@ VCMJitterBuffer::GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, } // Answers the question: -// Will the packet sequence be complete if the next frame is grabbed for decoding right now? -// That is, have we lost a frame between the last decoded frame and the next, or is the next +// Will the packet sequence be complete if the next frame is grabbed for +// decoding right now? That is, have we lost a frame between the last decoded +// frame and the next, or is the next // frame missing one or more packets? bool VCMJitterBuffer::CompleteSequenceWithNextFrame() { CriticalSectionScoped cs(_critSect); - // Finding oldest frame ready for decoder, but check sequence number and size + // Finding oldest frame ready for decoder, check sequence number and size CleanUpOldFrames(); CleanUpSizeZeroFrames(); @@ -1011,7 +1034,8 @@ VCMJitterBuffer::CompleteSequenceWithNextFrame() } VCMFrameBuffer* oldestFrame = oldestFrameListItem->GetItem(); - const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameListItem); + const VCMFrameListItem* nextFrameItem = + _frameBuffersTSOrder.Next(oldestFrameListItem); if (nextFrameItem == NULL && !oldestFrame->HaveLastPacket()) { // Frame not ready to be decoded. @@ -1031,7 +1055,8 @@ VCMJitterBuffer::CompleteSequenceWithNextFrame() { return false; } - else if (oldestFrame->GetLowSeqNum() != (_lastDecodedSeqNum + 1) % 0x00010000) + else if (oldestFrame->GetLowSeqNum() != (_lastDecodedSeqNum + 1) + % 0x00010000) { return false; } @@ -1063,7 +1088,8 @@ VCMJitterBuffer::GetFrameForDecoding() } VCMFrameBuffer* oldestFrame = oldestFrameListItem->GetItem(); - const VCMFrameListItem* nextFrameItem = _frameBuffersTSOrder.Next(oldestFrameListItem); + const VCMFrameListItem* nextFrameItem = + _frameBuffersTSOrder.Next(oldestFrameListItem); if (nextFrameItem == NULL && !oldestFrame->HaveLastPacket()) { return NULL; @@ -1088,7 +1114,8 @@ VCMJitterBuffer::GetFrameForDecoding() } // Then wait for this one to get complete _waitingForCompletion.frameSize = oldestFrame->Length(); - _waitingForCompletion.latestPacketTime = oldestFrame->LatestPacketTimeMs(); + _waitingForCompletion.latestPacketTime = + oldestFrame->LatestPacketTimeMs(); _waitingForCompletion.timestamp = oldestFrame->TimeStamp(); oldestFrame->SetState(kStateDecoding); } @@ -1100,11 +1127,9 @@ VCMJitterBuffer::GetFrameForDecoding() VerifyAndSetPreviousFrameLost(*oldestFrame); - // store current time - _lastDecodedTimeStamp = oldestFrame->TimeStamp(); - - // store seqnum + // Store current seqnum & time _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); + _lastDecodedTimeStamp = oldestFrame->TimeStamp(); return oldestFrame; } @@ -1143,13 +1168,6 @@ VCMJitterBuffer::GetFrameForDecodingNACK() return NULL; } } - - // We have a complete/decodable continuous frame, decode it. - // store seqnum - _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); - // store current time - _lastDecodedTimeStamp = oldestFrame->TimeStamp(); - // Update jitter estimate const bool retransmitted = (oldestFrame->GetNackCount() > 0); if (retransmitted) @@ -1171,13 +1189,19 @@ VCMJitterBuffer::GetFrameForDecodingNACK() CleanUpOldFrames(); CleanUpSizeZeroFrames(); + // We have a complete/decodable continuous frame, decode it. + // Store seqnum & timestamp + _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); + _lastDecodedTimeStamp = oldestFrame->TimeStamp(); + return oldestFrame; } // Must be called under the critical section _critSect. Should never be called with // retransmitted frames, they must be filtered out before this function is called. void -VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMJitterSample& sample, bool incompleteFrame) +VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMJitterSample& sample, + bool incompleteFrame) { if (sample.latestPacketTime == -1) { @@ -1185,17 +1209,19 @@ VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMJitterSample& sample, bool inc } if (incompleteFrame) { - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "Received incomplete frame timestamp %u frame size %u at time %u", - sample.timestamp, sample.frameSize, - MaskWord64ToUWord32(sample.latestPacketTime)); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), "Received incomplete frame " + "timestamp %u frame size %u at time %u", + sample.timestamp, sample.frameSize, + MaskWord64ToUWord32(sample.latestPacketTime)); } else { - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "Received complete frame timestamp %u frame size %u at time %u", - sample.timestamp, sample.frameSize, - MaskWord64ToUWord32(sample.latestPacketTime)); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), "Received complete frame " + "timestamp %u frame size %u at time %u", + sample.timestamp, sample.frameSize, + MaskWord64ToUWord32(sample.latestPacketTime)); } UpdateJitterAndDelayEstimates(sample.latestPacketTime, sample.timestamp, @@ -1203,10 +1229,12 @@ VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMJitterSample& sample, bool inc incompleteFrame); } -// Must be called under the critical section _critSect. Should never be called with -// retransmitted frames, they must be filtered out before this function is called. +// Must be called under the critical section _critSect. Should never be +// called with retransmitted frames, they must be filtered out before this +// function is called. void -VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, bool incompleteFrame) +VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, + bool incompleteFrame) { if (frame.LatestPacketTimeMs() == -1) { @@ -1216,28 +1244,31 @@ VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, bool incom // estimate. if (incompleteFrame) { - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "Received incomplete frame timestamp %u frame type %d frame size %u" - " at time %u, jitter estimate was %u", + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "Received incomplete frame timestamp %u frame type %d " + "frame size %u at time %u, jitter estimate was %u", frame.TimeStamp(), frame.FrameType(), frame.Length(), MaskWord64ToUWord32(frame.LatestPacketTimeMs()), GetEstimatedJitterMsInternal()); } else { - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "Received complete frame timestamp %u frame type %d frame size %u " - "at time %u, jitter estimate was %u", - frame.TimeStamp(), frame.FrameType(), frame.Length(), - MaskWord64ToUWord32(frame.LatestPacketTimeMs()), - GetEstimatedJitterMsInternal()); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId),"Received complete frame " + "timestamp %u frame type %d frame size %u at time %u, " + "jitter estimate was %u", + frame.TimeStamp(), frame.FrameType(), frame.Length(), + MaskWord64ToUWord32(frame.LatestPacketTimeMs()), + GetEstimatedJitterMsInternal()); } UpdateJitterAndDelayEstimates(frame.LatestPacketTimeMs(), frame.TimeStamp(), frame.Length(), incompleteFrame); } -// Must be called under the critical section _critSect. Should never be called with -// retransmitted frames, they must be filtered out before this function is called. +// Must be called under the critical section _critSect. Should never be called +// with retransmitted frames, they must be filtered out before this function +// is called. void VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs, WebRtc_UWord32 timestamp, @@ -1250,10 +1281,14 @@ VCMJitterBuffer::UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs, } WebRtc_Word64 frameDelay; // Calculate the delay estimate - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "Packet received and sent to jitter estimate with: timestamp=%u wallClock=%u", - timestamp, MaskWord64ToUWord32(latestPacketTimeMs)); - bool notReordered = _delayEstimate.CalculateDelay(timestamp, &frameDelay, latestPacketTimeMs); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), + "Packet received and sent to jitter estimate with: " + "timestamp=%u wallClock=%u", timestamp, + MaskWord64ToUWord32(latestPacketTimeMs)); + bool notReordered = _delayEstimate.CalculateDelay(timestamp, + &frameDelay, + latestPacketTimeMs); // Filter out frames which have been reordered in time by the network if (notReordered) { @@ -1273,8 +1308,8 @@ WebRtc_Word32 VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word32& highSeqNum) const { - int i = 0; - int seqNum = -1; + WebRtc_Word32 i = 0; + WebRtc_Word32 seqNum = -1; highSeqNum = -1; lowSeqNum = _lastDecodedSeqNum; @@ -1325,7 +1360,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) WebRtc_Word32 highSeqNum = -1; listExtended = false; - // don't create list, if we won't wait for it + // Don't create list, if we won't wait for it if (!WaitForNack()) { nackSize = 0; @@ -1341,7 +1376,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) // write a list of all seq num we have if (lowSeqNum == -1 || highSeqNum == -1) { - //This happens if we lose the first packet, nothing is popped + // This happens if we lose the first packet, nothing is popped if (highSeqNum == -1) { // we have not received any packets yet @@ -1429,8 +1464,8 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) nackSize = 0xffff; listExtended = true; WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1, - "\tNo key frame found, request one. _lastDecodedSeqNum[0] %d", - _lastDecodedSeqNum); + "\tNo key frame found, request one. _lastDecodedSeqNum[0] " + "%d", _lastDecodedSeqNum); } else { @@ -1479,15 +1514,15 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) rttScore); if (_frameBuffers[i]->IsRetransmitted() == false) { - // if no retransmission required,set the state to decodable - // meaning that we will not wait for NACK + // If no retransmission required,set the state to decodable + // meaning that we will not wait for NACK. _frameBuffers[i]->SetState(kStateDecodable); } } else { - // used when the frame is being processed by the decoding thread - // don't need to use that info in this loop + // Used when the frame is being processed by the decoding thread + // don't need to use that info in this loop. _frameBuffers[i]->ZeroOutSeqNum(_NACKSeqNumInternal, numberOfSeqNum); } @@ -1532,10 +1567,10 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) { nackSize = emptyIndex; } - // convert to unsigned short 16 bit and store in a list to be used externally. + if (nackSize > _NACKSeqNumLength) { - // Larger list means that the nack list was extended since the last call. + // Larger list: nack list was extended since the last call. listExtended = true; } @@ -1580,7 +1615,8 @@ VCMJitterBuffer::ReleaseFrame(VCMEncodedFrame* frame) } WebRtc_Word64 -VCMJitterBuffer::LastPacketTime(VCMEncodedFrame* frame, bool& retransmitted) const +VCMJitterBuffer::LastPacketTime(VCMEncodedFrame* frame, + bool& retransmitted) const { CriticalSectionScoped cs(_critSect); retransmitted = (static_cast(frame)->GetNackCount() > 0); @@ -1638,14 +1674,18 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) if (frame != NULL) { VCMFrameBufferStateEnum state = frame->GetState(); - if (state == kStateDecoding && packet.sizeBytes == 0) - { - // Filler packet, make sure we update the last decoded seq num - // since this packet should've been a part of the frame being decoded. - // If the filler packet belongs to a very old frame (already decoded - // and freed) a new frame will be created for the filler packet. - // That frame will be empty and later on cleaned up. - UpdateLastDecodedWithFiller(packet); + if ((packet.sizeBytes == 0) && + ((state == kStateDecoding) || + (state == kStateEmpty && + _lastDecodedTimeStamp == packet.timestamp))) + { + // Empty packet (sizeBytes = 0), make sure we update the last + // decoded seq num since this packet belongs either to a frame + // being decoded (condition 1) or to a frame which was already + // decoded and freed (condition 2). A new frame will be created + // for the empty packet. That frame will be empty and later on + // cleaned up. + UpdateLastDecodedWithEmpty(packet); } // Insert packet @@ -1715,9 +1755,9 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) } void -VCMJitterBuffer::UpdateLastDecodedWithFiller(const VCMPacket& packet) +VCMJitterBuffer::UpdateLastDecodedWithEmpty(const VCMPacket& packet) { - // Empty packet (filler) inserted to a frame which + // Empty packet inserted to a frame which // is already decoding. Update the last decoded seq no. if (_lastDecodedTimeStamp == packet.timestamp && (packet.seqNum > _lastDecodedSeqNum || @@ -1866,10 +1906,10 @@ VCMJitterBuffer::CleanUpOldFrames() ((frameLowSeqNum == 0) && (_lastDecodedSeqNum == 0xffff)))) { - // Could happen when sending filler data. - // Filler packet (size = 0) belonging to last decoded frame. + // Could happen when sending empty packets + // Empty packet (size = 0) belonging to last decoded frame. // Frame: | packet | packet | packet M=1 | - // filler data (size = 0) | filler data (size = 0)| ... + // empty data (size = 0) | empty data (size = 0)| ... // This frame follows the last decoded frame _lastDecodedSeqNum = frameHighSeqNum; @@ -1914,18 +1954,22 @@ VCMJitterBuffer::CleanUpSizeZeroFrames() else { bool releaseFrame = false; - const WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum(); - const WebRtc_Word32 frameLowSeqNum = ptrTempBuffer->GetLowSeqNum(); + const WebRtc_Word32 frameHighSeqNum = + ptrTempBuffer->GetHighSeqNum(); + const WebRtc_Word32 frameLowSeqNum = + ptrTempBuffer->GetLowSeqNum(); - if ((frameLowSeqNum == (_lastDecodedSeqNum + 1)) || // Frame is next in line + if ((frameLowSeqNum == (_lastDecodedSeqNum + 1)) || + // Frame is next in line ((frameLowSeqNum == 0) && (_lastDecodedSeqNum== 0xffff))) { // This frame follows the last decoded frame, release it. _lastDecodedSeqNum = frameHighSeqNum; releaseFrame = true; } - // If frameHighSeqNum < _lastDecodedSeqNum but need to take wrap into account. - else if(frameHighSeqNum < _lastDecodedSeqNum) + // If frameHighSeqNum < _lastDecodedSeqNum + // but need to take wrap into account. + else if (frameHighSeqNum < _lastDecodedSeqNum) { if (frameHighSeqNum < 0x0fff && _lastDecodedSeqNum> 0xf000) @@ -1939,7 +1983,7 @@ VCMJitterBuffer::CleanUpSizeZeroFrames() releaseFrame = true; } } - else if(frameHighSeqNum > _lastDecodedSeqNum && + else if (frameHighSeqNum > _lastDecodedSeqNum && _lastDecodedSeqNum < 0x0fff && frameHighSeqNum > 0xf000) { @@ -1970,7 +2014,7 @@ VCMJitterBuffer::CleanUpSizeZeroFrames() } } -// used in GetFrameForDecoding +// Used in GetFrameForDecoding void VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame) { @@ -1980,8 +2024,8 @@ VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame) // First frame frame.SetPreviousFrameLoss(); } - else if (frame.GetLowSeqNum() != ((WebRtc_UWord16)_lastDecodedSeqNum + - (WebRtc_UWord16)1)) + else if ((WebRtc_UWord16)frame.GetLowSeqNum() != + ((WebRtc_UWord16)_lastDecodedSeqNum + (WebRtc_UWord16)1)) { // Frame loss frame.SetPreviousFrameLoss(); diff --git a/src/modules/video_coding/main/source/jitter_buffer.h b/src/modules/video_coding/main/source/jitter_buffer.h index ab88f38bc..bad28d5a8 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.h +++ b/src/modules/video_coding/main/source/jitter_buffer.h @@ -71,7 +71,8 @@ public: // Statistics, Calculate frame and bit rates WebRtc_Word32 GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate); - // Wait for the first packet in the next frame to arrive, blocks for <= maxWaitTimeMS ms + // Wait for the first packet in the next frame to arrive, blocks + // for <= maxWaitTimeMS ms WebRtc_Word64 GetNextTimeStamp(WebRtc_UWord32 maxWaitTimeMS, FrameType& incomingFrameType, WebRtc_Word64& renderTimeMs); @@ -82,7 +83,8 @@ public: // or more packets? bool CompleteSequenceWithNextFrame(); - // Wait maxWaitTimeMS for a complete frame to arrive. After timeout NULL is returned. + // Wait maxWaitTimeMS for a complete frame to arrive. After timeout NULL + // is returned. VCMEncodedFrame* GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS); // Get a frame for decoding (even an incomplete) without delay. @@ -100,10 +102,12 @@ public: // Returns the time in ms when the latest packet was inserted into the frame. // Retransmitted is set to true if any of the packets belonging to the frame // has been retransmitted. - WebRtc_Word64 LastPacketTime(VCMEncodedFrame* frame, bool& retransmitted) const; + WebRtc_Word64 LastPacketTime(VCMEncodedFrame* frame, + bool& retransmitted) const; // Insert a packet into a frame - VCMFrameBufferEnum InsertPacket(VCMEncodedFrame* frame, const VCMPacket& packet); + VCMFrameBufferEnum InsertPacket(VCMEncodedFrame* frame, + const VCMPacket& packet); // Sync WebRtc_UWord32 GetEstimatedJitterMS(); @@ -113,7 +117,8 @@ public: void SetNackMode(VCMNackMode mode); // Enable/disable nack VCMNackMode GetNackMode() const; // Get nack mode // Get list of missing sequence numbers (size in number of elements) - WebRtc_UWord16* GetNackList(WebRtc_UWord16& nackSize, bool& listExtended); + WebRtc_UWord16* GetNackList(WebRtc_UWord16& nackSize, + bool& listExtended); WebRtc_Word64 LastDecodedTimestamp() const; static WebRtc_UWord32 LatestTimestamp(const WebRtc_UWord32 existingTimestamp, @@ -134,7 +139,8 @@ protected: VCMFrameBuffer* GetEmptyFrame(); // Recycle oldest frames up to a key frame, used if JB is completely full bool RecycleFramesUntilKeyFrame(); - // Update frame state (set as complete or reconstructable if conditions are met) + // Update frame state + // (set as complete or reconstructable if conditions are met) void UpdateFrameState(VCMFrameBuffer* frameListItem); // Help functions for getting a frame @@ -150,8 +156,10 @@ protected: void VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame); bool IsPacketRetransmitted(const VCMPacket& packet) const; - void UpdateJitterAndDelayEstimates(VCMJitterSample& sample, bool incompleteFrame); - void UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, bool incompleteFrame); + void UpdateJitterAndDelayEstimates(VCMJitterSample& sample, + bool incompleteFrame); + void UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, + bool incompleteFrame); void UpdateJitterAndDelayEstimates(WebRtc_Word64 latestPacketTimeMs, WebRtc_UWord32 timestamp, WebRtc_UWord32 frameSize, @@ -160,15 +168,17 @@ protected: WebRtc_UWord32 GetEstimatedJitterMsInternal(); // NACK help - WebRtc_UWord16* CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended); + WebRtc_UWord16* CreateNackList(WebRtc_UWord16& nackSize, + bool& listExtended); WebRtc_Word32 GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word32& highSeqNum) const; - void UpdateLastDecodedWithFiller(const VCMPacket& packet); + void UpdateLastDecodedWithEmpty(const VCMPacket& packet); private: - static bool FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp); + static bool FrameEqualTimestamp(VCMFrameBuffer* frame, + const void* timestamp); static bool CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame, const void* notUsed); // Decide whether should wait for NACK (mainly relevant for hybrid mode) @@ -222,7 +232,7 @@ private: // NACK VCMNackMode _nackMode; - // Holds the internal nack list (the missing seqence numbers) + // Holds the internal nack list (the missing sequence numbers) WebRtc_Word32 _NACKSeqNumInternal[kNackHistoryLength]; WebRtc_UWord16 _NACKSeqNum[kNackHistoryLength]; WebRtc_UWord32 _NACKSeqNumLength; diff --git a/src/modules/video_coding/main/source/session_info.cc b/src/modules/video_coding/main/source/session_info.cc index 7145b2902..298aa6306 100644 --- a/src/modules/video_coding/main/source/session_info.cc +++ b/src/modules/video_coding/main/source/session_info.cc @@ -75,7 +75,8 @@ VCMSessionInfo::Reset() memset(_ORwithPrevByte, 0, sizeof(_ORwithPrevByte)); } -WebRtc_UWord32 VCMSessionInfo::GetSessionLength() +WebRtc_UWord32 +VCMSessionInfo::GetSessionLength() { WebRtc_UWord32 length = 0; for (WebRtc_Word32 i = 0; i <= _highestPacketIndex; ++i) @@ -163,7 +164,7 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, if (packet.dataPtr != NULL) { const unsigned char startCode[] = {0, 0, 0, 1}; - if(packet.insertStartCode) + if (packet.insertStartCode) { memcpy((void*)(ptrStartOfLayer + offset), startCode, kH264StartCodeLengthBytes); @@ -185,8 +186,8 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, _markerBit = true; _markerSeqNum = packet.seqNum; } - // Store information about if the packet is decodable as is or not. - _naluCompleteness[packetIndex] = packet.completeNALU; + // Store information about if the packet is decodable as is or not. + _naluCompleteness[packetIndex] = packet.completeNALU; UpdateCompleteSession(); @@ -253,7 +254,7 @@ VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, } if (_naluCompleteness[packetIndex] == kNaluEnd || - _naluCompleteness[packetIndex] == kNaluComplete) + _naluCompleteness[packetIndex] == kNaluComplete) { endIndex = packetIndex; } @@ -271,7 +272,7 @@ VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, endIndex--; break; } - if ( _naluCompleteness[endIndex] == kNaluEnd) + if (_naluCompleteness[endIndex] == kNaluEnd) { // This is where the NALU end. break; @@ -472,7 +473,7 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, // found first packet, for now let's go only one back if ((list[index - 1] == -1) || (list[index - 1] == -2)) { - // this is indeed the first packet, as previous packet was populated + // This is indeed the first packet, as previous packet was populated isBaseAvailable = true; } } @@ -483,7 +484,7 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, } // Zero out between first entry and end point - int i = 0; + WebRtc_Word32 i = 0; // Score place holder - based on RTT and partition (when available). const float nackScoreTh = 0.25f; @@ -494,6 +495,7 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, } else { + // Estimation highMediaPacket = _emptySeqNumLow - 1 > _highSeqNum ? _emptySeqNumLow - 1: _highSeqNum; } @@ -605,9 +607,8 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, } if (packet.frameType == kFrameEmpty) { - // update seq number as an empty packet - InformOfEmptyPacket(packet.seqNum); - return 0; + // Update seq number as an empty packet + return InformOfEmptyPacket(packet.seqNum); } // Check sequence number and update highest and lowest sequence numbers @@ -621,7 +622,8 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, { _highSeqNum = packet.seqNum; } - } else if (_highSeqNum > 0xff00 && packet.seqNum < 0x00ff) + } + else if (_highSeqNum > 0xff00 && packet.seqNum < 0x00ff) { // wrap _highSeqNum = packet.seqNum; @@ -719,7 +721,7 @@ VCMSessionInfo::InformOfEmptyPacket(const WebRtc_UWord16 seqNum) // and low sequence numbers and may assume that the packets in between are // empty packets belonging to the same frame (timestamp). - if (_emptySeqNumLow == -1 && _emptySeqNumHigh == -1) + if (_emptySeqNumLow == -1 && _emptySeqNumHigh == -1) { _emptySeqNumLow = seqNum; _emptySeqNumHigh = seqNum; diff --git a/src/modules/video_coding/main/source/session_info.h b/src/modules/video_coding/main/source/session_info.h index ff71defe5..bf8cf622e 100644 --- a/src/modules/video_coding/main/source/session_info.h +++ b/src/modules/video_coding/main/source/session_info.h @@ -26,7 +26,8 @@ public: VCMSessionInfo(const VCMSessionInfo& rhs); - WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 numberOfSeqNum); + WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, + WebRtc_Word32 numberOfSeqNum); // Hybrid version: Zero out seq num for NACK list // apply a score based on the packet location and the external rttScore WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list, @@ -34,7 +35,8 @@ public: float rttScore); virtual void Reset(); - WebRtc_Word64 InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer); + WebRtc_Word64 InsertPacket(const VCMPacket& packet, + WebRtc_UWord8* ptrStartOfLayer); WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum); virtual bool IsSessionComplete(); @@ -47,7 +49,8 @@ public: webrtc::FrameType FrameType() const { return _frameType; } virtual WebRtc_Word32 GetHighestPacketIndex(); - virtual void UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32 length); + virtual void UpdatePacketSize(WebRtc_Word32 packetIndex, + WebRtc_UWord32 length); void SetStartSeqNumber(WebRtc_UWord16 seqNumber); @@ -57,7 +60,8 @@ public: // returns highest seqNum, media or empty WebRtc_Word32 GetHighSeqNum() const; - WebRtc_UWord32 PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec); + WebRtc_UWord32 PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, + VideoCodecType codec); void SetPreviousFrameLoss() { _previousFrameLoss = true; } bool PreviousFrameLoss() const { return _previousFrameLoss; } @@ -73,16 +77,18 @@ protected: WebRtc_Word32 startIndex, WebRtc_Word32 endIndex); void UpdateCompleteSession(); - - bool _haveFirstPacket; // If we have inserted the first packet into this frame - bool _markerBit; // If we have inserted a packet with markerbit into this frame - bool _sessionNACK; // If this session has been NACKed by JB + // If we have inserted the first packet into this frame + bool _haveFirstPacket; + // If we have inserted a packet with markerbit into this frame + bool _markerBit; + // If this session has been NACKed by JB + bool _sessionNACK; bool _completeSession; webrtc::FrameType _frameType; bool _previousFrameLoss; - - WebRtc_Word32 _lowSeqNum; // Lowest packet sequence number in a session - WebRtc_Word32 _highSeqNum; // Highest packet sequence number in a session + // Lowest/Highest packet sequence number in a session + WebRtc_Word32 _lowSeqNum; + WebRtc_Word32 _highSeqNum; // Highest packet index in this frame WebRtc_UWord16 _highestPacketIndex;