From 2b83acef3edc37bba0a8d29110a813a4360308d4 Mon Sep 17 00:00:00 2001 From: "mikhal@google.com" Date: Wed, 22 Jun 2011 17:25:06 +0000 Subject: [PATCH] VCM/JB: Setting only non-empty frames for decoding (when not waiting for NACK). Review URL: http://webrtc-codereview.appspot.com/49001 git-svn-id: http://webrtc.googlecode.com/svn/trunk@117 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../video_coding/main/source/jitter_buffer.cc | 34 +++-- .../video_coding/main/source/session_info.cc | 139 +++++++++++------- 2 files changed, 106 insertions(+), 67 deletions(-) diff --git a/modules/video_coding/main/source/jitter_buffer.cc b/modules/video_coding/main/source/jitter_buffer.cc index 90075191c..746432d7e 100644 --- a/modules/video_coding/main/source/jitter_buffer.cc +++ b/modules/video_coding/main/source/jitter_buffer.cc @@ -58,7 +58,8 @@ VCMJitterBuffer::CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame, } // Constructor -VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId, bool master) : +VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId, + bool master) : _vcmId(vcmId), _receiverId(receiverId), _running(false), @@ -216,7 +217,8 @@ VCMJitterBuffer::Start() _NACKSeqNumLength = 0; _rttMs = 0; - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: start", this); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, + _receiverId), "JB(0x%x): Jitter buffer: start", this); } @@ -240,7 +242,8 @@ VCMJitterBuffer::Stop() _critSect.Leave(); _frameEvent.Set(); // Make sure we exit from trying to get a frame to decoder _packetEvent.Set(); // Make sure we exit from trying to get a sequence number - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: stop", this); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, + _receiverId), "JB(0x%x): Jitter buffer: stop", this); } bool @@ -290,8 +293,8 @@ VCMJitterBuffer::FlushInternal() _NACKSeqNumLength = 0; - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "JB(0x%x): Jitter buffer: flush", this); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, + _receiverId), "JB(0x%x): Jitter buffer: flush", this); } // Set the frame state to free and remove it from the sorted @@ -314,8 +317,9 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) { if (frame == NULL) { - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), - "JB(0x%x) FB(0x%x): UpdateFrameState NULL frame pointer", this, frame); + WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), "JB(0x%x) FB(0x%x): " + "UpdateFrameState NULL frame pointer", this, frame); return; } @@ -518,8 +522,9 @@ VCMJitterBuffer::GetEmptyFrame() _maxNumberOfFrames++; _critSect.Leave(); - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x) FB(0x%x): Jitter buffer increased to:%d frames", - this, ptrNewBuffer, _maxNumberOfFrames); + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, + VCMId(_vcmId, _receiverId), "JB(0x%x) FB(0x%x): Jitter buffer " + "increased to:%d frames", this, ptrNewBuffer, _maxNumberOfFrames); return ptrNewBuffer; } _critSect.Leave(); @@ -1084,9 +1089,8 @@ VCMJitterBuffer::GetFrameForDecoding() _waitingForCompletion.frameSize = oldestFrame->Length(); _waitingForCompletion.latestPacketTime = oldestFrame->LatestPacketTimeMs(); _waitingForCompletion.timestamp = oldestFrame->TimeStamp(); + oldestFrame->SetState(kStateDecoding); } - - oldestFrame->SetState(kStateDecoding); _frameBuffersTSOrder.Erase(oldestFrameListItem); oldestFrameListItem = NULL; @@ -1098,7 +1102,7 @@ VCMJitterBuffer::GetFrameForDecoding() // store current time _lastDecodedTimeStamp = oldestFrame->TimeStamp(); - // store seqnum + // store seqnum _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); return oldestFrame; @@ -1282,9 +1286,9 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word VCMFrameBufferStateEnum state = _frameBuffers[i]->GetState(); if ((kStateFree != state) && - (kStateEmpty != state) && - (kStateDecoding != state) && - seqNum != -1) + (kStateEmpty != state) && + (kStateDecoding != state) && + seqNum != -1) { if (highSeqNum == -1) { diff --git a/modules/video_coding/main/source/session_info.cc b/modules/video_coding/main/source/session_info.cc index 5a5617730..2624da838 100644 --- a/modules/video_coding/main/source/session_info.cc +++ b/modules/video_coding/main/source/session_info.cc @@ -38,17 +38,20 @@ VCMSessionInfo::~VCMSessionInfo() { } -WebRtc_Word32 VCMSessionInfo::GetLowSeqNum() const +WebRtc_Word32 +VCMSessionInfo::GetLowSeqNum() const { return _lowSeqNum; } -WebRtc_Word32 VCMSessionInfo::GetHighSeqNum() const +WebRtc_Word32 +VCMSessionInfo::GetHighSeqNum() const { return _highSeqNum; } -void VCMSessionInfo::Reset() +void +VCMSessionInfo::Reset() { _lowSeqNum = -1; _highSeqNum = -1; @@ -93,9 +96,10 @@ VCMSessionInfo::HaveStartSeqNumber() return true; } -WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, - WebRtc_Word32 packetIndex, - const VCMPacket& packet) +WebRtc_UWord32 +VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, + WebRtc_Word32 packetIndex, + const VCMPacket& packet) { WebRtc_UWord32 moveLength = 0; WebRtc_UWord32 returnLength = 0; @@ -184,7 +188,8 @@ WebRtc_UWord32 VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, return returnLength; } -void VCMSessionInfo::UpdateCompleteSession() +void +VCMSessionInfo::UpdateCompleteSession() { if (_haveFirstPacket && _markerBit) { @@ -209,19 +214,20 @@ bool VCMSessionInfo::IsSessionComplete() // Find the start and end index of packetIndex packet. // startIndex -1 if start not found endIndex=-1 if end index not found -void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, - WebRtc_Word32& startIndex, - WebRtc_Word32& endIndex) +void +VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, + WebRtc_Word32& startIndex, + WebRtc_Word32& endIndex) { - if(_naluCompleteness[packetIndex]==kNaluStart || - _naluCompleteness[packetIndex]==kNaluComplete) + if (_naluCompleteness[packetIndex] == kNaluStart || + _naluCompleteness[packetIndex] == kNaluComplete) { startIndex = packetIndex; } else // Need to find the start { - for(startIndex = packetIndex - 1; startIndex >= 0; --startIndex) + for (startIndex = packetIndex - 1; startIndex >= 0; --startIndex) { if( (_naluCompleteness[startIndex] == kNaluComplete && @@ -233,7 +239,7 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, break; } // This is where the NALU start. - if( _naluCompleteness[startIndex] == kNaluStart) + if (_naluCompleteness[startIndex] == kNaluStart) { break; } @@ -248,38 +254,40 @@ void VCMSessionInfo::FindNaluBorder(WebRtc_Word32 packetIndex, else { // Find the next NALU - for(endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex) + for (endIndex=packetIndex+1;endIndex<=_highestPacketIndex;++endIndex) { - if((_naluCompleteness[endIndex]==kNaluComplete && + if ((_naluCompleteness[endIndex]==kNaluComplete && _packetSizeBytes[endIndex]>0) || _naluCompleteness[endIndex]==kNaluStart) // Found next NALU. { endIndex--; break; } - if( _naluCompleteness[endIndex]==kNaluEnd) // This is where the NALU end. + if ( _naluCompleteness[endIndex]==kNaluEnd) { + // This is where the NALU end. break; } } - if(endIndex > _highestPacketIndex) + if (endIndex > _highestPacketIndex) endIndex = -1; } } // Deletes all packets between startIndex and endIndex -WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, - WebRtc_Word32 startIndex, - WebRtc_Word32 endIndex) +WebRtc_UWord32 +VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, + WebRtc_Word32 startIndex, + WebRtc_Word32 endIndex) { //Get the number of bytes to delete. //Clear the size of these packets. WebRtc_UWord32 bytesToDelete = 0; /// The number of bytes to delete. - for(int j = startIndex;j <= endIndex; ++j) + for (int j = startIndex;j <= endIndex; ++j) { bytesToDelete += _packetSizeBytes[j]; - _packetSizeBytes[j]=0; + _packetSizeBytes[j] = 0; } if (bytesToDelete > 0) { @@ -291,7 +299,7 @@ WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, } //Get the number of bytes to move - WebRtc_UWord32 numberOfBytesToMove=0; + WebRtc_UWord32 numberOfBytesToMove = 0; for (int j = endIndex + 1; j <= _highestPacketIndex; ++j) { numberOfBytesToMove += _packetSizeBytes[j]; @@ -307,10 +315,18 @@ WebRtc_UWord32 VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, // Makes the layer decodable. Ie only contain decodable NALU // return the number of bytes deleted from the session. -1 if an error occurs -WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer) +WebRtc_UWord32 +VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLayer) { if(_lowSeqNum < 0) // No packets in this session + { return 0; + } + else if (_lowSeqNum == _emptySeqNumLow) + { + // no data packets in this session + return 0; + } WebRtc_Word32 startIndex = 0; WebRtc_Word32 endIndex = 0; @@ -321,10 +337,14 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay if (_naluCompleteness[packetIndex] == kNaluUnset) // Found a lost packet { FindNaluBorder(packetIndex,startIndex,endIndex); - if(startIndex == -1) + if (startIndex == -1) + { startIndex = 0; - if(endIndex == -1) + } + if (endIndex == -1) + { endIndex = _highestPacketIndex; + } returnLength += DeletePackets(ptrStartOfLayer,packetIndex,endIndex); packetIndex = endIndex; @@ -332,9 +352,9 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay } //Make sure the first packet is decodable (Either complete nalu or start of NALU) - if(_packetSizeBytes[0] > 0) + if (_packetSizeBytes[0] > 0) { - switch(_naluCompleteness[0]) + switch (_naluCompleteness[0]) { case kNaluComplete: //Packet can be decoded as is. break; @@ -363,8 +383,9 @@ WebRtc_UWord32 VCMSessionInfo::MakeSessionDecodable(WebRtc_UWord8* ptrStartOfLay return returnLength; } -WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, - WebRtc_Word32 numberOfSeqNum) +WebRtc_Word32 +VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, + WebRtc_Word32 numberOfSeqNum) { if ((NULL == list) || (numberOfSeqNum < 1)) { @@ -402,16 +423,17 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, i++; index++; } - if(!_haveFirstPacket) + if (!_haveFirstPacket) { _sessionNACK = true; } return 0; } -WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, - WebRtc_Word32 numberOfSeqNum, - float rttScore) +WebRtc_Word32 +VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, + WebRtc_Word32 numberOfSeqNum, + float rttScore) { if ((NULL == list) || (numberOfSeqNum < 1)) { @@ -508,31 +530,37 @@ WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, return 0; } -WebRtc_Word32 VCMSessionInfo::GetHighestPacketIndex() +WebRtc_Word32 +VCMSessionInfo::GetHighestPacketIndex() { return _highestPacketIndex; } -bool VCMSessionInfo::HaveLastPacket() +bool +VCMSessionInfo::HaveLastPacket() { return _markerBit; } -void VCMSessionInfo::ForceSetHaveLastPacket() +void +VCMSessionInfo::ForceSetHaveLastPacket() { _markerBit = true; UpdateCompleteSession(); } -bool VCMSessionInfo::IsRetransmitted() +bool +VCMSessionInfo::IsRetransmitted() { return _sessionNACK; } -void VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32 length) +void +VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, + WebRtc_UWord32 length) { // sanity - if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) + if (packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) { // not allowed assert(!"SessionInfo::UpdatePacketSize Error: invalid packetIndex"); @@ -541,11 +569,13 @@ void VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, WebRtc_UWord32 _packetSizeBytes[packetIndex] = length; } -void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices) +void +VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices) { // sanity - if((numberOfPacketIndices + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer) - || numberOfPacketIndices < 0) + if ((numberOfPacketIndices + + GetHighestPacketIndex() >= kMaxPacketsInJitterBuffer) + || numberOfPacketIndices < 0) { // not allowed assert(!"SessionInfo::PrependPacketIndexes Error: invalid packetIndex"); @@ -560,10 +590,11 @@ void VCMSessionInfo::PrependPacketIndices(WebRtc_Word32 numberOfPacketIndices) _highestPacketIndex += (WebRtc_UWord16)numberOfPacketIndices; } -void VCMSessionInfo::ClearPacketSize(WebRtc_Word32 packetIndex) +void +VCMSessionInfo::ClearPacketSize(WebRtc_Word32 packetIndex) { // sanity - if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) + if (packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) { // not allowed assert(!"SessionInfo::ClearPacketSize Error: invalid packetIndex"); @@ -572,10 +603,11 @@ void VCMSessionInfo::ClearPacketSize(WebRtc_Word32 packetIndex) _packetSizeBytes[packetIndex] = 0; } -WebRtc_UWord32 VCMSessionInfo::GetPacketSize(WebRtc_Word32 packetIndex) +WebRtc_UWord32 +VCMSessionInfo::GetPacketSize(WebRtc_Word32 packetIndex) { // sanity - if(packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) + if (packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) { //not allowed assert(!"SessionInfo::GetPacketSize Error: invalid packetIndex"); @@ -585,9 +617,10 @@ WebRtc_UWord32 VCMSessionInfo::GetPacketSize(WebRtc_Word32 packetIndex) } WebRtc_Word64 -VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer) +VCMSessionInfo::InsertPacket(const VCMPacket& packet, + WebRtc_UWord8* ptrStartOfLayer) { - //not allowed + // not allowed assert(!packet.insertStartCode || !packet.bits); if (packet.frameType == kFrameEmpty) @@ -695,7 +728,8 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfL } // update highest packet index - _highestPacketIndex = packetIndex > _highestPacketIndex ? packetIndex :_highestPacketIndex; + _highestPacketIndex = packetIndex > _highestPacketIndex ? + packetIndex :_highestPacketIndex; return InsertBuffer(ptrStartOfLayer, packetIndex, packet); } @@ -743,7 +777,8 @@ VCMSessionInfo::InformOfEmptyPacket(const WebRtc_UWord16 seqNum) } WebRtc_UWord32 -VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, VideoCodecType codec) +VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, + VideoCodecType codec) { WebRtc_UWord32 currentPacketOffset = 0; WebRtc_UWord32 length = GetSessionLength();