diff --git a/src/modules/video_coding/main/source/frame_buffer.cc b/src/modules/video_coding/main/source/frame_buffer.cc index f143044c2..14f448b0b 100644 --- a/src/modules/video_coding/main/source/frame_buffer.cc +++ b/src/modules/video_coding/main/source/frame_buffer.cc @@ -81,7 +81,8 @@ VCMFrameBuffer::IsSessionComplete() const // Insert packet VCMFrameBufferEnum -VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) +VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs, + bool enableDecodableState, WebRtc_UWord32 rttMS) { if (_state == kStateDecoding) { @@ -159,7 +160,9 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) CopyCodecSpecific(&packet.codecSpecificHeader); - WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer); + WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer, + enableDecodableState, + rttMS); if (retVal == -1) { return kSizeError; @@ -173,19 +176,18 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs) _latestPacketTimeMs = timeInMs; - if (_sessionInfo.IsSessionComplete()) - { - return kCompleteSession; - } - else - { - // this layer is not complete - if (_state == kStateComplete) - { - // we already have a complete layer - // wait for all independent layers belonging to the same frame - _state = kStateIncomplete; - } + if (_sessionInfo.IsSessionComplete()) { + return kCompleteSession; + } else if (_sessionInfo.IsSessionDecodable()) { + SetState(kStateDecodable); + return kDecodableSession; + } else { + // this layer is not complete + if (_state == kStateComplete) { + // we already have a complete layer + // wait for all independent layers belonging to the same frame + _state = kStateIncomplete; + } } return kIncomplete; } @@ -215,9 +217,9 @@ VCMFrameBuffer::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num) WebRtc_Word32 VCMFrameBuffer::ZeroOutSeqNumHybrid(WebRtc_Word32* list, WebRtc_Word32 num, - float rttScore) + WebRtc_UWord32 rttMs) { - return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttScore); + return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttMs); } void @@ -308,7 +310,7 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) break; case kStateDecoding: - // A frame migth have received empty packets, or media packets might + // A frame might have received empty packets, or media packets might // have been removed when making the frame decodable. The frame can // still be set to decodable since it can be used to inform the // decoder of a frame loss. @@ -320,11 +322,6 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) break; case kStateDecodable: - if (_state == kStateComplete) - { - // if complete, obviously decodable, keep as is. - return; - } assert(_state == kStateEmpty || _state == kStateIncomplete); break; diff --git a/src/modules/video_coding/main/source/frame_buffer.h b/src/modules/video_coding/main/source/frame_buffer.h index b55440865..fcc6f0b49 100644 --- a/src/modules/video_coding/main/source/frame_buffer.h +++ b/src/modules/video_coding/main/source/frame_buffer.h @@ -33,7 +33,9 @@ public: virtual void Reset(); VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, - WebRtc_Word64 timeInMs); + WebRtc_Word64 timeInMs, + bool enableDecodableState, + WebRtc_UWord32 rttMs); // State // Get current state of frame @@ -66,7 +68,7 @@ public: // Hybrid extension: only NACK important packets, discard FEC packets WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list, WebRtc_Word32 num, - float rttScore); + WebRtc_UWord32 rttMs); void IncrementNackCount(); WebRtc_Word16 GetNackCount() const; diff --git a/src/modules/video_coding/main/source/jitter_buffer.cc b/src/modules/video_coding/main/source/jitter_buffer.cc index 42aa9fb43..5cc92b165 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.cc +++ b/src/modules/video_coding/main/source/jitter_buffer.cc @@ -391,7 +391,8 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) } } - const VCMFrameListItem* oldFrameListItem = FindOldestCompleteContinuousFrame(); + const VCMFrameListItem* + oldFrameListItem = FindOldestCompleteContinuousFrame(false); VCMFrameBuffer* oldFrame = NULL; if (oldFrameListItem != NULL) { @@ -590,59 +591,56 @@ VCMJitterBuffer::FindOldestSequenceNum() const // Based on sequence number // Return NULL for lost packets VCMFrameListItem* -VCMJitterBuffer::FindOldestCompleteContinuousFrame() -{ - // if we have more than one frame done since last time, pick oldest - VCMFrameBuffer* oldestFrame = NULL; - int currentLow = -1; +VCMJitterBuffer::FindOldestCompleteContinuousFrame(bool enableDecodable) { + // If we have more than one frame done since last time, pick oldest. + VCMFrameBuffer* oldestFrame = NULL; + int currentLow = -1; - VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First(); - if (oldestFrameItem != NULL) - { - oldestFrame = oldestFrameItem->GetItem(); - } - // is the frame complete? - if (oldestFrame != NULL && kStateComplete != oldestFrame->GetState()) - { - oldestFrame = NULL; - } - if (oldestFrame == NULL) - { - // no complete frame no point to continue - return NULL; + VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First(); + if (oldestFrameItem != NULL) { + oldestFrame = oldestFrameItem->GetItem(); + } + if (oldestFrame != NULL) { + // Check for a complete or decodable frame (when enabled). + VCMFrameBufferStateEnum state = oldestFrame->GetState(); + bool decodable = enableDecodable && (state == kStateDecodable); + if ((state != kStateComplete) && !decodable) { + oldestFrame = NULL; } + } + if (oldestFrame == NULL) { + // No complete frame no point to continue + return NULL; + } - // we have a complete frame - // check if it's continuous, otherwise we are missing a full frame + // We have a complete frame. + // Check if it's continuous, otherwise we are missing a full frame. - // Use pictureId if available. Otherwise use seqNum and not timestamps, as - // a complete frame might be lost - // First determine if we are waiting for a key frame. - if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey) { - return NULL; - } - // Is this the first frame to be decoded? - // Otherwise, verify continuity - use PictureId when available. - if (_lastDecodedSeqNum == -1) { - return oldestFrameItem; - } else if (oldestFrame->PictureId() == kNoPictureId) { - // It's not enough that we have a complete frame we need the layers to - // be continuous too. - currentLow = oldestFrame->GetLowSeqNum(); - WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum; - - // We could have received the first packet of the last frame before a - // long period if drop, that case is handled by GetNackList - if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) { - // Wait since we want a complete continuous frame - return NULL; - } - } - else if (!ContinuousPictureId(oldestFrame->PictureId())) { - // We don't have a continuous frame - return NULL; - } + // Use pictureId if available. Otherwise use seqNum and not timestamps, as + // a complete frame might be lost. + // First determine if we are waiting for a key frame. + if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey) { + return NULL; + } + // Is this the first frame to be decoded? + // Otherwise, establish continuity - use PictureId when available. + if (_lastDecodedSeqNum == -1) { return oldestFrameItem; + } else if (oldestFrame->PictureId() == kNoPictureId) { + currentLow = oldestFrame->GetLowSeqNum(); + WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum; + + // We could have received the first packet of the last frame before a long + // period if drop, that case is handled by GetNackList (when applicable). + if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) { + // Wait since we want a complete continuous frame. + return NULL; + } + } else if (!ContinuousPictureId(oldestFrame->PictureId())) { + // We don't have a continuous frame. + return NULL; + } + return oldestFrameItem; } // Call from inside the critical section _critSect @@ -731,7 +729,7 @@ VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate) return 0; } -// Returns immediately or a X ms event hang waiting for a decodable frame, +// Returns immediately or a X ms event hang waiting for a complete frame, // X decided by caller VCMEncodedFrame* VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) @@ -749,7 +747,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) if (_lastDecodedSeqNum == -1 && WaitForNack()) { _waitingForKeyFrame = true; } - VCMFrameListItem* oldestFrameListItem = FindOldestCompleteContinuousFrame(); + VCMFrameListItem* + oldestFrameListItem = FindOldestCompleteContinuousFrame(false); VCMFrameBuffer* oldestFrame = NULL; if (oldestFrameListItem != NULL) { @@ -785,7 +784,7 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS) // sequence number and size CleanUpOldFrames(); CleanUpSizeZeroFrames(); - oldestFrameListItem = FindOldestCompleteContinuousFrame(); + oldestFrameListItem = FindOldestCompleteContinuousFrame(false); if (oldestFrameListItem != NULL) { oldestFrame = oldestFrameListItem->GetItem(); @@ -1117,7 +1116,11 @@ VCMJitterBuffer::GetFrameForDecodingNACK() if (_lastDecodedSeqNum == -1) { _waitingForKeyFrame = true; } - VCMFrameListItem* oldestFrameListItem = FindOldestCompleteContinuousFrame(); + + // Allow for a decodable frame when in Hybrid mode. + bool enableDecodable = _nackMode == kNackHybrid ? true : false; + VCMFrameListItem* + oldestFrameListItem = FindOldestCompleteContinuousFrame(enableDecodable); VCMFrameBuffer* oldestFrame = NULL; if (oldestFrameListItem != NULL) { @@ -1473,18 +1476,9 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended) // not to add empty packets to the nack list if (_nackMode == kNackHybrid) { - // build external rttScore based on RTT value - float rttScore = 1.0f; - int ret = _frameBuffers[i]->ZeroOutSeqNumHybrid(_NACKSeqNumInternal, numberOfSeqNum, - rttScore); - if (_frameBuffers[i]->IsRetransmitted() == false && ret != -1) - { - // If no retransmission required,set the state to decodable - // meaning that we will not wait for NACK. - _frameBuffers[i]->SetState(kStateDecodable); - } + _rttMs); } else { @@ -1659,7 +1653,13 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) // High sequence number will be -1 if neither an empty packet nor // a media packet has been inserted. bool first = (frame->GetHighSeqNum() == -1); - bufferReturn = frame->InsertPacket(packet, nowMs); + // When in Hybrid mode, we allow for a decodable state + // Note: Under current version, a decodable frame will never be + // triggered, as the body of the function is empty. + // TODO (mikhal): Update when decodable is enabled. + bufferReturn = frame->InsertPacket(packet, nowMs, + _nackMode == kNackHybrid, + _rttMs); ret = bufferReturn; if (bufferReturn > 0) @@ -1702,11 +1702,12 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) _packetEvent.Set(); break; } + case kDecodableSession: case kIncomplete: { - // Signal that we have a received packet - _packetEvent.Set(); - break; + // Signal that we have a received packet + _packetEvent.Set(); + break; } case kNoError: case kDuplicatePacket: diff --git a/src/modules/video_coding/main/source/jitter_buffer.h b/src/modules/video_coding/main/source/jitter_buffer.h index 225e76a32..2a5a5054a 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.h +++ b/src/modules/video_coding/main/source/jitter_buffer.h @@ -89,6 +89,7 @@ public: // or more packets? bool CompleteSequenceWithNextFrame(); + // TODO (mikhal/stefan): Merge all GetFrameForDecoding into one. // Wait maxWaitTimeMS for a complete frame to arrive. After timeout NULL // is returned. VCMEncodedFrame* GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS); @@ -148,7 +149,8 @@ private: // Help functions for getting a frame // Find oldest complete frame, used for getting next frame to decode - VCMFrameListItem* FindOldestCompleteContinuousFrame(); + // When enabled, will return a decodable frame + VCMFrameListItem* FindOldestCompleteContinuousFrame(bool enableDecodable); void CleanUpOldFrames(); void CleanUpSizeZeroFrames(); diff --git a/src/modules/video_coding/main/source/jitter_buffer_common.h b/src/modules/video_coding/main/source/jitter_buffer_common.h index f86b2173e..167399d41 100644 --- a/src/modules/video_coding/main/source/jitter_buffer_common.h +++ b/src/modules/video_coding/main/source/jitter_buffer_common.h @@ -39,6 +39,7 @@ enum VCMFrameBufferEnum kIncomplete = 1, // Frame incomplete kFirstPacket = 2, kCompleteSession = 3, // at least one layer in the frame complete + kDecodableSession = 4, // Frame incomplete, but ready to be decoded kDuplicatePacket = 5 // We're receiving a duplicate packet. }; diff --git a/src/modules/video_coding/main/source/session_info.cc b/src/modules/video_coding/main/source/session_info.cc index 799a5d3e7..58206d0d0 100644 --- a/src/modules/video_coding/main/source/session_info.cc +++ b/src/modules/video_coding/main/source/session_info.cc @@ -18,6 +18,7 @@ VCMSessionInfo::VCMSessionInfo(): _markerBit(false), _sessionNACK(false), _completeSession(false), + _decodableSession(false), _frameType(kVideoFrameDelta), _previousFrameLoss(false), _lowSeqNum(-1), @@ -70,6 +71,7 @@ VCMSessionInfo::Reset() { _emptySeqNumHigh = -1; _markerBit = false; _completeSession = false; + _decodableSession = false; _frameType = kVideoFrameDelta; _previousFrameLoss = false; _sessionNACK = false; @@ -186,34 +188,37 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer, _markerSeqNum = packet.seqNum; } - UpdateCompleteSession(); - return returnLength; } -void -VCMSessionInfo::UpdateCompleteSession() -{ - if (_packets[0].isFirstPacket && _markerBit) - { - // Do we have all the packets in this session? - bool completeSession = true; +void VCMSessionInfo::UpdateCompleteSession() { + if (_packets[0].isFirstPacket && _markerBit) { + // Do we have all the packets in this session? + bool completeSession = true; - for (int i = 0; i <= _highestPacketIndex; ++i) - { - if (_packets[i].completeNALU == kNaluUnset) - { - completeSession = false; - break; - } - } - _completeSession = completeSession; + for (int i = 0; i <= _highestPacketIndex; ++i) { + if (_packets[i].completeNALU == kNaluUnset) { + completeSession = false; + break; + } } + _completeSession = completeSession; + } } -bool VCMSessionInfo::IsSessionComplete() const -{ - return _completeSession; +void VCMSessionInfo::UpdateDecodableSession(WebRtc_UWord32 rttMs) { + // Irrelevant if session is already complete or decodable + if (_completeSession || _decodableSession) + return; + // First iteration - do nothing +} + +bool VCMSessionInfo::IsSessionComplete() const { + return _completeSession; +} + +bool VCMSessionInfo::IsSessionDecodable() const { + return _decodableSession; } // Find the start and end index of packetIndex packet. @@ -537,7 +542,7 @@ VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, WebRtc_Word32 numberOfSeqNum, - float rttScore) + WebRtc_UWord32 rttMs) { if (NULL == list || numberOfSeqNum < 1) { @@ -590,6 +595,9 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list, _emptySeqNumLow - 1: _highSeqNum; } + // Place holder + int rttScore = 1.0f; + while (list[index] <= highMediaPacket && index < numberOfSeqNum) { if (_packets[i].completeNALU != kNaluUnset) @@ -659,7 +667,9 @@ VCMSessionInfo::IsRetransmitted() const WebRtc_Word64 VCMSessionInfo::InsertPacket(const VCMPacket& packet, - WebRtc_UWord8* ptrStartOfLayer) + WebRtc_UWord8* ptrStartOfLayer, + bool enableDecodableState, + WebRtc_UWord32 rttMs) { // not allowed assert(!packet.insertStartCode || !packet.bits); @@ -765,7 +775,11 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet, _highestPacketIndex = packetIndex > _highestPacketIndex ? packetIndex :_highestPacketIndex; - return InsertBuffer(ptrStartOfLayer, packetIndex, packet); + int returnLength = InsertBuffer(ptrStartOfLayer, packetIndex, packet); + UpdateCompleteSession(); + if (enableDecodableState) + UpdateDecodableSession(rttMs); + return returnLength; } diff --git a/src/modules/video_coding/main/source/session_info.h b/src/modules/video_coding/main/source/session_info.h index 942c85de2..3a1aa8eb3 100644 --- a/src/modules/video_coding/main/source/session_info.h +++ b/src/modules/video_coding/main/source/session_info.h @@ -34,17 +34,20 @@ public: WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 numberOfSeqNum); // Hybrid version: Zero out seq num for NACK list - // apply a score based on the packet location and the external rttScore + // Selectively NACK packets. WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list, WebRtc_Word32 numberOfSeqNum, - float rttScore); + WebRtc_UWord32 rttMs); virtual void Reset(); WebRtc_Word64 InsertPacket(const VCMPacket& packet, - WebRtc_UWord8* ptrStartOfLayer); + WebRtc_UWord8* ptrStartOfLayer, + bool enableDecodableState, + WebRtc_UWord32 rttMs); WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum); virtual bool IsSessionComplete() const; + virtual bool IsSessionDecodable() const; // Builds fragmentation headers for VP8, each fragment being a decodable // VP8 partition. Returns the total number of bytes which are decodable. Is @@ -106,11 +109,15 @@ private: WebRtc_Word32 startIndex, WebRtc_Word32 endIndex); void UpdateCompleteSession(); + // When enabled, determine if session is decodable, i.e. incomplete but + // would be sent to the decoder. + void UpdateDecodableSession(WebRtc_UWord32 rttMs); // If we have inserted a packet with markerbit into this frame bool _markerBit; // If this session has been NACKed by JB bool _sessionNACK; bool _completeSession; + bool _decodableSession; webrtc::FrameType _frameType; bool _previousFrameLoss; // Lowest/Highest packet sequence number in a session