video_coding: Allowing for a decodable state independent of selective nacking

Review URL: http://webrtc-codereview.appspot.com/263001

git-svn-id: http://webrtc.googlecode.com/svn/trunk@993 4adac7df-926f-26a2-2b94-8c16560cd09d

src/modules/video_coding/main/source/frame_buffer.cc

@@ -81,7 +81,8 @@ VCMFrameBuffer::IsSessionComplete() const
 
 // Insert packet
 VCMFrameBufferEnum
-VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
+VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs,
+                             bool enableDecodableState, WebRtc_UWord32 rttMS)
 {
     if (_state == kStateDecoding)
     {
@@ -159,7 +160,9 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
 
     CopyCodecSpecific(&packet.codecSpecificHeader);
 
-    WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer);
+    WebRtc_Word64 retVal = _sessionInfo.InsertPacket(packet, _buffer,
+                                                     enableDecodableState,
+                                                     rttMS);
     if (retVal == -1)
     {
         return kSizeError;
@@ -173,19 +176,18 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, WebRtc_Word64 timeInMs)
 
     _latestPacketTimeMs = timeInMs;
 
-    if (_sessionInfo.IsSessionComplete())
+    if (_sessionInfo.IsSessionComplete()) {
-    {
+      return kCompleteSession;
-        return kCompleteSession;
+    } else if (_sessionInfo.IsSessionDecodable()) {
-    }
+      SetState(kStateDecodable);
-    else
+      return kDecodableSession;
-    {
+    } else {
-        // this layer is not complete
+      // this layer is not complete
-        if (_state == kStateComplete)
+      if (_state == kStateComplete) {
-        {
+        // we already have a complete layer
-            // we already have a complete layer
+        // wait for all independent layers belonging to the same frame
-            // wait for all independent layers belonging to the same frame
+        _state = kStateIncomplete;
-            _state = kStateIncomplete;
+      }
-        }
     }
     return kIncomplete;
 }
@@ -215,9 +217,9 @@ VCMFrameBuffer::ZeroOutSeqNum(WebRtc_Word32* list, WebRtc_Word32 num)
 WebRtc_Word32
 VCMFrameBuffer::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
                                     WebRtc_Word32 num,
-                                    float rttScore)
+                                    WebRtc_UWord32 rttMs)
 {
-    return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttScore);
+    return _sessionInfo.ZeroOutSeqNumHybrid(list, num, rttMs);
 }
 
 void
@@ -308,7 +310,7 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
         break;
 
     case kStateDecoding:
-        // A frame migth have received empty packets, or media packets might
+        // A frame might have received empty packets, or media packets might
         // have been removed when making the frame decodable. The frame can
         // still be set to decodable since it can be used to inform the
         // decoder of a frame loss.
@@ -320,11 +322,6 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state)
         break;
 
     case kStateDecodable:
-        if (_state == kStateComplete)
-        {
-            // if complete, obviously decodable, keep as is.
-            return;
-        }
         assert(_state == kStateEmpty ||
                _state == kStateIncomplete);
         break;

src/modules/video_coding/main/source/frame_buffer.h

@@ -33,7 +33,9 @@ public:
     virtual void Reset();
 
     VCMFrameBufferEnum InsertPacket(const VCMPacket& packet,
-                                    WebRtc_Word64 timeInMs);
+                                    WebRtc_Word64 timeInMs,
+                                    bool enableDecodableState,
+                                    WebRtc_UWord32 rttMs);
 
     // State
     // Get current state of frame
@@ -66,7 +68,7 @@ public:
     // Hybrid extension: only NACK important packets, discard FEC packets
     WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
                                       WebRtc_Word32 num,
-                                      float rttScore);
+                                      WebRtc_UWord32 rttMs);
     void IncrementNackCount();
     WebRtc_Word16 GetNackCount() const;
 

src/modules/video_coding/main/source/jitter_buffer.cc

@@ -391,7 +391,8 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
 
         }
     }
-    const VCMFrameListItem* oldFrameListItem = FindOldestCompleteContinuousFrame();
+    const VCMFrameListItem*
+      oldFrameListItem = FindOldestCompleteContinuousFrame(false);
     VCMFrameBuffer* oldFrame = NULL;
     if (oldFrameListItem != NULL)
     {
@@ -590,59 +591,56 @@ VCMJitterBuffer::FindOldestSequenceNum() const
 // Based on sequence number
 // Return NULL for lost packets
 VCMFrameListItem*
-VCMJitterBuffer::FindOldestCompleteContinuousFrame()
+VCMJitterBuffer::FindOldestCompleteContinuousFrame(bool enableDecodable) {
-{
+  // If we have more than one frame done since last time, pick oldest.
-    // if we have more than one frame done since last time, pick oldest
+  VCMFrameBuffer* oldestFrame = NULL;
-    VCMFrameBuffer* oldestFrame = NULL;
+  int currentLow = -1;
-    int currentLow = -1;
 
-    VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First();
+  VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First();
-    if (oldestFrameItem != NULL)
+  if (oldestFrameItem != NULL) {
-    {
+    oldestFrame = oldestFrameItem->GetItem();
-        oldestFrame = oldestFrameItem->GetItem();
+  }
-    }
+  if (oldestFrame != NULL) {
-    // is the frame complete?
+    // Check for a complete or decodable frame (when enabled).
-    if (oldestFrame != NULL && kStateComplete != oldestFrame->GetState())
+    VCMFrameBufferStateEnum state = oldestFrame->GetState();
-    {
+    bool decodable = enableDecodable && (state == kStateDecodable);
-        oldestFrame = NULL;
+    if ((state != kStateComplete) && !decodable) {
-    }
+      oldestFrame = NULL;
-    if (oldestFrame == NULL)
-    {
-        // no complete frame no point to continue
-        return NULL;
     }
+  }
+  if (oldestFrame == NULL) {
+    // No complete frame no point to continue
+    return NULL;
+  }
 
-    // we have a complete frame
+  // We have a complete frame.
-    // check if it's continuous, otherwise we are missing a full frame
+  // Check if it's continuous, otherwise we are missing a full frame.
 
-    // Use pictureId if available. Otherwise use seqNum and not timestamps, as
+  // Use pictureId if available. Otherwise use seqNum and not timestamps, as
-    // a complete frame might be lost
+  // a complete frame might be lost.
-    // First determine if we are waiting for a key frame.
+  // First determine if we are waiting for a key frame.
-    if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey)  {
+  if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey) {
-      return NULL;
+    return NULL;
-    }
+  }
-    // Is this the first frame to be decoded?
+  // Is this the first frame to be decoded?
-    // Otherwise, verify continuity - use PictureId when available.
+  // Otherwise, establish continuity - use PictureId when available.
-    if (_lastDecodedSeqNum == -1)  {
+  if (_lastDecodedSeqNum == -1) {
-      return oldestFrameItem;
-    } else if (oldestFrame->PictureId() == kNoPictureId)  {
-      // It's not enough that we have a complete frame we need the layers to
-      // be continuous too.
-      currentLow = oldestFrame->GetLowSeqNum();
-      WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
-
-      // We could have received the first packet of the last frame before a
-      // long period if drop, that case is handled by GetNackList
-      if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow)  {
-        // Wait since we want a complete continuous frame
-        return NULL;
-      }
-    }
-    else if (!ContinuousPictureId(oldestFrame->PictureId()))  {
-      // We don't have a continuous frame
-      return NULL;
-    }
     return oldestFrameItem;
+  } else if (oldestFrame->PictureId() == kNoPictureId) {
+    currentLow = oldestFrame->GetLowSeqNum();
+    WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
+
+    // We could have received the first packet of the last frame before a long
+    // period if drop, that case is handled by GetNackList (when applicable).
+    if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) {
+      // Wait since we want a complete continuous frame.
+      return NULL;
+    }
+  } else if (!ContinuousPictureId(oldestFrame->PictureId())) {
+    // We don't have a continuous frame.
+    return NULL;
+  }
+  return oldestFrameItem;
 }
 
 // Call from inside the critical section _critSect
@@ -731,7 +729,7 @@ VCMJitterBuffer::GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate)
     return 0;
 }
 
-// Returns immediately or a X ms event hang waiting for a decodable frame,
+// Returns immediately or a X ms event hang waiting for a complete frame,
 // X decided by caller
 VCMEncodedFrame*
 VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
@@ -749,7 +747,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
     if (_lastDecodedSeqNum == -1 && WaitForNack()) {
       _waitingForKeyFrame = true;
     }
-    VCMFrameListItem* oldestFrameListItem = FindOldestCompleteContinuousFrame();
+    VCMFrameListItem*
+      oldestFrameListItem = FindOldestCompleteContinuousFrame(false);
     VCMFrameBuffer* oldestFrame = NULL;
     if (oldestFrameListItem != NULL)
     {
@@ -785,7 +784,7 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
                 // sequence number and size
                 CleanUpOldFrames();
                 CleanUpSizeZeroFrames();
-                oldestFrameListItem = FindOldestCompleteContinuousFrame();
+                oldestFrameListItem = FindOldestCompleteContinuousFrame(false);
                 if (oldestFrameListItem != NULL)
                 {
                     oldestFrame = oldestFrameListItem->GetItem();
@@ -1117,7 +1116,11 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
     if (_lastDecodedSeqNum == -1) {
       _waitingForKeyFrame = true;
     }
-    VCMFrameListItem* oldestFrameListItem = FindOldestCompleteContinuousFrame();
+
+    // Allow for a decodable frame when in Hybrid mode.
+    bool enableDecodable = _nackMode == kNackHybrid ? true : false;
+    VCMFrameListItem*
+    oldestFrameListItem = FindOldestCompleteContinuousFrame(enableDecodable);
     VCMFrameBuffer* oldestFrame = NULL;
     if (oldestFrameListItem != NULL)
     {
@@ -1473,18 +1476,9 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
             // not to add empty packets to the nack list
             if (_nackMode == kNackHybrid)
             {
-                // build external rttScore based on RTT value
-                float rttScore = 1.0f;
-                int ret =
                 _frameBuffers[i]->ZeroOutSeqNumHybrid(_NACKSeqNumInternal,
                                                       numberOfSeqNum,
-                                                      rttScore);
+                                                      _rttMs);
-                if (_frameBuffers[i]->IsRetransmitted() == false && ret != -1)
-                {
-                    // If no retransmission required,set the state to decodable
-                    // meaning that we will not wait for NACK.
-                    _frameBuffers[i]->SetState(kStateDecodable);
-                }
             }
             else
             {
@@ -1659,7 +1653,13 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
         // High sequence number will be -1 if neither an empty packet nor
         // a media packet has been inserted.
         bool first = (frame->GetHighSeqNum() == -1);
-        bufferReturn = frame->InsertPacket(packet, nowMs);
+        // When in Hybrid mode, we allow for a decodable state
+        // Note: Under current version, a decodable frame will never be
+        // triggered, as the body of the function is empty.
+        // TODO (mikhal): Update when decodable is enabled.
+        bufferReturn = frame->InsertPacket(packet, nowMs,
+                                           _nackMode == kNackHybrid,
+                                           _rttMs);
         ret = bufferReturn;
 
         if (bufferReturn > 0)
@@ -1702,11 +1702,12 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
             _packetEvent.Set();
             break;
         }
+    case kDecodableSession:
     case kIncomplete:
         {
-            // Signal that we have a received packet
+          // Signal that we have a received packet
-            _packetEvent.Set();
+          _packetEvent.Set();
-            break;
+          break;
         }
     case kNoError:
     case kDuplicatePacket:

src/modules/video_coding/main/source/jitter_buffer.h

@@ -89,6 +89,7 @@ public:
     // or more packets?
     bool CompleteSequenceWithNextFrame();
 
+    // TODO (mikhal/stefan): Merge all GetFrameForDecoding into one.
     // Wait maxWaitTimeMS for a complete frame to arrive. After timeout NULL
     // is returned.
     VCMEncodedFrame* GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS);
@@ -148,7 +149,8 @@ private:
 
     // Help functions for getting a frame
     // Find oldest complete frame, used for getting next frame to decode
-    VCMFrameListItem* FindOldestCompleteContinuousFrame();
+    // When enabled, will return a decodable frame
+    VCMFrameListItem* FindOldestCompleteContinuousFrame(bool enableDecodable);
 
     void CleanUpOldFrames();
     void CleanUpSizeZeroFrames();

src/modules/video_coding/main/source/jitter_buffer_common.h

@@ -39,6 +39,7 @@ enum VCMFrameBufferEnum
     kIncomplete           = 1,    // Frame incomplete
     kFirstPacket          = 2,
     kCompleteSession      = 3,    // at least one layer in the frame complete
+    kDecodableSession     = 4,    // Frame incomplete, but ready to be decoded
     kDuplicatePacket      = 5     // We're receiving a duplicate packet.
 };
 

src/modules/video_coding/main/source/session_info.cc

@@ -18,6 +18,7 @@ VCMSessionInfo::VCMSessionInfo():
     _markerBit(false),
     _sessionNACK(false),
     _completeSession(false),
+    _decodableSession(false),
     _frameType(kVideoFrameDelta),
     _previousFrameLoss(false),
     _lowSeqNum(-1),
@@ -70,6 +71,7 @@ VCMSessionInfo::Reset() {
   _emptySeqNumHigh = -1;
   _markerBit = false;
   _completeSession = false;
+  _decodableSession = false;
   _frameType = kVideoFrameDelta;
   _previousFrameLoss = false;
   _sessionNACK = false;
@@ -186,34 +188,37 @@ VCMSessionInfo::InsertBuffer(WebRtc_UWord8* ptrStartOfLayer,
         _markerSeqNum = packet.seqNum;
     }
 
-    UpdateCompleteSession();
-
     return returnLength;
 }
 
-void
+void VCMSessionInfo::UpdateCompleteSession() {
-VCMSessionInfo::UpdateCompleteSession()
+  if (_packets[0].isFirstPacket && _markerBit) {
-{
+    // Do we have all the packets in this session?
-    if (_packets[0].isFirstPacket && _markerBit)
+    bool completeSession = true;
-    {
-        // Do we have all the packets in this session?
-        bool completeSession = true;
 
-        for (int i = 0; i <= _highestPacketIndex; ++i)
+    for (int i = 0; i <= _highestPacketIndex; ++i) {
-        {
+      if (_packets[i].completeNALU == kNaluUnset) {
-            if (_packets[i].completeNALU == kNaluUnset)
+        completeSession = false;
-            {
+        break;
-                completeSession = false;
+      }
-                break;
-            }
-        }
-        _completeSession = completeSession;
     }
+    _completeSession = completeSession;
+  }
 }
 
-bool VCMSessionInfo::IsSessionComplete() const
+void VCMSessionInfo::UpdateDecodableSession(WebRtc_UWord32 rttMs) {
-{
+  // Irrelevant if session is already complete or decodable
-    return _completeSession;
+  if (_completeSession || _decodableSession)
+    return;
+  // First iteration - do nothing
+}
+
+bool VCMSessionInfo::IsSessionComplete() const {
+  return _completeSession;
+}
+
+bool VCMSessionInfo::IsSessionDecodable() const {
+  return _decodableSession;
 }
 
 // Find the start and end index of packetIndex packet.
@@ -537,7 +542,7 @@ VCMSessionInfo::ZeroOutSeqNum(WebRtc_Word32* list,
 WebRtc_Word32
 VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
                                     WebRtc_Word32 numberOfSeqNum,
-                                    float rttScore)
+                                    WebRtc_UWord32 rttMs)
 {
     if (NULL == list || numberOfSeqNum < 1)
     {
@@ -590,6 +595,9 @@ VCMSessionInfo::ZeroOutSeqNumHybrid(WebRtc_Word32* list,
                           _emptySeqNumLow - 1: _highSeqNum;
     }
 
+    // Place holder
+    int rttScore = 1.0f;
+
     while (list[index] <= highMediaPacket && index < numberOfSeqNum)
     {
         if (_packets[i].completeNALU != kNaluUnset)
@@ -659,7 +667,9 @@ VCMSessionInfo::IsRetransmitted() const
 
 WebRtc_Word64
 VCMSessionInfo::InsertPacket(const VCMPacket& packet,
-                             WebRtc_UWord8* ptrStartOfLayer)
+                             WebRtc_UWord8* ptrStartOfLayer,
+                             bool enableDecodableState,
+                             WebRtc_UWord32 rttMs)
 {
     // not allowed
     assert(!packet.insertStartCode || !packet.bits);
@@ -765,7 +775,11 @@ VCMSessionInfo::InsertPacket(const VCMPacket& packet,
     _highestPacketIndex = packetIndex > _highestPacketIndex ?
                                         packetIndex :_highestPacketIndex;
 
-    return InsertBuffer(ptrStartOfLayer, packetIndex, packet);
+    int returnLength =  InsertBuffer(ptrStartOfLayer, packetIndex, packet);
+    UpdateCompleteSession();
+    if (enableDecodableState)
+      UpdateDecodableSession(rttMs);
+    return returnLength;
 }
 
 

src/modules/video_coding/main/source/session_info.h

@@ -34,17 +34,20 @@ public:
     WebRtc_Word32 ZeroOutSeqNum(WebRtc_Word32* list,
                                 WebRtc_Word32 numberOfSeqNum);
     // Hybrid version: Zero out seq num for NACK list
-    // apply a score based on the packet location and the external rttScore
+    // Selectively NACK packets.
     WebRtc_Word32 ZeroOutSeqNumHybrid(WebRtc_Word32* list,
                                       WebRtc_Word32 numberOfSeqNum,
-                                      float rttScore);
+                                      WebRtc_UWord32 rttMs);
     virtual void Reset();
 
     WebRtc_Word64 InsertPacket(const VCMPacket& packet,
-                               WebRtc_UWord8* ptrStartOfLayer);
+                               WebRtc_UWord8* ptrStartOfLayer,
+                               bool enableDecodableState,
+                               WebRtc_UWord32 rttMs);
     WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum);
 
     virtual bool IsSessionComplete() const;
+    virtual bool IsSessionDecodable() const;
 
     // Builds fragmentation headers for VP8, each fragment being a decodable
     // VP8 partition. Returns the total number of bytes which are decodable. Is
@@ -106,11 +109,15 @@ private:
                                  WebRtc_Word32 startIndex,
                                  WebRtc_Word32 endIndex);
     void UpdateCompleteSession();
+    // When enabled, determine if session is decodable, i.e. incomplete but
+    // would be sent to the decoder.
+    void UpdateDecodableSession(WebRtc_UWord32 rttMs);
     // If we have inserted a packet with markerbit into this frame
     bool               _markerBit;
     // If this session has been NACKed by JB
     bool               _sessionNACK;
     bool               _completeSession;
+    bool               _decodableSession;
     webrtc::FrameType  _frameType;
     bool               _previousFrameLoss;
     // Lowest/Highest packet sequence number in a session