video-coding: Adding a decoded state to the JB logic (JB refactor).

This new class stores the last decoded info, including temporal info. 
Review URL: http://webrtc-codereview.appspot.com/300005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1174 4adac7df-926f-26a2-2b94-8c16560cd09d

src/modules/video_coding/main/source/decoding_state.cc

@@ -0,0 +1,203 @@
+/*
+ *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/main/source/decoding_state.h"
+#include "modules/video_coding/main/source/frame_buffer.h"
+#include "modules/video_coding/main/source/jitter_buffer_common.h"
+#include "modules/video_coding/main/source/packet.h"
+
+#include "modules/interface/module_common_types.h"
+
+namespace webrtc {
+
+VCMDecodingState::VCMDecodingState()
+    : sequence_num_(0),
+      time_stamp_(0),
+      picture_id_(kNoPictureId),
+      temporal_id_(kNoTemporalIdx),
+      tl0_pic_id_(kNoTl0PicIdx),
+      full_sync_(true),
+      init_(true) {}
+
+VCMDecodingState::~VCMDecodingState() {}
+
+void VCMDecodingState::Reset() {
+  // TODO(mikhal): Verify - not always would want to reset the sync
+  sequence_num_ = 0;
+  time_stamp_ = 0;
+  picture_id_ = kNoPictureId;
+  temporal_id_ = kNoTemporalIdx;
+  tl0_pic_id_ = kNoTl0PicIdx;
+  full_sync_ = true;
+  init_ = true;
+}
+
+uint32_t VCMDecodingState::time_stamp() const {
+  return time_stamp_;
+}
+
+uint16_t VCMDecodingState::sequence_num() const {
+  return sequence_num_;
+}
+
+bool VCMDecodingState::IsOldFrame(const VCMFrameBuffer* frame) const {
+  assert(frame != NULL);
+  if (init_)
+    return false;
+  return (LatestTimestamp(time_stamp_, frame->TimeStamp(), NULL)
+          == time_stamp_);
+}
+
+bool VCMDecodingState::IsOldPacket(const VCMPacket* packet) const {
+  assert(packet != NULL);
+  if (init_)
+    return false;
+  return (LatestTimestamp(time_stamp_, packet->timestamp, NULL)
+           == time_stamp_);
+}
+
+void VCMDecodingState::SetState(const VCMFrameBuffer* frame) {
+  assert(frame != NULL && frame->GetHighSeqNum() >= 0);
+  UpdateSyncState(frame);
+  sequence_num_ = static_cast<uint16_t>(frame->GetHighSeqNum());
+  time_stamp_ = frame->TimeStamp();
+  picture_id_ = frame->PictureId();
+  temporal_id_ = frame->TemporalId();
+  tl0_pic_id_ = frame->Tl0PicId();
+  init_ = false;
+}
+
+void VCMDecodingState::SetStateOneBack(const VCMFrameBuffer* frame) {
+  assert(frame != NULL && frame->GetHighSeqNum() >= 0);
+  sequence_num_ = static_cast <uint16_t>(frame->GetHighSeqNum()) - 1u;
+  time_stamp_ = frame->TimeStamp() - 1u;
+  temporal_id_ = frame->TemporalId();
+  if (frame->PictureId() != kNoPictureId) {
+    if (frame->PictureId() == 0)
+      picture_id_ = 0x7FFF;
+    else
+      picture_id_ =  frame->PictureId() - 1;
+  }
+  if (frame->Tl0PicId() != kNoTl0PicIdx) {
+    if (frame->Tl0PicId() == 0)
+      tl0_pic_id_ = 0x00FF;
+    else
+      tl0_pic_id_ = frame->Tl0PicId() - 1;
+  }
+  init_ = false;
+}
+
+void VCMDecodingState::UpdateEmptyPacket(const VCMPacket* packet) {
+  assert(packet != NULL);
+  if (packet->sizeBytes == 0 && packet->timestamp == time_stamp_) {
+    // Empty packet (sizeBytes = 0), make sure we update the last
+    // decoded sequence number.
+    sequence_num_ = LatestSequenceNumber(packet->seqNum, sequence_num_, NULL);
+  }
+}
+
+void VCMDecodingState::SetSeqNum(uint16_t new_seq_num) {
+  sequence_num_ = new_seq_num;
+}
+
+bool VCMDecodingState::init() const {
+  return init_;
+}
+
+bool VCMDecodingState::full_sync() const {
+  return full_sync_;
+}
+
+void VCMDecodingState::UpdateSyncState(const VCMFrameBuffer* frame) {
+  if (init_)
+    return;
+  if (frame->TemporalId() == kNoTemporalIdx ||
+      frame->Tl0PicId() == kNoTl0PicIdx) {
+    full_sync_ = true;
+  } else if (frame->FrameType() == kVideoFrameKey ||
+             frame->NonReference()) {
+    full_sync_ = true;
+  } else if (full_sync_) {
+    // Verify that we are still in sync.
+    // Sync will be broken if continuity is true for layers but not for the
+    // other methods (PictureId and SeqNum).
+    if (!ContinuousPictureId(frame->PictureId()) &&
+        !ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum()))) {
+      // Non-layered methods have failed.
+      full_sync_ = false;
+    }
+  }
+}
+
+bool VCMDecodingState::ContinuousFrame(const VCMFrameBuffer* frame) const {
+  // Check continuity based on the following hierarchy:
+  // - Temporal layers (stop here if out of sync).
+  // - Picture Id when available.
+  // - Sequence numbers.
+  // Return true when in initial state.
+  // Note that when a method is not applicable it will return false.
+  assert(frame != NULL);
+  if (init_)
+    return true;
+
+  if (!ContinuousLayer(frame->TemporalId(), frame->Tl0PicId())) {
+    // Base layers are not continuous or temporal layers are inactive.
+    if (!full_sync_)
+      return false;
+    else if (!ContinuousPictureId(frame->PictureId()))
+      return ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum()));
+  }
+  return true;
+}
+
+bool VCMDecodingState::ContinuousPictureId(int picture_id) const {
+  // First, check if applicable.
+  if (picture_id == kNoPictureId || picture_id_ == kNoPictureId)
+    return false;
+  uint16_t local_pic_id = static_cast<uint16_t>(picture_id_);
+  uint16_t new_pic_id = static_cast<uint16_t>(picture_id);
+
+  if (new_pic_id < local_pic_id) {
+    // Wrap
+    if (local_pic_id >= (1 << 8)) {
+      // 15 bits used for picture id
+      return (((local_pic_id + 1) % 0x7FFF) == new_pic_id);
+    } else {
+      // 7 bits used for picture id
+      return (((local_pic_id + 1) % 0x7F) == new_pic_id);
+    }
+  }
+  // No wrap
+  return (local_pic_id + 1 == new_pic_id);
+}
+
+bool VCMDecodingState::ContinuousSeqNum(uint16_t seq_num) const {
+  return (seq_num == static_cast<uint16_t>(sequence_num_ + 1));
+}
+
+bool VCMDecodingState::ContinuousLayer(int temporal_id,
+                                       int tl0_pic_id) const {
+  // First, check if applicable.
+  if (temporal_id == kNoTemporalIdx || tl0_pic_id == kNoTl0PicIdx)
+    return false;
+  // If this is the first frame to use temporal layers, make sure we start
+  // from base.
+  else if (tl0_pic_id_ == kNoTl0PicIdx && temporal_id_ == kNoTemporalIdx &&
+           temporal_id == 0)
+    return true;
+
+  // Current implementation: applicable for base layer only.
+  if (temporal_id != 0)
+    return false;
+  return (static_cast<uint8_t>(tl0_pic_id_ + 1) ==
+      static_cast<uint8_t>(tl0_pic_id));
+}
+
+}  // namespace webrtc

src/modules/video_coding/main/source/decoding_state.h

@@ -0,0 +1,68 @@
+/*
+ *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_
+#define WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_
+
+#include "typedefs.h"
+
+namespace webrtc {
+
+// Forward declarations
+class VCMFrameBuffer;
+class VCMPacket;
+
+class VCMDecodingState {
+ public:
+  VCMDecodingState();
+  ~VCMDecodingState();
+  // Check for old frame
+  bool IsOldFrame(const VCMFrameBuffer* frame) const;
+  // Check for old packet
+  bool IsOldPacket(const VCMPacket* packet) const;
+  // Check for frame continuity based on current decoded state. Use best method
+  // possible, i.e. temporal info, picture ID or sequence number.
+  bool ContinuousFrame(const VCMFrameBuffer* frame) const;
+  void SetState(const VCMFrameBuffer* frame);
+  // Set the decoding state one frame back.
+  void SetStateOneBack(const VCMFrameBuffer* frame);
+  // Update the sequence number if the timestamp matches current state and the
+  // packet is empty.
+  void UpdateEmptyPacket(const VCMPacket* packet);
+  void SetSeqNum(uint16_t new_seq_num);
+  void Reset();
+  uint32_t time_stamp() const;
+  uint16_t sequence_num() const;
+  // Return true if at initial state.
+  bool init() const;
+  // Return true when sync is on - decode all layers.
+  bool full_sync() const;
+
+ private:
+  void UpdateSyncState(const VCMFrameBuffer* frame);
+  // Designated continuity functions
+  bool ContinuousPictureId(int picture_id) const;
+  bool ContinuousSeqNum(uint16_t seq_num) const;
+  bool ContinuousLayer(int temporal_id, int tl0_pic_id) const;
+
+  // Keep state of last decoded frame.
+  // TODO(mikhal/stefan): create designated classes to handle these types.
+  uint16_t    sequence_num_;
+  uint32_t    time_stamp_;
+  int         picture_id_;
+  int         temporal_id_;
+  int         tl0_pic_id_;
+  bool        full_sync_;  // Sync flag when temporal layers are used.
+  bool        init_;
+};
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_

src/modules/video_coding/main/source/jitter_buffer.cc

@@ -69,9 +69,7 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId,
     _maxNumberOfFrames(kStartNumberOfFrames),
     _frameBuffers(),
     _frameBuffersTSOrder(),
-    _lastDecodedSeqNum(-1),
+    _lastDecodedState(),
-    _lastDecodedTimeStamp(-1),
-    _lastDecodedPictureId(-1),
     _packetsNotDecodable(0),
     _receiveStatistics(),
     _incomingFrameRate(0),
@@ -143,9 +141,7 @@ VCMJitterBuffer::CopyFrom(const VCMJitterBuffer& rhs)
         _NACKSeqNumLength = rhs._NACKSeqNumLength;
         _waitingForKeyFrame = rhs._waitingForKeyFrame;
         _firstPacket = rhs._firstPacket;
-        _lastDecodedSeqNum =  rhs._lastDecodedSeqNum;
+        _lastDecodedState =  rhs._lastDecodedState;
-        _lastDecodedTimeStamp = rhs._lastDecodedTimeStamp;
-        _lastDecodedPictureId = rhs._lastDecodedPictureId;
         _packetsNotDecodable = rhs._packetsNotDecodable;
         memcpy(_receiveStatistics, rhs._receiveStatistics,
                sizeof(_receiveStatistics));
@@ -213,9 +209,7 @@ VCMJitterBuffer::Stop()
 {
     _critSect->Enter();
     _running = false;
-    _lastDecodedTimeStamp = -1;
+    _lastDecodedState.Reset();
-    _lastDecodedSeqNum = -1;
-    _lastDecodedPictureId = -1;
     _frameBuffersTSOrder.Flush();
     for (int i = 0; i < kMaxNumberOfFrames; i++)
     {
@@ -257,9 +251,7 @@ VCMJitterBuffer::FlushInternal()
     {
         ReleaseFrameInternal(_frameBuffers[i]);
     }
-    _lastDecodedSeqNum = -1;
+    _lastDecodedState.Reset(); // TODO (mikhal): sync reset
-    _lastDecodedTimeStamp = -1;
-    _lastDecodedPictureId = -1;
     _packetsNotDecodable = 0;
 
     _frameEvent.Reset();
@@ -331,9 +323,7 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame)
 
     // Check if we should drop frame
     // an old complete frame can arrive too late
-    if (_lastDecodedTimeStamp > 0 &&
+    if (_lastDecodedState.IsOldFrame(frame))
-            LatestTimestamp(static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp),
-                            frame->TimeStamp(), NULL) == _lastDecodedTimeStamp)
     {
         // Frame is older than the latest decoded frame, drop it.
         // This will trigger a release in CleanUpSizeZeroFrames later.
@@ -442,14 +432,15 @@ VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMEncodedFrame*& frame)
     }
 
     _critSect->Enter();
-    // Make sure that old empty packets are inserted.
+    // Does this packet belong to an old frame?
-    if (LatestTimestamp(static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp),
+    if (_lastDecodedState.IsOldPacket(&packet))
-                        packet.timestamp, NULL) == _lastDecodedTimeStamp
-        && packet.sizeBytes > 0)
     {
-        _discardedPackets++;  // Only counts discarded media packets
+        // Account only for media packets
-        // Trying to get an old frame.
+        if (packet.sizeBytes > 0)
-        _numConsecutiveOldPackets++;
+        {
+            _discardedPackets++;
+            _numConsecutiveOldPackets++;
+        }
         if (_numConsecutiveOldPackets > kMaxConsecutiveOldPackets)
         {
             FlushInternal();
@@ -544,51 +535,6 @@ VCMJitterBuffer::GetEmptyFrame()
     return NULL;
 }
 
-// Must be called under the critical section _critSect.
-VCMFrameListItem*
-VCMJitterBuffer::FindOldestSequenceNum() const
-{
-    WebRtc_UWord16 currentLow = 0xffff;
-    WebRtc_UWord16 sequenceNumber = 0;
-    bool first = true;
-    VCMFrameListItem* frameListItem = _frameBuffersTSOrder.First();
-    VCMFrameListItem* oldestFrameListItem = NULL;
-
-    while (frameListItem != NULL)
-    {
-        // if we have more than one frame done since last time,
-        // pick oldest
-        VCMFrameBuffer* ptrFrame = NULL;
-        ptrFrame = frameListItem->GetItem();
-        sequenceNumber = static_cast<WebRtc_UWord16>(ptrFrame->GetLowSeqNum());
-
-        // Find the oldest, hence lowest, using sequence numbers
-        if (first)
-        {
-            currentLow = sequenceNumber;
-            oldestFrameListItem = frameListItem;
-            first = false;
-        }
-        else if ((currentLow < 0x0fff) && (sequenceNumber > 0xf000))
-        {
-            // We have a wrap and this one is older
-            currentLow = sequenceNumber;
-            oldestFrameListItem = frameListItem;
-        }
-        else if ((sequenceNumber < 0x0fff) && (currentLow > 0xf000))
-        {
-            // This one is after a wrap, leave as is
-        }
-        else if (currentLow > sequenceNumber)
-        {
-            // Normal case, this one is lower.
-            currentLow = sequenceNumber;
-            oldestFrameListItem = frameListItem;
-        }
-        frameListItem = _frameBuffersTSOrder.Next(frameListItem);
-    }
-    return oldestFrameListItem;
-}
 
 // Find oldest complete frame used for getting next frame to decode
 // Must be called under critical section
@@ -598,7 +544,6 @@ VCMFrameListItem*
 VCMJitterBuffer::FindOldestCompleteContinuousFrame(bool enableDecodable) {
   // If we have more than one frame done since last time, pick oldest.
   VCMFrameBuffer* oldestFrame = NULL;
-  int currentLow = -1;
 
   VCMFrameListItem* oldestFrameItem = _frameBuffersTSOrder.First();
   if (oldestFrameItem != NULL) {
@@ -626,23 +571,11 @@ VCMJitterBuffer::FindOldestCompleteContinuousFrame(bool enableDecodable) {
   if (_waitingForKeyFrame && oldestFrame->FrameType() != kVideoFrameKey) {
     return NULL;
   }
-  // Is this the first frame to be decoded?
+  // We have a complete frame - establish continuity.
-  // Otherwise, establish continuity - use PictureId when available.
+  if (_lastDecodedState.init()) {
-  if (_lastDecodedSeqNum == -1) {
     return oldestFrameItem;
-  } else if (oldestFrame->PictureId() == kNoPictureId) {
+  } else if (!_lastDecodedState.ContinuousFrame(oldestFrame)) {
-    currentLow = oldestFrame->GetLowSeqNum();
-    WebRtc_UWord16 lastDecodedSeqNum = (WebRtc_UWord16)_lastDecodedSeqNum;
-
-    // We could have received the first packet of the last frame before a long
-    // period if drop, that case is handled by GetNackList (when applicable).
-    if (((WebRtc_UWord16)(lastDecodedSeqNum + 1)) != currentLow) {
-      // Wait since we want a complete continuous frame.
       return NULL;
-    }
-  } else if (!ContinuousPictureId(oldestFrame->PictureId())) {
-    // We don't have a continuous frame.
-    return NULL;
   }
   return oldestFrameItem;
 }
@@ -748,7 +681,7 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
     CleanUpOldFrames();
     CleanUpSizeZeroFrames();
 
-    if (_lastDecodedSeqNum == -1 && WaitForNack()) {
+    if (_lastDecodedState.init() && WaitForNack()) {
       _waitingForKeyFrame = true;
     }
     VCMFrameListItem*
@@ -850,10 +783,8 @@ VCMJitterBuffer::GetCompleteFrameForDecoding(WebRtc_UWord32 maxWaitTimeMS)
 
     _critSect->Leave();
 
-    // We have a frame - store seqnum & timestamp
+    // We have a frame - update decoded state with frame info.
-    _lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
+    _lastDecodedState.SetState(oldestFrame);
-    _lastDecodedTimeStamp = oldestFrame->TimeStamp();
-    _lastDecodedPictureId = oldestFrame->PictureId();
 
     return oldestFrame;
 }
@@ -990,9 +921,9 @@ VCMJitterBuffer::CompleteSequenceWithNextFrame()
     }
 
     // See if we have lost a frame before this one.
-    if (_lastDecodedSeqNum == -1)
+    if (_lastDecodedState.init())
     {
-        // The sequence is not complete since we haven't yet.
+        // Following start, reset or flush -> check for key frame.
         if (oldestFrame->FrameType() != kVideoFrameKey)
         {
             return false;
@@ -1002,18 +933,7 @@ VCMJitterBuffer::CompleteSequenceWithNextFrame()
     {
         return false;
     }
-    // We can't use sequence numbers to detect frame loss when FEC is enabled.
+    else if (!_lastDecodedState.ContinuousFrame(oldestFrame))
-    // Assume FEC is only enabled for VP8 with picture ids, and use picture ids
-    // to detect frame loss in that situation.
-    else if (oldestFrame->PictureId() == kNoPictureId)
-    {
-        if (oldestFrame->GetLowSeqNum() !=
-            (_lastDecodedSeqNum + 1) % 0x00010000)
-        {
-            return false;
-        }
-    }
-    else if (!ContinuousPictureId(oldestFrame->PictureId()))
     {
         return false;
     }
@@ -1098,10 +1018,8 @@ VCMJitterBuffer::GetFrameForDecoding()
 
     _packetsNotDecodable += oldestFrame->NotDecodablePackets();
 
-    // Store current seqnum & time
+    // We have a frame - update decoded state with frame info.
-    _lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
+    _lastDecodedState.SetState(oldestFrame);
-    _lastDecodedTimeStamp = oldestFrame->TimeStamp();
-    _lastDecodedPictureId = oldestFrame->PictureId();
 
     return oldestFrame;
 }
@@ -1119,8 +1037,8 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
 
     // First look for a complete _continuous_ frame.
     // When waiting for nack, wait for a key frame, if a continuous frame cannot
-    // be determined (i.e. lastDecodedSeqNum = -1)
+    // be determined (i.e. initial decoding state).
-    if (_lastDecodedSeqNum == -1) {
+    if (_lastDecodedState.init()) {
       _waitingForKeyFrame = true;
     }
 
@@ -1178,11 +1096,8 @@ VCMJitterBuffer::GetFrameForDecodingNACK()
         _waitingForKeyFrame = false;
     }
 
-    // We have a complete/decodable continuous frame, decode it.
+    // We have a frame - update decoded state with frame info.
-    // Store seqnum & timestamp
+    _lastDecodedState.SetState(oldestFrame);
-    _lastDecodedSeqNum = oldestFrame->GetHighSeqNum();
-    _lastDecodedTimeStamp = oldestFrame->TimeStamp();
-    _lastDecodedPictureId = oldestFrame->PictureId();
 
     return oldestFrame;
 }
@@ -1299,11 +1214,12 @@ WebRtc_Word32
 VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum,
                                            WebRtc_Word32& highSeqNum) const
 {
+    // TODO (mikhal/stefan): refactor to use lastDecodedState
     WebRtc_Word32 i = 0;
     WebRtc_Word32 seqNum = -1;
 
     highSeqNum = -1;
-    lowSeqNum = _lastDecodedSeqNum;
+    lowSeqNum = _lastDecodedState.sequence_num();
 
     // find highest seq numbers
     for (i = 0; i < _maxNumberOfFrames; ++i)
@@ -1329,6 +1245,7 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum,
 WebRtc_UWord16*
 VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
 {
+    // TODO (mikhal/stefan): Refactor to use lastDecodedState.
     CriticalSectionScoped cs(_critSect);
     int i = 0;
     WebRtc_Word32 lowSeqNum = -1;
@@ -1388,13 +1305,13 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
                      "from seq: %d. Lowest seq in jb %d", highSeqNum,lowSeqNum);
 
         // This nack size will trigger a key request...
-        bool foundIFrame = false;
+        bool foundKeyFrame = false;
 
         while (numberOfSeqNum > kNackHistoryLength)
         {
-            foundIFrame = RecycleFramesUntilKeyFrame();
+          foundKeyFrame = RecycleFramesUntilKeyFrame();
 
-            if (!foundIFrame)
+            if (!foundKeyFrame)
             {
                 break;
             }
@@ -1428,20 +1345,20 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
 
         } // end while
 
-        if (!foundIFrame)
+        if (!foundKeyFrame)
         {
-            // No I frame in JB.
+            // No key frame in JB.
 
             // Set the last decoded sequence number to current high.
             // This is to not get a large nack list again right away
-            _lastDecodedSeqNum = highSeqNum;
+            _lastDecodedState.SetSeqNum(static_cast<uint16_t>(highSeqNum));
             _waitingForKeyFrame = true;
             // Set to trigger key frame signal
             nackSize = 0xffff;
             listExtended = true;
             WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1,
                     "\tNo key frame found, request one. _lastDecodedSeqNum[0] "
-                    "%d", _lastDecodedSeqNum);
+                    "%d", _lastDecodedState.sequence_num());
         }
         else
         {
@@ -1449,7 +1366,7 @@ VCMJitterBuffer::CreateNackList(WebRtc_UWord16& nackSize, bool& listExtended)
             // The function itself has set last decoded seq.
             WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, -1,
                     "\tKey frame found. _lastDecodedSeqNum[0] %d",
-                    _lastDecodedSeqNum);
+                    _lastDecodedState.sequence_num());
             nackSize = 0;
         }
 
@@ -1598,7 +1515,7 @@ WebRtc_Word64
 VCMJitterBuffer::LastDecodedTimestamp() const
 {
     CriticalSectionScoped cs(_critSect);
-    return _lastDecodedTimeStamp;
+    return _lastDecodedState.time_stamp();
 }
 
 // Insert packet
@@ -1647,14 +1564,7 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet)
     if (frame != NULL)
     {
         VCMFrameBufferStateEnum state = frame->GetState();
-        if (packet.sizeBytes == 0 && packet.timestamp == _lastDecodedTimeStamp)
+        _lastDecodedState.UpdateEmptyPacket(&packet);
-        {
-            // Empty packet (sizeBytes = 0), make sure we update the last
-            // decoded sequence number
-            _lastDecodedSeqNum = LatestSequenceNumber(packet.seqNum,
-                                                      _lastDecodedSeqNum, NULL);
-        }
-
         // Insert packet
         // Check for first packet
         // High sequence number will be -1 if neither an empty packet nor
@@ -1811,8 +1721,8 @@ VCMJitterBuffer::RecycleFramesUntilKeyFrame()
     }
 
     // Remove up to oldest key frame
-    bool foundIFrame = false;
+    bool foundKeyFrame = false;
-    while (oldestFrameListItem != NULL && !foundIFrame)
+    while (oldestFrameListItem != NULL && !foundKeyFrame)
     {
         // Throw at least one frame.
         _dropCount++;
@@ -1831,209 +1741,126 @@ VCMJitterBuffer::RecycleFramesUntilKeyFrame()
 
         if (oldestFrame != NULL)
         {
-            foundIFrame = foundIFrame ||
+            foundKeyFrame = foundKeyFrame ||
-                          (oldestFrame->FrameType() != kVideoFrameDelta);
+                            (oldestFrame->FrameType() != kVideoFrameDelta);
-            if (foundIFrame)
+            if (foundKeyFrame)
             {
-                // fake the last played out to match the start of this key frame
+                // Fake the lastDecodedState to match this key frame.
-                _lastDecodedSeqNum = (WebRtc_UWord16)((WebRtc_UWord16)
+                _lastDecodedState.SetStateOneBack(oldestFrame);
-                                     (oldestFrame->GetLowSeqNum()) - 1);
-                _lastDecodedTimeStamp = (WebRtc_UWord32)
-                                        (oldestFrame->TimeStamp() - 1);
                 break;
             }
         }
     }
-    _lastDecodedSeqNum = -1;
+    _lastDecodedState.Reset(); // TODO (mikhal): no sync
-    return foundIFrame;
+    return foundKeyFrame;
 }
 
 // Must be called under the critical section _critSect.
-void
+VCMFrameListItem*
-VCMJitterBuffer::CleanUpOldFrames()
+VCMJitterBuffer::FindOldestSequenceNum() const
 {
-    VCMFrameListItem* oldestFrameListItem = _frameBuffersTSOrder.First();
+    WebRtc_UWord16 currentLow = 0xffff;
-    VCMFrameBuffer* oldestFrame = NULL;
+    WebRtc_UWord16 sequenceNumber = 0;
-
+    bool first = true;
-    if (_lastDecodedTimeStamp == -1)
+    VCMFrameListItem* frameListItem = _frameBuffersTSOrder.First();
-    {
+    VCMFrameListItem* oldestFrameListItem = NULL;
-        return;
-    }
-
-    while (oldestFrameListItem != NULL)
-    {
-        oldestFrame = oldestFrameListItem->GetItem();
-        WebRtc_UWord32 frameTimeStamp = oldestFrame->TimeStamp();
-
-        // Release the frame if it's older than the last decoded frame.
-        if (_lastDecodedTimeStamp > -1 &&
-            LatestTimestamp(static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp),
-                            frameTimeStamp, NULL) ==
-                            static_cast<WebRtc_UWord32>(_lastDecodedTimeStamp))
-        {
-            const WebRtc_Word32 frameLowSeqNum = oldestFrame->GetLowSeqNum();
-            const WebRtc_Word32 frameHighSeqNum = oldestFrame->GetHighSeqNum();
-            if (frameTimeStamp == _lastDecodedTimeStamp &&
-                ((frameLowSeqNum == (_lastDecodedSeqNum + 1)) ||
-                ((frameLowSeqNum == 0) &&
-                (_lastDecodedSeqNum == 0xffff))))
-            {
-                // Could happen when sending empty packets
-                // Empty packet (size = 0) belonging to last decoded frame.
-                // Frame: | packet | packet | packet M=1 |
-                // empty data (size = 0) | empty data (size = 0)| ...
-
-                // This frame follows the last decoded frame
-                _lastDecodedSeqNum = frameHighSeqNum;
-            }
-
-            _frameBuffersTSOrder.Erase(oldestFrameListItem);
-            ReleaseFrameInternal(oldestFrame);
-            oldestFrameListItem = _frameBuffersTSOrder.First();
-        }
-        else
-        {
-            break;
-        }
-    }
-}
-
-// This function has changed to use sequence numbers
-// Using timestamp won't work since can get
-// nack requests with a higher time stamp than
-// the following encoded frame, but with a lower sequence number.
-// Must be called under _critSect.
-void
-VCMJitterBuffer::CleanUpSizeZeroFrames()
-{
-    VCMFrameListItem* frameListItem = FindOldestSequenceNum();
 
     while (frameListItem != NULL)
     {
-        VCMFrameBuffer* ptrTempBuffer = frameListItem->GetItem();
+        // if we have more than one frame done since last time,
+        // pick oldest
+        VCMFrameBuffer* ptrFrame = NULL;
+        ptrFrame = frameListItem->GetItem();
+        sequenceNumber = static_cast<WebRtc_UWord16>(ptrFrame->GetLowSeqNum());
 
-        // pop frame if its size zero but store seqnum
+        // Find the oldest, hence lowest, using sequence numbers
-        if (ptrTempBuffer->Length() == 0)
+        if (first)
         {
-            WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum();
+            currentLow = sequenceNumber;
-            if (frameHighSeqNum == -1)
+            oldestFrameListItem = frameListItem;
-            {
+            first = false;
-                // This frame has been Reset for this function to clean it up
-                _frameBuffersTSOrder.Erase(frameListItem);
-                ReleaseFrameInternal(ptrTempBuffer);
-                frameListItem = FindOldestSequenceNum();
-            }
-            else
-            {
-                bool releaseFrame = false;
-                const WebRtc_Word32 frameHighSeqNum =
-                                    ptrTempBuffer->GetHighSeqNum();
-                const WebRtc_Word32 frameLowSeqNum =
-                                    ptrTempBuffer->GetLowSeqNum();
-
-                if ((frameLowSeqNum == (_lastDecodedSeqNum + 1)) ||
-                    // Frame is next in line
-                    ((frameLowSeqNum == 0) && (_lastDecodedSeqNum== 0xffff)))
-                {
-                    // This frame follows the last decoded frame, release it.
-                    _lastDecodedSeqNum = frameHighSeqNum;
-                    releaseFrame = true;
-                }
-                // If frameHighSeqNum < _lastDecodedSeqNum
-                // but need to take wrap into account.
-                else if (frameHighSeqNum < _lastDecodedSeqNum)
-                {
-                    if (frameHighSeqNum < 0x0fff &&
-                        _lastDecodedSeqNum> 0xf000)
-                    {
-                        // Wrap, we don't want release this one. It's newer...
-                    }
-                    else
-                    {
-                        // This frame has lower seq than last decoded,
-                        // and we have no wrap -> it's older.
-                        releaseFrame = true;
-                    }
-                }
-                else if (frameHighSeqNum > _lastDecodedSeqNum &&
-                        _lastDecodedSeqNum < 0x0fff &&
-                        frameHighSeqNum > 0xf000)
-                {
-                    // Higher seq than last decoded,
-                    // but last decoded has recently wrapped.
-                    releaseFrame = true;
-                }
-
-                if (releaseFrame)
-                {
-                    _frameBuffersTSOrder.Erase(frameListItem);
-                    ReleaseFrameInternal(ptrTempBuffer);
-                    frameListItem = FindOldestSequenceNum();
-                }
-                else
-                {
-                    // We couldn't release this one and we're using nack,
-                    // stop trying...
-                    frameListItem = NULL;
-                }
-            }
         }
-        else
+        else if ((currentLow < 0x0fff) && (sequenceNumber > 0xf000))
         {
-            // we have a length
+            // We have a wrap and this one is older
-            break;
+            currentLow = sequenceNumber;
+            oldestFrameListItem = frameListItem;
         }
+        else if ((sequenceNumber < 0x0fff) && (currentLow > 0xf000))
+        {
+            // This one is after a wrap, leave as is
+        }
+        else if (currentLow > sequenceNumber)
+        {
+            // Normal case, this one is lower.
+            currentLow = sequenceNumber;
+            oldestFrameListItem = frameListItem;
+        }
+        frameListItem = _frameBuffersTSOrder.Next(frameListItem);
     }
+    return oldestFrameListItem;
+}
+
+
+// Must be called under the critical section _critSect.
+void VCMJitterBuffer::CleanUpOldFrames() {
+  if (_lastDecodedState.init())
+    return;
+
+  VCMFrameListItem* oldestFrameListItem = _frameBuffersTSOrder.First();
+  VCMFrameBuffer* oldestFrame = NULL;
+
+  while (oldestFrameListItem != NULL) {
+    oldestFrame = oldestFrameListItem->GetItem();
+    if (_lastDecodedState.IsOldFrame(oldestFrame)) {
+      _frameBuffersTSOrder.Erase(oldestFrameListItem);
+      ReleaseFrameInternal(oldestFrame);
+      oldestFrameListItem = _frameBuffersTSOrder.First();
+    } else {
+      break;
+    }
+  }
+}
+
+// TODO (mikhal):
+// 1. Merge with previous function.
+// 2. Modify to use timestamps and not seqnum.
+// Must be called under _critSect.
+void VCMJitterBuffer::CleanUpSizeZeroFrames() {
+  VCMFrameListItem* frameListItem = FindOldestSequenceNum();
+
+  while (frameListItem != NULL) {
+    VCMFrameBuffer* ptrTempBuffer = frameListItem->GetItem();
+
+    // Pop frame if its size zero but store seqnum.
+    if (ptrTempBuffer->Length() == 0) {
+      WebRtc_Word32 frameHighSeqNum = ptrTempBuffer->GetHighSeqNum();
+      if (frameHighSeqNum == -1) {
+        // This frame has been Reset for this function to clean it up
+        _frameBuffersTSOrder.Erase(frameListItem);
+        ReleaseFrameInternal(ptrTempBuffer);
+        frameListItem = FindOldestSequenceNum();
+      }
+    } else {
+      // we have a length
+      break;
+    }
+  }
 }
 
 // Used in GetFrameForDecoding
-void
+void VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame) {
-VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame)
+  frame.MakeSessionDecodable();  // Make sure the session can be decoded.
-{
+  if (frame.FrameType() == kVideoFrameKey)
-    frame.MakeSessionDecodable(); // make sure the session can be decoded.
+    return;
-    if (frame.FrameType() == kVideoFrameKey)
-        return;
 
-    if (_lastDecodedSeqNum == -1)
+  if (_lastDecodedState.init() ||
-    {
+      !_lastDecodedState.ContinuousFrame(&frame)) {
-        // First frame is not a key frame
+    frame.SetPreviousFrameLoss();
-        frame.SetPreviousFrameLoss();
-        return;
-    }
-    // We can't use sequence numbers to detect frame loss when FEC is enabled.
-    // Assume FEC is only enabled for VP8 with picture ids, and use picture ids
-    // to detect frame loss in that situation.
-    if (frame.PictureId() == kNoPictureId)
-    {
-        WebRtc_UWord16 nextExpectedSeqNum =
-            static_cast<WebRtc_UWord16>(_lastDecodedSeqNum + 1);
-        if (static_cast<WebRtc_UWord16>(frame.GetLowSeqNum()) !=
-            nextExpectedSeqNum)
-        {
-            frame.SetPreviousFrameLoss();
-        }
-    }
-    else if (!ContinuousPictureId(frame.PictureId()))
-    {
-        frame.SetPreviousFrameLoss();
-    }
-}
-
-bool VCMJitterBuffer::ContinuousPictureId(int pictureId) const {
-  if (pictureId < _lastDecodedPictureId) {
-    // Wrap
-    if (_lastDecodedPictureId >= (1<<8)) {
-      // 15 bits used for picture id
-      return (((_lastDecodedPictureId + 1) % 0x7FFF) == pictureId);
-    } else {
-      // 7 bits used for picture id
-      return (((_lastDecodedPictureId + 1) % 0x7F) == pictureId);
-    }
   }
-  // No wrap
-  return (_lastDecodedPictureId + 1 == pictureId);
 }
 
+
 bool
 VCMJitterBuffer::WaitForNack()
 {

src/modules/video_coding/main/source/jitter_buffer.h

@@ -13,6 +13,7 @@
 
 #include "typedefs.h"
 #include "critical_section_wrapper.h"
+#include "decoding_state.h"
 #include "module_common_types.h"
 #include "video_coding_defines.h"
 #include "inter_frame_delay.h"
@@ -144,7 +145,6 @@ private:
     void ReleaseFrameInternal(VCMFrameBuffer* frame);
     // Flush and reset the jitter buffer. Call under critical section.
     void FlushInternal();
-    VCMFrameListItem* FindOldestSequenceNum() const;
 
     // Help functions for insert packet
     // Get empty frame, creates new (i.e. increases JB size) if necessary
@@ -187,9 +187,9 @@ private:
                                     const void* timestamp);
     static bool CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame,
                                                   const void* notUsed);
-    bool ContinuousPictureId(int pictureId) const;
     // Decide whether should wait for NACK (mainly relevant for hybrid mode)
     bool WaitForNack();
+    VCMFrameListItem* FindOldestSequenceNum() const;
 
     WebRtc_Word32                 _vcmId;
     WebRtc_Word32                 _receiverId;
@@ -208,11 +208,7 @@ private:
     VCMFrameListTimestampOrderAsc _frameBuffersTSOrder;
 
     // timing
-    // Sequence number of last frame that was given to decoder
+    VCMDecodingState       _lastDecodedState;
-    WebRtc_Word32           _lastDecodedSeqNum;
-    // Timestamp of last frame that was given to decoder
-    WebRtc_Word64           _lastDecodedTimeStamp;
-    int                     _lastDecodedPictureId;
     WebRtc_UWord32          _packetsNotDecodable;
 
     // Statistics

src/modules/video_coding/main/source/video_coding.gypi

@@ -38,6 +38,7 @@
         'codec_database.h',
         'codec_timer.h',
         'content_metrics_processing.h',
+        'decoding_state.h',
         'encoded_frame.h',
         'er_tables_xor.h',
         'event.h',
@@ -72,6 +73,7 @@
         'codec_database.cc',
         'codec_timer.cc',
         'content_metrics_processing.cc',
+        'decoding_state.cc',
         'encoded_frame.cc',
         'exp_filter.cc',
         'frame_buffer.cc',