diff --git a/src/modules/video_coding/codecs/vp8/main/source/vp8.cc b/src/modules/video_coding/codecs/vp8/main/source/vp8.cc index a90e52643..4a645a3e2 100644 --- a/src/modules/video_coding/codecs/vp8/main/source/vp8.cc +++ b/src/modules/video_coding/codecs/vp8/main/source/vp8.cc @@ -793,18 +793,10 @@ VP8Decoder::Decode(const EncodedImage& inputImage, { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - if (inputImage._buffer == NULL) - { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } if (_decodeCompleteCallback == NULL) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - if (inputImage._length <= 0) - { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } if (inputImage._completeFrame == false) { // future improvement @@ -815,6 +807,11 @@ VP8Decoder::Decode(const EncodedImage& inputImage, } // otherwise allow for incomplete frames to be decoded. } + if (inputImage._buffer == NULL && inputImage._length > 0) + { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + #ifdef INDEPENDENT_PARTITIONS if (fragmentation == NULL) { @@ -844,8 +841,13 @@ VP8Decoder::Decode(const EncodedImage& inputImage, return WEBRTC_VIDEO_CODEC_ERROR; } #else + WebRtc_UWord8* buffer = inputImage._buffer; + if (inputImage._length == 0) + { + buffer = NULL; // Triggers full frame concealment + } if (vpx_codec_decode(_decoder, - inputImage._buffer, + buffer, inputImage._length, 0, VPX_DL_REALTIME)) @@ -947,11 +949,12 @@ VP8Decoder::DecodePartitions(const EncodedImage& input_image, partition_length, 0, VPX_DL_REALTIME)) { - return WEBRTC_VIDEO_CODEC_ERROR; + return WEBRTC_VIDEO_CODEC_ERROR; } } - // Signal end of frame data + // Signal end of frame data. If there was no frame data this will trigger + // a full frame concealment. if (vpx_codec_decode(_decoder, NULL, 0, 0, VPX_DL_REALTIME)) return WEBRTC_VIDEO_CODEC_ERROR; return WEBRTC_VIDEO_CODEC_OK; diff --git a/src/modules/video_coding/main/source/frame_buffer.cc b/src/modules/video_coding/main/source/frame_buffer.cc index efc78c88a..bb4ec87f1 100644 --- a/src/modules/video_coding/main/source/frame_buffer.cc +++ b/src/modules/video_coding/main/source/frame_buffer.cc @@ -58,19 +58,19 @@ VCMFrameBuffer::SetPreviousFrameLoss() } WebRtc_Word32 -VCMFrameBuffer::GetLowSeqNum() +VCMFrameBuffer::GetLowSeqNum() const { return _sessionInfo.GetLowSeqNum(); } WebRtc_Word32 -VCMFrameBuffer::GetHighSeqNum() +VCMFrameBuffer::GetHighSeqNum() const { return _sessionInfo.GetHighSeqNum(); } bool -VCMFrameBuffer::IsSessionComplete() +VCMFrameBuffer::IsSessionComplete() const { return _sessionInfo.IsSessionComplete(); } @@ -229,7 +229,7 @@ VCMFrameBuffer::GetNackCount() const } bool -VCMFrameBuffer::HaveLastPacket() +VCMFrameBuffer::HaveLastPacket() const { return _sessionInfo.HaveLastPacket(); } @@ -311,9 +311,12 @@ VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) break; case kStateDecoding: - // we can go to this state from state kStateComplete kStateIncomplete + // A frame migth have received empty packets, or media packets might + // have been removed when making the frame decodable. The frame can + // still be set to decodable since it can be used to inform the + // decoder of a frame loss. assert(_state == kStateComplete || _state == kStateIncomplete || - _state == kStateDecodable); + _state == kStateDecodable || _state == kStateEmpty); // Transfer frame information to EncodedFrame and create any codec // specific information RestructureFrameInformation(); @@ -372,13 +375,17 @@ VCMFrameBuffer::ExtractFromStorage(const EncodedVideoData& frameFromStorage) return VCM_OK; } +int VCMFrameBuffer::NotDecodablePackets() const { + return _sessionInfo.NotDecodablePackets(); +} + // Set counted status (as counted by JB or not) void VCMFrameBuffer::SetCountedFrame(bool frameCounted) { _frameCounted = frameCounted; } -bool VCMFrameBuffer::GetCountedFrame() +bool VCMFrameBuffer::GetCountedFrame() const { return _frameCounted; } @@ -399,7 +406,7 @@ VCMFrameBuffer::GetState(WebRtc_UWord32& timeStamp) const } bool -VCMFrameBuffer::IsRetransmitted() +VCMFrameBuffer::IsRetransmitted() const { return _sessionInfo.IsRetransmitted(); } diff --git a/src/modules/video_coding/main/source/frame_buffer.h b/src/modules/video_coding/main/source/frame_buffer.h index d5ba3238c..619834fce 100644 --- a/src/modules/video_coding/main/source/frame_buffer.h +++ b/src/modules/video_coding/main/source/frame_buffer.h @@ -42,22 +42,22 @@ public: VCMFrameBufferStateEnum GetState(WebRtc_UWord32& timeStamp) const; void SetState(VCMFrameBufferStateEnum state); // Set state of frame - bool IsRetransmitted(); - bool IsSessionComplete(); - bool HaveLastPacket(); + bool IsRetransmitted() const; + bool IsSessionComplete() const; + bool HaveLastPacket() const; bool ForceSetHaveLastPacket(); // Makes sure the session contain a decodable stream. void MakeSessionDecodable(); // Sequence numbers // Get lowest packet sequence number in frame - WebRtc_Word32 GetLowSeqNum(); + WebRtc_Word32 GetLowSeqNum() const; // Get highest packet sequence number in frame - WebRtc_Word32 GetHighSeqNum(); + WebRtc_Word32 GetHighSeqNum() const; // Set counted status (as counted by JB or not) void SetCountedFrame(bool frameCounted); - bool GetCountedFrame(); + bool GetCountedFrame() const; // NACK // Zero out all entries in list up to and including _lowSeqNum @@ -76,6 +76,10 @@ public: WebRtc_Word32 ExtractFromStorage(const EncodedVideoData& frameFromStorage); + // The number of packets discarded because the decoder can't make use of + // them. + int NotDecodablePackets() const; + protected: void RestructureFrameInformation(); void PrepareForDecode(); diff --git a/src/modules/video_coding/main/source/frame_list.cc b/src/modules/video_coding/main/source/frame_list.cc index e79dc9153..3908892e2 100644 --- a/src/modules/video_coding/main/source/frame_list.cc +++ b/src/modules/video_coding/main/source/frame_list.cc @@ -26,7 +26,8 @@ VCMFrameListTimestampOrderAsc::Flush() while(Erase(First()) != -1) { } } -// Inserts frame in timestamp order, with the oldest timestamp first. Takes wrap arounds into account +// Inserts frame in timestamp order, with the oldest timestamp first. Takes wrap +// arounds into account WebRtc_Word32 VCMFrameListTimestampOrderAsc::Insert(VCMFrameBuffer* frame) { @@ -40,7 +41,8 @@ VCMFrameListTimestampOrderAsc::Insert(VCMFrameBuffer* frame) while (item != NULL) { const WebRtc_UWord32 itemTimestamp = item->GetItem()->TimeStamp(); - if (VCMJitterBuffer::LatestTimestamp(itemTimestamp, frame->TimeStamp()) == itemTimestamp) + if (LatestTimestamp(itemTimestamp, frame->TimeStamp(), NULL) == + itemTimestamp) { if (InsertBefore(item, newItem) < 0) { @@ -101,7 +103,9 @@ VCMFrameListTimestampOrderAsc::FindFrame(FindFrameCriteria criteria, const void* compareWith, VCMFrameListItem* startItem) const { - const VCMFrameListItem* frameListItem = FindFrameListItem(criteria, compareWith, startItem); + const VCMFrameListItem* frameListItem = FindFrameListItem(criteria, + compareWith, + startItem); if (frameListItem == NULL) { return NULL; diff --git a/src/modules/video_coding/main/source/jitter_buffer.cc b/src/modules/video_coding/main/source/jitter_buffer.cc index cfada1ace..1dd5745c3 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.cc +++ b/src/modules/video_coding/main/source/jitter_buffer.cc @@ -71,8 +71,9 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId, _maxNumberOfFrames(kStartNumberOfFrames), _frameBuffers(), _frameBuffersTSOrder(), - _lastDecodedSeqNum(), + _lastDecodedSeqNum(-1), _lastDecodedTimeStamp(-1), + _packetsNotDecodable(0), _receiveStatistics(), _incomingFrameRate(0), _incomingFrameCount(0), @@ -92,7 +93,6 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId, { memset(_frameBuffers, 0, sizeof(_frameBuffers)); memset(_receiveStatistics, 0, sizeof(_receiveStatistics)); - _lastDecodedSeqNum = -1; memset(_NACKSeqNumInternal, -1, sizeof(_NACKSeqNumInternal)); for (int i = 0; i< kStartNumberOfFrames; i++) @@ -127,7 +127,6 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) _running = rhs._running; _master = !rhs._master; _maxNumberOfFrames = rhs._maxNumberOfFrames; - _lastDecodedTimeStamp = rhs._lastDecodedTimeStamp; _incomingFrameRate = rhs._incomingFrameRate; _incomingFrameCount = rhs._incomingFrameCount; _timeLastIncomingFrameCount = rhs._timeLastIncomingFrameCount; @@ -145,6 +144,8 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) _missingMarkerBits = rhs._missingMarkerBits; _firstPacket = rhs._firstPacket; _lastDecodedSeqNum = rhs._lastDecodedSeqNum; + _lastDecodedTimeStamp = rhs._lastDecodedTimeStamp; + _packetsNotDecodable = rhs._packetsNotDecodable; memcpy(_receiveStatistics, rhs._receiveStatistics, sizeof(_receiveStatistics)); memcpy(_NACKSeqNumInternal, rhs._NACKSeqNumInternal, @@ -174,30 +175,6 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) return *this; } -WebRtc_UWord32 -VCMJitterBuffer::LatestTimestamp(const WebRtc_UWord32 existingTimestamp, - const WebRtc_UWord32 newTimestamp) -{ - bool wrap = (newTimestamp < 0x0000ffff && existingTimestamp > 0xffff0000) || - (newTimestamp > 0xffff0000 && existingTimestamp < 0x0000ffff); - if (existingTimestamp > newTimestamp && !wrap) - { - return existingTimestamp; - } - else if (existingTimestamp <= newTimestamp && !wrap) - { - return newTimestamp; - } - else if (existingTimestamp < newTimestamp && wrap) - { - return existingTimestamp; - } - else - { - return newTimestamp; - } -} - // Start jitter buffer void VCMJitterBuffer::Start() @@ -223,6 +200,7 @@ VCMJitterBuffer::Start() _firstPacket = true; _NACKSeqNumLength = 0; _rttMs = 0; + _packetsNotDecodable = 0; WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(_vcmId, _receiverId), "JB(0x%x): Jitter buffer: start", this); @@ -280,6 +258,7 @@ VCMJitterBuffer::FlushInternal() } _lastDecodedSeqNum = -1; _lastDecodedTimeStamp = -1; + _packetsNotDecodable = 0; _frameEvent.Reset(); _packetEvent.Reset(); @@ -353,7 +332,7 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) // an old complete frame can arrive too late if (_lastDecodedTimeStamp > 0 && LatestTimestamp(static_cast(_lastDecodedTimeStamp), - frame->TimeStamp()) == _lastDecodedTimeStamp) + frame->TimeStamp(), NULL) == _lastDecodedTimeStamp) { // Frame is older than the latest decoded frame, drop it. // This will trigger a release in CleanUpSizeZeroFrames later. @@ -427,7 +406,6 @@ VCMJitterBuffer::UpdateFrameState(VCMFrameBuffer* frame) } } - // Get received key and delta frames WebRtc_Word32 VCMJitterBuffer::GetFrameStatistics(WebRtc_UWord32& receivedDeltaFrames, @@ -441,6 +419,11 @@ VCMJitterBuffer::GetFrameStatistics(WebRtc_UWord32& receivedDeltaFrames, return 0; } +WebRtc_UWord32 VCMJitterBuffer::NumNotDecodablePackets() const { + CriticalSectionScoped cs(_critSect); + return _packetsNotDecodable; +} + WebRtc_UWord32 VCMJitterBuffer::DiscardedPackets() const { CriticalSectionScoped cs(_critSect); return _discardedPackets; @@ -458,7 +441,7 @@ VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMEncodedFrame*& frame) _critSect.Enter(); // Make sure that old empty packets are inserted. if (LatestTimestamp(static_cast(_lastDecodedTimeStamp), - packet.timestamp) == _lastDecodedTimeStamp + packet.timestamp, NULL) == _lastDecodedTimeStamp && packet.sizeBytes > 0) { _discardedPackets++; // Only counts discarded media packets @@ -1136,6 +1119,8 @@ VCMJitterBuffer::GetFrameForDecoding() // Set as decoding. Propagates the missingFrame bit. oldestFrame->SetState(kStateDecoding); + _packetsNotDecodable += oldestFrame->NotDecodablePackets(); + // Store current seqnum & time _lastDecodedSeqNum = oldestFrame->GetHighSeqNum(); _lastDecodedTimeStamp = oldestFrame->TimeStamp(); @@ -1204,8 +1189,9 @@ VCMJitterBuffer::GetFrameForDecodingNACK() return oldestFrame; } -// Must be called under the critical section _critSect. Should never be called with -// retransmitted frames, they must be filtered out before this function is called. +// Must be called under the critical section _critSect. Should never be called +// with retransmitted frames, they must be filtered out before this function is +// called. void VCMJitterBuffer::UpdateJitterAndDelayEstimates(VCMJitterSample& sample, bool incompleteFrame) @@ -1334,24 +1320,8 @@ VCMJitterBuffer::GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, (kStateDecoding != state) && seqNum != -1) { - if (highSeqNum == -1) - { - // first - highSeqNum = seqNum; - } - else if (seqNum < 0x0fff && highSeqNum > 0xf000) - { - // wrap - highSeqNum = seqNum; - } - else if(seqNum > 0xf000 && highSeqNum < 0x0fff) - { - // Do nothing since it is a wrap and this one is older - } - else if (seqNum > highSeqNum) - { - highSeqNum = seqNum; - } + bool wrap; + highSeqNum = LatestSequenceNumber(seqNum, highSeqNum, &wrap); } } // for return 0; @@ -1648,18 +1618,20 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) VCMFrameBufferEnum ret = kSizeError; VCMFrameBuffer* frame = static_cast(buffer); + // We are keeping track of the first seq num, the latest seq num and + // the number of wraps to be able to calculate how many packets we expect. + if (_firstPacket) + { + // Now it's time to start estimating jitter + // reset the delay estimate. + _delayEstimate.Reset(); + _firstPacket = false; + } + // Empty packets may bias the jitter estimate (lacking size component), // therefore don't let empty packet trigger the following updates: if (packet.frameType != kFrameEmpty) { - if (_firstPacket) - { - // Now it's time to start estimating jitter - // reset the delay estimate. - _delayEstimate.Reset(); - _firstPacket = false; - } - if (_waitingForCompletion.timestamp == packet.timestamp) { // This can get bad if we have a lot of duplicate packets, @@ -1681,24 +1653,19 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) if (frame != NULL) { VCMFrameBufferStateEnum state = frame->GetState(); - if ((packet.sizeBytes == 0) && - ((state == kStateDecoding) || - (state == kStateEmpty && - _lastDecodedTimeStamp == packet.timestamp))) - { + if (packet.sizeBytes == 0 && packet.timestamp == _lastDecodedTimeStamp) + { // Empty packet (sizeBytes = 0), make sure we update the last - // decoded seq num since this packet belongs either to a frame - // being decoded (condition 1) or to a frame which was already - // decoded and freed (condition 2). A new frame will be created - // for the empty packet. That frame will be empty and later on - // cleaned up. - UpdateLastDecodedWithEmpty(packet); + // decoded sequence number + _lastDecodedSeqNum = LatestSequenceNumber(packet.seqNum, + _lastDecodedSeqNum, NULL); } // Insert packet - // check for first packet - // high sequence number will not be set - bool first = frame->GetHighSeqNum() == -1; + // Check for first packet + // High sequence number will be -1 if neither an empty packet nor + // a media packet has been inserted. + bool first = (frame->GetHighSeqNum() == -1); bufferReturn = frame->InsertPacket(packet, nowMs); ret = bufferReturn; @@ -1762,26 +1729,13 @@ VCMJitterBuffer::InsertPacket(VCMEncodedFrame* buffer, const VCMPacket& packet) return ret; } -void -VCMJitterBuffer::UpdateLastDecodedWithEmpty(const VCMPacket& packet) -{ - // Empty packet inserted to a frame which - // is already decoding. Update the last decoded seq no. - if (_lastDecodedTimeStamp == packet.timestamp && - (packet.seqNum > _lastDecodedSeqNum || - (packet.seqNum < 0x0fff && _lastDecodedSeqNum > 0xf000))) - { - _lastDecodedSeqNum = packet.seqNum; - } -} - // Must be called from within _critSect void VCMJitterBuffer::UpdateOldJitterSample(const VCMPacket& packet) { if (_waitingForCompletion.timestamp != packet.timestamp && - LatestTimestamp(_waitingForCompletion.timestamp, packet.timestamp) == - packet.timestamp) + LatestTimestamp(_waitingForCompletion.timestamp, packet.timestamp, + NULL) == packet.timestamp) { // This is a newer frame than the one waiting for completion. _waitingForCompletion.frameSize = packet.sizeBytes; @@ -1904,8 +1858,8 @@ VCMJitterBuffer::CleanUpOldFrames() // Release the frame if it's older than the last decoded frame. if (_lastDecodedTimeStamp > -1 && LatestTimestamp(static_cast(_lastDecodedTimeStamp), - frameTimeStamp) - == static_cast(_lastDecodedTimeStamp)) + frameTimeStamp, NULL) == + static_cast(_lastDecodedTimeStamp)) { const WebRtc_Word32 frameLowSeqNum = oldestFrame->GetLowSeqNum(); const WebRtc_Word32 frameHighSeqNum = oldestFrame->GetHighSeqNum(); @@ -2027,13 +1981,17 @@ void VCMJitterBuffer::VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame) { frame.MakeSessionDecodable(); // make sure the session can be decoded. + if (frame.FrameType() == kVideoFrameKey) + return; + WebRtc_UWord16 nextExpectedSeqNum = + static_cast(_lastDecodedSeqNum + 1); if (_lastDecodedSeqNum == -1) { // First frame frame.SetPreviousFrameLoss(); } - else if ((WebRtc_UWord16)frame.GetLowSeqNum() != - ((WebRtc_UWord16)_lastDecodedSeqNum + (WebRtc_UWord16)1)) + else if (static_cast(frame.GetLowSeqNum()) != + nextExpectedSeqNum) { // Frame loss frame.SetPreviousFrameLoss(); diff --git a/src/modules/video_coding/main/source/jitter_buffer.h b/src/modules/video_coding/main/source/jitter_buffer.h index 0baecb5ac..fc027e164 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.h +++ b/src/modules/video_coding/main/source/jitter_buffer.h @@ -68,6 +68,9 @@ public: WebRtc_Word32 GetFrameStatistics(WebRtc_UWord32& receivedDeltaFrames, WebRtc_UWord32& receivedKeyFrames) const; + // The number of packets discarded by the jitter buffer because the decoder + // won't be able to decode them. + WebRtc_UWord32 NumNotDecodablePackets() const; // Get number of packets discarded by the jitter buffer WebRtc_UWord32 DiscardedPackets() const; @@ -124,11 +127,8 @@ public: bool& listExtended); WebRtc_Word64 LastDecodedTimestamp() const; - static WebRtc_UWord32 LatestTimestamp(const WebRtc_UWord32 existingTimestamp, - const WebRtc_UWord32 newTimestamp); - -protected: +private: // Misc help functions // Recycle (release) frame, used if we didn't receive whole frame void RecycleFrame(VCMFrameBuffer* frame); @@ -159,6 +159,7 @@ protected: void VerifyAndSetPreviousFrameLost(VCMFrameBuffer& frame); bool IsPacketRetransmitted(const VCMPacket& packet) const; + void UpdateJitterAndDelayEstimates(VCMJitterSample& sample, bool incompleteFrame); void UpdateJitterAndDelayEstimates(VCMFrameBuffer& frame, @@ -176,10 +177,6 @@ protected: WebRtc_Word32 GetLowHighSequenceNumbers(WebRtc_Word32& lowSeqNum, WebRtc_Word32& highSeqNum) const; - void UpdateLastDecodedWithEmpty(const VCMPacket& packet); - -private: - static bool FrameEqualTimestamp(VCMFrameBuffer* frame, const void* timestamp); static bool CompleteDecodableKeyFrameCriteria(VCMFrameBuffer* frame, @@ -208,6 +205,7 @@ private: WebRtc_Word32 _lastDecodedSeqNum; // Timestamp of last frame that was given to decoder WebRtc_Word64 _lastDecodedTimeStamp; + WebRtc_UWord32 _packetsNotDecodable; // Statistics // Frame counter for each type (key, delta, golden, key-delta) diff --git a/src/modules/video_coding/main/source/jitter_buffer_common.cc b/src/modules/video_coding/main/source/jitter_buffer_common.cc new file mode 100644 index 000000000..79a21b440 --- /dev/null +++ b/src/modules/video_coding/main/source/jitter_buffer_common.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "jitter_buffer_common.h" + +#include + +namespace webrtc { + +WebRtc_UWord32 LatestTimestamp(WebRtc_UWord32 timestamp1, + WebRtc_UWord32 timestamp2, + bool* has_wrapped) { + bool wrap = (timestamp2 < 0x0000ffff && timestamp1 > 0xffff0000) || + (timestamp2 > 0xffff0000 && timestamp1 < 0x0000ffff); + if (has_wrapped != NULL) + *has_wrapped = wrap; + if (timestamp1 > timestamp2 && !wrap) + return timestamp1; + else if (timestamp1 <= timestamp2 && !wrap) + return timestamp2; + else if (timestamp1 < timestamp2 && wrap) + return timestamp1; + else + return timestamp2; +} + +WebRtc_Word32 LatestSequenceNumber(WebRtc_Word32 seq_num1, + WebRtc_Word32 seq_num2, + bool* has_wrapped) { + if (seq_num1 < 0 && seq_num2 < 0) + return -1; + else if (seq_num1 < 0) + return seq_num2; + else if (seq_num2 < 0) + return seq_num1; + + bool wrap = (seq_num1 < 0x00ff && seq_num2 > 0xff00) || + (seq_num1 > 0xff00 && seq_num2 < 0x00ff); + + if (has_wrapped != NULL) + *has_wrapped = wrap; + + if (seq_num2 > seq_num1 && !wrap) + return seq_num2; + else if (seq_num2 <= seq_num1 && !wrap) + return seq_num1; + else if (seq_num2 < seq_num1 && wrap) + return seq_num2; + else + return seq_num1; +} + +} // namespace webrtc diff --git a/src/modules/video_coding/main/source/jitter_buffer_common.h b/src/modules/video_coding/main/source/jitter_buffer_common.h index 035a3c117..f86b2173e 100644 --- a/src/modules/video_coding/main/source/jitter_buffer_common.h +++ b/src/modules/video_coding/main/source/jitter_buffer_common.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_COMMON_H_ #define WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_COMMON_H_ +#include "typedefs.h" + namespace webrtc { @@ -62,6 +64,19 @@ enum VCMNaluCompleteness kNaluEnd, // Packet is the end of a NALU }; +// Returns the latest of the two timestamps, compensating for wrap arounds. +// This function assumes that the two timestamps are close in time. +WebRtc_UWord32 LatestTimestamp(WebRtc_UWord32 timestamp1, + WebRtc_UWord32 timestamp2, + bool* has_wrapped); + +// Returns the latest of the two sequence numbers, compensating for wrap +// arounds. This function assumes that the two sequence numbers are close in +// time. +WebRtc_Word32 LatestSequenceNumber(WebRtc_Word32 seq_num1, + WebRtc_Word32 seq_num2, + bool* has_wrapped); + } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_COMMON_H_ diff --git a/src/modules/video_coding/main/source/session_info.cc b/src/modules/video_coding/main/source/session_info.cc index 3b416d2b5..58d95dc6e 100644 --- a/src/modules/video_coding/main/source/session_info.cc +++ b/src/modules/video_coding/main/source/session_info.cc @@ -8,13 +8,9 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "packet.h" #include "session_info.h" -#include -#include - -#include "internal_defines.h" +#include "packet.h" namespace webrtc { @@ -29,7 +25,8 @@ VCMSessionInfo::VCMSessionInfo(): _highestPacketIndex(0), _emptySeqNumLow(-1), _emptySeqNumHigh(-1), - _markerSeqNum(-1) + _markerSeqNum(-1), + _packetsNotDecodable(0) { } @@ -55,7 +52,7 @@ VCMSessionInfo::GetLowSeqNum() const WebRtc_Word32 VCMSessionInfo::GetHighSeqNum() const { - return VCM_MAX(_emptySeqNumHigh, _highSeqNum); + return LatestSequenceNumber(_emptySeqNumHigh, _highSeqNum, NULL); } void @@ -73,6 +70,7 @@ VCMSessionInfo::Reset() { _sessionNACK = false; _highestPacketIndex = 0; _markerSeqNum = -1; + _packetsNotDecodable = 0; } WebRtc_UWord32 @@ -202,7 +200,7 @@ VCMSessionInfo::UpdateCompleteSession() } } -bool VCMSessionInfo::IsSessionComplete() +bool VCMSessionInfo::IsSessionComplete() const { return _completeSession; } @@ -287,6 +285,7 @@ VCMSessionInfo::DeletePackets(WebRtc_UWord8* ptrStartOfLayer, { bytesToDelete += _packets[j].sizeBytes; _packets[j].Reset(); + ++_packetsNotDecodable; } if (bytesToDelete > 0) { @@ -362,7 +361,7 @@ VCMSessionInfo::BuildVP8FragmentationHeader( return new_length; } -int VCMSessionInfo::FindNextPartitionBeginning(int packet_index) const { +int VCMSessionInfo::FindNextPartitionBeginning(int packet_index) { while (packet_index <= _highestPacketIndex) { if (_packets[packet_index].completeNALU == kNaluUnset) { // Missing packet @@ -371,8 +370,13 @@ int VCMSessionInfo::FindNextPartitionBeginning(int packet_index) const { } const bool beginning = _packets[packet_index].codecSpecificHeader. codecHeader.VP8.beginningOfPartition; - if (beginning) + if (beginning) { return packet_index; + } else { + // This packet belongs to a partition with a previous loss and can't + // be decoded. + ++_packetsNotDecodable; + } ++packet_index; } return packet_index; @@ -433,7 +437,7 @@ VCMSessionInfo::MakeDecodable(WebRtc_UWord8* ptrStartOfLayer) } returnLength += DeletePackets(ptrStartOfLayer, - packetIndex, endIndex); + packetIndex + 1, endIndex); packetIndex = endIndex; }// end lost packet } @@ -636,7 +640,7 @@ VCMSessionInfo::GetHighestPacketIndex() } bool -VCMSessionInfo::HaveLastPacket() +VCMSessionInfo::HaveLastPacket() const { return _markerBit; } @@ -649,25 +653,11 @@ VCMSessionInfo::ForceSetHaveLastPacket() } bool -VCMSessionInfo::IsRetransmitted() +VCMSessionInfo::IsRetransmitted() const { return _sessionNACK; } -void -VCMSessionInfo::UpdatePacketSize(WebRtc_Word32 packetIndex, - WebRtc_UWord32 length) -{ - // sanity - if (packetIndex >= kMaxPacketsInJitterBuffer || packetIndex < 0) - { - // not allowed - assert(!"SessionInfo::UpdatePacketSize Error: invalid packetIndex"); - return; - } - _packets[packetIndex].sizeBytes = length; -} - WebRtc_Word64 VCMSessionInfo::InsertPacket(const VCMPacket& packet, WebRtc_UWord8* ptrStartOfLayer) @@ -848,6 +838,7 @@ VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, // missing the previous packet. memset(ptrFirstByte, 0, _packets[i].sizeBytes); previousLost = true; + ++_packetsNotDecodable; } else if (_packets[i].sizeBytes > 0) // Ignore if empty packet { @@ -870,6 +861,7 @@ VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, { memset(ptrStartOfLayer, 0, _packets[i].sizeBytes); previousLost = true; + ++_packetsNotDecodable; } } else if (_packets[i].sizeBytes == 0 && codec == kVideoCodecH263) @@ -899,4 +891,8 @@ VCMSessionInfo::PrepareForDecode(WebRtc_UWord8* ptrStartOfLayer, return length; } +int VCMSessionInfo::NotDecodablePackets() const { + return _packetsNotDecodable; +} + } diff --git a/src/modules/video_coding/main/source/session_info.h b/src/modules/video_coding/main/source/session_info.h index 8ea664a56..64878a5ae 100644 --- a/src/modules/video_coding/main/source/session_info.h +++ b/src/modules/video_coding/main/source/session_info.h @@ -44,7 +44,7 @@ public: WebRtc_UWord8* ptrStartOfLayer); WebRtc_Word32 InformOfEmptyPacket(const WebRtc_UWord16 seqNum); - virtual bool IsSessionComplete(); + virtual bool IsSessionComplete() const; // Builds fragmentation headers for VP8, each fragment being a decodable // VP8 partition. Returns the total number of bytes which are decodable. Is @@ -60,14 +60,12 @@ public: WebRtc_UWord32 MakeDecodable(WebRtc_UWord8* ptrStartOfLayer); WebRtc_UWord32 GetSessionLength(); - bool HaveLastPacket(); + bool HaveLastPacket() const; void ForceSetHaveLastPacket(); - bool IsRetransmitted(); + bool IsRetransmitted() const; webrtc::FrameType FrameType() const { return _frameType; } virtual WebRtc_Word32 GetHighestPacketIndex(); - virtual void UpdatePacketSize(WebRtc_Word32 packetIndex, - WebRtc_UWord32 length); void SetStartSeqNumber(WebRtc_UWord16 seqNumber); @@ -83,10 +81,17 @@ public: void SetPreviousFrameLoss() { _previousFrameLoss = true; } bool PreviousFrameLoss() const { return _previousFrameLoss; } -protected: - // Finds the packet index of the next VP8 partition. If none is found - // _highestPacketIndex + 1 is returned. - int FindNextPartitionBeginning(int packet_index) const; + // The number of packets discarded because the decoder can't make use of + // them. + int NotDecodablePackets() const; + +private: + // Finds the packet index of the beginning of the next VP8 partition. If + // none is found _highestPacketIndex + 1 is returned. packet_index is + // expected to be the index of the last decodable packet of the previous + // partitions + 1, or 0 for the first partition. + int FindNextPartitionBeginning(int packet_index); + // Finds the packet index of the end of the partition with index // partitionIndex. int FindPartitionEnd(int packet_index) const; @@ -123,6 +128,8 @@ protected: WebRtc_Word32 _emptySeqNumHigh; // Store the sequence number that marks the last media packet WebRtc_Word32 _markerSeqNum; + // Number of packets discarded because the decoder can't use them. + int _packetsNotDecodable; }; } // namespace webrtc diff --git a/src/modules/video_coding/main/source/video_coding.gypi b/src/modules/video_coding/main/source/video_coding.gypi index 49add57f1..88fce2a4e 100644 --- a/src/modules/video_coding/main/source/video_coding.gypi +++ b/src/modules/video_coding/main/source/video_coding.gypi @@ -50,8 +50,8 @@ 'generic_encoder.h', 'inter_frame_delay.h', 'internal_defines.h', - 'jitter_buffer_common.h', 'jitter_buffer.h', + 'jitter_buffer_common.h', 'jitter_estimator.h', 'media_opt_util.h', 'media_optimization.h', @@ -81,6 +81,7 @@ 'generic_encoder.cc', 'inter_frame_delay.cc', 'jitter_buffer.cc', + 'jitter_buffer_common.cc', 'jitter_estimator.cc', 'media_opt_util.cc', 'media_optimization.cc', diff --git a/src/modules/video_coding/main/test/jitter_buffer_test.cc b/src/modules/video_coding/main/test/jitter_buffer_test.cc index a125228bd..8a0f36c25 100644 --- a/src/modules/video_coding/main/test/jitter_buffer_test.cc +++ b/src/modules/video_coding/main/test/jitter_buffer_test.cc @@ -819,6 +819,94 @@ int JitterBufferTest(CmdArgs& args) TEST(bitRate > 10000000); + jb.Flush(); + + // + // TEST packet loss. Verify missing packets statistics and not decodable + // packets statistics. + // Insert 10 frames consisting of 4 packets and remove one from all of them. + // The last packet is a empty (non-media) packet + // + + // Select a start seqNum which triggers a difficult wrap situation + seqNum = 0xffff - 4; + for (int i=0; i < 10; ++i) { + webrtc::FrameType frametype = kVideoFrameDelta; + if (i == 0) + frametype = kVideoFrameKey; + seqNum++; + timeStamp += 33*90; + packet.frameType = frametype; + if (i == 0) + packet.frameType = frametype; + packet.isFirstPacket = true; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.timestamp = timeStamp; + packet.completeNALU = kNaluStart; + + frameIn = jb.GetFrame(packet); + TEST(frameIn != 0); + + // Insert a packet into a frame + TEST(kFirstPacket == jb.InsertPacket(frameIn, packet)); + + // get packet notification + TEST(timeStamp == jb.GetNextTimeStamp(10, incomingFrameType, renderTimeMs)); + + // check incoming frame type + TEST(incomingFrameType == frametype); + + // get the frame + frameOut = jb.GetCompleteFrameForDecoding(10); + + // it should not be complete + TEST(frameOut == 0); + + seqNum += 2; + packet.isFirstPacket = false; + packet.markerBit = true; + packet.seqNum = seqNum; + packet.completeNALU = kNaluEnd; + + frameIn = jb.GetFrame(packet); + TEST(frameIn != 0); + + // Insert a packet into a frame + TEST(kIncomplete == jb.InsertPacket(frameIn, packet)); + + + // Insert an empty (non-media) packet + seqNum++; + packet.isFirstPacket = false; + packet.markerBit = false; + packet.seqNum = seqNum; + packet.completeNALU = kNaluEnd; + packet.frameType = kFrameEmpty; + + frameIn = jb.GetFrame(packet); + TEST(frameIn != 0); + + // Insert a packet into a frame + TEST(kIncomplete == jb.InsertPacket(frameIn, packet)); + + // get the frame + frameOut = jb.GetFrameForDecoding(); + + // One of the packets has been discarded by the jitter buffer + CheckOutFrame(frameOut, size, false); + + // check the frame type + TEST(frameOut->FrameType() == frametype); + TEST(frameOut->Complete() == false); + TEST(frameOut->MissingFrame() == false); + + // Release frame (when done with decoding) + jb.ReleaseFrame(frameOut); + } + + TEST(jb.NumNotDecodablePackets() == 10); + // Insert 3 old packets and verify that we have 3 discarded packets packet.timestamp = timeStamp - 1000; frameIn = jb.GetFrame(packet);