From 0a3c1471b873ea7f81bff2faa7cf0d9e563c7d53 Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Thu, 5 Dec 2013 14:05:07 +0000 Subject: [PATCH] Add API to query video engine for the send-side delay. R=mflodman@webrtc.org Review URL: https://webrtc-codereview.appspot.com/4559005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@5225 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h | 3 ++ webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h | 2 ++ .../modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 12 +++++++ .../modules/rtp_rtcp/source/rtp_rtcp_impl.h | 4 +++ webrtc/modules/rtp_rtcp/source/rtp_sender.cc | 31 +++++++++++++++++ webrtc/modules/rtp_rtcp/source/rtp_sender.h | 12 +++++++ webrtc/video_engine/include/vie_codec.h | 4 +++ .../auto_test/source/vie_autotest_codec.cc | 14 +++++--- webrtc/video_engine/vie_channel.cc | 34 +++++++++++++++++++ webrtc/video_engine/vie_channel.h | 1 + webrtc/video_engine/vie_codec_impl.cc | 14 ++++++++ webrtc/video_engine/vie_codec_impl.h | 2 ++ 12 files changed, 129 insertions(+), 4 deletions(-) diff --git a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h index 51471ba3d..f8687a552 100644 --- a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h +++ b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h @@ -335,6 +335,9 @@ class RtpRtcp : public Module { FrameCountObserver* observer) = 0; virtual FrameCountObserver* GetSendFrameCountObserver() const = 0; + virtual bool GetSendSideDelay(int* avg_send_delay_ms, + int* max_send_delay_ms) const = 0; + /************************************************************************** * * RTCP diff --git a/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h b/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h index 9ca48975a..293412b6f 100644 --- a/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h +++ b/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h @@ -116,6 +116,8 @@ class MockRtpRtcp : public RtpRtcp { bool retransmission)); MOCK_METHOD1(TimeToSendPadding, int(int bytes)); + MOCK_CONST_METHOD2(GetSendSideDelay, + bool(int* avg_send_delay_ms, int* max_send_delay_ms)); MOCK_METHOD3(RegisterRtcpObservers, void(RtcpIntraFrameObserver* intraFrameCallback, RtcpBandwidthObserver* bandwidthCallback, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 4af8cde5f..00e34125c 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -732,6 +732,18 @@ int ModuleRtpRtcpImpl::TimeToSendPadding(int bytes) { return 0; } +bool ModuleRtpRtcpImpl::GetSendSideDelay(int* avg_send_delay_ms, + int* max_send_delay_ms) const { + assert(avg_send_delay_ms); + assert(max_send_delay_ms); + + if (!child_modules_.empty()) { + // This API is only supported for child modules. + return false; + } + return rtp_sender_.GetSendSideDelay(avg_send_delay_ms, max_send_delay_ms); +} + uint16_t ModuleRtpRtcpImpl::MaxPayloadLength() const { WEBRTC_TRACE(kTraceModuleCall, kTraceRtpRtcp, id_, "MaxPayloadLength()"); diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h index 0dd25492d..ae070f925 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h @@ -134,6 +134,10 @@ class ModuleRtpRtcpImpl : public RtpRtcp { // Returns the number of padding bytes actually sent, which can be more or // less than |bytes|. virtual int TimeToSendPadding(int bytes) OVERRIDE; + + virtual bool GetSendSideDelay(int* avg_send_delay_ms, + int* max_send_delay_ms) const OVERRIDE; + // RTCP part. // Get RTCP status. diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc index c7179646b..787f8717f 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc @@ -22,6 +22,7 @@ namespace webrtc { // Max in the RFC 3550 is 255 bytes, we limit it to be modulus 32 for SRTP. const int kMaxPaddingLength = 224; +const int kSendSideDelayWindowMs = 1000; namespace { @@ -127,6 +128,23 @@ uint32_t RTPSender::NackOverheadRate() const { return nack_bitrate_.BitrateLast(); } +bool RTPSender::GetSendSideDelay(int* avg_send_delay_ms, + int* max_send_delay_ms) const { + CriticalSectionScoped cs(statistics_crit_.get()); + SendDelayMap::const_iterator it = send_delays_.upper_bound( + clock_->TimeInMilliseconds() - kSendSideDelayWindowMs); + if (!sending_media_ || it == send_delays_.end()) + return false; + int num_delays = 0; + for (; it != send_delays_.end(); ++it) { + *max_send_delay_ms = std::max(*max_send_delay_ms, it->second); + *avg_send_delay_ms += it->second; + ++num_delays; + } + *avg_send_delay_ms = (*avg_send_delay_ms + num_delays / 2) / num_delays; + return true; +} + int32_t RTPSender::SetTransmissionTimeOffset( const int32_t transmission_time_offset) { if (transmission_time_offset > (0x800000 - 1) || @@ -756,6 +774,9 @@ bool RTPSender::TimeToSendPacket(uint16_t sequence_number, // Packet cannot be found. Allow sending to continue. return true; } + if (!retransmission && capture_time_ms > 0) { + UpdateDelayStatistics(capture_time_ms, clock_->TimeInMilliseconds()); + } return PrepareAndSendPacket(data_buffer, length, capture_time_ms, retransmission && (rtx_ & kRtxRetransmitted) > 0); } @@ -871,12 +892,22 @@ int32_t RTPSender::SendToNetwork( return 0; } } + if (capture_time_ms > 0) { + UpdateDelayStatistics(capture_time_ms, now_ms); + } if (SendPacketToNetwork(buffer, payload_length + rtp_header_length)) { return 0; } return -1; } +void RTPSender::UpdateDelayStatistics(int64_t capture_time_ms, int64_t now_ms) { + CriticalSectionScoped cs(statistics_crit_.get()); + send_delays_[now_ms] = now_ms - capture_time_ms; + send_delays_.erase(send_delays_.begin(), + send_delays_.lower_bound(now_ms - kSendSideDelayWindowMs)); +} + void RTPSender::ProcessBitrate() { CriticalSectionScoped cs(send_critsect_); Bitrate::Process(); diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.h b/webrtc/modules/rtp_rtcp/source/rtp_sender.h index cd8cd2180..becd22ad1 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.h @@ -78,6 +78,10 @@ class RTPSender : public Bitrate, public RTPSenderInterface { uint32_t FecOverheadRate() const; uint32_t NackOverheadRate() const; + // Returns true if the statistics have been calculated, and false if no frame + // was sent within the statistics window. + bool GetSendSideDelay(int* avg_send_delay_ms, int* max_send_delay_ms) const; + void SetTargetSendBitrate(const uint32_t bits); virtual uint16_t MaxDataPayloadLength() const @@ -272,6 +276,11 @@ class RTPSender : public Bitrate, public RTPSenderInterface { RtpVideoCodecTypes *video_type); private: + // Maps capture time in milliseconds to send-side delay in milliseconds. + // Send-side delay is the difference between transmission time and capture + // time. + typedef std::map SendDelayMap; + int CreateRTPHeader(uint8_t* header, int8_t payload_type, uint32_t ssrc, bool marker_bit, uint32_t timestamp, uint16_t sequence_number, @@ -296,6 +305,8 @@ class RTPSender : public Bitrate, public RTPSenderInterface { bool SendPacketToNetwork(const uint8_t *packet, uint32_t size); + void UpdateDelayStatistics(int64_t capture_time_ms, int64_t now_ms); + int32_t id_; const bool audio_configured_; RTPSenderAudio *audio_; @@ -329,6 +340,7 @@ class RTPSender : public Bitrate, public RTPSenderInterface { scoped_ptr statistics_crit_; uint32_t packets_sent_; uint32_t payload_bytes_sent_; + SendDelayMap send_delays_; // RTP variables bool start_time_stamp_forced_; diff --git a/webrtc/video_engine/include/vie_codec.h b/webrtc/video_engine/include/vie_codec.h index aaf017d2b..f658aaa24 100644 --- a/webrtc/video_engine/include/vie_codec.h +++ b/webrtc/video_engine/include/vie_codec.h @@ -198,6 +198,10 @@ class WEBRTC_DLLEXPORT ViECodec { // This is under development; not tested. virtual void SuspendBelowMinBitrate(int video_channel) = 0; + // TODO(holmer): Remove this default implementation when possible. + virtual bool GetSendSideDelay(int video_channel, int* avg_delay_ms, + int* max_delay_ms) const { return false; } + protected: ViECodec() {} virtual ~ViECodec() {} diff --git a/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc b/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc index 0ebfbecbe..773192ec5 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc @@ -258,10 +258,16 @@ void ViEAutoTest::ViECodecStandardTest() { } AutoTestSleep(kAutoTestSleepTimeMs); - // Verify the delay estimate is larger than 0. - int delay_ms = 0; - EXPECT_EQ(0, codec->GetReceiveSideDelay(video_channel, &delay_ms)); - EXPECT_GT(delay_ms, 0); + // Verify the delay estimates are larger than 0. + int avg_send_delay = 0; + int max_send_delay = 0; + EXPECT_TRUE(codec->GetSendSideDelay(video_channel, &avg_send_delay, + &max_send_delay)); + EXPECT_GT(avg_send_delay, 0); + EXPECT_GE(max_send_delay, avg_send_delay); + int receive_delay_ms = 0; + EXPECT_EQ(0, codec->GetReceiveSideDelay(video_channel, &receive_delay_ms)); + EXPECT_GT(receive_delay_ms, 0); EXPECT_EQ(0, base->StopSend(video_channel)); EXPECT_EQ(0, codec->DeregisterEncoderObserver(video_channel)); diff --git a/webrtc/video_engine/vie_channel.cc b/webrtc/video_engine/vie_channel.cc index eaa60447a..8b71a198c 100644 --- a/webrtc/video_engine/vie_channel.cc +++ b/webrtc/video_engine/vie_channel.cc @@ -1374,6 +1374,40 @@ void ViEChannel::GetBandwidthUsage(uint32_t* total_bitrate_sent, } } +bool ViEChannel::GetSendSideDelay(int* avg_send_delay, + int* max_send_delay) const { + WEBRTC_TRACE(kTraceInfo, kTraceVideo, ViEId(engine_id_, channel_id_), "%s", + __FUNCTION__); + + *avg_send_delay = 0; + *max_send_delay = 0; + bool valid_estimate = false; + int num_send_delays = 0; + if (rtp_rtcp_->GetSendSideDelay(avg_send_delay, max_send_delay)) { + ++num_send_delays; + valid_estimate = true; + } + CriticalSectionScoped cs(rtp_rtcp_cs_.get()); + for (std::list::const_iterator it = simulcast_rtp_rtcp_.begin(); + it != simulcast_rtp_rtcp_.end(); it++) { + RtpRtcp* rtp_rtcp = *it; + int sub_stream_avg_delay = 0; + int sub_stream_max_delay = 0; + if (rtp_rtcp->GetSendSideDelay(&sub_stream_avg_delay, + &sub_stream_max_delay)) { + *avg_send_delay += sub_stream_avg_delay; + *max_send_delay = std::max(*max_send_delay, sub_stream_max_delay); + ++num_send_delays; + } + } + if (num_send_delays > 0) { + valid_estimate = true; + *avg_send_delay = *avg_send_delay / num_send_delays; + *avg_send_delay = (*avg_send_delay + num_send_delays / 2) / num_send_delays; + } + return valid_estimate; +} + void ViEChannel::GetEstimatedReceiveBandwidth( uint32_t* estimated_bandwidth) const { vie_receiver_.EstimatedReceiveBandwidth(estimated_bandwidth); diff --git a/webrtc/video_engine/vie_channel.h b/webrtc/video_engine/vie_channel.h index a17bb2b92..35981fcca 100644 --- a/webrtc/video_engine/vie_channel.h +++ b/webrtc/video_engine/vie_channel.h @@ -194,6 +194,7 @@ class ViEChannel uint32_t* video_bitrate_sent, uint32_t* fec_bitrate_sent, uint32_t* nackBitrateSent) const; + bool GetSendSideDelay(int* avg_send_delay, int* max_send_delay) const; void GetEstimatedReceiveBandwidth(uint32_t* estimated_bandwidth) const; int32_t StartRTPDump(const char file_nameUTF8[1024], diff --git a/webrtc/video_engine/vie_codec_impl.cc b/webrtc/video_engine/vie_codec_impl.cc index 364862fe2..b46eb88d3 100644 --- a/webrtc/video_engine/vie_codec_impl.cc +++ b/webrtc/video_engine/vie_codec_impl.cc @@ -738,6 +738,20 @@ void ViECodecImpl::SuspendBelowMinBitrate(int video_channel) { vie_channel->SetTransmissionSmoothingStatus(true); } +bool ViECodecImpl::GetSendSideDelay(int video_channel, int* avg_delay_ms, + int* max_delay_ms) const { + ViEChannelManagerScoped cs(*(shared_data_->channel_manager())); + ViEChannel* vie_channel = cs.Channel(video_channel); + if (!vie_channel) { + WEBRTC_TRACE(kTraceError, kTraceVideo, + ViEId(shared_data_->instance_id(), video_channel), + "%s: No channel %d", __FUNCTION__, video_channel); + shared_data_->SetLastError(kViECodecInvalidChannelId); + return false; + } + return vie_channel->GetSendSideDelay(avg_delay_ms, max_delay_ms); +} + bool ViECodecImpl::CodecValid(const VideoCodec& video_codec) { // Check pl_name matches codec_type. if (video_codec.codecType == kVideoCodecRED) { diff --git a/webrtc/video_engine/vie_codec_impl.h b/webrtc/video_engine/vie_codec_impl.h index 372ffc997..cee3ed2b2 100644 --- a/webrtc/video_engine/vie_codec_impl.h +++ b/webrtc/video_engine/vie_codec_impl.h @@ -71,6 +71,8 @@ class ViECodecImpl const char* file_name_utf8); virtual int StopDebugRecording(int video_channel); virtual void SuspendBelowMinBitrate(int video_channel); + virtual bool GetSendSideDelay(int video_channel, int* avg_delay_ms, + int* max_delay_ms) const; protected: explicit ViECodecImpl(ViESharedData* shared_data);