diff --git a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 69e9d3aa7..4b031a4b8 100644 --- a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "gflags/gflags.h" #include "gtest/gtest.h" #include "testsupport/fileutils.h" @@ -30,17 +32,18 @@ class ViEVideoVerificationTest : public testing::Test { protected: void SetUp() { input_file_ = webrtc::test::ResourcePath("paris_qcif", "yuv"); - local_file_renderer_ = new ViEToFileRenderer(); - remote_file_renderer_ = new ViEToFileRenderer(); - SetUpLocalFileRenderer(local_file_renderer_); - SetUpRemoteFileRenderer(remote_file_renderer_); } void TearDown() { TearDownFileRenderer(local_file_renderer_); - delete local_file_renderer_; TearDownFileRenderer(remote_file_renderer_); - delete remote_file_renderer_; + } + + void InitializeFileRenderers() { + local_file_renderer_ = new ViEToFileRenderer(); + remote_file_renderer_ = new ViEToFileRenderer(); + SetUpLocalFileRenderer(local_file_renderer_); + SetUpRemoteFileRenderer(remote_file_renderer_); } void SetUpLocalFileRenderer(ViEToFileRenderer* file_renderer) { @@ -58,6 +61,7 @@ class ViEVideoVerificationTest : public testing::Test { } void TearDownFileRenderer(ViEToFileRenderer* file_renderer) { + assert(file_renderer); bool test_failed = ::testing::UnitTest::GetInstance()-> current_test_info()->result()->Failed(); if (test_failed) { @@ -67,6 +71,7 @@ class ViEVideoVerificationTest : public testing::Test { // No reason to keep the files if we succeeded. file_renderer->DeleteOutputFile(); } + delete file_renderer; } void CompareFiles(const std::string& reference_file, @@ -115,7 +120,8 @@ class ViEVideoVerificationTest : public testing::Test { } }; -TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { +TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { + InitializeFileRenderers(); ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight, local_file_renderer_, remote_file_renderer_)); @@ -132,6 +138,7 @@ TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { } TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) { + InitializeFileRenderers(); ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight, local_file_renderer_, remote_file_renderer_)); @@ -157,58 +164,63 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) { // in the encoder. The local and remote file will not be of equal size because // of unknown reasons. Tests show that they start at the same frame, which is // the important thing when doing frame-to-frame comparison with PSNR/SSIM. -TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) { - // Use our own FrameDropMonitoringRemoteFileRenderer instead of the - // ViEToFileRenderer from the test fixture: - // TODO(kjellander): Find a better way to reuse this code without duplication. - remote_file_renderer_->StopRendering(); - TearDownFileRenderer(remote_file_renderer_); - delete remote_file_renderer_; - +TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) { FrameDropDetector detector; + local_file_renderer_ = new ViEToFileRenderer(); remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector); + SetUpLocalFileRenderer(local_file_renderer_); SetUpRemoteFileRenderer(remote_file_renderer_); // Set a low bit rate so the encoder budget will be tight, causing it to drop // frames every now and then. const int kBitRateKbps = 50; - ViETest::Log("Bit rate: %d kbps.\n", kBitRateKbps); + const int kPacketLossPercent = 10; + const int kNetworkDelayMs = 100; + ViETest::Log("Bit rate : %5d kbps", kBitRateKbps); + ViETest::Log("Packet loss : %5d %%", kPacketLossPercent); + ViETest::Log("Network delay: %5d ms", kNetworkDelayMs); tests_.TestFullStack(input_file_, kInputWidth, kInputHeight, kBitRateKbps, + kPacketLossPercent, kNetworkDelayMs, local_file_renderer_, remote_file_renderer_, &detector); const std::string reference_file = local_file_renderer_->GetFullOutputPath(); const std::string output_file = remote_file_renderer_->GetFullOutputPath(); StopRenderers(); - ASSERT_EQ(detector.GetFramesDroppedAtRenderStep().size(), - detector.GetFramesDroppedAtDecodeStep().size()) - << "The number of dropped frames on the decode and render are not equal, " - "this may be because we have a major problem in the jitter buffer?"; - + detector.CalculateResults(); detector.PrintReport(); + if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) != + detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) { + detector.PrintDebugDump(); + } + + ASSERT_EQ(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered), + detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) + << "The number of dropped frames on the decode and render steps are not " + "equal. This may be because we have a major problem in the buffers of " + "the ViEToFileRenderer?"; + // We may have dropped frames during the processing, which means the output // file does not contain all the frames that are present in the input file. // To make the quality measurement correct, we must adjust the output file to // that by copying the last successful frame into the place where the dropped // frame would be, for all dropped frames. const int frame_length_in_bytes = 3 * kInputHeight * kInputWidth / 2; - int num_frames = detector.NumberSentFrames(); - ViETest::Log("Frame length: %d bytes\n", frame_length_in_bytes); - FixOutputFileForComparison(output_file, num_frames, frame_length_in_bytes, - detector.GetFramesDroppedAtDecodeStep()); + ViETest::Log("Frame length: %d bytes", frame_length_in_bytes); + std::vector all_frames = detector.GetAllFrames(); + FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames); // Verify all sent frames are present in the output file. size_t output_file_size = webrtc::test::GetFileSize(output_file); - EXPECT_EQ(num_frames, - static_cast(output_file_size / frame_length_in_bytes)) - << "The output file size is incorrect. It should be equal to the number" + EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes) + << "The output file size is incorrect. It should be equal to the number " "of frames multiplied by the frame size. This will likely affect " "PSNR/SSIM calculations in a bad way."; // We are running on a lower bitrate here so we need to settle for somewhat // lower PSNR and SSIM values. - const double kExpectedMinimumPSNR = 25; - const double kExpectedMinimumSSIM = 0.8; + const double kExpectedMinimumPSNR = 24; + const double kExpectedMinimumSSIM = 0.7; CompareFiles(reference_file, output_file, kExpectedMinimumPSNR, kExpectedMinimumSSIM); } diff --git a/src/video_engine/test/auto_test/interface/tb_external_transport.h b/src/video_engine/test/auto_test/interface/tb_external_transport.h index 46a1448fc..65788116d 100644 --- a/src/video_engine/test/auto_test/interface/tb_external_transport.h +++ b/src/video_engine/test/auto_test/interface/tb_external_transport.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -12,8 +12,8 @@ // tb_external_transport.h // -#ifndef WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_ -#define WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_ +#ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_ +#define WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_ #include @@ -27,7 +27,38 @@ class ThreadWrapper; class ViENetwork; } -class TbExternalTransport: public webrtc::Transport +// Allows to subscribe for callback when a frame is started being sent. +class SendFrameCallback +{ +public: + // Called once per frame (when a new RTP timestamp is detected) when the + // first data packet of the frame is being sent using the + // TbExternalTransport.SendPacket method. + virtual void FrameSent(unsigned int rtp_timestamp) = 0; +protected: + SendFrameCallback() {} + virtual ~SendFrameCallback() {} +}; + +// Allows to subscribe for callback when the first packet of a frame is +// received. +class ReceiveFrameCallback +{ +public: + // Called once per frame (when a new RTP timestamp is detected) + // during the processing of the RTP packet queue in + // TbExternalTransport::ViEExternalTransportProcess. + virtual void FrameReceived(unsigned int rtp_timestamp) = 0; +protected: + ReceiveFrameCallback() {} + virtual ~ReceiveFrameCallback() {} +}; + +// External transport implementation for testing purposes. +// A packet loss probability must be set in order to drop packets from the data +// being sent to this class. +// Will never drop packets from the first frame of a video sequence. +class TbExternalTransport : public webrtc::Transport { public: TbExternalTransport(webrtc::ViENetwork& vieNetwork); @@ -36,7 +67,17 @@ public: virtual int SendPacket(int channel, const void *data, int len); virtual int SendRTCPPacket(int channel, const void *data, int len); - WebRtc_Word32 SetPacketLoss(WebRtc_Word32 lossRate); // Rate in % + // Should only be called before/after traffic is being processed. + // Only one observer can be set (multiple calls will overwrite each other). + virtual void RegisterSendFrameCallback(SendFrameCallback* callback); + + // Should only be called before/after traffic is being processed. + // Only one observer can be set (multiple calls will overwrite each other). + virtual void RegisterReceiveFrameCallback(ReceiveFrameCallback* callback); + + // The probability of a packet of being dropped. Packets belonging to the + // first packet (same RTP timestamp) will never be dropped. + WebRtc_Word32 SetPacketLoss(WebRtc_Word32 lossRate); // Rate in % void SetNetworkDelay(WebRtc_Word64 delayMs); void SetSSRCFilter(WebRtc_UWord32 SSRC); @@ -89,6 +130,9 @@ private: std::list _rtpPackets; std::list _rtcpPackets; + SendFrameCallback* _send_frame_callback; + ReceiveFrameCallback* _receive_frame_callback; + unsigned char _temporalLayers; unsigned short _seqNum; unsigned short _sendPID; @@ -103,6 +147,13 @@ private: WebRtc_UWord32 _SSRC; bool _checkSequenceNumber; WebRtc_UWord16 _firstSequenceNumber; + + // Keep track of the first RTP timestamp so we don't do packet loss on + // the first frame. + WebRtc_UWord32 _firstRTPTimestamp; + // Track RTP timestamps so we invoke callbacks properly (if registered). + WebRtc_UWord32 _lastSendRTPTimestamp; + WebRtc_UWord32 _lastReceiveRTPTimestamp; }; -#endif // WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_ +#endif // WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_ diff --git a/src/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h b/src/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h index 009ec3093..a9560bcf0 100644 --- a/src/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h +++ b/src/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -58,6 +58,8 @@ class ViEFileBasedComparisonTests { int width, int height, int bit_rate_kbps, + int packet_loss_percent, + int network_delay_ms, ViEToFileRenderer* local_file_renderer, ViEToFileRenderer* remote_file_renderer, FrameDropDetector* frame_drop_detector); diff --git a/src/video_engine/test/auto_test/primitives/framedrop_primitives.cc b/src/video_engine/test/auto_test/primitives/framedrop_primitives.cc index ae4a10a3a..28ae7020a 100644 --- a/src/video_engine/test/auto_test/primitives/framedrop_primitives.cc +++ b/src/video_engine/test/auto_test/primitives/framedrop_primitives.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -11,49 +11,85 @@ #include #include -#include "framedrop_primitives.h" - -#include "general_primitives.h" +#include "modules/video_capture/main/interface/video_capture_factory.h" #include "system_wrappers/interface/tick_util.h" -#include "tb_interfaces.h" #include "testsupport/fileutils.h" #include "testsupport/frame_reader.h" #include "testsupport/frame_writer.h" -#include "video_capture_factory.h" -#include "vie_autotest.h" -#include "vie_autotest_defines.h" -#include "vie_to_file_renderer.h" +#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h" +#include "video_engine/test/auto_test/interface/tb_interfaces.h" +#include "video_engine/test/auto_test/interface/tb_external_transport.h" +#include "video_engine/test/auto_test/interface/vie_autotest.h" +#include "video_engine/test/auto_test/interface/vie_autotest_defines.h" +#include "video_engine/test/auto_test/primitives/framedrop_primitives.h" +#include "video_engine/test/auto_test/primitives/general_primitives.h" -// Tracks which frames are sent on the local side and reports them to the +// Tracks which frames are created on the local side and reports them to the // FrameDropDetector class. -class SendTimestampEffectFilter: public webrtc::ViEEffectFilter { +class CreatedTimestampEffectFilter : public webrtc::ViEEffectFilter { public: - explicit SendTimestampEffectFilter(FrameDropDetector* frame_drop_detector) + explicit CreatedTimestampEffectFilter(FrameDropDetector* frame_drop_detector) : frame_drop_detector_(frame_drop_detector) {} - virtual ~SendTimestampEffectFilter() {} + virtual ~CreatedTimestampEffectFilter() {} virtual int Transform(int size, unsigned char* frameBuffer, unsigned int timeStamp90KHz, unsigned int width, unsigned int height) { - frame_drop_detector_->ReportSent(timeStamp90KHz); + frame_drop_detector_->ReportFrameState(FrameDropDetector::kCreated, + timeStamp90KHz); return 0; } + + private: + FrameDropDetector* frame_drop_detector_; +}; + +// Tracks which frames are sent in external transport on the local side +// and reports them to the FrameDropDetector class. +class FrameSentCallback : public SendFrameCallback { + public: + explicit FrameSentCallback(FrameDropDetector* frame_drop_detector) + : frame_drop_detector_(frame_drop_detector) {} + virtual ~FrameSentCallback() {} + virtual void FrameSent(unsigned int rtp_timestamp) { + frame_drop_detector_->ReportFrameState(FrameDropDetector::kSent, + rtp_timestamp); + } + + private: + FrameDropDetector* frame_drop_detector_; +}; + +// Tracks which frames are received in external transport on the remote side +// and reports them to the FrameDropDetector class. +class FrameReceivedCallback : public ReceiveFrameCallback { + public: + explicit FrameReceivedCallback(FrameDropDetector* frame_drop_detector) + : frame_drop_detector_(frame_drop_detector) {} + virtual ~FrameReceivedCallback() {} + virtual void FrameReceived(unsigned int rtp_timestamp) { + frame_drop_detector_->ReportFrameState(FrameDropDetector::kReceived, + rtp_timestamp); + } + private: FrameDropDetector* frame_drop_detector_; }; // Tracks when frames are decoded on the remote side (received from the // jitter buffer) and reports them to the FrameDropDetector class. -class DecodeTimestampEffectFilter: public webrtc::ViEEffectFilter { +class DecodedTimestampEffectFilter : public webrtc::ViEEffectFilter { public: - explicit DecodeTimestampEffectFilter(FrameDropDetector* frame_drop_detector) + explicit DecodedTimestampEffectFilter(FrameDropDetector* frame_drop_detector) : frame_drop_detector_(frame_drop_detector) {} - virtual ~DecodeTimestampEffectFilter() {} + virtual ~DecodedTimestampEffectFilter() {} virtual int Transform(int size, unsigned char* frameBuffer, unsigned int timeStamp90KHz, unsigned int width, unsigned int height) { - frame_drop_detector_->ReportDecoded(timeStamp90KHz); + frame_drop_detector_->ReportFrameState(FrameDropDetector::kDecoded, + timeStamp90KHz); return 0; } + private: FrameDropDetector* frame_drop_detector_; }; @@ -64,6 +100,8 @@ void TestFullStack(const TbInterfaces& interfaces, int width, int height, int bit_rate_kbps, + int packet_loss_percent, + int network_delay_ms, FrameDropDetector* frame_drop_detector) { webrtc::VideoEngine *video_engine_interface = interfaces.video_engine; webrtc::ViEBase *base_interface = interfaces.base; @@ -86,12 +124,20 @@ void TestFullStack(const TbInterfaces& interfaces, EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec)); } - const char *ip_address = "127.0.0.1"; - const unsigned short rtp_port = 6000; - EXPECT_EQ(0, network_interface->SetLocalReceiver(video_channel, rtp_port)); + + // Configure External transport to simulate network interference: + TbExternalTransport external_transport(*interfaces.network); + external_transport.SetPacketLoss(packet_loss_percent); + external_transport.SetNetworkDelay(network_delay_ms); + + FrameSentCallback frame_sent_callback(frame_drop_detector); + FrameReceivedCallback frame_received_callback(frame_drop_detector); + external_transport.RegisterSendFrameCallback(&frame_sent_callback); + external_transport.RegisterReceiveFrameCallback(&frame_received_callback); + EXPECT_EQ(0, network_interface->RegisterSendTransport(video_channel, + external_transport)); EXPECT_EQ(0, base_interface->StartReceive(video_channel)); - EXPECT_EQ(0, network_interface->SetSendDestination(video_channel, ip_address, - rtp_port)); + // Setup only the VP8 codec, which is what we'll use. webrtc::VideoCodec codec; EXPECT_TRUE(FindSpecificCodec(webrtc::kVideoCodecVP8, codec_interface, @@ -107,27 +153,37 @@ void TestFullStack(const TbInterfaces& interfaces, EXPECT_TRUE(image_process); // Setup the effect filters - DecodeTimestampEffectFilter decode_filter(frame_drop_detector); + CreatedTimestampEffectFilter create_filter(frame_drop_detector); + EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel, + create_filter)); + DecodedTimestampEffectFilter decode_filter(frame_drop_detector); EXPECT_EQ(0, image_process->RegisterRenderEffectFilter(video_channel, decode_filter)); - SendTimestampEffectFilter send_filter(frame_drop_detector); - EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel, - send_filter)); // Send video. EXPECT_EQ(0, base_interface->StartSend(video_channel)); AutoTestSleep(KAutoTestSleepTimeMs); // Cleanup. - EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel)); EXPECT_EQ(0, image_process->DeregisterSendEffectFilter(video_channel)); + EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel)); image_process->Release(); ViETest::Log("Done!"); + WebRtc_Word32 num_rtp_packets = 0; + WebRtc_Word32 num_dropped_packets = 0; + WebRtc_Word32 num_rtcp_packets = 0; + external_transport.GetStats(num_rtp_packets, num_dropped_packets, + num_rtcp_packets); + ViETest::Log("RTP packets : %5d", num_rtp_packets); + ViETest::Log("Dropped packets: %5d", num_dropped_packets); + ViETest::Log("RTCP packets : %5d", num_rtcp_packets); + // *************************************************************** // Testing finished. Tear down Video Engine // *************************************************************** EXPECT_EQ(0, base_interface->StopSend(video_channel)); EXPECT_EQ(0, base_interface->StopReceive(video_channel)); + EXPECT_EQ(0, network_interface->DeregisterSendTransport(video_channel)); EXPECT_EQ(0, render_interface->StopRender(capture_id)); EXPECT_EQ(0, render_interface->StopRender(video_channel)); EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id)); @@ -137,14 +193,8 @@ void TestFullStack(const TbInterfaces& interfaces, } void FixOutputFileForComparison(const std::string& output_file, - int total_number_of_frames, int frame_length_in_bytes, - std::list dropped_frames) { - if (dropped_frames.size() == 0) { - // No need to modify if no frames are dropped, since the file is already - // frame-per-frame in sync in that case. - return; - } + const std::vector& frames) { webrtc::test::FrameReaderImpl frame_reader(output_file, frame_length_in_bytes); const std::string temp_file = output_file + ".fixed"; @@ -152,159 +202,308 @@ void FixOutputFileForComparison(const std::string& output_file, frame_reader.Init(); frame_writer.Init(); - // Assume the dropped_frames list is sorted by frame number. - int next_dropped_frame = dropped_frames.front()->number_; - dropped_frames.pop_front(); - ASSERT_NE(0, next_dropped_frame) << "It should not be possible to drop the " - "first frame. Both because we don't have anything useful to fill that " - "gap with and it is impossible to detect it without any previous " - "timestamps to compare with."; + ASSERT_FALSE(frames.front()->dropped_at_render) << "It should not be " + "possible to drop the first frame. Both because we don't have anything " + "useful to fill that gap with and it is impossible to detect it without " + "any previous timestamps to compare with."; - WebRtc_UWord8* last_read_frame_data = - new WebRtc_UWord8[frame_length_in_bytes]; - - // Write the first frame now since it will always be the same. - EXPECT_TRUE(frame_reader.ReadFrame(last_read_frame_data)); - EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data)); + WebRtc_UWord8* last_frame_data = new WebRtc_UWord8[frame_length_in_bytes]; // Process the file and write frame duplicates for all dropped frames. - for (int i = 1; i < total_number_of_frames; ++i) { - if (i == next_dropped_frame) { + for (std::vector::const_iterator it = frames.begin(); + it != frames.end(); ++it) { + if ((*it)->dropped_at_render) { // Write the previous frame to the output file: - EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data)); - if (!dropped_frames.empty()) { - next_dropped_frame = dropped_frames.front()->number_; - dropped_frames.pop_front(); - } + EXPECT_TRUE(frame_writer.WriteFrame(last_frame_data)); } else { - // Read a new frame and write it to the output file. - EXPECT_TRUE(frame_reader.ReadFrame(last_read_frame_data)); - EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data)); + EXPECT_TRUE(frame_reader.ReadFrame(last_frame_data)); + EXPECT_TRUE(frame_writer.WriteFrame(last_frame_data)); } } - delete[] last_read_frame_data; + delete[] last_frame_data; frame_reader.Close(); frame_writer.Close(); ASSERT_EQ(0, std::remove(output_file.c_str())); ASSERT_EQ(0, std::rename(temp_file.c_str(), output_file.c_str())); } -void FrameDropDetector::ReportSent(unsigned int timestamp) { - int number = sent_frames_list_.size(); - Frame* frame = new Frame(number, timestamp); - frame->sent_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp(); - sent_frames_list_.push_back(frame); - sent_frames_[timestamp] = frame; -} - -void FrameDropDetector::ReportDecoded(unsigned int timestamp) { - // When the first sent frame arrives we calculate the fixed difference - // between the timestamps of the sent frames and the decoded/rendered frames. - // This diff is then used to identify the frames from the sent_frames_ map. - if (frame_timestamp_diff_ == 0) { - Frame* first_sent_frame = sent_frames_list_.front(); - frame_timestamp_diff_ = timestamp - first_sent_frame->frame_timestamp_; +void FrameDropDetector::ReportFrameState(State state, unsigned int timestamp) { + dirty_ = true; + switch (state) { + case kCreated: { + int number = created_frames_vector_.size(); + Frame* frame = new Frame(number, timestamp); + frame->created_timestamp_in_us_ = + webrtc::TickTime::MicrosecondTimestamp(); + created_frames_vector_.push_back(frame); + created_frames_[timestamp] = frame; + num_created_frames_++; + break; + } + case kSent: + sent_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp(); + if (timestamp_diff_ == 0) { + // When the first created frame arrives we calculate the fixed + // difference between the timestamps of the frames entering and leaving + // the encoder. This diff is used to identify the frames from the + // created_frames_ map. + timestamp_diff_ = + timestamp - created_frames_vector_.front()->frame_timestamp_; + } + num_sent_frames_++; + break; + case kReceived: + received_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp(); + num_received_frames_++; + break; + case kDecoded: + decoded_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp(); + num_decoded_frames_++; + break; + case kRendered: + rendered_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp(); + num_rendered_frames_++; + break; + default: + assert(false); } - // Calculate the sent timestamp required to identify the frame: - unsigned int sent_timestamp = timestamp - frame_timestamp_diff_; - - // Find the right Frame object in the map of sent frames: - Frame* frame = sent_frames_[sent_timestamp]; - frame->decoded_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp(); - decoded_frames_[sent_timestamp] = frame; } -void FrameDropDetector::ReportRendered(unsigned int timestamp) { - // Calculate the sent timestamp required to identify the frame: - unsigned int sent_timestamp = timestamp - frame_timestamp_diff_; - - // Find this frame in the map of sent frames: - Frame* frame = sent_frames_[sent_timestamp]; - frame->rendered_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp(); - rendered_frames_[sent_timestamp] = frame; -} - -int FrameDropDetector::NumberSentFrames() { - return static_cast(sent_frames_.size()); +void FrameDropDetector::CalculateResults() { + // Fill in all fields of the Frame objects in the created_frames_ map. + // Iterate over the maps from converted timestamps to the arrival timestamps. + std::map::const_iterator it; + for (it = sent_frames_.begin(); it != sent_frames_.end(); ++it) { + int created_timestamp = it->first - timestamp_diff_; + created_frames_[created_timestamp]->sent_timestamp_in_us_ = it->second; + } + for (it = received_frames_.begin(); it != received_frames_.end(); ++it) { + int created_timestamp = it->first - timestamp_diff_; + created_frames_[created_timestamp]->received_timestamp_in_us_ = it->second; + } + for (it = decoded_frames_.begin(); it != decoded_frames_.end(); ++it) { + int created_timestamp = it->first - timestamp_diff_; + created_frames_[created_timestamp]->decoded_timestamp_in_us_ =it->second; + } + for (it = rendered_frames_.begin(); it != rendered_frames_.end(); ++it) { + int created_timestamp = it->first - timestamp_diff_; + created_frames_[created_timestamp]->rendered_timestamp_in_us_ = it->second; + } + // Find out where the frames were not present in the different states. + dropped_frames_at_send_ = 0; + dropped_frames_at_receive_ = 0; + dropped_frames_at_decode_ = 0; + dropped_frames_at_render_ = 0; + for (std::vector::const_iterator it = created_frames_vector_.begin(); + it != created_frames_vector_.end(); ++it) { + int encoded_timestamp = (*it)->frame_timestamp_ + timestamp_diff_; + if (sent_frames_.find(encoded_timestamp) == sent_frames_.end()) { + (*it)->dropped_at_send = true; + dropped_frames_at_send_++; + } + if (received_frames_.find(encoded_timestamp) == received_frames_.end()) { + (*it)->dropped_at_receive = true; + dropped_frames_at_receive_++; + } + if (decoded_frames_.find(encoded_timestamp) == decoded_frames_.end()) { + (*it)->dropped_at_decode = true; + dropped_frames_at_decode_++; + } + if (rendered_frames_.find(encoded_timestamp) == rendered_frames_.end()) { + (*it)->dropped_at_render = true; + dropped_frames_at_render_++; + } + } + dirty_ = false; } void FrameDropDetector::PrintReport() { + assert(!dirty_); ViETest::Log("Frame Drop Detector report:"); - ViETest::Log("Sent frames: %ld", sent_frames_.size()); - ViETest::Log("Decoded frames: %ld", decoded_frames_.size()); - ViETest::Log("Rendered frames: %ld", rendered_frames_.size()); + ViETest::Log(" Created frames: %ld", created_frames_.size()); + ViETest::Log(" Sent frames: %ld", sent_frames_.size()); + ViETest::Log(" Received frames: %ld", received_frames_.size()); + ViETest::Log(" Decoded frames: %ld", decoded_frames_.size()); + ViETest::Log(" Rendered frames: %ld", rendered_frames_.size()); // Display all frames and stats for them: + long last_created = 0; long last_sent = 0; + long last_received = 0; long last_decoded = 0; long last_rendered = 0; - ViETest::Log("Sent frames summary:"); - ViETest::Log("Deltas are in microseconds and only cover existing frames."); - ViETest::Log("Frame no SentDelta DecodedDelta RenderedDelta DecodedDrop? " - "RenderedDrop?"); - for (std::list::iterator it = sent_frames_list_.begin(); - it != sent_frames_list_.end(); it++) { - bool dropped_decode = (decoded_frames_.find((*it)->frame_timestamp_) == - decoded_frames_.end()); - bool dropped_render = (rendered_frames_.find((*it)->frame_timestamp_) == - rendered_frames_.end()); - int sent_delta = static_cast((*it)->sent_timestamp_in_us_ - last_sent); - int decoded_delta = dropped_decode ? 0 : + ViETest::Log("\nDeltas between sent frames and drop status:"); + ViETest::Log("Unit: Microseconds"); + ViETest::Log("Frame Created Sent Received Decoded Rendered " + "Dropped at Dropped at Dropped at Dropped at"); + ViETest::Log(" nbr delta delta delta delta delta " + " Send? Receive? Decode? Render?"); + for (std::vector::const_iterator it = created_frames_vector_.begin(); + it != created_frames_vector_.end(); ++it) { + int created_delta = + static_cast((*it)->created_timestamp_in_us_ - last_created); + int sent_delta = (*it)->dropped_at_send ? -1 : + static_cast((*it)->sent_timestamp_in_us_ - last_sent); + int received_delta = (*it)->dropped_at_receive ? -1 : + static_cast((*it)->received_timestamp_in_us_ - last_received); + int decoded_delta = (*it)->dropped_at_decode ? -1 : static_cast((*it)->decoded_timestamp_in_us_ - last_decoded); - int rendered_delta = dropped_render ? 0 : + int rendered_delta = (*it)->dropped_at_render ? -1 : static_cast((*it)->rendered_timestamp_in_us_ - last_rendered); - // Set values to 0 for the first frame: + // Set values to -1 for the first frame: if ((*it)->number_ == 0) { - sent_delta = 0; - decoded_delta = 0; - rendered_delta = 0; + created_delta = -1; + sent_delta = -1; + received_delta = -1; + decoded_delta = -1; + rendered_delta = -1; } - ViETest::Log("%8d %10d %10d %10d %s %s", (*it)->number_, - sent_delta, decoded_delta, rendered_delta, - dropped_decode ? "DROPPED" : " ", - dropped_render ? "DROPPED" : " "); - last_sent = (*it)->sent_timestamp_in_us_; - if (!dropped_render) { + ViETest::Log("%5d %8d %8d %8d %8d %8d %10s %10s %10s %10s", + (*it)->number_, + created_delta, + sent_delta, + received_delta, + decoded_delta, + rendered_delta, + (*it)->dropped_at_send ? "DROPPED" : " ", + (*it)->dropped_at_receive ? "DROPPED" : " ", + (*it)->dropped_at_decode ? "DROPPED" : " ", + (*it)->dropped_at_render ? "DROPPED" : " "); + last_created = (*it)->created_timestamp_in_us_; + if (!(*it)->dropped_at_send) { + last_sent = (*it)->sent_timestamp_in_us_; + } + if (!(*it)->dropped_at_receive) { + last_received = (*it)->received_timestamp_in_us_; + } + if (!(*it)->dropped_at_decode) { last_decoded = (*it)->decoded_timestamp_in_us_; + } + if (!(*it)->dropped_at_render) { last_rendered = (*it)->rendered_timestamp_in_us_; } } - // Find and print the dropped frames. Work at a copy of the sent_frames_ map. - std::list decode_dropped_frames = GetFramesDroppedAtDecodeStep(); - ViETest::Log("Number of dropped frames at the decode step: %d", - static_cast(decode_dropped_frames.size())); - std::list render_dropped_frames = GetFramesDroppedAtRenderStep(); - ViETest::Log("Number of dropped frames at the render step: %d", - static_cast(render_dropped_frames.size())); + ViETest::Log("\nLatency between states (-1 means N/A because of drop):"); + ViETest::Log("Unit: Microseconds"); + ViETest::Log("Frame Created Sent Received Decoded Total " + " Total"); + ViETest::Log(" nbr ->Sent ->Received ->Decoded ->Rendered latency " + " latency"); + ViETest::Log(" (incl network)" + "(excl network)"); + for (std::vector::const_iterator it = created_frames_vector_.begin(); + it != created_frames_vector_.end(); ++it) { + int created_to_sent = (*it)->dropped_at_send ? -1 : + static_cast((*it)->sent_timestamp_in_us_ - + (*it)->created_timestamp_in_us_); + int sent_to_received = (*it)->dropped_at_receive ? -1 : + static_cast((*it)->received_timestamp_in_us_ - + (*it)->sent_timestamp_in_us_); + int received_to_decoded = (*it)->dropped_at_decode ? -1 : + static_cast((*it)->decoded_timestamp_in_us_ - + (*it)->received_timestamp_in_us_); + int decoded_to_render = (*it)->dropped_at_render ? -1 : + static_cast((*it)->rendered_timestamp_in_us_ - + (*it)->decoded_timestamp_in_us_); + int total_latency_incl_network = (*it)->dropped_at_render ? -1 : + static_cast((*it)->rendered_timestamp_in_us_ - + (*it)->created_timestamp_in_us_); + int total_latency_excl_network = (*it)->dropped_at_render ? -1 : + static_cast((*it)->rendered_timestamp_in_us_ - + (*it)->created_timestamp_in_us_ - sent_to_received); + ViETest::Log("%5d %9d %9d %9d %9d %12d %12d", + (*it)->number_, + created_to_sent, + sent_to_received, + received_to_decoded, + decoded_to_render, + total_latency_incl_network, + total_latency_excl_network); + } + // Find and print the dropped frames. + ViETest::Log("\nTotal # dropped frames at:"); + ViETest::Log(" Send : %d", dropped_frames_at_send_); + ViETest::Log(" Receive: %d", dropped_frames_at_receive_); + ViETest::Log(" Decode : %d", dropped_frames_at_decode_); + ViETest::Log(" Render : %d", dropped_frames_at_render_); } -const std::list FrameDropDetector::GetFramesDroppedAtDecodeStep() { - std::list dropped_frames; - std::map::iterator it; - for (it = sent_frames_.begin(); it != sent_frames_.end(); it++) { - if (decoded_frames_.find(it->first) == decoded_frames_.end()) { - dropped_frames.push_back(it->second); +void FrameDropDetector::PrintDebugDump() { + assert(!dirty_); + ViETest::Log("\nPrintDebugDump: Frame objects:"); + ViETest::Log("Frame FrTimeStamp Created Sent Received Decoded" + " Rendered "); + for (std::vector::const_iterator it = created_frames_vector_.begin(); + it != created_frames_vector_.end(); ++it) { + ViETest::Log("%5d %11d %11d %11d %11d %11d %11d", + (*it)->number_, + (*it)->frame_timestamp_, + (*it)->created_timestamp_in_us_, + (*it)->sent_timestamp_in_us_, + (*it)->received_timestamp_in_us_, + (*it)->decoded_timestamp_in_us_, + (*it)->rendered_timestamp_in_us_); + } + std::vector mismatch_frame_num_list; + for (std::vector::const_iterator it = created_frames_vector_.begin(); + it != created_frames_vector_.end(); ++it) { + if ((*it)->dropped_at_render != (*it)->dropped_at_decode) { + mismatch_frame_num_list.push_back((*it)->number_); } } - return dropped_frames; + if (mismatch_frame_num_list.size() > 0) { + ViETest::Log("\nDecoded/Rendered mismatches:"); + ViETest::Log("Frame FrTimeStamp Created Sent Received " + "Decoded Rendered "); + for (std::vector::const_iterator it = mismatch_frame_num_list.begin(); + it != mismatch_frame_num_list.end(); ++it) { + Frame* frame = created_frames_vector_[*it]; + ViETest::Log("%5d %11d %11d %11d %11d %11d %11d", + frame->number_, + frame->frame_timestamp_, + frame->created_timestamp_in_us_, + frame->sent_timestamp_in_us_, + frame->received_timestamp_in_us_, + frame->decoded_timestamp_in_us_, + frame->rendered_timestamp_in_us_); + } + } + + ViETest::Log("\nReportFrameState method invocations:"); + ViETest::Log(" Created : %d", num_created_frames_); + ViETest::Log(" Send : %d", num_sent_frames_); + ViETest::Log(" Received: %d", num_received_frames_); + ViETest::Log(" Decoded : %d", num_decoded_frames_); + ViETest::Log(" Rendered: %d", num_rendered_frames_); } -const std::list FrameDropDetector::GetFramesDroppedAtRenderStep() { - std::list dropped_frames; - std::map::iterator it; - for (it = sent_frames_.begin(); it != sent_frames_.end(); it++) { - if (rendered_frames_.find(it->first) == rendered_frames_.end()) { - dropped_frames.push_back(it->second); - } +const std::vector& FrameDropDetector::GetAllFrames() { + assert(!dirty_); + return created_frames_vector_; +} + +int FrameDropDetector::GetNumberOfFramesDroppedAt(State state) { + assert(!dirty_); + switch (state) { + case kSent: + return dropped_frames_at_send_; + case kReceived: + return dropped_frames_at_receive_; + case kDecoded: + return dropped_frames_at_decode_; + case kRendered: + return dropped_frames_at_render_; + default: + return 0; } - return dropped_frames; } int FrameDropMonitoringRemoteFileRenderer::DeliverFrame( unsigned char *buffer, int buffer_size, unsigned int time_stamp) { // Register that this frame has been rendered: - frame_drop_detector_->ReportRendered(time_stamp); + frame_drop_detector_->ReportFrameState(FrameDropDetector::kRendered, + time_stamp); return ViEToFileRenderer::DeliverFrame(buffer, buffer_size, time_stamp); } diff --git a/src/video_engine/test/auto_test/primitives/framedrop_primitives.h b/src/video_engine/test/auto_test/primitives/framedrop_primitives.h index 44012774b..e5091f14f 100644 --- a/src/video_engine/test/auto_test/primitives/framedrop_primitives.h +++ b/src/video_engine/test/auto_test/primitives/framedrop_primitives.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -11,8 +11,8 @@ #ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_ #define WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_ -#include #include +#include #include "video_engine/include/vie_codec.h" #include "video_engine/include/vie_image_process.h" @@ -24,25 +24,36 @@ class TbInterfaces; // Initializes the Video engine and its components, runs video playback using // for KAutoTestSleepTimeMs milliseconds, then shuts down everything. -// The bit rate should be low enough to make the video encoder being forced to -// drop some frames, in order to test the frame drop detection that is performed -// by the FrameDropDetector class. +// The bit rate and packet loss parameters should be configured so that +// frames are dropped, in order to test the frame drop detection that is +// performed by the FrameDropDetector class. void TestFullStack(const TbInterfaces& interfaces, int capture_id, int video_channel, int width, int height, int bit_rate_kbps, + int packet_loss_percent, + int network_delay_ms, FrameDropDetector* frame_drop_detector); -// A frame in a video file. The three different points in the stack when -// register the frame state are (in time order): sent, decoded, rendered. +// A frame in a video file. The four different points in the stack when +// register the frame state are (in time order): created, transmitted, decoded, +// rendered. class Frame { public: Frame(int number, unsigned int timestamp) - : number_(number), frame_timestamp_(timestamp), - sent_timestamp_in_us_(0), decoded_timestamp_in_us_(0), - rendered_timestamp_in_us_(0) {} + : number_(number), + frame_timestamp_(timestamp), + created_timestamp_in_us_(-1), + sent_timestamp_in_us_(-1), + received_timestamp_in_us_(-1), + decoded_timestamp_in_us_(-1), + rendered_timestamp_in_us_(-1), + dropped_at_send(false), + dropped_at_receive(false), + dropped_at_decode(false), + dropped_at_render(false) {} // Frame number, starting at 0. int number_; @@ -52,9 +63,17 @@ class Frame { unsigned int frame_timestamp_; // Timestamps for our measurements of when the frame is in different states. + int64_t created_timestamp_in_us_; int64_t sent_timestamp_in_us_; + int64_t received_timestamp_in_us_; int64_t decoded_timestamp_in_us_; int64_t rendered_timestamp_in_us_; + + // Where the frame was dropped (more than one may be true). + bool dropped_at_send; + bool dropped_at_receive; + bool dropped_at_decode; + bool dropped_at_render; }; // Fixes the output file by copying the last successful frame into the place @@ -63,70 +82,129 @@ class Frame { // dropped, since there'll be no previous frame to copy. This case should never // happen because of encoder frame dropping at least. // Parameters: -// output_file The output file to modify (pad with frame copies -// for all dropped frames) -// total_number_of_frames Number of frames in the reference file we want -// to match. -// frame_length_in_bytes Byte length of each frame. -// dropped_frames List of Frame objects. Must be sorted by frame -// number. If empty this method will do nothing. +// output_file The output file to modify (pad with frame copies +// for all dropped frames) +// frame_length_in_bytes Byte length of each frame. +// frames A vector of all Frame objects. Must be sorted by +// frame number. If empty this method will do nothing. void FixOutputFileForComparison(const std::string& output_file, - int total_number_of_frames, int frame_length_in_bytes, - std::list dropped_frames); + const std::vector& frames); // Handles statistics about dropped frames. Frames travel through the stack -// with different timestamps. The sent frames have one timestamp on the sending -// side while the decoded/rendered frames have another timestamp on the -// receiving side. However the difference between these timestamps is fixed, -// which we can use to identify the frames when they arrive, since the -// FrameDropDetector class gets data reported from both sides. -// The three different points in the stack when this class examines the frame -// states are (in time order): sent, decoded, rendered. +// with different timestamps. The frames created and sent to the encoder have +// one timestamp on the sending side while the decoded/rendered frames have +// another timestamp on the receiving side. The difference between these +// timestamps is fixed, which we can use to identify the frames when they +// arrive, since the FrameDropDetector class gets data reported from both sides. +// The four different points in the stack when this class examines the frame +// states are (in time order): created, sent, received, decoded, rendered. +// +// The flow can be visualized like this: +// +// Created Sent Received Decoded Rendered +// +-------+ | +-------+ | +---------+ | +------+ +-------+ | +--------+ +// |Capture| | |Encoder| | | Ext. | | |Jitter| |Decoder| | | Ext. | +// | device|---->| |-->|transport|-->|buffer|->| |---->|renderer| +// +-------+ +-------+ +---------+ +------+ +-------+ +--------+ +// +// This class has no intention of being thread-safe. class FrameDropDetector { public: + enum State { + // A frame being created, i.e. sent to the encoder; the first step of + // a frame's life cycle. This timestamp becomes the frame timestamp in the + // Frame objects. + kCreated, + // A frame being sent in external transport (to the simulated network). This + // timestamp differs from the one in the Created state by a constant diff. + kSent, + // A frame being received in external transport (from the simulated + // network). This timestamp differs from the one in the Created state by a + // constant diff. + kReceived, + // A frame that has been decoded in the decoder. This timestamp differs + // from the one in the Created state by a constant diff. + kDecoded, + // A frame that has been rendered; the last step of a frame's life cycle. + // This timestamp differs from the one in the Created state by a constant + // diff. + kRendered + }; + FrameDropDetector() - : frame_timestamp_diff_(0) {} + : dirty_(true), + dropped_frames_at_send_(0), + dropped_frames_at_receive_(0), + dropped_frames_at_decode_(0), + dropped_frames_at_render_(0), + num_created_frames_(0), + num_sent_frames_(0), + num_received_frames_(0), + num_decoded_frames_(0), + num_rendered_frames_(0), + timestamp_diff_(0) {} - // Report a frame being sent; the first step of a frame transfer. - // This timestamp becomes the frame timestamp in the Frame objects. - void ReportSent(unsigned int timestamp); + // Reports a frame has reached a state in the frame life cycle. + void ReportFrameState(State state, unsigned int timestamp); - // Report a frame being rendered; happens right before it is received. - // This timestamp differs from the one in ReportSent timestamp. - void ReportDecoded(unsigned int timestamp); + // Uses all the gathered timestamp information to calculate which frames have + // been dropped during the test and where they were dropped. Not until + // this method has been executed, the Frame objects will have all fields + // filled with the proper timestamp information. + void CalculateResults(); - // Report a frame being rendered; the last step of a frame transfer. - // This timestamp differs from the one in ReportSent timestamp, but is the - // same as the ReportRendered timestamp. - void ReportRendered(unsigned int timestamp); + // Calculates the number of frames have been registered as dropped at the + // specified state of the frame life cycle. + // CalculateResults() must be called before calling this method. + int GetNumberOfFramesDroppedAt(State state); - // The number of sent frames, i.e. the number of times the ReportSent has been - // called successfully. - int NumberSentFrames(); - - // Calculates which frames have been registered as dropped at the decode step. - const std::list GetFramesDroppedAtDecodeStep(); - - // Calculates which frames have been registered as dropped at the render step. - const std::list GetFramesDroppedAtRenderStep(); + // Gets a vector of all the created frames. + // CalculateResults() must be called before calling this method to have all + // fields of the Frame objects to represent the current state. + const std::vector& GetAllFrames(); // Prints a detailed report about all the different frame states and which // ones are detected as dropped, using ViETest::Log. + // CalculateResults() must be called before calling this method. void PrintReport(); + // Prints all the timestamp maps. Mainly used for debugging purposes to find + // missing timestamps. + void PrintDebugDump(); private: - // Maps mapping frame timestamps to Frame objects. - std::map sent_frames_; - std::map decoded_frames_; - std::map rendered_frames_; + // Will be false until CalculateResults() is called. Switches to true + // as soon as new timestamps are reported using ReportFrameState(). + bool dirty_; - // A list with the frames sorted in their sent order: - std::list sent_frames_list_; + // Map of frame creation timestamps to all Frame objects. + std::map created_frames_; - // The constant diff between the sent and rendered frames, since their + // Maps converted frame timestamps (differ from creation timestamp) to the + // time they arrived in the different states of the frame's life cycle. + std::map sent_frames_; + std::map received_frames_; + std::map decoded_frames_; + std::map rendered_frames_; + + // A vector with the frames sorted in their created order. + std::vector created_frames_vector_; + + // Statistics. + int dropped_frames_at_send_; + int dropped_frames_at_receive_; + int dropped_frames_at_decode_; + int dropped_frames_at_render_; + + int num_created_frames_; + int num_sent_frames_; + int num_received_frames_; + int num_decoded_frames_; + int num_rendered_frames_; + + // The constant diff between the created and transmitted frames, since their // timestamps are converted. - unsigned int frame_timestamp_diff_; + unsigned int timestamp_diff_; }; // Tracks which frames are received on the remote side and reports back to the diff --git a/src/video_engine/test/auto_test/primitives/framedrop_primitives_unittest.cc b/src/video_engine/test/auto_test/primitives/framedrop_primitives_unittest.cc new file mode 100644 index 000000000..f4cc390c8 --- /dev/null +++ b/src/video_engine/test/auto_test/primitives/framedrop_primitives_unittest.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "framedrop_primitives.h" + +#include +#include + +#include "gtest/gtest.h" +#include "testsupport/fileutils.h" +#include "testsupport/frame_reader.h" +#include "testsupport/frame_writer.h" + +namespace webrtc { + +const std::string kOutputFilename = "temp_outputfile.tmp"; +const int kFrameLength = 1000; + +class FrameDropPrimitivesTest: public testing::Test { + protected: + FrameDropPrimitivesTest() {} + virtual ~FrameDropPrimitivesTest() {} + void SetUp() { + // Cleanup any previous output file. + std::remove(kOutputFilename.c_str()); + } + void TearDown() { + // Cleanup the temporary file. + std::remove(kOutputFilename.c_str()); + } +}; + +TEST_F(FrameDropPrimitivesTest, FixOutputFileForComparison) { + // Create test frame objects, where the second and fourth frame is marked + // as dropped at rendering. + std::vector frames; + Frame first_frame(0, kFrameLength); + Frame second_frame(0, kFrameLength); + Frame third_frame(0, kFrameLength); + Frame fourth_frame(0, kFrameLength); + + second_frame.dropped_at_render = true; + fourth_frame.dropped_at_render = true; + + frames.push_back(&first_frame); + frames.push_back(&second_frame); + frames.push_back(&third_frame); + frames.push_back(&fourth_frame); + + // Prepare data for the first and third frames: + WebRtc_UWord8 first_frame_data[kFrameLength]; + memset(first_frame_data, 5, kFrameLength); // Fill it with 5's to identify. + WebRtc_UWord8 third_frame_data[kFrameLength]; + memset(third_frame_data, 7, kFrameLength); // Fill it with 7's to identify. + + // Write the first and third frames to the temporary file. This means the fix + // method should add two frames of data by filling the file with data from + // the first and third frames after executing. + webrtc::test::FrameWriterImpl frame_writer(kOutputFilename, kFrameLength); + EXPECT_TRUE(frame_writer.Init()); + EXPECT_TRUE(frame_writer.WriteFrame(first_frame_data)); + EXPECT_TRUE(frame_writer.WriteFrame(third_frame_data)); + frame_writer.Close(); + EXPECT_EQ(2 * kFrameLength, + static_cast(webrtc::test::GetFileSize(kOutputFilename))); + + FixOutputFileForComparison(kOutputFilename, kFrameLength, frames); + + // Verify that the output file has correct size. + EXPECT_EQ(4 * kFrameLength, + static_cast(webrtc::test::GetFileSize(kOutputFilename))); + + webrtc::test::FrameReaderImpl frame_reader(kOutputFilename, kFrameLength); + frame_reader.Init(); + WebRtc_UWord8 read_buffer[kFrameLength]; + EXPECT_TRUE(frame_reader.ReadFrame(read_buffer)); + EXPECT_EQ(0, memcmp(read_buffer, first_frame_data, kFrameLength)); + EXPECT_TRUE(frame_reader.ReadFrame(read_buffer)); + EXPECT_EQ(0, memcmp(read_buffer, first_frame_data, kFrameLength)); + + EXPECT_TRUE(frame_reader.ReadFrame(read_buffer)); + EXPECT_EQ(0, memcmp(read_buffer, third_frame_data, kFrameLength)); + EXPECT_TRUE(frame_reader.ReadFrame(read_buffer)); + EXPECT_EQ(0, memcmp(read_buffer, third_frame_data, kFrameLength)); + + frame_reader.Close(); +} + +} // namespace webrtc diff --git a/src/video_engine/test/auto_test/source/tb_external_transport.cc b/src/video_engine/test/auto_test/source/tb_external_transport.cc index 1e5375ba7..8670c40fe 100644 --- a/src/video_engine/test/auto_test/source/tb_external_transport.cc +++ b/src/video_engine/test/auto_test/source/tb_external_transport.cc @@ -8,10 +8,6 @@ * be found in the AUTHORS file in the root of the source tree. */ -// -// tb_external_transport.cc -// - #include "tb_external_transport.h" #include // printf @@ -48,6 +44,10 @@ TbExternalTransport::TbExternalTransport(webrtc::ViENetwork& vieNetwork) : _rtpCount(0), _rtcpCount(0), _dropCount(0), + _rtpPackets(), + _rtcpPackets(), + _send_frame_callback(NULL), + _receive_frame_callback(NULL), _temporalLayers(0), _seqNum(0), _sendPID(0), @@ -60,7 +60,10 @@ TbExternalTransport::TbExternalTransport(webrtc::ViENetwork& vieNetwork) : _filterSSRC(false), _SSRC(0), _checkSequenceNumber(0), - _firstSequenceNumber(0) + _firstSequenceNumber(0), + _firstRTPTimestamp(0), + _lastSendRTPTimestamp(0), + _lastReceiveRTPTimestamp(0) { srand((int) webrtc::TickTime::MicrosecondTimestamp()); unsigned int tId = 0; @@ -82,6 +85,23 @@ TbExternalTransport::~TbExternalTransport() int TbExternalTransport::SendPacket(int channel, const void *data, int len) { + // Parse timestamp from RTP header according to RFC 3550, section 5.1. + WebRtc_UWord8* ptr = (WebRtc_UWord8*)data; + WebRtc_UWord32 rtp_timestamp = ptr[4] << 24; + rtp_timestamp += ptr[5] << 16; + rtp_timestamp += ptr[6] << 8; + rtp_timestamp += ptr[7]; + _crit.Enter(); + if (_firstRTPTimestamp == 0) { + _firstRTPTimestamp = rtp_timestamp; + } + _crit.Leave(); + if (_send_frame_callback != NULL && + _lastSendRTPTimestamp != rtp_timestamp) { + _send_frame_callback->FrameSent(rtp_timestamp); + } + _lastSendRTPTimestamp = rtp_timestamp; + if (_filterSSRC) { WebRtc_UWord8* ptr = (WebRtc_UWord8*)data; @@ -159,9 +179,10 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len) _rtpCount++; _statCrit.Leave(); - // Packet loss + // Packet loss. Never drop packets from the first RTP timestamp, i.e. the + // first frame being transmitted. int dropThis = rand() % 100; - if (dropThis < _lossRate) + if (dropThis < _lossRate && _firstRTPTimestamp != rtp_timestamp) { _statCrit.Enter(); _dropCount++; @@ -200,6 +221,16 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len) return len; } +void TbExternalTransport::RegisterSendFrameCallback( + SendFrameCallback* callback) { + _send_frame_callback = callback; +} + +void TbExternalTransport::RegisterReceiveFrameCallback( + ReceiveFrameCallback* callback) { + _receive_frame_callback = callback; +} + // Set to 0 to disable. void TbExternalTransport::SetTemporalToggle(unsigned char layers) { @@ -348,6 +379,18 @@ bool TbExternalTransport::ViEExternalTransportProcess() _checkSequenceNumber = false; } } + // Signal received packet of frame + WebRtc_UWord8* ptr = (WebRtc_UWord8*)packet->packetBuffer; + WebRtc_UWord32 rtp_timestamp = ptr[4] << 24; + rtp_timestamp += ptr[5] << 16; + rtp_timestamp += ptr[6] << 8; + rtp_timestamp += ptr[7]; + if (_receive_frame_callback != NULL && + _lastReceiveRTPTimestamp != rtp_timestamp) { + _receive_frame_callback->FrameReceived(rtp_timestamp); + } + _lastReceiveRTPTimestamp = rtp_timestamp; + _vieNetwork.ReceivedRTPPacket(packet->channel, packet->packetBuffer, packet->length); delete packet; diff --git a/src/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc b/src/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc index f72c9685c..aff5118b2 100644 --- a/src/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc +++ b/src/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,16 +8,16 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "vie_file_based_comparison_tests.h" +#include "video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h" -#include "base_primitives.h" -#include "codec_primitives.h" -#include "framedrop_primitives.h" -#include "general_primitives.h" -#include "tb_interfaces.h" -#include "vie_autotest_defines.h" -#include "vie_fake_camera.h" -#include "vie_to_file_renderer.h" +#include "video_engine/test/auto_test/interface/tb_interfaces.h" +#include "video_engine/test/auto_test/interface/vie_autotest_defines.h" +#include "video_engine/test/auto_test/helpers/vie_fake_camera.h" +#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h" +#include "video_engine/test/auto_test/primitives/base_primitives.h" +#include "video_engine/test/auto_test/primitives/codec_primitives.h" +#include "video_engine/test/auto_test/primitives/framedrop_primitives.h" +#include "video_engine/test/auto_test/primitives/general_primitives.h" bool ViEFileBasedComparisonTests::TestCallSetup( const std::string& i420_video_file, @@ -123,11 +123,14 @@ void ViEFileBasedComparisonTests::TestFullStack( int width, int height, int bit_rate_kbps, + int packet_loss_percent, + int network_delay_ms, ViEToFileRenderer* local_file_renderer, ViEToFileRenderer* remote_file_renderer, FrameDropDetector* frame_drop_detector) { TbInterfaces interfaces = TbInterfaces("TestFullStack"); + // Setup camera capturing from file. ViEFakeCamera fake_camera(interfaces.capture); if (!fake_camera.StartCameraInNewThread(i420_video_file, width, height)) { // No point in continuing if we have no proper video source @@ -138,6 +141,12 @@ void ViEFileBasedComparisonTests::TestFullStack( int video_channel = -1; int capture_id = fake_camera.capture_id(); EXPECT_EQ(0, interfaces.base->CreateChannel(video_channel)); + + // Must set SSRC to avoid SSRC collision detection since we're sending and + // receiving from the same machine (that would cause frames being discarded + // and decoder reset). + EXPECT_EQ(0, interfaces.rtp_rtcp->SetLocalSSRC(video_channel, 12345)); + EXPECT_EQ(0, interfaces.capture->ConnectCaptureDevice( capture_id, video_channel)); ConfigureRtpRtcp(interfaces.rtp_rtcp, video_channel); @@ -145,6 +154,7 @@ void ViEFileBasedComparisonTests::TestFullStack( RenderToFile(interfaces.render, video_channel, remote_file_renderer); ::TestFullStack(interfaces, capture_id, video_channel, width, height, - bit_rate_kbps, frame_drop_detector); - fake_camera.StopCamera(); + bit_rate_kbps, packet_loss_percent, network_delay_ms, + frame_drop_detector); + EXPECT_TRUE(fake_camera.StopCamera()); } diff --git a/src/video_engine/test/auto_test/vie_auto_test.gypi b/src/video_engine/test/auto_test/vie_auto_test.gypi index 28d8f5349..b21e4784b 100644 --- a/src/video_engine/test/auto_test/vie_auto_test.gypi +++ b/src/video_engine/test/auto_test/vie_auto_test.gypi @@ -1,4 +1,4 @@ -# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license # that can be found in the LICENSE file in the root of the source @@ -12,19 +12,17 @@ 'target_name': 'vie_auto_test', 'type': 'executable', 'dependencies': [ + 'vie_auto_test_lib', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', '<(webrtc_root)/modules/modules.gyp:video_render_module', '<(webrtc_root)/modules/modules.gyp:video_capture_module', '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core', '<(webrtc_root)/../testing/gtest.gyp:gtest', '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', - '<(webrtc_root)/../test/metrics.gyp:metrics', - '<(webrtc_root)/../test/test.gyp:test_support', - 'video_engine_core', ], 'include_dirs': [ - 'interface/', - 'helpers/', + 'interface', + 'helpers', 'primitives', '../../include', '../..', @@ -32,116 +30,24 @@ '../../../common_video/interface', ], 'sources': [ - 'interface/tb_capture_device.h', - 'interface/tb_external_transport.h', - 'interface/tb_I420_codec.h', - 'interface/tb_interfaces.h', - 'interface/tb_video_channel.h', - 'interface/vie_autotest.h', - 'interface/vie_autotest_defines.h', - 'interface/vie_autotest_linux.h', - 'interface/vie_autotest_mac_carbon.h', - 'interface/vie_autotest_mac_cocoa.h', - 'interface/vie_autotest_main.h', - 'interface/vie_autotest_window_manager_interface.h', - 'interface/vie_autotest_windows.h', - 'interface/vie_file_based_comparison_tests.h', - 'interface/vie_window_manager_factory.h', - - # Helper classes - 'helpers/vie_fake_camera.cc', - 'helpers/vie_fake_camera.h', - 'helpers/vie_file_capture_device.cc', - 'helpers/vie_file_capture_device.h', - 'helpers/vie_to_file_renderer.cc', - 'helpers/vie_to_file_renderer.h', - 'helpers/vie_window_creator.cc', - 'helpers/vie_window_creator.h', - - # New, fully automated tests + 'source/vie_autotest_main.cc', + # Automated tests 'automated/vie_api_integration_test.cc', 'automated/vie_extended_integration_test.cc', 'automated/vie_integration_test_base.cc', 'automated/vie_integration_test_base.h', 'automated/vie_standard_integration_test.cc', 'automated/vie_video_verification_test.cc', - - # Test primitives - 'primitives/base_primitives.cc', - 'primitives/base_primitives.h', - 'primitives/codec_primitives.cc', - 'primitives/codec_primitives.h', - 'primitives/framedrop_primitives.h', - 'primitives/framedrop_primitives.cc', - 'primitives/general_primitives.cc', - 'primitives/general_primitives.h', - - # Platform independent - 'source/tb_capture_device.cc', - 'source/tb_external_transport.cc', - 'source/tb_I420_codec.cc', - 'source/tb_interfaces.cc', - 'source/tb_video_channel.cc', - 'source/vie_autotest.cc', - 'source/vie_autotest_base.cc', - 'source/vie_autotest_capture.cc', - 'source/vie_autotest_codec.cc', - 'source/vie_autotest_encryption.cc', - 'source/vie_autotest_file.cc', - 'source/vie_autotest_image_process.cc', - 'source/vie_autotest_loopback.cc', - 'source/vie_autotest_main.cc', - 'source/vie_autotest_network.cc', - 'source/vie_autotest_render.cc', - 'source/vie_autotest_rtp_rtcp.cc', - 'source/vie_autotest_custom_call.cc', - 'source/vie_autotest_simulcast.cc', - 'source/vie_file_based_comparison_tests.cc', - # Platform dependent - # Linux 'source/vie_autotest_linux.cc', - 'source/vie_window_manager_factory_linux.cc', - # Mac 'source/vie_autotest_mac_cocoa.mm', 'source/vie_autotest_mac_carbon.cc', - 'source/vie_window_manager_factory_mac.mm', - # Windows 'source/vie_autotest_windows.cc', + 'source/vie_window_manager_factory_linux.cc', + 'source/vie_window_manager_factory_mac.mm', 'source/vie_window_manager_factory_win.cc', ], - 'copies': [{ - 'destination': '/tmp', - 'files': [ - 'media/captureDeviceImage.bmp', - 'media/captureDeviceImage.jpg', - 'media/renderStartImage.bmp', - 'media/renderStartImage.jpg', - 'media/renderTimeoutImage.bmp', - 'media/renderTimeoutImage.jpg', - ], - }], 'conditions': [ - # TODO(andrew): rename these to be suffixed with _mac and _win. They - # will then be automatically excluded. - ['OS!="mac"', { - 'sources!': [ - 'source/vie_autotest_mac_cocoa.cc', - 'source/vie_autotest_mac_carbon.cc', - 'source/vie_window_manager_factory_mac.mm', - ], - }], - ['OS!="win"', { - 'sources!': [ - 'source/vie_autotest_windows.cc', - ], - }], - ['OS!="linux"', { - 'sources!': [ - 'source/vie_window_manager_factory_linux.cc', - ], - }], - # TODO(andrew): this likely isn't an actual dependency. It should be # included in webrtc.gyp or video_engine.gyp instead. ['OS=="win"', { @@ -167,8 +73,130 @@ ], }, }], + # TODO(andrew): rename these to be suffixed with _mac and _win. They + # will then be automatically excluded. + ['OS!="mac"', { + 'sources!': [ + 'source/vie_autotest_mac_cocoa.cc', + 'source/vie_autotest_mac_carbon.cc', + 'source/vie_window_manager_factory_mac.mm', + ], + }], + ['OS!="win"', { + 'sources!': [ + 'source/vie_autotest_windows.cc', + ], + }], + ['OS!="linux"', { + 'sources!': [ + 'source/vie_window_manager_factory_linux.cc', + ], + }], ], # conditions }, + { + 'target_name': 'vie_auto_test_lib', + 'type': '<(library)', + 'dependencies': [ + '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', + '<(webrtc_root)/modules/modules.gyp:video_render_module', + '<(webrtc_root)/modules/modules.gyp:video_capture_module', + '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core', + '<(webrtc_root)/../testing/gtest.gyp:gtest', + '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', + '<(webrtc_root)/../test/metrics.gyp:metrics', + '<(webrtc_root)/../test/test.gyp:test_support', + 'video_engine_core', + ], + 'include_dirs': [ + 'interface', + 'helpers', + 'primitives', + '../../include', + '../..', + '../../../modules/video_coding/codecs/interface', + '../../../common_video/interface', + ], + 'sources': [ + 'interface/tb_capture_device.h', + 'interface/tb_external_transport.h', + 'interface/tb_I420_codec.h', + 'interface/tb_interfaces.h', + 'interface/tb_video_channel.h', + 'interface/vie_autotest.h', + 'interface/vie_autotest_defines.h', + 'interface/vie_autotest_linux.h', + 'interface/vie_autotest_mac_carbon.h', + 'interface/vie_autotest_mac_cocoa.h', + 'interface/vie_autotest_main.h', + 'interface/vie_autotest_window_manager_interface.h', + 'interface/vie_autotest_windows.h', + 'interface/vie_file_based_comparison_tests.h', + 'interface/vie_window_manager_factory.h', + # Helper classes + 'helpers/vie_fake_camera.cc', + 'helpers/vie_fake_camera.h', + 'helpers/vie_file_capture_device.cc', + 'helpers/vie_file_capture_device.h', + 'helpers/vie_to_file_renderer.cc', + 'helpers/vie_to_file_renderer.h', + 'helpers/vie_window_creator.cc', + 'helpers/vie_window_creator.h', + # Test primitives + 'primitives/base_primitives.cc', + 'primitives/base_primitives.h', + 'primitives/codec_primitives.cc', + 'primitives/codec_primitives.h', + 'primitives/framedrop_primitives.h', + 'primitives/framedrop_primitives.cc', + 'primitives/general_primitives.cc', + 'primitives/general_primitives.h', + # Platform independent + 'source/tb_capture_device.cc', + 'source/tb_external_transport.cc', + 'source/tb_I420_codec.cc', + 'source/tb_interfaces.cc', + 'source/tb_video_channel.cc', + 'source/vie_autotest.cc', + 'source/vie_autotest_base.cc', + 'source/vie_autotest_capture.cc', + 'source/vie_autotest_codec.cc', + 'source/vie_autotest_encryption.cc', + 'source/vie_autotest_file.cc', + 'source/vie_autotest_image_process.cc', + 'source/vie_autotest_loopback.cc', + 'source/vie_autotest_main.cc', + 'source/vie_autotest_network.cc', + 'source/vie_autotest_render.cc', + 'source/vie_autotest_rtp_rtcp.cc', + 'source/vie_autotest_custom_call.cc', + 'source/vie_autotest_simulcast.cc', + 'source/vie_file_based_comparison_tests.cc', + ], + 'copies': [{ + 'destination': '/tmp', + 'files': [ + 'media/captureDeviceImage.bmp', + 'media/captureDeviceImage.jpg', + 'media/renderStartImage.bmp', + 'media/renderStartImage.jpg', + 'media/renderTimeoutImage.bmp', + 'media/renderTimeoutImage.jpg', + ], + }], + }, + { + 'target_name': 'vie_auto_test_unittests', + 'type': 'executable', + 'dependencies': [ + 'vie_auto_test_lib', + '<(webrtc_root)/../test/test.gyp:test_support_main', + '<(webrtc_root)/../testing/gtest.gyp:gtest', + ], + 'sources': [ + 'primitives/framedrop_primitives_unittest.cc', + ], + }, ], }