From e861359925a975198b4c414ca3317ff963d63d50 Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Tue, 4 Dec 2012 09:40:46 +0000 Subject: [PATCH] Adds two full stack performance metrics for end-to-end delay. Review URL: https://webrtc-codereview.appspot.com/937034 git-svn-id: http://webrtc.googlecode.com/svn/trunk@3235 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../automated/vie_video_verification_test.cc | 13 +++-- .../primitives/framedrop_primitives.cc | 57 ++++++++++++++++++- .../primitives/framedrop_primitives.h | 6 +- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc index fbe97aa4b..502b4ec89 100644 --- a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -116,25 +116,28 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest, protected: struct TestParameters { int packet_loss_rate; - int round_trip_time; + int one_way_delay; int bitrate; double avg_psnr_threshold; double avg_ssim_threshold; + std::string test_label; }; void SetUp() { int i = 0; parameter_table_[i].packet_loss_rate = 0; - parameter_table_[i].round_trip_time = 0; + parameter_table_[i].one_way_delay = 0; parameter_table_[i].bitrate = 300; parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_ssim_threshold = 0.96; + parameter_table_[i].test_label = "net delay 0, plr 0"; ++i; parameter_table_[i].packet_loss_rate = 5; - parameter_table_[i].round_trip_time = 50; + parameter_table_[i].one_way_delay = 50; parameter_table_[i].bitrate = 300; parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_ssim_threshold = 0.96; + parameter_table_[i].test_label = "net delay 50, plr 5"; } TestParameters parameter_table_[2]; @@ -193,7 +196,7 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) { // frames every now and then. const int kBitRateKbps = parameter_table_[GetParam()].bitrate; const int kPacketLossPercent = parameter_table_[GetParam()].packet_loss_rate; - const int kNetworkDelayMs = parameter_table_[GetParam()].round_trip_time; + const int kNetworkDelayMs = parameter_table_[GetParam()].one_way_delay; int width = 352; int height = 288; ViETest::Log("Bit rate : %5d kbps", kBitRateKbps); @@ -207,7 +210,7 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) { StopRenderers(); detector.CalculateResults(); - detector.PrintReport(); + detector.PrintReport(parameter_table_[GetParam()].test_label); if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) > detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) { diff --git a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc index 321c74c45..c64201a75 100644 --- a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc +++ b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc @@ -9,6 +9,8 @@ */ #include +#include +#include #include #include "webrtc/modules/video_capture/include/video_capture_factory.h" @@ -16,6 +18,7 @@ #include "testsupport/fileutils.h" #include "testsupport/frame_reader.h" #include "testsupport/frame_writer.h" +#include "testsupport/perf_test.h" #include "video_engine/test/auto_test/interface/vie_autotest.h" #include "video_engine/test/auto_test/interface/vie_autotest_defines.h" #include "video_engine/test/auto_test/primitives/framedrop_primitives.h" @@ -99,6 +102,41 @@ class DecodedTimestampEffectFilter : public webrtc::ViEEffectFilter { FrameDropDetector* frame_drop_detector_; }; +class Statistics { + public: + Statistics() : sum_(0.0f), sum_squared_(0.0f), count_(0) {}; + + void AddSample(float sample) { + sum_ += sample; + sum_squared_ += sample * sample; + ++count_; + } + + float Mean() { + if (count_ == 0) + return -1.0f; + return sum_ / count_; + } + + float Variance() { + if (count_ == 0) + return -1.0f; + return sum_squared_ / count_ - Mean() * Mean(); + } + + std::string AsString() { + std::stringstream ss; + ss << (Mean() >= 0 ? Mean() : -1) << ", " << + (Variance() >= 0 ? sqrt(Variance()) : -1); + return ss.str(); + } + + private: + float sum_; + float sum_squared_; + int count_; +}; + void TestFullStack(const TbInterfaces& interfaces, int capture_id, int video_channel, @@ -338,7 +376,7 @@ void FrameDropDetector::CalculateResults() { dirty_ = false; } -void FrameDropDetector::PrintReport() { +void FrameDropDetector::PrintReport(const std::string& test_label) { assert(!dirty_); ViETest::Log("Frame Drop Detector report:"); ViETest::Log(" Created frames: %ld", created_frames_.size()); @@ -413,6 +451,8 @@ void FrameDropDetector::PrintReport() { " latency"); ViETest::Log(" (incl network)" "(excl network)"); + Statistics latency_incl_network_stats; + Statistics latency_excl_network_stats; for (std::vector::const_iterator it = created_frames_vector_.begin(); it != created_frames_vector_.end(); ++it) { int created_to_sent = (*it)->dropped_at_send ? -1 : @@ -433,6 +473,12 @@ void FrameDropDetector::PrintReport() { int total_latency_excl_network = (*it)->dropped_at_render ? -1 : static_cast((*it)->rendered_timestamp_in_us_ - (*it)->created_timestamp_in_us_ - sent_to_received); + if (total_latency_incl_network >= 0) + latency_incl_network_stats.AddSample(total_latency_incl_network / + 1000.0f); + if (total_latency_excl_network >= 0) + latency_excl_network_stats.AddSample(total_latency_excl_network / + 1000.0f); ViETest::Log("%5d %9d %9d %9d %9d %12d %12d", (*it)->number_, created_to_sent, @@ -442,6 +488,15 @@ void FrameDropDetector::PrintReport() { total_latency_incl_network, total_latency_excl_network); } + + // Print dashboard data. + webrtc::test::PrintResultMeanAndError( + "total delay (excl. network)", " " + test_label, "", + latency_excl_network_stats.AsString(), "ms", false); + webrtc::test::PrintResultMeanAndError( + "total delay (incl. network)", " " + test_label, "", + latency_incl_network_stats.AsString(), "ms", false); + // Find and print the dropped frames. ViETest::Log("\nTotal # dropped frames at:"); ViETest::Log(" Send : %d", dropped_frames_at_send_); diff --git a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h index fd5aab393..a79032ab9 100644 --- a/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h +++ b/webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h @@ -167,9 +167,11 @@ class FrameDropDetector { const std::vector& GetAllFrames(); // Prints a detailed report about all the different frame states and which - // ones are detected as dropped, using ViETest::Log. + // ones are detected as dropped, using ViETest::Log. Also prints + // perf-formatted output and adds |test_label| as a modifier to the perf + // output. // CalculateResults() must be called before calling this method. - void PrintReport(); + void PrintReport(const std::string& test_label); // Prints all the timestamp maps. Mainly used for debugging purposes to find // missing timestamps.