generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Disable full stack PSNR/SSIM triggers on Mac and Win for now due to flakiness. Adding plots of PSNR and SSIM.

Browse Source 
TBR=mflodman

BUG=1271

Review URL: https://webrtc-codereview.appspot.com/1032005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@3360 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
stefan@webrtc.org
2013-01-11 12:55:19 +00:00
parent 26901c262c
commit e7dc7f8553
1 changed files with 28 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@@ -12,6 +12,7 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "webrtc/test/testsupport/fileutils.h" #include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/test/testsupport/perf_test.h"
#include "webrtc/test/testsupport/metrics/video_metrics.h" #include "webrtc/test/testsupport/metrics/video_metrics.h"
#include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h" #include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h"
#include "webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h" #include "webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h"
@@ -146,8 +147,15 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest,
parameter_table_[i].network.mean_one_way_delay = 0; parameter_table_[i].network.mean_one_way_delay = 0;
parameter_table_[i].network.std_dev_one_way_delay = 0; parameter_table_[i].network.std_dev_one_way_delay = 0;
parameter_table_[i].bitrate = 300; parameter_table_[i].bitrate = 300;
// TODO(holmer): Enable for Win and Mac when the file rendering has been
// moved to a separate thread.
#ifdef WEBRTC_LINUX
parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_psnr_threshold = 35;
parameter_table_[i].avg_ssim_threshold = 0.96; parameter_table_[i].avg_ssim_threshold = 0.96;
#else
parameter_table_[i].avg_psnr_threshold = 0;
parameter_table_[i].avg_ssim_threshold = 0.0;
#endif
parameter_table_[i].test_label = "net_delay_0_0_plr_0"; parameter_table_[i].test_label = "net_delay_0_0_plr_0";
++i; ++i;
parameter_table_[i].protection_method = kNack; parameter_table_[i].protection_method = kNack;
@@ -157,8 +165,15 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest,
parameter_table_[i].network.mean_one_way_delay = 50; parameter_table_[i].network.mean_one_way_delay = 50;
parameter_table_[i].network.std_dev_one_way_delay = 5; parameter_table_[i].network.std_dev_one_way_delay = 5;
parameter_table_[i].bitrate = 300; parameter_table_[i].bitrate = 300;
// TODO(holmer): Enable for Win and Mac when the file rendering has been
// moved to a separate thread.
#ifdef WEBRTC_LINUX
parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_psnr_threshold = 35;
parameter_table_[i].avg_ssim_threshold = 0.96; parameter_table_[i].avg_ssim_threshold = 0.96;
#else
parameter_table_[i].avg_psnr_threshold = 0;
parameter_table_[i].avg_ssim_threshold = 0.0;
#endif
parameter_table_[i].test_label = "net_delay_50_5_plr_5"; parameter_table_[i].test_label = "net_delay_50_5_plr_5";
++i; ++i;
parameter_table_[i].protection_method = kNack; parameter_table_[i].protection_method = kNack;
@@ -168,8 +183,15 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest,
parameter_table_[i].network.mean_one_way_delay = 100; parameter_table_[i].network.mean_one_way_delay = 100;
parameter_table_[i].network.std_dev_one_way_delay = 10; parameter_table_[i].network.std_dev_one_way_delay = 10;
parameter_table_[i].bitrate = 300; parameter_table_[i].bitrate = 300;
// TODO(holmer): Enable for Win and Mac when the file rendering has been
// moved to a separate thread.
#ifdef WEBRTC_LINUX
parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_psnr_threshold = 35;
parameter_table_[i].avg_ssim_threshold = 0.96; parameter_table_[i].avg_ssim_threshold = 0.96;
#else
parameter_table_[i].avg_psnr_threshold = 0;
parameter_table_[i].avg_ssim_threshold = 0.0;
#endif
parameter_table_[i].test_label = "net_delay_100_10_plr_0"; parameter_table_[i].test_label = "net_delay_100_10_plr_0";
++i; ++i;
parameter_table_[i].protection_method = kNack; parameter_table_[i].protection_method = kNack;
@@ -305,6 +327,12 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) {
EXPECT_GE(actual_psnr, kExpectedMinimumPSNR); EXPECT_GE(actual_psnr, kExpectedMinimumPSNR);
EXPECT_GE(actual_ssim, kExpectedMinimumSSIM); EXPECT_GE(actual_ssim, kExpectedMinimumSSIM);
webrtc::test::PrintResult(
"psnr", "", parameter_table_[GetParam()].test_label,
actual_psnr, "dB", false);
webrtc::test::PrintResult(
"ssim", "", parameter_table_[GetParam()].test_label,
actual_ssim, "", false);
} }
INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest, INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest,