generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added a retry mechanism to vie_auto_test's verifying tests to make them less flaky.

Browse Source 
BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/392015

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1716 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
phoglund@webrtc.org 2012-02-17 13:08:57 +00:00
parent 4cb060127c
commit b3172860d7
1 changed files with 57 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
src/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@ -26,6 +26,7 @@ namespace {
// wouldn't like scaling, so this will work when we compare with the original.
const int kInputWidth = 176;
const int kInputHeight = 144;
const int kVerifyingTestMaxNumAttempts = 3;
class ViEVideoVerificationTest : public testing::Test {
protected:
@ -75,24 +76,19 @@ class ViEVideoVerificationTest : public testing::Test {
void CompareFiles(const std::string& reference_file,
const std::string& test_file,
double minimum_psnr, double minimum_ssim) {
static const char* kPsnrSsimExplanation =
"Don't worry too much about this error if it only happens once. "
"It may be because mundane things like unfortunate OS scheduling. "
"If it keeps happening over and over though it's a cause of concern.";
double* psnr_result, double *ssim_result) {
webrtc::test::QualityMetricsResult psnr;
int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &psnr);
EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?";
EXPECT_GT(psnr.average, minimum_psnr) << kPsnrSsimExplanation;
*psnr_result = psnr.average;
webrtc::test::QualityMetricsResult ssim;
error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &ssim);
EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
EXPECT_GT(ssim.average, minimum_ssim) << kPsnrSsimExplanation;
*ssim_result = ssim.average;
ViETest::Log("Results: PSNR is %f (dB), SSIM is %f (1 is perfect)",
psnr.average, ssim.average);
@ -120,31 +116,38 @@ class ViEVideoVerificationTest : public testing::Test {
};
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
// The I420 test should give pretty good values since it's a lossless codec
// running on the default bitrate. It should average about 30 dB but there
// may be cases where it dips as low as 26 under adverse conditions.
const double kExpectedMinimumPSNR = 28;
// may be cases where it dips as low as 26 under adverse conditions. That's
// why we have a retrying mechanism in place for this test.
const double kExpectedMinimumPSNR = 30;
const double kExpectedMinimumSSIM = 0.95;
CompareFiles(input_file_, output_file, kExpectedMinimumPSNR,
kExpectedMinimumSSIM);
for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
double actual_psnr = 0;
double actual_ssim = 0;
CompareFiles(input_file_, output_file, &actual_psnr, &actual_ssim);
if (actual_psnr >= kExpectedMinimumPSNR &&
actual_ssim >= kExpectedMinimumSSIM) {
// Test succeeded!
return;
}
}
ADD_FAILURE() << "Failed to achieve PSNR " << kExpectedMinimumPSNR <<
" and SSIM " << kExpectedMinimumSSIM << " after " <<
kVerifyingTestMaxNumAttempts << " attempts.";
}
TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
std::string reference_file = local_file_renderer_->GetFullOutputPath();
std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
// We compare the local and remote here instead of with the original.
// The reason is that it is hard to say when the three consecutive tests
// switch over into each other, at which point we would have to restart the
@ -155,8 +158,26 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) {
// acceptable, but it probably shouldn't get worse than this.
const double kExpectedMinimumPSNR = 20;
const double kExpectedMinimumSSIM = 0.7;
CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
kExpectedMinimumSSIM);
for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
std::string reference_file = local_file_renderer_->GetFullOutputPath();
std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
double actual_psnr = 0;
double actual_ssim = 0;
CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim);
if (actual_psnr >= kExpectedMinimumPSNR &&
actual_ssim >= kExpectedMinimumSSIM) {
// Test succeeded!
return;
}
}
}
// Runs a whole stack processing with tracking of which frames are dropped
@ -218,10 +239,15 @@ TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
// We are running on a lower bitrate here so we need to settle for somewhat
// lower PSNR and SSIM values.
double actual_psnr = 0;
double actual_ssim = 0;
CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim);
const double kExpectedMinimumPSNR = 24;
const double kExpectedMinimumSSIM = 0.7;
CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
kExpectedMinimumSSIM);
EXPECT_GE(actual_psnr, kExpectedMinimumPSNR);
EXPECT_GE(actual_ssim, kExpectedMinimumSSIM);
}
} // namespace