generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Prepared tests for running on build bot.

Browse Source 
The changes will mostly clarify why PSNR and SSIM thresholds are chosen. I think those explanations will be good since these tests may well be a bit flaky.

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/343017

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1496 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
phoglund@webrtc.org 2012-01-20 12:48:13 +00:00
parent c672d34ac7
commit d056abd62f
1 changed files with 34 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@ -26,13 +26,6 @@ namespace {
const int kInputWidth = 176;
const int kInputHeight = 144;
const double kMinPSNR420DefaultBitRateQCIF = 28;
const double kMinSSIM420DefaultBitRateQCIF = 0.95;
const double kMinPSNRCodecTestsQCIF = 20;
const double kMinSSIMCodecTestsQCIF = 0.7;
const double kMinPSNR50kbpsQCIF = 25;
const double kMinSSIM50kbpsQCIF = 0.8;
class ViEVideoVerificationTest : public testing::Test {
protected:
void SetUp() {
@ -79,19 +72,25 @@ class ViEVideoVerificationTest : public testing::Test {
void CompareFiles(const std::string& reference_file,
const std::string& test_file,
double minimum_psnr, double minimum_ssim) {
static const char* kPsnrSsimExplanation =
"Don't worry too much about this error if it only happens once. "
"It may be because mundane things like unfortunate OS scheduling. "
"If it keeps happening over and over though it's a cause of concern.";
webrtc::test::QualityMetricsResult psnr;
int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &psnr);
EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?";
EXPECT_GT(psnr.average, minimum_psnr);
EXPECT_GT(psnr.average, minimum_psnr) << kPsnrSsimExplanation;
webrtc::test::QualityMetricsResult ssim;
error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &ssim);
EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
EXPECT_GT(ssim.average, minimum_ssim); // 1 = perfect, -1 = terrible
EXPECT_GT(ssim.average, minimum_ssim) << kPsnrSsimExplanation;
ViETest::Log("Results: PSNR: %f (db) SSIM: %f",
ViETest::Log("Results: PSNR is %f (dB), SSIM is %f (1 is perfect)",
psnr.average, ssim.average);
}
@ -116,14 +115,20 @@ class ViEVideoVerificationTest : public testing::Test {
}
};
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
CompareFiles(input_file_, output_file, kMinPSNR420DefaultBitRateQCIF,
kMinSSIM420DefaultBitRateQCIF);
// The I420 test should give pretty good values since it's a lossless codec
// running on the default bitrate. It should average about 30 dB but there
// may be cases where it dips as low as 26 under adverse conditions.
const double kExpectedMinimumPSNR = 28;
const double kExpectedMinimumSSIM = 0.95;
CompareFiles(input_file_, output_file, kExpectedMinimumPSNR,
kExpectedMinimumSSIM);
}
TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) {
@ -138,18 +143,21 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) {
// The reason is that it is hard to say when the three consecutive tests
// switch over into each other, at which point we would have to restart the
// original to get a fair comparison.
CompareFiles(reference_file, output_file, kMinPSNRCodecTestsQCIF,
kMinSSIMCodecTestsQCIF);
// TODO(phoglund): The values should be higher. Investigate why the remote
// file turns out 6 seconds shorter than the local file (frame dropping?).
//
// The PSNR and SSIM values are quite low here, and they have to be since
// the codec switches will lead to lag in the output. This is considered
// acceptable, but it probably shouldn't get worse than this.
const double kExpectedMinimumPSNR = 20;
const double kExpectedMinimumSSIM = 0.7;
CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
kExpectedMinimumSSIM);
}
// Runs a whole stack processing with tracking of which frames are dropped
// in the encoder. The local and remote file will not be of equal size because
// of unknown reasons. Tests show that they start at the same frame, which is
// the important thing when doing frame-to-frame comparison with PSNR/SSIM.
TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
// Use our own FrameDropMonitoringRemoteFileRenderer instead of the
// ViEToFileRenderer from the test fixture:
// TODO(kjellander): Find a better way to reuse this code without duplication.
@ -196,8 +204,13 @@ TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
<< "The output file size is incorrect. It should be equal to the number"
"of frames multiplied by the frame size. This will likely affect "
"PSNR/SSIM calculations in a bad way.";
CompareFiles(reference_file, output_file, kMinPSNR50kbpsQCIF,
kMinSSIM50kbpsQCIF);
// We are running on a lower bitrate here so we need to settle for somewhat
// lower PSNR and SSIM values.
const double kExpectedMinimumPSNR = 25;
const double kExpectedMinimumSSIM = 0.8;
CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
kExpectedMinimumSSIM);
}
} // namespace