generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Changing all PSNR/SSIM calculations to use libyuv.

Browse Source 
Removed old PSNR/SSIM implementations in:
* test/testsupport/metrics/video_metrics.cc
* src/modules/video_coding/codecs/test_framework/test.cc
The functions in video_metrics.cc is now using code in libyuv instead. Old code in test.cc is using the same functions.
The code for video_metrics.h had to be moved into a separate GYP file to avoid circular dependency error on Mac (see issue 160 for more details). The reason for this is that libyuv's unittest target depends on test_support_main.

BUG=
TEST=metrics_unittests in Debug+Release on Linux, Mac and Windows.

Review URL: http://webrtc-codereview.appspot.com/333025

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1325 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
kjellander@webrtc.org 2012-01-04 08:09:32 +00:00
parent ec0f54954d
commit cc33737a80
21 changed files with 453 additions and 893 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi
View File

@ -50,6 +50,7 @@
'webrtc_video_coding',
'webrtc_vp8',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
'<(webrtc_root)/../test/metrics.gyp:metrics',
'<(webrtc_root)/../test/test.gyp:test_support_main',
],
'sources': [

26
src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
View File

@ -117,20 +117,18 @@ class VideoProcessorIntegrationTest: public testing::Test {
frame_reader_->Close();
frame_writer_->Close();
QualityMetricsResult result;
EXPECT_EQ(0, PsnrFromFiles(config_.input_filename.c_str(),
config_.output_filename.c_str(),
config_.codec_settings->width,
config_.codec_settings->height, &result));
EXPECT_GT(result.average, minimum_psnr + 2.0);
EXPECT_GT(result.min, minimum_psnr);
EXPECT_EQ(0, SsimFromFiles(config_.input_filename.c_str(),
config_.output_filename.c_str(),
config_.codec_settings->width,
config_.codec_settings->height, &result));
EXPECT_GT(result.average, minimum_ssim + 0.1);
EXPECT_GT(result.min, minimum_ssim);
webrtc::test::QualityMetricsResult psnr_result, ssim_result;
EXPECT_EQ(0, webrtc::test::I420MetricsFromFiles(
config_.input_filename.c_str(),
config_.output_filename.c_str(),
config_.codec_settings->width,
config_.codec_settings->height,
&psnr_result,
&ssim_result));
EXPECT_GT(psnr_result.average, minimum_psnr + 2.0);
EXPECT_GT(psnr_result.min, minimum_psnr);
EXPECT_GT(ssim_result.average, minimum_ssim + 0.1);
EXPECT_GT(ssim_result.min, minimum_ssim);
}
};

32
src/modules/video_coding/codecs/test_framework/benchmark.cc
View File

@ -11,7 +11,6 @@
#include "benchmark.h"
#include <cassert>
#include <fstream>
#include <iostream>
#include <sstream>
#include <vector>
@ -19,12 +18,11 @@
#include <windows.h>
#endif
#include "event_wrapper.h"
#include "testsupport/fileutils.h"
#include "common_video/libyuv/include/libyuv.h"
#include "video_codec_interface.h"
#include "video_source.h"
#include "system_wrappers/interface/event_wrapper.h"
#include "modules/video_coding/codecs/test_framework/video_source.h"
#include "testsupport/fileutils.h"
#include "testsupport/metrics/video_metrics.h"
#define SSIM_CALC 0 // by default, don't compute SSIM
@ -80,8 +78,8 @@ Benchmark::Perform()
const int nBitrates = sizeof(bitRate)/sizeof(*bitRate);
int testIterations = 10;
double psnr[nBitrates];
double ssim[nBitrates];
webrtc::test::QualityMetricsResult psnr[nBitrates];
webrtc::test::QualityMetricsResult ssim[nBitrates];
double fps[nBitrates];
double totalEncodeTime[nBitrates];
double totalDecodeTime[nBitrates];
@ -153,12 +151,14 @@ Benchmark::Perform()
double actualBitRate = ActualBitRate(_framecnt) / 1000.0;
std::cout << " " << actualBitRate;
_results << "," << actualBitRate;
PSNRfromFiles(_inname.c_str(), _outname.c_str(), _inst.width,
_inst.height, &psnr[k]);
webrtc::test::QualityMetricsResult psnr_result;
I420PSNRFromFiles(_inname.c_str(), _outname.c_str(),
_inst.width, _inst.height, &psnr[k]);
if (SSIM_CALC)
{
SSIMfromFiles(_inname.c_str(), _outname.c_str(), _inst.width,
_inst.height, &ssim[k]);
webrtc::test::QualityMetricsResult ssim_result;
I420SSIMFromFiles(_inname.c_str(), _outname.c_str(),
_inst.width, _inst.height, &ssim[k]);
}
fps[k] = avgFps;
@ -167,8 +167,8 @@ Benchmark::Perform()
_results << std::endl << "Y-PSNR [dB]";
for (int k = 0; k < nBitrates; k++)
{
std::cout << " " << psnr[k];
_results << "," << psnr[k];
std::cout << " " << psnr[k].average;
_results << "," << psnr[k].average;
}
if (SSIM_CALC)
@ -177,8 +177,8 @@ Benchmark::Perform()
_results << std::endl << "SSIM ";
for (int k = 0; k < nBitrates; k++)
{
std::cout << " " << ssim[k];
_results << "," << ssim[k];
std::cout << " " << ssim[k].average;
_results << "," << ssim[k].average;
}
}

411
src/modules/video_coding/codecs/test_framework/test.cc
View File

@ -9,41 +9,16 @@
*/
#include "test.h"
#include "video_source.h"
#include "common_video/libyuv/include/libyuv.h"
#include "event_wrapper.h"
#include "thread_wrapper.h"
#include <cstring>
#include <iostream>
#include <fstream>
#include <cmath>
#include <ctime>
#include <string.h>
#include <cassert>
#include <vector>
#include "testsupport/metrics/video_metrics.h"
using namespace webrtc;
long filesize(const char *filename); // local function defined at end of file
struct SSIMcontext
{
SSIMcontext() :
refFileName(NULL), testFileName(NULL), width(0), height(0),
SSIMptr(NULL), startFrame(-1), endFrame(-1), evnt(NULL) {};
SSIMcontext(const char *ref, const char *test, int w, int h, double *Sptr,
int start, int end, EventWrapper* ev) :
refFileName(ref), testFileName(test), width(w), height(h),
SSIMptr(Sptr), startFrame(start), endFrame(end), evnt(ev) {};
const char *refFileName;
const char *testFileName;
int width;
int height;
double *SSIMptr;
int startFrame;
int endFrame;
EventWrapper* evnt;
};
Test::Test(std::string name, std::string description)
:
_bitRate(0),
@ -80,14 +55,17 @@ Test::Print()
(*_log) << _description << std::endl;
(*_log) << "Input file: " << _inname << std::endl;
(*_log) << "Output file: " << _outname << std::endl;
double psnr = -1.0, ssim = -1.0;
PSNRfromFiles(_inname.c_str(), _outname.c_str(), _inst.width, _inst.height, &psnr);
ssim = SSIMfromFilesMT(4 /* number of threads*/);
webrtc::test::QualityMetricsResult psnr;
webrtc::test::QualityMetricsResult ssim;
I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _inst.width,
_inst.height, &psnr);
I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _inst.width,
_inst.height, &ssim);
(*_log) << "PSNR: " << psnr << std::endl;
std::cout << "PSNR: " << psnr << std::endl << std::endl;
(*_log) << "SSIM: " << ssim << std::endl;
std::cout << "SSIM: " << ssim << std::endl << std::endl;
(*_log) << "PSNR: " << psnr.average << std::endl;
std::cout << "PSNR: " << psnr.average << std::endl << std::endl;
(*_log) << "SSIM: " << ssim.average << std::endl;
std::cout << "SSIM: " << ssim.average << std::endl << std::endl;
(*_log) << std::endl;
}
@ -143,354 +121,6 @@ Test::SetLog(std::fstream* log)
_log = log;
}
int
Test::PSNRfromFiles(const char *refFileName, const char *testFileName, int width, int height, double *YPSNRptr)
{
FILE *refFp = fopen(refFileName, "rb");
if( refFp == NULL ) {
// cannot open reference file
fprintf(stderr, "Cannot open file %s\n", refFileName);
return -1;
}
FILE *testFp = fopen(testFileName, "rb");
if( testFp == NULL ) {
// cannot open test file
fprintf(stderr, "Cannot open file %s\n", testFileName);
return -2;
}
double mse = 0.0;
double mseLogSum = 0.0;
int frames = 0;
int frameBytes = 3*width*height/2; // bytes in one frame I420
unsigned char *ref = new unsigned char[frameBytes]; // space for one frame I420
unsigned char *test = new unsigned char[frameBytes]; // space for one frame I420
int refBytes = (int) fread(ref, 1, frameBytes, refFp);
int testBytes = (int) fread(test, 1, frameBytes, testFp);
while( refBytes == frameBytes && testBytes == frameBytes )
{
mse = 0.0;
// calculate Y sum-square-difference
for( int k = 0; k < width * height; k++ )
{
mse += (test[k] - ref[k]) * (test[k] - ref[k]);
}
// divide by number of pixels
mse /= (double) (width * height);
// accumulate for total average
mseLogSum += std::log10( mse );
frames++;
refBytes = (int) fread(ref, 1, frameBytes, refFp);
testBytes = (int) fread(test, 1, frameBytes, testFp);
}
// ypsnrAvg = sum( 10 log (255^2 / MSE) ) / frames
// = 20 * log(255) - 10 * mseLogSum / frames
*YPSNRptr = 20.0 * std::log10(255.0) - 10.0 * mseLogSum / frames;
delete [] ref;
delete [] test;
fclose(refFp);
fclose(testFp);
return 0;
}
int
Test::SSIMfromFiles(const char *refFileName, const char *testFileName, int width, int height, double *SSIMptr,
int startFrame /*= -1*/, int endFrame /*= -1*/)
{
FILE *refFp = fopen(refFileName, "rb");
if( refFp == NULL ) {
// cannot open reference file
fprintf(stderr, "Cannot open file %s\n", refFileName);
return -1;
}
FILE *testFp = fopen(testFileName, "rb");
if( testFp == NULL ) {
// cannot open test file
fprintf(stderr, "Cannot open file %s\n", testFileName);
return -2;
}
int frames = 0;
int frameBytes = 3*width*height/2; // bytes in one frame I420
unsigned char *ref = new unsigned char[frameBytes]; // space for one frame I420
unsigned char *test = new unsigned char[frameBytes]; // space for one frame I420
if (startFrame >= 0)
{
if (fseek(refFp, frameBytes * startFrame, SEEK_SET) != 0){
fprintf(stderr, "Cannot go to frame %i in %s\n", startFrame, refFileName);
return -1;
}
if (fseek(testFp, frameBytes * startFrame, SEEK_SET) != 0){
fprintf(stderr, "Cannot go to frame %i in %s\n", startFrame, testFileName);
return -1;
}
}
int refBytes = (int) fread(ref, 1, frameBytes, refFp);
int testBytes = (int) fread(test, 1, frameBytes, testFp);
//
// SSIM: variable definition, window function, initialization
int window = 10;
int flag_window = 0; //0 for uniform window filter, 1 for gaussian symmetric window
float variance_window = 2.0; //variance for window function
float ssimFilter[121]; //2d window filter: typically 11x11 = (window+1)*(window+1)
//statistics per column of window (#columns = window+1), 0 element for avg over all columns
float avgTest[12];
float avgRef[12];
float contrastTest[12];
float contrastRef[12];
float crossCorr[12];
//
//offsets for stability
float offset1 = 0.1f;
float offset2 = 0.1f;
float offset3 = offset2/2;
//
//define window for SSIM: take uniform filter for now
float sumfil = 0.0;
int nn=-1;
for(int j=-window/2;j<=window/2;j++)
for(int i=-window/2;i<=window/2;i++)
{
nn+=1;
if (flag_window == 0)
ssimFilter[nn] = 1.0;
else
{
float dist = (float)(i*i) + (float)(j*j);
float tmp = 0.5f*dist/variance_window;
ssimFilter[nn] = exp(-tmp);
}
sumfil +=ssimFilter[nn];
}
//normalize window
nn=-1;
for(int j=-window/2;j<=window/2;j++)
for(int i=-window/2;i<=window/2;i++)
{
nn+=1;
ssimFilter[nn] = ssimFilter[nn]/((float)sumfil);
}
//
float ssimScene = 0.0; //avgerage SSIM for sequence
//
//SSIM: done with variables and defintion
//
while( refBytes == frameBytes && testBytes == frameBytes &&
!(endFrame >= 0 && frames > endFrame - startFrame))
{
float ssimFrame = 0.0;
int sh = window/2+1;
int numPixels = 0;
for(int i=sh;i<height-sh;i++)
for(int j=sh;j<width-sh;j++)
{
avgTest[0] = 0.0;
avgRef[0] = 0.0;
contrastTest[0] = 0.0;
contrastRef[0] = 0.0;
crossCorr[0] = 0.0;
numPixels +=1;
//for uniform window, only need to loop over whole window for first column pixel in image, and then shift
if (j == sh || flag_window == 1)
{
//initialize statistics
for(int k=1;k<window+2;k++)
{
avgTest[k] = 0.0;
avgRef[k] = 0.0;
contrastTest[k] = 0.0;
contrastRef[k] = 0.0;
crossCorr[k] = 0.0;
}
int nn=-1;
//compute contrast and correlation
for(int jj=-window/2;jj<=window/2;jj++)
for(int ii=-window/2;ii<=window/2;ii++)
{
nn+=1;
int i2 = i+ii;
int j2 = j+jj;
float tmp1 = (float)test[i2*width+j2];
float tmp2 = (float)ref[i2*width+j2];
//local average of each signal
avgTest[jj+window/2+1] += ssimFilter[nn]*tmp1;
avgRef[jj+window/2+1] += ssimFilter[nn]*tmp2;
//local correlation/contrast of each signal
contrastTest[jj+window/2+1] += ssimFilter[nn]*tmp1*tmp1;
contrastRef[jj+window/2+1] += ssimFilter[nn]*tmp2*tmp2;
//local cross correlation
crossCorr[jj+window/2+1] += ssimFilter[nn]*tmp1*tmp2;
}
}
//for uniform window case, can shift window horiz, then compute statistics for last column in window
else
{
//shift statistics horiz.
for(int k=1;k<window+1;k++)
{
avgTest[k]=avgTest[k+1];
avgRef[k]=avgRef[k+1];
contrastTest[k] = contrastTest[k+1];
contrastRef[k] = contrastRef[k+1];
crossCorr[k] = crossCorr[k+1];
}
//compute statistics for last column
avgTest[window+1] = 0.0;
avgRef[window+1] = 0.0;
contrastTest[window+1] = 0.0;
contrastRef[window+1] = 0.0;
crossCorr[window+1] = 0.0;
int nn = (window+1)*window - 1;
int jj = window/2;
int j2 = j + jj;
for(int ii=-window/2;ii<=window/2;ii++)
{
nn+=1;
int i2 = i+ii;
float tmp1 = (float)test[i2*width+j2];
float tmp2 = (float)ref[i2*width+j2];
//local average of each signal
avgTest[jj+window/2+1] += ssimFilter[nn]*tmp1;
avgRef[jj+window/2+1] += ssimFilter[nn]*tmp2;
//local correlation/contrast of each signal
contrastTest[jj+window/2+1] += ssimFilter[nn]*tmp1*tmp1;
contrastRef[jj+window/2+1] += ssimFilter[nn]*tmp2*tmp2;
//local cross correlation
crossCorr[jj+window/2+1] += ssimFilter[nn]*tmp1*tmp2;
}
}
//sum over all columns
for(int k=1;k<window+2;k++)
{
avgTest[0] += avgTest[k];
avgRef[0] += avgRef[k];
contrastTest[0] += contrastTest[k];
contrastRef[0] += contrastRef[k];
crossCorr[0] += crossCorr[k];
}
float tmp1 = (contrastTest[0] - avgTest[0]*avgTest[0]);
if (tmp1 < 0.0) tmp1 = 0.0;
contrastTest[0] = sqrt(tmp1);
float tmp2 = (contrastRef[0] - avgRef[0]*avgRef[0]);
if (tmp2 < 0.0) tmp2 = 0.0;
contrastRef[0] = sqrt(tmp2);
crossCorr[0] = crossCorr[0] - avgTest[0]*avgRef[0];
float ssimCorrCoeff = (crossCorr[0]+offset3)/(contrastTest[0]*contrastRef[0] + offset3);
float ssimLuminance = (2*avgTest[0]*avgRef[0]+offset1)/(avgTest[0]*avgTest[0] + avgRef[0]*avgRef[0] + offset1);
float ssimContrast = (2*contrastTest[0]*contrastRef[0]+offset2)/(contrastTest[0]*contrastTest[0] + contrastRef[0]*contrastRef[0] + offset2);
float ssimPixel = ssimCorrCoeff * ssimLuminance * ssimContrast;
ssimFrame += ssimPixel;
}
ssimFrame = ssimFrame / (numPixels);
//printf("***SSIM for frame ***%f \n",ssimFrame);
ssimScene += ssimFrame;
//
//SSIM: done with SSIM computation
//
frames++;
refBytes = (int) fread(ref, 1, frameBytes, refFp);
testBytes = (int) fread(test, 1, frameBytes, testFp);
}
//SSIM: normalize/average for sequence
ssimScene = ssimScene / frames;
*SSIMptr = ssimScene;
delete [] ref;
delete [] test;
fclose(refFp);
fclose(testFp);
return 0;
}
bool
Test::SSIMthread(void *vctx)
{
SSIMcontext *ctx = (SSIMcontext *) vctx;
SSIMfromFiles(ctx->refFileName, ctx->testFileName, ctx->width, ctx->height, ctx->SSIMptr, ctx->startFrame, ctx->endFrame);
ctx->evnt->Set();
return false;
}
double Test::SSIMfromFilesMT(const int numThreads)
{
int numFrames = filesize(_inname.c_str()) / _lengthSourceFrame;
std::vector<int> nFramesVec(numThreads);
std::vector<double> ssimVec(numThreads);
int framesPerCore = (numFrames + numThreads - 1) / numThreads; // rounding up
int i = 0;
int nFrames;
for (nFrames = numFrames; nFrames >= framesPerCore; nFrames -= framesPerCore)
{
nFramesVec[i++] = framesPerCore;
}
if (nFrames > 0)
{
assert(i == numThreads - 1);
nFramesVec[i] = nFrames; // remainder
}
int frameIx = 0;
std::vector<EventWrapper*> eventVec(numThreads);
std::vector<ThreadWrapper*> threadVec(numThreads);
std::vector<SSIMcontext> ctxVec(numThreads);
for (i = 0; i < numThreads; i++)
{
eventVec[i] = EventWrapper::Create();
ctxVec[i] = SSIMcontext(_inname.c_str(), _outname.c_str(), _inst.width, _inst.height, &ssimVec[i], frameIx, frameIx + nFramesVec[i] - 1, eventVec[i]);
threadVec[i] = ThreadWrapper::CreateThread(SSIMthread, &(ctxVec[i]), kLowPriority);
unsigned int id;
threadVec[i]->Start(id);
frameIx += nFramesVec[i];
}
// wait for all events
for (i = 0; i < numThreads; i++) {
eventVec[i]->Wait(100000 /* ms*/);
threadVec[i]->Stop();
delete threadVec[i];
delete eventVec[i];
}
double avgSsim = 0;
for (i = 0; i < numThreads; i++)
{
avgSsim += (ssimVec[i] * nFramesVec[i]);
}
avgSsim /= numFrames;
return avgSsim;
}
double Test::ActualBitRate(int nFrames)
{
return 8.0 * _sumEncBytes / (nFrames / _inst.maxFramerate);
@ -526,11 +156,10 @@ Test::VideoEncodedBufferToEncodedImage(TestVideoEncodedBuffer& videoBuffer, Enco
long filesize(const char *filename)
{
FILE *f = fopen(filename,"rb"); /* open the file in read only */
long size = 0;
if (fseek(f,0,SEEK_END)==0) /* seek was successful */
size = ftell(f);
fclose(f);
return size;
FILE *f = fopen(filename,"rb"); /* open the file in read only */
long size = 0;
if (fseek(f,0,SEEK_END)==0) /* seek was successful */
size = ftell(f);
fclose(f);
return size;
}

14
src/modules/video_coding/codecs/test_framework/test.h
View File

@ -36,20 +36,6 @@ protected:
WebRtc_UWord32 frameRate=30,
WebRtc_UWord32 bitRate=0);
virtual void Teardown();
static int PSNRfromFiles(const char *refFileName,
const char *testFileName,
int width,
int height,
double *YPSNRptr);
static int SSIMfromFiles(const char *refFileName,
const char *testFileName,
int width,
int height,
double *SSIMptr,
int startByte = -1, int endByte = -1);
double SSIMfromFilesMT(int numThreads);
static bool SSIMthread(void *ctx);
double ActualBitRate(int nFrames);
virtual bool PacketLoss(double lossRate, int /*thrown*/);
static double RandUniform() { return (std::rand() + 1.0)/(RAND_MAX + 1.0); }

1
src/modules/video_coding/codecs/test_framework/test_framework.gypi
View File

@ -16,6 +16,7 @@
'type': '<(library)',
'dependencies': [
'<(webrtc_root)/../test/metrics.gyp:metrics',
'<(webrtc_root)/../test/test.gyp:test_support',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
'<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',

1
src/modules/video_coding/codecs/tools/video_codecs_tools.gypi
View File

@ -18,6 +18,7 @@
'video_codecs_test_framework',
'webrtc_video_coding',
'webrtc_vp8',
'<(webrtc_root)/../test/metrics.gyp:metrics',
'<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
],
'sources': [

52
src/modules/video_coding/codecs/tools/video_quality_measurement.cc
View File

@ -265,29 +265,27 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
}
void CalculateSsimVideoMetrics(webrtc::test::TestConfig* config,
QualityMetricsResult* ssimResult) {
webrtc::test::QualityMetricsResult* result) {
Log("Calculating SSIM...\n");
SsimFromFiles(config->input_filename.c_str(), config->output_filename.c_str(),
config->codec_settings->width,
config->codec_settings->height, ssimResult);
Log(" Average: %3.2f\n", ssimResult->average);
Log(" Min : %3.2f (frame %d)\n", ssimResult->min,
ssimResult->min_frame_number);
Log(" Max : %3.2f (frame %d)\n", ssimResult->max,
ssimResult->max_frame_number);
I420SSIMFromFiles(config->input_filename.c_str(),
config->output_filename.c_str(),
config->codec_settings->width,
config->codec_settings->height, result);
Log(" Average: %3.2f\n", result->average);
Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number);
Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number);
}
void CalculatePsnrVideoMetrics(webrtc::test::TestConfig* config,
QualityMetricsResult* psnrResult) {
webrtc::test::QualityMetricsResult* result) {
Log("Calculating PSNR...\n");
PsnrFromFiles(config->input_filename.c_str(), config->output_filename.c_str(),
I420PSNRFromFiles(config->input_filename.c_str(),
config->output_filename.c_str(),
config->codec_settings->width,
config->codec_settings->height, psnrResult);
Log(" Average: %3.2f\n", psnrResult->average);
Log(" Min : %3.2f (frame %d)\n", psnrResult->min,
psnrResult->min_frame_number);
Log(" Max : %3.2f (frame %d)\n", psnrResult->max,
psnrResult->max_frame_number);
config->codec_settings->height, result);
Log(" Average: %3.2f\n", result->average);
Log(" Min : %3.2f (frame %d)\n", result->min, result->min_frame_number);
Log(" Max : %3.2f (frame %d)\n", result->max, result->max_frame_number);
}
void PrintConfigurationSummary(const webrtc::test::TestConfig& config) {
@ -312,8 +310,8 @@ void PrintConfigurationSummary(const webrtc::test::TestConfig& config) {
}
void PrintCsvOutput(const webrtc::test::Stats& stats,
const QualityMetricsResult& ssimResult,
const QualityMetricsResult& psnrResult) {
const webrtc::test::QualityMetricsResult& ssim_result,
const webrtc::test::QualityMetricsResult& psnr_result) {
Log("\nCSV output (recommended to run with --noverbose to skip the "
"above output)\n");
printf("frame_number encoding_successful decoding_successful "
@ -325,8 +323,8 @@ void PrintCsvOutput(const webrtc::test::Stats& stats,
for (unsigned int i = 0; i < stats.stats_.size(); ++i) {
const webrtc::test::FrameStatistic& f = stats.stats_[i];
const FrameResult& ssim = ssimResult.frames[i];
const FrameResult& psnr = psnrResult.frames[i];
const webrtc::test::FrameResult& ssim = ssim_result.frames[i];
const webrtc::test::FrameResult& psnr = psnr_result.frames[i];
printf("%4d, %d, %d, %2d, %2d, %6d, %6d, %5d, %7d, %d, %2d, %2d, "
"%5.3f, %5.2f\n",
f.frame_number,
@ -348,8 +346,8 @@ void PrintCsvOutput(const webrtc::test::Stats& stats,
void PrintPythonOutput(const webrtc::test::TestConfig& config,
const webrtc::test::Stats& stats,
const QualityMetricsResult& ssimResult,
const QualityMetricsResult& psnrResult) {
const webrtc::test::QualityMetricsResult& ssim_result,
const webrtc::test::QualityMetricsResult& psnr_result) {
Log("\nPython output (recommended to run with --noverbose to skip the "
"above output)\n");
printf("test_configuration = ["
@ -412,8 +410,8 @@ void PrintPythonOutput(const webrtc::test::TestConfig& config,
printf("frame_data = [");
for (unsigned int i = 0; i < stats.stats_.size(); ++i) {
const webrtc::test::FrameStatistic& f = stats.stats_[i];
const FrameResult& ssim = ssimResult.frames[i];
const FrameResult& psnr = psnrResult.frames[i];
const webrtc::test::FrameResult& ssim = ssim_result.frames[i];
const webrtc::test::FrameResult& psnr = psnr_result.frames[i];
printf("{'frame_number': %d, "
"'encoding_successful': %s, 'decoding_successful': %s, "
"'encode_time': %d, 'decode_time': %d, "
@ -508,9 +506,9 @@ int main(int argc, char* argv[]) {
stats.PrintSummary();
QualityMetricsResult ssim_result;
webrtc::test::QualityMetricsResult ssim_result;
CalculateSsimVideoMetrics(&config, &ssim_result);
QualityMetricsResult psnr_result;
webrtc::test::QualityMetricsResult psnr_result;
CalculatePsnrVideoMetrics(&config, &psnr_result);
if (FLAGS_csv) {

1
src/modules/video_coding/main/source/video_coding_test.gypi
View File

@ -13,6 +13,7 @@
'dependencies': [
'<(webrtc_root)/../testing/gtest.gyp:gtest',
'<(webrtc_root)/../test/test.gyp:test_support',
'<(webrtc_root)/../test/metrics.gyp:metrics',
'webrtc_video_coding',
'rtp_rtcp',
'webrtc_utility',

5
src/modules/video_coding/main/test/codec_database_test.cc
View File

@ -368,8 +368,9 @@ CodecDataBaseTest::Perform(CmdArgs& args)
_vcm->Decode();
// Don't measure PSNR for I420 since it will be perfect.
if (sendCodec.codecType != kVideoCodecI420) {
QualityMetricsResult psnr;
PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr);
webrtc::test::QualityMetricsResult psnr;
I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _width,
_height, &psnr);
printf("\n @ %d KBPS: ", sendCodec.startBitrate);
printf("PSNR from encoder-decoder send-receive control test"
"is %f\n\n", psnr.average);

5
src/modules/video_coding/main/test/media_opt_test.cc
View File

@ -480,8 +480,9 @@ MediaOptTest::Print(int mode)
{
double ActualBitRate = 8.0 *( _sumEncBytes / (_frameCnt / _frameRate));
double actualBitRate = ActualBitRate / 1000.0;
QualityMetricsResult psnr;
PsnrFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width, _height, &psnr);
webrtc::test::QualityMetricsResult psnr;
I420PSNRFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width,
_height, &psnr);
(_log) << "VCM: Media Optimization Test Cycle Completed!" << std::endl;
(_log) << "Input file: " << _inname << std::endl;

8
src/modules/video_coding/main/test/normal_test.cc
View File

@ -367,9 +367,11 @@ NormalTest::Print()
double actualBitRate = ActualBitRate / 1000.0;
double avgEncTime = _totalEncodeTime / _frameCnt;
double avgDecTime = _totalDecodeTime / _frameCnt;
QualityMetricsResult psnr, ssim;
PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr);
SsimFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &ssim);
webrtc::test::QualityMetricsResult psnr, ssim;
I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _width, _height,
&psnr);
I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _width, _height,
&ssim);
printf("Actual bitrate: %f kbps\n", actualBitRate);
printf("Target bitrate: %f kbps\n", _bitRate);
( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " << _bitRate << " kbps" << std::endl;

8
src/modules/video_coding/main/test/quality_modes_test.cc
View File

@ -113,8 +113,9 @@ QualityModesTest::Print()
double actualBitRate = ActualBitRate / 1000.0;
double avgEncTime = _totalEncodeTime / _frameCnt;
double avgDecTime = _totalDecodeTime / _frameCnt;
QualityMetricsResult psnr,ssim;
PsnrFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &psnr);
webrtc::test::QualityMetricsResult psnr,ssim;
I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth,
_nativeHeight, &psnr);
printf("Actual bitrate: %f kbps\n", actualBitRate);
printf("Target bitrate: %f kbps\n", _bitRate);
( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " << _bitRate << " kbps" << std::endl;
@ -128,7 +129,8 @@ QualityModesTest::Print()
if (_flagSSIM == 1)
{
printf("***computing SSIM***\n");
SsimFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &ssim);
I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth,
_nativeHeight, &ssim);
printf("SSIM: %f \n", ssim.average);
}
(_log) << std::endl;

30
src/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@ -8,15 +8,15 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "framedrop_primitives.h"
#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include "testsupport/fileutils.h"
#include "testsupport/metrics/video_metrics.h"
#include "vie_autotest.h"
#include "vie_file_based_comparison_tests.h"
#include "vie_integration_test_base.h"
#include "vie_to_file_renderer.h"
#include "video_engine/test/auto_test/automated/vie_integration_test_base.h"
#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h"
#include "video_engine/test/auto_test/interface/vie_autotest.h"
#include "video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h"
#include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
namespace {
@ -79,17 +79,17 @@ class ViEVideoVerificationTest : public testing::Test {
void CompareFiles(const std::string& reference_file,
const std::string& test_file,
double minimum_psnr, double minimum_ssim) {
QualityMetricsResult psnr;
int psnr_error = PsnrFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &psnr);
ASSERT_EQ(0, psnr_error) << "PSNR routine failed - output files missing?";
ASSERT_GT(psnr.average, minimum_psnr);
webrtc::test::QualityMetricsResult psnr;
int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &psnr);
EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?";
EXPECT_GT(psnr.average, minimum_psnr);
QualityMetricsResult ssim;
int ssim_error = SsimFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &ssim);
ASSERT_EQ(0, ssim_error) << "SSIM routine failed - output files missing?";
ASSERT_GT(ssim.average, minimum_ssim); // 1 = perfect, -1 = terrible
webrtc::test::QualityMetricsResult ssim;
error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(),
kInputWidth, kInputHeight, &ssim);
EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
EXPECT_GT(ssim.average, minimum_ssim); // 1 = perfect, -1 = terrible
ViETest::Log("Results: PSNR: %f (db) SSIM: %f",
psnr.average, ssim.average);

1
src/video_engine/test/auto_test/vie_auto_test.gypi
View File

@ -18,6 +18,7 @@
'<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
'<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
'<(webrtc_root)/../test/metrics.gyp:metrics',
'<(webrtc_root)/../test/test.gyp:test_support',
'video_engine_core',
],

46
test/metrics.gyp Normal file
View File

@ -0,0 +1,46 @@
# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
{
'includes': [
'../src/build/common.gypi',
],
'targets': [
{
# The metrics code must be kept in its own GYP file in order to
# avoid a circular dependency error due to the dependency on libyuv.
# If the code would be put in test.gyp a circular dependency error during
# GYP generation would occur, because the libyuv.gypi unittest target
# depends on test_support_main. See issue #160 for more info.
'target_name': 'metrics',
'type': '<(library)',
'dependencies': [
'<(webrtc_root)/common_video/common_video.gyp:webrtc_libyuv',
],
'include_dirs': [
'.',
],
'sources': [
'testsupport/metrics/video_metrics.h',
'testsupport/metrics/video_metrics.cc',
],
},
{
'target_name': 'metrics_unittests',
'type': 'executable',
'dependencies': [
'metrics',
'<(webrtc_root)/../test/test.gyp:test_support_main',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
],
'sources': [
'testsupport/metrics/video_metrics_unittest.cc',
],
},
],
}

3
test/test.gyp
View File

@ -43,8 +43,6 @@
'testsupport/frame_writer.cc',
'testsupport/packet_reader.h',
'testsupport/packet_reader.cc',
'testsupport/metrics/video_metrics.h',
'testsupport/metrics/video_metrics.cc',
'testsupport/mock/mock_frame_reader.h',
'testsupport/mock/mock_frame_writer.h',
],
@ -74,7 +72,6 @@
'testsupport/frame_reader_unittest.cc',
'testsupport/frame_writer_unittest.cc',
'testsupport/packet_reader_unittest.cc',
'testsupport/metrics/video_metrics_unittest.cc',
],
},
],

490
test/testsupport/metrics/video_metrics.cc
View File

@ -11,14 +11,13 @@
#include "testsupport/metrics/video_metrics.h"
#include <algorithm> // min_element, max_element
#include <cmath>
#include <fstream>
#include <cassert>
#include <cstdio>
// Calculates PSNR from MSE
static inline double CalcPsnr(double mse) {
// Formula: PSNR = 10 log (255^2 / MSE) = 20 log(255) - 10 log(MSE)
return 20.0 * std::log10(255.0) - 10.0 * std::log10(mse);
}
#include "common_video/libyuv/include/libyuv.h"
namespace webrtc {
namespace test {
// Used for calculating min and max values
static bool LessForFrameResultValue (const FrameResult& s1,
@ -26,342 +25,163 @@ static bool LessForFrameResultValue (const FrameResult& s1,
return s1.value < s2.value;
}
int
PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result)
{
FILE *refFp = fopen(refFileName, "rb");
if (refFp == NULL )
{
// cannot open reference file
fprintf(stderr, "Cannot open file %s\n", refFileName);
return -1;
}
enum VideoMetricsType { kPSNR, kSSIM, kBoth };
FILE *testFp = fopen(testFileName, "rb");
if (testFp == NULL )
{
// cannot open test file
fprintf(stderr, "Cannot open file %s\n", testFileName);
fclose(refFp);
return -2;
}
double mse = 0.0;
double mseSum = 0.0;
WebRtc_Word32 frames = 0;
// Allocating size for one I420 frame.
WebRtc_Word32 frameBytes = 3 * width * height >> 1;
WebRtc_UWord8 *ref = new WebRtc_UWord8[frameBytes];
WebRtc_UWord8 *test = new WebRtc_UWord8[frameBytes];
WebRtc_Word32 refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp);
WebRtc_Word32 testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp);
while (refBytes == frameBytes && testBytes == frameBytes)
{
mse = 0.0;
WebRtc_Word32 sh = 8; //boundary offset
for (WebRtc_Word32 k2 = sh; k2 < height - sh; k2++)
for (WebRtc_Word32 k = sh; k < width - sh; k++)
{
WebRtc_Word32 kk = k2 * width + k;
mse += (test[kk] - ref[kk]) * (test[kk] - ref[kk]);
}
// divide by number of pixels
mse /= (double) (width * height);
// Save statistics for this specific frame
FrameResult frame_result;
frame_result.value = CalcPsnr(mse);
frame_result.frame_number = frames;
result->frames.push_back(frame_result);
// accumulate for total average
mseSum += mse;
frames++;
refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp);
testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp);
}
if (mseSum == 0)
{
// The PSNR value is undefined in this case.
// This value effectively means that the files are equal.
result->average = std::numeric_limits<double>::max();
}
else
{
result->average = CalcPsnr(mseSum / frames);
}
int return_code = 0;
if (result->frames.size() == 0)
{
fprintf(stderr, "Tried to measure SSIM from empty files (reference "
"file: %s test file: %s\n", refFileName, testFileName);
return_code = -3;
}
else
{
// Calculate min/max statistics
std::vector<FrameResult>::iterator element;
element = min_element(result->frames.begin(),
result->frames.end(), LessForFrameResultValue);
result->min = element->value;
result->min_frame_number = element->frame_number;
element = max_element(result->frames.begin(),
result->frames.end(), LessForFrameResultValue);
result->max = element->value;
result->max_frame_number = element->frame_number;
}
delete [] ref;
delete [] test;
fclose(refFp);
fclose(testFp);
return return_code;
// Calculates metrics for a frame and adds statistics to the result for it.
void CalculateFrame(VideoMetricsType video_metrics_type,
uint8_t* ref,
uint8_t* test,
int width,
int height,
int frame_number,
QualityMetricsResult* result) {
FrameResult frame_result;
frame_result.frame_number = frame_number;
switch (video_metrics_type) {
case kPSNR:
frame_result.value = I420PSNR(ref, test, width, height);
break;
case kSSIM:
frame_result.value = I420SSIM(ref, test, width, height);
break;
default:
assert(false);
}
result->frames.push_back(frame_result);
}
static double
Similarity(WebRtc_UWord64 sum_s, WebRtc_UWord64 sum_r, WebRtc_UWord64 sum_sq_s,
WebRtc_UWord64 sum_sq_r, WebRtc_UWord64 sum_sxr, WebRtc_Word32 count)
{
WebRtc_Word64 ssim_n, ssim_d;
WebRtc_Word64 c1, c2;
const WebRtc_Word64 cc1 = 26634; // (64^2*(.01*255)^2
const WebRtc_Word64 cc2 = 239708; // (64^2*(.03*255)^2
// Calculates average, min and max values for the supplied struct, if non-NULL.
void CalculateStats(QualityMetricsResult* result) {
if (result == NULL || result->frames.size() == 0) {
return;
}
// Calculate average
std::vector<FrameResult>::iterator iter;
double metrics_values_sum = 0.0;
for (iter = result->frames.begin(); iter != result->frames.end(); ++iter) {
metrics_values_sum += iter->value;
}
result->average = metrics_values_sum / result->frames.size();
// Scale the constants by number of pixels
c1 = (cc1 * count * count) >> 12;
c2 = (cc2 * count * count) >> 12;
ssim_n = (2 * sum_s * sum_r + c1) * ((WebRtc_Word64) 2 * count * sum_sxr-
(WebRtc_Word64) 2 * sum_s * sum_r + c2);
ssim_d = (sum_s * sum_s + sum_r * sum_r + c1)*
((WebRtc_Word64)count * sum_sq_s - (WebRtc_Word64)sum_s * sum_s +
(WebRtc_Word64)count * sum_sq_r - (WebRtc_Word64) sum_r * sum_r + c2);
return ssim_n * 1.0 / ssim_d;
// Calculate min/max statistics
iter = min_element(result->frames.begin(), result->frames.end(),
LessForFrameResultValue);
result->min = iter->value;
result->min_frame_number = iter->frame_number;
iter = max_element(result->frames.begin(), result->frames.end(),
LessForFrameResultValue);
result->max = iter->value;
result->max_frame_number = iter->frame_number;
}
#if !defined(WEBRTC_USE_SSE2)
static double
Ssim8x8C(WebRtc_UWord8 *s, WebRtc_Word32 sp,
WebRtc_UWord8 *r, WebRtc_Word32 rp)
{
WebRtc_UWord64 sum_s = 0;
WebRtc_UWord64 sum_r = 0;
WebRtc_UWord64 sum_sq_s = 0;
WebRtc_UWord64 sum_sq_r = 0;
WebRtc_UWord64 sum_sxr = 0;
// Single method that handles all combinations of video metrics calculation, to
// minimize code duplication. Either psnr_result or ssim_result may be NULL,
// depending on which VideoMetricsType is targeted.
int CalculateMetrics(VideoMetricsType video_metrics_type,
const char* ref_filename,
const char* test_filename,
int width,
int height,
QualityMetricsResult* psnr_result,
QualityMetricsResult* ssim_result) {
assert(ref_filename != NULL);
assert(test_filename != NULL);
assert(width > 0);
assert(height > 0);
WebRtc_Word32 i, j;
for (i = 0; i < 8; i++, s += sp,r += rp)
{
for (j = 0; j < 8; j++)
{
sum_s += s[j];
sum_r += r[j];
sum_sq_s += s[j] * s[j];
sum_sq_r += r[j] * r[j];
sum_sxr += s[j] * r[j];
}
FILE* ref_fp = fopen(ref_filename, "rb");
if (ref_fp == NULL) {
// cannot open reference file
fprintf(stderr, "Cannot open file %s\n", ref_filename);
return -1;
}
FILE* test_fp = fopen(test_filename, "rb");
if (test_fp == NULL) {
// cannot open test file
fprintf(stderr, "Cannot open file %s\n", test_filename);
fclose(ref_fp);
return -2;
}
int frame_number = 0;
// Allocating size for one I420 frame.
const int frame_length = 3 * width * height >> 1;
uint8_t* ref = new uint8_t[frame_length];
uint8_t* test = new uint8_t[frame_length];
int ref_bytes = fread(ref, 1, frame_length, ref_fp);
int test_bytes = fread(test, 1, frame_length, test_fp);
while (ref_bytes == frame_length && test_bytes == frame_length) {
switch (video_metrics_type) {
case kPSNR:
CalculateFrame(kPSNR, ref, test, width, height, frame_number,
psnr_result);
break;
case kSSIM:
CalculateFrame(kSSIM, ref, test, width, height, frame_number,
ssim_result);
break;
case kBoth:
CalculateFrame(kPSNR, ref, test, width, height, frame_number,
psnr_result);
CalculateFrame(kSSIM, ref, test, width, height, frame_number,
ssim_result);
break;
default:
assert(false);
}
return Similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64);
}
#endif
#if defined(WEBRTC_USE_SSE2)
#include <emmintrin.h>
#include <xmmintrin.h>
static double
Ssim8x8Sse2(WebRtc_UWord8 *s, WebRtc_Word32 sp,
WebRtc_UWord8 *r, WebRtc_Word32 rp)
{
WebRtc_Word32 i;
const __m128i z = _mm_setzero_si128();
__m128i sum_s_16 = _mm_setzero_si128();
__m128i sum_r_16 = _mm_setzero_si128();
__m128i sum_sq_s_32 = _mm_setzero_si128();
__m128i sum_sq_r_32 = _mm_setzero_si128();
__m128i sum_sxr_32 = _mm_setzero_si128();
for (i = 0; i < 8; i++, s += sp,r += rp)
{
const __m128i s_8 = _mm_loadl_epi64((__m128i*)(s));
const __m128i r_8 = _mm_loadl_epi64((__m128i*)(r));
const __m128i s_16 = _mm_unpacklo_epi8(s_8,z);
const __m128i r_16 = _mm_unpacklo_epi8(r_8,z);
sum_s_16 = _mm_adds_epu16(sum_s_16, s_16);
sum_r_16 = _mm_adds_epu16(sum_r_16, r_16);
const __m128i sq_s_32 = _mm_madd_epi16(s_16, s_16);
sum_sq_s_32 = _mm_add_epi32(sum_sq_s_32, sq_s_32);
const __m128i sq_r_32 = _mm_madd_epi16(r_16, r_16);
sum_sq_r_32 = _mm_add_epi32(sum_sq_r_32, sq_r_32);
const __m128i sxr_32 = _mm_madd_epi16(s_16, r_16);
sum_sxr_32 = _mm_add_epi32(sum_sxr_32, sxr_32);
}
const __m128i sum_s_32 = _mm_add_epi32(_mm_unpackhi_epi16(sum_s_16, z),
_mm_unpacklo_epi16(sum_s_16, z));
const __m128i sum_r_32 = _mm_add_epi32(_mm_unpackhi_epi16(sum_r_16, z),
_mm_unpacklo_epi16(sum_r_16, z));
__m128i sum_s_128;
__m128i sum_r_128;
__m128i sum_sq_s_128;
__m128i sum_sq_r_128;
__m128i sum_sxr_128;
_mm_store_si128 (&sum_s_128,
_mm_add_epi64(_mm_unpackhi_epi32(sum_s_32, z),
_mm_unpacklo_epi32(sum_s_32, z)));
_mm_store_si128 (&sum_r_128,
_mm_add_epi64(_mm_unpackhi_epi32(sum_r_32, z),
_mm_unpacklo_epi32(sum_r_32, z)));
_mm_store_si128 (&sum_sq_s_128,
_mm_add_epi64(_mm_unpackhi_epi32(sum_sq_s_32, z),
_mm_unpacklo_epi32(sum_sq_s_32, z)));
_mm_store_si128 (&sum_sq_r_128,
_mm_add_epi64(_mm_unpackhi_epi32(sum_sq_r_32, z),
_mm_unpacklo_epi32(sum_sq_r_32, z)));
_mm_store_si128 (&sum_sxr_128,
_mm_add_epi64(_mm_unpackhi_epi32(sum_sxr_32, z),
_mm_unpacklo_epi32(sum_sxr_32, z)));
const WebRtc_UWord64 *sum_s_64 =
reinterpret_cast<WebRtc_UWord64*>(&sum_s_128);
const WebRtc_UWord64 *sum_r_64 =
reinterpret_cast<WebRtc_UWord64*>(&sum_r_128);
const WebRtc_UWord64 *sum_sq_s_64 =
reinterpret_cast<WebRtc_UWord64*>(&sum_sq_s_128);
const WebRtc_UWord64 *sum_sq_r_64 =
reinterpret_cast<WebRtc_UWord64*>(&sum_sq_r_128);
const WebRtc_UWord64 *sum_sxr_64 =
reinterpret_cast<WebRtc_UWord64*>(&sum_sxr_128);
const WebRtc_UWord64 sum_s = sum_s_64[0] + sum_s_64[1];
const WebRtc_UWord64 sum_r = sum_r_64[0] + sum_r_64[1];
const WebRtc_UWord64 sum_sq_s = sum_sq_s_64[0] + sum_sq_s_64[1];
const WebRtc_UWord64 sum_sq_r = sum_sq_r_64[0] + sum_sq_r_64[1];
const WebRtc_UWord64 sum_sxr = sum_sxr_64[0] + sum_sxr_64[1];
return Similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64);
}
#endif
double
SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1,
WebRtc_Word32 stride_img2, WebRtc_Word32 width, WebRtc_Word32 height)
{
WebRtc_Word32 i,j;
WebRtc_UWord32 samples = 0;
double ssim_total = 0;
double (*ssim_8x8)(WebRtc_UWord8*, WebRtc_Word32,
WebRtc_UWord8*, WebRtc_Word32 rp);
#if defined(WEBRTC_USE_SSE2)
ssim_8x8 = Ssim8x8Sse2;
#else
ssim_8x8 = Ssim8x8C;
#endif
// Sample point start with each 4x4 location
for (i = 0; i < height - 8; i += 4, img1 += stride_img1 * 4,
img2 += stride_img2 * 4)
{
for (j = 0; j < width - 8; j += 4 )
{
double v = ssim_8x8(img1 + j, stride_img1, img2 + j, stride_img2);
ssim_total += v;
samples++;
}
}
ssim_total /= samples;
return ssim_total;
frame_number++;
ref_bytes = fread(ref, 1, frame_length, ref_fp);
test_bytes = fread(test, 1, frame_length, test_fp);
}
int return_code = 0;
if (frame_number == 0) {
fprintf(stderr, "Tried to measure video metrics from empty files "
"(reference file: %s test file: %s)\n", ref_filename,
test_filename);
return_code = -3;
} else {
CalculateStats(psnr_result);
CalculateStats(ssim_result);
}
delete [] ref;
delete [] test;
fclose(ref_fp);
fclose(test_fp);
return return_code;
}
int
SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result)
{
FILE *refFp = fopen(refFileName, "rb");
if (refFp == NULL)
{
// cannot open reference file
fprintf(stderr, "Cannot open file %s\n", refFileName);
return -1;
}
FILE *testFp = fopen(testFileName, "rb");
if (testFp == NULL)
{
// cannot open test file
fprintf(stderr, "Cannot open file %s\n", testFileName);
fclose(refFp);
return -2;
}
WebRtc_Word32 frames = 0;
// Bytes in one frame I420
const WebRtc_Word32 frameBytes = 3 * width * height / 2;
WebRtc_UWord8 *ref = new WebRtc_UWord8[frameBytes];
WebRtc_UWord8 *test = new WebRtc_UWord8[frameBytes];
WebRtc_Word32 refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp);
WebRtc_Word32 testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp);
double ssimScene = 0.0; //average SSIM for sequence
while (refBytes == frameBytes && testBytes == frameBytes )
{
double ssimFrameValue = SsimFrame(ref, test, width, width, width, height);
// Save statistics for this specific frame
FrameResult frame_result;
frame_result.value = ssimFrameValue;
frame_result.frame_number = frames;
result->frames.push_back(frame_result);
ssimScene += ssimFrameValue;
frames++;
refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp);
testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp);
}
int return_code = 0;
if (result->frames.size() == 0)
{
fprintf(stderr, "Tried to measure SSIM from empty files (reference "
"file: %s test file: %s\n", refFileName, testFileName);
return_code = -3;
}
else
{
// SSIM: normalize/average for sequence
ssimScene = ssimScene / frames;
result->average = ssimScene;
// Calculate min/max statistics
std::vector<FrameResult>::iterator element;
element = min_element(result->frames.begin(),
result->frames.end(), LessForFrameResultValue);
result->min = element->value;
result->min_frame_number = element->frame_number;
element = max_element(result->frames.begin(),
result->frames.end(), LessForFrameResultValue);
result->max = element->value;
result->max_frame_number = element->frame_number;
}
delete [] ref;
delete [] test;
fclose(refFp);
fclose(testFp);
return return_code;
int I420MetricsFromFiles(const char* ref_filename,
const char* test_filename,
int width,
int height,
QualityMetricsResult* psnr_result,
QualityMetricsResult* ssim_result) {
assert(psnr_result != NULL);
assert(ssim_result != NULL);
return CalculateMetrics(kBoth, ref_filename, test_filename, width, height,
psnr_result, ssim_result);
}
int I420PSNRFromFiles(const char* ref_filename,
const char* test_filename,
int width,
int height,
QualityMetricsResult* result) {
assert(result != NULL);
return CalculateMetrics(kPSNR, ref_filename, test_filename, width, height,
result, NULL);
}
int I420SSIMFromFiles(const char* ref_filename,
const char* test_filename,
int width,
int height,
QualityMetricsResult* result) {
assert(result != NULL);
return CalculateMetrics(kSSIM, ref_filename, test_filename, width, height,
NULL, result);
}
} // namespace test
} // namespace webrtc

100
test/testsupport/metrics/video_metrics.h
View File

@ -8,17 +8,18 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_
#define WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_
#ifndef WEBRTC_TESTSUPPORT_METRICS_VIDEO_METRICS_H_
#define WEBRTC_TESTSUPPORT_METRICS_VIDEO_METRICS_H_
#include <limits>
#include <vector>
#include "typedefs.h"
namespace webrtc {
namespace test {
// Contains video quality metrics result for a single frame.
struct FrameResult {
WebRtc_Word32 frame_number;
int frame_number;
double value;
};
@ -35,46 +36,77 @@ struct QualityMetricsResult {
double average;
double min;
double max;
WebRtc_Word32 min_frame_number;
WebRtc_Word32 max_frame_number;
int min_frame_number;
int max_frame_number;
std::vector<FrameResult> frames;
};
// PSNR & SSIM calculations
// PSNR values are filled into the QualityMetricsResult struct.
// If the result is std::numerical_limits<double>::max() the videos were
// equal. Otherwise, PSNR values are in decibel (higher is better). This
// algorithm only compares up to the point when the shortest video ends.
// By definition of PSNR, the result value is undefined if the reference file
// and the test file are identical. In that case the max value for double
// will be set in the result struct.
//
// Returns 0 if successful, negative on errors:
// Calculates PSNR and SSIM values for the reference and test video files
// (must be in I420 format). All calculated values are filled into the
// QualityMetricsResult stucts.
// PSNR values have the unit decibel (dB) where a high value means the test file
// is similar to the reference file. The higher value, the more similar.
// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR
// SSIM values range between -1.0 and 1.0, where 1.0 means the files are
// identical. For more info about SSIM, see http://en.wikipedia.org/wiki/SSIM
// This function only compares video frames up to the point when the shortest
// video ends.
// Return value:
// 0 if successful, negative on errors:
// -1 if the source file cannot be opened
// -2 if the test file cannot be opened
// -3 if any of the files are empty
int PsnrFromFiles(const WebRtc_Word8 *refFileName,
const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
WebRtc_Word32 height, QualityMetricsResult *result);
// -4 if any arguments are invalid.
int I420MetricsFromFiles(const char* ref_filename,
const char* test_filename,
int width,
int height,
QualityMetricsResult* psnr_result,
QualityMetricsResult* ssim_result);
// SSIM values are filled into the QualityMetricsResult struct.
// Values range between -1 and 1, where 1 means the files were identical. This
// algorithm only compares up to the point when the shortest video ends.
// By definition, SSIM values varies from -1.0, when everything is different
// between the reference file and the test file, up to 1.0 for two identical
// files.
// Calculates PSNR values for the reference and test video files (must be in
// I420 format). All calculated values are filled into the QualityMetricsResult
// struct.
// PSNR values have the unit decibel (dB) where a high value means the test file
// is similar to the reference file. The higher value, the more similar.
// This function only compares video frames up to the point when the shortest
// video ends.
// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR
//
// Returns 0 if successful, negative on errors:
// Return value:
// 0 if successful, negative on errors:
// -1 if the source file cannot be opened
// -2 if the test file cannot be opened
// -3 if any of the files are empty
int SsimFromFiles(const WebRtc_Word8 *refFileName,
const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
WebRtc_Word32 height, QualityMetricsResult *result);
// -4 if any arguments are invalid.
int I420PSNRFromFiles(const char* ref_filename,
const char* test_filename,
int width,
int height,
QualityMetricsResult* result);
double SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2,
WebRtc_Word32 stride_img1, WebRtc_Word32 stride_img2,
WebRtc_Word32 width, WebRtc_Word32 height);
// Calculates SSIM values for the reference and test video files (must be in
// I420 format). All calculated values are filled into the QualityMetricsResult
// struct.
// SSIM values range between -1.0 and 1.0, where 1.0 means the files are
// identical.
// This function only compares video frames up to the point when the shortest
// video ends.
// For more info about SSIM, see http://en.wikipedia.org/wiki/SSIM
//
// Return value:
// 0 if successful, negative on errors:
// -1 if the source file cannot be opened
// -2 if the test file cannot be opened
// -3 if any of the files are empty
// -4 if any arguments are invalid.
int I420SSIMFromFiles(const char* ref_filename,
const char* test_filename,
int width,
int height,
QualityMetricsResult* result);
#endif // WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_
} // namespace test
} // namespace webrtc
#endif // WEBRTC_TESTSUPPORT_METRICS_VIDEO_METRICS_H_

110
test/testsupport/metrics/video_metrics_unittest.cc
View File

@ -10,8 +10,6 @@
#include "testsupport/metrics/video_metrics.h"
#include <cstdio>
#include "gtest/gtest.h"
#include "testsupport/fileutils.h"
@ -25,13 +23,13 @@ static const int kHeight = 288;
static const int kMissingReferenceFileReturnCode = -1;
static const int kMissingTestFileReturnCode = -2;
static const int kEmptyFileReturnCode = -3;
static const double kPsnrPerfectResult = std::numeric_limits<double>::max();
static const double kPsnrPerfectResult = 48.0;
static const double kSsimPerfectResult = 1.0;
class VideoMetricsTest: public testing::Test {
protected:
VideoMetricsTest() {
video_file = webrtc::test::ProjectRootPath() + "resources/foreman_cif.yuv";
video_file_ = webrtc::test::ResourcePath("foreman_cif_short", "yuv");
}
virtual ~VideoMetricsTest() {}
void SetUp() {
@ -42,56 +40,100 @@ class VideoMetricsTest: public testing::Test {
void TearDown() {
std::remove(kEmptyFileName);
}
QualityMetricsResult result_;
std::string video_file;
webrtc::test::QualityMetricsResult psnr_result_;
webrtc::test::QualityMetricsResult ssim_result_;
std::string video_file_;
};
// Tests that it is possible to run with the same reference as test file
TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFiles) {
EXPECT_EQ(0, PsnrFromFiles(video_file.c_str(), video_file.c_str(),
kWidth, kHeight, &result_));
EXPECT_EQ(kPsnrPerfectResult, result_.average);
EXPECT_EQ(SsimFromFiles(video_file.c_str(), video_file.c_str(), kWidth, kHeight,
&result_), 0);
EXPECT_EQ(kSsimPerfectResult, result_.average);
TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFilesPSNR) {
EXPECT_EQ(0, I420PSNRFromFiles(video_file_.c_str(), video_file_.c_str(),
kWidth, kHeight, &psnr_result_));
EXPECT_EQ(kPsnrPerfectResult, psnr_result_.average);
}
TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFilesSSIM) {
EXPECT_EQ(0, I420SSIMFromFiles(video_file_.c_str(), video_file_.c_str(),
kWidth, kHeight, &ssim_result_));
EXPECT_EQ(kSsimPerfectResult, ssim_result_.average);
}
TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFilesBothMetrics) {
EXPECT_EQ(0, I420MetricsFromFiles(video_file_.c_str(), video_file_.c_str(),
kWidth, kHeight, &psnr_result_,
&ssim_result_));
EXPECT_EQ(kPsnrPerfectResult, psnr_result_.average);
EXPECT_EQ(kSsimPerfectResult, ssim_result_.average);
}
// Tests that the right return code is given when the reference file is missing.
TEST_F(VideoMetricsTest, MissingReferenceFile) {
TEST_F(VideoMetricsTest, MissingReferenceFilePSNR) {
EXPECT_EQ(kMissingReferenceFileReturnCode,
PsnrFromFiles(kNonExistingFileName, video_file.c_str(), kWidth,
kHeight, &result_));
I420PSNRFromFiles(kNonExistingFileName, video_file_.c_str(),
kWidth, kHeight, &ssim_result_));
}
TEST_F(VideoMetricsTest, MissingReferenceFileSSIM) {
EXPECT_EQ(kMissingReferenceFileReturnCode,
SsimFromFiles(kNonExistingFileName, video_file.c_str(), kWidth,
kHeight, &result_));
I420SSIMFromFiles(kNonExistingFileName, video_file_.c_str(),
kWidth, kHeight, &ssim_result_));
}
TEST_F(VideoMetricsTest, MissingReferenceFileBothMetrics) {
EXPECT_EQ(kMissingReferenceFileReturnCode,
I420MetricsFromFiles(kNonExistingFileName, video_file_.c_str(),
kWidth, kHeight,
&psnr_result_, &ssim_result_));
}
// Tests that the right return code is given when the test file is missing.
TEST_F(VideoMetricsTest, MissingTestFile) {
TEST_F(VideoMetricsTest, MissingTestFilePSNR) {
EXPECT_EQ(kMissingTestFileReturnCode,
PsnrFromFiles(video_file.c_str(), kNonExistingFileName, kWidth,
kHeight, &result_));
I420PSNRFromFiles(video_file_.c_str(), kNonExistingFileName,
kWidth, kHeight, &ssim_result_));
}
TEST_F(VideoMetricsTest, MissingTestFileSSIM) {
EXPECT_EQ(kMissingTestFileReturnCode,
SsimFromFiles(video_file.c_str(), kNonExistingFileName, kWidth,
kHeight, &result_));
I420SSIMFromFiles(video_file_.c_str(), kNonExistingFileName,
kWidth, kHeight, &ssim_result_));
}
TEST_F(VideoMetricsTest, MissingTestFileBothMetrics) {
EXPECT_EQ(kMissingTestFileReturnCode,
I420MetricsFromFiles(video_file_.c_str(), kNonExistingFileName,
kWidth, kHeight,
&psnr_result_, &ssim_result_));
}
// Tests that the method can be executed with empty files.
TEST_F(VideoMetricsTest, EmptyFiles) {
TEST_F(VideoMetricsTest, EmptyFilesPSNR) {
EXPECT_EQ(kEmptyFileReturnCode,
PsnrFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight,
&result_));
I420PSNRFromFiles(kEmptyFileName, video_file_.c_str(),
kWidth, kHeight, &ssim_result_));
EXPECT_EQ(kEmptyFileReturnCode,
SsimFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight,
&result_));
// Run the same again with the empty file switched.
I420PSNRFromFiles(video_file_.c_str(), kEmptyFileName,
kWidth, kHeight, &ssim_result_));
}
TEST_F(VideoMetricsTest, EmptyFilesSSIM) {
EXPECT_EQ(kEmptyFileReturnCode,
PsnrFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight,
&result_));
I420SSIMFromFiles(kEmptyFileName, video_file_.c_str(),
kWidth, kHeight, &ssim_result_));
EXPECT_EQ(kEmptyFileReturnCode,
SsimFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight,
&result_));
I420SSIMFromFiles(video_file_.c_str(), kEmptyFileName,
kWidth, kHeight, &ssim_result_));
}
TEST_F(VideoMetricsTest, EmptyFilesBothMetrics) {
EXPECT_EQ(kEmptyFileReturnCode,
I420MetricsFromFiles(kEmptyFileName, video_file_.c_str(),
kWidth, kHeight,
&psnr_result_, &ssim_result_));
EXPECT_EQ(kEmptyFileReturnCode,
I420MetricsFromFiles(video_file_.c_str(), kEmptyFileName,
kWidth, kHeight,
&psnr_result_, &ssim_result_));
}
} // namespace webrtc

1
webrtc.gyp
View File

@ -20,6 +20,7 @@
'src/system_wrappers/source/system_wrappers.gyp:*',
'src/video_engine/video_engine.gyp:*',
'src/voice_engine/voice_engine.gyp:*',
'test/metrics.gyp:*',
'test/test.gyp:*',
],
},