From f0a8464b74a2abd670e4be99faf864f100b51c04 Mon Sep 17 00:00:00 2001 From: "kjellander@webrtc.org" Date: Mon, 12 Sep 2011 13:45:39 +0000 Subject: [PATCH] Added more statistics during SSIM/PSNR calculation, including calculation of min/max value. Moved video_metrics.h into a GYP library so it can be used from other projects. Review URL: http://webrtc-codereview.appspot.com/132013 git-svn-id: http://webrtc.googlecode.com/svn/trunk@582 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/source/video_coding_test.gypi | 22 ++++++- .../main/test/codec_database_test.cc | 4 +- .../video_coding/main/test/media_opt_test.cc | 14 ++-- .../video_coding/main/test/normal_test.cc | 11 ++-- .../main/test/quality_modes_test.cc | 8 +-- .../video_coding/main/test/video_metrics.cc | 66 ++++++++++++++++--- .../video_coding/main/test/video_metrics.h | 32 ++++++++- 7 files changed, 124 insertions(+), 33 deletions(-) diff --git a/src/modules/video_coding/main/source/video_coding_test.gypi b/src/modules/video_coding/main/source/video_coding_test.gypi index 7f47cf72b..e017f822f 100644 --- a/src/modules/video_coding/main/source/video_coding_test.gypi +++ b/src/modules/video_coding/main/source/video_coding_test.gypi @@ -8,10 +8,30 @@ { 'targets': [ + { + 'target_name': 'video_coding_test_lib', + 'type': '<(library)', + 'dependencies': [ + ], + 'include_dirs': [ + ], + 'direct_dependent_settings': { + 'include_dirs': [ + '../test', + ], + }, + 'sources': [ + # headers + '../test/video_metrics.h', + # sources + '../test/video_metrics.cc', + ], + }, { 'target_name': 'video_coding_test', 'type': 'executable', 'dependencies': [ + 'video_coding_test_lib', 'webrtc_video_coding', 'rtp_rtcp', 'webrtc_utility', @@ -39,7 +59,6 @@ '../test/release_test.h', '../test/rtp_player.h', '../test/test_util.h', - '../test/video_metrics.h', '../test/video_source.h', # sources @@ -56,7 +75,6 @@ '../test/rtp_player.cc', '../test/test_util.cc', '../test/tester_main.cc', - '../test/video_metrics.cc', '../test/video_rtp_play_mt.cc', '../test/video_rtp_play.cc', '../test/video_source.cc', diff --git a/src/modules/video_coding/main/test/codec_database_test.cc b/src/modules/video_coding/main/test/codec_database_test.cc index 7a1f75bf3..5030bf626 100644 --- a/src/modules/video_coding/main/test/codec_database_test.cc +++ b/src/modules/video_coding/main/test/codec_database_test.cc @@ -364,10 +364,10 @@ CodecDataBaseTest::Perform(CmdArgs& args) delete encodeCallCDT; // closing and calculating PSNR for prior encoder-decoder test TearDown(); // closing open files - double psnr = 0; + QualityMetricsResult psnr; PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr); printf(" \n @ %d KBPS: ", sendCodec.startBitrate); - printf("PSNR from encoder-decoder send-receive control test is %f \n \n", psnr); + printf("PSNR from encoder-decoder send-receive control test is %f \n \n", psnr.average); } // end of #codecs >1 delete waitEvent; diff --git a/src/modules/video_coding/main/test/media_opt_test.cc b/src/modules/video_coding/main/test/media_opt_test.cc index a6ff6e9dc..db6828e0f 100644 --- a/src/modules/video_coding/main/test/media_opt_test.cc +++ b/src/modules/video_coding/main/test/media_opt_test.cc @@ -467,7 +467,7 @@ MediaOptTest::Print(int mode) { double ActualBitRate = 8.0 *( _sumEncBytes / (_frameCnt / _frameRate)); double actualBitRate = ActualBitRate / 1000.0; - double psnr; + QualityMetricsResult psnr; PsnrFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width, _height, &psnr); (_log) << "VCM: Media Optimization Test Cycle Completed!" << std::endl; @@ -477,7 +477,7 @@ MediaOptTest::Print(int mode) (_log) << "Error Reslience: NACK:" << _nackEnabled << "; FEC: " << _fecEnabled << std::endl; (_log) << "Packet Loss applied= %f " << _lossRate << std::endl; (_log) << _numFramesDropped << " FRames were dropped" << std::endl; - ( _log) << "PSNR: " << psnr << std::endl; + ( _log) << "PSNR: " << psnr.average << std::endl; (_log) << std::endl; if (_testType == 2) @@ -490,7 +490,7 @@ MediaOptTest::Print(int mode) fprintf(_outputRes,"FEC: %s \n ",(_fecEnabled)?"true":"false"); fprintf(_outputRes,"Packet loss applied = %f\n", _lossRate); fprintf(_outputRes,"%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt); - fprintf(_outputRes,"PSNR: %f \n", psnr); + fprintf(_outputRes,"PSNR: %f \n", psnr.average); fprintf(_outputRes,"************\n"); } @@ -506,13 +506,13 @@ MediaOptTest::Print(int mode) fprintf(_fpout,"FEC: %s \n ",(_fecEnabled)?"true":"false"); fprintf(_fpout,"Packet loss applied = %f\n", _lossRate); fprintf(_fpout,"%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt); - fprintf(_fpout,"PSNR: %f \n", psnr); + fprintf(_fpout,"PSNR: %f \n", psnr.average); fprintf(_fpout,"************\n"); int testNum1 = _testNum/(_numParRuns +1) + 1; int testNum2 = _testNum%_numParRuns; if (testNum2 == 0) testNum2 = _numParRuns; - fprintf(_fpout2,"%d %d %f %f %f %f \n",testNum1,testNum2,_bitRate,actualBitRate,_lossRate,psnr); + fprintf(_fpout2,"%d %d %f %f %f %f \n",testNum1,testNum2,_bitRate,actualBitRate,_lossRate,psnr.average); fclose(_fpinp); _fpinp = fopen("dat_inp","wb"); fprintf(_fpinp,"%f %f %d \n",_bitRate,_lossRate,_testNum); @@ -530,9 +530,9 @@ MediaOptTest::Print(int mode) printf("FEC: %s \n",(_fecEnabled)?"true":"false"); printf("Packet loss applied = %f\n", _lossRate); printf("%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt); - printf("PSNR: %f \n", psnr); + printf("PSNR: %f \n", psnr.average); } - TEST(psnr > 10); // low becuase of possible frame dropping (need to verify that OK for all packet loss values/ rates) + TEST(psnr.average > 10); // low becuase of possible frame dropping (need to verify that OK for all packet loss values/ rates) } void diff --git a/src/modules/video_coding/main/test/normal_test.cc b/src/modules/video_coding/main/test/normal_test.cc index e4f4fcf94..bfded77b5 100644 --- a/src/modules/video_coding/main/test/normal_test.cc +++ b/src/modules/video_coding/main/test/normal_test.cc @@ -362,8 +362,7 @@ NormalTest::Print() double actualBitRate = ActualBitRate / 1000.0; double avgEncTime = _totalEncodeTime / _frameCnt; double avgDecTime = _totalDecodeTime / _frameCnt; - double psnr; - double ssim; + QualityMetricsResult psnr, ssim; PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr); SsimFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &ssim); printf("Actual bitrate: %f kbps\n", actualBitRate); @@ -373,10 +372,10 @@ NormalTest::Print() ( _log) << "Average encode time: " << avgEncTime << " s" << std::endl; printf("Average decode time: %f s\n", avgDecTime); ( _log) << "Average decode time: " << avgDecTime << " s" << std::endl; - printf("PSNR: %f \n", psnr); - ( _log) << "PSNR: " << psnr << std::endl; - printf("SSIM: %f \n", ssim); - ( _log) << "SSIM: " << ssim << std::endl; + printf("PSNR: %f \n", psnr.average); + ( _log) << "PSNR: " << psnr.average << std::endl; + printf("SSIM: %f \n", ssim.average); + ( _log) << "SSIM: " << ssim.average << std::endl; (_log) << std::endl; printf("\nVCM Normal Test: \n\n%i tests completed\n", vcmMacrosTests); diff --git a/src/modules/video_coding/main/test/quality_modes_test.cc b/src/modules/video_coding/main/test/quality_modes_test.cc index dd464fe5e..da8517d95 100644 --- a/src/modules/video_coding/main/test/quality_modes_test.cc +++ b/src/modules/video_coding/main/test/quality_modes_test.cc @@ -108,7 +108,7 @@ QualityModesTest::Print() double actualBitRate = ActualBitRate / 1000.0; double avgEncTime = _totalEncodeTime / _frameCnt; double avgDecTime = _totalDecodeTime / _frameCnt; - double psnr,ssim; + QualityMetricsResult psnr,ssim; PsnrFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &psnr); printf("Actual bitrate: %f kbps\n", actualBitRate); printf("Target bitrate: %f kbps\n", _bitRate); @@ -117,14 +117,14 @@ QualityModesTest::Print() ( _log) << "Average encode time: " << avgEncTime << " s" << std::endl; printf("Average decode time: %f s\n", avgDecTime); ( _log) << "Average decode time: " << avgDecTime << " s" << std::endl; - printf("PSNR: %f \n", psnr); + printf("PSNR: %f \n", psnr.average); printf("**Number of frames dropped in VPM***%d \n",_numFramesDroppedVPM); - ( _log) << "PSNR: " << psnr << std::endl; + ( _log) << "PSNR: " << psnr.average << std::endl; if (_flagSSIM == 1) { printf("***computing SSIM***\n"); SsimFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &ssim); - printf("SSIM: %f \n", ssim); + printf("SSIM: %f \n", ssim.average); } (_log) << std::endl; diff --git a/src/modules/video_coding/main/test/video_metrics.cc b/src/modules/video_coding/main/test/video_metrics.cc index ca58010dc..feee4821f 100644 --- a/src/modules/video_coding/main/test/video_metrics.cc +++ b/src/modules/video_coding/main/test/video_metrics.cc @@ -8,13 +8,26 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "video_metrics.h" +#include // min_element, max_element #include +#include #include "system_wrappers/interface/cpu_features_wrapper.h" -#include "test_util.h" + +// Calculates PSNR from MSE +static inline double CalcPsnr(double mse) { + // Formula: PSNR = 10 log (255^2 / MSE) = 20 log(255) - 10 log(MSE) + return 20.0 * std::log10(255.0) - 10.0 * std::log10(mse); +} + +// Used for calculating min and max values +static bool lessForFrameResultValue (const FrameResult& s1, const FrameResult& s2) { + return s1.value < s2.value; +} WebRtc_Word32 PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, - WebRtc_Word32 width, WebRtc_Word32 height, double *YPsnrPtr) + WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result) { FILE *refFp = fopen(refFileName, "rb"); if (refFp == NULL ) @@ -33,7 +46,7 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, } double mse = 0.0; - double mseLogSum = 0.0; + double mseSum = 0.0; WebRtc_Word32 frames = 0; // Allocating size for one I420 frame. @@ -57,10 +70,16 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, } // divide by number of pixels - mse /= (double) (width * height); + mse /= (double) (width * height); + + // Save statistics for this specific frame + FrameResult frame_result; + frame_result.value = CalcPsnr(mse); + frame_result.frame_number = frames; + result->frames.push_back(frame_result); // accumulate for total average - mseLogSum += std::log10(mse); + mseSum += mse; frames++; refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp); @@ -69,13 +88,23 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, // for identical reproduction: if (mse == 0) { - *YPsnrPtr = 48; + result->average = 48; } else { - *YPsnrPtr = 20.0 * std::log10(255.0) - 10.0 * mseLogSum / frames; + result->average = CalcPsnr(mseSum / frames); } + // Calculate min/max statistics + std::vector::iterator element; + element = min_element(result->frames.begin(), + result->frames.end(), lessForFrameResultValue); + result->min = element->value; + result->min_frame_number = element->frame_number; + element = max_element(result->frames.begin(), + result->frames.end(), lessForFrameResultValue); + result->max = element->value; + result->max_frame_number = element->frame_number; delete [] ref; delete [] test; @@ -238,7 +267,7 @@ SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1, WebRtc_Word32 SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, - WebRtc_Word32 width, WebRtc_Word32 height, double *YSsimPtr) + WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result) { FILE *refFp = fopen(refFileName, "rb"); if (refFp == NULL) @@ -270,8 +299,14 @@ SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, while (refBytes == frameBytes && testBytes == frameBytes ) { - ssimScene += SsimFrame(ref, test, width, width, width, height); + double ssimFrameValue = SsimFrame(ref, test, width, width, width, height); + // Save statistics for this specific frame + FrameResult frame_result; + frame_result.value = ssimFrameValue; + frame_result.frame_number = frames; + result->frames.push_back(frame_result); + ssimScene += ssimFrameValue; frames++; refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp); @@ -280,7 +315,18 @@ SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, // SSIM: normalize/average for sequence ssimScene = ssimScene / frames; - *YSsimPtr = ssimScene; + result->average = ssimScene; + + // Calculate min/max statistics + std::vector::iterator element; + element = min_element(result->frames.begin(), + result->frames.end(), lessForFrameResultValue); + result->min = element->value; + result->min_frame_number = element->frame_number; + element = max_element(result->frames.begin(), + result->frames.end(), lessForFrameResultValue); + result->max = element->value; + result->max_frame_number = element->frame_number; delete [] ref; delete [] test; diff --git a/src/modules/video_coding/main/test/video_metrics.h b/src/modules/video_coding/main/test/video_metrics.h index 592b696f3..1bbb66c21 100644 --- a/src/modules/video_coding/main/test/video_metrics.h +++ b/src/modules/video_coding/main/test/video_metrics.h @@ -12,17 +12,45 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ #define WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ +#include "typedefs.h" +#include +#include + + +// Contains video quality metrics result for a single frame. +struct FrameResult { + WebRtc_Word32 frame_number; + double value; +}; + +// Result from a PSNR/SSIM calculation operation. +// The frames in this data structure are 0-indexed. +struct QualityMetricsResult { + QualityMetricsResult() : + average(0.0), + min(std::numeric_limits::max()), + max(std::numeric_limits::min()), + min_frame_number(-1), + max_frame_number(-1) + {}; + double average; + double min; + double max; + WebRtc_Word32 min_frame_number; + WebRtc_Word32 max_frame_number; + std::vector frames; +}; // PSNR & SSIM calculations WebRtc_Word32 PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, WebRtc_Word32 width, - WebRtc_Word32 height, double *YPsnrPtr); + WebRtc_Word32 height, QualityMetricsResult *result); WebRtc_Word32 SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, WebRtc_Word32 width, - WebRtc_Word32 height, double *YSsimPtr); + WebRtc_Word32 height, QualityMetricsResult *result); #endif // WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_