Added more statistics during SSIM/PSNR calculation, including calculation of min/max value.

Moved video_metrics.h into a GYP library so it can be used from other projects.

Review URL: http://webrtc-codereview.appspot.com/132013

git-svn-id: http://webrtc.googlecode.com/svn/trunk@582 4adac7df-926f-26a2-2b94-8c16560cd09d

src/modules/video_coding/main/source/video_coding_test.gypi

@@ -8,10 +8,30 @@
 
 {
   'targets': [
+    {
+      'target_name': 'video_coding_test_lib',
+      'type': '<(library)',
+      'dependencies': [
+      ],
+      'include_dirs': [
+      ],
+      'direct_dependent_settings': {
+        'include_dirs': [
+          '../test',
+        ],
+      },
+      'sources': [
+        # headers
+        '../test/video_metrics.h',
+        # sources
+        '../test/video_metrics.cc',
+      ], 
+    },
     {
       'target_name': 'video_coding_test',
       'type': 'executable',
       'dependencies': [
+         'video_coding_test_lib',
          'webrtc_video_coding',
          'rtp_rtcp',
          'webrtc_utility',
@@ -39,7 +59,6 @@
         '../test/release_test.h',
         '../test/rtp_player.h',
         '../test/test_util.h',
-        '../test/video_metrics.h',
         '../test/video_source.h',
 
         # sources
@@ -56,7 +75,6 @@
         '../test/rtp_player.cc',
         '../test/test_util.cc',
         '../test/tester_main.cc',
-        '../test/video_metrics.cc',
         '../test/video_rtp_play_mt.cc',
         '../test/video_rtp_play.cc',
         '../test/video_source.cc',

src/modules/video_coding/main/test/codec_database_test.cc

@@ -364,10 +364,10 @@ CodecDataBaseTest::Perform(CmdArgs& args)
         delete encodeCallCDT;
         // closing and calculating PSNR for prior encoder-decoder test
         TearDown(); // closing open files
-        double psnr = 0;
+        QualityMetricsResult psnr;
         PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr);
         printf(" \n @ %d KBPS:  ", sendCodec.startBitrate);
-        printf("PSNR from encoder-decoder send-receive control test is %f \n \n", psnr);
+        printf("PSNR from encoder-decoder send-receive control test is %f \n \n", psnr.average);
     } // end of #codecs >1
 
     delete waitEvent;

src/modules/video_coding/main/test/media_opt_test.cc

@@ -467,7 +467,7 @@ MediaOptTest::Print(int mode)
 {
     double ActualBitRate =  8.0 *( _sumEncBytes / (_frameCnt / _frameRate));
     double actualBitRate = ActualBitRate / 1000.0;
-    double psnr;
+    QualityMetricsResult psnr;
     PsnrFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width, _height, &psnr);
 
     (_log) << "VCM: Media Optimization Test Cycle Completed!" << std::endl;
@@ -477,7 +477,7 @@ MediaOptTest::Print(int mode)
     (_log) << "Error Reslience: NACK:" << _nackEnabled << "; FEC: " << _fecEnabled << std::endl;
     (_log) << "Packet Loss applied= %f " << _lossRate << std::endl;
     (_log) << _numFramesDropped << " FRames were dropped" << std::endl;
-     ( _log) << "PSNR: " << psnr << std::endl;
+     ( _log) << "PSNR: " << psnr.average << std::endl;
     (_log) << std::endl;
 
     if (_testType == 2)
@@ -490,7 +490,7 @@ MediaOptTest::Print(int mode)
         fprintf(_outputRes,"FEC: %s \n ",(_fecEnabled)?"true":"false");
         fprintf(_outputRes,"Packet loss applied = %f\n", _lossRate);
         fprintf(_outputRes,"%d frames were dropped, and total number of frames processed %d  \n",_numFramesDropped,_frameCnt);
-        fprintf(_outputRes,"PSNR: %f \n", psnr);
+        fprintf(_outputRes,"PSNR: %f \n", psnr.average);
         fprintf(_outputRes,"************\n");
     }
 
@@ -506,13 +506,13 @@ MediaOptTest::Print(int mode)
         fprintf(_fpout,"FEC: %s \n ",(_fecEnabled)?"true":"false");
         fprintf(_fpout,"Packet loss applied = %f\n", _lossRate);
         fprintf(_fpout,"%d frames were dropped, and total number of frames processed %d  \n",_numFramesDropped,_frameCnt);
-        fprintf(_fpout,"PSNR: %f \n", psnr);
+        fprintf(_fpout,"PSNR: %f \n", psnr.average);
         fprintf(_fpout,"************\n");
 
         int testNum1 = _testNum/(_numParRuns +1) + 1;
         int testNum2 = _testNum%_numParRuns;
         if (testNum2 == 0) testNum2 = _numParRuns;
-        fprintf(_fpout2,"%d %d %f %f %f %f \n",testNum1,testNum2,_bitRate,actualBitRate,_lossRate,psnr);
+        fprintf(_fpout2,"%d %d %f %f %f %f \n",testNum1,testNum2,_bitRate,actualBitRate,_lossRate,psnr.average);
         fclose(_fpinp);
         _fpinp = fopen("dat_inp","wb");
         fprintf(_fpinp,"%f %f %d \n",_bitRate,_lossRate,_testNum);
@@ -530,9 +530,9 @@ MediaOptTest::Print(int mode)
         printf("FEC: %s \n",(_fecEnabled)?"true":"false");
         printf("Packet loss applied = %f\n", _lossRate);
         printf("%d frames were dropped, and total number of frames processed %d  \n",_numFramesDropped,_frameCnt);
-        printf("PSNR: %f \n", psnr);
+        printf("PSNR: %f \n", psnr.average);
     }
-    TEST(psnr > 10); // low becuase of possible frame dropping (need to verify that OK for all packet loss values/ rates)
+    TEST(psnr.average > 10); // low becuase of possible frame dropping (need to verify that OK for all packet loss values/ rates)
 }
 
 void

src/modules/video_coding/main/test/normal_test.cc

@@ -362,8 +362,7 @@ NormalTest::Print()
     double actualBitRate = ActualBitRate / 1000.0;
     double avgEncTime = _totalEncodeTime / _frameCnt;
     double avgDecTime = _totalDecodeTime / _frameCnt;
-    double psnr;
+    QualityMetricsResult psnr, ssim;
-    double ssim;
     PsnrFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &psnr);
     SsimFromFiles(_inname.c_str(), _outname.c_str(), _width, _height, &ssim);
     printf("Actual bitrate: %f kbps\n", actualBitRate);
@@ -373,10 +372,10 @@ NormalTest::Print()
     ( _log) << "Average encode time: " << avgEncTime << " s" << std::endl;
     printf("Average decode time: %f s\n", avgDecTime);
     ( _log) << "Average decode time: " << avgDecTime << " s" << std::endl;
-    printf("PSNR: %f \n", psnr);
+    printf("PSNR: %f \n", psnr.average);
-    ( _log) << "PSNR: " << psnr << std::endl;
+    ( _log) << "PSNR: " << psnr.average << std::endl;
-    printf("SSIM: %f \n", ssim);
+    printf("SSIM: %f \n", ssim.average);
-    ( _log) << "SSIM: " << ssim << std::endl;
+    ( _log) << "SSIM: " << ssim.average << std::endl;
     (_log) << std::endl;
 
     printf("\nVCM Normal Test: \n\n%i tests completed\n", vcmMacrosTests);

src/modules/video_coding/main/test/quality_modes_test.cc

@@ -108,7 +108,7 @@ QualityModesTest::Print()
     double actualBitRate = ActualBitRate / 1000.0;
     double avgEncTime = _totalEncodeTime / _frameCnt;
     double avgDecTime = _totalDecodeTime / _frameCnt;
-    double psnr,ssim;
+    QualityMetricsResult psnr,ssim;
     PsnrFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &psnr);
     printf("Actual bitrate: %f kbps\n", actualBitRate);
     printf("Target bitrate: %f kbps\n", _bitRate);
@@ -117,14 +117,14 @@ QualityModesTest::Print()
     ( _log) << "Average encode time: " << avgEncTime << " s" << std::endl;
     printf("Average decode time: %f s\n", avgDecTime);
     ( _log) << "Average decode time: " << avgDecTime << " s" << std::endl;
-    printf("PSNR: %f \n", psnr);
+    printf("PSNR: %f \n", psnr.average);
     printf("**Number of frames dropped in VPM***%d \n",_numFramesDroppedVPM);
-    ( _log) << "PSNR: " << psnr << std::endl;
+    ( _log) << "PSNR: " << psnr.average << std::endl;
     if (_flagSSIM == 1)
     {
         printf("***computing SSIM***\n");
         SsimFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth, _nativeHeight, &ssim);
-        printf("SSIM: %f \n", ssim);
+        printf("SSIM: %f \n", ssim.average);
     }
     (_log) << std::endl;
 

src/modules/video_coding/main/test/video_metrics.cc

@@ -8,13 +8,26 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include "video_metrics.h"
+#include <algorithm> // min_element, max_element
 #include <cmath>
+#include <fstream>
 #include "system_wrappers/interface/cpu_features_wrapper.h"
-#include "test_util.h"
+
+// Calculates PSNR from MSE
+static inline double CalcPsnr(double mse) {
+  // Formula: PSNR = 10 log (255^2 / MSE) = 20 log(255) - 10 log(MSE)
+  return 20.0 * std::log10(255.0) - 10.0 * std::log10(mse);
+}
+
+// Used for calculating min and max values
+static bool lessForFrameResultValue (const FrameResult& s1, const FrameResult& s2) {
+    return s1.value < s2.value;
+}
 
 WebRtc_Word32
 PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
-              WebRtc_Word32 width, WebRtc_Word32 height, double *YPsnrPtr)
+              WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result)
 {
     FILE *refFp = fopen(refFileName, "rb");
     if (refFp == NULL )
@@ -33,7 +46,7 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
     }
 
     double mse = 0.0;
-    double mseLogSum = 0.0;
+    double mseSum = 0.0;
     WebRtc_Word32 frames = 0;
 
     // Allocating size for one I420 frame.
@@ -57,10 +70,16 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
         }
 
         // divide by number of pixels
-          mse /= (double) (width * height);
+        mse /= (double) (width * height);
+
+        // Save statistics for this specific frame
+        FrameResult frame_result;
+        frame_result.value = CalcPsnr(mse);
+        frame_result.frame_number = frames;
+        result->frames.push_back(frame_result);
 
         // accumulate for total average
-        mseLogSum += std::log10(mse);
+        mseSum += mse;
         frames++;
 
         refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp);
@@ -69,13 +88,23 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
      // for identical reproduction:
     if (mse == 0)
     {
-        *YPsnrPtr = 48;
+        result->average = 48;
     }
     else
     {
-        *YPsnrPtr = 20.0 * std::log10(255.0) - 10.0 * mseLogSum / frames;
+        result->average = CalcPsnr(mseSum / frames);
     }
 
+    // Calculate min/max statistics
+    std::vector<FrameResult>::iterator element;
+    element = min_element(result->frames.begin(),
+                          result->frames.end(), lessForFrameResultValue);
+    result->min = element->value;
+    result->min_frame_number = element->frame_number;
+    element = max_element(result->frames.begin(),
+                          result->frames.end(), lessForFrameResultValue);
+    result->max = element->value;
+    result->max_frame_number = element->frame_number;
 
     delete [] ref;
     delete [] test;
@@ -238,7 +267,7 @@ SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1,
 
 WebRtc_Word32
 SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
-              WebRtc_Word32 width, WebRtc_Word32 height, double *YSsimPtr)
+              WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result)
 {
     FILE *refFp = fopen(refFileName, "rb");
     if (refFp == NULL)
@@ -270,8 +299,14 @@ SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
 
     while (refBytes == frameBytes && testBytes == frameBytes )
     {
-        ssimScene += SsimFrame(ref, test, width, width, width, height);
+        double ssimFrameValue = SsimFrame(ref, test, width, width, width, height);
+        // Save statistics for this specific frame
+        FrameResult frame_result;
+        frame_result.value = ssimFrameValue;
+        frame_result.frame_number = frames;
+        result->frames.push_back(frame_result);
 
+        ssimScene += ssimFrameValue;
         frames++;
 
         refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp);
@@ -280,7 +315,18 @@ SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
 
     // SSIM: normalize/average for sequence
     ssimScene  = ssimScene / frames;
-    *YSsimPtr = ssimScene;
+    result->average = ssimScene;
+
+    // Calculate min/max statistics
+    std::vector<FrameResult>::iterator element;
+    element = min_element(result->frames.begin(),
+                          result->frames.end(), lessForFrameResultValue);
+    result->min = element->value;
+    result->min_frame_number = element->frame_number;
+    element = max_element(result->frames.begin(),
+                          result->frames.end(), lessForFrameResultValue);
+    result->max = element->value;
+    result->max_frame_number = element->frame_number;
 
     delete [] ref;
     delete [] test;

src/modules/video_coding/main/test/video_metrics.h

@@ -12,17 +12,45 @@
 #ifndef WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_
 #define WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_
 
+#include "typedefs.h"
+#include <limits>
+#include <vector>
+
+
+// Contains video quality metrics result for a single frame.
+struct FrameResult {
+  WebRtc_Word32 frame_number;
+  double value;
+};
+
+// Result from a PSNR/SSIM calculation operation.
+// The frames in this data structure are 0-indexed.
+struct QualityMetricsResult {
+  QualityMetricsResult() :
+    average(0.0),
+    min(std::numeric_limits<double>::max()),
+    max(std::numeric_limits<double>::min()),
+    min_frame_number(-1),
+    max_frame_number(-1)
+  {};
+  double average;
+  double min;
+  double max;
+  WebRtc_Word32 min_frame_number;
+  WebRtc_Word32 max_frame_number;
+  std::vector<FrameResult> frames;
+};
 
 // PSNR & SSIM calculations
 WebRtc_Word32
 PsnrFromFiles(const WebRtc_Word8 *refFileName,
         const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
-        WebRtc_Word32 height, double *YPsnrPtr);
+        WebRtc_Word32 height, QualityMetricsResult *result);
 
 WebRtc_Word32
 SsimFromFiles(const WebRtc_Word8 *refFileName,
         const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
-        WebRtc_Word32 height, double *YSsimPtr);
+        WebRtc_Word32 height, QualityMetricsResult *result);
 
 
 #endif // WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_