Fixing crash when calculating SSIM and PSNR with empty video files in video_metrics.cc

There were previously a dependency on system_wrappers that is now removed (uses defines to check for SEE2 instructions during compilation time instead).

Review URL: http://webrtc-codereview.appspot.com/296008

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1102 4adac7df-926f-26a2-2b94-8c16560cd09d

test/test.gyp

@@ -34,6 +34,7 @@
         'test_suite.h',
         'testsupport/fileutils.h',
         'testsupport/fileutils.cc',
+        'testsupport/metrics/video_metrics.h',
         'testsupport/metrics/video_metrics.cc',
       ],
     },
@@ -58,6 +59,7 @@
       ],
       'sources': [
         'testsupport/fileutils_unittest.cc',
+        'testsupport/metrics/video_metrics_unittest.cc',
       ],
     },
   ],

test/testsupport/metrics/video_metrics.cc

@@ -9,10 +9,10 @@
  */
 
 #include "video_metrics.h"
+
 #include <algorithm> // min_element, max_element
 #include <cmath>
 #include <fstream>
-#include "system_wrappers/interface/cpu_features_wrapper.h"
 
 // Calculates PSNR from MSE
 static inline double CalcPsnr(double mse) {
@@ -21,11 +21,12 @@ static inline double CalcPsnr(double mse) {
 }
 
 // Used for calculating min and max values
-static bool lessForFrameResultValue (const FrameResult& s1, const FrameResult& s2) {
+static bool LessForFrameResultValue (const FrameResult& s1,
+                                     const FrameResult& s2) {
     return s1.value < s2.value;
 }
 
-WebRtc_Word32
+int
 PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
               WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result)
 {
@@ -86,7 +87,7 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
         testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp);
     }
 
-    if (mse == 0)
+    if (mseSum == 0)
     {
         // The PSNR value is undefined in this case.
         // This value effectively means that the files are equal.
@@ -97,17 +98,25 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
         result->average = CalcPsnr(mseSum / frames);
     }
 
-    // Calculate min/max statistics
+    if (result->frames.size() == 0)
-    std::vector<FrameResult>::iterator element;
+    {
-    element = min_element(result->frames.begin(),
+      fprintf(stderr, "Tried to measure SSIM from empty files (reference "
-                          result->frames.end(), lessForFrameResultValue);
+              "file: %s  test file: %s\n", refFileName, testFileName);
-    result->min = element->value;
+      return -3;
-    result->min_frame_number = element->frame_number;
+    }
-    element = max_element(result->frames.begin(),
+    else
-                          result->frames.end(), lessForFrameResultValue);
+    {
-    result->max = element->value;
+        // Calculate min/max statistics
-    result->max_frame_number = element->frame_number;
+        std::vector<FrameResult>::iterator element;
-
+        element = min_element(result->frames.begin(),
+                              result->frames.end(), LessForFrameResultValue);
+        result->min = element->value;
+        result->min_frame_number = element->frame_number;
+        element = max_element(result->frames.begin(),
+                              result->frames.end(), LessForFrameResultValue);
+        result->max = element->value;
+        result->max_frame_number = element->frame_number;
+    }
     delete [] ref;
     delete [] test;
 
@@ -140,6 +149,7 @@ Similarity(WebRtc_UWord64 sum_s, WebRtc_UWord64 sum_r, WebRtc_UWord64 sum_sq_s,
     return ssim_n * 1.0 / ssim_d;
 }
 
+#if !defined(WEBRTC_USE_SSE2)
 static double
 Ssim8x8C(WebRtc_UWord8 *s, WebRtc_Word32 sp,
          WebRtc_UWord8 *r, WebRtc_Word32 rp)
@@ -164,6 +174,7 @@ Ssim8x8C(WebRtc_UWord8 *s, WebRtc_Word32 sp,
     }
     return Similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64);
 }
+#endif
 
 #if defined(WEBRTC_USE_SSE2)
 #include <emmintrin.h>
@@ -256,13 +267,11 @@ SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1,
     double (*ssim_8x8)(WebRtc_UWord8*, WebRtc_Word32,
                        WebRtc_UWord8*, WebRtc_Word32 rp);
 
-    ssim_8x8 = Ssim8x8C;
-    if (WebRtc_GetCPUInfo(kSSE2))
-    {
 #if defined(WEBRTC_USE_SSE2)
-        ssim_8x8 = Ssim8x8Sse2;
+    ssim_8x8 = Ssim8x8Sse2;
+#else
+    ssim_8x8 = Ssim8x8C;
 #endif
-    }
 
     // Sample point start with each 4x4 location
     for (i = 0; i < height - 8; i += 4, img1 += stride_img1 * 4,
@@ -279,7 +288,7 @@ SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1,
     return ssim_total;
 }
 
-WebRtc_Word32
+int
 SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
               WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result)
 {
@@ -327,21 +336,29 @@ SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName,
         testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp);
     }
 
-    // SSIM: normalize/average for sequence
+    if (result->frames.size() == 0)
-    ssimScene  = ssimScene / frames;
+    {
-    result->average = ssimScene;
+      fprintf(stderr, "Tried to measure SSIM from empty files (reference "
-
+              "file: %s  test file: %s\n", refFileName, testFileName);
-    // Calculate min/max statistics
+      return -3;
-    std::vector<FrameResult>::iterator element;
+    }
-    element = min_element(result->frames.begin(),
+    else
-                          result->frames.end(), lessForFrameResultValue);
+    {
-    result->min = element->value;
+        // SSIM: normalize/average for sequence
-    result->min_frame_number = element->frame_number;
+        ssimScene  = ssimScene / frames;
-    element = max_element(result->frames.begin(),
+        result->average = ssimScene;
-                          result->frames.end(), lessForFrameResultValue);
-    result->max = element->value;
-    result->max_frame_number = element->frame_number;
 
+        // Calculate min/max statistics
+        std::vector<FrameResult>::iterator element;
+        element = min_element(result->frames.begin(),
+                              result->frames.end(), LessForFrameResultValue);
+        result->min = element->value;
+        result->min_frame_number = element->frame_number;
+        element = max_element(result->frames.begin(),
+                              result->frames.end(), LessForFrameResultValue);
+        result->max = element->value;
+        result->max_frame_number = element->frame_number;
+    }
     delete [] ref;
     delete [] test;
 

test/testsupport/metrics/video_metrics.h

@@ -8,14 +8,13 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-
 #ifndef WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_
 #define WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_
 
-#include "typedefs.h"
 #include <limits>
 #include <vector>
 
+#include "typedefs.h"
 
 // Contains video quality metrics result for a single frame.
 struct FrameResult {
@@ -47,23 +46,35 @@ struct QualityMetricsResult {
 // If the result is std::numerical_limits<double>::max() the videos were
 // equal. Otherwise, PSNR values are in decibel (higher is better). This
 // algorithm only compares up to the point when the shortest video ends.
-WebRtc_Word32
+// By definition of PSNR, the result value is undefined if the reference file
-PsnrFromFiles(const WebRtc_Word8 *refFileName,
+// and the test file are identical. In that case the max value for double
-        const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
+// will be set in the result struct.
-        WebRtc_Word32 height, QualityMetricsResult *result);
+//
-
+// Returns 0 if successful, negative on errors:
+// -1 if the source file cannot be opened
+// -2 if the test file cannot be opened
+// -3 if any of the files are empty
+int PsnrFromFiles(const WebRtc_Word8 *refFileName,
+                  const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
+                  WebRtc_Word32 height, QualityMetricsResult *result);
 
 // SSIM values are filled into the QualityMetricsResult struct.
 // Values range between -1 and 1, where 1 means the files were identical. This
 // algorithm only compares up to the point when the shortest video ends.
-WebRtc_Word32
+// By definition, SSIM values varies from -1.0, when everything is different
-SsimFromFiles(const WebRtc_Word8 *refFileName,
+// between the reference file and the test file, up to 1.0 for two identical
-        const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
+// files.
-        WebRtc_Word32 height, QualityMetricsResult *result);
+//
-
+// Returns 0 if successful, negative on errors:
-double
+// -1 if the source file cannot be opened
-SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1,
+// -2 if the test file cannot be opened
-          WebRtc_Word32 stride_img2, WebRtc_Word32 width, WebRtc_Word32 height);
+// -3 if any of the files are empty
+int SsimFromFiles(const WebRtc_Word8 *refFileName,
+                  const WebRtc_Word8 *testFileName, WebRtc_Word32 width,
+                  WebRtc_Word32 height, QualityMetricsResult *result);
 
+double SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2,
+                 WebRtc_Word32 stride_img1, WebRtc_Word32 stride_img2,
+                 WebRtc_Word32 width, WebRtc_Word32 height);
 
 #endif // WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_

test/testsupport/metrics/video_metrics_unittest.cc

@@ -0,0 +1,97 @@
+/*
+ *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video_metrics.h"
+
+#include <cstdio>
+
+#include "gtest/gtest.h"
+#include "testsupport/fileutils.h"
+
+namespace webrtc {
+
+static const char* kEmptyFileName = "video_metrics_unittest_empty_file.tmp";
+static const char* kNonExistingFileName = "video_metrics_unittest_non_existing";
+static const int kWidth = 352;
+static const int kHeight = 288;
+
+static const int kMissingReferenceFileReturnCode = -1;
+static const int kMissingTestFileReturnCode = -2;
+static const int kEmptyFileReturnCode = -3;
+static const double kPsnrPerfectResult =  std::numeric_limits<double>::max();
+static const double kSsimPerfectResult = 1.0;
+
+class VideoMetricsTest: public testing::Test {
+ protected:
+  VideoMetricsTest() {
+    video_file = webrtc::test::ProjectRootPath() + "resources/foreman_cif.yuv";
+  }
+  virtual ~VideoMetricsTest() {}
+  void SetUp() {
+    // Create an empty file:
+    FILE* dummy = fopen(kEmptyFileName, "wb");
+    fclose(dummy);
+  }
+  void TearDown() {
+    std::remove(kEmptyFileName);
+  }
+  QualityMetricsResult result_;
+  std::string video_file;
+};
+
+// Tests that it is possible to run with the same reference as test file
+TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFiles) {
+  EXPECT_EQ(0, PsnrFromFiles(video_file.c_str(), video_file.c_str(),
+                             kWidth, kHeight, &result_));
+  EXPECT_EQ(kPsnrPerfectResult, result_.average);
+  EXPECT_EQ(SsimFromFiles(video_file.c_str(), video_file.c_str(), kWidth, kHeight,
+                          &result_), 0);
+  EXPECT_EQ(kSsimPerfectResult, result_.average);
+}
+
+// Tests that the right return code is given when the reference file is missing.
+TEST_F(VideoMetricsTest, MissingReferenceFile) {
+  EXPECT_EQ(kMissingReferenceFileReturnCode,
+            PsnrFromFiles(kNonExistingFileName, video_file.c_str(), kWidth,
+                          kHeight, &result_));
+  EXPECT_EQ(kMissingReferenceFileReturnCode,
+            SsimFromFiles(kNonExistingFileName, video_file.c_str(), kWidth,
+                          kHeight, &result_));
+}
+
+// Tests that the right return code is given when the test file is missing.
+TEST_F(VideoMetricsTest, MissingTestFile) {
+  EXPECT_EQ(kMissingTestFileReturnCode,
+            PsnrFromFiles(video_file.c_str(), kNonExistingFileName, kWidth,
+                          kHeight, &result_));
+  EXPECT_EQ(kMissingTestFileReturnCode,
+            SsimFromFiles(video_file.c_str(), kNonExistingFileName, kWidth,
+                          kHeight, &result_));
+}
+
+// Tests that the method can be executed with empty files.
+TEST_F(VideoMetricsTest, EmptyFiles) {
+  EXPECT_EQ(kEmptyFileReturnCode,
+            PsnrFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight,
+                          &result_));
+  EXPECT_EQ(kEmptyFileReturnCode,
+            SsimFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight,
+                          &result_));
+  // Run the same again with the empty file switched.
+  EXPECT_EQ(kEmptyFileReturnCode,
+            PsnrFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight,
+                          &result_));
+  EXPECT_EQ(kEmptyFileReturnCode,
+            SsimFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight,
+                          &result_));
+}
+
+}  // namespace webrtc
+

webrtc.gyp

@@ -21,6 +21,7 @@
         'src/system_wrappers/source/system_wrappers.gyp:*',
         'src/video_engine/video_engine.gyp:*',
         'src/voice_engine/voice_engine.gyp:*',
+        'test/test.gyp:*',
       ],
     },
     # TODO(andrew): move peerconnection to its own gyp.