A tool to do PSNR and SSIM analysis over frames.

This is a very simple tool which takes a test and reference YUV files and compares the frames in them.
The interesting part is that the test video is the video formed by the frames captured from the browser
(WebRTC output). The test video frames has been decoded and the correspondence between evrey frame in the
test video and the decoded barcode, i.e. the frame number in the refrence video has been written to a stats
file in the form frame_xxxx yyyy, where xxxx is the number of the frame in the test video and yyyy is the
number of the frame in the reference video.

We can have jumping over frames or duplicate frames in the test video, as well as incorrectly decoded barcodes.
The tool takes care of these cases.

I haven't used the video_metrics.h because the functions in there seem to do much more than we need and not to
do things that I needed.

The tool may need to be changed so that it could produce output which in turn will be used by PythonCharts or
other chart-drawing tool or library that we decide to use.

BUG=
TEST=
./out/Debug/frame_analyzer --reference_file=reference.yuv --test_file=test.yuv --stats_file=stats.txt --width=352 --height=288

Review URL: https://webrtc-codereview.appspot.com/701005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2620 4adac7df-926f-26a2-2b94-8c16560cd09d

src/test/test.gyp

@@ -97,8 +97,8 @@
       'type': 'executable',
       'dependencies': [
         'test_support',
-        '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
+        '<(DEPTH)/third_party/google-gflags/google-gflags.gyp:google-gflags',
-        '<(webrtc_root)/../third_party/libyuv/libyuv.gyp:libyuv',
+        '<(DEPTH)/third_party/libyuv/libyuv.gyp:libyuv',
       ],
       'sources': [
         'testsupport/converter/converter.h',
@@ -106,5 +106,19 @@
         'testsupport/converter/rgba_to_i420_converter.cc',
       ],
     },
+    {
+      'target_name': 'frame_analyzer',
+      'type': 'executable',
+      'dependencies': [
+        '<(webrtc_root)/test/test.gyp:test_support',
+        '<(DEPTH)/third_party/google-gflags/google-gflags.gyp:google-gflags',
+        '<(DEPTH)/third_party/libyuv/libyuv.gyp:libyuv',
+      ],
+      'sources': [
+        'testsupport/frame_analyzer/frame_analyzer.cc',
+        'testsupport/frame_analyzer/video_quality_analysis.h',
+        'testsupport/frame_analyzer/video_quality_analysis.cc',
+      ],
+    },
   ],
 }

src/test/testsupport/frame_analyzer/frame_analyzer.cc

@@ -0,0 +1,55 @@
+/*
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <cassert>
+#include <cstdio>
+#include <sstream>
+#include <string>
+
+#include "google/gflags.h"
+
+#include "testsupport/frame_analyzer/video_quality_analysis.h"
+
+#define STATS_LINE_LENGTH 25
+
+DEFINE_string(stats_file, "stats.txt", "The full name of the file "
+              "containing the stats after decoding of the received YUV video");
+DEFINE_string(reference_file, "ref.yuv", "The reference YUV file to compare "
+              "against.");
+DEFINE_string(test_file, "test.yuv", "The test YUV file to run the analysis "
+              "for.");
+DEFINE_int32(width, 352, "The width of the refence and test files.");
+DEFINE_int32(height, 288, "The height of the reference and test files.");
+
+
+int main(int argc, char** argv) {
+  std::string program_name = argv[0];
+  std::string usage = "Compares the output video with the initially sent video."
+      "\nRun " + program_name + " --helpshort for usage.\n"
+      "Example usage:\n" + program_name + " --stats_file=stats.txt "
+      "--reference_file=ref.yuv --test_file=test.yuv --width=352 --height=288";
+  google::SetUsageMessage(usage);
+
+  google::ParseCommandLineFlags(&argc, &argv, true);
+
+  fprintf(stdout, "You have entered:\n");
+  fprintf(stdout, "stats_file=%s, reference_file=%s, test_file=%s, width=%d, "
+          "height=%d\n", FLAGS_stats_file.c_str(), FLAGS_reference_file.c_str(),
+          FLAGS_test_file.c_str(), FLAGS_width, FLAGS_height);
+
+  webrtc::test::ResultsContainer results;
+
+  webrtc::test::RunAnalysis(FLAGS_reference_file.c_str(),
+                            FLAGS_test_file.c_str(), FLAGS_stats_file.c_str(),
+                            FLAGS_width, FLAGS_height, &results);
+
+  webrtc::test::PrintAnalysisResults(&results);
+  webrtc::test::PrintMaxRepeatedAndSkippedFrames(FLAGS_stats_file.c_str());
+}

src/test/testsupport/frame_analyzer/video_quality_analysis.cc

@@ -0,0 +1,283 @@
+/*
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/testsupport/frame_analyzer/video_quality_analysis.h"
+
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+
+#define STATS_LINE_LENGTH 32
+
+namespace webrtc {
+namespace test {
+
+unsigned int GetI420FrameSize(unsigned int width, unsigned int height) {
+  unsigned int half_width = (width + 1) >> 1;
+  unsigned int half_height = (height + 1) >> 1;
+
+  unsigned int y_plane = width * height;  // I420 Y plane.
+  unsigned int u_plane = half_width * half_height;  // I420 U plane.
+  unsigned int v_plane = half_width * half_height;  // I420 V plane.
+
+  return y_plane + u_plane + v_plane;
+}
+
+int ExtractFrameSequenceNumber(std::string line) {
+  int space_position = line.find(' ');
+  if (space_position == -1) {
+    return -1;
+  }
+  std::string frame = line.substr(0, space_position);
+
+  int underscore_position = frame.find('_');
+  if (underscore_position == -1) {
+    return -1;
+  }
+  std::string frame_number = frame.substr(underscore_position + 1);
+
+  return strtol(frame_number.c_str(), NULL, 10);
+}
+
+int ExtractDecodedFrameNumber(std::string line) {
+  int space_position = line.find(' ');
+  if (space_position == -1) {
+    return -1;
+  }
+  std::string decoded_number = line.substr(space_position + 1);
+
+  return strtol(decoded_number.c_str(), NULL, 10);
+}
+
+bool IsThereBarcodeError(std::string line) {
+  int barcode_error_position = line.find("Barcode error");
+  if (barcode_error_position != -1) {
+    return true;
+  }
+  return false;
+}
+
+bool GetNextStatsLine(FILE* stats_file, char* line) {
+  int chars = 0;
+  char buf = 0;
+
+  while (buf != '\n') {
+    size_t chars_read = fread(&buf, 1, 1, stats_file);
+    if (chars_read != 1 || feof(stats_file)) {
+      return false;
+    }
+    line[chars] = buf;
+    ++chars;
+  }
+  line[chars-1] = '\0';  // Strip the trailing \n and put end of string.
+  return true;
+}
+
+bool GetNextI420Frame(FILE* input_file, int width, int height,
+                      uint8* result_frame) {
+  unsigned int frame_size = GetI420FrameSize(width, height);
+  bool errors = false;
+
+  size_t bytes_read = fread(result_frame, 1, frame_size, input_file);
+  if (bytes_read != frame_size) {
+    fprintf(stdout, "Error while reading frame from file\n");
+    errors = true;
+  }
+  return !errors;
+}
+
+bool ExtractFrameFromI420(const char* i420_file_name, int width, int height,
+                          int frame_number, uint8* result_frame) {
+  unsigned int frame_size = GetI420FrameSize(width, height);
+  int offset = frame_number * frame_size;  // Calculate offset for the frame.
+  bool errors = false;
+
+  FILE* input_file = fopen(i420_file_name, "rb");
+  if (input_file == NULL) {
+    fprintf(stderr, "Couldn't open input file for reading: %s\n",
+            i420_file_name);
+    return false;
+  }
+
+  // Change stream pointer to new offset.
+  fseek(input_file, offset, SEEK_SET);
+
+  size_t bytes_read = fread(result_frame, 1, frame_size, input_file);
+  if (bytes_read != frame_size &&
+      ferror(input_file)) {
+    fprintf(stdout, "Error while reading frame no %d from file %s\n",
+            frame_number, i420_file_name);
+    errors = true;
+  }
+  fclose(input_file);
+  return !errors;
+}
+
+double CalculateMetrics(VideoAnalysisMetricsType video_metrics_type,
+                        const uint8* ref_frame,  const uint8* test_frame,
+                        int width, int height) {
+  if (!ref_frame || !test_frame)
+    return -1;
+  else if (height < 0 || width < 0)
+    return -1;
+  int half_width = (width + 1) >> 1;
+  int half_height = (height + 1) >> 1;
+  const uint8* src_y_a = ref_frame;
+  const uint8* src_u_a = src_y_a + width * height;
+  const uint8* src_v_a = src_u_a + half_width * half_height;
+  const uint8* src_y_b = test_frame;
+  const uint8* src_u_b = src_y_b + width * height;
+  const uint8* src_v_b = src_u_b + half_width * half_height;
+
+  int stride_y = width;
+  int stride_uv = half_width;
+
+  double result = 0.0;
+
+  switch (video_metrics_type) {
+    case kPSNR:
+      // In the following: stride is determined by width.
+      result = libyuv::I420Psnr(src_y_a, width, src_u_a, half_width,
+                                src_v_a, half_width, src_y_b, width,
+                                src_u_b, half_width, src_v_b, half_width,
+                                width, height);
+      // LibYuv sets the max psnr value to 128, we restrict it to 48.
+      // In case of 0 mse in one frame, 128 can skew the results significantly.
+      result = (result > 48.0) ? 48.0 : result;
+      break;
+    case kSSIM:
+      result = libyuv::I420Ssim(src_y_a, stride_y, src_u_a, stride_uv,
+                                src_v_a, stride_uv, src_y_b, stride_y,
+                                src_u_b, stride_uv, src_v_b, stride_uv,
+                                width, height);
+      break;
+    default:
+      assert(false);
+  }
+
+  return result;
+}
+
+void RunAnalysis(const char* reference_file_name, const char* test_file_name,
+                 const char* stats_file_name, int width, int height,
+                 ResultsContainer* results) {
+  int size = GetI420FrameSize(width, height);
+  FILE* stats_file = fopen(stats_file_name, "r");
+
+  // String buffer for the lines in the stats file.
+  char line[STATS_LINE_LENGTH];
+
+  // Allocate buffers for test and reference frames.
+  uint8* test_frame = new uint8[size];
+  uint8* reference_frame = new uint8[size];
+  int previous_frame_number = -1;
+
+  // While there are entries in the stats file.
+  while (GetNextStatsLine(stats_file, line)) {
+    int extracted_test_frame = ExtractFrameSequenceNumber(line);
+    int decoded_frame_number = ExtractDecodedFrameNumber(line);
+
+    // If there was problem decoding the barcode in this frame or the frame has
+    // been duplicated, continue.
+    if (IsThereBarcodeError(line) ||
+        decoded_frame_number == previous_frame_number) {
+      continue;
+    }
+
+    assert(extracted_test_frame != -1);
+    assert(decoded_frame_number != -1);
+
+    ExtractFrameFromI420(test_file_name, width, height, extracted_test_frame,
+                         test_frame);
+    ExtractFrameFromI420(reference_file_name, width, height,
+                         decoded_frame_number, reference_frame);
+
+    // Calculate the PSNR and SSIM.
+    double result_psnr = CalculateMetrics(kPSNR, reference_frame, test_frame,
+                                          width, height);
+    double result_ssim = CalculateMetrics(kSSIM, reference_frame, test_frame,
+                                          width, height);
+
+    previous_frame_number = decoded_frame_number;
+
+    // Fill in the result struct.
+    AnalysisResult result;
+    result.frame_number = decoded_frame_number;
+    result.psnr_value = result_psnr;
+    result.ssim_value = result_ssim;
+
+    results->frames.push_back(result);
+  }
+
+  // Cleanup.
+  fclose(stats_file);
+  delete[] test_frame;
+  delete[] reference_frame;
+}
+
+void PrintMaxRepeatedAndSkippedFrames(const char* stats_file_name) {
+  FILE* stats_file = fopen(stats_file_name, "r");
+  char line[STATS_LINE_LENGTH];
+
+  int repeated_frames = 1;
+  int max_repeated_frames = 1;
+  int max_skipped_frames = 1;
+  int previous_frame_number = -1;
+
+  while (GetNextStatsLine(stats_file, line)) {
+    int decoded_frame_number = ExtractDecodedFrameNumber(line);
+
+    if (decoded_frame_number == -1) {
+      continue;
+    }
+
+    // Calculate how many frames a cluster of repeated frames contains.
+    if (decoded_frame_number == previous_frame_number) {
+      ++repeated_frames;
+      if (repeated_frames > max_repeated_frames) {
+        max_repeated_frames = repeated_frames;
+      }
+    } else {
+      repeated_frames = 1;
+    }
+
+    // Calculate how much frames have been skipped.
+    if (decoded_frame_number != 0 && previous_frame_number != -1) {
+      int skipped_frames = decoded_frame_number - previous_frame_number - 1;
+      if (skipped_frames > max_skipped_frames) {
+        max_skipped_frames = skipped_frames;
+      }
+    }
+    previous_frame_number = decoded_frame_number;
+  }
+  fprintf(stdout, "Max_repeated:%d Max_skipped:%d\n", max_repeated_frames,
+          max_skipped_frames);
+}
+
+void PrintAnalysisResults(ResultsContainer* results) {
+  std::vector<AnalysisResult>::iterator iter;
+  int frames_counter = 0;
+
+  fprintf(stdout, "BSTATS\n");
+  for (iter = results->frames.begin(); iter != results->frames.end(); ++iter) {
+    ++frames_counter;
+    fprintf(stdout, "%f %f;", iter->psnr_value, iter->ssim_value);
+  }
+  fprintf(stdout, "ESTATS\n");
+  if (frames_counter > 0) {
+    fprintf(stdout, "Unique_frames_count:%d\n", frames_counter);
+  } else {
+    fprintf(stdout, "Unique_frames_count:undef\n");
+  }
+}
+
+}  // namespace test
+}  // namespace webrtc

src/test/testsupport/frame_analyzer/video_quality_analysis.h

@@ -0,0 +1,97 @@
+/*
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_TEST_TESTSUPPORT_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_
+#define WEBRTC_TEST_TESTSUPPORT_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_
+
+#include <string>
+#include <vector>
+
+#include "third_party/libyuv/include/libyuv.h"
+
+namespace webrtc {
+namespace test {
+
+struct AnalysisResult {
+  int frame_number;
+  double psnr_value;
+  double ssim_value;
+};
+
+struct ResultsContainer {
+  std::vector<AnalysisResult> frames;
+};
+
+enum VideoAnalysisMetricsType {kPSNR, kSSIM};
+
+// A function to run the PSNR and SSIM analysis on the test file. The test file
+// comprises the frames that were captured during the quality measurement test.
+// There may be missing or duplicate frames. Also the frames start at a random
+// position in the original video. We should provide a statistics file along
+// with the test video. The stats file contains the connection between the
+// actual frames in the test file and their position in the reference video, so
+// that the analysis could run with the right frames from both videos. The stats
+// file should be in the form 'frame_xxxx yyyy', where xxxx is the consecutive
+// number of the frame in the test video, and yyyy is the equivalent frame in
+// the reference video. The stats file could be produced by
+// tools/barcode_tools/barcode_decoder.py. This script decodes the barcodes
+// integrated in every video and generates the stats file. If three was some
+// problem with the decoding there would be 'Barcode error' instead of yyyy.
+void RunAnalysis(const char* reference_file_name, const char* test_file_name,
+                 const char* stats_file_name, int width, int height,
+                 ResultsContainer* results);
+
+// Compute PSNR or SSIM for an I420 frame (all planes). When we are calculating
+// PSNR values, the max return value (in the case where the test and reference
+// frames are exactly the same) will be 48. In the case of SSIM the max return
+// value will be 1.
+double CalculateMetrics(VideoAnalysisMetricsType video_metrics_type,
+                        const uint8* ref_frame,  const uint8* test_frame,
+                        int width, int height);
+
+// Function to print the result from the analysis.
+void PrintAnalysisResults(ResultsContainer* results);
+
+// Calculates max repeated and skipped frames.
+void PrintMaxRepeatedAndSkippedFrames(const char* stats_file_name);
+
+// Gets the next line from an open stats file.
+bool GetNextStatsLine(FILE* stats_file, char* line);
+
+// Calculates the size of a I420 frame if given the width and height.
+unsigned int GetI420FrameSize(unsigned int width, unsigned int height);
+
+// Converts a string to an int.
+int StringToInt(std::string number);
+
+// Extract the sequence of the frame in the video. I.e. if line is
+// frame_0023 0284, we will get 23.
+int ExtractFrameSequenceNumber(std::string line);
+
+// Checks if there is 'Barcode error' for the given line.
+bool IsThereBarcodeError(std::string line);
+
+// Extract the frame number in the reference video. I.e. if line is
+// frame_0023 0284, we will get 284.
+int ExtractDecodedFrameNumber(std::string line);
+
+// Gets the next frame from an open I420 file.
+bool GetNextI420Frame(FILE* input_file, int width, int height,
+                      uint8* result_frame);
+
+// Extracts an I420 frame at position frame_number from the file.
+bool ExtractFrameFromI420(const char* i420_file_name, int width, int height,
+                          int frame_number, uint8* result_frame);
+
+
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // WEBRTC_TEST_TESTSUPPORT_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_