From f880f863dd9c0b1347ef185f5cc856f3ad045861 Mon Sep 17 00:00:00 2001
From: "kjellander@webrtc.org"
 <kjellander@webrtc.org@4adac7df-926f-26a2-2b94-8c16560cd09d>
Date: Tue, 10 Sep 2013 12:10:01 +0000
Subject: [PATCH] Convert printing in video quality tests to Chromium's perf
 format.

Add support for --label flag to the frame_analyzer, that
decides what label shall be used for the perf output.

BUG=none
TEST=
Make sure to have zxing and ffmpeg in the PATH.
Create a captured video (from running vie_auto_test custom call)
webrtc/tools/compare_videos.py --ref_video=reference_video.yuv --test_video=captured_output.yuv --frame_analyzer=out/Release/frame_analyzer --label=TEST_VGA
And then inspecting the output that is prefixed with RESULT.

R=phoglund@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/2190005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4714 4adac7df-926f-26a2-2b94-8c16560cd09d
---
 webrtc/tools/compare_videos.py                |  6 +-
 webrtc/tools/frame_analyzer/frame_analyzer.cc | 24 +++---
 .../frame_analyzer/video_quality_analysis.cc  | 45 ++++++++----
 .../frame_analyzer/video_quality_analysis.h   | 17 ++++-
 .../video_quality_analysis_unittest.cc        | 73 +++++++++++++++++++
 webrtc/tools/tools.gyp                        |  2 +
 6 files changed, 135 insertions(+), 32 deletions(-)
 create mode 100644 webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc

diff --git a/webrtc/tools/compare_videos.py b/webrtc/tools/compare_videos.py
index 0e0a4f659..afdd6a467 100755
--- a/webrtc/tools/compare_videos.py
+++ b/webrtc/tools/compare_videos.py
@@ -24,10 +24,13 @@ def _ParseArgs():
   usage = 'usage: %prog [options]'
   parser = optparse.OptionParser(usage=usage)
 
+  parser.add_option('--label', type='string', default="MY_TEST",
+                    help=('Label of the test, used to identify different '
+                          'tests. Default: %default'))
   parser.add_option('--ref_video', type='string',
                     help='Reference video to compare with (YUV).')
   parser.add_option('--test_video', type='string',
-                    help=('Test video to be comared with the reference '
+                    help=('Test video to be compared with the reference '
                           'video (YUV).'))
   parser.add_option('--frame_analyzer', type='string',
                     help='Path to the frame analyzer executable.')
@@ -94,6 +97,7 @@ def main():
   # Run frame analyzer to compare the videos and print output.
   cmd = [
     options.frame_analyzer,
+    '--label=%s' % options.label,
     '--reference_file=%s' % options.ref_video,
     '--test_file=%s' % options.test_video,
     '--stats_file=%s' % options.stats_file,
diff --git a/webrtc/tools/frame_analyzer/frame_analyzer.cc b/webrtc/tools/frame_analyzer/frame_analyzer.cc
index d368379ed..aed956562 100644
--- a/webrtc/tools/frame_analyzer/frame_analyzer.cc
+++ b/webrtc/tools/frame_analyzer/frame_analyzer.cc
@@ -29,19 +29,15 @@
  * where xxxx is the frame number in the test video and yyyy is the
  * corresponding frame number in the original video.
  * The video files should be 1420 YUV videos.
- * The tool prints the result to the standard output in the following format:
- * BSTATS
- * <psnr_value> <ssim_value>; <psnr_value> <ssim_value>; ....
- * ESTATS
- * Unique_frames_count:<value>
- * Max_repeated:<value>
- * Max_skipped<value>
+ * The tool prints the result to standard output in the Chromium perf format:
+ * RESULT <metric>:<label>= <values>
  *
  * The max value for PSNR is 48.0 (between equal frames), as for SSIM it is 1.0.
  *
  * Usage:
- * frame_analyzer --reference_file=<name_of_file> --test_file=<name_of_file>
- * --stats_file=<name_of_file> --width=<frame_width> --height=<frame_height>
+ * frame_analyzer --label=<test_label> --reference_file=<name_of_file>
+ * --test_file=<name_of_file> --stats_file=<name_of_file> --width=<frame_width>
+ * --height=<frame_height>
  */
 int main(int argc, char** argv) {
   std::string program_name = argv[0];
@@ -52,6 +48,8 @@ int main(int argc, char** argv) {
       "  - width(int): The width of the reference and test files. Default: -1\n"
       "  - height(int): The height of the reference and test files. "
       " Default: -1\n"
+      "  - label(string): The label to use for the perf output."
+      " Default: MY_TEST\n"
       "  - stats_file(string): The full name of the file containing the stats"
       " after decoding of the received YUV video. Default: stats.txt\n"
       "  - reference_file(string): The reference YUV file to compare against."
@@ -67,6 +65,7 @@ int main(int argc, char** argv) {
 
   parser.SetFlag("width", "-1");
   parser.SetFlag("height", "-1");
+  parser.SetFlag("label", "MY_TEST");
   parser.SetFlag("stats_file", "stats.txt");
   parser.SetFlag("reference_file", "ref.yuv");
   parser.SetFlag("test_file", "test.yuv");
@@ -93,7 +92,8 @@ int main(int argc, char** argv) {
                             parser.GetFlag("stats_file").c_str(), width, height,
                             &results);
 
-  webrtc::test::PrintAnalysisResults(&results);
-  webrtc::test::PrintMaxRepeatedAndSkippedFrames(
-      parser.GetFlag("stats_file").c_str());
+  std::string label = parser.GetFlag("label");
+  webrtc::test::PrintAnalysisResults(label, &results);
+  webrtc::test::PrintMaxRepeatedAndSkippedFrames(label,
+                                                 parser.GetFlag("stats_file"));
 }
diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis.cc b/webrtc/tools/frame_analyzer/video_quality_analysis.cc
index 14f5ec62d..fe3d83473 100644
--- a/webrtc/tools/frame_analyzer/video_quality_analysis.cc
+++ b/webrtc/tools/frame_analyzer/video_quality_analysis.cc
@@ -230,8 +230,14 @@ void RunAnalysis(const char* reference_file_name, const char* test_file_name,
   delete[] reference_frame;
 }
 
-void PrintMaxRepeatedAndSkippedFrames(const char* stats_file_name) {
-  FILE* stats_file = fopen(stats_file_name, "r");
+void PrintMaxRepeatedAndSkippedFrames(const std::string& label,
+                                      const std::string& stats_file_name) {
+  FILE* stats_file = fopen(stats_file_name.c_str(), "r");
+  if (stats_file == NULL) {
+    fprintf(stderr, "Couldn't open stats file for reading: %s\n",
+            stats_file_name.c_str());
+    return;
+  }
   char line[STATS_LINE_LENGTH];
 
   int repeated_frames = 1;
@@ -265,24 +271,33 @@ void PrintMaxRepeatedAndSkippedFrames(const char* stats_file_name) {
     }
     previous_frame_number = decoded_frame_number;
   }
-  fprintf(stdout, "Max_repeated:%d Max_skipped:%d\n", max_repeated_frames,
+  fprintf(stdout, "RESULT Max_repeated: %s= %d\n", label.c_str(),
+          max_repeated_frames);
+  fprintf(stdout, "RESULT Max_skipped: %s= %d\n", label.c_str(),
           max_skipped_frames);
+  fclose(stats_file);
 }
 
-void PrintAnalysisResults(ResultsContainer* results) {
+void PrintAnalysisResults(const std::string& label, ResultsContainer* results) {
   std::vector<AnalysisResult>::iterator iter;
-  int frames_counter = 0;
 
-  fprintf(stdout, "BSTATS\n");
-  for (iter = results->frames.begin(); iter != results->frames.end(); ++iter) {
-    ++frames_counter;
-    fprintf(stdout, "%f %f;", iter->psnr_value, iter->ssim_value);
-  }
-  fprintf(stdout, "ESTATS\n");
-  if (frames_counter > 0) {
-    fprintf(stdout, "Unique_frames_count:%d\n", frames_counter);
-  } else {
-    fprintf(stdout, "Unique_frames_count:undef\n");
+  fprintf(stdout, "RESULT Unique_frames_count: %s= %ld\n", label.c_str(),
+          results->frames.size());
+
+  if (results->frames.size() > 0u) {
+    fprintf(stdout, "RESULT PSNR: %s= [", label.c_str());
+    for (iter = results->frames.begin(); iter != results->frames.end() - 1;
+         ++iter) {
+      fprintf(stdout, "%f,", iter->psnr_value);
+    }
+    fprintf(stdout, "%f] dB\n", iter->psnr_value);
+
+    fprintf(stdout, "RESULT SSIM: %s= [", label.c_str());
+    for (iter = results->frames.begin(); iter != results->frames.end() - 1;
+         ++iter) {
+      fprintf(stdout, "%f,", iter->ssim_value);
+    }
+    fprintf(stdout, "%f]\n", iter->ssim_value);
   }
 }
 
diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis.h b/webrtc/tools/frame_analyzer/video_quality_analysis.h
index 354fe2f14..e78fa616b 100644
--- a/webrtc/tools/frame_analyzer/video_quality_analysis.h
+++ b/webrtc/tools/frame_analyzer/video_quality_analysis.h
@@ -21,6 +21,11 @@ namespace webrtc {
 namespace test {
 
 struct AnalysisResult {
+  AnalysisResult() {}
+  AnalysisResult(int frame_number, double psnr_value, double ssim_value)
+      : frame_number(frame_number),
+        psnr_value(psnr_value),
+        ssim_value(ssim_value) {}
   int frame_number;
   double psnr_value;
   double ssim_value;
@@ -57,11 +62,15 @@ double CalculateMetrics(VideoAnalysisMetricsType video_metrics_type,
                         const uint8* ref_frame,  const uint8* test_frame,
                         int width, int height);
 
-// Function to print the result from the analysis.
-void PrintAnalysisResults(ResultsContainer* results);
+// Prints the result from the analysis in Chromium performance
+// numbers compatible format to stdout. If the results object contains no frames
+// no output will be written.
+void PrintAnalysisResults(const std::string& label, ResultsContainer* results);
 
-// Calculates max repeated and skipped frames.
-void PrintMaxRepeatedAndSkippedFrames(const char* stats_file_name);
+// Calculates max repeated and skipped frames and prints them to stdout in a
+// format that is compatible with Chromium performance numbers.
+void PrintMaxRepeatedAndSkippedFrames(const std::string& label,
+                                      const std::string& stats_file_name);
 
 // Gets the next line from an open stats file.
 bool GetNextStatsLine(FILE* stats_file, char* line);
diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc b/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc
new file mode 100644
index 000000000..a24d4ae3d
--- /dev/null
+++ b/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc
@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+// This test doesn't actually verify the output since it's just printed
+// to stdout by void functions, but it's still useful as it executes the code.
+
+#include <fstream>
+#include <string>
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/test/testsupport/fileutils.h"
+#include "webrtc/tools/frame_analyzer/video_quality_analysis.h"
+
+namespace webrtc {
+namespace test {
+
+
+TEST(VideoQualityAnalysisTest, PrintAnalysisResultsEmpty) {
+  ResultsContainer result;
+  PrintAnalysisResults("Empty", &result);
+}
+
+TEST(VideoQualityAnalysisTest, PrintAnalysisResultsOneFrame) {
+  ResultsContainer result;
+  result.frames.push_back(AnalysisResult(0, 35.0, 0.9));
+  PrintAnalysisResults("OneFrame", &result);
+}
+
+TEST(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) {
+  ResultsContainer result;
+  result.frames.push_back(AnalysisResult(0, 35.0, 0.9));
+  result.frames.push_back(AnalysisResult(1, 34.0, 0.8));
+  result.frames.push_back(AnalysisResult(2, 33.0, 0.7));
+  PrintAnalysisResults("ThreeFrames", &result);
+}
+
+TEST(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesInvalidFile) {
+  std::string stats_filename = OutputPath() + "non-existing-stats-file.txt";
+  remove(stats_filename.c_str());
+  PrintMaxRepeatedAndSkippedFrames("NonExistingStatsFile", stats_filename);
+}
+
+TEST(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesEmptyStatsFile) {
+  std::string stats_filename = OutputPath() + "empty-stats.txt";
+  std::ofstream stats_file;
+  stats_file.open(stats_filename.c_str());
+  stats_file.close();
+  PrintMaxRepeatedAndSkippedFrames("EmptyStatsFile", stats_filename);
+}
+
+TEST(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) {
+  std::string stats_filename = OutputPath() + "stats.txt";
+  std::ofstream stats_file;
+  stats_file.open(stats_filename.c_str());
+  stats_file << "frame_0001 0100\n";
+  stats_file << "frame_0002 0101\n";
+  stats_file << "frame_0003 0101\n";
+  stats_file << "frame_0004 0106\n";
+  stats_file.close();
+
+  PrintMaxRepeatedAndSkippedFrames("NormalStatsFile", stats_filename);
+}
+
+
+}  // namespace test
+}  // namespace webrtc
diff --git a/webrtc/tools/tools.gyp b/webrtc/tools/tools.gyp
index 04ebd5e1b..386f0257e 100644
--- a/webrtc/tools/tools.gyp
+++ b/webrtc/tools/tools.gyp
@@ -113,6 +113,7 @@
           'type': '<(gtest_target_type)',
           'dependencies': [
             'frame_editing_lib',
+            'video_quality_analysis',
             '<(webrtc_root)/tools/internal_tools.gyp:command_line_parser',
             '<(webrtc_root)/test/test.gyp:test_support_main',
             '<(DEPTH)/testing/gtest.gyp:gtest',
@@ -120,6 +121,7 @@
           'sources': [
             'simple_command_line_parser_unittest.cc',
             'frame_editing/frame_editing_unittest.cc',
+            'frame_analyzer/video_quality_analysis_unittest.cc',
           ],
           # Disable warnings to enable Win64 build, issue 1323.
           'msvs_disabled_warnings': [