Making video codecs test framework integration test execute in a reproducable fashion.

Fixed reproducable random behavior in packet_manipulator.h.
Test is now fully reproducable (runs on only one core) so much tighter limits are now set for the SSIM/PSNR values for the encoding/decoding (verified on all platforms)

BUG=
TEST=out/Debug/video_codecs_test_framework_integrationtests in Debug+Release on Linux, Mac, Windows and in Linux Valgrind.

Review URL: https://webrtc-codereview.appspot.com/381005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1649 4adac7df-926f-26a2-2b94-8c16560cd09d

src/modules/video_coding/codecs/test/packet_manipulator.cc

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -22,11 +22,14 @@ PacketManipulatorImpl::PacketManipulatorImpl(PacketReader* packet_reader,
     : packet_reader_(packet_reader),
       config_(config),
       active_burst_packets_(0),
+      critsect_(CriticalSectionWrapper::CreateCriticalSection()),
+      random_seed_(1),
       verbose_(verbose) {
   assert(packet_reader);
 }
 
 PacketManipulatorImpl::~PacketManipulatorImpl() {
+  delete critsect_;
 }
 
 int PacketManipulatorImpl::ManipulatePackets(
@@ -77,8 +80,19 @@ int PacketManipulatorImpl::ManipulatePackets(
   return nbr_packets_dropped;
 }
 
+void PacketManipulatorImpl::InitializeRandomSeed(unsigned int seed) {
+  random_seed_ = seed;
+}
+
 inline double PacketManipulatorImpl::RandomUniform() {
-  return (std::rand() + 1.0)/(RAND_MAX + 1.0);
+  // Use the previous result as new seed before each rand() call. Doing this
+  // it doesn't matter if other threads are calling rand() since we'll always
+  // get the same behavior as long as we're using a fixed initial seed.
+  critsect_->Enter();
+  srand(random_seed_);
+  random_seed_ = std::rand();
+  critsect_->Leave();
+  return (random_seed_ + 1.0)/(RAND_MAX + 1.0);
 }
 
 const char* PacketLossModeToStr(PacketLossMode e) {

src/modules/video_coding/codecs/test/packet_manipulator.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -14,6 +14,7 @@
 #include <cstdlib>
 
 #include "modules/video_coding/codecs/interface/video_codec_interface.h"
+#include "system_wrappers/interface/critical_section_wrapper.h"
 #include "testsupport/packet_reader.h"
 
 namespace webrtc {
@@ -70,8 +71,8 @@ struct NetworkingConfig {
 // when CL 172001 has been submitted. This also requires a correct
 // fragmentation header to be passed to the decoder.
 //
-// To get a deterministic behavior of the packet dropping, initialize the
-// random generator with a fixed value before using this class, e.g. srand(0);
+// To get a repeatable packet drop pattern, re-initialize the random seed
+// using InitializeRandomSeed before each test run.
 class PacketManipulator {
  public:
   virtual ~PacketManipulator() {}
@@ -88,9 +89,11 @@ class PacketManipulator {
 class PacketManipulatorImpl : public PacketManipulator {
  public:
   PacketManipulatorImpl(PacketReader* packet_reader,
-                        const NetworkingConfig& config, bool verbose);
+                        const NetworkingConfig& config,
+                        bool verbose);
   virtual ~PacketManipulatorImpl();
   virtual int ManipulatePackets(webrtc::EncodedImage* encoded_image);
+  virtual void InitializeRandomSeed(unsigned int seed);
  protected:
   // Returns a uniformly distributed random value between 0.0 and 1.0
   virtual double RandomUniform();
@@ -99,6 +102,8 @@ class PacketManipulatorImpl : public PacketManipulator {
   const NetworkingConfig& config_;
   // Used to simulate a burst over several frames.
   int active_burst_packets_;
+  CriticalSectionWrapper* critsect_;
+  unsigned int random_seed_;
   bool verbose_;
 };
 

src/modules/video_coding/codecs/test/packet_manipulator_unittest.cc

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -14,6 +14,7 @@
 
 #include "gtest/gtest.h"
 #include "modules/video_coding/codecs/interface/video_codec_interface.h"
+#include "modules/video_coding/codecs/test/predictive_packet_manipulator.h"
 #include "testsupport/unittest_utils.h"
 #include "typedefs.h"
 
@@ -69,35 +70,6 @@ class PacketManipulatorTest: public PacketRelatedTest {
   }
 };
 
-// Predictive packet manipulator that allows for setup of the result of
-// the random invocations.
-class PredictivePacketManipulatorImpl : public PacketManipulatorImpl {
- public:
-  PredictivePacketManipulatorImpl(PacketReader* packet_reader,
-                                  const NetworkingConfig& config)
-    : PacketManipulatorImpl(packet_reader, config, false) {
-  }
-  // Adds a result. You must add at least the same number of results as the
-  // expected calls to the RandomUniform method. The results are added to a
-  // FIFO queue so they will be returned in the same order they were added.
-  void AddRandomResult(double result) {
-    ASSERT_TRUE(result >= 0.0 || result <= 1.0)
-        << "Cannot add results outside the range 0.0 - 1.0, was:" << result;
-    random_results_.push(result);
-  }
- protected:
-  double RandomUniform() {
-    EXPECT_GT(random_results_.size(), 0u) << "No more stored results, please "
-        "make sure AddRandomResult() is called same amount of times you're "
-        "going to invoke the RandomUniform() function, i.e. once per packet.";
-    double result = random_results_.front();
-    random_results_.pop();
-    return result;
-  }
- private:
-  std::queue<double> random_results_;
-};
-
 TEST_F(PacketManipulatorTest, Constructor) {
   PacketManipulatorImpl manipulator(&packet_reader_, no_drop_config_, false);
 }
@@ -129,7 +101,7 @@ TEST_F(PacketManipulatorTest, UniformDropAll) {
 // Use our customized test class to make the second packet being lost
 TEST_F(PacketManipulatorTest, UniformDropSinglePacket) {
   drop_config_.packet_loss_probability = 0.5;
-  PredictivePacketManipulatorImpl manipulator(&packet_reader_, drop_config_);
+  PredictivePacketManipulator manipulator(&packet_reader_, drop_config_);
   manipulator.AddRandomResult(1.0);
   manipulator.AddRandomResult(0.3);  // less than 0.5 will cause packet loss
   manipulator.AddRandomResult(1.0);
@@ -163,7 +135,7 @@ TEST_F(PacketManipulatorTest, BurstDropNinePackets) {
   drop_config_.packet_loss_probability = 0.5;
   drop_config_.packet_loss_burst_length = 5;
   drop_config_.packet_loss_mode = kBurst;
-  PredictivePacketManipulatorImpl manipulator(&packet_reader_, drop_config_);
+  PredictivePacketManipulator manipulator(&packet_reader_, drop_config_);
   manipulator.AddRandomResult(1.0);
   manipulator.AddRandomResult(0.3);  // less than 0.5 will cause packet loss
   for (int i = 0; i < kNbrPackets - 2; ++i) {

src/modules/video_coding/codecs/test/predictive_packet_manipulator.cc

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/test/predictive_packet_manipulator.h"
+
+#include <cassert>
+#include <cstdio>
+
+#include "testsupport/packet_reader.h"
+
+namespace webrtc {
+namespace test {
+
+PredictivePacketManipulator::PredictivePacketManipulator(
+    PacketReader* packet_reader, const NetworkingConfig& config)
+    : PacketManipulatorImpl(packet_reader, config, false) {
+}
+
+PredictivePacketManipulator::~PredictivePacketManipulator() {
+}
+
+
+void PredictivePacketManipulator::AddRandomResult(double result) {
+  assert(result >= 0.0 && result <= 1.0);
+  random_results_.push(result);
+}
+
+double PredictivePacketManipulator::RandomUniform() {
+  if(random_results_.size() == 0u) {
+    fprintf(stderr, "No more stored results, please make sure AddRandomResult()"
+            "is called same amount of times you're going to invoke the "
+            "RandomUniform() function, i.e. once per packet.\n");
+    assert(false);
+  }
+  double result = random_results_.front();
+  random_results_.pop();
+  return result;
+}
+
+}  // namespace test
+}  // namespace webrtcc

src/modules/video_coding/codecs/test/predictive_packet_manipulator.h

@@ -0,0 +1,45 @@
+/*
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_PREDICTIVE_PACKET_MANIPULATOR_H_
+#define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_PREDICTIVE_PACKET_MANIPULATOR_H_
+
+#include <queue>
+
+#include "modules/video_coding/codecs/test/packet_manipulator.h"
+#include "testsupport/packet_reader.h"
+
+namespace webrtc {
+namespace test {
+
+// Predictive packet manipulator that allows for setup of the result of
+// the random invocations.
+class PredictivePacketManipulator : public PacketManipulatorImpl {
+ public:
+  PredictivePacketManipulator(PacketReader* packet_reader,
+                              const NetworkingConfig& config);
+  virtual ~PredictivePacketManipulator();
+  // Adds a result. You must add at least the same number of results as the
+  // expected calls to the RandomUniform method. The results are added to a
+  // FIFO queue so they will be returned in the same order they were added.
+  // Result parameter must be 0.0 to 1.0.
+  void AddRandomResult(double result);
+ protected:
+  // Returns a uniformly distributed random value between 0.0 and 1.0
+  virtual double RandomUniform();
+
+ private:
+  std::queue<double> random_results_;
+};
+
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_PREDICTIVE_PACKET_MANIPULATOR_H_

src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi

@@ -21,6 +21,8 @@
             'mock/mock_packet_manipulator.h',
             'packet_manipulator.h',
             'packet_manipulator.cc',
+            'predictive_packet_manipulator.h',
+            'predictive_packet_manipulator.cc',
             'stats.h',
             'stats.cc',
             'videoprocessor.h',

src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc

@@ -61,6 +61,8 @@ class VideoProcessorIntegrationTest: public testing::Test {
         "foreman_cif_short_video_codecs_test_framework_integrationtests.yuv";
     config_.frame_length_in_bytes = 3 * kCIFWidth * kCIFHeight / 2;
     config_.verbose = false;
+    // Only allow encoder/decoder to use single core, for predictability.
+    config_.use_single_core = true;
 
     // Get a codec configuration struct and configure it.
     VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_);
@@ -98,11 +100,10 @@ class VideoProcessorIntegrationTest: public testing::Test {
   }
 
   // Processes all frames in the clip and verifies the result.
-  // The average PSNR for all frames is required to be 2.0 higher than the
-  // minimum_psnr parameter.
-  // The minimum SSIM for all frames is required to be 0.1 higher than the
-  // minimum_ssim parameter.
-  void ProcessFramesAndVerify(double minimum_psnr, double minimum_ssim) {
+  void ProcessFramesAndVerify(double minimum_avg_psnr,
+                              double minimum_min_psnr,
+                              double minimum_avg_ssim,
+                              double minimum_min_ssim) {
     int frame_number = 0;
     while (processor_->ProcessFrame(frame_number)) {
       frame_number++;
@@ -125,36 +126,48 @@ class VideoProcessorIntegrationTest: public testing::Test {
         config_.codec_settings->height,
         &psnr_result,
         &ssim_result));
-    EXPECT_GT(psnr_result.average, minimum_psnr + 2.0);
-    EXPECT_GT(psnr_result.min, minimum_psnr);
-    EXPECT_GT(ssim_result.average, minimum_ssim + 0.1);
-    EXPECT_GT(ssim_result.min, minimum_ssim);
+    printf("PSNR avg: %f, min: %f    SSIM avg: %f, min: %f\n",
+           psnr_result.average, psnr_result.min,
+           ssim_result.average, ssim_result.min);
+    EXPECT_GT(psnr_result.average, minimum_avg_psnr);
+    EXPECT_GT(psnr_result.min, minimum_min_psnr);
+    EXPECT_GT(ssim_result.average, minimum_avg_ssim);
+    EXPECT_GT(ssim_result.min, minimum_min_ssim);
   }
 };
 
 // Run with no packet loss. Quality should be very high.
 TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) {
   config_.networking_config.packet_loss_probability = 0;
-  double minimum_psnr = 30;
-  double minimum_ssim = 0.7;
-  ProcessFramesAndVerify(minimum_psnr, minimum_ssim);
+  double minimum_avg_psnr = 36;
+  double minimum_min_psnr = 34;
+  double minimum_avg_ssim = 0.9;
+  double minimum_min_ssim = 0.9;
+  ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
+                         minimum_avg_ssim, minimum_min_ssim);
 }
 
 // Run with 5% packet loss. Quality should be a bit lower.
 // TODO(mflodman): Reenable this once it's not flaky.
-TEST_F(VideoProcessorIntegrationTest, DISABLED_Process5PercentPacketLoss) {
+TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) {
   config_.networking_config.packet_loss_probability = 0.05;
-  double minimum_psnr = 14;
-  double minimum_ssim = 0.3;
-  ProcessFramesAndVerify(minimum_psnr, minimum_ssim);
+  double minimum_avg_psnr = 21;
+  double minimum_min_psnr = 17;
+  double minimum_avg_ssim = 0.6;
+  double minimum_min_ssim = 0.4;
+  ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
+                         minimum_avg_ssim, minimum_min_ssim);
 }
 
 // Run with 10% packet loss. Quality should be even lower.
 TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) {
   config_.networking_config.packet_loss_probability = 0.10;
-  double minimum_psnr = 12;
-  double minimum_ssim = 0.2;
-  ProcessFramesAndVerify(minimum_psnr, minimum_ssim);
+  double minimum_avg_psnr = 19;
+  double minimum_min_psnr = 16;
+  double minimum_avg_ssim = 0.6;
+  double minimum_min_ssim = 0.4;
+  ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
+                         minimum_avg_ssim, minimum_min_ssim);
 }
 
 }  // namespace webrtc

src/modules/video_coding/codecs/test/videoprocessor_unittest.cc

@@ -75,7 +75,7 @@ TEST_F(VideoProcessorTest, Init) {
                                      &frame_writer_mock_,
                                      &packet_manipulator_mock_, config_,
                                      &stats_);
-  video_processor.Init();
+  ASSERT_TRUE(video_processor.Init());
 }
 
 TEST_F(VideoProcessorTest, ProcessFrame) {
@@ -91,7 +91,7 @@ TEST_F(VideoProcessorTest, ProcessFrame) {
                                      &frame_writer_mock_,
                                      &packet_manipulator_mock_, config_,
                                      &stats_);
-  video_processor.Init();
+  ASSERT_TRUE(video_processor.Init());
   video_processor.ProcessFrame(0);
 }
 

src/modules/video_coding/codecs/tools/video_quality_measurement.cc

@@ -13,6 +13,7 @@
 
 #include <cassert>
 #include <cstdio>
+#include <ctime>
 
 #ifndef S_ISDIR  // Not defined in stat.h on Windows.
 #define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR)
@@ -168,12 +169,6 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
   // Check single core flag.
   config->use_single_core = FLAGS_use_single_core;
 
-  // Seed our random function if that flag is enabled. This will force
-  // repeatable behaviour between runs.
-  if (!FLAGS_disable_fixed_random_seed) {
-    srand(0);
-  }
-
   // Get codec specific configuration.
   webrtc::VideoCodingModule::Codec(webrtc::kVideoCodecVP8,
                                    config->codec_settings);
@@ -475,15 +470,21 @@ int main(int argc, char* argv[]) {
 
   webrtc::test::PacketManipulatorImpl packet_manipulator(
       &packet_reader, config.networking_config, config.verbose);
-  webrtc::test::VideoProcessorImpl processor(encoder, decoder,
-                                             &frame_reader,
-                                             &frame_writer,
-                                             &packet_manipulator,
-                                             config, &stats);
-  processor.Init();
+  // By default the packet manipulator is seeded with a fixed random.
+  // If disabled we must generate a new seed.
+  if (FLAGS_disable_fixed_random_seed) {
+    packet_manipulator.InitializeRandomSeed(time(NULL));
+  }
+  webrtc::test::VideoProcessor* processor =
+      new webrtc::test::VideoProcessorImpl(encoder, decoder,
+                                           &frame_reader,
+                                           &frame_writer,
+                                           &packet_manipulator,
+                                           config, &stats);
+  processor->Init();
 
   int frame_number = 0;
-  while (processor.ProcessFrame(frame_number)) {
+  while (processor->ProcessFrame(frame_number)) {
     if (frame_number % 80 == 0) {
       Log("\n");  // make the output a bit nicer.
     }
@@ -517,6 +518,7 @@ int main(int argc, char* argv[]) {
   if (FLAGS_python) {
     PrintPythonOutput(config, stats, ssim_result, psnr_result);
   }
+  delete processor;
   delete encoder;
   delete decoder;
   Log("Quality test finished!");

test/testsupport/unittest_utils.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -46,10 +46,7 @@ class PacketRelatedTest: public testing::Test {
     memcpy(packet_data_pointer_ + kPacketSizeInBytes * 2, packet3_, 1);
   }
   virtual ~PacketRelatedTest() {}
-  void SetUp() {
-    // Initialize the random generator with 0 to get deterministic behavior
-    srand(0);
-  }
+  void SetUp() {}
   void TearDown() {}
 };