Initial port of FullStackTest to new VideoEngine API.

Deferring network loss, delay and such to a later CL.

BUG=1872
R=stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/1756004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4310 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/video_engine/internal/video_receive_stream.cc

@@ -114,7 +114,7 @@ int VideoReceiveStream::FrameSizeChange(unsigned int width, unsigned int height,
 }
 
 int VideoReceiveStream::DeliverFrame(uint8_t* frame, int buffer_size,
-                                     uint32_t time_stamp, int64_t render_time) {
+                                     uint32_t timestamp, int64_t render_time) {
   if (config_.renderer == NULL) {
     return 0;
   }
@@ -123,6 +123,8 @@ int VideoReceiveStream::DeliverFrame(uint8_t* frame, int buffer_size,
   video_frame.CreateEmptyFrame(width_, height_, width_, height_, height_);
   ConvertToI420(kI420, frame, 0, 0, width_, height_, buffer_size,
                 webrtc::kRotateNone, &video_frame);
+  video_frame.set_timestamp(timestamp);
+  video_frame.set_render_time_ms(render_time);
 
   if (config_.post_decode_callback != NULL) {
     config_.post_decode_callback->FrameCallback(&video_frame);
@@ -137,16 +139,22 @@ int VideoReceiveStream::DeliverFrame(uint8_t* frame, int buffer_size,
   return 0;
 }
 
-int VideoReceiveStream::SendPacket(int /*channel*/, const void* packet,
+int VideoReceiveStream::SendPacket(int /*channel*/,
+                                   const void* packet,
                                    int length) {
   assert(length >= 0);
-  return transport_->SendRTP(packet, static_cast<size_t>(length)) ? 0 : -1;
+  bool success = transport_->SendRTP(static_cast<const uint8_t*>(packet),
+                                     static_cast<size_t>(length));
+  return success ? 0 : -1;
 }
 
-int VideoReceiveStream::SendRTCPPacket(int /*channel*/, const void* packet,
+int VideoReceiveStream::SendRTCPPacket(int /*channel*/,
+                                       const void* packet,
                                        int length) {
   assert(length >= 0);
-  return transport_->SendRTCP(packet, static_cast<size_t>(length)) ? 0 : -1;
+  bool success = transport_->SendRTCP(static_cast<const uint8_t*>(packet),
+                                      static_cast<size_t>(length));
+  return success ? 0 : -1;
 }
 }  // internal
 }  // webrtc

webrtc/video_engine/internal/video_receive_stream.h

@@ -49,7 +49,7 @@ class VideoReceiveStream : public newapi::VideoReceiveStream,
   virtual int FrameSizeChange(unsigned int width, unsigned int height,
                               unsigned int /*number_of_streams*/) OVERRIDE;
 
-  virtual int DeliverFrame(uint8_t* frame, int buffer_size, uint32_t time_stamp,
+  virtual int DeliverFrame(uint8_t* frame, int buffer_size, uint32_t timestamp,
                            int64_t render_time) OVERRIDE;
 
   virtual int SendPacket(int /*channel*/, const void* packet, int length)

webrtc/video_engine/internal/video_send_stream.cc

@@ -23,10 +23,9 @@
 namespace webrtc {
 namespace internal {
 
-VideoSendStream::VideoSendStream(
+VideoSendStream::VideoSendStream(newapi::Transport* transport,
-    newapi::Transport* transport,
+                                 webrtc::VideoEngine* video_engine,
-    webrtc::VideoEngine* video_engine,
+                                 const newapi::VideoSendStream::Config& config)
-    const newapi::VideoSendStream::Config& config)
     : transport_(transport), config_(config) {
 
   if (config_.codec.numberOfSimulcastStreams > 0) {
@@ -134,14 +133,18 @@ int VideoSendStream::SendPacket(int /*channel*/,
   // TODO(pbos): Lock these methods and the destructor so it can't be processing
   //             a packet when the destructor has been called.
   assert(length >= 0);
-  return transport_->SendRTP(packet, static_cast<size_t>(length)) ? 0 : -1;
+  bool success = transport_->SendRTP(static_cast<const uint8_t*>(packet),
+                                     static_cast<size_t>(length));
+  return success ? 0 : -1;
 }
 
 int VideoSendStream::SendRTCPPacket(int /*channel*/,
                                     const void* packet,
                                     int length) {
   assert(length >= 0);
-  return transport_->SendRTCP(packet, static_cast<size_t>(length)) ? 0 : -1;
+  bool success = transport_->SendRTCP(static_cast<const uint8_t*>(packet),
+                                      static_cast<size_t>(length));
+  return success ? 0 : -1;
 }
 
 }  // namespace internal

webrtc/video_engine/new_include/transport.h

@@ -13,13 +13,15 @@
 
 #include <stddef.h>
 
+#include "webrtc/typedefs.h"
+
 namespace webrtc {
 namespace newapi {
 
 class Transport {
  public:
-  virtual bool SendRTP(const void* packet, size_t length) = 0;
+  virtual bool SendRTP(const uint8_t* packet, size_t length) = 0;
-  virtual bool SendRTCP(const void* packet, size_t length) = 0;
+  virtual bool SendRTCP(const uint8_t* packet, size_t length) = 0;
 
  protected:
   virtual ~Transport() {}

webrtc/video_engine/test/common/direct_transport.h

@@ -22,11 +22,11 @@ class DirectTransport : public newapi::Transport {
 
   void SetReceiver(newapi::PacketReceiver* receiver) { receiver_ = receiver; }
 
-  bool SendRTP(const void* data, size_t length) OVERRIDE {
+  virtual bool SendRTP(const uint8_t* data, size_t length) OVERRIDE {
     return receiver_->DeliverPacket(data, length);
   }
 
-  bool SendRTCP(const void* data, size_t length) OVERRIDE {
+  virtual bool SendRTCP(const uint8_t* data, size_t length) OVERRIDE {
     return SendRTP(data, length);
   }
 

webrtc/video_engine/test/common/file_capturer.cc

@@ -0,0 +1,56 @@
+/*
+ *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video_engine/test/common/file_capturer.h"
+
+#include <stdio.h>
+
+#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
+
+namespace webrtc {
+namespace test {
+
+YuvFileFrameGenerator* YuvFileFrameGenerator::Create(const char* file,
+                                                     size_t width,
+                                                     size_t height,
+                                                     Clock* clock) {
+  FILE* file_handle = fopen(file, "r");
+  if (file_handle == NULL) {
+    return NULL;
+  }
+  return new YuvFileFrameGenerator(file_handle, width, height, clock);
+}
+
+YuvFileFrameGenerator::YuvFileFrameGenerator(FILE* file,
+                                             size_t width,
+                                             size_t height,
+                                             Clock* clock)
+    : FrameGenerator(width, height, clock), file_(file) {
+  frame_size_ = CalcBufferSize(kI420, width_, height_);
+  frame_buffer_ = new uint8_t[frame_size_];
+}
+
+YuvFileFrameGenerator::~YuvFileFrameGenerator() {
+  fclose(file_);
+  delete[] frame_buffer_;
+}
+
+void YuvFileFrameGenerator::GenerateNextFrame() {
+  size_t count = fread(frame_buffer_, 1, frame_size_, file_);
+  if (count < frame_size_) {
+    rewind(file_);
+    return;
+  }
+
+  ConvertToI420(kI420, frame_buffer_, 0, 0, width_, height_, frame_size_,
+                kRotateNone, &frame_);
+}
+}  // namespace test
+}  // namespace webrtc

webrtc/video_engine/test/common/file_capturer.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef WEBRTC_VIDEO_ENGINE_TEST_COMMON_FILE_CAPTURER_H_
+#define WEBRTC_VIDEO_ENGINE_TEST_COMMON_FILE_CAPTURER_H_
+
+#include <stdio.h>
+
+#include "webrtc/typedefs.h"
+#include "webrtc/video_engine/test/common/frame_generator.h"
+#include "webrtc/video_engine/test/common/video_capturer.h"
+
+namespace webrtc {
+
+class Clock;
+
+namespace newapi {
+class VideoSendStreamInput;
+}  // namespace newapi
+
+namespace test {
+
+class YuvFileFrameGenerator : public FrameGenerator {
+ public:
+  static YuvFileFrameGenerator* Create(const char* file_name,
+                                       size_t width,
+                                       size_t height,
+                                       Clock* clock);
+  virtual ~YuvFileFrameGenerator();
+
+ private:
+  YuvFileFrameGenerator(FILE* file, size_t width, size_t height, Clock* clock);
+  virtual void GenerateNextFrame() OVERRIDE;
+
+  FILE* file_;
+  size_t frame_size_;
+  uint8_t* frame_buffer_;
+};
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // WEBRTC_VIDEO_ENGINE_TEST_COMMON_VIDEO_CAPTURER_H_

webrtc/video_engine/test/common/frame_generator.cc

@@ -14,6 +14,7 @@
 #include <cstring>
 
 #include "webrtc/system_wrappers/interface/clock.h"
+#include "webrtc/video_engine/new_include/video_send_stream.h"
 
 namespace webrtc {
 namespace test {
@@ -25,12 +26,12 @@ FrameGenerator* FrameGenerator::Create(size_t width,
 }
 
 void FrameGenerator::InsertFrame(newapi::VideoSendStreamInput* input) {
-  int64_t time_before = clock_->TimeInMilliseconds();
+  int64_t time_before = clock_->CurrentNtpInMilliseconds();
   frame_.set_render_time_ms(time_before);
 
   GenerateNextFrame();
 
-  int64_t time_after = clock_->TimeInMilliseconds();
+  int64_t time_after = clock_->CurrentNtpInMilliseconds();
   input->PutFrame(frame_, static_cast<uint32_t>(time_after - time_before));
 }
 

webrtc/video_engine/test/common/frame_generator.h

@@ -12,12 +12,15 @@
 
 #include "webrtc/common_video/interface/i420_video_frame.h"
 #include "webrtc/typedefs.h"
-#include "webrtc/video_engine/new_include/video_send_stream.h"
 
 namespace webrtc {
 
 class Clock;
 
+namespace newapi {
+class VideoSendStreamInput;
+}  // newapi
+
 namespace test {
 
 // A set of classes that generate sequences of I420VideoFrames for testing

webrtc/video_engine/test/common/frame_generator_capturer.cc

@@ -10,8 +10,8 @@
 
 #include "webrtc/video_engine/test/common/frame_generator_capturer.h"
 
-#include "webrtc/system_wrappers/interface/clock.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
+#include "webrtc/system_wrappers/interface/event_wrapper.h"
 #include "webrtc/system_wrappers/interface/sleep.h"
 #include "webrtc/system_wrappers/interface/thread_wrapper.h"
 #include "webrtc/video_engine/test/common/frame_generator.h"
@@ -22,10 +22,9 @@ namespace test {
 FrameGeneratorCapturer* FrameGeneratorCapturer::Create(
     newapi::VideoSendStreamInput* input,
     FrameGenerator* frame_generator,
-    int target_fps,
+    int target_fps) {
-    Clock* clock) {
   FrameGeneratorCapturer* capturer =
-      new FrameGeneratorCapturer(input, frame_generator, target_fps, clock);
+      new FrameGeneratorCapturer(input, frame_generator, target_fps);
 
   if (!capturer->Init()) {
     delete capturer;
@@ -38,11 +37,10 @@ FrameGeneratorCapturer* FrameGeneratorCapturer::Create(
 FrameGeneratorCapturer::FrameGeneratorCapturer(
     newapi::VideoSendStreamInput* input,
     FrameGenerator* frame_generator,
-    int target_fps,
+    int target_fps)
-    Clock* clock)
     : VideoCapturer(input),
       sending_(false),
-      clock_(clock),
+      tick_(EventWrapper::Create()),
       lock_(CriticalSectionWrapper::CreateCriticalSection()),
       thread_(NULL),
       frame_generator_(frame_generator),
@@ -55,26 +53,22 @@ FrameGeneratorCapturer::FrameGeneratorCapturer(
 FrameGeneratorCapturer::~FrameGeneratorCapturer() {
   Stop();
 
-  if (thread_ != NULL) {
+  if (thread_.get() != NULL)
-    if (!thread_->Stop()) {
+    thread_->Stop();
-      // TODO(pbos): Log a warning. This will leak a thread.
-    } else {
-      delete thread_;
-    }
-  }
 }
 
 bool FrameGeneratorCapturer::Init() {
-  thread_ = ThreadWrapper::CreateThread(FrameGeneratorCapturer::Run,
+  if (!tick_->StartTimer(true, 1000 / target_fps_))
-                                        this,
+    return false;
-                                        webrtc::kHighPriority,
+  thread_.reset(ThreadWrapper::CreateThread(FrameGeneratorCapturer::Run,
-                                        "FrameGeneratorCapturer");
+                                            this,
-  if (thread_ == NULL)
+                                            webrtc::kHighPriority,
+                                            "FrameGeneratorCapturer"));
+  if (thread_.get() == NULL)
     return false;
   unsigned int thread_id;
   if (!thread_->Start(thread_id)) {
-    delete thread_;
+    thread_.reset();
-    thread_ = NULL;
     return false;
   }
   return true;
@@ -86,29 +80,21 @@ bool FrameGeneratorCapturer::Run(void* obj) {
 }
 
 void FrameGeneratorCapturer::InsertFrame() {
-  int64_t time_start = clock_->TimeInMilliseconds();
-
   {
-    CriticalSectionScoped cs(lock_);
+    CriticalSectionScoped cs(lock_.get());
     if (sending_)
       frame_generator_->InsertFrame(input_);
   }
-
+  tick_->Wait(WEBRTC_EVENT_INFINITE);
-  int64_t remaining_sleep =
-      1000 / target_fps_ - (clock_->TimeInMilliseconds() - time_start);
-
-  if (remaining_sleep > 0) {
-    SleepMs(static_cast<int>(remaining_sleep));
-  }
 }
 
 void FrameGeneratorCapturer::Start() {
-  CriticalSectionScoped cs(lock_);
+  CriticalSectionScoped cs(lock_.get());
   sending_ = true;
 }
 
 void FrameGeneratorCapturer::Stop() {
-  CriticalSectionScoped cs(lock_);
+  CriticalSectionScoped cs(lock_.get());
   sending_ = false;
 }
 }  // test

webrtc/video_engine/test/common/frame_generator_capturer.h

@@ -10,13 +10,14 @@
 #ifndef WEBRTC_VIDEO_ENGINE_TEST_COMMON_FRAME_GENERATOR_CAPTURER_H_
 #define WEBRTC_VIDEO_ENGINE_TEST_COMMON_FRAME_GENERATOR_CAPTURER_H_
 
-#include "webrtc/video_engine/test/common/video_capturer.h"
+#include "webrtc/system_wrappers/interface/scoped_ptr.h"
 #include "webrtc/typedefs.h"
+#include "webrtc/video_engine/test/common/video_capturer.h"
 
 namespace webrtc {
 
-class Clock;
 class CriticalSectionWrapper;
+class EventWrapper;
 class ThreadWrapper;
 
 namespace test {
@@ -27,8 +28,7 @@ class FrameGeneratorCapturer : public VideoCapturer {
  public:
   static FrameGeneratorCapturer* Create(newapi::VideoSendStreamInput* input,
                                         FrameGenerator* frame_generator,
-                                        int target_fps,
+                                        int target_fps);
-                                        Clock* clock);
   virtual ~FrameGeneratorCapturer();
 
   virtual void Start() OVERRIDE;
@@ -37,17 +37,16 @@ class FrameGeneratorCapturer : public VideoCapturer {
  private:
   FrameGeneratorCapturer(newapi::VideoSendStreamInput* input,
                          FrameGenerator* frame_generator,
-                         int target_fps,
+                         int target_fps);
-                         Clock* clock);
   bool Init();
   void InsertFrame();
   static bool Run(void* obj);
 
   bool sending_;
 
-  Clock* clock_;
+  scoped_ptr<EventWrapper> tick_;
-  CriticalSectionWrapper* lock_;
+  scoped_ptr<CriticalSectionWrapper> lock_;
-  ThreadWrapper* thread_;
+  scoped_ptr<ThreadWrapper> thread_;
   FrameGenerator* frame_generator_;
 
   int target_fps_;

webrtc/video_engine/test/common/statistics.cc

@@ -0,0 +1,41 @@
+/*
+ *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "webrtc/video_engine/test/common/statistics.h"
+
+#include <math.h>
+
+namespace webrtc {
+namespace test {
+
+Statistics::Statistics() : sum_(0.0), sum_squared_(0.0), count_(0) {}
+
+void Statistics::AddSample(double sample) {
+  sum_ += sample;
+  sum_squared_ += sample * sample;
+  ++count_;
+}
+
+double Statistics::Mean() const {
+  if (count_ == 0)
+    return 0.0;
+  return sum_ / count_;
+}
+
+double Statistics::Variance() const {
+  if (count_ == 0)
+    return 0.0;
+  return sum_squared_ / count_ - Mean() * Mean();
+}
+
+double Statistics::StandardDeviation() const {
+  return sqrt(Variance());
+}
+}  // namespace test
+}  // namespace webrtc

webrtc/video_engine/test/common/statistics.h

@@ -0,0 +1,36 @@
+/*
+ *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef WEBRTC_VIDEO_ENGINE_TEST_COMMON_STATISTICS_H_
+#define WEBRTC_VIDEO_ENGINE_TEST_COMMON_STATISTICS_H_
+
+#include "webrtc/typedefs.h"
+
+namespace webrtc {
+namespace test {
+
+class Statistics {
+ public:
+  Statistics();
+
+  void AddSample(double sample);
+
+  double Mean() const;
+  double Variance() const;
+  double StandardDeviation() const;
+
+ private:
+  double sum_;
+  double sum_squared_;
+  uint64_t count_;
+};
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // WEBRTC_VIDEO_ENGINE_TEST_COMMON_STATISTICS_H_

webrtc/video_engine/test/common/video_capturer.cc

@@ -10,6 +10,8 @@
 
 #include "webrtc/video_engine/test/common/video_capturer.h"
 
+#include "test/testsupport/fileutils.h"
+#include "webrtc/video_engine/test/common/file_capturer.h"
 #include "webrtc/video_engine/test/common/frame_generator.h"
 #include "webrtc/video_engine/test/common/frame_generator_capturer.h"
 #include "webrtc/video_engine/test/common/vcm_capturer.h"
@@ -43,7 +45,7 @@ VideoCapturer* VideoCapturer::Create(newapi::VideoSendStreamInput* input,
 
   FrameGeneratorCapturer* frame_generator_capturer =
       FrameGeneratorCapturer::Create(
-          input, FrameGenerator::Create(width, height, clock), fps, clock);
+          input, FrameGenerator::Create(width, height, clock), fps);
   if (frame_generator_capturer != NULL) {
     return frame_generator_capturer;
   }

webrtc/video_engine/test/full_stack.cc

@@ -0,0 +1,347 @@
+/*
+ *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include <stdio.h>
+
+#include <deque>
+#include <map>
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "gflags/gflags.h"
+
+#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
+#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
+#include "webrtc/system_wrappers/interface/clock.h"
+#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
+#include "webrtc/system_wrappers/interface/event_wrapper.h"
+#include "webrtc/system_wrappers/interface/scoped_ptr.h"
+#include "webrtc/test/testsupport/fileutils.h"
+#include "webrtc/typedefs.h"
+#include "webrtc/video_engine/new_include/video_engine.h"
+#include "webrtc/video_engine/test/common/direct_transport.h"
+#include "webrtc/video_engine/test/common/file_capturer.h"
+#include "webrtc/video_engine/test/common/frame_generator_capturer.h"
+#include "webrtc/video_engine/test/common/generate_ssrcs.h"
+#include "webrtc/video_engine/test/common/statistics.h"
+#include "webrtc/video_engine/test/common/video_renderer.h"
+
+DEFINE_int32(seconds, 10, "Seconds to run each clip.");
+
+namespace webrtc {
+
+struct FullStackTestParams {
+  const char* test_label;
+  struct {
+    const char* name;
+    size_t width, height, fps;
+  } clip;
+  size_t bitrate;
+  double avg_psnr_threshold;
+  double avg_ssim_threshold;
+};
+
+FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
+                                  {"paris_qcif", 176, 144, 30}, 300, 36.0,
+                                  0.96};
+
+// TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
+FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
+                                   {"foreman_cif", 352, 288, 30}, 700, 0.0,
+                                   0.0};
+
+class FullStackTest : public ::testing::TestWithParam<FullStackTestParams> {
+ protected:
+  std::map<uint32_t, bool> reserved_ssrcs_;
+};
+
+class VideoAnalyzer : public newapi::PacketReceiver,
+                      public newapi::Transport,
+                      public newapi::VideoRenderer,
+                      public newapi::VideoSendStreamInput {
+ public:
+  VideoAnalyzer(newapi::VideoSendStreamInput* input,
+                newapi::Transport* transport,
+                newapi::VideoRenderer* loopback_video,
+                const char* test_label,
+                double avg_psnr_threshold,
+                double avg_ssim_threshold,
+                uint64_t duration_frames)
+      : input_(input),
+        transport_(transport),
+        renderer_(loopback_video),
+        receiver_(NULL),
+        test_label_(test_label),
+        rtp_timestamp_delta_(0),
+        first_send_frame_(NULL),
+        last_render_time_(0),
+        avg_psnr_threshold_(avg_psnr_threshold),
+        avg_ssim_threshold_(avg_ssim_threshold),
+        frames_left_(duration_frames),
+        crit_(CriticalSectionWrapper::CreateCriticalSection()),
+        trigger_(EventWrapper::Create()) {}
+
+  ~VideoAnalyzer() {
+    while (!frames_.empty()) {
+      delete frames_.back();
+      frames_.pop_back();
+    }
+    while (!frame_pool_.empty()) {
+      delete frame_pool_.back();
+      frame_pool_.pop_back();
+    }
+  }
+
+  virtual bool DeliverPacket(const void* packet, size_t length) OVERRIDE {
+    scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
+    RTPHeader header;
+    parser->Parse(
+        static_cast<const uint8_t*>(packet), static_cast<int>(length), &header);
+    {
+      CriticalSectionScoped cs(crit_.get());
+      recv_times_[header.timestamp - rtp_timestamp_delta_] =
+          Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
+    }
+
+    return receiver_->DeliverPacket(packet, length);
+  }
+
+  virtual void PutFrame(const I420VideoFrame& video_frame,
+                        uint32_t delta_capture_ms) OVERRIDE {
+    I420VideoFrame* copy = NULL;
+    {
+      CriticalSectionScoped cs(crit_.get());
+      if (frame_pool_.size() > 0) {
+        copy = frame_pool_.front();
+        frame_pool_.pop_front();
+      }
+    }
+    if (copy == NULL)
+      copy = new I420VideoFrame();
+
+    copy->CopyFrame(video_frame);
+    copy->set_timestamp(copy->render_time_ms() * 90);
+
+    {
+      CriticalSectionScoped cs(crit_.get());
+      if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
+        first_send_frame_ = copy;
+
+      frames_.push_back(copy);
+    }
+
+    input_->PutFrame(video_frame, delta_capture_ms);
+  }
+
+  virtual bool SendRTP(const uint8_t* packet, size_t length) OVERRIDE {
+    scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
+    RTPHeader header;
+    parser->Parse(packet, static_cast<int>(length), &header);
+
+    {
+      CriticalSectionScoped cs(crit_.get());
+      if (rtp_timestamp_delta_ == 0) {
+        rtp_timestamp_delta_ =
+            header.timestamp - first_send_frame_->timestamp();
+        first_send_frame_ = NULL;
+        send_times_[header.timestamp - rtp_timestamp_delta_] =
+            Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
+      }
+    }
+
+    return transport_->SendRTP(packet, length);
+  }
+
+  virtual bool SendRTCP(const uint8_t* packet, size_t length) OVERRIDE {
+    return transport_->SendRTCP(packet, length);
+  }
+
+  virtual void RenderFrame(const I420VideoFrame& video_frame,
+                           int time_to_render_ms) OVERRIDE {
+    uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
+
+    {
+      CriticalSectionScoped cs(crit_.get());
+      while (frames_.front()->timestamp() < send_timestamp) {
+        AddFrameComparison(frames_.front(), &last_rendered_frame_, true);
+        frame_pool_.push_back(frames_.front());
+        frames_.pop_front();
+      }
+
+      I420VideoFrame* reference_frame = frames_.front();
+      frames_.pop_front();
+      assert(reference_frame != NULL);
+      assert(reference_frame->timestamp() == send_timestamp);
+
+      AddFrameComparison(reference_frame, &video_frame, false);
+      frame_pool_.push_back(reference_frame);
+
+      if (--frames_left_ == 0) {
+        PrintResult("psnr", psnr_, " dB");
+        PrintResult("ssim", ssim_, "");
+        PrintResult("sender_time", sender_time_, " ms");
+        PrintResult("receiver_time", receiver_time_, " ms");
+        PrintResult("total_delay_incl_network", end_to_end_, " ms");
+        PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
+        EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
+        EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
+        trigger_->Set();
+      }
+    }
+
+    renderer_->RenderFrame(video_frame, time_to_render_ms);
+    last_rendered_frame_.CopyFrame(video_frame);
+  }
+
+  void Wait() { trigger_->Wait(WEBRTC_EVENT_INFINITE); }
+
+  newapi::VideoSendStreamInput* input_;
+  newapi::Transport* transport_;
+  newapi::VideoRenderer* renderer_;
+  newapi::PacketReceiver* receiver_;
+
+ private:
+  void AddFrameComparison(const I420VideoFrame* reference_frame,
+                          const I420VideoFrame* render,
+                          bool dropped) {
+    int64_t render_time = Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
+    psnr_.AddSample(I420PSNR(reference_frame, render));
+    ssim_.AddSample(I420SSIM(reference_frame, render));
+    if (dropped)
+      return;
+    if (last_render_time_ != 0)
+      rendered_delta_.AddSample(render_time - last_render_time_);
+    last_render_time_ = render_time;
+
+    int64_t input_time = reference_frame->render_time_ms();
+    int64_t send_time = send_times_[reference_frame->timestamp()];
+    send_times_.erase(reference_frame->timestamp());
+    sender_time_.AddSample(send_time - input_time);
+    int64_t recv_time = recv_times_[reference_frame->timestamp()];
+    recv_times_.erase(reference_frame->timestamp());
+    receiver_time_.AddSample(render_time - recv_time);
+    end_to_end_.AddSample(render_time - input_time);
+  }
+
+  void PrintResult(const char* result_type,
+                   test::Statistics stats,
+                   const char* unit) {
+    printf("RESULT %s: %s = {%f, %f}%s\n",
+           result_type,
+           test_label_,
+           stats.Mean(),
+           stats.StandardDeviation(),
+           unit);
+  }
+
+  const char* test_label_;
+  test::Statistics sender_time_;
+  test::Statistics receiver_time_;
+  test::Statistics psnr_;
+  test::Statistics ssim_;
+  test::Statistics end_to_end_;
+  test::Statistics rendered_delta_;
+
+  std::deque<I420VideoFrame*> frames_;
+  std::deque<I420VideoFrame*> frame_pool_;
+  I420VideoFrame last_rendered_frame_;
+  std::map<uint32_t, int64_t> send_times_;
+  std::map<uint32_t, int64_t> recv_times_;
+  uint32_t rtp_timestamp_delta_;
+  I420VideoFrame* first_send_frame_;
+  int64_t last_render_time_;
+  double avg_psnr_threshold_;
+  double avg_ssim_threshold_;
+  uint32_t frames_left_;
+  scoped_ptr<CriticalSectionWrapper> crit_;
+  scoped_ptr<EventWrapper> trigger_;
+};
+
+TEST_P(FullStackTest, NoPacketLoss) {
+  FullStackTestParams params = GetParam();
+
+  scoped_ptr<test::VideoRenderer> local_preview(test::VideoRenderer::Create(
+      "Local Preview", params.clip.width, params.clip.height));
+  scoped_ptr<test::VideoRenderer> loopback_video(test::VideoRenderer::Create(
+      "Loopback Video", params.clip.width, params.clip.height));
+
+  scoped_ptr<newapi::VideoEngine> video_engine(
+      newapi::VideoEngine::Create(newapi::VideoEngineConfig()));
+
+  test::DirectTransport transport(NULL);
+  VideoAnalyzer analyzer(NULL,
+                         &transport,
+                         loopback_video.get(),
+                         params.test_label,
+                         params.avg_psnr_threshold,
+                         params.avg_ssim_threshold,
+                         FLAGS_seconds * params.clip.fps);
+
+  scoped_ptr<newapi::VideoCall> call(video_engine->CreateCall(&analyzer));
+  analyzer.receiver_ = call->Receiver();
+  transport.SetReceiver(&analyzer);
+
+  newapi::VideoSendStream::Config send_config = call->GetDefaultSendConfig();
+  test::GenerateRandomSsrcs(&send_config, &reserved_ssrcs_);
+
+  send_config.local_renderer = local_preview.get();
+
+  // TODO(pbos): static_cast shouldn't be required after mflodman refactors the
+  //             VideoCodec struct.
+  send_config.codec.width = static_cast<uint16_t>(params.clip.width);
+  send_config.codec.height = static_cast<uint16_t>(params.clip.height);
+  send_config.codec.minBitrate = params.bitrate;
+  send_config.codec.startBitrate = params.bitrate;
+  send_config.codec.maxBitrate = params.bitrate;
+
+  newapi::VideoSendStream* send_stream = call->CreateSendStream(send_config);
+  analyzer.input_ = send_stream->Input();
+
+  Clock* test_clock = Clock::GetRealTimeClock();
+
+  scoped_ptr<test::YuvFileFrameGenerator> file_frame_generator(
+      test::YuvFileFrameGenerator::Create(
+          test::ResourcePath(params.clip.name, "yuv").c_str(),
+          params.clip.width,
+          params.clip.height,
+          test_clock));
+  ASSERT_TRUE(file_frame_generator.get() != NULL);
+
+  scoped_ptr<test::FrameGeneratorCapturer> file_capturer(
+      test::FrameGeneratorCapturer::Create(
+          &analyzer, file_frame_generator.get(), params.clip.fps));
+  ASSERT_TRUE(file_capturer.get() != NULL);
+
+  newapi::VideoReceiveStream::Config receive_config =
+      call->GetDefaultReceiveConfig();
+  receive_config.rtp.ssrc = send_config.rtp.ssrcs[0];
+  receive_config.renderer = &analyzer;
+
+  newapi::VideoReceiveStream* receive_stream =
+      call->CreateReceiveStream(receive_config);
+
+  receive_stream->StartReceive();
+  send_stream->StartSend();
+
+  file_capturer->Start();
+
+  analyzer.Wait();
+
+  file_capturer->Stop();
+  send_stream->StopSend();
+  receive_stream->StopReceive();
+
+  call->DestroyReceiveStream(receive_stream);
+  call->DestroySendStream(send_stream);
+}
+
+INSTANTIATE_TEST_CASE_P(FullStack,
+                        FullStackTest,
+                        ::testing::Values(paris_qcif, foreman_cif));
+
+}  // namespace webrtc

webrtc/video_engine/test/loopback.cc

@@ -20,7 +20,6 @@
 #include "webrtc/video_engine/test/common/direct_transport.h"
 #include "webrtc/video_engine/test/common/flags.h"
 #include "webrtc/video_engine/test/common/generate_ssrcs.h"
-#include "webrtc/video_engine/test/common/run_tests.h"
 #include "webrtc/video_engine/test/common/video_capturer.h"
 #include "webrtc/video_engine/test/common/video_renderer.h"
 
@@ -28,7 +27,7 @@ namespace webrtc {
 
 class LoopbackTest : public ::testing::Test {
  protected:
-  std::map<uint32_t, bool> reserved_ssrcs;
+  std::map<uint32_t, bool> reserved_ssrcs_;
 };
 
 TEST_F(LoopbackTest, Test) {
@@ -47,7 +46,7 @@ TEST_F(LoopbackTest, Test) {
   transport.SetReceiver(call->Receiver());
 
   newapi::VideoSendStream::Config send_config = call->GetDefaultSendConfig();
-  test::GenerateRandomSsrcs(&send_config, &reserved_ssrcs);
+  test::GenerateRandomSsrcs(&send_config, &reserved_ssrcs_);
 
   send_config.local_renderer = local_preview.get();
 
@@ -99,10 +98,3 @@ TEST_F(LoopbackTest, Test) {
   call->DestroySendStream(send_stream);
 }
 }  // webrtc
-
-int main(int argc, char* argv[]) {
-  ::testing::InitGoogleTest(&argc, argv);
-  webrtc::test::flags::Init(&argc, &argv);
-
-  return webrtc::test::RunAllTests();
-}

webrtc/video_engine/test/test_main.cc

@@ -0,0 +1,21 @@
+/*
+ *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+#include "webrtc/video_engine/test/common/flags.h"
+#include "webrtc/video_engine/test/common/run_tests.h"
+
+int main(int argc, char* argv[]) {
+  ::testing::InitGoogleTest(&argc, argv);
+  webrtc::test::flags::Init(&argc, &argv);
+
+  return webrtc::test::RunAllTests();
+}

webrtc/video_engine/test/tests.gypi

@@ -12,6 +12,8 @@
       'target_name': 'video_tests_common',
       'type': 'static_library',
       'sources': [
+        'common/file_capturer.cc',
+        'common/file_capturer.h',
         'common/flags.cc',
         'common/flags.h',
         'common/frame_generator.cc',
@@ -30,6 +32,8 @@
         'common/null_platform_renderer.cc',
         'common/run_tests.cc',
         'common/run_tests.h',
+        'common/statistics.cc',
+        'common/statistics.h',
         'common/vcm_capturer.cc',
         'common/vcm_capturer.h',
         'common/video_capturer.cc',
@@ -100,11 +104,26 @@
       'type': 'executable',
       'sources': [
         'loopback.cc',
+        'test_main.cc',
       ],
       'dependencies': [
         '<(DEPTH)/testing/gtest.gyp:gtest',
         'video_tests_common',
       ],
     },
+    {
+      'target_name': 'video_full_stack',
+      'type': 'executable',
+      'sources': [
+        'full_stack.cc',
+        'test_main.cc',
+      ],
+      'dependencies': [
+        '<(DEPTH)/testing/gtest.gyp:gtest',
+        '<(DEPTH)/third_party/google-gflags/google-gflags.gyp:google-gflags',
+        '<(webrtc_root)/test/test.gyp:test_support',
+        'video_tests_common',
+      ],
+    },
   ],
 }