External transport is modified to never drop packets from the first frame.

Refactoring of FrameDropHandler: It now also tracks when frames are leaving the encoder and is being sent to external transport.

Previous 'Sent' state is now renamed to 'Created'.

NOTICE: The test seems to be a little flaky on Linux so it's not ready for buildbots yet. Since this might be caused by unstable production code further investigation should be performed to clear out the flakiness. I will file an issue for this when this CL is submitted (since I don't have any code to refer to before that). Usually the flakiness is caused by a decoded/rendered callback that is left out for the last frame, but I have seen other flaky failures too, which means it's not as simple as ignoring the last frame.
These errors occur even if 400kbps bit rate and 0% PL and 0 delay is configured.

BUG=
TEST=vie_auto_test --automated --gtest_filter="ViEVideoVerificationTest.RunsFullStackWithoutErrors" in Debug+Release on Linux, Mac and Windows.

Review URL: http://webrtc-codereview.appspot.com/339005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1597 4adac7df-926f-26a2-2b94-8c16560cd09d

src/video_engine/test/auto_test/automated/vie_video_verification_test.cc

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -8,6 +8,8 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <vector>
+
 #include "gflags/gflags.h"
 #include "gtest/gtest.h"
 #include "testsupport/fileutils.h"
@@ -30,17 +32,18 @@ class ViEVideoVerificationTest : public testing::Test {
  protected:
   void SetUp() {
     input_file_ = webrtc::test::ResourcePath("paris_qcif", "yuv");
-    local_file_renderer_ = new ViEToFileRenderer();
-    remote_file_renderer_ = new ViEToFileRenderer();
-    SetUpLocalFileRenderer(local_file_renderer_);
-    SetUpRemoteFileRenderer(remote_file_renderer_);
   }
 
   void TearDown() {
     TearDownFileRenderer(local_file_renderer_);
-    delete local_file_renderer_;
     TearDownFileRenderer(remote_file_renderer_);
-    delete remote_file_renderer_;
+  }
+
+  void InitializeFileRenderers() {
+    local_file_renderer_ = new ViEToFileRenderer();
+    remote_file_renderer_ = new ViEToFileRenderer();
+    SetUpLocalFileRenderer(local_file_renderer_);
+    SetUpRemoteFileRenderer(remote_file_renderer_);
   }
 
   void SetUpLocalFileRenderer(ViEToFileRenderer* file_renderer) {
@@ -58,6 +61,7 @@ class ViEVideoVerificationTest : public testing::Test {
   }
 
   void TearDownFileRenderer(ViEToFileRenderer* file_renderer) {
+    assert(file_renderer);
     bool test_failed = ::testing::UnitTest::GetInstance()->
         current_test_info()->result()->Failed();
     if (test_failed) {
@@ -67,6 +71,7 @@ class ViEVideoVerificationTest : public testing::Test {
       // No reason to keep the files if we succeeded.
       file_renderer->DeleteOutputFile();
     }
+    delete file_renderer;
   }
 
   void CompareFiles(const std::string& reference_file,
@@ -115,7 +120,8 @@ class ViEVideoVerificationTest : public testing::Test {
   }
 };
 
-TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
+TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors)  {
+  InitializeFileRenderers();
   ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
                                    local_file_renderer_,
                                    remote_file_renderer_));
@@ -132,6 +138,7 @@ TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
 }
 
 TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors)  {
+  InitializeFileRenderers();
   ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight,
                                 local_file_renderer_,
                                 remote_file_renderer_));
@@ -157,58 +164,63 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors)  {
 // in the encoder. The local and remote file will not be of equal size because
 // of unknown reasons. Tests show that they start at the same frame, which is
 // the important thing when doing frame-to-frame comparison with PSNR/SSIM.
-TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
+TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors)  {
-  // Use our own FrameDropMonitoringRemoteFileRenderer instead of the
-  // ViEToFileRenderer from the test fixture:
-  // TODO(kjellander): Find a better way to reuse this code without duplication.
-  remote_file_renderer_->StopRendering();
-  TearDownFileRenderer(remote_file_renderer_);
-  delete remote_file_renderer_;
-
   FrameDropDetector detector;
+  local_file_renderer_ = new ViEToFileRenderer();
   remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector);
+  SetUpLocalFileRenderer(local_file_renderer_);
   SetUpRemoteFileRenderer(remote_file_renderer_);
 
   // Set a low bit rate so the encoder budget will be tight, causing it to drop
   // frames every now and then.
   const int kBitRateKbps = 50;
-  ViETest::Log("Bit rate: %d kbps.\n", kBitRateKbps);
+  const int kPacketLossPercent = 10;
+  const int kNetworkDelayMs = 100;
+  ViETest::Log("Bit rate     : %5d kbps", kBitRateKbps);
+  ViETest::Log("Packet loss  : %5d %%", kPacketLossPercent);
+  ViETest::Log("Network delay: %5d ms", kNetworkDelayMs);
   tests_.TestFullStack(input_file_, kInputWidth, kInputHeight, kBitRateKbps,
+                       kPacketLossPercent, kNetworkDelayMs,
                        local_file_renderer_, remote_file_renderer_, &detector);
   const std::string reference_file = local_file_renderer_->GetFullOutputPath();
   const std::string output_file = remote_file_renderer_->GetFullOutputPath();
   StopRenderers();
 
-  ASSERT_EQ(detector.GetFramesDroppedAtRenderStep().size(),
+  detector.CalculateResults();
-      detector.GetFramesDroppedAtDecodeStep().size())
-      << "The number of dropped frames on the decode and render are not equal, "
-      "this may be because we have a major problem in the jitter buffer?";
-
   detector.PrintReport();
 
+  if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) !=
+      detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) {
+    detector.PrintDebugDump();
+  }
+
+  ASSERT_EQ(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered),
+      detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded))
+      << "The number of dropped frames on the decode and render steps are not "
+      "equal. This may be because we have a major problem in the buffers of "
+      "the ViEToFileRenderer?";
+
   // We may have dropped frames during the processing, which means the output
   // file does not contain all the frames that are present in the input file.
   // To make the quality measurement correct, we must adjust the output file to
   // that by copying the last successful frame into the place where the dropped
   // frame would be, for all dropped frames.
   const int frame_length_in_bytes = 3 * kInputHeight * kInputWidth / 2;
-  int num_frames = detector.NumberSentFrames();
+  ViETest::Log("Frame length: %d bytes", frame_length_in_bytes);
-  ViETest::Log("Frame length: %d bytes\n", frame_length_in_bytes);
+  std::vector<Frame*> all_frames = detector.GetAllFrames();
-  FixOutputFileForComparison(output_file, num_frames, frame_length_in_bytes,
+  FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames);
-                             detector.GetFramesDroppedAtDecodeStep());
 
   // Verify all sent frames are present in the output file.
   size_t output_file_size = webrtc::test::GetFileSize(output_file);
-  EXPECT_EQ(num_frames,
+  EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes)
-      static_cast<int>(output_file_size / frame_length_in_bytes))
+      << "The output file size is incorrect. It should be equal to the number "
-      << "The output file size is incorrect. It should be equal to the number"
       "of frames multiplied by the frame size. This will likely affect "
       "PSNR/SSIM calculations in a bad way.";
 
   // We are running on a lower bitrate here so we need to settle for somewhat
   // lower PSNR and SSIM values.
-  const double kExpectedMinimumPSNR = 25;
+  const double kExpectedMinimumPSNR = 24;
-  const double kExpectedMinimumSSIM = 0.8;
+  const double kExpectedMinimumSSIM = 0.7;
   CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
                kExpectedMinimumSSIM);
 }

src/video_engine/test/auto_test/interface/tb_external_transport.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -12,8 +12,8 @@
 // tb_external_transport.h
 //
 
-#ifndef WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
+#ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
-#define WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
+#define WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
 
 #include <list>
 
@@ -27,7 +27,38 @@ class ThreadWrapper;
 class ViENetwork;
 }
 
-class TbExternalTransport: public webrtc::Transport
+// Allows to subscribe for callback when a frame is started being sent.
+class SendFrameCallback
+{
+public:
+    // Called once per frame (when a new RTP timestamp is detected) when the
+    // first data packet of the frame is being sent using the
+    // TbExternalTransport.SendPacket method.
+    virtual void FrameSent(unsigned int rtp_timestamp) = 0;
+protected:
+    SendFrameCallback() {}
+    virtual ~SendFrameCallback() {}
+};
+
+// Allows to subscribe for callback when the first packet of a frame is
+// received.
+class ReceiveFrameCallback
+{
+public:
+    // Called once per frame (when a new RTP timestamp is detected)
+    // during the processing of the RTP packet queue in
+    // TbExternalTransport::ViEExternalTransportProcess.
+    virtual void FrameReceived(unsigned int rtp_timestamp) = 0;
+protected:
+    ReceiveFrameCallback() {}
+    virtual ~ReceiveFrameCallback() {}
+};
+
+// External transport implementation for testing purposes.
+// A packet loss probability must be set in order to drop packets from the data
+// being sent to this class.
+// Will never drop packets from the first frame of a video sequence.
+class TbExternalTransport : public webrtc::Transport
 {
 public:
     TbExternalTransport(webrtc::ViENetwork& vieNetwork);
@@ -36,7 +67,17 @@ public:
     virtual int SendPacket(int channel, const void *data, int len);
     virtual int SendRTCPPacket(int channel, const void *data, int len);
 
-    WebRtc_Word32 SetPacketLoss(WebRtc_Word32 lossRate); // Rate in %
+    // Should only be called before/after traffic is being processed.
+    // Only one observer can be set (multiple calls will overwrite each other).
+    virtual void RegisterSendFrameCallback(SendFrameCallback* callback);
+
+    // Should only be called before/after traffic is being processed.
+    // Only one observer can be set (multiple calls will overwrite each other).
+    virtual void RegisterReceiveFrameCallback(ReceiveFrameCallback* callback);
+
+    // The probability of a packet of being dropped. Packets belonging to the
+    // first packet (same RTP timestamp) will never be dropped.
+    WebRtc_Word32 SetPacketLoss(WebRtc_Word32 lossRate);  // Rate in %
     void SetNetworkDelay(WebRtc_Word64 delayMs);
     void SetSSRCFilter(WebRtc_UWord32 SSRC);
 
@@ -89,6 +130,9 @@ private:
     std::list<VideoPacket*> _rtpPackets;
     std::list<VideoPacket*> _rtcpPackets;
 
+    SendFrameCallback* _send_frame_callback;
+    ReceiveFrameCallback* _receive_frame_callback;
+
     unsigned char _temporalLayers;
     unsigned short _seqNum;
     unsigned short _sendPID;
@@ -103,6 +147,13 @@ private:
     WebRtc_UWord32 _SSRC;
     bool _checkSequenceNumber;
     WebRtc_UWord16 _firstSequenceNumber;
+
+    // Keep track of the first RTP timestamp so we don't do packet loss on
+    // the first frame.
+    WebRtc_UWord32 _firstRTPTimestamp;
+    // Track RTP timestamps so we invoke callbacks properly (if registered).
+    WebRtc_UWord32 _lastSendRTPTimestamp;
+    WebRtc_UWord32 _lastReceiveRTPTimestamp;
 };
 
-#endif  // WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
+#endif  // WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_

src/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -58,6 +58,8 @@ class ViEFileBasedComparisonTests {
       int width,
       int height,
       int bit_rate_kbps,
+      int packet_loss_percent,
+      int network_delay_ms,
       ViEToFileRenderer* local_file_renderer,
       ViEToFileRenderer* remote_file_renderer,
       FrameDropDetector* frame_drop_detector);

src/video_engine/test/auto_test/primitives/framedrop_primitives.cc

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -11,49 +11,85 @@
 #include <cassert>
 #include <string>
 
-#include "framedrop_primitives.h"
+#include "modules/video_capture/main/interface/video_capture_factory.h"
-
-#include "general_primitives.h"
 #include "system_wrappers/interface/tick_util.h"
-#include "tb_interfaces.h"
 #include "testsupport/fileutils.h"
 #include "testsupport/frame_reader.h"
 #include "testsupport/frame_writer.h"
-#include "video_capture_factory.h"
+#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h"
-#include "vie_autotest.h"
+#include "video_engine/test/auto_test/interface/tb_interfaces.h"
-#include "vie_autotest_defines.h"
+#include "video_engine/test/auto_test/interface/tb_external_transport.h"
-#include "vie_to_file_renderer.h"
+#include "video_engine/test/auto_test/interface/vie_autotest.h"
+#include "video_engine/test/auto_test/interface/vie_autotest_defines.h"
+#include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
+#include "video_engine/test/auto_test/primitives/general_primitives.h"
 
-// Tracks which frames are sent on the local side and reports them to the
+// Tracks which frames are created on the local side and reports them to the
 // FrameDropDetector class.
-class SendTimestampEffectFilter: public webrtc::ViEEffectFilter {
+class CreatedTimestampEffectFilter : public webrtc::ViEEffectFilter {
  public:
-  explicit SendTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
+  explicit CreatedTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
       : frame_drop_detector_(frame_drop_detector) {}
-  virtual ~SendTimestampEffectFilter() {}
+  virtual ~CreatedTimestampEffectFilter() {}
   virtual int Transform(int size, unsigned char* frameBuffer,
                         unsigned int timeStamp90KHz, unsigned int width,
                         unsigned int height) {
-    frame_drop_detector_->ReportSent(timeStamp90KHz);
+    frame_drop_detector_->ReportFrameState(FrameDropDetector::kCreated,
+                                           timeStamp90KHz);
     return 0;
   }
+
+ private:
+  FrameDropDetector* frame_drop_detector_;
+};
+
+// Tracks which frames are sent in external transport on the local side
+// and reports them to the FrameDropDetector class.
+class FrameSentCallback : public SendFrameCallback {
+ public:
+  explicit FrameSentCallback(FrameDropDetector* frame_drop_detector)
+      : frame_drop_detector_(frame_drop_detector) {}
+  virtual ~FrameSentCallback() {}
+  virtual void FrameSent(unsigned int rtp_timestamp) {
+    frame_drop_detector_->ReportFrameState(FrameDropDetector::kSent,
+                                           rtp_timestamp);
+  }
+
+ private:
+  FrameDropDetector* frame_drop_detector_;
+};
+
+// Tracks which frames are received in external transport on the remote side
+// and reports them to the FrameDropDetector class.
+class FrameReceivedCallback : public ReceiveFrameCallback {
+ public:
+  explicit FrameReceivedCallback(FrameDropDetector* frame_drop_detector)
+      : frame_drop_detector_(frame_drop_detector) {}
+  virtual ~FrameReceivedCallback() {}
+  virtual void FrameReceived(unsigned int rtp_timestamp) {
+    frame_drop_detector_->ReportFrameState(FrameDropDetector::kReceived,
+                                           rtp_timestamp);
+  }
+
  private:
   FrameDropDetector* frame_drop_detector_;
 };
 
 // Tracks when frames are decoded on the remote side (received from the
 // jitter buffer) and reports them to the FrameDropDetector class.
-class DecodeTimestampEffectFilter: public webrtc::ViEEffectFilter {
+class DecodedTimestampEffectFilter : public webrtc::ViEEffectFilter {
  public:
-  explicit DecodeTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
+  explicit DecodedTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
       : frame_drop_detector_(frame_drop_detector) {}
-  virtual ~DecodeTimestampEffectFilter() {}
+  virtual ~DecodedTimestampEffectFilter() {}
   virtual int Transform(int size, unsigned char* frameBuffer,
                         unsigned int timeStamp90KHz, unsigned int width,
                         unsigned int height) {
-    frame_drop_detector_->ReportDecoded(timeStamp90KHz);
+    frame_drop_detector_->ReportFrameState(FrameDropDetector::kDecoded,
+                                           timeStamp90KHz);
     return 0;
   }
+
  private:
   FrameDropDetector* frame_drop_detector_;
 };
@@ -64,6 +100,8 @@ void TestFullStack(const TbInterfaces& interfaces,
                    int width,
                    int height,
                    int bit_rate_kbps,
+                   int packet_loss_percent,
+                   int network_delay_ms,
                    FrameDropDetector* frame_drop_detector) {
   webrtc::VideoEngine *video_engine_interface = interfaces.video_engine;
   webrtc::ViEBase *base_interface = interfaces.base;
@@ -86,12 +124,20 @@ void TestFullStack(const TbInterfaces& interfaces,
 
     EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec));
   }
-  const char *ip_address = "127.0.0.1";
+
-  const unsigned short rtp_port = 6000;
+  // Configure External transport to simulate network interference:
-  EXPECT_EQ(0, network_interface->SetLocalReceiver(video_channel, rtp_port));
+  TbExternalTransport external_transport(*interfaces.network);
+  external_transport.SetPacketLoss(packet_loss_percent);
+  external_transport.SetNetworkDelay(network_delay_ms);
+
+  FrameSentCallback frame_sent_callback(frame_drop_detector);
+  FrameReceivedCallback frame_received_callback(frame_drop_detector);
+  external_transport.RegisterSendFrameCallback(&frame_sent_callback);
+  external_transport.RegisterReceiveFrameCallback(&frame_received_callback);
+  EXPECT_EQ(0, network_interface->RegisterSendTransport(video_channel,
+                                                        external_transport));
   EXPECT_EQ(0, base_interface->StartReceive(video_channel));
-  EXPECT_EQ(0, network_interface->SetSendDestination(video_channel, ip_address,
+
-                                                     rtp_port));
   // Setup only the VP8 codec, which is what we'll use.
   webrtc::VideoCodec codec;
   EXPECT_TRUE(FindSpecificCodec(webrtc::kVideoCodecVP8, codec_interface,
@@ -107,27 +153,37 @@ void TestFullStack(const TbInterfaces& interfaces,
   EXPECT_TRUE(image_process);
 
   // Setup the effect filters
-  DecodeTimestampEffectFilter decode_filter(frame_drop_detector);
+  CreatedTimestampEffectFilter create_filter(frame_drop_detector);
+  EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel,
+                                                       create_filter));
+  DecodedTimestampEffectFilter decode_filter(frame_drop_detector);
   EXPECT_EQ(0, image_process->RegisterRenderEffectFilter(video_channel,
                                                          decode_filter));
-  SendTimestampEffectFilter send_filter(frame_drop_detector);
-  EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel,
-                                                       send_filter));
   // Send video.
   EXPECT_EQ(0, base_interface->StartSend(video_channel));
   AutoTestSleep(KAutoTestSleepTimeMs);
 
   // Cleanup.
-  EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel));
   EXPECT_EQ(0, image_process->DeregisterSendEffectFilter(video_channel));
+  EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel));
   image_process->Release();
   ViETest::Log("Done!");
 
+  WebRtc_Word32 num_rtp_packets = 0;
+  WebRtc_Word32 num_dropped_packets = 0;
+  WebRtc_Word32 num_rtcp_packets = 0;
+  external_transport.GetStats(num_rtp_packets, num_dropped_packets,
+                              num_rtcp_packets);
+  ViETest::Log("RTP packets    : %5d", num_rtp_packets);
+  ViETest::Log("Dropped packets: %5d", num_dropped_packets);
+  ViETest::Log("RTCP packets   : %5d", num_rtcp_packets);
+
   // ***************************************************************
   // Testing finished. Tear down Video Engine
   // ***************************************************************
   EXPECT_EQ(0, base_interface->StopSend(video_channel));
   EXPECT_EQ(0, base_interface->StopReceive(video_channel));
+  EXPECT_EQ(0, network_interface->DeregisterSendTransport(video_channel));
   EXPECT_EQ(0, render_interface->StopRender(capture_id));
   EXPECT_EQ(0, render_interface->StopRender(video_channel));
   EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id));
@@ -137,14 +193,8 @@ void TestFullStack(const TbInterfaces& interfaces,
 }
 
 void FixOutputFileForComparison(const std::string& output_file,
-                                int total_number_of_frames,
                                 int frame_length_in_bytes,
-                                std::list<Frame*> dropped_frames) {
+                                const std::vector<Frame*>& frames) {
-  if (dropped_frames.size() == 0) {
-    // No need to modify if no frames are dropped, since the file is already
-    // frame-per-frame in sync in that case.
-    return;
-  }
   webrtc::test::FrameReaderImpl frame_reader(output_file,
                                              frame_length_in_bytes);
   const std::string temp_file = output_file + ".fixed";
@@ -152,159 +202,308 @@ void FixOutputFileForComparison(const std::string& output_file,
   frame_reader.Init();
   frame_writer.Init();
 
-  // Assume the dropped_frames list is sorted by frame number.
+  ASSERT_FALSE(frames.front()->dropped_at_render) << "It should not be "
-  int next_dropped_frame = dropped_frames.front()->number_;
+      "possible to drop the first frame. Both because we don't have anything "
-  dropped_frames.pop_front();
+      "useful to fill that gap with and it is impossible to detect it without "
-  ASSERT_NE(0, next_dropped_frame) << "It should not be possible to drop the "
+      "any previous timestamps to compare with.";
-      "first frame. Both because we don't have anything useful to fill that "
-      "gap with and it is impossible to detect it without any previous "
-      "timestamps to compare with.";
 
-  WebRtc_UWord8* last_read_frame_data =
+  WebRtc_UWord8* last_frame_data = new WebRtc_UWord8[frame_length_in_bytes];
-      new WebRtc_UWord8[frame_length_in_bytes];
-
-  // Write the first frame now since it will always be the same.
-  EXPECT_TRUE(frame_reader.ReadFrame(last_read_frame_data));
-  EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data));
 
   // Process the file and write frame duplicates for all dropped frames.
-  for (int i = 1; i < total_number_of_frames; ++i) {
+  for (std::vector<Frame*>::const_iterator it = frames.begin();
-    if (i == next_dropped_frame) {
+       it != frames.end(); ++it) {
+    if ((*it)->dropped_at_render) {
       // Write the previous frame to the output file:
-      EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data));
+      EXPECT_TRUE(frame_writer.WriteFrame(last_frame_data));
-      if (!dropped_frames.empty()) {
-        next_dropped_frame = dropped_frames.front()->number_;
-        dropped_frames.pop_front();
-      }
     } else {
-      // Read a new frame and write it to the output file.
+      EXPECT_TRUE(frame_reader.ReadFrame(last_frame_data));
-      EXPECT_TRUE(frame_reader.ReadFrame(last_read_frame_data));
+      EXPECT_TRUE(frame_writer.WriteFrame(last_frame_data));
-      EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data));
     }
   }
-  delete[] last_read_frame_data;
+  delete[] last_frame_data;
   frame_reader.Close();
   frame_writer.Close();
   ASSERT_EQ(0, std::remove(output_file.c_str()));
   ASSERT_EQ(0, std::rename(temp_file.c_str(), output_file.c_str()));
 }
 
-void FrameDropDetector::ReportSent(unsigned int timestamp) {
+void FrameDropDetector::ReportFrameState(State state, unsigned int timestamp) {
-  int number = sent_frames_list_.size();
+  dirty_ = true;
-  Frame* frame = new Frame(number, timestamp);
+  switch (state) {
-  frame->sent_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp();
+    case kCreated: {
-  sent_frames_list_.push_back(frame);
+      int number = created_frames_vector_.size();
-  sent_frames_[timestamp] = frame;
+      Frame* frame = new Frame(number, timestamp);
-}
+      frame->created_timestamp_in_us_ =
-
+          webrtc::TickTime::MicrosecondTimestamp();
-void FrameDropDetector::ReportDecoded(unsigned int timestamp) {
+      created_frames_vector_.push_back(frame);
-  // When the first sent frame arrives we calculate the fixed difference
+      created_frames_[timestamp] = frame;
-  // between the timestamps of the sent frames and the decoded/rendered frames.
+      num_created_frames_++;
-  // This diff is then used to identify the frames from the sent_frames_ map.
+      break;
-  if (frame_timestamp_diff_ == 0) {
+    }
-    Frame* first_sent_frame = sent_frames_list_.front();
+    case kSent:
-    frame_timestamp_diff_ = timestamp - first_sent_frame->frame_timestamp_;
+      sent_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
+      if (timestamp_diff_ == 0) {
+        // When the first created frame arrives we calculate the fixed
+        // difference between the timestamps of the frames entering and leaving
+        // the encoder. This diff is used to identify the frames from the
+        // created_frames_ map.
+        timestamp_diff_ =
+            timestamp - created_frames_vector_.front()->frame_timestamp_;
+      }
+      num_sent_frames_++;
+      break;
+    case kReceived:
+      received_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
+      num_received_frames_++;
+      break;
+    case kDecoded:
+      decoded_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
+      num_decoded_frames_++;
+      break;
+    case kRendered:
+      rendered_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
+      num_rendered_frames_++;
+      break;
+    default:
+      assert(false);
   }
-  // Calculate the sent timestamp required to identify the frame:
-  unsigned int sent_timestamp = timestamp - frame_timestamp_diff_;
-
-  // Find the right Frame object in the map of sent frames:
-  Frame* frame = sent_frames_[sent_timestamp];
-  frame->decoded_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp();
-  decoded_frames_[sent_timestamp] = frame;
 }
 
-void FrameDropDetector::ReportRendered(unsigned int timestamp) {
+void FrameDropDetector::CalculateResults() {
-  // Calculate the sent timestamp required to identify the frame:
+  // Fill in all fields of the Frame objects in the created_frames_ map.
-  unsigned int sent_timestamp = timestamp - frame_timestamp_diff_;
+  // Iterate over the maps from converted timestamps to the arrival timestamps.
-
+  std::map<unsigned int, int64_t>::const_iterator it;
-  // Find this frame in the map of sent frames:
+  for (it = sent_frames_.begin(); it != sent_frames_.end(); ++it) {
-  Frame* frame = sent_frames_[sent_timestamp];
+    int created_timestamp = it->first - timestamp_diff_;
-  frame->rendered_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp();
+    created_frames_[created_timestamp]->sent_timestamp_in_us_ = it->second;
-  rendered_frames_[sent_timestamp] = frame;
+  }
-}
+  for (it = received_frames_.begin(); it != received_frames_.end(); ++it) {
-
+    int created_timestamp = it->first - timestamp_diff_;
-int FrameDropDetector::NumberSentFrames() {
+    created_frames_[created_timestamp]->received_timestamp_in_us_ = it->second;
-  return static_cast<int>(sent_frames_.size());
+  }
+  for (it = decoded_frames_.begin(); it != decoded_frames_.end(); ++it) {
+    int created_timestamp = it->first - timestamp_diff_;
+    created_frames_[created_timestamp]->decoded_timestamp_in_us_ =it->second;
+  }
+  for (it = rendered_frames_.begin(); it != rendered_frames_.end(); ++it) {
+    int created_timestamp = it->first - timestamp_diff_;
+    created_frames_[created_timestamp]->rendered_timestamp_in_us_ = it->second;
+  }
+  // Find out where the frames were not present in the different states.
+  dropped_frames_at_send_ = 0;
+  dropped_frames_at_receive_ = 0;
+  dropped_frames_at_decode_ = 0;
+  dropped_frames_at_render_ = 0;
+  for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
+       it != created_frames_vector_.end(); ++it) {
+    int encoded_timestamp = (*it)->frame_timestamp_ + timestamp_diff_;
+    if (sent_frames_.find(encoded_timestamp) == sent_frames_.end()) {
+      (*it)->dropped_at_send = true;
+      dropped_frames_at_send_++;
+    }
+    if (received_frames_.find(encoded_timestamp) == received_frames_.end()) {
+      (*it)->dropped_at_receive = true;
+      dropped_frames_at_receive_++;
+    }
+    if (decoded_frames_.find(encoded_timestamp) == decoded_frames_.end()) {
+      (*it)->dropped_at_decode = true;
+      dropped_frames_at_decode_++;
+    }
+    if (rendered_frames_.find(encoded_timestamp) == rendered_frames_.end()) {
+      (*it)->dropped_at_render = true;
+      dropped_frames_at_render_++;
+    }
+  }
+  dirty_ = false;
 }
 
 void FrameDropDetector::PrintReport() {
+  assert(!dirty_);
   ViETest::Log("Frame Drop Detector report:");
-  ViETest::Log("Sent     frames: %ld", sent_frames_.size());
+  ViETest::Log("  Created  frames: %ld", created_frames_.size());
-  ViETest::Log("Decoded  frames: %ld", decoded_frames_.size());
+  ViETest::Log("  Sent     frames: %ld", sent_frames_.size());
-  ViETest::Log("Rendered frames: %ld", rendered_frames_.size());
+  ViETest::Log("  Received frames: %ld", received_frames_.size());
+  ViETest::Log("  Decoded  frames: %ld", decoded_frames_.size());
+  ViETest::Log("  Rendered frames: %ld", rendered_frames_.size());
 
   // Display all frames and stats for them:
+  long last_created = 0;
   long last_sent = 0;
+  long last_received = 0;
   long last_decoded = 0;
   long last_rendered = 0;
-  ViETest::Log("Sent frames summary:");
+  ViETest::Log("\nDeltas between sent frames and drop status:");
-  ViETest::Log("Deltas are in microseconds and only cover existing frames.");
+  ViETest::Log("Unit: Microseconds");
-  ViETest::Log("Frame no  SentDelta  DecodedDelta  RenderedDelta  DecodedDrop? "
+  ViETest::Log("Frame  Created    Sent    Received Decoded Rendered "
-      "RenderedDrop?");
+      "Dropped at  Dropped at  Dropped at  Dropped at");
-  for (std::list<Frame*>::iterator it = sent_frames_list_.begin();
+  ViETest::Log(" nbr    delta     delta    delta    delta   delta   "
-       it != sent_frames_list_.end(); it++) {
+      " Send?       Receive?    Decode?     Render?");
-    bool dropped_decode = (decoded_frames_.find((*it)->frame_timestamp_) ==
+  for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
-        decoded_frames_.end());
+       it != created_frames_vector_.end(); ++it) {
-    bool dropped_render = (rendered_frames_.find((*it)->frame_timestamp_) ==
+    int created_delta =
-        rendered_frames_.end());
+        static_cast<int>((*it)->created_timestamp_in_us_ - last_created);
-    int sent_delta = static_cast<int>((*it)->sent_timestamp_in_us_ - last_sent);
+    int sent_delta = (*it)->dropped_at_send ? -1 :
-    int decoded_delta = dropped_decode ? 0 :
+        static_cast<int>((*it)->sent_timestamp_in_us_ - last_sent);
+    int received_delta = (*it)->dropped_at_receive ? -1 :
+        static_cast<int>((*it)->received_timestamp_in_us_ - last_received);
+    int decoded_delta = (*it)->dropped_at_decode ? -1 :
         static_cast<int>((*it)->decoded_timestamp_in_us_ - last_decoded);
-    int rendered_delta = dropped_render ? 0 :
+    int rendered_delta = (*it)->dropped_at_render ? -1 :
         static_cast<int>((*it)->rendered_timestamp_in_us_ - last_rendered);
 
-    // Set values to 0 for the first frame:
+    // Set values to -1 for the first frame:
     if ((*it)->number_ == 0) {
-      sent_delta = 0;
+      created_delta = -1;
-      decoded_delta = 0;
+      sent_delta = -1;
-      rendered_delta = 0;
+      received_delta = -1;
+      decoded_delta = -1;
+      rendered_delta = -1;
     }
-    ViETest::Log("%8d %10d    %10d     %10d    %s     %s", (*it)->number_,
+    ViETest::Log("%5d %8d %8d %8d %8d %8d %10s %10s %10s %10s",
-           sent_delta, decoded_delta, rendered_delta,
+                 (*it)->number_,
-           dropped_decode ? "DROPPED" : "      ",
+                 created_delta,
-           dropped_render ? "DROPPED" : "      ");
+                 sent_delta,
-    last_sent = (*it)->sent_timestamp_in_us_;
+                 received_delta,
-    if (!dropped_render) {
+                 decoded_delta,
+                 rendered_delta,
+                 (*it)->dropped_at_send ? "DROPPED" : "      ",
+                 (*it)->dropped_at_receive ? "DROPPED" : "      ",
+                 (*it)->dropped_at_decode ? "DROPPED" : "      ",
+                 (*it)->dropped_at_render ? "DROPPED" : "      ");
+    last_created = (*it)->created_timestamp_in_us_;
+    if (!(*it)->dropped_at_send) {
+      last_sent = (*it)->sent_timestamp_in_us_;
+    }
+     if (!(*it)->dropped_at_receive) {
+      last_received = (*it)->received_timestamp_in_us_;
+    }
+    if (!(*it)->dropped_at_decode) {
       last_decoded = (*it)->decoded_timestamp_in_us_;
+    }
+    if (!(*it)->dropped_at_render) {
       last_rendered = (*it)->rendered_timestamp_in_us_;
     }
   }
-  // Find and print the dropped frames. Work at a copy of the sent_frames_ map.
+  ViETest::Log("\nLatency between states (-1 means N/A because of drop):");
-  std::list<Frame*> decode_dropped_frames = GetFramesDroppedAtDecodeStep();
+  ViETest::Log("Unit: Microseconds");
-  ViETest::Log("Number of dropped frames at the decode step: %d",
+  ViETest::Log("Frame  Created    Sent      Received   Decoded      Total    "
-               static_cast<int>(decode_dropped_frames.size()));
+      "   Total");
-  std::list<Frame*> render_dropped_frames = GetFramesDroppedAtRenderStep();
+  ViETest::Log(" nbr   ->Sent  ->Received  ->Decoded ->Rendered    latency   "
-  ViETest::Log("Number of dropped frames at the render step: %d",
+      "  latency");
-               static_cast<int>(render_dropped_frames.size()));
+  ViETest::Log("                                               (incl network)"
+      "(excl network)");
+  for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
+       it != created_frames_vector_.end(); ++it) {
+    int created_to_sent = (*it)->dropped_at_send ? -1 :
+        static_cast<int>((*it)->sent_timestamp_in_us_ -
+                         (*it)->created_timestamp_in_us_);
+    int sent_to_received = (*it)->dropped_at_receive ? -1 :
+        static_cast<int>((*it)->received_timestamp_in_us_ -
+                         (*it)->sent_timestamp_in_us_);
+    int received_to_decoded = (*it)->dropped_at_decode ? -1 :
+        static_cast<int>((*it)->decoded_timestamp_in_us_ -
+                         (*it)->received_timestamp_in_us_);
+    int decoded_to_render = (*it)->dropped_at_render ? -1 :
+        static_cast<int>((*it)->rendered_timestamp_in_us_ -
+                         (*it)->decoded_timestamp_in_us_);
+    int total_latency_incl_network = (*it)->dropped_at_render ? -1 :
+        static_cast<int>((*it)->rendered_timestamp_in_us_ -
+                         (*it)->created_timestamp_in_us_);
+    int total_latency_excl_network = (*it)->dropped_at_render ? -1 :
+        static_cast<int>((*it)->rendered_timestamp_in_us_ -
+                         (*it)->created_timestamp_in_us_ - sent_to_received);
+    ViETest::Log("%5d %9d %9d %9d %9d %12d %12d",
+                 (*it)->number_,
+                 created_to_sent,
+                 sent_to_received,
+                 received_to_decoded,
+                 decoded_to_render,
+                 total_latency_incl_network,
+                 total_latency_excl_network);
+  }
+  // Find and print the dropped frames.
+  ViETest::Log("\nTotal # dropped frames at:");
+  ViETest::Log("  Send   : %d", dropped_frames_at_send_);
+  ViETest::Log("  Receive: %d", dropped_frames_at_receive_);
+  ViETest::Log("  Decode : %d", dropped_frames_at_decode_);
+  ViETest::Log("  Render : %d", dropped_frames_at_render_);
 }
 
-const std::list<Frame*> FrameDropDetector::GetFramesDroppedAtDecodeStep() {
+void FrameDropDetector::PrintDebugDump() {
-  std::list<Frame*> dropped_frames;
+  assert(!dirty_);
-  std::map<unsigned int, Frame*>::iterator it;
+  ViETest::Log("\nPrintDebugDump: Frame objects:");
-  for (it = sent_frames_.begin(); it != sent_frames_.end(); it++) {
+  ViETest::Log("Frame FrTimeStamp Created       Sent      Received    Decoded"
-    if (decoded_frames_.find(it->first) == decoded_frames_.end()) {
+      "    Rendered ");
-      dropped_frames.push_back(it->second);
+  for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
+       it != created_frames_vector_.end(); ++it) {
+    ViETest::Log("%5d %11d %11d %11d %11d %11d %11d",
+                 (*it)->number_,
+                 (*it)->frame_timestamp_,
+                 (*it)->created_timestamp_in_us_,
+                 (*it)->sent_timestamp_in_us_,
+                 (*it)->received_timestamp_in_us_,
+                 (*it)->decoded_timestamp_in_us_,
+                 (*it)->rendered_timestamp_in_us_);
+  }
+  std::vector<int> mismatch_frame_num_list;
+  for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
+       it != created_frames_vector_.end(); ++it) {
+    if ((*it)->dropped_at_render != (*it)->dropped_at_decode) {
+      mismatch_frame_num_list.push_back((*it)->number_);
     }
   }
-  return dropped_frames;
+  if (mismatch_frame_num_list.size() > 0) {
+    ViETest::Log("\nDecoded/Rendered mismatches:");
+    ViETest::Log("Frame FrTimeStamp    Created       Sent      Received    "
+        "Decoded    Rendered ");
+    for (std::vector<int>::const_iterator it = mismatch_frame_num_list.begin();
+         it != mismatch_frame_num_list.end(); ++it) {
+      Frame* frame = created_frames_vector_[*it];
+      ViETest::Log("%5d %11d %11d %11d %11d %11d %11d",
+                 frame->number_,
+                 frame->frame_timestamp_,
+                 frame->created_timestamp_in_us_,
+                 frame->sent_timestamp_in_us_,
+                 frame->received_timestamp_in_us_,
+                 frame->decoded_timestamp_in_us_,
+                 frame->rendered_timestamp_in_us_);
+    }
+  }
+
+  ViETest::Log("\nReportFrameState method invocations:");
+  ViETest::Log("  Created : %d", num_created_frames_);
+  ViETest::Log("  Send    : %d", num_sent_frames_);
+  ViETest::Log("  Received: %d", num_received_frames_);
+  ViETest::Log("  Decoded : %d", num_decoded_frames_);
+  ViETest::Log("  Rendered: %d", num_rendered_frames_);
 }
 
-const std::list<Frame*> FrameDropDetector::GetFramesDroppedAtRenderStep() {
+const std::vector<Frame*>& FrameDropDetector::GetAllFrames() {
-  std::list<Frame*> dropped_frames;
+  assert(!dirty_);
-  std::map<unsigned int, Frame*>::iterator it;
+  return created_frames_vector_;
-  for (it = sent_frames_.begin(); it != sent_frames_.end(); it++) {
+}
-    if (rendered_frames_.find(it->first) == rendered_frames_.end()) {
+
-      dropped_frames.push_back(it->second);
+int FrameDropDetector::GetNumberOfFramesDroppedAt(State state) {
-    }
+  assert(!dirty_);
+  switch (state) {
+    case kSent:
+      return dropped_frames_at_send_;
+    case kReceived:
+      return dropped_frames_at_receive_;
+    case kDecoded:
+      return dropped_frames_at_decode_;
+    case kRendered:
+      return dropped_frames_at_render_;
+    default:
+      return 0;
   }
-  return dropped_frames;
 }
 
 int FrameDropMonitoringRemoteFileRenderer::DeliverFrame(
     unsigned char *buffer, int buffer_size, unsigned int time_stamp) {
   // Register that this frame has been rendered:
-  frame_drop_detector_->ReportRendered(time_stamp);
+  frame_drop_detector_->ReportFrameState(FrameDropDetector::kRendered,
+                                         time_stamp);
   return ViEToFileRenderer::DeliverFrame(buffer, buffer_size, time_stamp);
 }
 

src/video_engine/test/auto_test/primitives/framedrop_primitives.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -11,8 +11,8 @@
 #ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_
 #define WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_
 
-#include <list>
 #include <map>
+#include <vector>
 
 #include "video_engine/include/vie_codec.h"
 #include "video_engine/include/vie_image_process.h"
@@ -24,25 +24,36 @@ class TbInterfaces;
 
 // Initializes the Video engine and its components, runs video playback using
 // for KAutoTestSleepTimeMs milliseconds, then shuts down everything.
-// The bit rate should be low enough to make the video encoder being forced to
+// The bit rate and packet loss parameters should be configured so that
-// drop some frames, in order to test the frame drop detection that is performed
+// frames are dropped, in order to test the frame drop detection that is
-// by the FrameDropDetector class.
+// performed by the FrameDropDetector class.
 void TestFullStack(const TbInterfaces& interfaces,
                    int capture_id,
                    int video_channel,
                    int width,
                    int height,
                    int bit_rate_kbps,
+                   int packet_loss_percent,
+                   int network_delay_ms,
                    FrameDropDetector* frame_drop_detector);
 
-// A frame in a video file. The three different points in the stack when
+// A frame in a video file. The four different points in the stack when
-// register the frame state are (in time order): sent, decoded, rendered.
+// register the frame state are (in time order): created, transmitted, decoded,
+// rendered.
 class Frame {
  public:
   Frame(int number, unsigned int timestamp)
-    : number_(number), frame_timestamp_(timestamp),
+    : number_(number),
-      sent_timestamp_in_us_(0), decoded_timestamp_in_us_(0),
+      frame_timestamp_(timestamp),
-      rendered_timestamp_in_us_(0) {}
+      created_timestamp_in_us_(-1),
+      sent_timestamp_in_us_(-1),
+      received_timestamp_in_us_(-1),
+      decoded_timestamp_in_us_(-1),
+      rendered_timestamp_in_us_(-1),
+      dropped_at_send(false),
+      dropped_at_receive(false),
+      dropped_at_decode(false),
+      dropped_at_render(false) {}
 
   // Frame number, starting at 0.
   int number_;
@@ -52,9 +63,17 @@ class Frame {
   unsigned int frame_timestamp_;
 
   // Timestamps for our measurements of when the frame is in different states.
+  int64_t created_timestamp_in_us_;
   int64_t sent_timestamp_in_us_;
+  int64_t received_timestamp_in_us_;
   int64_t decoded_timestamp_in_us_;
   int64_t rendered_timestamp_in_us_;
+
+  // Where the frame was dropped (more than one may be true).
+  bool dropped_at_send;
+  bool dropped_at_receive;
+  bool dropped_at_decode;
+  bool dropped_at_render;
 };
 
 // Fixes the output file by copying the last successful frame into the place
@@ -63,70 +82,129 @@ class Frame {
 // dropped, since there'll be no previous frame to copy. This case should never
 // happen because of encoder frame dropping at least.
 // Parameters:
-//    output_file              The output file to modify (pad with frame copies
+//    output_file            The output file to modify (pad with frame copies
-//                             for all dropped frames)
+//                           for all dropped frames)
-//    total_number_of_frames   Number of frames in the reference file we want
+//    frame_length_in_bytes  Byte length of each frame.
-//                             to match.
+//    frames                 A vector of all Frame objects. Must be sorted by
-//    frame_length_in_bytes    Byte length of each frame.
+//                           frame number. If empty this method will do nothing.
-//    dropped_frames           List of Frame objects. Must be sorted by frame
-//                             number. If empty this method will do nothing.
 void FixOutputFileForComparison(const std::string& output_file,
-                                int total_number_of_frames,
                                 int frame_length_in_bytes,
-                                std::list<Frame*> dropped_frames);
+                                const std::vector<Frame*>& frames);
 
 // Handles statistics about dropped frames. Frames travel through the stack
-// with different timestamps. The sent frames have one timestamp on the sending
+// with different timestamps. The frames created and sent to the encoder have
-// side while the decoded/rendered frames have another timestamp on the
+// one timestamp on the sending side while the decoded/rendered frames have
-// receiving side. However the difference between these timestamps is fixed,
+// another timestamp on the receiving side. The difference between these
-// which we can use to identify the frames when they arrive, since the
+// timestamps is fixed, which we can use to identify the frames when they
-// FrameDropDetector class gets data reported from both sides.
+// arrive, since the FrameDropDetector class gets data reported from both sides.
-// The three different points in the stack when this class examines the frame
+// The four different points in the stack when this class examines the frame
-// states are (in time order): sent, decoded, rendered.
+// states are (in time order): created, sent, received, decoded, rendered.
+//
+// The flow can be visualized like this:
+//
+//         Created        Sent        Received               Decoded   Rendered
+// +-------+  |  +-------+ | +---------+ | +------+  +-------+  |  +--------+
+// |Capture|  |  |Encoder| | |  Ext.   | | |Jitter|  |Decoder|  |  |  Ext.  |
+// | device|---->|       |-->|transport|-->|buffer|->|       |---->|renderer|
+// +-------+     +-------+   +---------+   +------+  +-------+     +--------+
+//
+// This class has no intention of being thread-safe.
 class FrameDropDetector {
  public:
+  enum State {
+    // A frame being created, i.e. sent to the encoder; the first step of
+    // a frame's life cycle. This timestamp becomes the frame timestamp in the
+    // Frame objects.
+    kCreated,
+    // A frame being sent in external transport (to the simulated network). This
+    // timestamp differs from the one in the Created state by a constant diff.
+    kSent,
+    // A frame being received in external transport (from the simulated
+    // network). This timestamp differs from the one in the Created state by a
+    // constant diff.
+    kReceived,
+    // A frame that has been decoded in the decoder. This timestamp differs
+    // from the one in the Created state by a constant diff.
+    kDecoded,
+    // A frame that has been rendered; the last step of a frame's life cycle.
+    // This timestamp differs from the one in the Created state by a constant
+    // diff.
+    kRendered
+  };
+
   FrameDropDetector()
-      : frame_timestamp_diff_(0) {}
+      : dirty_(true),
+        dropped_frames_at_send_(0),
+        dropped_frames_at_receive_(0),
+        dropped_frames_at_decode_(0),
+        dropped_frames_at_render_(0),
+        num_created_frames_(0),
+        num_sent_frames_(0),
+        num_received_frames_(0),
+        num_decoded_frames_(0),
+        num_rendered_frames_(0),
+        timestamp_diff_(0) {}
 
-  // Report a frame being sent; the first step of a frame transfer.
+  // Reports a frame has reached a state in the frame life cycle.
-  // This timestamp becomes the frame timestamp in the Frame objects.
+  void ReportFrameState(State state, unsigned int timestamp);
-  void ReportSent(unsigned int timestamp);
 
-  // Report a frame being rendered; happens right before it is received.
+  // Uses all the gathered timestamp information to calculate which frames have
-  // This timestamp differs from the one in ReportSent timestamp.
+  // been dropped during the test and where they were dropped. Not until
-  void ReportDecoded(unsigned int timestamp);
+  // this method has been executed, the Frame objects will have all fields
+  // filled with the proper timestamp information.
+  void CalculateResults();
 
-  // Report a frame being rendered; the last step of a frame transfer.
+  // Calculates the number of frames have been registered as dropped at the
-  // This timestamp differs from the one in ReportSent timestamp, but is the
+  // specified state of the frame life cycle.
-  // same as the ReportRendered timestamp.
+  // CalculateResults() must be called before calling this method.
-  void ReportRendered(unsigned int timestamp);
+  int GetNumberOfFramesDroppedAt(State state);
 
-  // The number of sent frames, i.e. the number of times the ReportSent has been
+  // Gets a vector of all the created frames.
-  // called successfully.
+  // CalculateResults() must be called before calling this method to have all
-  int NumberSentFrames();
+  // fields of the Frame objects to represent the current state.
-
+  const std::vector<Frame*>& GetAllFrames();
-  // Calculates which frames have been registered as dropped at the decode step.
-  const std::list<Frame*> GetFramesDroppedAtDecodeStep();
-
-  // Calculates which frames have been registered as dropped at the render step.
-  const std::list<Frame*> GetFramesDroppedAtRenderStep();
 
   // Prints a detailed report about all the different frame states and which
   // ones are detected as dropped, using ViETest::Log.
+  // CalculateResults() must be called before calling this method.
   void PrintReport();
 
+  // Prints all the timestamp maps. Mainly used for debugging purposes to find
+  // missing timestamps.
+  void PrintDebugDump();
  private:
-  // Maps mapping frame timestamps to Frame objects.
+  // Will be false until CalculateResults() is called. Switches to true
-  std::map<unsigned int, Frame*> sent_frames_;
+  // as soon as new timestamps are reported using ReportFrameState().
-  std::map<unsigned int, Frame*> decoded_frames_;
+  bool dirty_;
-  std::map<unsigned int, Frame*> rendered_frames_;
 
-  // A list with the frames sorted in their sent order:
+  // Map of frame creation timestamps to all Frame objects.
-  std::list<Frame*> sent_frames_list_;
+  std::map<unsigned int, Frame*> created_frames_;
 
-  // The constant diff between the sent and rendered frames, since their
+  // Maps converted frame timestamps (differ from creation timestamp) to the
+  // time they arrived in the different states of the frame's life cycle.
+  std::map<unsigned int, int64_t> sent_frames_;
+  std::map<unsigned int, int64_t> received_frames_;
+  std::map<unsigned int, int64_t> decoded_frames_;
+  std::map<unsigned int, int64_t> rendered_frames_;
+
+  // A vector with the frames sorted in their created order.
+  std::vector<Frame*> created_frames_vector_;
+
+  // Statistics.
+  int dropped_frames_at_send_;
+  int dropped_frames_at_receive_;
+  int dropped_frames_at_decode_;
+  int dropped_frames_at_render_;
+
+  int num_created_frames_;
+  int num_sent_frames_;
+  int num_received_frames_;
+  int num_decoded_frames_;
+  int num_rendered_frames_;
+
+  // The constant diff between the created and transmitted frames, since their
   // timestamps are converted.
-  unsigned int frame_timestamp_diff_;
+  unsigned int timestamp_diff_;
 };
 
 // Tracks which frames are received on the remote side and reports back to the

src/video_engine/test/auto_test/primitives/framedrop_primitives_unittest.cc

@@ -0,0 +1,96 @@
+/*
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "framedrop_primitives.h"
+
+#include <cstdio>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "testsupport/fileutils.h"
+#include "testsupport/frame_reader.h"
+#include "testsupport/frame_writer.h"
+
+namespace webrtc {
+
+const std::string kOutputFilename = "temp_outputfile.tmp";
+const int kFrameLength = 1000;
+
+class FrameDropPrimitivesTest: public testing::Test {
+ protected:
+  FrameDropPrimitivesTest() {}
+  virtual ~FrameDropPrimitivesTest() {}
+  void SetUp() {
+    // Cleanup any previous output file.
+    std::remove(kOutputFilename.c_str());
+  }
+  void TearDown() {
+    // Cleanup the temporary file.
+    std::remove(kOutputFilename.c_str());
+  }
+};
+
+TEST_F(FrameDropPrimitivesTest, FixOutputFileForComparison) {
+  // Create test frame objects, where the second and fourth frame is marked
+  // as dropped at rendering.
+  std::vector<Frame*> frames;
+  Frame first_frame(0, kFrameLength);
+  Frame second_frame(0, kFrameLength);
+  Frame third_frame(0, kFrameLength);
+  Frame fourth_frame(0, kFrameLength);
+
+  second_frame.dropped_at_render = true;
+  fourth_frame.dropped_at_render = true;
+
+  frames.push_back(&first_frame);
+  frames.push_back(&second_frame);
+  frames.push_back(&third_frame);
+  frames.push_back(&fourth_frame);
+
+  // Prepare data for the first and third frames:
+  WebRtc_UWord8 first_frame_data[kFrameLength];
+  memset(first_frame_data, 5, kFrameLength);  // Fill it with 5's to identify.
+  WebRtc_UWord8 third_frame_data[kFrameLength];
+  memset(third_frame_data, 7, kFrameLength);  // Fill it with 7's to identify.
+
+  // Write the first and third frames to the temporary file. This means the fix
+  // method should add two frames of data by filling the file with data from
+  // the first and third frames after executing.
+  webrtc::test::FrameWriterImpl frame_writer(kOutputFilename, kFrameLength);
+  EXPECT_TRUE(frame_writer.Init());
+  EXPECT_TRUE(frame_writer.WriteFrame(first_frame_data));
+  EXPECT_TRUE(frame_writer.WriteFrame(third_frame_data));
+  frame_writer.Close();
+  EXPECT_EQ(2 * kFrameLength,
+            static_cast<int>(webrtc::test::GetFileSize(kOutputFilename)));
+
+  FixOutputFileForComparison(kOutputFilename, kFrameLength, frames);
+
+  // Verify that the output file has correct size.
+  EXPECT_EQ(4 * kFrameLength,
+            static_cast<int>(webrtc::test::GetFileSize(kOutputFilename)));
+
+  webrtc::test::FrameReaderImpl frame_reader(kOutputFilename, kFrameLength);
+  frame_reader.Init();
+  WebRtc_UWord8 read_buffer[kFrameLength];
+  EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
+  EXPECT_EQ(0, memcmp(read_buffer, first_frame_data, kFrameLength));
+  EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
+  EXPECT_EQ(0, memcmp(read_buffer, first_frame_data, kFrameLength));
+
+  EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
+  EXPECT_EQ(0, memcmp(read_buffer, third_frame_data, kFrameLength));
+  EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
+  EXPECT_EQ(0, memcmp(read_buffer, third_frame_data, kFrameLength));
+
+  frame_reader.Close();
+}
+
+}  // namespace webrtc

src/video_engine/test/auto_test/source/tb_external_transport.cc

@@ -8,10 +8,6 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-//
-// tb_external_transport.cc
-//
-
 #include "tb_external_transport.h"
 
 #include <stdio.h> // printf
@@ -48,6 +44,10 @@ TbExternalTransport::TbExternalTransport(webrtc::ViENetwork& vieNetwork) :
         _rtpCount(0),
         _rtcpCount(0),
         _dropCount(0),
+        _rtpPackets(),
+        _rtcpPackets(),
+        _send_frame_callback(NULL),
+        _receive_frame_callback(NULL),
         _temporalLayers(0),
         _seqNum(0),
         _sendPID(0),
@@ -60,7 +60,10 @@ TbExternalTransport::TbExternalTransport(webrtc::ViENetwork& vieNetwork) :
         _filterSSRC(false),
         _SSRC(0),
         _checkSequenceNumber(0),
-        _firstSequenceNumber(0)
+        _firstSequenceNumber(0),
+        _firstRTPTimestamp(0),
+        _lastSendRTPTimestamp(0),
+        _lastReceiveRTPTimestamp(0)
 {
     srand((int) webrtc::TickTime::MicrosecondTimestamp());
     unsigned int tId = 0;
@@ -82,6 +85,23 @@ TbExternalTransport::~TbExternalTransport()
 
 int TbExternalTransport::SendPacket(int channel, const void *data, int len)
 {
+  // Parse timestamp from RTP header according to RFC 3550, section 5.1.
+    WebRtc_UWord8* ptr = (WebRtc_UWord8*)data;
+    WebRtc_UWord32 rtp_timestamp = ptr[4] << 24;
+    rtp_timestamp += ptr[5] << 16;
+    rtp_timestamp += ptr[6] << 8;
+    rtp_timestamp += ptr[7];
+    _crit.Enter();
+    if (_firstRTPTimestamp == 0) {
+      _firstRTPTimestamp = rtp_timestamp;
+    }
+    _crit.Leave();
+    if (_send_frame_callback != NULL &&
+        _lastSendRTPTimestamp != rtp_timestamp) {
+      _send_frame_callback->FrameSent(rtp_timestamp);
+    }
+    _lastSendRTPTimestamp = rtp_timestamp;
+
     if (_filterSSRC)
     {
         WebRtc_UWord8* ptr = (WebRtc_UWord8*)data;
@@ -159,9 +179,10 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len)
     _rtpCount++;
     _statCrit.Leave();
 
-    // Packet loss
+    // Packet loss. Never drop packets from the first RTP timestamp, i.e. the
+    // first frame being transmitted.
     int dropThis = rand() % 100;
-    if (dropThis < _lossRate)
+    if (dropThis < _lossRate && _firstRTPTimestamp != rtp_timestamp)
     {
         _statCrit.Enter();
         _dropCount++;
@@ -200,6 +221,16 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len)
     return len;
 }
 
+void TbExternalTransport::RegisterSendFrameCallback(
+    SendFrameCallback* callback) {
+  _send_frame_callback = callback;
+}
+
+void TbExternalTransport::RegisterReceiveFrameCallback(
+    ReceiveFrameCallback* callback) {
+  _receive_frame_callback = callback;
+}
+
 // Set to 0 to disable.
 void TbExternalTransport::SetTemporalToggle(unsigned char layers)
 {
@@ -348,6 +379,18 @@ bool TbExternalTransport::ViEExternalTransportProcess()
                     _checkSequenceNumber = false;
                 }
             }
+            // Signal received packet of frame
+            WebRtc_UWord8* ptr = (WebRtc_UWord8*)packet->packetBuffer;
+            WebRtc_UWord32 rtp_timestamp = ptr[4] << 24;
+            rtp_timestamp += ptr[5] << 16;
+            rtp_timestamp += ptr[6] << 8;
+            rtp_timestamp += ptr[7];
+            if (_receive_frame_callback != NULL &&
+                _lastReceiveRTPTimestamp != rtp_timestamp) {
+              _receive_frame_callback->FrameReceived(rtp_timestamp);
+            }
+            _lastReceiveRTPTimestamp = rtp_timestamp;
+
             _vieNetwork.ReceivedRTPPacket(packet->channel,
                                           packet->packetBuffer, packet->length);
             delete packet;

src/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -8,16 +8,16 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "vie_file_based_comparison_tests.h"
+#include "video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h"
 
-#include "base_primitives.h"
+#include "video_engine/test/auto_test/interface/tb_interfaces.h"
-#include "codec_primitives.h"
+#include "video_engine/test/auto_test/interface/vie_autotest_defines.h"
-#include "framedrop_primitives.h"
+#include "video_engine/test/auto_test/helpers/vie_fake_camera.h"
-#include "general_primitives.h"
+#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h"
-#include "tb_interfaces.h"
+#include "video_engine/test/auto_test/primitives/base_primitives.h"
-#include "vie_autotest_defines.h"
+#include "video_engine/test/auto_test/primitives/codec_primitives.h"
-#include "vie_fake_camera.h"
+#include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
-#include "vie_to_file_renderer.h"
+#include "video_engine/test/auto_test/primitives/general_primitives.h"
 
 bool ViEFileBasedComparisonTests::TestCallSetup(
     const std::string& i420_video_file,
@@ -123,11 +123,14 @@ void ViEFileBasedComparisonTests::TestFullStack(
     int width,
     int height,
     int bit_rate_kbps,
+    int packet_loss_percent,
+    int network_delay_ms,
     ViEToFileRenderer* local_file_renderer,
     ViEToFileRenderer* remote_file_renderer,
     FrameDropDetector* frame_drop_detector) {
   TbInterfaces interfaces = TbInterfaces("TestFullStack");
 
+  // Setup camera capturing from file.
   ViEFakeCamera fake_camera(interfaces.capture);
   if (!fake_camera.StartCameraInNewThread(i420_video_file, width, height)) {
     // No point in continuing if we have no proper video source
@@ -138,6 +141,12 @@ void ViEFileBasedComparisonTests::TestFullStack(
   int video_channel = -1;
   int capture_id = fake_camera.capture_id();
   EXPECT_EQ(0, interfaces.base->CreateChannel(video_channel));
+
+  // Must set SSRC to avoid SSRC collision detection since we're sending and
+  // receiving from the same machine (that would cause frames being discarded
+  // and decoder reset).
+  EXPECT_EQ(0, interfaces.rtp_rtcp->SetLocalSSRC(video_channel, 12345));
+
   EXPECT_EQ(0, interfaces.capture->ConnectCaptureDevice(
       capture_id, video_channel));
   ConfigureRtpRtcp(interfaces.rtp_rtcp, video_channel);
@@ -145,6 +154,7 @@ void ViEFileBasedComparisonTests::TestFullStack(
   RenderToFile(interfaces.render, video_channel, remote_file_renderer);
 
   ::TestFullStack(interfaces, capture_id, video_channel, width, height,
-                  bit_rate_kbps, frame_drop_detector);
+                  bit_rate_kbps, packet_loss_percent, network_delay_ms,
-  fake_camera.StopCamera();
+                  frame_drop_detector);
+  EXPECT_TRUE(fake_camera.StopCamera());
 }

src/video_engine/test/auto_test/vie_auto_test.gypi

@@ -1,4 +1,4 @@
-# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
 # that can be found in the LICENSE file in the root of the source
@@ -12,19 +12,17 @@
       'target_name': 'vie_auto_test',
       'type': 'executable',
       'dependencies': [
+        'vie_auto_test_lib',
         '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',
         '<(webrtc_root)/modules/modules.gyp:video_render_module',
         '<(webrtc_root)/modules/modules.gyp:video_capture_module',
         '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core',
         '<(webrtc_root)/../testing/gtest.gyp:gtest',
         '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
-        '<(webrtc_root)/../test/metrics.gyp:metrics',
-        '<(webrtc_root)/../test/test.gyp:test_support',
-        'video_engine_core',
       ],
       'include_dirs': [
-        'interface/',
+        'interface',
-        'helpers/',
+        'helpers',
         'primitives',
         '../../include',
         '../..',
@@ -32,116 +30,24 @@
         '../../../common_video/interface',
       ],
       'sources': [
-        'interface/tb_capture_device.h',
+        'source/vie_autotest_main.cc',
-        'interface/tb_external_transport.h',
+        # Automated tests
-        'interface/tb_I420_codec.h',
-        'interface/tb_interfaces.h',
-        'interface/tb_video_channel.h',
-        'interface/vie_autotest.h',
-        'interface/vie_autotest_defines.h',
-        'interface/vie_autotest_linux.h',
-        'interface/vie_autotest_mac_carbon.h',
-        'interface/vie_autotest_mac_cocoa.h',
-        'interface/vie_autotest_main.h',
-        'interface/vie_autotest_window_manager_interface.h',
-        'interface/vie_autotest_windows.h',
-        'interface/vie_file_based_comparison_tests.h',
-        'interface/vie_window_manager_factory.h',
-
-        # Helper classes
-        'helpers/vie_fake_camera.cc',
-        'helpers/vie_fake_camera.h',
-        'helpers/vie_file_capture_device.cc',
-        'helpers/vie_file_capture_device.h',
-        'helpers/vie_to_file_renderer.cc',
-        'helpers/vie_to_file_renderer.h',
-        'helpers/vie_window_creator.cc',
-        'helpers/vie_window_creator.h',
-
-        # New, fully automated tests
         'automated/vie_api_integration_test.cc',
         'automated/vie_extended_integration_test.cc',
         'automated/vie_integration_test_base.cc',
         'automated/vie_integration_test_base.h',
         'automated/vie_standard_integration_test.cc',
         'automated/vie_video_verification_test.cc',
-
-        # Test primitives
-        'primitives/base_primitives.cc',
-        'primitives/base_primitives.h',
-        'primitives/codec_primitives.cc',
-        'primitives/codec_primitives.h',
-        'primitives/framedrop_primitives.h',
-        'primitives/framedrop_primitives.cc',
-        'primitives/general_primitives.cc',
-        'primitives/general_primitives.h',
-
-        # Platform independent
-        'source/tb_capture_device.cc',
-        'source/tb_external_transport.cc',
-        'source/tb_I420_codec.cc',
-        'source/tb_interfaces.cc',
-        'source/tb_video_channel.cc',
-        'source/vie_autotest.cc',
-        'source/vie_autotest_base.cc',
-        'source/vie_autotest_capture.cc',
-        'source/vie_autotest_codec.cc',
-        'source/vie_autotest_encryption.cc',
-        'source/vie_autotest_file.cc',
-        'source/vie_autotest_image_process.cc',
-        'source/vie_autotest_loopback.cc',
-        'source/vie_autotest_main.cc',
-        'source/vie_autotest_network.cc',
-        'source/vie_autotest_render.cc',
-        'source/vie_autotest_rtp_rtcp.cc',
-        'source/vie_autotest_custom_call.cc',
-        'source/vie_autotest_simulcast.cc',
-        'source/vie_file_based_comparison_tests.cc',
-
         # Platform dependent
-        # Linux
         'source/vie_autotest_linux.cc',
-        'source/vie_window_manager_factory_linux.cc',
-        # Mac
         'source/vie_autotest_mac_cocoa.mm',
         'source/vie_autotest_mac_carbon.cc',
-        'source/vie_window_manager_factory_mac.mm',
-        # Windows
         'source/vie_autotest_windows.cc',
+        'source/vie_window_manager_factory_linux.cc',
+        'source/vie_window_manager_factory_mac.mm',
         'source/vie_window_manager_factory_win.cc',
       ],
-      'copies': [{
-        'destination': '/tmp',
-        'files': [
-          'media/captureDeviceImage.bmp',
-          'media/captureDeviceImage.jpg',
-          'media/renderStartImage.bmp',
-          'media/renderStartImage.jpg',
-          'media/renderTimeoutImage.bmp',
-          'media/renderTimeoutImage.jpg',
-        ],
-      }],
       'conditions': [
-        # TODO(andrew): rename these to be suffixed with _mac and _win. They
-        # will then be automatically excluded.
-        ['OS!="mac"', {
-          'sources!': [
-            'source/vie_autotest_mac_cocoa.cc',
-            'source/vie_autotest_mac_carbon.cc',
-            'source/vie_window_manager_factory_mac.mm',
-          ],
-        }],
-        ['OS!="win"', {
-          'sources!': [
-            'source/vie_autotest_windows.cc',
-          ],
-        }],
-        ['OS!="linux"', {
-          'sources!': [
-            'source/vie_window_manager_factory_linux.cc',
-          ],
-        }],
-
         # TODO(andrew): this likely isn't an actual dependency. It should be
         # included in webrtc.gyp or video_engine.gyp instead.
         ['OS=="win"', {
@@ -167,8 +73,130 @@
             ],
           },
         }],
+        # TODO(andrew): rename these to be suffixed with _mac and _win. They
+        # will then be automatically excluded.
+        ['OS!="mac"', {
+          'sources!': [
+            'source/vie_autotest_mac_cocoa.cc',
+            'source/vie_autotest_mac_carbon.cc',
+            'source/vie_window_manager_factory_mac.mm',
+          ],
+        }],
+        ['OS!="win"', {
+          'sources!': [
+            'source/vie_autotest_windows.cc',
+          ],
+        }],
+        ['OS!="linux"', {
+          'sources!': [
+            'source/vie_window_manager_factory_linux.cc',
+          ],
+        }],
       ], # conditions
     },
+    {
+      'target_name': 'vie_auto_test_lib',
+      'type': '<(library)',
+      'dependencies': [
+        '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',
+        '<(webrtc_root)/modules/modules.gyp:video_render_module',
+        '<(webrtc_root)/modules/modules.gyp:video_capture_module',
+        '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core',
+        '<(webrtc_root)/../testing/gtest.gyp:gtest',
+        '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
+        '<(webrtc_root)/../test/metrics.gyp:metrics',
+        '<(webrtc_root)/../test/test.gyp:test_support',
+        'video_engine_core',
+      ],
+      'include_dirs': [
+        'interface',
+        'helpers',
+        'primitives',
+        '../../include',
+        '../..',
+        '../../../modules/video_coding/codecs/interface',
+        '../../../common_video/interface',
+      ],
+      'sources': [
+        'interface/tb_capture_device.h',
+        'interface/tb_external_transport.h',
+        'interface/tb_I420_codec.h',
+        'interface/tb_interfaces.h',
+        'interface/tb_video_channel.h',
+        'interface/vie_autotest.h',
+        'interface/vie_autotest_defines.h',
+        'interface/vie_autotest_linux.h',
+        'interface/vie_autotest_mac_carbon.h',
+        'interface/vie_autotest_mac_cocoa.h',
+        'interface/vie_autotest_main.h',
+        'interface/vie_autotest_window_manager_interface.h',
+        'interface/vie_autotest_windows.h',
+        'interface/vie_file_based_comparison_tests.h',
+        'interface/vie_window_manager_factory.h',
+        # Helper classes
+        'helpers/vie_fake_camera.cc',
+        'helpers/vie_fake_camera.h',
+        'helpers/vie_file_capture_device.cc',
+        'helpers/vie_file_capture_device.h',
+        'helpers/vie_to_file_renderer.cc',
+        'helpers/vie_to_file_renderer.h',
+        'helpers/vie_window_creator.cc',
+        'helpers/vie_window_creator.h',
+        # Test primitives
+        'primitives/base_primitives.cc',
+        'primitives/base_primitives.h',
+        'primitives/codec_primitives.cc',
+        'primitives/codec_primitives.h',
+        'primitives/framedrop_primitives.h',
+        'primitives/framedrop_primitives.cc',
+        'primitives/general_primitives.cc',
+        'primitives/general_primitives.h',
+        # Platform independent
+        'source/tb_capture_device.cc',
+        'source/tb_external_transport.cc',
+        'source/tb_I420_codec.cc',
+        'source/tb_interfaces.cc',
+        'source/tb_video_channel.cc',
+        'source/vie_autotest.cc',
+        'source/vie_autotest_base.cc',
+        'source/vie_autotest_capture.cc',
+        'source/vie_autotest_codec.cc',
+        'source/vie_autotest_encryption.cc',
+        'source/vie_autotest_file.cc',
+        'source/vie_autotest_image_process.cc',
+        'source/vie_autotest_loopback.cc',
+        'source/vie_autotest_main.cc',
+        'source/vie_autotest_network.cc',
+        'source/vie_autotest_render.cc',
+        'source/vie_autotest_rtp_rtcp.cc',
+        'source/vie_autotest_custom_call.cc',
+        'source/vie_autotest_simulcast.cc',
+        'source/vie_file_based_comparison_tests.cc',
+      ],
+      'copies': [{
+        'destination': '/tmp',
+        'files': [
+          'media/captureDeviceImage.bmp',
+          'media/captureDeviceImage.jpg',
+          'media/renderStartImage.bmp',
+          'media/renderStartImage.jpg',
+          'media/renderTimeoutImage.bmp',
+          'media/renderTimeoutImage.jpg',
+        ],
+      }],
+    },
+    {
+      'target_name': 'vie_auto_test_unittests',
+      'type': 'executable',
+      'dependencies': [
+        'vie_auto_test_lib',
+        '<(webrtc_root)/../test/test.gyp:test_support_main',
+        '<(webrtc_root)/../testing/gtest.gyp:gtest',
+      ],
+      'sources': [
+        'primitives/framedrop_primitives_unittest.cc',
+      ],
+    },
   ],
 }