generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

External transport is modified to never drop packets from the first frame.

Browse Source 
Refactoring of FrameDropHandler: It now also tracks when frames are leaving the encoder and is being sent to external transport.

Previous 'Sent' state is now renamed to 'Created'.

NOTICE: The test seems to be a little flaky on Linux so it's not ready for buildbots yet. Since this might be caused by unstable production code further investigation should be performed to clear out the flakiness. I will file an issue for this when this CL is submitted (since I don't have any code to refer to before that). Usually the flakiness is caused by a decoded/rendered callback that is left out for the last frame, but I have seen other flaky failures too, which means it's not as simple as ignoring the last frame.
These errors occur even if 400kbps bit rate and 0% PL and 0 delay is configured.

BUG=
TEST=vie_auto_test --automated --gtest_filter="ViEVideoVerificationTest.RunsFullStackWithoutErrors" in Debug+Release on Linux, Mac and Windows.

Review URL: http://webrtc-codereview.appspot.com/339005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1597 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
kjellander@webrtc.org 2012-02-03 12:40:28 +00:00
parent 683833442a
commit 918a8bf40c
9 changed files with 874 additions and 355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
src/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -8,6 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include <vector>
#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include "testsupport/fileutils.h"
@ -30,17 +32,18 @@ class ViEVideoVerificationTest : public testing::Test {
protected:
void SetUp() {
input_file_ = webrtc::test::ResourcePath("paris_qcif", "yuv");
local_file_renderer_ = new ViEToFileRenderer();
remote_file_renderer_ = new ViEToFileRenderer();
SetUpLocalFileRenderer(local_file_renderer_);
SetUpRemoteFileRenderer(remote_file_renderer_);
}
void TearDown() {
TearDownFileRenderer(local_file_renderer_);
delete local_file_renderer_;
TearDownFileRenderer(remote_file_renderer_);
delete remote_file_renderer_;
}
void InitializeFileRenderers() {
local_file_renderer_ = new ViEToFileRenderer();
remote_file_renderer_ = new ViEToFileRenderer();
SetUpLocalFileRenderer(local_file_renderer_);
SetUpRemoteFileRenderer(remote_file_renderer_);
}
void SetUpLocalFileRenderer(ViEToFileRenderer* file_renderer) {
@ -58,6 +61,7 @@ class ViEVideoVerificationTest : public testing::Test {
}
void TearDownFileRenderer(ViEToFileRenderer* file_renderer) {
assert(file_renderer);
bool test_failed = ::testing::UnitTest::GetInstance()->
current_test_info()->result()->Failed();
if (test_failed) {
@ -67,6 +71,7 @@ class ViEVideoVerificationTest : public testing::Test {
// No reason to keep the files if we succeeded.
file_renderer->DeleteOutputFile();
}
delete file_renderer;
}
void CompareFiles(const std::string& reference_file,
@ -115,7 +120,8 @@ class ViEVideoVerificationTest : public testing::Test {
}
};
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
@ -132,6 +138,7 @@ TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
}
TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
@ -157,58 +164,63 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) {
// in the encoder. The local and remote file will not be of equal size because
// of unknown reasons. Tests show that they start at the same frame, which is
// the important thing when doing frame-to-frame comparison with PSNR/SSIM.
TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
// Use our own FrameDropMonitoringRemoteFileRenderer instead of the
// ViEToFileRenderer from the test fixture:
// TODO(kjellander): Find a better way to reuse this code without duplication.
remote_file_renderer_->StopRendering();
TearDownFileRenderer(remote_file_renderer_);
delete remote_file_renderer_;
TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
FrameDropDetector detector;
local_file_renderer_ = new ViEToFileRenderer();
remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector);
SetUpLocalFileRenderer(local_file_renderer_);
SetUpRemoteFileRenderer(remote_file_renderer_);
// Set a low bit rate so the encoder budget will be tight, causing it to drop
// frames every now and then.
const int kBitRateKbps = 50;
ViETest::Log("Bit rate: %d kbps.\n", kBitRateKbps);
const int kPacketLossPercent = 10;
const int kNetworkDelayMs = 100;
ViETest::Log("Bit rate : %5d kbps", kBitRateKbps);
ViETest::Log("Packet loss : %5d %%", kPacketLossPercent);
ViETest::Log("Network delay: %5d ms", kNetworkDelayMs);
tests_.TestFullStack(input_file_, kInputWidth, kInputHeight, kBitRateKbps,
kPacketLossPercent, kNetworkDelayMs,
local_file_renderer_, remote_file_renderer_, &detector);
const std::string reference_file = local_file_renderer_->GetFullOutputPath();
const std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
ASSERT_EQ(detector.GetFramesDroppedAtRenderStep().size(),
detector.GetFramesDroppedAtDecodeStep().size())
<< "The number of dropped frames on the decode and render are not equal, "
"this may be because we have a major problem in the jitter buffer?";
detector.CalculateResults();
detector.PrintReport();
if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) !=
detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) {
detector.PrintDebugDump();
}
ASSERT_EQ(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered),
detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded))
<< "The number of dropped frames on the decode and render steps are not "
"equal. This may be because we have a major problem in the buffers of "
"the ViEToFileRenderer?";
// We may have dropped frames during the processing, which means the output
// file does not contain all the frames that are present in the input file.
// To make the quality measurement correct, we must adjust the output file to
// that by copying the last successful frame into the place where the dropped
// frame would be, for all dropped frames.
const int frame_length_in_bytes = 3 * kInputHeight * kInputWidth / 2;
int num_frames = detector.NumberSentFrames();
ViETest::Log("Frame length: %d bytes\n", frame_length_in_bytes);
FixOutputFileForComparison(output_file, num_frames, frame_length_in_bytes,
detector.GetFramesDroppedAtDecodeStep());
ViETest::Log("Frame length: %d bytes", frame_length_in_bytes);
std::vector<Frame*> all_frames = detector.GetAllFrames();
FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames);
// Verify all sent frames are present in the output file.
size_t output_file_size = webrtc::test::GetFileSize(output_file);
EXPECT_EQ(num_frames,
static_cast<int>(output_file_size / frame_length_in_bytes))
<< "The output file size is incorrect. It should be equal to the number"
EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes)
<< "The output file size is incorrect. It should be equal to the number "
"of frames multiplied by the frame size. This will likely affect "
"PSNR/SSIM calculations in a bad way.";
// We are running on a lower bitrate here so we need to settle for somewhat
// lower PSNR and SSIM values.
const double kExpectedMinimumPSNR = 25;
const double kExpectedMinimumSSIM = 0.8;
const double kExpectedMinimumPSNR = 24;
const double kExpectedMinimumSSIM = 0.7;
CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
kExpectedMinimumSSIM);
}

63
src/video_engine/test/auto_test/interface/tb_external_transport.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -12,8 +12,8 @@
// tb_external_transport.h
//
#ifndef WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
#define WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
#ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
#define WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
#include <list>
@ -27,7 +27,38 @@ class ThreadWrapper;
class ViENetwork;
}
class TbExternalTransport: public webrtc::Transport
// Allows to subscribe for callback when a frame is started being sent.
class SendFrameCallback
{
public:
// Called once per frame (when a new RTP timestamp is detected) when the
// first data packet of the frame is being sent using the
// TbExternalTransport.SendPacket method.
virtual void FrameSent(unsigned int rtp_timestamp) = 0;
protected:
SendFrameCallback() {}
virtual ~SendFrameCallback() {}
};
// Allows to subscribe for callback when the first packet of a frame is
// received.
class ReceiveFrameCallback
{
public:
// Called once per frame (when a new RTP timestamp is detected)
// during the processing of the RTP packet queue in
// TbExternalTransport::ViEExternalTransportProcess.
virtual void FrameReceived(unsigned int rtp_timestamp) = 0;
protected:
ReceiveFrameCallback() {}
virtual ~ReceiveFrameCallback() {}
};
// External transport implementation for testing purposes.
// A packet loss probability must be set in order to drop packets from the data
// being sent to this class.
// Will never drop packets from the first frame of a video sequence.
class TbExternalTransport : public webrtc::Transport
{
public:
TbExternalTransport(webrtc::ViENetwork& vieNetwork);
@ -36,7 +67,17 @@ public:
virtual int SendPacket(int channel, const void *data, int len);
virtual int SendRTCPPacket(int channel, const void *data, int len);
WebRtc_Word32 SetPacketLoss(WebRtc_Word32 lossRate); // Rate in %
// Should only be called before/after traffic is being processed.
// Only one observer can be set (multiple calls will overwrite each other).
virtual void RegisterSendFrameCallback(SendFrameCallback* callback);
// Should only be called before/after traffic is being processed.
// Only one observer can be set (multiple calls will overwrite each other).
virtual void RegisterReceiveFrameCallback(ReceiveFrameCallback* callback);
// The probability of a packet of being dropped. Packets belonging to the
// first packet (same RTP timestamp) will never be dropped.
WebRtc_Word32 SetPacketLoss(WebRtc_Word32 lossRate); // Rate in %
void SetNetworkDelay(WebRtc_Word64 delayMs);
void SetSSRCFilter(WebRtc_UWord32 SSRC);
@ -89,6 +130,9 @@ private:
std::list<VideoPacket*> _rtpPackets;
std::list<VideoPacket*> _rtcpPackets;
SendFrameCallback* _send_frame_callback;
ReceiveFrameCallback* _receive_frame_callback;
unsigned char _temporalLayers;
unsigned short _seqNum;
unsigned short _sendPID;
@ -103,6 +147,13 @@ private:
WebRtc_UWord32 _SSRC;
bool _checkSequenceNumber;
WebRtc_UWord16 _firstSequenceNumber;
// Keep track of the first RTP timestamp so we don't do packet loss on
// the first frame.
WebRtc_UWord32 _firstRTPTimestamp;
// Track RTP timestamps so we invoke callbacks properly (if registered).
WebRtc_UWord32 _lastSendRTPTimestamp;
WebRtc_UWord32 _lastReceiveRTPTimestamp;
};
#endif // WEBRTC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_
#endif // WEBRTC_VIDEO_ENGINE_TEST_AUTOTEST_INTERFACE_TB_EXTERNAL_TRANSPORT_H_

4
src/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -58,6 +58,8 @@ class ViEFileBasedComparisonTests {
int width,
int height,
int bit_rate_kbps,
int packet_loss_percent,
int network_delay_ms,
ViEToFileRenderer* local_file_renderer,
ViEToFileRenderer* remote_file_renderer,
FrameDropDetector* frame_drop_detector);

483
src/video_engine/test/auto_test/primitives/framedrop_primitives.cc
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -11,49 +11,85 @@
#include <cassert>
#include <string>
#include "framedrop_primitives.h"
#include "general_primitives.h"
#include "modules/video_capture/main/interface/video_capture_factory.h"
#include "system_wrappers/interface/tick_util.h"
#include "tb_interfaces.h"
#include "testsupport/fileutils.h"
#include "testsupport/frame_reader.h"
#include "testsupport/frame_writer.h"
#include "video_capture_factory.h"
#include "vie_autotest.h"
#include "vie_autotest_defines.h"
#include "vie_to_file_renderer.h"
#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h"
#include "video_engine/test/auto_test/interface/tb_interfaces.h"
#include "video_engine/test/auto_test/interface/tb_external_transport.h"
#include "video_engine/test/auto_test/interface/vie_autotest.h"
#include "video_engine/test/auto_test/interface/vie_autotest_defines.h"
#include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
#include "video_engine/test/auto_test/primitives/general_primitives.h"
// Tracks which frames are sent on the local side and reports them to the
// Tracks which frames are created on the local side and reports them to the
// FrameDropDetector class.
class SendTimestampEffectFilter: public webrtc::ViEEffectFilter {
class CreatedTimestampEffectFilter : public webrtc::ViEEffectFilter {
public:
explicit SendTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
explicit CreatedTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
: frame_drop_detector_(frame_drop_detector) {}
virtual ~SendTimestampEffectFilter() {}
virtual ~CreatedTimestampEffectFilter() {}
virtual int Transform(int size, unsigned char* frameBuffer,
unsigned int timeStamp90KHz, unsigned int width,
unsigned int height) {
frame_drop_detector_->ReportSent(timeStamp90KHz);
frame_drop_detector_->ReportFrameState(FrameDropDetector::kCreated,
timeStamp90KHz);
return 0;
}
private:
FrameDropDetector* frame_drop_detector_;
};
// Tracks which frames are sent in external transport on the local side
// and reports them to the FrameDropDetector class.
class FrameSentCallback : public SendFrameCallback {
public:
explicit FrameSentCallback(FrameDropDetector* frame_drop_detector)
: frame_drop_detector_(frame_drop_detector) {}
virtual ~FrameSentCallback() {}
virtual void FrameSent(unsigned int rtp_timestamp) {
frame_drop_detector_->ReportFrameState(FrameDropDetector::kSent,
rtp_timestamp);
}
private:
FrameDropDetector* frame_drop_detector_;
};
// Tracks which frames are received in external transport on the remote side
// and reports them to the FrameDropDetector class.
class FrameReceivedCallback : public ReceiveFrameCallback {
public:
explicit FrameReceivedCallback(FrameDropDetector* frame_drop_detector)
: frame_drop_detector_(frame_drop_detector) {}
virtual ~FrameReceivedCallback() {}
virtual void FrameReceived(unsigned int rtp_timestamp) {
frame_drop_detector_->ReportFrameState(FrameDropDetector::kReceived,
rtp_timestamp);
}
private:
FrameDropDetector* frame_drop_detector_;
};
// Tracks when frames are decoded on the remote side (received from the
// jitter buffer) and reports them to the FrameDropDetector class.
class DecodeTimestampEffectFilter: public webrtc::ViEEffectFilter {
class DecodedTimestampEffectFilter : public webrtc::ViEEffectFilter {
public:
explicit DecodeTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
explicit DecodedTimestampEffectFilter(FrameDropDetector* frame_drop_detector)
: frame_drop_detector_(frame_drop_detector) {}
virtual ~DecodeTimestampEffectFilter() {}
virtual ~DecodedTimestampEffectFilter() {}
virtual int Transform(int size, unsigned char* frameBuffer,
unsigned int timeStamp90KHz, unsigned int width,
unsigned int height) {
frame_drop_detector_->ReportDecoded(timeStamp90KHz);
frame_drop_detector_->ReportFrameState(FrameDropDetector::kDecoded,
timeStamp90KHz);
return 0;
}
private:
FrameDropDetector* frame_drop_detector_;
};
@ -64,6 +100,8 @@ void TestFullStack(const TbInterfaces& interfaces,
int width,
int height,
int bit_rate_kbps,
int packet_loss_percent,
int network_delay_ms,
FrameDropDetector* frame_drop_detector) {
webrtc::VideoEngine *video_engine_interface = interfaces.video_engine;
webrtc::ViEBase *base_interface = interfaces.base;
@ -86,12 +124,20 @@ void TestFullStack(const TbInterfaces& interfaces,
EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec));
}
const char *ip_address = "127.0.0.1";
const unsigned short rtp_port = 6000;
EXPECT_EQ(0, network_interface->SetLocalReceiver(video_channel, rtp_port));
// Configure External transport to simulate network interference:
TbExternalTransport external_transport(*interfaces.network);
external_transport.SetPacketLoss(packet_loss_percent);
external_transport.SetNetworkDelay(network_delay_ms);
FrameSentCallback frame_sent_callback(frame_drop_detector);
FrameReceivedCallback frame_received_callback(frame_drop_detector);
external_transport.RegisterSendFrameCallback(&frame_sent_callback);
external_transport.RegisterReceiveFrameCallback(&frame_received_callback);
EXPECT_EQ(0, network_interface->RegisterSendTransport(video_channel,
external_transport));
EXPECT_EQ(0, base_interface->StartReceive(video_channel));
EXPECT_EQ(0, network_interface->SetSendDestination(video_channel, ip_address,
rtp_port));
// Setup only the VP8 codec, which is what we'll use.
webrtc::VideoCodec codec;
EXPECT_TRUE(FindSpecificCodec(webrtc::kVideoCodecVP8, codec_interface,
@ -107,27 +153,37 @@ void TestFullStack(const TbInterfaces& interfaces,
EXPECT_TRUE(image_process);
// Setup the effect filters
DecodeTimestampEffectFilter decode_filter(frame_drop_detector);
CreatedTimestampEffectFilter create_filter(frame_drop_detector);
EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel,
create_filter));
DecodedTimestampEffectFilter decode_filter(frame_drop_detector);
EXPECT_EQ(0, image_process->RegisterRenderEffectFilter(video_channel,
decode_filter));
SendTimestampEffectFilter send_filter(frame_drop_detector);
EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel,
send_filter));
// Send video.
EXPECT_EQ(0, base_interface->StartSend(video_channel));
AutoTestSleep(KAutoTestSleepTimeMs);
// Cleanup.
EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel));
EXPECT_EQ(0, image_process->DeregisterSendEffectFilter(video_channel));
EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel));
image_process->Release();
ViETest::Log("Done!");
WebRtc_Word32 num_rtp_packets = 0;
WebRtc_Word32 num_dropped_packets = 0;
WebRtc_Word32 num_rtcp_packets = 0;
external_transport.GetStats(num_rtp_packets, num_dropped_packets,
num_rtcp_packets);
ViETest::Log("RTP packets : %5d", num_rtp_packets);
ViETest::Log("Dropped packets: %5d", num_dropped_packets);
ViETest::Log("RTCP packets : %5d", num_rtcp_packets);
// ***************************************************************
// Testing finished. Tear down Video Engine
// ***************************************************************
EXPECT_EQ(0, base_interface->StopSend(video_channel));
EXPECT_EQ(0, base_interface->StopReceive(video_channel));
EXPECT_EQ(0, network_interface->DeregisterSendTransport(video_channel));
EXPECT_EQ(0, render_interface->StopRender(capture_id));
EXPECT_EQ(0, render_interface->StopRender(video_channel));
EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id));
@ -137,14 +193,8 @@ void TestFullStack(const TbInterfaces& interfaces,
}
void FixOutputFileForComparison(const std::string& output_file,
int total_number_of_frames,
int frame_length_in_bytes,
std::list<Frame*> dropped_frames) {
if (dropped_frames.size() == 0) {
// No need to modify if no frames are dropped, since the file is already
// frame-per-frame in sync in that case.
return;
}
const std::vector<Frame*>& frames) {
webrtc::test::FrameReaderImpl frame_reader(output_file,
frame_length_in_bytes);
const std::string temp_file = output_file + ".fixed";
@ -152,159 +202,308 @@ void FixOutputFileForComparison(const std::string& output_file,
frame_reader.Init();
frame_writer.Init();
// Assume the dropped_frames list is sorted by frame number.
int next_dropped_frame = dropped_frames.front()->number_;
dropped_frames.pop_front();
ASSERT_NE(0, next_dropped_frame) << "It should not be possible to drop the "
"first frame. Both because we don't have anything useful to fill that "
"gap with and it is impossible to detect it without any previous "
"timestamps to compare with.";
ASSERT_FALSE(frames.front()->dropped_at_render) << "It should not be "
"possible to drop the first frame. Both because we don't have anything "
"useful to fill that gap with and it is impossible to detect it without "
"any previous timestamps to compare with.";
WebRtc_UWord8* last_read_frame_data =
new WebRtc_UWord8[frame_length_in_bytes];
// Write the first frame now since it will always be the same.
EXPECT_TRUE(frame_reader.ReadFrame(last_read_frame_data));
EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data));
WebRtc_UWord8* last_frame_data = new WebRtc_UWord8[frame_length_in_bytes];
// Process the file and write frame duplicates for all dropped frames.
for (int i = 1; i < total_number_of_frames; ++i) {
if (i == next_dropped_frame) {
for (std::vector<Frame*>::const_iterator it = frames.begin();
it != frames.end(); ++it) {
if ((*it)->dropped_at_render) {
// Write the previous frame to the output file:
EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data));
if (!dropped_frames.empty()) {
next_dropped_frame = dropped_frames.front()->number_;
dropped_frames.pop_front();
}
EXPECT_TRUE(frame_writer.WriteFrame(last_frame_data));
} else {
// Read a new frame and write it to the output file.
EXPECT_TRUE(frame_reader.ReadFrame(last_read_frame_data));
EXPECT_TRUE(frame_writer.WriteFrame(last_read_frame_data));
EXPECT_TRUE(frame_reader.ReadFrame(last_frame_data));
EXPECT_TRUE(frame_writer.WriteFrame(last_frame_data));
}
}
delete[] last_read_frame_data;
delete[] last_frame_data;
frame_reader.Close();
frame_writer.Close();
ASSERT_EQ(0, std::remove(output_file.c_str()));
ASSERT_EQ(0, std::rename(temp_file.c_str(), output_file.c_str()));
}
void FrameDropDetector::ReportSent(unsigned int timestamp) {
int number = sent_frames_list_.size();
Frame* frame = new Frame(number, timestamp);
frame->sent_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp();
sent_frames_list_.push_back(frame);
sent_frames_[timestamp] = frame;
}
void FrameDropDetector::ReportDecoded(unsigned int timestamp) {
// When the first sent frame arrives we calculate the fixed difference
// between the timestamps of the sent frames and the decoded/rendered frames.
// This diff is then used to identify the frames from the sent_frames_ map.
if (frame_timestamp_diff_ == 0) {
Frame* first_sent_frame = sent_frames_list_.front();
frame_timestamp_diff_ = timestamp - first_sent_frame->frame_timestamp_;
void FrameDropDetector::ReportFrameState(State state, unsigned int timestamp) {
dirty_ = true;
switch (state) {
case kCreated: {
int number = created_frames_vector_.size();
Frame* frame = new Frame(number, timestamp);
frame->created_timestamp_in_us_ =
webrtc::TickTime::MicrosecondTimestamp();
created_frames_vector_.push_back(frame);
created_frames_[timestamp] = frame;
num_created_frames_++;
break;
}
case kSent:
sent_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
if (timestamp_diff_ == 0) {
// When the first created frame arrives we calculate the fixed
// difference between the timestamps of the frames entering and leaving
// the encoder. This diff is used to identify the frames from the
// created_frames_ map.
timestamp_diff_ =
timestamp - created_frames_vector_.front()->frame_timestamp_;
}
num_sent_frames_++;
break;
case kReceived:
received_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
num_received_frames_++;
break;
case kDecoded:
decoded_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
num_decoded_frames_++;
break;
case kRendered:
rendered_frames_[timestamp] = webrtc::TickTime::MicrosecondTimestamp();
num_rendered_frames_++;
break;
default:
assert(false);
}
// Calculate the sent timestamp required to identify the frame:
unsigned int sent_timestamp = timestamp - frame_timestamp_diff_;
// Find the right Frame object in the map of sent frames:
Frame* frame = sent_frames_[sent_timestamp];
frame->decoded_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp();
decoded_frames_[sent_timestamp] = frame;
}
void FrameDropDetector::ReportRendered(unsigned int timestamp) {
// Calculate the sent timestamp required to identify the frame:
unsigned int sent_timestamp = timestamp - frame_timestamp_diff_;
// Find this frame in the map of sent frames:
Frame* frame = sent_frames_[sent_timestamp];
frame->rendered_timestamp_in_us_ = webrtc::TickTime::MicrosecondTimestamp();
rendered_frames_[sent_timestamp] = frame;
}
int FrameDropDetector::NumberSentFrames() {
return static_cast<int>(sent_frames_.size());
void FrameDropDetector::CalculateResults() {
// Fill in all fields of the Frame objects in the created_frames_ map.
// Iterate over the maps from converted timestamps to the arrival timestamps.
std::map<unsigned int, int64_t>::const_iterator it;
for (it = sent_frames_.begin(); it != sent_frames_.end(); ++it) {
int created_timestamp = it->first - timestamp_diff_;
created_frames_[created_timestamp]->sent_timestamp_in_us_ = it->second;
}
for (it = received_frames_.begin(); it != received_frames_.end(); ++it) {
int created_timestamp = it->first - timestamp_diff_;
created_frames_[created_timestamp]->received_timestamp_in_us_ = it->second;
}
for (it = decoded_frames_.begin(); it != decoded_frames_.end(); ++it) {
int created_timestamp = it->first - timestamp_diff_;
created_frames_[created_timestamp]->decoded_timestamp_in_us_ =it->second;
}
for (it = rendered_frames_.begin(); it != rendered_frames_.end(); ++it) {
int created_timestamp = it->first - timestamp_diff_;
created_frames_[created_timestamp]->rendered_timestamp_in_us_ = it->second;
}
// Find out where the frames were not present in the different states.
dropped_frames_at_send_ = 0;
dropped_frames_at_receive_ = 0;
dropped_frames_at_decode_ = 0;
dropped_frames_at_render_ = 0;
for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
it != created_frames_vector_.end(); ++it) {
int encoded_timestamp = (*it)->frame_timestamp_ + timestamp_diff_;
if (sent_frames_.find(encoded_timestamp) == sent_frames_.end()) {
(*it)->dropped_at_send = true;
dropped_frames_at_send_++;
}
if (received_frames_.find(encoded_timestamp) == received_frames_.end()) {
(*it)->dropped_at_receive = true;
dropped_frames_at_receive_++;
}
if (decoded_frames_.find(encoded_timestamp) == decoded_frames_.end()) {
(*it)->dropped_at_decode = true;
dropped_frames_at_decode_++;
}
if (rendered_frames_.find(encoded_timestamp) == rendered_frames_.end()) {
(*it)->dropped_at_render = true;
dropped_frames_at_render_++;
}
}
dirty_ = false;
}
void FrameDropDetector::PrintReport() {
assert(!dirty_);
ViETest::Log("Frame Drop Detector report:");
ViETest::Log("Sent frames: %ld", sent_frames_.size());
ViETest::Log("Decoded frames: %ld", decoded_frames_.size());
ViETest::Log("Rendered frames: %ld", rendered_frames_.size());
ViETest::Log(" Created frames: %ld", created_frames_.size());
ViETest::Log(" Sent frames: %ld", sent_frames_.size());
ViETest::Log(" Received frames: %ld", received_frames_.size());
ViETest::Log(" Decoded frames: %ld", decoded_frames_.size());
ViETest::Log(" Rendered frames: %ld", rendered_frames_.size());
// Display all frames and stats for them:
long last_created = 0;
long last_sent = 0;
long last_received = 0;
long last_decoded = 0;
long last_rendered = 0;
ViETest::Log("Sent frames summary:");
ViETest::Log("Deltas are in microseconds and only cover existing frames.");
ViETest::Log("Frame no SentDelta DecodedDelta RenderedDelta DecodedDrop? "
"RenderedDrop?");
for (std::list<Frame*>::iterator it = sent_frames_list_.begin();
it != sent_frames_list_.end(); it++) {
bool dropped_decode = (decoded_frames_.find((*it)->frame_timestamp_) ==
decoded_frames_.end());
bool dropped_render = (rendered_frames_.find((*it)->frame_timestamp_) ==
rendered_frames_.end());
int sent_delta = static_cast<int>((*it)->sent_timestamp_in_us_ - last_sent);
int decoded_delta = dropped_decode ? 0 :
ViETest::Log("\nDeltas between sent frames and drop status:");
ViETest::Log("Unit: Microseconds");
ViETest::Log("Frame Created Sent Received Decoded Rendered "
"Dropped at Dropped at Dropped at Dropped at");
ViETest::Log(" nbr delta delta delta delta delta "
" Send? Receive? Decode? Render?");
for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
it != created_frames_vector_.end(); ++it) {
int created_delta =
static_cast<int>((*it)->created_timestamp_in_us_ - last_created);
int sent_delta = (*it)->dropped_at_send ? -1 :
static_cast<int>((*it)->sent_timestamp_in_us_ - last_sent);
int received_delta = (*it)->dropped_at_receive ? -1 :
static_cast<int>((*it)->received_timestamp_in_us_ - last_received);
int decoded_delta = (*it)->dropped_at_decode ? -1 :
static_cast<int>((*it)->decoded_timestamp_in_us_ - last_decoded);
int rendered_delta = dropped_render ? 0 :
int rendered_delta = (*it)->dropped_at_render ? -1 :
static_cast<int>((*it)->rendered_timestamp_in_us_ - last_rendered);
// Set values to 0 for the first frame:
// Set values to -1 for the first frame:
if ((*it)->number_ == 0) {
sent_delta = 0;
decoded_delta = 0;
rendered_delta = 0;
created_delta = -1;
sent_delta = -1;
received_delta = -1;
decoded_delta = -1;
rendered_delta = -1;
}
ViETest::Log("%8d %10d %10d %10d %s %s", (*it)->number_,
sent_delta, decoded_delta, rendered_delta,
dropped_decode ? "DROPPED" : " ",
dropped_render ? "DROPPED" : " ");
last_sent = (*it)->sent_timestamp_in_us_;
if (!dropped_render) {
ViETest::Log("%5d %8d %8d %8d %8d %8d %10s %10s %10s %10s",
(*it)->number_,
created_delta,
sent_delta,
received_delta,
decoded_delta,
rendered_delta,
(*it)->dropped_at_send ? "DROPPED" : " ",
(*it)->dropped_at_receive ? "DROPPED" : " ",
(*it)->dropped_at_decode ? "DROPPED" : " ",
(*it)->dropped_at_render ? "DROPPED" : " ");
last_created = (*it)->created_timestamp_in_us_;
if (!(*it)->dropped_at_send) {
last_sent = (*it)->sent_timestamp_in_us_;
}
if (!(*it)->dropped_at_receive) {
last_received = (*it)->received_timestamp_in_us_;
}
if (!(*it)->dropped_at_decode) {
last_decoded = (*it)->decoded_timestamp_in_us_;
}
if (!(*it)->dropped_at_render) {
last_rendered = (*it)->rendered_timestamp_in_us_;
}
}
// Find and print the dropped frames. Work at a copy of the sent_frames_ map.
std::list<Frame*> decode_dropped_frames = GetFramesDroppedAtDecodeStep();
ViETest::Log("Number of dropped frames at the decode step: %d",
static_cast<int>(decode_dropped_frames.size()));
std::list<Frame*> render_dropped_frames = GetFramesDroppedAtRenderStep();
ViETest::Log("Number of dropped frames at the render step: %d",
static_cast<int>(render_dropped_frames.size()));
ViETest::Log("\nLatency between states (-1 means N/A because of drop):");
ViETest::Log("Unit: Microseconds");
ViETest::Log("Frame Created Sent Received Decoded Total "
" Total");
ViETest::Log(" nbr ->Sent ->Received ->Decoded ->Rendered latency "
" latency");
ViETest::Log(" (incl network)"
"(excl network)");
for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
it != created_frames_vector_.end(); ++it) {
int created_to_sent = (*it)->dropped_at_send ? -1 :
static_cast<int>((*it)->sent_timestamp_in_us_ -
(*it)->created_timestamp_in_us_);
int sent_to_received = (*it)->dropped_at_receive ? -1 :
static_cast<int>((*it)->received_timestamp_in_us_ -
(*it)->sent_timestamp_in_us_);
int received_to_decoded = (*it)->dropped_at_decode ? -1 :
static_cast<int>((*it)->decoded_timestamp_in_us_ -
(*it)->received_timestamp_in_us_);
int decoded_to_render = (*it)->dropped_at_render ? -1 :
static_cast<int>((*it)->rendered_timestamp_in_us_ -
(*it)->decoded_timestamp_in_us_);
int total_latency_incl_network = (*it)->dropped_at_render ? -1 :
static_cast<int>((*it)->rendered_timestamp_in_us_ -
(*it)->created_timestamp_in_us_);
int total_latency_excl_network = (*it)->dropped_at_render ? -1 :
static_cast<int>((*it)->rendered_timestamp_in_us_ -
(*it)->created_timestamp_in_us_ - sent_to_received);
ViETest::Log("%5d %9d %9d %9d %9d %12d %12d",
(*it)->number_,
created_to_sent,
sent_to_received,
received_to_decoded,
decoded_to_render,
total_latency_incl_network,
total_latency_excl_network);
}
// Find and print the dropped frames.
ViETest::Log("\nTotal # dropped frames at:");
ViETest::Log(" Send : %d", dropped_frames_at_send_);
ViETest::Log(" Receive: %d", dropped_frames_at_receive_);
ViETest::Log(" Decode : %d", dropped_frames_at_decode_);
ViETest::Log(" Render : %d", dropped_frames_at_render_);
}
const std::list<Frame*> FrameDropDetector::GetFramesDroppedAtDecodeStep() {
std::list<Frame*> dropped_frames;
std::map<unsigned int, Frame*>::iterator it;
for (it = sent_frames_.begin(); it != sent_frames_.end(); it++) {
if (decoded_frames_.find(it->first) == decoded_frames_.end()) {
dropped_frames.push_back(it->second);
void FrameDropDetector::PrintDebugDump() {
assert(!dirty_);
ViETest::Log("\nPrintDebugDump: Frame objects:");
ViETest::Log("Frame FrTimeStamp Created Sent Received Decoded"
" Rendered ");
for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
it != created_frames_vector_.end(); ++it) {
ViETest::Log("%5d %11d %11d %11d %11d %11d %11d",
(*it)->number_,
(*it)->frame_timestamp_,
(*it)->created_timestamp_in_us_,
(*it)->sent_timestamp_in_us_,
(*it)->received_timestamp_in_us_,
(*it)->decoded_timestamp_in_us_,
(*it)->rendered_timestamp_in_us_);
}
std::vector<int> mismatch_frame_num_list;
for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
it != created_frames_vector_.end(); ++it) {
if ((*it)->dropped_at_render != (*it)->dropped_at_decode) {
mismatch_frame_num_list.push_back((*it)->number_);
}
}
return dropped_frames;
if (mismatch_frame_num_list.size() > 0) {
ViETest::Log("\nDecoded/Rendered mismatches:");
ViETest::Log("Frame FrTimeStamp Created Sent Received "
"Decoded Rendered ");
for (std::vector<int>::const_iterator it = mismatch_frame_num_list.begin();
it != mismatch_frame_num_list.end(); ++it) {
Frame* frame = created_frames_vector_[*it];
ViETest::Log("%5d %11d %11d %11d %11d %11d %11d",
frame->number_,
frame->frame_timestamp_,
frame->created_timestamp_in_us_,
frame->sent_timestamp_in_us_,
frame->received_timestamp_in_us_,
frame->decoded_timestamp_in_us_,
frame->rendered_timestamp_in_us_);
}
}
ViETest::Log("\nReportFrameState method invocations:");
ViETest::Log(" Created : %d", num_created_frames_);
ViETest::Log(" Send : %d", num_sent_frames_);
ViETest::Log(" Received: %d", num_received_frames_);
ViETest::Log(" Decoded : %d", num_decoded_frames_);
ViETest::Log(" Rendered: %d", num_rendered_frames_);
}
const std::list<Frame*> FrameDropDetector::GetFramesDroppedAtRenderStep() {
std::list<Frame*> dropped_frames;
std::map<unsigned int, Frame*>::iterator it;
for (it = sent_frames_.begin(); it != sent_frames_.end(); it++) {
if (rendered_frames_.find(it->first) == rendered_frames_.end()) {
dropped_frames.push_back(it->second);
}
const std::vector<Frame*>& FrameDropDetector::GetAllFrames() {
assert(!dirty_);
return created_frames_vector_;
}
int FrameDropDetector::GetNumberOfFramesDroppedAt(State state) {
assert(!dirty_);
switch (state) {
case kSent:
return dropped_frames_at_send_;
case kReceived:
return dropped_frames_at_receive_;
case kDecoded:
return dropped_frames_at_decode_;
case kRendered:
return dropped_frames_at_render_;
default:
return 0;
}
return dropped_frames;
}
int FrameDropMonitoringRemoteFileRenderer::DeliverFrame(
unsigned char *buffer, int buffer_size, unsigned int time_stamp) {
// Register that this frame has been rendered:
frame_drop_detector_->ReportRendered(time_stamp);
frame_drop_detector_->ReportFrameState(FrameDropDetector::kRendered,
time_stamp);
return ViEToFileRenderer::DeliverFrame(buffer, buffer_size, time_stamp);
}

186
src/video_engine/test/auto_test/primitives/framedrop_primitives.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -11,8 +11,8 @@
#ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_
#define WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_SOURCE_FRAMEDROP_PRIMITIVES_H_
#include <list>
#include <map>
#include <vector>
#include "video_engine/include/vie_codec.h"
#include "video_engine/include/vie_image_process.h"
@ -24,25 +24,36 @@ class TbInterfaces;
// Initializes the Video engine and its components, runs video playback using
// for KAutoTestSleepTimeMs milliseconds, then shuts down everything.
// The bit rate should be low enough to make the video encoder being forced to
// drop some frames, in order to test the frame drop detection that is performed
// by the FrameDropDetector class.
// The bit rate and packet loss parameters should be configured so that
// frames are dropped, in order to test the frame drop detection that is
// performed by the FrameDropDetector class.
void TestFullStack(const TbInterfaces& interfaces,
int capture_id,
int video_channel,
int width,
int height,
int bit_rate_kbps,
int packet_loss_percent,
int network_delay_ms,
FrameDropDetector* frame_drop_detector);
// A frame in a video file. The three different points in the stack when
// register the frame state are (in time order): sent, decoded, rendered.
// A frame in a video file. The four different points in the stack when
// register the frame state are (in time order): created, transmitted, decoded,
// rendered.
class Frame {
public:
Frame(int number, unsigned int timestamp)
: number_(number), frame_timestamp_(timestamp),
sent_timestamp_in_us_(0), decoded_timestamp_in_us_(0),
rendered_timestamp_in_us_(0) {}
: number_(number),
frame_timestamp_(timestamp),
created_timestamp_in_us_(-1),
sent_timestamp_in_us_(-1),
received_timestamp_in_us_(-1),
decoded_timestamp_in_us_(-1),
rendered_timestamp_in_us_(-1),
dropped_at_send(false),
dropped_at_receive(false),
dropped_at_decode(false),
dropped_at_render(false) {}
// Frame number, starting at 0.
int number_;
@ -52,9 +63,17 @@ class Frame {
unsigned int frame_timestamp_;
// Timestamps for our measurements of when the frame is in different states.
int64_t created_timestamp_in_us_;
int64_t sent_timestamp_in_us_;
int64_t received_timestamp_in_us_;
int64_t decoded_timestamp_in_us_;
int64_t rendered_timestamp_in_us_;
// Where the frame was dropped (more than one may be true).
bool dropped_at_send;
bool dropped_at_receive;
bool dropped_at_decode;
bool dropped_at_render;
};
// Fixes the output file by copying the last successful frame into the place
@ -63,70 +82,129 @@ class Frame {
// dropped, since there'll be no previous frame to copy. This case should never
// happen because of encoder frame dropping at least.
// Parameters:
// output_file The output file to modify (pad with frame copies
// for all dropped frames)
// total_number_of_frames Number of frames in the reference file we want
// to match.
// frame_length_in_bytes Byte length of each frame.
// dropped_frames List of Frame objects. Must be sorted by frame
// number. If empty this method will do nothing.
// output_file The output file to modify (pad with frame copies
// for all dropped frames)
// frame_length_in_bytes Byte length of each frame.
// frames A vector of all Frame objects. Must be sorted by
// frame number. If empty this method will do nothing.
void FixOutputFileForComparison(const std::string& output_file,
int total_number_of_frames,
int frame_length_in_bytes,
std::list<Frame*> dropped_frames);
const std::vector<Frame*>& frames);
// Handles statistics about dropped frames. Frames travel through the stack
// with different timestamps. The sent frames have one timestamp on the sending
// side while the decoded/rendered frames have another timestamp on the
// receiving side. However the difference between these timestamps is fixed,
// which we can use to identify the frames when they arrive, since the
// FrameDropDetector class gets data reported from both sides.
// The three different points in the stack when this class examines the frame
// states are (in time order): sent, decoded, rendered.
// with different timestamps. The frames created and sent to the encoder have
// one timestamp on the sending side while the decoded/rendered frames have
// another timestamp on the receiving side. The difference between these
// timestamps is fixed, which we can use to identify the frames when they
// arrive, since the FrameDropDetector class gets data reported from both sides.
// The four different points in the stack when this class examines the frame
// states are (in time order): created, sent, received, decoded, rendered.
//
// The flow can be visualized like this:
//
// Created Sent Received Decoded Rendered
// +-------+ | +-------+ | +---------+ | +------+ +-------+ | +--------+
// |Capture| | |Encoder| | | Ext. | | |Jitter| |Decoder| | | Ext. |
// | device|---->| |-->|transport|-->|buffer|->| |---->|renderer|
// +-------+ +-------+ +---------+ +------+ +-------+ +--------+
//
// This class has no intention of being thread-safe.
class FrameDropDetector {
public:
enum State {
// A frame being created, i.e. sent to the encoder; the first step of
// a frame's life cycle. This timestamp becomes the frame timestamp in the
// Frame objects.
kCreated,
// A frame being sent in external transport (to the simulated network). This
// timestamp differs from the one in the Created state by a constant diff.
kSent,
// A frame being received in external transport (from the simulated
// network). This timestamp differs from the one in the Created state by a
// constant diff.
kReceived,
// A frame that has been decoded in the decoder. This timestamp differs
// from the one in the Created state by a constant diff.
kDecoded,
// A frame that has been rendered; the last step of a frame's life cycle.
// This timestamp differs from the one in the Created state by a constant
// diff.
kRendered
};
FrameDropDetector()
: frame_timestamp_diff_(0) {}
: dirty_(true),
dropped_frames_at_send_(0),
dropped_frames_at_receive_(0),
dropped_frames_at_decode_(0),
dropped_frames_at_render_(0),
num_created_frames_(0),
num_sent_frames_(0),
num_received_frames_(0),
num_decoded_frames_(0),
num_rendered_frames_(0),
timestamp_diff_(0) {}
// Report a frame being sent; the first step of a frame transfer.
// This timestamp becomes the frame timestamp in the Frame objects.
void ReportSent(unsigned int timestamp);
// Reports a frame has reached a state in the frame life cycle.
void ReportFrameState(State state, unsigned int timestamp);
// Report a frame being rendered; happens right before it is received.
// This timestamp differs from the one in ReportSent timestamp.
void ReportDecoded(unsigned int timestamp);
// Uses all the gathered timestamp information to calculate which frames have
// been dropped during the test and where they were dropped. Not until
// this method has been executed, the Frame objects will have all fields
// filled with the proper timestamp information.
void CalculateResults();
// Report a frame being rendered; the last step of a frame transfer.
// This timestamp differs from the one in ReportSent timestamp, but is the
// same as the ReportRendered timestamp.
void ReportRendered(unsigned int timestamp);
// Calculates the number of frames have been registered as dropped at the
// specified state of the frame life cycle.
// CalculateResults() must be called before calling this method.
int GetNumberOfFramesDroppedAt(State state);
// The number of sent frames, i.e. the number of times the ReportSent has been
// called successfully.
int NumberSentFrames();
// Calculates which frames have been registered as dropped at the decode step.
const std::list<Frame*> GetFramesDroppedAtDecodeStep();
// Calculates which frames have been registered as dropped at the render step.
const std::list<Frame*> GetFramesDroppedAtRenderStep();
// Gets a vector of all the created frames.
// CalculateResults() must be called before calling this method to have all
// fields of the Frame objects to represent the current state.
const std::vector<Frame*>& GetAllFrames();
// Prints a detailed report about all the different frame states and which
// ones are detected as dropped, using ViETest::Log.
// CalculateResults() must be called before calling this method.
void PrintReport();
// Prints all the timestamp maps. Mainly used for debugging purposes to find
// missing timestamps.
void PrintDebugDump();
private:
// Maps mapping frame timestamps to Frame objects.
std::map<unsigned int, Frame*> sent_frames_;
std::map<unsigned int, Frame*> decoded_frames_;
std::map<unsigned int, Frame*> rendered_frames_;
// Will be false until CalculateResults() is called. Switches to true
// as soon as new timestamps are reported using ReportFrameState().
bool dirty_;
// A list with the frames sorted in their sent order:
std::list<Frame*> sent_frames_list_;
// Map of frame creation timestamps to all Frame objects.
std::map<unsigned int, Frame*> created_frames_;
// The constant diff between the sent and rendered frames, since their
// Maps converted frame timestamps (differ from creation timestamp) to the
// time they arrived in the different states of the frame's life cycle.
std::map<unsigned int, int64_t> sent_frames_;
std::map<unsigned int, int64_t> received_frames_;
std::map<unsigned int, int64_t> decoded_frames_;
std::map<unsigned int, int64_t> rendered_frames_;
// A vector with the frames sorted in their created order.
std::vector<Frame*> created_frames_vector_;
// Statistics.
int dropped_frames_at_send_;
int dropped_frames_at_receive_;
int dropped_frames_at_decode_;
int dropped_frames_at_render_;
int num_created_frames_;
int num_sent_frames_;
int num_received_frames_;
int num_decoded_frames_;
int num_rendered_frames_;
// The constant diff between the created and transmitted frames, since their
// timestamps are converted.
unsigned int frame_timestamp_diff_;
unsigned int timestamp_diff_;
};
// Tracks which frames are received on the remote side and reports back to the

96
src/video_engine/test/auto_test/primitives/framedrop_primitives_unittest.cc Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "framedrop_primitives.h"
#include <cstdio>
#include <vector>
#include "gtest/gtest.h"
#include "testsupport/fileutils.h"
#include "testsupport/frame_reader.h"
#include "testsupport/frame_writer.h"
namespace webrtc {
const std::string kOutputFilename = "temp_outputfile.tmp";
const int kFrameLength = 1000;
class FrameDropPrimitivesTest: public testing::Test {
protected:
FrameDropPrimitivesTest() {}
virtual ~FrameDropPrimitivesTest() {}
void SetUp() {
// Cleanup any previous output file.
std::remove(kOutputFilename.c_str());
}
void TearDown() {
// Cleanup the temporary file.
std::remove(kOutputFilename.c_str());
}
};
TEST_F(FrameDropPrimitivesTest, FixOutputFileForComparison) {
// Create test frame objects, where the second and fourth frame is marked
// as dropped at rendering.
std::vector<Frame*> frames;
Frame first_frame(0, kFrameLength);
Frame second_frame(0, kFrameLength);
Frame third_frame(0, kFrameLength);
Frame fourth_frame(0, kFrameLength);
second_frame.dropped_at_render = true;
fourth_frame.dropped_at_render = true;
frames.push_back(&first_frame);
frames.push_back(&second_frame);
frames.push_back(&third_frame);
frames.push_back(&fourth_frame);
// Prepare data for the first and third frames:
WebRtc_UWord8 first_frame_data[kFrameLength];
memset(first_frame_data, 5, kFrameLength); // Fill it with 5's to identify.
WebRtc_UWord8 third_frame_data[kFrameLength];
memset(third_frame_data, 7, kFrameLength); // Fill it with 7's to identify.
// Write the first and third frames to the temporary file. This means the fix
// method should add two frames of data by filling the file with data from
// the first and third frames after executing.
webrtc::test::FrameWriterImpl frame_writer(kOutputFilename, kFrameLength);
EXPECT_TRUE(frame_writer.Init());
EXPECT_TRUE(frame_writer.WriteFrame(first_frame_data));
EXPECT_TRUE(frame_writer.WriteFrame(third_frame_data));
frame_writer.Close();
EXPECT_EQ(2 * kFrameLength,
static_cast<int>(webrtc::test::GetFileSize(kOutputFilename)));
FixOutputFileForComparison(kOutputFilename, kFrameLength, frames);
// Verify that the output file has correct size.
EXPECT_EQ(4 * kFrameLength,
static_cast<int>(webrtc::test::GetFileSize(kOutputFilename)));
webrtc::test::FrameReaderImpl frame_reader(kOutputFilename, kFrameLength);
frame_reader.Init();
WebRtc_UWord8 read_buffer[kFrameLength];
EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
EXPECT_EQ(0, memcmp(read_buffer, first_frame_data, kFrameLength));
EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
EXPECT_EQ(0, memcmp(read_buffer, first_frame_data, kFrameLength));
EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
EXPECT_EQ(0, memcmp(read_buffer, third_frame_data, kFrameLength));
EXPECT_TRUE(frame_reader.ReadFrame(read_buffer));
EXPECT_EQ(0, memcmp(read_buffer, third_frame_data, kFrameLength));
frame_reader.Close();
}
} // namespace webrtc

57
src/video_engine/test/auto_test/source/tb_external_transport.cc
View File

@ -8,10 +8,6 @@
* be found in the AUTHORS file in the root of the source tree.
*/
//
// tb_external_transport.cc
//
#include "tb_external_transport.h"
#include <stdio.h> // printf
@ -48,6 +44,10 @@ TbExternalTransport::TbExternalTransport(webrtc::ViENetwork& vieNetwork) :
_rtpCount(0),
_rtcpCount(0),
_dropCount(0),
_rtpPackets(),
_rtcpPackets(),
_send_frame_callback(NULL),
_receive_frame_callback(NULL),
_temporalLayers(0),
_seqNum(0),
_sendPID(0),
@ -60,7 +60,10 @@ TbExternalTransport::TbExternalTransport(webrtc::ViENetwork& vieNetwork) :
_filterSSRC(false),
_SSRC(0),
_checkSequenceNumber(0),
_firstSequenceNumber(0)
_firstSequenceNumber(0),
_firstRTPTimestamp(0),
_lastSendRTPTimestamp(0),
_lastReceiveRTPTimestamp(0)
{
srand((int) webrtc::TickTime::MicrosecondTimestamp());
unsigned int tId = 0;
@ -82,6 +85,23 @@ TbExternalTransport::~TbExternalTransport()
int TbExternalTransport::SendPacket(int channel, const void *data, int len)
{
// Parse timestamp from RTP header according to RFC 3550, section 5.1.
WebRtc_UWord8* ptr = (WebRtc_UWord8*)data;
WebRtc_UWord32 rtp_timestamp = ptr[4] << 24;
rtp_timestamp += ptr[5] << 16;
rtp_timestamp += ptr[6] << 8;
rtp_timestamp += ptr[7];
_crit.Enter();
if (_firstRTPTimestamp == 0) {
_firstRTPTimestamp = rtp_timestamp;
}
_crit.Leave();
if (_send_frame_callback != NULL &&
_lastSendRTPTimestamp != rtp_timestamp) {
_send_frame_callback->FrameSent(rtp_timestamp);
}
_lastSendRTPTimestamp = rtp_timestamp;
if (_filterSSRC)
{
WebRtc_UWord8* ptr = (WebRtc_UWord8*)data;
@ -159,9 +179,10 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len)
_rtpCount++;
_statCrit.Leave();
// Packet loss
// Packet loss. Never drop packets from the first RTP timestamp, i.e. the
// first frame being transmitted.
int dropThis = rand() % 100;
if (dropThis < _lossRate)
if (dropThis < _lossRate && _firstRTPTimestamp != rtp_timestamp)
{
_statCrit.Enter();
_dropCount++;
@ -200,6 +221,16 @@ int TbExternalTransport::SendPacket(int channel, const void *data, int len)
return len;
}
void TbExternalTransport::RegisterSendFrameCallback(
SendFrameCallback* callback) {
_send_frame_callback = callback;
}
void TbExternalTransport::RegisterReceiveFrameCallback(
ReceiveFrameCallback* callback) {
_receive_frame_callback = callback;
}
// Set to 0 to disable.
void TbExternalTransport::SetTemporalToggle(unsigned char layers)
{
@ -348,6 +379,18 @@ bool TbExternalTransport::ViEExternalTransportProcess()
_checkSequenceNumber = false;
}
}
// Signal received packet of frame
WebRtc_UWord8* ptr = (WebRtc_UWord8*)packet->packetBuffer;
WebRtc_UWord32 rtp_timestamp = ptr[4] << 24;
rtp_timestamp += ptr[5] << 16;
rtp_timestamp += ptr[6] << 8;
rtp_timestamp += ptr[7];
if (_receive_frame_callback != NULL &&
_lastReceiveRTPTimestamp != rtp_timestamp) {
_receive_frame_callback->FrameReceived(rtp_timestamp);
}
_lastReceiveRTPTimestamp = rtp_timestamp;
_vieNetwork.ReceivedRTPPacket(packet->channel,
packet->packetBuffer, packet->length);
delete packet;

34
src/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -8,16 +8,16 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "vie_file_based_comparison_tests.h"
#include "video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h"
#include "base_primitives.h"
#include "codec_primitives.h"
#include "framedrop_primitives.h"
#include "general_primitives.h"
#include "tb_interfaces.h"
#include "vie_autotest_defines.h"
#include "vie_fake_camera.h"
#include "vie_to_file_renderer.h"
#include "video_engine/test/auto_test/interface/tb_interfaces.h"
#include "video_engine/test/auto_test/interface/vie_autotest_defines.h"
#include "video_engine/test/auto_test/helpers/vie_fake_camera.h"
#include "video_engine/test/auto_test/helpers/vie_to_file_renderer.h"
#include "video_engine/test/auto_test/primitives/base_primitives.h"
#include "video_engine/test/auto_test/primitives/codec_primitives.h"
#include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
#include "video_engine/test/auto_test/primitives/general_primitives.h"
bool ViEFileBasedComparisonTests::TestCallSetup(
const std::string& i420_video_file,
@ -123,11 +123,14 @@ void ViEFileBasedComparisonTests::TestFullStack(
int width,
int height,
int bit_rate_kbps,
int packet_loss_percent,
int network_delay_ms,
ViEToFileRenderer* local_file_renderer,
ViEToFileRenderer* remote_file_renderer,
FrameDropDetector* frame_drop_detector) {
TbInterfaces interfaces = TbInterfaces("TestFullStack");
// Setup camera capturing from file.
ViEFakeCamera fake_camera(interfaces.capture);
if (!fake_camera.StartCameraInNewThread(i420_video_file, width, height)) {
// No point in continuing if we have no proper video source
@ -138,6 +141,12 @@ void ViEFileBasedComparisonTests::TestFullStack(
int video_channel = -1;
int capture_id = fake_camera.capture_id();
EXPECT_EQ(0, interfaces.base->CreateChannel(video_channel));
// Must set SSRC to avoid SSRC collision detection since we're sending and
// receiving from the same machine (that would cause frames being discarded
// and decoder reset).
EXPECT_EQ(0, interfaces.rtp_rtcp->SetLocalSSRC(video_channel, 12345));
EXPECT_EQ(0, interfaces.capture->ConnectCaptureDevice(
capture_id, video_channel));
ConfigureRtpRtcp(interfaces.rtp_rtcp, video_channel);
@ -145,6 +154,7 @@ void ViEFileBasedComparisonTests::TestFullStack(
RenderToFile(interfaces.render, video_channel, remote_file_renderer);
::TestFullStack(interfaces, capture_id, video_channel, width, height,
bit_rate_kbps, frame_drop_detector);
fake_camera.StopCamera();
bit_rate_kbps, packet_loss_percent, network_delay_ms,
frame_drop_detector);
EXPECT_TRUE(fake_camera.StopCamera());
}

232
src/video_engine/test/auto_test/vie_auto_test.gypi
View File

@ -1,4 +1,4 @@
# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
@ -12,19 +12,17 @@
'target_name': 'vie_auto_test',
'type': 'executable',
'dependencies': [
'vie_auto_test_lib',
'<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',
'<(webrtc_root)/modules/modules.gyp:video_render_module',
'<(webrtc_root)/modules/modules.gyp:video_capture_module',
'<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
'<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
'<(webrtc_root)/../test/metrics.gyp:metrics',
'<(webrtc_root)/../test/test.gyp:test_support',
'video_engine_core',
],
'include_dirs': [
'interface/',
'helpers/',
'interface',
'helpers',
'primitives',
'../../include',
'../..',
@ -32,116 +30,24 @@
'../../../common_video/interface',
],
'sources': [
'interface/tb_capture_device.h',
'interface/tb_external_transport.h',
'interface/tb_I420_codec.h',
'interface/tb_interfaces.h',
'interface/tb_video_channel.h',
'interface/vie_autotest.h',
'interface/vie_autotest_defines.h',
'interface/vie_autotest_linux.h',
'interface/vie_autotest_mac_carbon.h',
'interface/vie_autotest_mac_cocoa.h',
'interface/vie_autotest_main.h',
'interface/vie_autotest_window_manager_interface.h',
'interface/vie_autotest_windows.h',
'interface/vie_file_based_comparison_tests.h',
'interface/vie_window_manager_factory.h',
# Helper classes
'helpers/vie_fake_camera.cc',
'helpers/vie_fake_camera.h',
'helpers/vie_file_capture_device.cc',
'helpers/vie_file_capture_device.h',
'helpers/vie_to_file_renderer.cc',
'helpers/vie_to_file_renderer.h',
'helpers/vie_window_creator.cc',
'helpers/vie_window_creator.h',
# New, fully automated tests
'source/vie_autotest_main.cc',
# Automated tests
'automated/vie_api_integration_test.cc',
'automated/vie_extended_integration_test.cc',
'automated/vie_integration_test_base.cc',
'automated/vie_integration_test_base.h',
'automated/vie_standard_integration_test.cc',
'automated/vie_video_verification_test.cc',
# Test primitives
'primitives/base_primitives.cc',
'primitives/base_primitives.h',
'primitives/codec_primitives.cc',
'primitives/codec_primitives.h',
'primitives/framedrop_primitives.h',
'primitives/framedrop_primitives.cc',
'primitives/general_primitives.cc',
'primitives/general_primitives.h',
# Platform independent
'source/tb_capture_device.cc',
'source/tb_external_transport.cc',
'source/tb_I420_codec.cc',
'source/tb_interfaces.cc',
'source/tb_video_channel.cc',
'source/vie_autotest.cc',
'source/vie_autotest_base.cc',
'source/vie_autotest_capture.cc',
'source/vie_autotest_codec.cc',
'source/vie_autotest_encryption.cc',
'source/vie_autotest_file.cc',
'source/vie_autotest_image_process.cc',
'source/vie_autotest_loopback.cc',
'source/vie_autotest_main.cc',
'source/vie_autotest_network.cc',
'source/vie_autotest_render.cc',
'source/vie_autotest_rtp_rtcp.cc',
'source/vie_autotest_custom_call.cc',
'source/vie_autotest_simulcast.cc',
'source/vie_file_based_comparison_tests.cc',
# Platform dependent
# Linux
'source/vie_autotest_linux.cc',
'source/vie_window_manager_factory_linux.cc',
# Mac
'source/vie_autotest_mac_cocoa.mm',
'source/vie_autotest_mac_carbon.cc',
'source/vie_window_manager_factory_mac.mm',
# Windows
'source/vie_autotest_windows.cc',
'source/vie_window_manager_factory_linux.cc',
'source/vie_window_manager_factory_mac.mm',
'source/vie_window_manager_factory_win.cc',
],
'copies': [{
'destination': '/tmp',
'files': [
'media/captureDeviceImage.bmp',
'media/captureDeviceImage.jpg',
'media/renderStartImage.bmp',
'media/renderStartImage.jpg',
'media/renderTimeoutImage.bmp',
'media/renderTimeoutImage.jpg',
],
}],
'conditions': [
# TODO(andrew): rename these to be suffixed with _mac and _win. They
# will then be automatically excluded.
['OS!="mac"', {
'sources!': [
'source/vie_autotest_mac_cocoa.cc',
'source/vie_autotest_mac_carbon.cc',
'source/vie_window_manager_factory_mac.mm',
],
}],
['OS!="win"', {
'sources!': [
'source/vie_autotest_windows.cc',
],
}],
['OS!="linux"', {
'sources!': [
'source/vie_window_manager_factory_linux.cc',
],
}],
# TODO(andrew): this likely isn't an actual dependency. It should be
# included in webrtc.gyp or video_engine.gyp instead.
['OS=="win"', {
@ -167,8 +73,130 @@
],
},
}],
# TODO(andrew): rename these to be suffixed with _mac and _win. They
# will then be automatically excluded.
['OS!="mac"', {
'sources!': [
'source/vie_autotest_mac_cocoa.cc',
'source/vie_autotest_mac_carbon.cc',
'source/vie_window_manager_factory_mac.mm',
],
}],
['OS!="win"', {
'sources!': [
'source/vie_autotest_windows.cc',
],
}],
['OS!="linux"', {
'sources!': [
'source/vie_window_manager_factory_linux.cc',
],
}],
], # conditions
},
{
'target_name': 'vie_auto_test_lib',
'type': '<(library)',
'dependencies': [
'<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',
'<(webrtc_root)/modules/modules.gyp:video_render_module',
'<(webrtc_root)/modules/modules.gyp:video_capture_module',
'<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
'<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
'<(webrtc_root)/../test/metrics.gyp:metrics',
'<(webrtc_root)/../test/test.gyp:test_support',
'video_engine_core',
],
'include_dirs': [
'interface',
'helpers',
'primitives',
'../../include',
'../..',
'../../../modules/video_coding/codecs/interface',
'../../../common_video/interface',
],
'sources': [
'interface/tb_capture_device.h',
'interface/tb_external_transport.h',
'interface/tb_I420_codec.h',
'interface/tb_interfaces.h',
'interface/tb_video_channel.h',
'interface/vie_autotest.h',
'interface/vie_autotest_defines.h',
'interface/vie_autotest_linux.h',
'interface/vie_autotest_mac_carbon.h',
'interface/vie_autotest_mac_cocoa.h',
'interface/vie_autotest_main.h',
'interface/vie_autotest_window_manager_interface.h',
'interface/vie_autotest_windows.h',
'interface/vie_file_based_comparison_tests.h',
'interface/vie_window_manager_factory.h',
# Helper classes
'helpers/vie_fake_camera.cc',
'helpers/vie_fake_camera.h',
'helpers/vie_file_capture_device.cc',
'helpers/vie_file_capture_device.h',
'helpers/vie_to_file_renderer.cc',
'helpers/vie_to_file_renderer.h',
'helpers/vie_window_creator.cc',
'helpers/vie_window_creator.h',
# Test primitives
'primitives/base_primitives.cc',
'primitives/base_primitives.h',
'primitives/codec_primitives.cc',
'primitives/codec_primitives.h',
'primitives/framedrop_primitives.h',
'primitives/framedrop_primitives.cc',
'primitives/general_primitives.cc',
'primitives/general_primitives.h',
# Platform independent
'source/tb_capture_device.cc',
'source/tb_external_transport.cc',
'source/tb_I420_codec.cc',
'source/tb_interfaces.cc',
'source/tb_video_channel.cc',
'source/vie_autotest.cc',
'source/vie_autotest_base.cc',
'source/vie_autotest_capture.cc',
'source/vie_autotest_codec.cc',
'source/vie_autotest_encryption.cc',
'source/vie_autotest_file.cc',
'source/vie_autotest_image_process.cc',
'source/vie_autotest_loopback.cc',
'source/vie_autotest_main.cc',
'source/vie_autotest_network.cc',
'source/vie_autotest_render.cc',
'source/vie_autotest_rtp_rtcp.cc',
'source/vie_autotest_custom_call.cc',
'source/vie_autotest_simulcast.cc',
'source/vie_file_based_comparison_tests.cc',
],
'copies': [{
'destination': '/tmp',
'files': [
'media/captureDeviceImage.bmp',
'media/captureDeviceImage.jpg',
'media/renderStartImage.bmp',
'media/renderStartImage.jpg',
'media/renderTimeoutImage.bmp',
'media/renderTimeoutImage.jpg',
],
}],
},
{
'target_name': 'vie_auto_test_unittests',
'type': 'executable',
'dependencies': [
'vie_auto_test_lib',
'<(webrtc_root)/../test/test.gyp:test_support_main',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
],
'sources': [
'primitives/framedrop_primitives_unittest.cc',
],
},
],
}