Add a packet loss full stack test to the new API.
Remove all full stack tests for the old API. BUG=3750 R=pbos@webrtc.org Review URL: https://webrtc-codereview.appspot.com/23029004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@7442 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
stefan@webrtc.org
parent
a57678a70e
commit
c216b9aeaf
@ -392,6 +392,7 @@ void FullStackTest::RunTest(const FullStackTestParams& params) {
|
||||
send_config_.encoder_settings.encoder = encoder.get();
|
||||
send_config_.encoder_settings.payload_name = "VP8";
|
||||
send_config_.encoder_settings.payload_type = 124;
|
||||
send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
|
||||
|
||||
VideoStream* stream = &encoder_config_.streams[0];
|
||||
stream->width = params.clip.width;
|
||||
@ -403,6 +404,7 @@ void FullStackTest::RunTest(const FullStackTestParams& params) {
|
||||
|
||||
CreateMatchingReceiveConfigs();
|
||||
receive_configs_[0].renderer = &analyzer;
|
||||
receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
|
||||
|
||||
CreateStreams();
|
||||
analyzer.input_ = send_stream_->Input();
|
||||
@ -457,6 +459,20 @@ TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
|
||||
RunTest(foreman_cif);
|
||||
}
|
||||
|
||||
TEST_F(FullStackTest, ForemanCifPlr5) {
|
||||
FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5",
|
||||
{"foreman_cif", 352, 288, 30},
|
||||
30000,
|
||||
500000,
|
||||
2000000,
|
||||
0.0,
|
||||
0.0
|
||||
};
|
||||
foreman_cif.link.loss_percent = 5;
|
||||
foreman_cif.link.queue_delay_ms = 50;
|
||||
RunTest(foreman_cif);
|
||||
}
|
||||
|
||||
TEST_F(FullStackTest, ForemanCif500kbps) {
|
||||
FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
|
||||
{"foreman_cif", 352, 288, 30},
|
||||
|
||||
@ -115,105 +115,6 @@ class ViEVideoVerificationTest : public testing::Test {
|
||||
}
|
||||
};
|
||||
|
||||
class ParameterizedFullStackTest : public ViEVideoVerificationTest,
|
||||
public ::testing::WithParamInterface<int> {
|
||||
public:
|
||||
static const int kNumFullStackInstances = 4;
|
||||
|
||||
protected:
|
||||
struct TestParameters {
|
||||
NetworkParameters network;
|
||||
std::string file_name;
|
||||
int width;
|
||||
int height;
|
||||
int bitrate;
|
||||
double avg_psnr_threshold;
|
||||
double avg_ssim_threshold;
|
||||
ProtectionMethod protection_method;
|
||||
std::string test_label;
|
||||
};
|
||||
|
||||
void SetUp() {
|
||||
for (int i = 0; i < kNumFullStackInstances; ++i) {
|
||||
parameter_table_[i].file_name = webrtc::test::ResourcePath("foreman_cif",
|
||||
"yuv");
|
||||
parameter_table_[i].width = 352;
|
||||
parameter_table_[i].height = 288;
|
||||
}
|
||||
int i = 0;
|
||||
parameter_table_[i].protection_method = kNack;
|
||||
// Uniform loss => Setting burst length to -1.
|
||||
parameter_table_[i].network.loss_model = kUniformLoss;
|
||||
parameter_table_[i].network.packet_loss_rate = 0;
|
||||
parameter_table_[i].network.burst_length = -1;
|
||||
parameter_table_[i].network.mean_one_way_delay = 0;
|
||||
parameter_table_[i].network.std_dev_one_way_delay = 0;
|
||||
parameter_table_[i].bitrate = 300;
|
||||
// TODO(holmer): Enable for Win and Mac when the file rendering has been
|
||||
// moved to a separate thread.
|
||||
#ifdef WEBRTC_LINUX
|
||||
parameter_table_[i].avg_psnr_threshold = 35;
|
||||
parameter_table_[i].avg_ssim_threshold = 0.96;
|
||||
#else
|
||||
parameter_table_[i].avg_psnr_threshold = 0;
|
||||
parameter_table_[i].avg_ssim_threshold = 0.0;
|
||||
#endif
|
||||
parameter_table_[i].test_label = "net_delay_0_0_plr_0";
|
||||
++i;
|
||||
parameter_table_[i].protection_method = kNack;
|
||||
parameter_table_[i].network.loss_model = kUniformLoss;
|
||||
parameter_table_[i].network.packet_loss_rate = 5;
|
||||
parameter_table_[i].network.burst_length = -1;
|
||||
parameter_table_[i].network.mean_one_way_delay = 50;
|
||||
parameter_table_[i].network.std_dev_one_way_delay = 5;
|
||||
parameter_table_[i].bitrate = 300;
|
||||
// TODO(holmer): Enable for Win and Mac when the file rendering has been
|
||||
// moved to a separate thread.
|
||||
#ifdef WEBRTC_LINUX
|
||||
parameter_table_[i].avg_psnr_threshold = 35;
|
||||
parameter_table_[i].avg_ssim_threshold = 0.96;
|
||||
#else
|
||||
parameter_table_[i].avg_psnr_threshold = 0;
|
||||
parameter_table_[i].avg_ssim_threshold = 0.0;
|
||||
#endif
|
||||
parameter_table_[i].test_label = "net_delay_50_5_plr_5";
|
||||
++i;
|
||||
parameter_table_[i].protection_method = kNack;
|
||||
parameter_table_[i].network.loss_model = kUniformLoss;
|
||||
parameter_table_[i].network.packet_loss_rate = 0;
|
||||
parameter_table_[i].network.burst_length = -1;
|
||||
parameter_table_[i].network.mean_one_way_delay = 100;
|
||||
parameter_table_[i].network.std_dev_one_way_delay = 10;
|
||||
parameter_table_[i].bitrate = 300;
|
||||
// TODO(holmer): Enable for Win and Mac when the file rendering has been
|
||||
// moved to a separate thread.
|
||||
#ifdef WEBRTC_LINUX
|
||||
parameter_table_[i].avg_psnr_threshold = 35;
|
||||
parameter_table_[i].avg_ssim_threshold = 0.96;
|
||||
#else
|
||||
parameter_table_[i].avg_psnr_threshold = 0;
|
||||
parameter_table_[i].avg_ssim_threshold = 0.0;
|
||||
#endif
|
||||
parameter_table_[i].test_label = "net_delay_100_10_plr_0";
|
||||
++i;
|
||||
parameter_table_[i].protection_method = kNack;
|
||||
parameter_table_[i].network.loss_model = kGilbertElliotLoss;
|
||||
parameter_table_[i].network.packet_loss_rate = 5;
|
||||
parameter_table_[i].network.burst_length = 3;
|
||||
parameter_table_[i].network.mean_one_way_delay = 100;
|
||||
parameter_table_[i].network.std_dev_one_way_delay = 10;
|
||||
parameter_table_[i].bitrate = 300;
|
||||
// Thresholds disabled for now. This is being run mainly to get a graph.
|
||||
parameter_table_[i].avg_psnr_threshold = 0;
|
||||
parameter_table_[i].avg_ssim_threshold = 0.0;
|
||||
parameter_table_[i].test_label = "net_delay_100_10_plr_5_gilbert_elliot";
|
||||
|
||||
ASSERT_EQ(kNumFullStackInstances - 1, i);
|
||||
}
|
||||
|
||||
TestParameters parameter_table_[kNumFullStackInstances];
|
||||
};
|
||||
|
||||
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
|
||||
// I420 is lossless, so the I420 test should obviously get perfect results -
|
||||
// the local preview and remote output files should be bit-exact. This test
|
||||
@ -251,99 +152,4 @@ TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
|
||||
kNumAttempts << " attempts.";
|
||||
}
|
||||
|
||||
// Runs a whole stack processing with tracking of which frames are dropped
|
||||
// in the encoder. Tests show that they start at the same frame, which is
|
||||
// the important thing when doing frame-to-frame comparison with PSNR/SSIM.
|
||||
TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) {
|
||||
// Using CIF here since it's a more common resolution than QCIF, and higher
|
||||
// resolutions shouldn't be a problem for a test using VP8.
|
||||
input_file_ = parameter_table_[GetParam()].file_name;
|
||||
FrameDropDetector detector;
|
||||
local_file_renderer_ = new ViEToFileRenderer();
|
||||
remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector);
|
||||
SetUpLocalFileRenderer(local_file_renderer_);
|
||||
SetUpRemoteFileRenderer(remote_file_renderer_);
|
||||
|
||||
// Set a low bit rate so the encoder budget will be tight, causing it to drop
|
||||
// frames every now and then.
|
||||
const int kBitRateKbps = parameter_table_[GetParam()].bitrate;
|
||||
const NetworkParameters network = parameter_table_[GetParam()].network;
|
||||
int width = parameter_table_[GetParam()].width;
|
||||
int height = parameter_table_[GetParam()].height;
|
||||
ProtectionMethod protection_method =
|
||||
parameter_table_[GetParam()].protection_method;
|
||||
ViETest::Log("Bit rate : %5d kbps", kBitRateKbps);
|
||||
ViETest::Log("Packet loss : %5d %%", network.packet_loss_rate);
|
||||
ViETest::Log("Network delay: mean=%dms std dev=%d ms",
|
||||
network.mean_one_way_delay, network.std_dev_one_way_delay);
|
||||
tests_.TestFullStack(input_file_, width, height, kBitRateKbps,
|
||||
protection_method, network, local_file_renderer_,
|
||||
remote_file_renderer_, &detector);
|
||||
const std::string reference_file = local_file_renderer_->GetFullOutputPath();
|
||||
const std::string output_file = remote_file_renderer_->GetFullOutputPath();
|
||||
StopRenderers();
|
||||
|
||||
detector.CalculateResults();
|
||||
detector.PrintReport(parameter_table_[GetParam()].test_label);
|
||||
|
||||
if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) >
|
||||
detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) {
|
||||
detector.PrintDebugDump();
|
||||
}
|
||||
|
||||
ASSERT_GE(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered),
|
||||
detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded))
|
||||
<< "The number of dropped frames on the decode and render steps are not "
|
||||
"equal. This may be because we have a major problem in the buffers of "
|
||||
"the ViEToFileRenderer?";
|
||||
|
||||
// We may have dropped frames during the processing, which means the output
|
||||
// file does not contain all the frames that are present in the input file.
|
||||
// To make the quality measurement correct, we must adjust the output file to
|
||||
// that by copying the last successful frame into the place where the dropped
|
||||
// frame would be, for all dropped frames.
|
||||
const int frame_length_in_bytes = 3 * width * height / 2;
|
||||
ViETest::Log("Frame length: %d bytes", frame_length_in_bytes);
|
||||
std::vector<Frame*> all_frames = detector.GetAllFrames();
|
||||
FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames);
|
||||
|
||||
// Verify all sent frames are present in the output file.
|
||||
size_t output_file_size = webrtc::test::GetFileSize(output_file);
|
||||
EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes)
|
||||
<< "The output file size is incorrect. It should be equal to the number "
|
||||
"of frames multiplied by the frame size. This will likely affect "
|
||||
"PSNR/SSIM calculations in a bad way.";
|
||||
|
||||
TearDownFileRenderers();
|
||||
|
||||
// We are running on a lower bitrate here so we need to settle for somewhat
|
||||
// lower PSNR and SSIM values.
|
||||
double actual_psnr = 0;
|
||||
double actual_ssim = 0;
|
||||
CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim);
|
||||
|
||||
const double kExpectedMinimumPSNR =
|
||||
parameter_table_[GetParam()].avg_psnr_threshold;
|
||||
const double kExpectedMinimumSSIM =
|
||||
parameter_table_[GetParam()].avg_ssim_threshold;
|
||||
|
||||
EXPECT_GE(actual_psnr, kExpectedMinimumPSNR);
|
||||
EXPECT_GE(actual_ssim, kExpectedMinimumSSIM);
|
||||
|
||||
std::stringstream ss;
|
||||
ss << std::setprecision(3) << std::fixed << actual_psnr;
|
||||
webrtc::test::PrintResult(
|
||||
"psnr", "", parameter_table_[GetParam()].test_label,
|
||||
ss.str(), "dB", false);
|
||||
|
||||
ss.str("");
|
||||
ss << std::setprecision(3) << std::fixed << actual_ssim;
|
||||
webrtc::test::PrintResult(
|
||||
"ssim", "", parameter_table_[GetParam()].test_label,
|
||||
ss.str(), "", false);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest,
|
||||
::testing::Range(0, ParameterizedFullStackTest::kNumFullStackInstances));
|
||||
|
||||
} // namespace
|
||||
|
||||
Loading…
Reference in New Issue
Block a user