generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Integration test for videoprocessor

Browse Source 
Added temporal layers number flag for video_quality_measurement tool.
This tool now also uses webrtc::VideoCodingModule::Codec() to get its
VideoCodec struct configuration instead of filling it in manually.

Updated paths for header files to use full directory paths.

Tested in Debug+Release on Linux, Mac and Windows. Passes Valgrind memcheck on Linux.

BUG=
TEST=video_codecs_test_framework_integrationtests. Also executed out/Debug/video_quality_measurement --input_filename=resources/foreman_cif.yuv  --width=352 --height=288

Review URL: http://webrtc-codereview.appspot.com/339001

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1310 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
kjellander@webrtc.org 2011-12-30 14:44:07 +00:00
parent 62665b8cd3
commit a643d5c4ef
8 changed files with 251 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi
View File

@ -31,9 +31,9 @@
'target_name': 'video_codecs_test_framework_unittests',
'type': 'executable',
'dependencies': [
'video_codecs_test_framework',
'video_codecs_test_framework',
'<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',
'<(webrtc_root)/../testing/gmock.gyp:gmock',
'<(webrtc_root)/../testing/gmock.gyp:gmock',
'<(webrtc_root)/../test/test.gyp:test_support_main',
],
'sources': [
@ -42,6 +42,20 @@
'videoprocessor_unittest.cc',
],
},
{
'target_name': 'video_codecs_test_framework_integrationtests',
'type': 'executable',
'dependencies': [
'video_codecs_test_framework',
'webrtc_video_coding',
'webrtc_vp8',
'<(webrtc_root)/../testing/gtest.gyp:gtest',
'<(webrtc_root)/../test/test.gyp:test_support_main',
],
'sources': [
'videoprocessor_integrationtest.cc',
],
},
], # targets
}], # build_with_chromium
], # conditions

16
src/modules/video_coding/codecs/test/videoprocessor.cc
View File

@ -49,7 +49,7 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder,
bool VideoProcessorImpl::Init() {
// Calculate a factor used for bit rate calculations:
bit_rate_factor_ = config_.codec_settings.maxFramerate * 0.001 * 8; // bits
bit_rate_factor_ = config_.codec_settings->maxFramerate * 0.001 * 8; // bits
int frame_length_in_bytes = frame_reader_->FrameLength();
@ -58,8 +58,8 @@ bool VideoProcessorImpl::Init() {
last_successful_frame_buffer_ = new WebRtc_UWord8[frame_length_in_bytes];
// Set fixed properties common for all frames:
source_frame_._width = config_.codec_settings.width;
source_frame_._height = config_.codec_settings.height;
source_frame_._width = config_.codec_settings->width;
source_frame_._height = config_.codec_settings->height;
source_frame_._length = frame_length_in_bytes;
source_frame_._size = frame_length_in_bytes;
@ -85,14 +85,14 @@ bool VideoProcessorImpl::Init() {
nbr_of_cores = CpuInfo::DetectNumberOfCores();
}
WebRtc_Word32 init_result =
encoder_->InitEncode(&config_.codec_settings, nbr_of_cores,
encoder_->InitEncode(config_.codec_settings, nbr_of_cores,
config_.networking_config.max_payload_size_in_bytes);
if (init_result != WEBRTC_VIDEO_CODEC_OK) {
fprintf(stderr, "Failed to initialize VideoEncoder, return code: %d\n",
init_result);
return false;
}
init_result = decoder_->InitDecode(&config_.codec_settings, nbr_of_cores);
init_result = decoder_->InitDecode(config_.codec_settings, nbr_of_cores);
if (init_result != WEBRTC_VIDEO_CODEC_OK) {
fprintf(stderr, "Failed to initialize VideoDecoder, return code: %d\n",
init_result);
@ -105,9 +105,9 @@ bool VideoProcessorImpl::Init() {
printf(" Total # of frames: %d\n", frame_reader_->NumberOfFrames());
printf(" Codec settings:\n");
printf(" Start bitrate : %d kbps\n",
config_.codec_settings.startBitrate);
printf(" Width : %d\n", config_.codec_settings.width);
printf(" Height : %d\n", config_.codec_settings.height);
config_.codec_settings->startBitrate);
printf(" Width : %d\n", config_.codec_settings->width);
printf(" Height : %d\n", config_.codec_settings->height);
}
initialized_ = true;
return true;

7
src/modules/video_coding/codecs/test/videoprocessor.h
View File

@ -42,7 +42,7 @@ struct TestConfig {
input_filename(""), output_filename(""), output_dir("out"),
networking_config(), exclude_frame_types(kExcludeOnlyFirstKeyFrame),
frame_length_in_bytes(-1), use_single_core(false), keyframe_interval(0),
verbose(true) {
codec_settings(NULL), verbose(true) {
};
// Name of the test. This is purely metadata and does not affect
@ -99,8 +99,9 @@ struct TestConfig {
int keyframe_interval;
// The codec settings to use for the test (target bitrate, video size,
// framerate and so on)
webrtc::VideoCodec codec_settings;
// framerate and so on). This struct must be created and filled in using
// the VideoCodingModule::Codec() method.
webrtc::VideoCodec* codec_settings;
// If printing of information to stdout shall be performed during processing.
bool verbose;

161
src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc Normal file
View File

@ -0,0 +1,161 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "gtest/gtest.h"
#include "modules/video_coding/codecs/interface/video_codec_interface.h"
#include "modules/video_coding/codecs/test/packet_manipulator.h"
#include "modules/video_coding/codecs/test/videoprocessor.h"
#include "modules/video_coding/codecs/vp8/main/interface/vp8.h"
#include "modules/video_coding/main/interface/video_coding.h"
#include "testsupport/fileutils.h"
#include "testsupport/frame_reader.h"
#include "testsupport/frame_writer.h"
#include "testsupport/metrics/video_metrics.h"
#include "testsupport/packet_reader.h"
#include "typedefs.h"
namespace webrtc {
const int kNbrFrames = 61; // foreman_cif_short.yuv
const int kCIFWidth = 352;
const int kCIFHeight = 288;
const int kBitRateKbps = 500;
// Integration test for video processor. Encodes+decodes a small clip and
// writes it to the output directory. After completion, PSNR and SSIM
// measurements are performed on the original and the processed clip to verify
// the quality is acceptable.
// The limits for the PSNR and SSIM values must be set quite low, since we have
// no control over the random function used for packet loss in this test.
class VideoProcessorIntegrationTest: public testing::Test {
protected:
VideoEncoder* encoder_;
VideoDecoder* decoder_;
webrtc::test::FrameReader* frame_reader_;
webrtc::test::FrameWriter* frame_writer_;
webrtc::test::PacketReader packet_reader_;
webrtc::test::PacketManipulator* packet_manipulator_;
webrtc::test::Stats stats_;
webrtc::test::TestConfig config_;
VideoCodec codec_settings_;
webrtc::test::VideoProcessor* processor_;
VideoProcessorIntegrationTest() {}
virtual ~VideoProcessorIntegrationTest() {}
void SetUp() {
encoder_ = new VP8Encoder();
decoder_ = new VP8Decoder();
// Setup the TestConfig struct for processing of a clip in CIF resolution.
config_.input_filename =
webrtc::test::ResourcePath("foreman_cif_short", "yuv");
config_.output_filename = webrtc::test::OutputPath() +
"foreman_cif_short_video_codecs_test_framework_integrationtests.yuv";
config_.frame_length_in_bytes = 3 * kCIFWidth * kCIFHeight / 2;
config_.verbose = false;
// Get a codec configuration struct and configure it.
VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_);
config_.codec_settings = &codec_settings_;
config_.codec_settings->startBitrate = kBitRateKbps;
config_.codec_settings->width = kCIFWidth;
config_.codec_settings->height = kCIFHeight;
frame_reader_ =
new webrtc::test::FrameReaderImpl(config_.input_filename,
config_.frame_length_in_bytes);
frame_writer_ =
new webrtc::test::FrameWriterImpl(config_.output_filename,
config_.frame_length_in_bytes);
ASSERT_TRUE(frame_reader_->Init());
ASSERT_TRUE(frame_writer_->Init());
packet_manipulator_ = new webrtc::test::PacketManipulatorImpl(
&packet_reader_, config_.networking_config, config_.verbose);
processor_ = new webrtc::test::VideoProcessorImpl(encoder_, decoder_,
frame_reader_,
frame_writer_,
packet_manipulator_,
config_, &stats_);
ASSERT_TRUE(processor_->Init());
}
void TearDown() {
delete processor_;
delete packet_manipulator_;
delete frame_writer_;
delete frame_reader_;
delete decoder_;
delete encoder_;
}
// Processes all frames in the clip and verifies the result.
// The average PSNR for all frames is required to be 2.0 higher than the
// minimum_psnr parameter.
// The minimum SSIM for all frames is required to be 0.1 higher than the
// minimum_ssim parameter.
void ProcessFramesAndVerify(double minimum_psnr, double minimum_ssim) {
int frame_number = 0;
while (processor_->ProcessFrame(frame_number)) {
frame_number++;
}
EXPECT_EQ(kNbrFrames, frame_number);
EXPECT_EQ(kNbrFrames, static_cast<int>(stats_.stats_.size()));
// Release encoder and decoder to make sure they have finished processing:
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
// Close the files before we start using them for SSIM/PSNR calculations.
frame_reader_->Close();
frame_writer_->Close();
QualityMetricsResult result;
EXPECT_EQ(0, PsnrFromFiles(config_.input_filename.c_str(),
config_.output_filename.c_str(),
config_.codec_settings->width,
config_.codec_settings->height, &result));
EXPECT_GT(result.average, minimum_psnr + 2.0);
EXPECT_GT(result.min, minimum_psnr);
EXPECT_EQ(0, SsimFromFiles(config_.input_filename.c_str(),
config_.output_filename.c_str(),
config_.codec_settings->width,
config_.codec_settings->height, &result));
EXPECT_GT(result.average, minimum_ssim + 0.1);
EXPECT_GT(result.min, minimum_ssim);
}
};
// Run with no packet loss. Quality should be very high.
TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) {
config_.networking_config.packet_loss_probability = 0;
double minimum_psnr = 30;
double minimum_ssim = 0.7;
ProcessFramesAndVerify(minimum_psnr, minimum_ssim);
}
// Run with 5% packet loss. Quality should be a bit lower.
TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) {
config_.networking_config.packet_loss_probability = 0.05;
double minimum_psnr = 14;
double minimum_ssim = 0.3;
ProcessFramesAndVerify(minimum_psnr, minimum_ssim);
}
// Run with 10% packet loss. Quality should be even lower.
TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) {
config_.networking_config.packet_loss_probability = 0.10;
double minimum_psnr = 12;
double minimum_ssim = 0.2;
ProcessFramesAndVerify(minimum_psnr, minimum_ssim);
}
} // namespace webrtc

1
src/modules/video_coding/codecs/tools/video_codecs_tools.gypi
View File

@ -16,6 +16,7 @@
'type': 'executable',
'dependencies': [
'video_codecs_test_framework',
'webrtc_video_coding',
'webrtc_vp8',
'<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags',
],

92
src/modules/video_coding/codecs/tools/video_quality_measurement.cc
View File

@ -18,11 +18,13 @@
#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR)
#endif
#include "common_types.h"
#include "google/gflags.h"
#include "modules/video_coding/codecs/test/packet_manipulator.h"
#include "modules/video_coding/codecs/test/stats.h"
#include "modules/video_coding/codecs/test/videoprocessor.h"
#include "modules/video_coding/codecs/vp8/main/interface/vp8.h"
#include "modules/video_coding/main/interface/video_coding.h"
#include "system_wrappers/interface/trace.h"
#include "testsupport/frame_reader.h"
#include "testsupport/frame_writer.h"
@ -63,6 +65,8 @@ DEFINE_int32(keyframe_interval, 0, "Forces a keyframe every Nth frame. "
"0 means the encoder decides when to insert keyframes. Note that "
"the encoder may create a keyframe in other locations in addition "
"to the interval that is set using this parameter.");
DEFINE_int32(temporal_layers, 0, "The number of temporal layers to use "
"(VP8 specific codec setting). Must be 0-4.");
DEFINE_int32(packet_size, 1500, "Simulated network packet size in bytes (MTU). "
"Used for packet loss simulation.");
DEFINE_int32(max_payload_size, 1440, "Max payload size in bytes for the "
@ -87,8 +91,8 @@ DEFINE_bool(verbose, true, "Verbose mode. Prints a lot of debugging info. "
"Suitable for tracking progress but not for capturing output. "
"Disable with --noverbose flag.");
// Custom log method that only prints if the verbose flag is given
// Supports all the standard printf parameters and formatting (just forwarded)
// Custom log method that only prints if the verbose flag is given.
// Supports all the standard printf parameters and formatting (just forwarded).
int Log(const char *format, ...) {
int result = 0;
if (FLAGS_verbose) {
@ -100,7 +104,8 @@ int Log(const char *format, ...) {
return result;
}
// Validates the arguments given as command line flags.
// Validates the arguments given as command line flags and fills in the
// TestConfig struct with all configurations needed for video processing.
// Returns 0 if everything is OK, otherwise an exit code.
int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
// Validate the mandatory flags:
@ -111,7 +116,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
config->name = FLAGS_test_name;
config->description = FLAGS_test_description;
// Verify the input file exists and is readable:
// Verify the input file exists and is readable.
FILE* test_file;
test_file = fopen(FLAGS_input_filename.c_str(), "rb");
if (test_file == NULL) {
@ -122,7 +127,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
fclose(test_file);
config->input_filename = FLAGS_input_filename;
// Verify the output dir exists:
// Verify the output dir exists.
struct stat dir_info;
if (!(stat(FLAGS_output_dir.c_str(), &dir_info) == 0 &&
S_ISDIR(dir_info.st_mode))) {
@ -132,7 +137,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
}
config->output_dir = FLAGS_output_dir;
// Manufacture an output filename if none was given:
// Manufacture an output filename if none was given.
if (FLAGS_output_filename == "") {
// Cut out the filename without extension from the given input file
// (which may include a path)
@ -146,7 +151,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
- startIndex) + "_out.yuv";
}
// Verify output file can be written
// Verify output file can be written.
if (FLAGS_output_dir == ".") {
config->output_filename = FLAGS_output_filename;
} else {
@ -160,23 +165,37 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
}
fclose(test_file);
// Check single core flag
// Check single core flag.
config->use_single_core = FLAGS_use_single_core;
// Seed our random function if that flag is enabled. This will force
// repeatable behaviour between runs
// repeatable behaviour between runs.
if (!FLAGS_disable_fixed_random_seed) {
srand(0);
}
// Check the bit rate
// Get codec specific configuration.
webrtc::VideoCodingModule::Codec(webrtc::kVideoCodecVP8,
config->codec_settings);
// Check the temporal layers.
if (FLAGS_temporal_layers < 0 ||
FLAGS_temporal_layers > webrtc::kMaxTemporalStreams) {
fprintf(stderr, "Temporal layers number must be 0-4, was: %d\n",
FLAGS_temporal_layers);
return 13;
}
config->codec_settings->codecSpecific.VP8.numberOfTemporalLayers =
FLAGS_temporal_layers;
// Check the bit rate.
if (FLAGS_bitrate <= 0) {
fprintf(stderr, "Bit rate must be >0 kbps, was: %d\n", FLAGS_bitrate);
return 5;
}
config->codec_settings.startBitrate = FLAGS_bitrate;
config->codec_settings->startBitrate = FLAGS_bitrate;
// Check the keyframe interval
// Check the keyframe interval.
if (FLAGS_keyframe_interval < 0) {
fprintf(stderr, "Keyframe interval must be >=0, was: %d\n",
FLAGS_keyframe_interval);
@ -184,7 +203,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
}
config->keyframe_interval = FLAGS_keyframe_interval;
// Check packet size and max payload size
// Check packet size and max payload size.
if (FLAGS_packet_size <= 0) {
fprintf(stderr, "Packet size must be >0 bytes, was: %d\n",
FLAGS_packet_size);
@ -205,14 +224,13 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) {
fprintf(stderr, "Width and height must be >0.");
return 9;
}
config->codec_settings.codecType = webrtc::kVideoCodecVP8;
config->codec_settings.width = FLAGS_width;
config->codec_settings.height = FLAGS_height;
config->codec_settings.maxFramerate = FLAGS_framerate;
config->codec_settings->width = FLAGS_width;
config->codec_settings->height = FLAGS_height;
config->codec_settings->maxFramerate = FLAGS_framerate;
// Calculate the size of each frame to read (according to YUV spec):
// Calculate the size of each frame to read (according to YUV spec).
config->frame_length_in_bytes =
3 * config->codec_settings.width * config->codec_settings.height / 2;
3 * config->codec_settings->width * config->codec_settings->height / 2;
// Check packet loss settings
if (FLAGS_packet_loss_mode != "uniform" &&
@ -250,8 +268,8 @@ void CalculateSsimVideoMetrics(webrtc::test::TestConfig* config,
QualityMetricsResult* ssimResult) {
Log("Calculating SSIM...\n");
SsimFromFiles(config->input_filename.c_str(), config->output_filename.c_str(),
config->codec_settings.width,
config->codec_settings.height, ssimResult);
config->codec_settings->width,
config->codec_settings->height, ssimResult);
Log(" Average: %3.2f\n", ssimResult->average);
Log(" Min : %3.2f (frame %d)\n", ssimResult->min,
ssimResult->min_frame_number);
@ -263,8 +281,8 @@ void CalculatePsnrVideoMetrics(webrtc::test::TestConfig* config,
QualityMetricsResult* psnrResult) {
Log("Calculating PSNR...\n");
PsnrFromFiles(config->input_filename.c_str(), config->output_filename.c_str(),
config->codec_settings.width,
config->codec_settings.height, psnrResult);
config->codec_settings->width,
config->codec_settings->height, psnrResult);
Log(" Average: %3.2f\n", psnrResult->average);
Log(" Min : %3.2f (frame %d)\n", psnrResult->min,
psnrResult->min_frame_number);
@ -370,10 +388,10 @@ void PrintPythonOutput(const webrtc::test::TestConfig& config,
config.frame_length_in_bytes,
config.use_single_core ? "True " : "False",
config.keyframe_interval,
webrtc::test::VideoCodecTypeToStr(config.codec_settings.codecType),
config.codec_settings.width,
config.codec_settings.height,
config.codec_settings.startBitrate);
webrtc::test::VideoCodecTypeToStr(config.codec_settings->codecType),
config.codec_settings->width,
config.codec_settings->height,
config.codec_settings->startBitrate);
printf("frame_data_types = {"
"'frame_number': ('number', 'Frame number'),\n"
"'encoding_successful': ('boolean', 'Encoding successful?'),\n"
@ -433,9 +451,13 @@ int main(int argc, char* argv[]) {
google::ParseCommandLineFlags(&argc, &argv, true);
// Create TestConfig and codec settings struct.
webrtc::test::TestConfig config;
webrtc::VideoCodec codec_settings;
config.codec_settings = &codec_settings;
int return_code = HandleCommandLineFlags(&config);
// Exit if an invalid argument is supplied:
// Exit if an invalid argument is supplied.
if (return_code != 0) {
return return_code;
}
@ -473,7 +495,7 @@ int main(int argc, char* argv[]) {
Log("\n");
Log("Processed %d frames\n", frame_number);
// Release encoder and decoder to make sure they have finished processing:
// Release encoder and decoder to make sure they have finished processing.
encoder.Release();
decoder.Release();
@ -486,16 +508,16 @@ int main(int argc, char* argv[]) {
stats.PrintSummary();
QualityMetricsResult ssimResult;
CalculateSsimVideoMetrics(&config, &ssimResult);
QualityMetricsResult psnrResult;
CalculatePsnrVideoMetrics(&config, &psnrResult);
QualityMetricsResult ssim_result;
CalculateSsimVideoMetrics(&config, &ssim_result);
QualityMetricsResult psnr_result;
CalculatePsnrVideoMetrics(&config, &psnr_result);
if (FLAGS_csv) {
PrintCsvOutput(stats, ssimResult, psnrResult);
PrintCsvOutput(stats, ssim_result, psnr_result);
}
if (FLAGS_python) {
PrintPythonOutput(config, stats, ssimResult, psnrResult);
PrintPythonOutput(config, stats, ssim_result, psnr_result);
}
Log("Quality test finished!");
return 0;

6
src/modules/video_coding/main/interface/video_coding.h
View File

@ -11,9 +11,9 @@
#ifndef WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_
#define WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_
#include "module.h"
#include "module_common_types.h"
#include "video_coding_defines.h"
#include "modules/interface/module.h"
#include "modules/interface/module_common_types.h"
#include "modules/video_coding/main/interface/video_coding_defines.h"
namespace webrtc
{

2
src/modules/video_coding/main/interface/video_coding_defines.h
View File

@ -12,7 +12,7 @@
#define WEBRTC_MODULES_INTERFACE_VIDEO_CODING_DEFINES_H_
#include "typedefs.h"
#include "module_common_types.h"
#include "modules/interface/module_common_types.h"
namespace webrtc
{