diff --git a/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi index 7e3174b13..c38a5c8c0 100644 --- a/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi +++ b/src/modules/video_coding/codecs/test/video_codecs_test_framework.gypi @@ -31,9 +31,9 @@ 'target_name': 'video_codecs_test_framework_unittests', 'type': 'executable', 'dependencies': [ - 'video_codecs_test_framework', + 'video_codecs_test_framework', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', - '<(webrtc_root)/../testing/gmock.gyp:gmock', + '<(webrtc_root)/../testing/gmock.gyp:gmock', '<(webrtc_root)/../test/test.gyp:test_support_main', ], 'sources': [ @@ -42,6 +42,20 @@ 'videoprocessor_unittest.cc', ], }, + { + 'target_name': 'video_codecs_test_framework_integrationtests', + 'type': 'executable', + 'dependencies': [ + 'video_codecs_test_framework', + 'webrtc_video_coding', + 'webrtc_vp8', + '<(webrtc_root)/../testing/gtest.gyp:gtest', + '<(webrtc_root)/../test/test.gyp:test_support_main', + ], + 'sources': [ + 'videoprocessor_integrationtest.cc', + ], + }, ], # targets }], # build_with_chromium ], # conditions diff --git a/src/modules/video_coding/codecs/test/videoprocessor.cc b/src/modules/video_coding/codecs/test/videoprocessor.cc index 9b210ef19..cf0869b2b 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor.cc +++ b/src/modules/video_coding/codecs/test/videoprocessor.cc @@ -49,7 +49,7 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, bool VideoProcessorImpl::Init() { // Calculate a factor used for bit rate calculations: - bit_rate_factor_ = config_.codec_settings.maxFramerate * 0.001 * 8; // bits + bit_rate_factor_ = config_.codec_settings->maxFramerate * 0.001 * 8; // bits int frame_length_in_bytes = frame_reader_->FrameLength(); @@ -58,8 +58,8 @@ bool VideoProcessorImpl::Init() { last_successful_frame_buffer_ = new WebRtc_UWord8[frame_length_in_bytes]; // Set fixed properties common for all frames: - source_frame_._width = config_.codec_settings.width; - source_frame_._height = config_.codec_settings.height; + source_frame_._width = config_.codec_settings->width; + source_frame_._height = config_.codec_settings->height; source_frame_._length = frame_length_in_bytes; source_frame_._size = frame_length_in_bytes; @@ -85,14 +85,14 @@ bool VideoProcessorImpl::Init() { nbr_of_cores = CpuInfo::DetectNumberOfCores(); } WebRtc_Word32 init_result = - encoder_->InitEncode(&config_.codec_settings, nbr_of_cores, + encoder_->InitEncode(config_.codec_settings, nbr_of_cores, config_.networking_config.max_payload_size_in_bytes); if (init_result != WEBRTC_VIDEO_CODEC_OK) { fprintf(stderr, "Failed to initialize VideoEncoder, return code: %d\n", init_result); return false; } - init_result = decoder_->InitDecode(&config_.codec_settings, nbr_of_cores); + init_result = decoder_->InitDecode(config_.codec_settings, nbr_of_cores); if (init_result != WEBRTC_VIDEO_CODEC_OK) { fprintf(stderr, "Failed to initialize VideoDecoder, return code: %d\n", init_result); @@ -105,9 +105,9 @@ bool VideoProcessorImpl::Init() { printf(" Total # of frames: %d\n", frame_reader_->NumberOfFrames()); printf(" Codec settings:\n"); printf(" Start bitrate : %d kbps\n", - config_.codec_settings.startBitrate); - printf(" Width : %d\n", config_.codec_settings.width); - printf(" Height : %d\n", config_.codec_settings.height); + config_.codec_settings->startBitrate); + printf(" Width : %d\n", config_.codec_settings->width); + printf(" Height : %d\n", config_.codec_settings->height); } initialized_ = true; return true; diff --git a/src/modules/video_coding/codecs/test/videoprocessor.h b/src/modules/video_coding/codecs/test/videoprocessor.h index 0ac0d6c77..7c2c14f52 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor.h +++ b/src/modules/video_coding/codecs/test/videoprocessor.h @@ -42,7 +42,7 @@ struct TestConfig { input_filename(""), output_filename(""), output_dir("out"), networking_config(), exclude_frame_types(kExcludeOnlyFirstKeyFrame), frame_length_in_bytes(-1), use_single_core(false), keyframe_interval(0), - verbose(true) { + codec_settings(NULL), verbose(true) { }; // Name of the test. This is purely metadata and does not affect @@ -99,8 +99,9 @@ struct TestConfig { int keyframe_interval; // The codec settings to use for the test (target bitrate, video size, - // framerate and so on) - webrtc::VideoCodec codec_settings; + // framerate and so on). This struct must be created and filled in using + // the VideoCodingModule::Codec() method. + webrtc::VideoCodec* codec_settings; // If printing of information to stdout shall be performed during processing. bool verbose; diff --git a/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc new file mode 100644 index 000000000..6cc710f18 --- /dev/null +++ b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "gtest/gtest.h" +#include "modules/video_coding/codecs/interface/video_codec_interface.h" +#include "modules/video_coding/codecs/test/packet_manipulator.h" +#include "modules/video_coding/codecs/test/videoprocessor.h" +#include "modules/video_coding/codecs/vp8/main/interface/vp8.h" +#include "modules/video_coding/main/interface/video_coding.h" +#include "testsupport/fileutils.h" +#include "testsupport/frame_reader.h" +#include "testsupport/frame_writer.h" +#include "testsupport/metrics/video_metrics.h" +#include "testsupport/packet_reader.h" +#include "typedefs.h" + +namespace webrtc { + +const int kNbrFrames = 61; // foreman_cif_short.yuv +const int kCIFWidth = 352; +const int kCIFHeight = 288; +const int kBitRateKbps = 500; + +// Integration test for video processor. Encodes+decodes a small clip and +// writes it to the output directory. After completion, PSNR and SSIM +// measurements are performed on the original and the processed clip to verify +// the quality is acceptable. +// The limits for the PSNR and SSIM values must be set quite low, since we have +// no control over the random function used for packet loss in this test. +class VideoProcessorIntegrationTest: public testing::Test { + protected: + VideoEncoder* encoder_; + VideoDecoder* decoder_; + webrtc::test::FrameReader* frame_reader_; + webrtc::test::FrameWriter* frame_writer_; + webrtc::test::PacketReader packet_reader_; + webrtc::test::PacketManipulator* packet_manipulator_; + webrtc::test::Stats stats_; + webrtc::test::TestConfig config_; + VideoCodec codec_settings_; + webrtc::test::VideoProcessor* processor_; + + VideoProcessorIntegrationTest() {} + virtual ~VideoProcessorIntegrationTest() {} + + void SetUp() { + encoder_ = new VP8Encoder(); + decoder_ = new VP8Decoder(); + + // Setup the TestConfig struct for processing of a clip in CIF resolution. + config_.input_filename = + webrtc::test::ResourcePath("foreman_cif_short", "yuv"); + config_.output_filename = webrtc::test::OutputPath() + + "foreman_cif_short_video_codecs_test_framework_integrationtests.yuv"; + config_.frame_length_in_bytes = 3 * kCIFWidth * kCIFHeight / 2; + config_.verbose = false; + + // Get a codec configuration struct and configure it. + VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); + config_.codec_settings = &codec_settings_; + config_.codec_settings->startBitrate = kBitRateKbps; + config_.codec_settings->width = kCIFWidth; + config_.codec_settings->height = kCIFHeight; + + frame_reader_ = + new webrtc::test::FrameReaderImpl(config_.input_filename, + config_.frame_length_in_bytes); + frame_writer_ = + new webrtc::test::FrameWriterImpl(config_.output_filename, + config_.frame_length_in_bytes); + ASSERT_TRUE(frame_reader_->Init()); + ASSERT_TRUE(frame_writer_->Init()); + + packet_manipulator_ = new webrtc::test::PacketManipulatorImpl( + &packet_reader_, config_.networking_config, config_.verbose); + processor_ = new webrtc::test::VideoProcessorImpl(encoder_, decoder_, + frame_reader_, + frame_writer_, + packet_manipulator_, + config_, &stats_); + ASSERT_TRUE(processor_->Init()); + } + + void TearDown() { + delete processor_; + delete packet_manipulator_; + delete frame_writer_; + delete frame_reader_; + delete decoder_; + delete encoder_; + } + + // Processes all frames in the clip and verifies the result. + // The average PSNR for all frames is required to be 2.0 higher than the + // minimum_psnr parameter. + // The minimum SSIM for all frames is required to be 0.1 higher than the + // minimum_ssim parameter. + void ProcessFramesAndVerify(double minimum_psnr, double minimum_ssim) { + int frame_number = 0; + while (processor_->ProcessFrame(frame_number)) { + frame_number++; + } + EXPECT_EQ(kNbrFrames, frame_number); + EXPECT_EQ(kNbrFrames, static_cast(stats_.stats_.size())); + + // Release encoder and decoder to make sure they have finished processing: + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); + // Close the files before we start using them for SSIM/PSNR calculations. + frame_reader_->Close(); + frame_writer_->Close(); + + QualityMetricsResult result; + EXPECT_EQ(0, PsnrFromFiles(config_.input_filename.c_str(), + config_.output_filename.c_str(), + config_.codec_settings->width, + config_.codec_settings->height, &result)); + EXPECT_GT(result.average, minimum_psnr + 2.0); + EXPECT_GT(result.min, minimum_psnr); + + EXPECT_EQ(0, SsimFromFiles(config_.input_filename.c_str(), + config_.output_filename.c_str(), + config_.codec_settings->width, + config_.codec_settings->height, &result)); + EXPECT_GT(result.average, minimum_ssim + 0.1); + EXPECT_GT(result.min, minimum_ssim); + } +}; + +// Run with no packet loss. Quality should be very high. +TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { + config_.networking_config.packet_loss_probability = 0; + double minimum_psnr = 30; + double minimum_ssim = 0.7; + ProcessFramesAndVerify(minimum_psnr, minimum_ssim); +} + +// Run with 5% packet loss. Quality should be a bit lower. +TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { + config_.networking_config.packet_loss_probability = 0.05; + double minimum_psnr = 14; + double minimum_ssim = 0.3; + ProcessFramesAndVerify(minimum_psnr, minimum_ssim); +} + +// Run with 10% packet loss. Quality should be even lower. +TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { + config_.networking_config.packet_loss_probability = 0.10; + double minimum_psnr = 12; + double minimum_ssim = 0.2; + ProcessFramesAndVerify(minimum_psnr, minimum_ssim); +} + +} // namespace webrtc diff --git a/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi index ba5d62ee8..2c72b27f0 100644 --- a/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi +++ b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi @@ -16,6 +16,7 @@ 'type': 'executable', 'dependencies': [ 'video_codecs_test_framework', + 'webrtc_video_coding', 'webrtc_vp8', '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', ], diff --git a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc index 0394358b5..59f042c0d 100644 --- a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc +++ b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc @@ -18,11 +18,13 @@ #define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) #endif +#include "common_types.h" #include "google/gflags.h" #include "modules/video_coding/codecs/test/packet_manipulator.h" #include "modules/video_coding/codecs/test/stats.h" #include "modules/video_coding/codecs/test/videoprocessor.h" #include "modules/video_coding/codecs/vp8/main/interface/vp8.h" +#include "modules/video_coding/main/interface/video_coding.h" #include "system_wrappers/interface/trace.h" #include "testsupport/frame_reader.h" #include "testsupport/frame_writer.h" @@ -63,6 +65,8 @@ DEFINE_int32(keyframe_interval, 0, "Forces a keyframe every Nth frame. " "0 means the encoder decides when to insert keyframes. Note that " "the encoder may create a keyframe in other locations in addition " "to the interval that is set using this parameter."); +DEFINE_int32(temporal_layers, 0, "The number of temporal layers to use " + "(VP8 specific codec setting). Must be 0-4."); DEFINE_int32(packet_size, 1500, "Simulated network packet size in bytes (MTU). " "Used for packet loss simulation."); DEFINE_int32(max_payload_size, 1440, "Max payload size in bytes for the " @@ -87,8 +91,8 @@ DEFINE_bool(verbose, true, "Verbose mode. Prints a lot of debugging info. " "Suitable for tracking progress but not for capturing output. " "Disable with --noverbose flag."); -// Custom log method that only prints if the verbose flag is given -// Supports all the standard printf parameters and formatting (just forwarded) +// Custom log method that only prints if the verbose flag is given. +// Supports all the standard printf parameters and formatting (just forwarded). int Log(const char *format, ...) { int result = 0; if (FLAGS_verbose) { @@ -100,7 +104,8 @@ int Log(const char *format, ...) { return result; } -// Validates the arguments given as command line flags. +// Validates the arguments given as command line flags and fills in the +// TestConfig struct with all configurations needed for video processing. // Returns 0 if everything is OK, otherwise an exit code. int HandleCommandLineFlags(webrtc::test::TestConfig* config) { // Validate the mandatory flags: @@ -111,7 +116,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { config->name = FLAGS_test_name; config->description = FLAGS_test_description; - // Verify the input file exists and is readable: + // Verify the input file exists and is readable. FILE* test_file; test_file = fopen(FLAGS_input_filename.c_str(), "rb"); if (test_file == NULL) { @@ -122,7 +127,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { fclose(test_file); config->input_filename = FLAGS_input_filename; - // Verify the output dir exists: + // Verify the output dir exists. struct stat dir_info; if (!(stat(FLAGS_output_dir.c_str(), &dir_info) == 0 && S_ISDIR(dir_info.st_mode))) { @@ -132,7 +137,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { } config->output_dir = FLAGS_output_dir; - // Manufacture an output filename if none was given: + // Manufacture an output filename if none was given. if (FLAGS_output_filename == "") { // Cut out the filename without extension from the given input file // (which may include a path) @@ -146,7 +151,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { - startIndex) + "_out.yuv"; } - // Verify output file can be written + // Verify output file can be written. if (FLAGS_output_dir == ".") { config->output_filename = FLAGS_output_filename; } else { @@ -160,23 +165,37 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { } fclose(test_file); - // Check single core flag + // Check single core flag. config->use_single_core = FLAGS_use_single_core; // Seed our random function if that flag is enabled. This will force - // repeatable behaviour between runs + // repeatable behaviour between runs. if (!FLAGS_disable_fixed_random_seed) { srand(0); } - // Check the bit rate + // Get codec specific configuration. + webrtc::VideoCodingModule::Codec(webrtc::kVideoCodecVP8, + config->codec_settings); + + // Check the temporal layers. + if (FLAGS_temporal_layers < 0 || + FLAGS_temporal_layers > webrtc::kMaxTemporalStreams) { + fprintf(stderr, "Temporal layers number must be 0-4, was: %d\n", + FLAGS_temporal_layers); + return 13; + } + config->codec_settings->codecSpecific.VP8.numberOfTemporalLayers = + FLAGS_temporal_layers; + + // Check the bit rate. if (FLAGS_bitrate <= 0) { fprintf(stderr, "Bit rate must be >0 kbps, was: %d\n", FLAGS_bitrate); return 5; } - config->codec_settings.startBitrate = FLAGS_bitrate; + config->codec_settings->startBitrate = FLAGS_bitrate; - // Check the keyframe interval + // Check the keyframe interval. if (FLAGS_keyframe_interval < 0) { fprintf(stderr, "Keyframe interval must be >=0, was: %d\n", FLAGS_keyframe_interval); @@ -184,7 +203,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { } config->keyframe_interval = FLAGS_keyframe_interval; - // Check packet size and max payload size + // Check packet size and max payload size. if (FLAGS_packet_size <= 0) { fprintf(stderr, "Packet size must be >0 bytes, was: %d\n", FLAGS_packet_size); @@ -205,14 +224,13 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { fprintf(stderr, "Width and height must be >0."); return 9; } - config->codec_settings.codecType = webrtc::kVideoCodecVP8; - config->codec_settings.width = FLAGS_width; - config->codec_settings.height = FLAGS_height; - config->codec_settings.maxFramerate = FLAGS_framerate; + config->codec_settings->width = FLAGS_width; + config->codec_settings->height = FLAGS_height; + config->codec_settings->maxFramerate = FLAGS_framerate; - // Calculate the size of each frame to read (according to YUV spec): + // Calculate the size of each frame to read (according to YUV spec). config->frame_length_in_bytes = - 3 * config->codec_settings.width * config->codec_settings.height / 2; + 3 * config->codec_settings->width * config->codec_settings->height / 2; // Check packet loss settings if (FLAGS_packet_loss_mode != "uniform" && @@ -250,8 +268,8 @@ void CalculateSsimVideoMetrics(webrtc::test::TestConfig* config, QualityMetricsResult* ssimResult) { Log("Calculating SSIM...\n"); SsimFromFiles(config->input_filename.c_str(), config->output_filename.c_str(), - config->codec_settings.width, - config->codec_settings.height, ssimResult); + config->codec_settings->width, + config->codec_settings->height, ssimResult); Log(" Average: %3.2f\n", ssimResult->average); Log(" Min : %3.2f (frame %d)\n", ssimResult->min, ssimResult->min_frame_number); @@ -263,8 +281,8 @@ void CalculatePsnrVideoMetrics(webrtc::test::TestConfig* config, QualityMetricsResult* psnrResult) { Log("Calculating PSNR...\n"); PsnrFromFiles(config->input_filename.c_str(), config->output_filename.c_str(), - config->codec_settings.width, - config->codec_settings.height, psnrResult); + config->codec_settings->width, + config->codec_settings->height, psnrResult); Log(" Average: %3.2f\n", psnrResult->average); Log(" Min : %3.2f (frame %d)\n", psnrResult->min, psnrResult->min_frame_number); @@ -370,10 +388,10 @@ void PrintPythonOutput(const webrtc::test::TestConfig& config, config.frame_length_in_bytes, config.use_single_core ? "True " : "False", config.keyframe_interval, - webrtc::test::VideoCodecTypeToStr(config.codec_settings.codecType), - config.codec_settings.width, - config.codec_settings.height, - config.codec_settings.startBitrate); + webrtc::test::VideoCodecTypeToStr(config.codec_settings->codecType), + config.codec_settings->width, + config.codec_settings->height, + config.codec_settings->startBitrate); printf("frame_data_types = {" "'frame_number': ('number', 'Frame number'),\n" "'encoding_successful': ('boolean', 'Encoding successful?'),\n" @@ -433,9 +451,13 @@ int main(int argc, char* argv[]) { google::ParseCommandLineFlags(&argc, &argv, true); + // Create TestConfig and codec settings struct. webrtc::test::TestConfig config; + webrtc::VideoCodec codec_settings; + config.codec_settings = &codec_settings; + int return_code = HandleCommandLineFlags(&config); - // Exit if an invalid argument is supplied: + // Exit if an invalid argument is supplied. if (return_code != 0) { return return_code; } @@ -473,7 +495,7 @@ int main(int argc, char* argv[]) { Log("\n"); Log("Processed %d frames\n", frame_number); - // Release encoder and decoder to make sure they have finished processing: + // Release encoder and decoder to make sure they have finished processing. encoder.Release(); decoder.Release(); @@ -486,16 +508,16 @@ int main(int argc, char* argv[]) { stats.PrintSummary(); - QualityMetricsResult ssimResult; - CalculateSsimVideoMetrics(&config, &ssimResult); - QualityMetricsResult psnrResult; - CalculatePsnrVideoMetrics(&config, &psnrResult); + QualityMetricsResult ssim_result; + CalculateSsimVideoMetrics(&config, &ssim_result); + QualityMetricsResult psnr_result; + CalculatePsnrVideoMetrics(&config, &psnr_result); if (FLAGS_csv) { - PrintCsvOutput(stats, ssimResult, psnrResult); + PrintCsvOutput(stats, ssim_result, psnr_result); } if (FLAGS_python) { - PrintPythonOutput(config, stats, ssimResult, psnrResult); + PrintPythonOutput(config, stats, ssim_result, psnr_result); } Log("Quality test finished!"); return 0; diff --git a/src/modules/video_coding/main/interface/video_coding.h b/src/modules/video_coding/main/interface/video_coding.h index 9b3d6f5d8..5840b0e45 100644 --- a/src/modules/video_coding/main/interface/video_coding.h +++ b/src/modules/video_coding/main/interface/video_coding.h @@ -11,9 +11,9 @@ #ifndef WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_ #define WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_ -#include "module.h" -#include "module_common_types.h" -#include "video_coding_defines.h" +#include "modules/interface/module.h" +#include "modules/interface/module_common_types.h" +#include "modules/video_coding/main/interface/video_coding_defines.h" namespace webrtc { diff --git a/src/modules/video_coding/main/interface/video_coding_defines.h b/src/modules/video_coding/main/interface/video_coding_defines.h index 21973cbe1..deddb9dc6 100644 --- a/src/modules/video_coding/main/interface/video_coding_defines.h +++ b/src/modules/video_coding/main/interface/video_coding_defines.h @@ -12,7 +12,7 @@ #define WEBRTC_MODULES_INTERFACE_VIDEO_CODING_DEFINES_H_ #include "typedefs.h" -#include "module_common_types.h" +#include "modules/interface/module_common_types.h" namespace webrtc {