diff --git a/src/test/test.gyp b/src/test/test.gyp index 6fa7f30da..8818631a3 100644 --- a/src/test/test.gyp +++ b/src/test/test.gyp @@ -97,8 +97,8 @@ 'type': 'executable', 'dependencies': [ 'test_support', - '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', - '<(webrtc_root)/../third_party/libyuv/libyuv.gyp:libyuv', + '<(DEPTH)/third_party/google-gflags/google-gflags.gyp:google-gflags', + '<(DEPTH)/third_party/libyuv/libyuv.gyp:libyuv', ], 'sources': [ 'testsupport/converter/converter.h', @@ -106,5 +106,19 @@ 'testsupport/converter/rgba_to_i420_converter.cc', ], }, + { + 'target_name': 'frame_analyzer', + 'type': 'executable', + 'dependencies': [ + '<(webrtc_root)/test/test.gyp:test_support', + '<(DEPTH)/third_party/google-gflags/google-gflags.gyp:google-gflags', + '<(DEPTH)/third_party/libyuv/libyuv.gyp:libyuv', + ], + 'sources': [ + 'testsupport/frame_analyzer/frame_analyzer.cc', + 'testsupport/frame_analyzer/video_quality_analysis.h', + 'testsupport/frame_analyzer/video_quality_analysis.cc', + ], + }, ], } diff --git a/src/test/testsupport/frame_analyzer/frame_analyzer.cc b/src/test/testsupport/frame_analyzer/frame_analyzer.cc new file mode 100644 index 000000000..84ad9f261 --- /dev/null +++ b/src/test/testsupport/frame_analyzer/frame_analyzer.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include + +#include "google/gflags.h" + +#include "testsupport/frame_analyzer/video_quality_analysis.h" + +#define STATS_LINE_LENGTH 25 + +DEFINE_string(stats_file, "stats.txt", "The full name of the file " + "containing the stats after decoding of the received YUV video"); +DEFINE_string(reference_file, "ref.yuv", "The reference YUV file to compare " + "against."); +DEFINE_string(test_file, "test.yuv", "The test YUV file to run the analysis " + "for."); +DEFINE_int32(width, 352, "The width of the refence and test files."); +DEFINE_int32(height, 288, "The height of the reference and test files."); + + +int main(int argc, char** argv) { + std::string program_name = argv[0]; + std::string usage = "Compares the output video with the initially sent video." + "\nRun " + program_name + " --helpshort for usage.\n" + "Example usage:\n" + program_name + " --stats_file=stats.txt " + "--reference_file=ref.yuv --test_file=test.yuv --width=352 --height=288"; + google::SetUsageMessage(usage); + + google::ParseCommandLineFlags(&argc, &argv, true); + + fprintf(stdout, "You have entered:\n"); + fprintf(stdout, "stats_file=%s, reference_file=%s, test_file=%s, width=%d, " + "height=%d\n", FLAGS_stats_file.c_str(), FLAGS_reference_file.c_str(), + FLAGS_test_file.c_str(), FLAGS_width, FLAGS_height); + + webrtc::test::ResultsContainer results; + + webrtc::test::RunAnalysis(FLAGS_reference_file.c_str(), + FLAGS_test_file.c_str(), FLAGS_stats_file.c_str(), + FLAGS_width, FLAGS_height, &results); + + webrtc::test::PrintAnalysisResults(&results); + webrtc::test::PrintMaxRepeatedAndSkippedFrames(FLAGS_stats_file.c_str()); +} diff --git a/src/test/testsupport/frame_analyzer/video_quality_analysis.cc b/src/test/testsupport/frame_analyzer/video_quality_analysis.cc new file mode 100644 index 000000000..590c3509b --- /dev/null +++ b/src/test/testsupport/frame_analyzer/video_quality_analysis.cc @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/testsupport/frame_analyzer/video_quality_analysis.h" + +#include +#include +#include +#include + +#define STATS_LINE_LENGTH 32 + +namespace webrtc { +namespace test { + +unsigned int GetI420FrameSize(unsigned int width, unsigned int height) { + unsigned int half_width = (width + 1) >> 1; + unsigned int half_height = (height + 1) >> 1; + + unsigned int y_plane = width * height; // I420 Y plane. + unsigned int u_plane = half_width * half_height; // I420 U plane. + unsigned int v_plane = half_width * half_height; // I420 V plane. + + return y_plane + u_plane + v_plane; +} + +int ExtractFrameSequenceNumber(std::string line) { + int space_position = line.find(' '); + if (space_position == -1) { + return -1; + } + std::string frame = line.substr(0, space_position); + + int underscore_position = frame.find('_'); + if (underscore_position == -1) { + return -1; + } + std::string frame_number = frame.substr(underscore_position + 1); + + return strtol(frame_number.c_str(), NULL, 10); +} + +int ExtractDecodedFrameNumber(std::string line) { + int space_position = line.find(' '); + if (space_position == -1) { + return -1; + } + std::string decoded_number = line.substr(space_position + 1); + + return strtol(decoded_number.c_str(), NULL, 10); +} + +bool IsThereBarcodeError(std::string line) { + int barcode_error_position = line.find("Barcode error"); + if (barcode_error_position != -1) { + return true; + } + return false; +} + +bool GetNextStatsLine(FILE* stats_file, char* line) { + int chars = 0; + char buf = 0; + + while (buf != '\n') { + size_t chars_read = fread(&buf, 1, 1, stats_file); + if (chars_read != 1 || feof(stats_file)) { + return false; + } + line[chars] = buf; + ++chars; + } + line[chars-1] = '\0'; // Strip the trailing \n and put end of string. + return true; +} + +bool GetNextI420Frame(FILE* input_file, int width, int height, + uint8* result_frame) { + unsigned int frame_size = GetI420FrameSize(width, height); + bool errors = false; + + size_t bytes_read = fread(result_frame, 1, frame_size, input_file); + if (bytes_read != frame_size) { + fprintf(stdout, "Error while reading frame from file\n"); + errors = true; + } + return !errors; +} + +bool ExtractFrameFromI420(const char* i420_file_name, int width, int height, + int frame_number, uint8* result_frame) { + unsigned int frame_size = GetI420FrameSize(width, height); + int offset = frame_number * frame_size; // Calculate offset for the frame. + bool errors = false; + + FILE* input_file = fopen(i420_file_name, "rb"); + if (input_file == NULL) { + fprintf(stderr, "Couldn't open input file for reading: %s\n", + i420_file_name); + return false; + } + + // Change stream pointer to new offset. + fseek(input_file, offset, SEEK_SET); + + size_t bytes_read = fread(result_frame, 1, frame_size, input_file); + if (bytes_read != frame_size && + ferror(input_file)) { + fprintf(stdout, "Error while reading frame no %d from file %s\n", + frame_number, i420_file_name); + errors = true; + } + fclose(input_file); + return !errors; +} + +double CalculateMetrics(VideoAnalysisMetricsType video_metrics_type, + const uint8* ref_frame, const uint8* test_frame, + int width, int height) { + if (!ref_frame || !test_frame) + return -1; + else if (height < 0 || width < 0) + return -1; + int half_width = (width + 1) >> 1; + int half_height = (height + 1) >> 1; + const uint8* src_y_a = ref_frame; + const uint8* src_u_a = src_y_a + width * height; + const uint8* src_v_a = src_u_a + half_width * half_height; + const uint8* src_y_b = test_frame; + const uint8* src_u_b = src_y_b + width * height; + const uint8* src_v_b = src_u_b + half_width * half_height; + + int stride_y = width; + int stride_uv = half_width; + + double result = 0.0; + + switch (video_metrics_type) { + case kPSNR: + // In the following: stride is determined by width. + result = libyuv::I420Psnr(src_y_a, width, src_u_a, half_width, + src_v_a, half_width, src_y_b, width, + src_u_b, half_width, src_v_b, half_width, + width, height); + // LibYuv sets the max psnr value to 128, we restrict it to 48. + // In case of 0 mse in one frame, 128 can skew the results significantly. + result = (result > 48.0) ? 48.0 : result; + break; + case kSSIM: + result = libyuv::I420Ssim(src_y_a, stride_y, src_u_a, stride_uv, + src_v_a, stride_uv, src_y_b, stride_y, + src_u_b, stride_uv, src_v_b, stride_uv, + width, height); + break; + default: + assert(false); + } + + return result; +} + +void RunAnalysis(const char* reference_file_name, const char* test_file_name, + const char* stats_file_name, int width, int height, + ResultsContainer* results) { + int size = GetI420FrameSize(width, height); + FILE* stats_file = fopen(stats_file_name, "r"); + + // String buffer for the lines in the stats file. + char line[STATS_LINE_LENGTH]; + + // Allocate buffers for test and reference frames. + uint8* test_frame = new uint8[size]; + uint8* reference_frame = new uint8[size]; + int previous_frame_number = -1; + + // While there are entries in the stats file. + while (GetNextStatsLine(stats_file, line)) { + int extracted_test_frame = ExtractFrameSequenceNumber(line); + int decoded_frame_number = ExtractDecodedFrameNumber(line); + + // If there was problem decoding the barcode in this frame or the frame has + // been duplicated, continue. + if (IsThereBarcodeError(line) || + decoded_frame_number == previous_frame_number) { + continue; + } + + assert(extracted_test_frame != -1); + assert(decoded_frame_number != -1); + + ExtractFrameFromI420(test_file_name, width, height, extracted_test_frame, + test_frame); + ExtractFrameFromI420(reference_file_name, width, height, + decoded_frame_number, reference_frame); + + // Calculate the PSNR and SSIM. + double result_psnr = CalculateMetrics(kPSNR, reference_frame, test_frame, + width, height); + double result_ssim = CalculateMetrics(kSSIM, reference_frame, test_frame, + width, height); + + previous_frame_number = decoded_frame_number; + + // Fill in the result struct. + AnalysisResult result; + result.frame_number = decoded_frame_number; + result.psnr_value = result_psnr; + result.ssim_value = result_ssim; + + results->frames.push_back(result); + } + + // Cleanup. + fclose(stats_file); + delete[] test_frame; + delete[] reference_frame; +} + +void PrintMaxRepeatedAndSkippedFrames(const char* stats_file_name) { + FILE* stats_file = fopen(stats_file_name, "r"); + char line[STATS_LINE_LENGTH]; + + int repeated_frames = 1; + int max_repeated_frames = 1; + int max_skipped_frames = 1; + int previous_frame_number = -1; + + while (GetNextStatsLine(stats_file, line)) { + int decoded_frame_number = ExtractDecodedFrameNumber(line); + + if (decoded_frame_number == -1) { + continue; + } + + // Calculate how many frames a cluster of repeated frames contains. + if (decoded_frame_number == previous_frame_number) { + ++repeated_frames; + if (repeated_frames > max_repeated_frames) { + max_repeated_frames = repeated_frames; + } + } else { + repeated_frames = 1; + } + + // Calculate how much frames have been skipped. + if (decoded_frame_number != 0 && previous_frame_number != -1) { + int skipped_frames = decoded_frame_number - previous_frame_number - 1; + if (skipped_frames > max_skipped_frames) { + max_skipped_frames = skipped_frames; + } + } + previous_frame_number = decoded_frame_number; + } + fprintf(stdout, "Max_repeated:%d Max_skipped:%d\n", max_repeated_frames, + max_skipped_frames); +} + +void PrintAnalysisResults(ResultsContainer* results) { + std::vector::iterator iter; + int frames_counter = 0; + + fprintf(stdout, "BSTATS\n"); + for (iter = results->frames.begin(); iter != results->frames.end(); ++iter) { + ++frames_counter; + fprintf(stdout, "%f %f;", iter->psnr_value, iter->ssim_value); + } + fprintf(stdout, "ESTATS\n"); + if (frames_counter > 0) { + fprintf(stdout, "Unique_frames_count:%d\n", frames_counter); + } else { + fprintf(stdout, "Unique_frames_count:undef\n"); + } +} + +} // namespace test +} // namespace webrtc diff --git a/src/test/testsupport/frame_analyzer/video_quality_analysis.h b/src/test/testsupport/frame_analyzer/video_quality_analysis.h new file mode 100644 index 000000000..920b6f9af --- /dev/null +++ b/src/test/testsupport/frame_analyzer/video_quality_analysis.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_TEST_TESTSUPPORT_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_ +#define WEBRTC_TEST_TESTSUPPORT_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_ + +#include +#include + +#include "third_party/libyuv/include/libyuv.h" + +namespace webrtc { +namespace test { + +struct AnalysisResult { + int frame_number; + double psnr_value; + double ssim_value; +}; + +struct ResultsContainer { + std::vector frames; +}; + +enum VideoAnalysisMetricsType {kPSNR, kSSIM}; + +// A function to run the PSNR and SSIM analysis on the test file. The test file +// comprises the frames that were captured during the quality measurement test. +// There may be missing or duplicate frames. Also the frames start at a random +// position in the original video. We should provide a statistics file along +// with the test video. The stats file contains the connection between the +// actual frames in the test file and their position in the reference video, so +// that the analysis could run with the right frames from both videos. The stats +// file should be in the form 'frame_xxxx yyyy', where xxxx is the consecutive +// number of the frame in the test video, and yyyy is the equivalent frame in +// the reference video. The stats file could be produced by +// tools/barcode_tools/barcode_decoder.py. This script decodes the barcodes +// integrated in every video and generates the stats file. If three was some +// problem with the decoding there would be 'Barcode error' instead of yyyy. +void RunAnalysis(const char* reference_file_name, const char* test_file_name, + const char* stats_file_name, int width, int height, + ResultsContainer* results); + +// Compute PSNR or SSIM for an I420 frame (all planes). When we are calculating +// PSNR values, the max return value (in the case where the test and reference +// frames are exactly the same) will be 48. In the case of SSIM the max return +// value will be 1. +double CalculateMetrics(VideoAnalysisMetricsType video_metrics_type, + const uint8* ref_frame, const uint8* test_frame, + int width, int height); + +// Function to print the result from the analysis. +void PrintAnalysisResults(ResultsContainer* results); + +// Calculates max repeated and skipped frames. +void PrintMaxRepeatedAndSkippedFrames(const char* stats_file_name); + +// Gets the next line from an open stats file. +bool GetNextStatsLine(FILE* stats_file, char* line); + +// Calculates the size of a I420 frame if given the width and height. +unsigned int GetI420FrameSize(unsigned int width, unsigned int height); + +// Converts a string to an int. +int StringToInt(std::string number); + +// Extract the sequence of the frame in the video. I.e. if line is +// frame_0023 0284, we will get 23. +int ExtractFrameSequenceNumber(std::string line); + +// Checks if there is 'Barcode error' for the given line. +bool IsThereBarcodeError(std::string line); + +// Extract the frame number in the reference video. I.e. if line is +// frame_0023 0284, we will get 284. +int ExtractDecodedFrameNumber(std::string line); + +// Gets the next frame from an open I420 file. +bool GetNextI420Frame(FILE* input_file, int width, int height, + uint8* result_frame); + +// Extracts an I420 frame at position frame_number from the file. +bool ExtractFrameFromI420(const char* i420_file_name, int width, int height, + int frame_number, uint8* result_frame); + + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_TEST_TESTSUPPORT_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_