generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed standard PSNR/SSIM test.

Browse Source 
BUG=1103

Review URL: https://webrtc-codereview.appspot.com/971005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@3197 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
phoglund@webrtc.org 2012-11-29 10:08:16 +00:00
parent bf41508807
commit 273ccad59d
14 changed files with 169 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
webrtc/common_video/libyuv/include/webrtc_libyuv.h
View File

@ -43,6 +43,9 @@ enum VideoType {
kBGRA,
};
// This is the max PSNR value our algorithms can return.
const double kInfinitePSNR = 48.0f;
// Conversion between the RawVideoType and the LibYuv videoType.
// TODO(wu): Consolidate types into one type throughout WebRtc.
VideoType RawVideoTypeToCommonVideoVideoType(RawVideoType type);
@ -163,6 +166,7 @@ int MirrorI420UpDown(const I420VideoFrame* src_frame,
I420VideoFrame* dst_frame);
// Compute PSNR for an I420 frame (all planes).
// Returns the PSNR in decibel, to a maximum of kInfinitePSNR.
double I420PSNR(const I420VideoFrame* ref_frame,
const I420VideoFrame* test_frame);
// Compute SSIM for an I420 frame (all planes).
@ -171,6 +175,7 @@ double I420SSIM(const I420VideoFrame* ref_frame,
// TODO(mikhal): Remove these functions and keep only the above functionality.
// Compute PSNR for an I420 buffer (all planes).
// Returns the PSNR in decibel, to a maximum of kInfinitePSNR.
double I420PSNR(const uint8_t* ref_frame,
const uint8_t* test_frame,
int width, int height);

8
webrtc/common_video/libyuv/webrtc_libyuv.cc
View File

@ -352,9 +352,9 @@ double I420PSNR(const I420VideoFrame* ref_frame,
test_frame->buffer(kVPlane),
test_frame->stride(kVPlane),
test_frame->width(), test_frame->height());
// LibYuv sets the max psnr value to 128, we restrict it to 48.
// LibYuv sets the max psnr value to 128, we restrict it here.
// In case of 0 mse in one frame, 128 can skew the results significantly.
return (psnr > 48.0) ? 48.0 : psnr;
return (psnr > kInfinitePSNR) ? kInfinitePSNR : psnr;
}
// Compute SSIM for an I420 frame (all planes)
@ -407,9 +407,9 @@ double I420PSNR(const uint8_t* ref_frame,
src_u_b, half_width,
src_v_b, half_width,
width, height);
// LibYuv sets the max psnr value to 128, we restrict it to 48.
// LibYuv sets the max psnr value to 128, we restrict it here.
// In case of 0 mse in one frame, 128 can skew the results significantly.
return (psnr > 48.0) ? 48.0 : psnr;
return (psnr > kInfinitePSNR) ? kInfinitePSNR : psnr;
}
// Compute SSIM for an I420 frame (all planes)
double I420SSIM(const uint8_t* ref_frame,

1
webrtc/modules/video_coding/main/source/media_optimization.cc
View File

@ -249,6 +249,7 @@ VCMMediaOptimization::DropFrame()
{
// leak appropriate number of bytes
_frameDropper->Leak((WebRtc_UWord32)(InputFrameRate() + 0.5f));
return _frameDropper->DropFrame();
}

3
webrtc/test/testsupport/metrics/video_metrics.cc
View File

@ -20,6 +20,9 @@
namespace webrtc {
namespace test {
// Copy here so our callers won't need to include libyuv for this constant.
double kMetricsInfinitePSNR = kInfinitePSNR;
// Used for calculating min and max values.
static bool LessForFrameResultValue (const FrameResult& s1,
const FrameResult& s2) {

19
webrtc/test/testsupport/metrics/video_metrics.h
View File

@ -17,6 +17,9 @@
namespace webrtc {
namespace test {
// The highest PSNR value our algorithms will return.
extern double kMetricsInfinitePSNR;
// Contains video quality metrics result for a single frame.
struct FrameResult {
int frame_number;
@ -43,10 +46,13 @@ struct QualityMetricsResult {
// Calculates PSNR and SSIM values for the reference and test video files
// (must be in I420 format). All calculated values are filled into the
// QualityMetricsResult stucts.
// QualityMetricsResult structs.
//
// PSNR values have the unit decibel (dB) where a high value means the test file
// is similar to the reference file. The higher value, the more similar.
// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR
// is similar to the reference file. The higher value, the more similar. The
// maximum PSNR value is kMetricsInfinitePSNR. For more info about PSNR, see
// http://en.wikipedia.org/wiki/PSNR.
//
// SSIM values range between -1.0 and 1.0, where 1.0 means the files are
// identical. For more info about SSIM, see http://en.wikipedia.org/wiki/SSIM
// This function only compares video frames up to the point when the shortest
@ -67,11 +73,14 @@ int I420MetricsFromFiles(const char* ref_filename,
// Calculates PSNR values for the reference and test video files (must be in
// I420 format). All calculated values are filled into the QualityMetricsResult
// struct.
//
// PSNR values have the unit decibel (dB) where a high value means the test file
// is similar to the reference file. The higher value, the more similar.
// is similar to the reference file. The higher value, the more similar. The
// maximum PSNR value is kMetricsInfinitePSNR. For more info about PSNR, see
// http://en.wikipedia.org/wiki/PSNR.
//
// This function only compares video frames up to the point when the shortest
// video ends.
// For more info about PSNR, see http://en.wikipedia.org/wiki/PSNR
//
// Return value:
// 0 if successful, negative on errors:

43
webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@ -25,7 +25,6 @@ namespace {
// wouldn't like scaling, so this will work when we compare with the original.
const int kInputWidth = 176;
const int kInputHeight = 144;
const int kVerifyingTestMaxNumAttempts = 3;
class ViEVideoVerificationTest : public testing::Test {
protected:
@ -72,7 +71,8 @@ class ViEVideoVerificationTest : public testing::Test {
EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
*ssim_result = ssim.average;
ViETest::Log("Results: PSNR is %f (dB), SSIM is %f (1 is perfect)",
ViETest::Log("Results: PSNR is %f (dB; 48 is max), "
"SSIM is %f (1 is perfect)",
psnr.average, ssim.average);
}
@ -140,41 +140,40 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest,
TestParameters parameter_table_[2];
};
// TODO(phoglund): Needs to be rewritten to use external transport. Currently
// the new memory-safe decoder is too slow with I420 with the default packet
// engine. See http://code.google.com/p/webrtc/issues/detail?id=1103.
TEST_F(ViEVideoVerificationTest, DISABLED_RunsBaseStandardTestWithoutErrors) {
// The I420 test should give pretty good values since it's a lossless codec
// running on the default bitrate. It should average about 30 dB but there
// may be cases where it dips as low as 26 under adverse conditions. That's
// why we have a retrying mechanism in place for this test.
const double kExpectedMinimumPSNR = 30;
const double kExpectedMinimumSSIM = 0.95;
for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) {
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
// I420 is lossless, so the I420 test should obviously get perfect results -
// the local preview and remote output files should be bit-exact. This test
// runs on external transport to ensure we do not drop packets.
// However, it's hard to make 100% stringent requirements on the video engine
// since for instance the jitter buffer has non-deterministic elements. If it
// breaks five times in a row though, you probably introduced a bug.
const int kNumAttempts = 5;
for (int attempt = 0; attempt < kNumAttempts; ++attempt) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
std::string output_file = remote_file_renderer_->GetFullOutputPath();
std::string remote_file = remote_file_renderer_->GetFullOutputPath();
std::string local_preview = local_file_renderer_->GetFullOutputPath();
StopRenderers();
double actual_psnr = 0;
double actual_ssim = 0;
CompareFiles(input_file_, output_file, &actual_psnr, &actual_ssim);
CompareFiles(local_preview, remote_file, &actual_psnr, &actual_ssim);
TearDownFileRenderers();
if (actual_psnr >= kExpectedMinimumPSNR &&
actual_ssim >= kExpectedMinimumSSIM) {
// Test succeeded!
if (actual_psnr == webrtc::test::kMetricsInfinitePSNR &&
actual_ssim == 1.0f) {
// Test successful.
return;
} else {
ViETest::Log("Retrying; attempt %d of %d.", attempt + 1, kNumAttempts);
}
}
ADD_FAILURE() << "Failed to achieve PSNR " << kExpectedMinimumPSNR <<
" and SSIM " << kExpectedMinimumSSIM << " after " <<
kVerifyingTestMaxNumAttempts << " attempts.";
FAIL() << "Failed to achieve perfect PSNR and SSIM results after " <<
kNumAttempts << " attempts.";
}
// Runs a whole stack processing with tracking of which frames are dropped

78
webrtc/video_engine/test/auto_test/primitives/base_primitives.cc
View File

@ -8,11 +8,36 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "base_primitives.h"
#include "webrtc/video_engine/test/auto_test/primitives/base_primitives.h"
#include "vie_autotest.h"
#include "vie_autotest_defines.h"
#include "webrtc/modules/video_capture/include/video_capture_factory.h"
#include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h"
#include "webrtc/video_engine/test/auto_test/interface/vie_autotest_defines.h"
#include "webrtc/video_engine/test/libvietest/include/tb_external_transport.h"
static void ConfigureCodecsToI420(int video_channel,
webrtc::VideoCodec video_codec,
webrtc::ViECodec* codec_interface) {
// Set up the codec interface with all known receive codecs and with
// I420 as the send codec.
for (int i = 0; i < codec_interface->NumberOfCodecs(); i++) {
EXPECT_EQ(0, codec_interface->GetCodec(i, video_codec));
// Try to keep the test frame size small and bit rate generous when I420.
if (video_codec.codecType == webrtc::kVideoCodecI420) {
video_codec.width = 176;
video_codec.height = 144;
video_codec.maxBitrate = 32000;
video_codec.startBitrate = 32000;
EXPECT_EQ(0, codec_interface->SetSendCodec(video_channel, video_codec));
}
EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec));
}
// Verify that we really found the I420 codec.
EXPECT_EQ(0, codec_interface->GetSendCodec(video_channel, video_codec));
EXPECT_EQ(webrtc::kVideoCodecI420, video_codec.codecType);
}
void TestI420CallSetup(webrtc::ViECodec* codec_interface,
webrtc::VideoEngine* video_engine,
@ -23,45 +48,26 @@ void TestI420CallSetup(webrtc::ViECodec* codec_interface,
webrtc::VideoCodec video_codec;
memset(&video_codec, 0, sizeof(webrtc::VideoCodec));
// Set up the codec interface with all known receive codecs and with
// I420 as the send codec.
for (int i = 0; i < codec_interface->NumberOfCodecs(); i++) {
EXPECT_EQ(0, codec_interface->GetCodec(i, video_codec));
ConfigureCodecsToI420(video_channel, video_codec, codec_interface);
// Try to keep the test frame size small when I420.
if (video_codec.codecType == webrtc::kVideoCodecI420) {
video_codec.width = 176;
video_codec.height = 144;
EXPECT_EQ(0, codec_interface->SetSendCodec(video_channel, video_codec));
}
EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec));
}
// Verify that we really found the I420 codec.
EXPECT_EQ(0, codec_interface->GetSendCodec(video_channel, video_codec));
EXPECT_EQ(webrtc::kVideoCodecI420, video_codec.codecType);
// Set up senders and receivers.
char version[1024] = "";
EXPECT_EQ(0, base_interface->GetVersion(version));
ViETest::Log("\nUsing WebRTC Video Engine version: %s", version);
const char *ipAddress = "127.0.0.1";
WebRtc_UWord16 rtpPortListen = 6100;
WebRtc_UWord16 rtpPortSend = 6100;
EXPECT_EQ(0, network_interface->SetLocalReceiver(video_channel,
rtpPortListen));
TbExternalTransport external_transport(
*network_interface, video_channel, NULL);
EXPECT_EQ(0, network_interface->RegisterSendTransport(
video_channel, external_transport));
EXPECT_EQ(0, base_interface->StartReceive(video_channel));
EXPECT_EQ(0, network_interface->SetSendDestination(video_channel, ipAddress,
rtpPortSend));
EXPECT_EQ(0, base_interface->StartSend(video_channel));
// Call started.
// Let the call run for a while.
ViETest::Log("Call started");
AutoTestSleep(KAutoTestSleepTimeMs);
// Done.
// Stop the call.
ViETest::Log("Stopping call.");
EXPECT_EQ(0, base_interface->StopSend(video_channel));
// Make sure we receive all packets.
AutoTestSleep(1000);
EXPECT_EQ(0, base_interface->StopReceive(video_channel));
EXPECT_EQ(0, network_interface->DeregisterSendTransport(video_channel));
}

37
webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc
View File

@ -22,6 +22,7 @@
#include "video_engine/test/auto_test/primitives/general_primitives.h"
#include "video_engine/test/libvietest/include/tb_interfaces.h"
#include "video_engine/test/libvietest/include/tb_external_transport.h"
#include "video_engine/test/libvietest/include/vie_external_render_filter.h"
#include "video_engine/test/libvietest/include/vie_to_file_renderer.h"
enum { kWaitTimeForFinalDecodeMs = 100 };
@ -29,34 +30,22 @@ enum { kWaitTimeForFinalDecodeMs = 100 };
// Writes the frames to be encoded to file and tracks which frames are sent in
// external transport on the local side and reports them to the
// FrameDropDetector class.
class LocalRendererEffectFilter : public webrtc::ViEEffectFilter {
class LocalRendererEffectFilter : public webrtc::ExternalRendererEffectFilter {
public:
explicit LocalRendererEffectFilter(FrameDropDetector* frame_drop_detector,
webrtc::ExternalRenderer* renderer)
: width_(0), height_(0), frame_drop_detector_(frame_drop_detector),
renderer_(renderer) {}
virtual ~LocalRendererEffectFilter() {}
virtual int Transform(int size, unsigned char* frameBuffer,
unsigned int timeStamp90KHz, unsigned int width,
unsigned int height) {
if (width != width_ || height_ != height) {
renderer_->FrameSizeChange(width, height, 1);
width_ = width;
height_ = height;
}
LocalRendererEffectFilter(webrtc::ExternalRenderer* renderer,
FrameDropDetector* frame_drop_detector)
: ExternalRendererEffectFilter(renderer),
frame_drop_detector_(frame_drop_detector) {}
int Transform(int size, unsigned char* frameBuffer,
unsigned int timeStamp90KHz, unsigned int width,
unsigned int height) {
frame_drop_detector_->ReportFrameState(FrameDropDetector::kCreated,
timeStamp90KHz);
return renderer_->DeliverFrame(frameBuffer,
size,
timeStamp90KHz,
webrtc::TickTime::MillisecondTimestamp());
return webrtc::ExternalRendererEffectFilter::Transform(
size, frameBuffer, timeStamp90KHz, width, height);
}
private:
unsigned int width_;
unsigned int height_;
FrameDropDetector* frame_drop_detector_;
webrtc::ExternalRenderer* renderer_;
};
// Tracks which frames are sent in external transport on the local side
@ -175,8 +164,8 @@ void TestFullStack(const TbInterfaces& interfaces,
// Setup the effect filters.
// Local rendering at the send-side is done in an effect filter to avoid
// synchronization issues with the remote renderer.
LocalRendererEffectFilter local_renderer_filter(frame_drop_detector,
local_file_renderer);
LocalRendererEffectFilter local_renderer_filter(local_file_renderer,
frame_drop_detector);
EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel,
local_renderer_filter));
DecodedTimestampEffectFilter decode_filter(frame_drop_detector);

9
webrtc/video_engine/test/auto_test/primitives/general_primitives.cc
View File

@ -73,15 +73,6 @@ void RenderToFile(webrtc::ViERender* renderer_interface,
EXPECT_EQ(0, renderer_interface->StartRender(frame_provider_id));
}
void StopAndRemoveRenderers(webrtc::ViEBase* base_interface,
webrtc::ViERender* render_interface,
int channel_id,
int capture_id) {
EXPECT_EQ(0, render_interface->StopRender(channel_id));
EXPECT_EQ(0, render_interface->RemoveRenderer(channel_id));
EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id));
}
void ConfigureRtpRtcp(webrtc::ViERTP_RTCP* rtcp_interface,
int video_channel) {
EXPECT_EQ(0, rtcp_interface->SetRTCPStatus(video_channel,

7
webrtc/video_engine/test/auto_test/primitives/general_primitives.h
View File

@ -55,13 +55,6 @@ void RenderToFile(webrtc::ViERender* renderer_interface,
int frame_provider_id,
ViEToFileRenderer* to_file_renderer);
// Stops all rendering given the normal case that we have a capture device
// and a video channel set up for rendering.
void StopAndRemoveRenderers(webrtc::ViEBase* base_interface,
webrtc::ViERender* render_interface,
int channel_id,
int capture_id);
// Configures RTP-RTCP.
void ConfigureRtpRtcp(webrtc::ViERTP_RTCP* rtcp_interface,
int video_channel);

5
webrtc/video_engine/test/auto_test/source/vie_autotest_base.cc
View File

@ -75,8 +75,9 @@ void ViEAutoTest::ViEBaseStandardTest() {
EXPECT_EQ(0, capture_interface->StopCapture(capture_id));
EXPECT_EQ(0, base_interface->StopReceive(video_channel));
StopAndRemoveRenderers(base_interface, render_interface, video_channel,
capture_id);
EXPECT_EQ(0, render_interface->StopRender(video_channel));
EXPECT_EQ(0, render_interface->RemoveRenderer(video_channel));
EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id));
EXPECT_EQ(0, render_interface->DeRegisterVideoRenderModule(*_vrm1));
EXPECT_EQ(0, render_interface->DeRegisterVideoRenderModule(*_vrm2));

21
webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc
View File

@ -15,6 +15,7 @@
#include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
#include "video_engine/test/auto_test/primitives/general_primitives.h"
#include "video_engine/test/libvietest/include/tb_interfaces.h"
#include "video_engine/test/libvietest/include/vie_external_render_filter.h"
#include "video_engine/test/libvietest/include/vie_fake_camera.h"
#include "video_engine/test/libvietest/include/vie_to_file_renderer.h"
@ -48,11 +49,18 @@ bool ViEFileBasedComparisonTests::TestCallSetup(
ConfigureRtpRtcp(interfaces.rtp_rtcp, video_channel);
webrtc::ViERender *render_interface = interfaces.render;
webrtc::ViERender* render_interface = interfaces.render;
webrtc::ViEImageProcess* image_process = interfaces.image_process;
RenderToFile(render_interface, capture_id, local_file_renderer);
RenderToFile(render_interface, video_channel, remote_file_renderer);
// We make a special hookup of the local renderer to use an effect filter
// instead of using the render interface for the capture device. This way
// we will only render frames that actually get sent.
webrtc::ExternalRendererEffectFilter renderer_filter(local_file_renderer);
EXPECT_EQ(0, image_process->RegisterSendEffectFilter(video_channel,
renderer_filter));
// Run the test itself:
const char* device_name = "Fake Capture Device";
@ -60,12 +68,8 @@ bool ViEFileBasedComparisonTests::TestCallSetup(
interfaces.base, interfaces.network, video_channel,
device_name);
AutoTestSleep(KAutoTestSleepTimeMs);
EXPECT_EQ(0, interfaces.base->StopReceive(video_channel));
StopAndRemoveRenderers(interfaces.base, render_interface, video_channel,
capture_id);
EXPECT_EQ(0, render_interface->StopRender(video_channel));
EXPECT_EQ(0, render_interface->RemoveRenderer(video_channel));
interfaces.capture->DisconnectCaptureDevice(video_channel);
@ -74,6 +78,7 @@ bool ViEFileBasedComparisonTests::TestCallSetup(
// tests that the system doesn't mind that the external capture device sends
// data after rendering has been stopped.
fake_camera.StopCamera();
EXPECT_EQ(0, image_process->DeregisterSendEffectFilter(video_channel));
EXPECT_EQ(0, interfaces.base->DeleteChannel(video_channel));
return true;

49
webrtc/video_engine/test/libvietest/include/vie_external_render_filter.h Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_VIDEO_ENGINE_TEST_LIBVIETEST_INCLUDE_VIE_EXTERNAL_RENDER_H_
#define WEBRTC_VIDEO_ENGINE_TEST_LIBVIETEST_INCLUDE_VIE_EXTERNAL_RENDER_H_
#include "webrtc/system_wrappers/interface/tick_util.h"
#include "webrtc/video_engine/include/vie_render.h"
namespace webrtc {
// A render filter which passes frames directly to an external renderer. This
// is different from plugging the external renderer directly into the sending
// side since this will only run on frames that actually get sent and not on
// frames that only get captured.
class ExternalRendererEffectFilter : public webrtc::ViEEffectFilter {
public:
explicit ExternalRendererEffectFilter(webrtc::ExternalRenderer* renderer)
: width_(0), height_(0), renderer_(renderer) {}
virtual ~ExternalRendererEffectFilter() {}
virtual int Transform(int size, unsigned char* frame_buffer,
unsigned int time_stamp90KHz, unsigned int width,
unsigned int height) {
if (width != width_ || height_ != height) {
renderer_->FrameSizeChange(width, height, 1);
width_ = width;
height_ = height;
}
return renderer_->DeliverFrame(frame_buffer,
size,
time_stamp90KHz,
webrtc::TickTime::MillisecondTimestamp());
}
private:
unsigned int width_;
unsigned int height_;
webrtc::ExternalRenderer* renderer_;
};
} // namespace webrtc
#endif // WEBRTC_VIDEO_ENGINE_TEST_LIBVIETEST_INCLUDE_VIE_EXTERNAL_RENDER_H_

1
webrtc/video_engine/test/libvietest/libvietest.gypi
View File

@ -27,6 +27,7 @@
],
'sources': [
# Helper classes
'include/vie_external_effect_filter.h',
'include/vie_fake_camera.h',
'include/vie_file_capture_device.h',
'include/vie_to_file_renderer.h',