generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Removed codec comparison test: it didn't work and probably never will.

Browse Source 
The central problem is that we cannot sync the frames in the input video with the output video, which makes PSNR/SSIM go crazy. The test only appeared to succeed earlier due to a bug in the test. We can consider this a failed experiment, but we did learn a lot from it :)

BUG=550

Review URL: https://webrtc-codereview.appspot.com/969006

git-svn-id: http://webrtc.googlecode.com/svn/trunk@3155 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
phoglund@webrtc.org 2012-11-23 09:21:44 +00:00
parent e3b2bc6c40
commit 849fb8ebd2
6 changed files with 0 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@ -154,41 +154,6 @@ TEST_F(ViEVideoVerificationTest, DISABLED_RunsBaseStandardTestWithoutErrors) {
kVerifyingTestMaxNumAttempts << " attempts.";
}
TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) {
// We compare the local and remote here instead of with the original.
// The reason is that it is hard to say when the three consecutive tests
// switch over into each other, at which point we would have to restart the
// original to get a fair comparison.
//
// The PSNR and SSIM values are quite low here, and they have to be since
// the codec switches will lead to lag in the output. This is considered
// acceptable, but it probably shouldn't get worse than this.
const double kExpectedMinimumPSNR = 20;
const double kExpectedMinimumSSIM = 0.7;
for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) {
InitializeFileRenderers();
ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight,
local_file_renderer_,
remote_file_renderer_));
std::string reference_file = local_file_renderer_->GetFullOutputPath();
std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
double actual_psnr = 0;
double actual_ssim = 0;
CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim);
TearDownFileRenderers();
if (actual_psnr >= kExpectedMinimumPSNR &&
actual_ssim >= kExpectedMinimumSSIM) {
// Test succeeded!
return;
}
}
}
// Runs a whole stack processing with tracking of which frames are dropped
// in the encoder. The local and remote file will not be of equal size because
// of unknown reasons. Tests show that they start at the same frame, which is

10
webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h
View File

@ -41,16 +41,6 @@ class ViEFileBasedComparisonTests {
ViEToFileRenderer* local_file_renderer,
ViEToFileRenderer* remote_file_renderer);
// Tries testing the I420 and VP8 codecs in turn. Returns false if the
// input file could not be opened; reports errors using googletest macros
// otherwise.
bool TestCodecs(
const std::string& i420_video_file,
int width,
int height,
ViEToFileRenderer* local_file_renderer,
ViEToFileRenderer* remote_file_renderer);
// Runs a full stack test using the VP8 codec. Tests the full stack and uses
// RTP timestamps to sync frames between the endpoints.
void TestFullStack(

184
webrtc/video_engine/test/auto_test/primitives/codec_primitives.cc
View File

@ -1,184 +0,0 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "codec_primitives.h"
#include "general_primitives.h"
#include "vie_autotest.h"
#include "vie_autotest_defines.h"
#include "vie_to_file_renderer.h"
#include "webrtc/modules/video_capture/include/video_capture_factory.h"
#include "tb_interfaces.h"
// Helper functions.
void TestCodecImageProcess(webrtc::VideoCodec video_codec,
webrtc::ViECodec* codec_interface,
int video_channel,
webrtc::ViEImageProcess* image_process) {
EXPECT_EQ(0, codec_interface->SetSendCodec(video_channel, video_codec));
FrameCounterEffectFilter frame_counter;
EXPECT_EQ(0, image_process->RegisterRenderEffectFilter(video_channel,
frame_counter));
AutoTestSleep (KAutoTestSleepTimeMs);
int max_number_of_rendered_frames = video_codec.maxFramerate *
KAutoTestSleepTimeMs / 1000;
if (video_codec.codecType == webrtc::kVideoCodecI420) {
// Due to that I420 needs a huge bandwidth, rate control can set
// frame rate very low. This happen since we use the same channel
// as we just tested with vp8.
EXPECT_GT(frame_counter.numFrames, 0);
} else {
#ifdef WEBRTC_ANDROID
// Special case to get the autotest to pass on some slow devices
EXPECT_GT(frame_counter.numFrames, max_number_of_rendered_frames / 6);
#else
EXPECT_GT(frame_counter.numFrames, max_number_of_rendered_frames / 4);
#endif
}
EXPECT_EQ(0, image_process->DeregisterRenderEffectFilter(video_channel));
}
// Test switching from i420 to VP8 as send codec and make sure that
// the codec observer gets called after the switch.
void TestCodecCallbacks(webrtc::ViEBase *& base_interface,
webrtc::ViECodec *codec_interface,
int video_channel,
int forced_codec_width,
int forced_codec_height) {
// Set I420 as send codec so we don't make any assumptions about what
// we currently have as send codec:
SetSendCodec(webrtc::kVideoCodecI420, codec_interface, video_channel,
forced_codec_width, forced_codec_height);
// Register the observer:
ViEAutotestCodecObserver codec_observer;
EXPECT_EQ(0, codec_interface->RegisterEncoderObserver(video_channel,
codec_observer));
EXPECT_EQ(0, codec_interface->RegisterDecoderObserver(video_channel,
codec_observer));
// Make the switch.
ViETest::Log("Testing codec callbacks...");
SetSendCodec(webrtc::kVideoCodecVP8, codec_interface, video_channel,
forced_codec_width, forced_codec_height);
AutoTestSleep (KAutoTestSleepTimeMs);
// Verify that we got the right codec.
EXPECT_EQ(webrtc::kVideoCodecVP8, codec_observer.incomingCodec.codecType);
// Clean up.
EXPECT_EQ(0, codec_interface->DeregisterEncoderObserver(video_channel));
EXPECT_EQ(0, codec_interface->DeregisterDecoderObserver(video_channel));
// Verify results.
EXPECT_GT(codec_observer.incomingCodecCalled, 0);
EXPECT_GT(codec_observer.incomingRatecalled, 0);
EXPECT_GT(codec_observer.outgoingRatecalled, 0);
}
void TestCodecs(const TbInterfaces& interfaces,
int capture_id,
int video_channel,
int forced_codec_width,
int forced_codec_height) {
webrtc::VideoEngine *video_engine_interface = interfaces.video_engine;
webrtc::ViEBase *base_interface = interfaces.base;
webrtc::ViECapture *capture_interface = interfaces.capture;
webrtc::ViERender *render_interface = interfaces.render;
webrtc::ViECodec *codec_interface = interfaces.codec;
webrtc::ViENetwork *network_interface = interfaces.network;
// ***************************************************************
// Engine ready. Begin testing class
// ***************************************************************
webrtc::VideoCodec video_codec;
memset(&video_codec, 0, sizeof (webrtc::VideoCodec));
// Set up all receive codecs. This basically trains the codec interface
// to be able to recognize all receive codecs based on payload type.
for (int idx = 0; idx < codec_interface->NumberOfCodecs(); idx++) {
EXPECT_EQ(0, codec_interface->GetCodec(idx, video_codec));
SetSuitableResolution(&video_codec,
forced_codec_width,
forced_codec_height);
EXPECT_EQ(0, codec_interface->SetReceiveCodec(video_channel, video_codec));
}
const char *ip_address = "127.0.0.1";
const unsigned short rtp_port = 6000;
EXPECT_EQ(0, network_interface->SetLocalReceiver(video_channel, rtp_port));
EXPECT_EQ(0, base_interface->StartReceive(video_channel));
EXPECT_EQ(0, network_interface->SetSendDestination(video_channel, ip_address,
rtp_port));
EXPECT_EQ(0, base_interface->StartSend(video_channel));
// Run all found codecs
webrtc::ViEImageProcess *image_process =
webrtc::ViEImageProcess::GetInterface(video_engine_interface);
EXPECT_TRUE(image_process != NULL);
ViETest::Log("Loop through all codecs for %d seconds",
KAutoTestSleepTimeMs / 1000);
for (int i = 0; i < codec_interface->NumberOfCodecs(); i++) {
EXPECT_EQ(0, codec_interface->GetCodec(i, video_codec));
if (video_codec.codecType == webrtc::kVideoCodecRED ||
video_codec.codecType == webrtc::kVideoCodecULPFEC) {
ViETest::Log("\t %d. %s not tested", i, video_codec.plName);
} else {
ViETest::Log("\t %d. %s", i, video_codec.plName);
SetSuitableResolution(&video_codec, forced_codec_width,
forced_codec_height);
TestCodecImageProcess(video_codec, codec_interface, video_channel,
image_process);
}
}
image_process->Release();
TestCodecCallbacks(base_interface, codec_interface, video_channel,
forced_codec_width, forced_codec_height);
ViETest::Log("Done!");
// ***************************************************************
// Testing finished. Tear down Video Engine
// ***************************************************************
EXPECT_EQ(0, base_interface->StopSend(video_channel));
EXPECT_EQ(0, base_interface->StopReceive(video_channel));
EXPECT_EQ(0, render_interface->StopRender(capture_id));
EXPECT_EQ(0, render_interface->StopRender(video_channel));
EXPECT_EQ(0, render_interface->RemoveRenderer(capture_id));
EXPECT_EQ(0, render_interface->RemoveRenderer(video_channel));
EXPECT_EQ(0, capture_interface->DisconnectCaptureDevice(video_channel));
EXPECT_EQ(0, base_interface->DeleteChannel(video_channel));
}
void SetSendCodec(webrtc::VideoCodecType of_type,
webrtc::ViECodec* codec_interface,
int video_channel,
int forced_codec_width,
int forced_codec_height) {
webrtc::VideoCodec codec;
bool ok;
EXPECT_TRUE(ok = FindSpecificCodec(of_type, codec_interface, &codec));
if (!ok) {
return;
}
SetSuitableResolution(&codec, forced_codec_width, forced_codec_height);
EXPECT_EQ(0, codec_interface->SetSendCodec(video_channel, codec));
}

126
webrtc/video_engine/test/auto_test/primitives/codec_primitives.h
View File

@ -1,126 +0,0 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_PRIMITIVES_CODEC_PRIMITIVES_H_
#define WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_PRIMITIVES_CODEC_PRIMITIVES_H_
#include "video_engine/include/vie_codec.h"
#include "video_engine/include/vie_image_process.h"
#include "video_engine/test/auto_test/interface/vie_autotest_defines.h"
#include "video_engine/test/auto_test/primitives/general_primitives.h"
class TbInterfaces;
// Tests that a codec actually renders frames by registering a basic
// render effect filter on the codec and then running it. This test is
// quite lenient on the number of frames that get rendered, so it should not
// be seen as a end-user-visible quality measure - it is more a sanity check
// that the codec at least gets some frames through.
// The codec resolution can be forced by specifying the forced* variables
// (pass in kDoNotForceResolution if you don't care).
void TestCodecs(const TbInterfaces& interfaces,
int capture_id,
int video_channel,
int forced_codec_width,
int forced_codec_height);
// This helper function will set the send codec in the codec interface to a
// codec of the specified type. It will generate a test failure if we do not
// support the provided codec type.
// The codec resolution can be forced by specifying the forced* variables
// (pass in kDoNotForceResolution if you don't care).
void SetSendCodec(webrtc::VideoCodecType of_type,
webrtc::ViECodec* codec_interface,
int video_channel,
int forced_codec_width,
int forced_codec_height);
class ViEAutotestCodecObserver: public webrtc::ViEEncoderObserver,
public webrtc::ViEDecoderObserver {
public:
int incomingCodecCalled;
int incomingRatecalled;
int outgoingRatecalled;
unsigned char lastPayloadType;
unsigned short lastWidth;
unsigned short lastHeight;
unsigned int lastOutgoingFramerate;
unsigned int lastOutgoingBitrate;
unsigned int lastIncomingFramerate;
unsigned int lastIncomingBitrate;
webrtc::VideoCodec incomingCodec;
ViEAutotestCodecObserver() {
incomingCodecCalled = 0;
incomingRatecalled = 0;
outgoingRatecalled = 0;
lastPayloadType = 0;
lastWidth = 0;
lastHeight = 0;
lastOutgoingFramerate = 0;
lastOutgoingBitrate = 0;
lastIncomingFramerate = 0;
lastIncomingBitrate = 0;
memset(&incomingCodec, 0, sizeof(incomingCodec));
}
virtual void IncomingCodecChanged(const int videoChannel,
const webrtc::VideoCodec& videoCodec) {
incomingCodecCalled++;
lastPayloadType = videoCodec.plType;
lastWidth = videoCodec.width;
lastHeight = videoCodec.height;
memcpy(&incomingCodec, &videoCodec, sizeof(videoCodec));
}
virtual void IncomingRate(const int videoChannel,
const unsigned int framerate,
const unsigned int bitrate) {
incomingRatecalled++;
lastIncomingFramerate += framerate;
lastIncomingBitrate += bitrate;
}
virtual void OutgoingRate(const int videoChannel,
const unsigned int framerate,
const unsigned int bitrate) {
outgoingRatecalled++;
lastOutgoingFramerate += framerate;
lastOutgoingBitrate += bitrate;
}
virtual void RequestNewKeyFrame(const int videoChannel) {
}
};
class FrameCounterEffectFilter : public webrtc::ViEEffectFilter
{
public:
int numFrames;
FrameCounterEffectFilter() {
numFrames = 0;
}
virtual ~FrameCounterEffectFilter() {
}
virtual int Transform(int size, unsigned char* frameBuffer,
unsigned int timeStamp90KHz, unsigned int width,
unsigned int height) {
numFrames++;
return 0;
}
};
#endif // WEBRTC_VIDEO_ENGINE_TEST_AUTO_TEST_PRIMITIVES_CODEC_PRIMITIVES_H_

38
webrtc/video_engine/test/auto_test/source/vie_file_based_comparison_tests.cc
View File

@ -12,7 +12,6 @@
#include "video_engine/test/auto_test/interface/vie_autotest_defines.h"
#include "video_engine/test/auto_test/primitives/base_primitives.h"
#include "video_engine/test/auto_test/primitives/codec_primitives.h"
#include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
#include "video_engine/test/auto_test/primitives/general_primitives.h"
#include "video_engine/test/libvietest/include/tb_interfaces.h"
@ -80,43 +79,6 @@ bool ViEFileBasedComparisonTests::TestCallSetup(
return true;
}
bool ViEFileBasedComparisonTests::TestCodecs(
const std::string& i420_video_file,
int width,
int height,
ViEToFileRenderer* local_file_renderer,
ViEToFileRenderer* remote_file_renderer) {
TbInterfaces interfaces("TestCodecs");
ViEFakeCamera fake_camera(interfaces.capture);
if (!fake_camera.StartCameraInNewThread(i420_video_file, width, height)) {
// No point in continuing if we have no proper video source
ADD_FAILURE() << "Could not open input video " << i420_video_file <<
": aborting test...";
return false;
}
int video_channel = -1;
int capture_id = fake_camera.capture_id();
EXPECT_EQ(0, interfaces.base->CreateChannel(video_channel));
EXPECT_EQ(0, interfaces.capture->ConnectCaptureDevice(
capture_id, video_channel));
ConfigureRtpRtcp(interfaces.rtp_rtcp, video_channel);
RenderToFile(interfaces.render, capture_id, local_file_renderer);
RenderToFile(interfaces.render, video_channel, remote_file_renderer);
// Force the codec resolution to what our input video is so we can make
// comparisons later. Our comparison algorithms wouldn't like scaling.
::TestCodecs(interfaces, capture_id, video_channel, width, height);
fake_camera.StopCamera();
return true;
}
void ViEFileBasedComparisonTests::TestFullStack(
const std::string& i420_video_file,
int width,

2
webrtc/video_engine/test/auto_test/vie_auto_test.gypi
View File

@ -60,8 +60,6 @@
'primitives/choice_helpers.cc',
'primitives/choice_helpers.h',
'primitives/choice_helpers_unittest.cc',
'primitives/codec_primitives.cc',
'primitives/codec_primitives.h',
'primitives/fake_stdin.h',
'primitives/fake_stdin.cc',
'primitives/framedrop_primitives.h',