generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reformatting VPM: First step - No functional changes.

Browse Source 
R=marpan@google.com, marpan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/2333004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4912 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
mikhal@webrtc.org 2013-10-03 16:42:41 +00:00
parent 26f78f7ecb
commit b43d8078a1
31 changed files with 2335 additions and 2809 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webrtc/modules/video_coding/main/test/quality_modes_test.cc
View File

@ -249,7 +249,7 @@ QualityModesTest::Perform(const CmdArgs& args)
VideoContentMetrics* contentMetrics = NULL; VideoContentMetrics* contentMetrics = NULL;
// setting user frame rate // setting user frame rate
_vpm->SetMaxFrameRate((uint32_t)(_nativeFrameRate+ 0.5f)); _vpm->SetMaxFramerate((uint32_t)(_nativeFrameRate+ 0.5f));
// for starters: keeping native values: // for starters: keeping native values:
_vpm->SetTargetResolution(_width, _height, _vpm->SetTargetResolution(_width, _height,
(uint32_t)(_frameRate+ 0.5f)); (uint32_t)(_frameRate+ 0.5f));

65
webrtc/modules/video_processing/main/interface/video_processing.h
View File

@ -36,42 +36,36 @@
namespace webrtc { namespace webrtc {
class VideoProcessingModule : public Module class VideoProcessingModule : public Module {
{ public:
public:
/** /**
Structure to hold frame statistics. Populate it with GetFrameStats(). Structure to hold frame statistics. Populate it with GetFrameStats().
*/ */
struct FrameStats struct FrameStats {
{
FrameStats() : FrameStats() :
mean(0), mean(0),
sum(0), sum(0),
numPixels(0), num_pixels(0),
subSamplWidth(0), subSamplWidth(0),
subSamplHeight(0) subSamplHeight(0) {
{
memset(hist, 0, sizeof(hist)); memset(hist, 0, sizeof(hist));
} }
uint32_t hist[256]; /**< Histogram of frame */ uint32_t hist[256]; // FRame histogram.
uint32_t mean; /**< Mean value of frame */ uint32_t mean; // Frame Mean value.
uint32_t sum; /**< Sum of frame */ uint32_t sum; // Sum of frame.
uint32_t numPixels; /**< Number of pixels */ uint32_t num_pixels; // Number of pixels.
uint8_t subSamplWidth; /**< Subsampling rate of width in powers uint8_t subSamplWidth; // Subsampling rate of width in powers of 2.
of 2 */ uint8_t subSamplHeight; // Subsampling rate of height in powers of 2.
uint8_t subSamplHeight; /**< Subsampling rate of height in powers };
of 2 */
};
/** /**
Specifies the warning types returned by BrightnessDetection(). Specifies the warning types returned by BrightnessDetection().
*/ */
enum BrightnessWarning enum BrightnessWarning {
{ kNoWarning, // Frame has acceptable brightness.
kNoWarning, /**< Frame has acceptable brightness */ kDarkWarning, // Frame is too dark.
kDarkWarning, /**< Frame is too dark */ kBrightWarning // Frame is too bright.
kBrightWarning /**< Frame is too bright */
}; };
/* /*
@ -231,28 +225,28 @@ public:
\param[in] height \param[in] height
Target height Target height
\param[in] frameRate \param[in] frame_rate
Target frameRate Target frame_rate
\return VPM_OK on success, a negative value on error (see error codes) \return VPM_OK on success, a negative value on error (see error codes)
*/ */
virtual int32_t SetTargetResolution(uint32_t width, virtual int32_t SetTargetResolution(uint32_t width,
uint32_t height, uint32_t height,
uint32_t frameRate) = 0; uint32_t frame_rate) = 0;
/** /**
Set max frame rate Set max frame rate
\param[in] maxFrameRate: maximum frame rate (limited to native frame rate) \param[in] max_frame_rate: maximum frame rate (limited to native frame rate)
\return VPM_OK on success, a negative value on error (see error codes) \return VPM_OK on success, a negative value on error (see error codes)
*/ */
virtual int32_t SetMaxFrameRate(uint32_t maxFrameRate) = 0; virtual int32_t SetMaxFramerate(uint32_t max_frame_rate) = 0;
/** /**
Get decimated(target) frame rate Get decimated(target) frame rate
*/ */
virtual uint32_t DecimatedFrameRate() = 0; virtual uint32_t Decimatedframe_rate() = 0;
/** /**
Get decimated(target) frame width Get decimated(target) frame width
@ -269,23 +263,23 @@ public:
disabled or one of the following: disabled or one of the following:
scaling to a close to target dimension followed by crop/pad scaling to a close to target dimension followed by crop/pad
\param[in] resamplingMode \param[in] resampling_mode
Set resampling mode (a member of VideoFrameResampling) Set resampling mode (a member of VideoFrameResampling)
*/ */
virtual void SetInputFrameResampleMode(VideoFrameResampling virtual void SetInputFrameResampleMode(VideoFrameResampling
resamplingMode) = 0; resampling_mode) = 0;
/** /**
Get Processed (decimated) frame Get Processed (decimated) frame
\param[in] frame pointer to the video frame. \param[in] frame pointer to the video frame.
\param[in] processedFrame pointer (double) to the processed frame. If no \param[in] processed_frame pointer (double) to the processed frame. If no
processing is required, processedFrame will be NULL. processing is required, processed_frame will be NULL.
\return VPM_OK on success, a negative value on error (see error codes) \return VPM_OK on success, a negative value on error (see error codes)
*/ */
virtual int32_t PreprocessFrame(const I420VideoFrame& frame, virtual int32_t PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processedFrame) = 0; I420VideoFrame** processed_frame) = 0;
/** /**
Return content metrics for the last processed frame Return content metrics for the last processed frame
@ -296,9 +290,8 @@ public:
Enable content analysis Enable content analysis
*/ */
virtual void EnableContentAnalysis(bool enable) = 0; virtual void EnableContentAnalysis(bool enable) = 0;
}; };
} // namespace } // namespace webrtc
#endif #endif // WEBRTC_MODULES_INTERFACE_VIDEO_PROCESSING_H

16
webrtc/modules/video_processing/main/interface/video_processing_defines.h
View File

@ -29,15 +29,13 @@ namespace webrtc {
#define VPM_UNINITIALIZED -5 #define VPM_UNINITIALIZED -5
#define VPM_UNIMPLEMENTED -6 #define VPM_UNIMPLEMENTED -6
enum VideoFrameResampling enum VideoFrameResampling {
{ kNoRescaling, // Disables rescaling.
// TODO: Do we still need crop/pad? kFastRescaling, // Point filter.
kNoRescaling, // disables rescaling kBiLinear, // Bi-linear interpolation.
kFastRescaling, // point kBox, // Box inteprolation.
kBiLinear, // bi-linear interpolation
kBox, // Box inteprolation
}; };
} // namespace } // namespace webrtc
#endif #endif // WEBRTC_MODULES_INTERFACE_VIDEO_PROCESSING_DEFINES_H

14
webrtc/modules/video_processing/main/source/brighten.cc
View File

@ -31,19 +31,19 @@ int32_t Brighten(I420VideoFrame* frame, int delta) {
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
} }
int numPixels = frame->width() * frame->height(); int num_pixels = frame->width() * frame->height();
int lookUp[256]; int look_up[256];
for (int i = 0; i < 256; i++) { for (int i = 0; i < 256; i++) {
int val = i + delta; int val = i + delta;
lookUp[i] = ((((val < 0) ? 0 : val) > 255) ? 255 : val); look_up[i] = ((((val < 0) ? 0 : val) > 255) ? 255 : val);
} }
uint8_t* tempPtr = frame->buffer(kYPlane); uint8_t* temp_ptr = frame->buffer(kYPlane);
for (int i = 0; i < numPixels; i++) { for (int i = 0; i < num_pixels; i++) {
*tempPtr = static_cast<uint8_t>(lookUp[*tempPtr]); *temp_ptr = static_cast<uint8_t>(look_up[*temp_ptr]);
tempPtr++; temp_ptr++;
} }
return VPM_OK; return VPM_OK;
} }

190
webrtc/modules/video_processing/main/source/brightness_detection.cc
View File

@ -17,180 +17,128 @@
namespace webrtc { namespace webrtc {
VPMBrightnessDetection::VPMBrightnessDetection() : VPMBrightnessDetection::VPMBrightnessDetection() :
_id(0) id_(0) {
{
Reset(); Reset();
} }
VPMBrightnessDetection::~VPMBrightnessDetection() VPMBrightnessDetection::~VPMBrightnessDetection() {}
{
}
int32_t int32_t VPMBrightnessDetection::ChangeUniqueId(const int32_t id) {
VPMBrightnessDetection::ChangeUniqueId(const int32_t id) id_ = id;
{
_id = id;
return VPM_OK; return VPM_OK;
} }
void void VPMBrightnessDetection::Reset() {
VPMBrightnessDetection::Reset() frame_cnt_bright_ = 0;
{ frame_cnt_dark_ = 0;
_frameCntBright = 0;
_frameCntDark = 0;
} }
int32_t int32_t VPMBrightnessDetection::ProcessFrame(
VPMBrightnessDetection::ProcessFrame(const I420VideoFrame& frame, const I420VideoFrame& frame,
const VideoProcessingModule::FrameStats& const VideoProcessingModule::FrameStats& stats) {
stats) if (frame.IsZeroSize()) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
if (frame.IsZeroSize())
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Null frame pointer"); "Null frame pointer");
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
} }
int width = frame.width(); int width = frame.width();
int height = frame.height(); int height = frame.height();
if (!VideoProcessingModule::ValidFrameStats(stats)) if (!VideoProcessingModule::ValidFrameStats(stats)) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Invalid frame stats"); "Invalid frame stats");
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
} }
const uint8_t frameCntAlarm = 2; const uint8_t frame_cnt_alarm = 2;
// Get proportion in lowest bins // Get proportion in lowest bins.
uint8_t lowTh = 20; uint8_t low_th = 20;
float propLow = 0; float prop_low = 0;
for (uint32_t i = 0; i < lowTh; i++) for (uint32_t i = 0; i < low_th; i++) {
{ prop_low += stats.hist[i];
propLow += stats.hist[i];
} }
propLow /= stats.numPixels; prop_low /= stats.num_pixels;
// Get proportion in highest bins // Get proportion in highest bins.
unsigned char highTh = 230; unsigned char high_th = 230;
float propHigh = 0; float prop_high = 0;
for (uint32_t i = highTh; i < 256; i++) for (uint32_t i = high_th; i < 256; i++) {
{ prop_high += stats.hist[i];
propHigh += stats.hist[i];
} }
propHigh /= stats.numPixels; prop_high /= stats.num_pixels;
if(propHigh < 0.4) if (prop_high < 0.4) {
{ if (stats.mean < 90 || stats.mean > 170) {
if (stats.mean < 90 || stats.mean > 170)
{
// Standard deviation of Y // Standard deviation of Y
const uint8_t* buffer = frame.buffer(kYPlane); const uint8_t* buffer = frame.buffer(kYPlane);
float stdY = 0; float std_y = 0;
for (int h = 0; h < height; h += (1 << stats.subSamplHeight)) for (int h = 0; h < height; h += (1 << stats.subSamplHeight)) {
{
int row = h*width; int row = h*width;
for (int w = 0; w < width; w += (1 << stats.subSamplWidth)) for (int w = 0; w < width; w += (1 << stats.subSamplWidth)) {
{ std_y += (buffer[w + row] - stats.mean) * (buffer[w + row] -
stdY += (buffer[w + row] - stats.mean) * (buffer[w + row] -
stats.mean); stats.mean);
} }
} }
stdY = sqrt(stdY / stats.numPixels); std_y = sqrt(std_y / stats.num_pixels);
// Get percentiles // Get percentiles.
uint32_t sum = 0; uint32_t sum = 0;
uint32_t medianY = 140; uint32_t median_y = 140;
uint32_t perc05 = 0; uint32_t perc05 = 0;
uint32_t perc95 = 255; uint32_t perc95 = 255;
float posPerc05 = stats.numPixels * 0.05f; float pos_perc05 = stats.num_pixels * 0.05f;
float posMedian = stats.numPixels * 0.5f; float pos_median = stats.num_pixels * 0.5f;
float posPerc95 = stats.numPixels * 0.95f; float posPerc95 = stats.num_pixels * 0.95f;
for (uint32_t i = 0; i < 256; i++) for (uint32_t i = 0; i < 256; i++) {
{
sum += stats.hist[i]; sum += stats.hist[i];
if (sum < pos_perc05) perc05 = i; // 5th perc.
if (sum < posPerc05) if (sum < pos_median) median_y = i; // 50th perc.
{
perc05 = i; // 5th perc
}
if (sum < posMedian)
{
medianY = i; // 50th perc
}
if (sum < posPerc95) if (sum < posPerc95)
{ perc95 = i; // 95th perc.
perc95 = i; // 95th perc
}
else else
{
break; break;
} }
}
// Check if image is too dark // Check if image is too dark
if ((stdY < 55) && (perc05 < 50)) if ((std_y < 55) && (perc05 < 50)) {
{ if (median_y < 60 || stats.mean < 80 || perc95 < 130 ||
if (medianY < 60 || stats.mean < 80 || perc95 < 130 || prop_low > 0.20) {
propLow > 0.20) frame_cnt_dark_++;
{ } else {
_frameCntDark++; frame_cnt_dark_ = 0;
} }
else } else {
{ frame_cnt_dark_ = 0;
_frameCntDark = 0;
}
}
else
{
_frameCntDark = 0;
} }
// Check if image is too bright // Check if image is too bright
if ((stdY < 52) && (perc95 > 200) && (medianY > 160)) if ((std_y < 52) && (perc95 > 200) && (median_y > 160)) {
{ if (median_y > 185 || stats.mean > 185 || perc05 > 140 ||
if (medianY > 185 || stats.mean > 185 || perc05 > 140 || prop_high > 0.25) {
propHigh > 0.25) frame_cnt_bright_++;
{ } else {
_frameCntBright++; frame_cnt_bright_ = 0;
} }
else } else {
{ frame_cnt_bright_ = 0;
_frameCntBright = 0;
} }
} else {
frame_cnt_dark_ = 0;
frame_cnt_bright_ = 0;
} }
else } else {
{ frame_cnt_bright_++;
_frameCntBright = 0; frame_cnt_dark_ = 0;
} }
} if (frame_cnt_dark_ > frame_cnt_alarm) {
else
{
_frameCntDark = 0;
_frameCntBright = 0;
}
}
else
{
_frameCntBright++;
_frameCntDark = 0;
}
if (_frameCntDark > frameCntAlarm)
{
return VideoProcessingModule::kDarkWarning; return VideoProcessingModule::kDarkWarning;
} } else if (frame_cnt_bright_ > frame_cnt_alarm) {
else if (_frameCntBright > frameCntAlarm)
{
return VideoProcessingModule::kBrightWarning; return VideoProcessingModule::kBrightWarning;
} } else {
else
{
return VideoProcessingModule::kNoWarning; return VideoProcessingModule::kNoWarning;
} }
} }
} // namespace } // namespace webrtc

24
webrtc/modules/video_processing/main/source/brightness_detection.h
View File

@ -11,34 +11,30 @@
/* /*
* brightness_detection.h * brightness_detection.h
*/ */
#ifndef VPM_BRIGHTNESS_DETECTION_H #ifndef MODULES_VIDEO_PROCESSING_MAIN_SOURCE_BRIGHTNESS_DETECTION_H
#define VPM_BRIGHTNESS_DETECTION_H #define MODULES_VIDEO_PROCESSING_MAIN_SOURCE_BRIGHTNESS_DETECTION_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
class VPMBrightnessDetection class VPMBrightnessDetection {
{ public:
public:
VPMBrightnessDetection(); VPMBrightnessDetection();
~VPMBrightnessDetection(); ~VPMBrightnessDetection();
int32_t ChangeUniqueId(int32_t id); int32_t ChangeUniqueId(int32_t id);
void Reset(); void Reset();
int32_t ProcessFrame(const I420VideoFrame& frame, int32_t ProcessFrame(const I420VideoFrame& frame,
const VideoProcessingModule::FrameStats& stats); const VideoProcessingModule::FrameStats& stats);
private: private:
int32_t _id; int32_t id_;
uint32_t _frameCntBright; uint32_t frame_cnt_bright_;
uint32_t _frameCntDark; uint32_t frame_cnt_dark_;
}; };
} // namespace } // namespace webrtc
#endif // VPM_BRIGHTNESS_DETECTION_H #endif // MODULES_VIDEO_PROCESSING_MAIN_SOURCE_BRIGHTNESS_DETECTION_H

46
webrtc/modules/video_processing/main/source/color_enhancement.cc
View File

@ -16,50 +16,42 @@
namespace webrtc { namespace webrtc {
namespace VideoProcessing namespace VideoProcessing {
{ int32_t ColorEnhancement(I420VideoFrame* frame) {
int32_t
ColorEnhancement(I420VideoFrame* frame)
{
assert(frame); assert(frame);
// pointers to U and V color pixels // Pointers to U and V color pixels.
uint8_t* ptrU; uint8_t* ptr_u;
uint8_t* ptrV; uint8_t* ptr_v;
uint8_t tempChroma; uint8_t temp_chroma;
if (frame->IsZeroSize()) {
if (frame->IsZeroSize())
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing,
-1, "Null frame pointer"); -1, "Null frame pointer");
return VPM_GENERAL_ERROR; return VPM_GENERAL_ERROR;
} }
if (frame->width() == 0 || frame->height() == 0) if (frame->width() == 0 || frame->height() == 0) {
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing,
-1, "Invalid frame size"); -1, "Invalid frame size");
return VPM_GENERAL_ERROR; return VPM_GENERAL_ERROR;
} }
// set pointers to first U and V pixels (skip luminance) // set pointers to first U and V pixels (skip luminance)
ptrU = frame->buffer(kUPlane); ptr_u = frame->buffer(kUPlane);
ptrV = frame->buffer(kVPlane); ptr_v = frame->buffer(kVPlane);
int size_uv = ((frame->width() + 1) / 2) * ((frame->height() + 1) / 2); int size_uv = ((frame->width() + 1) / 2) * ((frame->height() + 1) / 2);
// loop through all chrominance pixels and modify color // Loop through all chrominance pixels and modify color
for (int ix = 0; ix < size_uv; ix++) for (int ix = 0; ix < size_uv; ix++) {
{ temp_chroma = colorTable[*ptr_u][*ptr_v];
tempChroma = colorTable[*ptrU][*ptrV]; *ptr_v = colorTable[*ptr_v][*ptr_u];
*ptrV = colorTable[*ptrV][*ptrU]; *ptr_u = temp_chroma;
*ptrU = tempChroma;
// increment pointers ptr_u++;
ptrU++; ptr_v++;
ptrV++;
} }
return VPM_OK; return VPM_OK;
} }
} // namespace } // namespace VideoProcessing
} // namespace webrtc } // namespace webrtc

11
webrtc/modules/video_processing/main/source/color_enhancement.h
View File

@ -11,19 +11,18 @@
/* /*
* color_enhancement.h * color_enhancement.h
*/ */
#ifndef VPM_COLOR_ENHANCEMENT_H #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_COLOR_ENHANCEMENT_H
#define VPM_COLOR_ENHANCEMENT_H #define WEBRTC_MODULES_VIDEO_PROCESSING_COLOR_ENHANCEMENT_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
namespace VideoProcessing namespace VideoProcessing {
{
int32_t ColorEnhancement(I420VideoFrame* frame); int32_t ColorEnhancement(I420VideoFrame* frame);
} }
} // namespace } // namespace webrtc
#endif // VPM_COLOR_ENHANCEMENT_H #endif // WEBRTC_MODULES_VIDEO_PROCESSING_COLOR_ENHANCEMENT_H

10
webrtc/modules/video_processing/main/source/color_enhancement_private.h
View File

@ -1,3 +1,13 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPM_COLOR_ENHANCEMENT_PRIVATE_H #ifndef VPM_COLOR_ENHANCEMENT_PRIVATE_H
#define VPM_COLOR_ENHANCEMENT_PRIVATE_H #define VPM_COLOR_ENHANCEMENT_PRIVATE_H

277
webrtc/modules/video_processing/main/source/content_analysis.cc
View File

@ -17,153 +17,126 @@
namespace webrtc { namespace webrtc {
VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection): VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection)
_origFrame(NULL), : orig_frame_(NULL),
_prevFrame(NULL), prev_frame_(NULL),
_width(0), width_(0),
_height(0), height_(0),
_skipNum(1), skip_num_(1),
_border(8), border_(8),
_motionMagnitude(0.0f), motion_magnitude_(0.0f),
_spatialPredErr(0.0f), spatial_pred_err_(0.0f),
_spatialPredErrH(0.0f), spatial_pred_err_h_(0.0f),
_spatialPredErrV(0.0f), spatial_pred_err_v_(0.0f),
_firstFrame(true), first_frame_(true),
_CAInit(false), ca_Init_(false),
_cMetrics(NULL) content_metrics_(NULL) {
{
ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C; ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C;
TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C; TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C;
if (runtime_cpu_detection) if (runtime_cpu_detection) {
{
#if defined(WEBRTC_ARCH_X86_FAMILY) #if defined(WEBRTC_ARCH_X86_FAMILY)
if (WebRtc_GetCPUInfo(kSSE2)) if (WebRtc_GetCPUInfo(kSSE2)) {
{ ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_SSE2;
ComputeSpatialMetrics =
&VPMContentAnalysis::ComputeSpatialMetrics_SSE2;
TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2; TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2;
} }
#endif #endif
} }
Release(); Release();
} }
VPMContentAnalysis::~VPMContentAnalysis() VPMContentAnalysis::~VPMContentAnalysis() {
{
Release(); Release();
} }
VideoContentMetrics* VideoContentMetrics* VPMContentAnalysis::ComputeContentMetrics(
VPMContentAnalysis::ComputeContentMetrics(const I420VideoFrame& inputFrame) const I420VideoFrame& inputFrame) {
{
if (inputFrame.IsZeroSize()) if (inputFrame.IsZeroSize())
{
return NULL; return NULL;
}
// Init if needed (native dimension change) // Init if needed (native dimension change).
if (_width != inputFrame.width() || _height != inputFrame.height()) if (width_ != inputFrame.width() || height_ != inputFrame.height()) {
{
if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height())) if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height()))
{
return NULL; return NULL;
} }
}
// Only interested in the Y plane. // Only interested in the Y plane.
_origFrame = inputFrame.buffer(kYPlane); orig_frame_ = inputFrame.buffer(kYPlane);
// compute spatial metrics: 3 spatial prediction errors // Compute spatial metrics: 3 spatial prediction errors.
(this->*ComputeSpatialMetrics)(); (this->*ComputeSpatialMetrics)();
// compute motion metrics // Compute motion metrics
if (_firstFrame == false) if (first_frame_ == false)
ComputeMotionMetrics(); ComputeMotionMetrics();
// saving current frame as previous one: Y only // Saving current frame as previous one: Y only.
memcpy(_prevFrame, _origFrame, _width * _height); memcpy(prev_frame_, orig_frame_, width_ * height_);
_firstFrame = false; first_frame_ = false;
_CAInit = true; ca_Init_ = true;
return ContentMetrics(); return ContentMetrics();
} }
int32_t int32_t VPMContentAnalysis::Release() {
VPMContentAnalysis::Release() if (content_metrics_ != NULL) {
{ delete content_metrics_;
if (_cMetrics != NULL) content_metrics_ = NULL;
{
delete _cMetrics;
_cMetrics = NULL;
} }
if (_prevFrame != NULL) if (prev_frame_ != NULL) {
{ delete [] prev_frame_;
delete [] _prevFrame; prev_frame_ = NULL;
_prevFrame = NULL;
} }
_width = 0; width_ = 0;
_height = 0; height_ = 0;
_firstFrame = true; first_frame_ = true;
return VPM_OK; return VPM_OK;
} }
int32_t int32_t VPMContentAnalysis::Initialize(int width, int height) {
VPMContentAnalysis::Initialize(int width, int height) width_ = width;
{ height_ = height;
_width = width; first_frame_ = true;
_height = height;
_firstFrame = true;
// skip parameter: # of skipped rows: for complexity reduction // skip parameter: # of skipped rows: for complexity reduction
// temporal also currently uses it for column reduction. // temporal also currently uses it for column reduction.
_skipNum = 1; skip_num_ = 1;
// use skipNum = 2 for 4CIF, WHD // use skipNum = 2 for 4CIF, WHD
if ( (_height >= 576) && (_width >= 704) ) if ( (height_ >= 576) && (width_ >= 704) ) {
{ skip_num_ = 2;
_skipNum = 2;
} }
// use skipNum = 4 for FULLL_HD images // use skipNum = 4 for FULLL_HD images
if ( (_height >= 1080) && (_width >= 1920) ) if ( (height_ >= 1080) && (width_ >= 1920) ) {
{ skip_num_ = 4;
_skipNum = 4;
} }
if (_cMetrics != NULL) if (content_metrics_ != NULL) {
{ delete content_metrics_;
delete _cMetrics;
} }
if (_prevFrame != NULL) if (prev_frame_ != NULL) {
{ delete [] prev_frame_;
delete [] _prevFrame;
} }
// Spatial Metrics don't work on a border of 8. Minimum processing // Spatial Metrics don't work on a border of 8. Minimum processing
// block size is 16 pixels. So make sure the width and height support this. // block size is 16 pixels. So make sure the width and height support this.
if (_width <= 32 || _height <= 32) if (width_ <= 32 || height_ <= 32) {
{ ca_Init_ = false;
_CAInit = false;
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
} }
_cMetrics = new VideoContentMetrics(); content_metrics_ = new VideoContentMetrics();
if (_cMetrics == NULL) if (content_metrics_ == NULL) {
{
return VPM_MEMORY; return VPM_MEMORY;
} }
_prevFrame = new uint8_t[_width * _height] ; // Y only prev_frame_ = new uint8_t[width_ * height_]; // Y only.
if (_prevFrame == NULL) if (prev_frame_ == NULL) return VPM_MEMORY;
{
return VPM_MEMORY;
}
return VPM_OK; return VPM_OK;
} }
@ -171,14 +144,10 @@ VPMContentAnalysis::Initialize(int width, int height)
// Compute motion metrics: magnitude over non-zero motion vectors, // Compute motion metrics: magnitude over non-zero motion vectors,
// and size of zero cluster // and size of zero cluster
int32_t int32_t VPMContentAnalysis::ComputeMotionMetrics() {
VPMContentAnalysis::ComputeMotionMetrics()
{
// Motion metrics: only one is derived from normalized // Motion metrics: only one is derived from normalized
// (MAD) temporal difference // (MAD) temporal difference
(this->*TemporalDiffMetric)(); (this->*TemporalDiffMetric)();
return VPM_OK; return VPM_OK;
} }
@ -186,60 +155,47 @@ VPMContentAnalysis::ComputeMotionMetrics()
// Normalize MAD by spatial contrast: images with more contrast // Normalize MAD by spatial contrast: images with more contrast
// (pixel variance) likely have larger temporal difference // (pixel variance) likely have larger temporal difference
// To reduce complexity, we compute the metric for a reduced set of points. // To reduce complexity, we compute the metric for a reduced set of points.
int32_t int32_t VPMContentAnalysis::TemporalDiffMetric_C() {
VPMContentAnalysis::TemporalDiffMetric_C()
{
// size of original frame // size of original frame
int sizei = _height; int sizei = height_;
int sizej = _width; int sizej = width_;
uint32_t tempDiffSum = 0; uint32_t tempDiffSum = 0;
uint32_t pixelSum = 0; uint32_t pixelSum = 0;
uint64_t pixelSqSum = 0; uint64_t pixelSqSum = 0;
uint32_t numPixels = 0; // counter for # of pixels uint32_t num_pixels = 0; // Counter for # of pixels.
const int width_end = ((width_ - 2*border_) & -16) + border_;
const int width_end = ((_width - 2*_border) & -16) + _border; for (int i = border_; i < sizei - border_; i += skip_num_) {
for (int j = border_; j < width_end; j++) {
for(int i = _border; i < sizei - _border; i += _skipNum) num_pixels += 1;
{
for(int j = _border; j < width_end; j++)
{
numPixels += 1;
int ssn = i * sizej + j; int ssn = i * sizej + j;
uint8_t currPixel = _origFrame[ssn]; uint8_t currPixel = orig_frame_[ssn];
uint8_t prevPixel = _prevFrame[ssn]; uint8_t prevPixel = prev_frame_[ssn];
tempDiffSum += (uint32_t) tempDiffSum += (uint32_t)abs((int16_t)(currPixel - prevPixel));
abs((int16_t)(currPixel - prevPixel));
pixelSum += (uint32_t) currPixel; pixelSum += (uint32_t) currPixel;
pixelSqSum += (uint64_t) (currPixel * currPixel); pixelSqSum += (uint64_t) (currPixel * currPixel);
} }
} }
// default // Default.
_motionMagnitude = 0.0f; motion_magnitude_ = 0.0f;
if (tempDiffSum == 0) if (tempDiffSum == 0) return VPM_OK;
{
return VPM_OK;
}
// normalize over all pixels // Normalize over all pixels.
float const tempDiffAvg = (float)tempDiffSum / (float)(numPixels); float const tempDiffAvg = (float)tempDiffSum / (float)(num_pixels);
float const pixelSumAvg = (float)pixelSum / (float)(numPixels); float const pixelSumAvg = (float)pixelSum / (float)(num_pixels);
float const pixelSqSumAvg = (float)pixelSqSum / (float)(numPixels); float const pixelSqSumAvg = (float)pixelSqSum / (float)(num_pixels);
float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg); float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
if (contrast > 0.0) if (contrast > 0.0) {
{
contrast = sqrt(contrast); contrast = sqrt(contrast);
_motionMagnitude = tempDiffAvg/contrast; motion_magnitude_ = tempDiffAvg/contrast;
} }
return VPM_OK; return VPM_OK;
} }
// Compute spatial metrics: // Compute spatial metrics:
@ -249,14 +205,11 @@ VPMContentAnalysis::TemporalDiffMetric_C()
// The metrics are a simple estimate of the up-sampling prediction error, // The metrics are a simple estimate of the up-sampling prediction error,
// estimated assuming sub-sampling for decimation (no filtering), // estimated assuming sub-sampling for decimation (no filtering),
// and up-sampling back up with simple bilinear interpolation. // and up-sampling back up with simple bilinear interpolation.
int32_t int32_t VPMContentAnalysis::ComputeSpatialMetrics_C() {
VPMContentAnalysis::ComputeSpatialMetrics_C() const int sizei = height_;
{ const int sizej = width_;
//size of original frame
const int sizei = _height;
const int sizej = _width;
// pixel mean square average: used to normalize the spatial metrics // Pixel mean square average: used to normalize the spatial metrics.
uint32_t pixelMSA = 0; uint32_t pixelMSA = 0;
uint32_t spatialErrSum = 0; uint32_t spatialErrSum = 0;
@ -264,73 +217,59 @@ VPMContentAnalysis::ComputeSpatialMetrics_C()
uint32_t spatialErrHSum = 0; uint32_t spatialErrHSum = 0;
// make sure work section is a multiple of 16 // make sure work section is a multiple of 16
const int width_end = ((sizej - 2*_border) & -16) + _border; const int width_end = ((sizej - 2*border_) & -16) + border_;
for(int i = _border; i < sizei - _border; i += _skipNum)
{
for(int j = _border; j < width_end; j++)
{
for (int i = border_; i < sizei - border_; i += skip_num_) {
for (int j = border_; j < width_end; j++) {
int ssn1= i * sizej + j; int ssn1= i * sizej + j;
int ssn2 = (i + 1) * sizej + j; // bottom int ssn2 = (i + 1) * sizej + j; // bottom
int ssn3 = (i - 1) * sizej + j; // top int ssn3 = (i - 1) * sizej + j; // top
int ssn4 = i * sizej + j + 1; // right int ssn4 = i * sizej + j + 1; // right
int ssn5 = i * sizej + j - 1; // left int ssn5 = i * sizej + j - 1; // left
uint16_t refPixel1 = _origFrame[ssn1] << 1; uint16_t refPixel1 = orig_frame_[ssn1] << 1;
uint16_t refPixel2 = _origFrame[ssn1] << 2; uint16_t refPixel2 = orig_frame_[ssn1] << 2;
uint8_t bottPixel = _origFrame[ssn2]; uint8_t bottPixel = orig_frame_[ssn2];
uint8_t topPixel = _origFrame[ssn3]; uint8_t topPixel = orig_frame_[ssn3];
uint8_t rightPixel = _origFrame[ssn4]; uint8_t rightPixel = orig_frame_[ssn4];
uint8_t leftPixel = _origFrame[ssn5]; uint8_t leftPixel = orig_frame_[ssn5];
spatialErrSum += (uint32_t) abs((int16_t)(refPixel2 spatialErrSum += (uint32_t) abs((int16_t)(refPixel2
- (uint16_t)(bottPixel + topPixel - (uint16_t)(bottPixel + topPixel + leftPixel + rightPixel)));
+ leftPixel + rightPixel)));
spatialErrVSum += (uint32_t) abs((int16_t)(refPixel1 spatialErrVSum += (uint32_t) abs((int16_t)(refPixel1
- (uint16_t)(bottPixel + topPixel))); - (uint16_t)(bottPixel + topPixel)));
spatialErrHSum += (uint32_t) abs((int16_t)(refPixel1 spatialErrHSum += (uint32_t) abs((int16_t)(refPixel1
- (uint16_t)(leftPixel + rightPixel))); - (uint16_t)(leftPixel + rightPixel)));
pixelMSA += orig_frame_[ssn1];
pixelMSA += _origFrame[ssn1];
} }
} }
// normalize over all pixels // Normalize over all pixels.
const float spatialErr = (float)(spatialErrSum >> 2); const float spatialErr = (float)(spatialErrSum >> 2);
const float spatialErrH = (float)(spatialErrHSum >> 1); const float spatialErrH = (float)(spatialErrHSum >> 1);
const float spatialErrV = (float)(spatialErrVSum >> 1); const float spatialErrV = (float)(spatialErrVSum >> 1);
const float norm = (float)pixelMSA; const float norm = (float)pixelMSA;
// 2X2: // 2X2:
_spatialPredErr = spatialErr / norm; spatial_pred_err_ = spatialErr / norm;
// 1X2: // 1X2:
_spatialPredErrH = spatialErrH / norm; spatial_pred_err_h_ = spatialErrH / norm;
// 2X1: // 2X1:
_spatialPredErrV = spatialErrV / norm; spatial_pred_err_v_ = spatialErrV / norm;
return VPM_OK; return VPM_OK;
} }
VideoContentMetrics* VideoContentMetrics* VPMContentAnalysis::ContentMetrics() {
VPMContentAnalysis::ContentMetrics() if (ca_Init_ == false) return NULL;
{
if (_CAInit == false)
{
return NULL;
}
_cMetrics->spatial_pred_err = _spatialPredErr; content_metrics_->spatial_pred_err = spatial_pred_err_;
_cMetrics->spatial_pred_err_h = _spatialPredErrH; content_metrics_->spatial_pred_err_h = spatial_pred_err_h_;
_cMetrics->spatial_pred_err_v = _spatialPredErrV; content_metrics_->spatial_pred_err_v = spatial_pred_err_v_;
// Motion metric: normalized temporal difference (MAD) // Motion metric: normalized temporal difference (MAD).
_cMetrics->motion_magnitude = _motionMagnitude; content_metrics_->motion_magnitude = motion_magnitude_;
return _cMetrics;
return content_metrics_;
} }
} // namespace } // namespace webrtc

50
webrtc/modules/video_processing/main/source/content_analysis.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef VPM_CONTENT_ANALYSIS_H #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_CONTENT_ANALYSIS_H
#define VPM_CONTENT_ANALYSIS_H #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_CONTENT_ANALYSIS_H
#include "webrtc/common_video/interface/i420_video_frame.h" #include "webrtc/common_video/interface/i420_video_frame.h"
#include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/interface/module_common_types.h"
@ -18,12 +18,11 @@
namespace webrtc { namespace webrtc {
class VPMContentAnalysis class VPMContentAnalysis {
{ public:
public:
// When |runtime_cpu_detection| is true, runtime selection of an optimized // When |runtime_cpu_detection| is true, runtime selection of an optimized
// code path is allowed. // code path is allowed.
VPMContentAnalysis(bool runtime_cpu_detection); explicit VPMContentAnalysis(bool runtime_cpu_detection);
~VPMContentAnalysis(); ~VPMContentAnalysis();
// Initialize ContentAnalysis - should be called prior to // Initialize ContentAnalysis - should be called prior to
@ -43,8 +42,7 @@ public:
// Output: 0 if OK, negative value upon error // Output: 0 if OK, negative value upon error
int32_t Release(); int32_t Release();
private: private:
// return motion metrics // return motion metrics
VideoContentMetrics* ContentMetrics(); VideoContentMetrics* ContentMetrics();
@ -67,26 +65,24 @@ private:
int32_t TemporalDiffMetric_SSE2(); int32_t TemporalDiffMetric_SSE2();
#endif #endif
const uint8_t* _origFrame; const uint8_t* orig_frame_;
uint8_t* _prevFrame; uint8_t* prev_frame_;
int _width; int width_;
int _height; int height_;
int _skipNum; int skip_num_;
int _border; int border_;
// Content Metrics: // Content Metrics: Stores the local average of the metrics.
// stores the local average of the metrics float motion_magnitude_; // motion class
float _motionMagnitude; // motion class float spatial_pred_err_; // spatial class
float _spatialPredErr; // spatial class float spatial_pred_err_h_; // spatial class
float _spatialPredErrH; // spatial class float spatial_pred_err_v_; // spatial class
float _spatialPredErrV; // spatial class bool first_frame_;
bool _firstFrame; bool ca_Init_;
bool _CAInit;
VideoContentMetrics* _cMetrics; VideoContentMetrics* content_metrics_;
};
}; // end of VPMContentAnalysis class definition } // namespace webrtc
} // namespace #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_CONTENT_ANALYSIS_H
#endif

162
webrtc/modules/video_processing/main/source/content_analysis_sse2.cc
View File

@ -15,23 +15,19 @@
namespace webrtc { namespace webrtc {
int32_t int32_t VPMContentAnalysis::TemporalDiffMetric_SSE2() {
VPMContentAnalysis::TemporalDiffMetric_SSE2() uint32_t num_pixels = 0; // counter for # of pixels
{ const uint8_t* imgBufO = orig_frame_ + border_*width_ + border_;
uint32_t numPixels = 0; // counter for # of pixels const uint8_t* imgBufP = prev_frame_ + border_*width_ + border_;
const uint8_t* imgBufO = _origFrame + _border*_width + _border; const int32_t width_end = ((width_ - 2*border_) & -16) + border_;
const uint8_t* imgBufP = _prevFrame + _border*_width + _border;
const int32_t width_end = ((_width - 2*_border) & -16) + _border;
__m128i sad_64 = _mm_setzero_si128(); __m128i sad_64 = _mm_setzero_si128();
__m128i sum_64 = _mm_setzero_si128(); __m128i sum_64 = _mm_setzero_si128();
__m128i sqsum_64 = _mm_setzero_si128(); __m128i sqsum_64 = _mm_setzero_si128();
const __m128i z = _mm_setzero_si128(); const __m128i z = _mm_setzero_si128();
for(uint16_t i = 0; i < (_height - 2*_border); i += _skipNum) for (uint16_t i = 0; i < (height_ - 2*border_); i += skip_num_) {
{
__m128i sqsum_32 = _mm_setzero_si128(); __m128i sqsum_32 = _mm_setzero_si128();
const uint8_t *lineO = imgBufO; const uint8_t *lineO = imgBufO;
@ -45,21 +41,20 @@ VPMContentAnalysis::TemporalDiffMetric_SSE2()
// o*o will have a maximum of 255*255 = 65025. This will roll over // o*o will have a maximum of 255*255 = 65025. This will roll over
// a 16 bit accumulator as 67*65025 > 65535, but will fit in a // a 16 bit accumulator as 67*65025 > 65535, but will fit in a
// 32 bit accumulator. // 32 bit accumulator.
for(uint16_t j = 0; j < width_end - _border; j += 16) for (uint16_t j = 0; j < width_end - border_; j += 16) {
{
const __m128i o = _mm_loadu_si128((__m128i*)(lineO)); const __m128i o = _mm_loadu_si128((__m128i*)(lineO));
const __m128i p = _mm_loadu_si128((__m128i*)(lineP)); const __m128i p = _mm_loadu_si128((__m128i*)(lineP));
lineO += 16; lineO += 16;
lineP += 16; lineP += 16;
// abs pixel difference between frames // Abs pixel difference between frames.
sad_64 = _mm_add_epi64 (sad_64, _mm_sad_epu8(o, p)); sad_64 = _mm_add_epi64 (sad_64, _mm_sad_epu8(o, p));
// sum of all pixels in frame // sum of all pixels in frame
sum_64 = _mm_add_epi64 (sum_64, _mm_sad_epu8(o, z)); sum_64 = _mm_add_epi64 (sum_64, _mm_sad_epu8(o, z));
// squared sum of all pixels in frame // Squared sum of all pixels in frame.
const __m128i olo = _mm_unpacklo_epi8(o,z); const __m128i olo = _mm_unpacklo_epi8(o,z);
const __m128i ohi = _mm_unpackhi_epi8(o,z); const __m128i ohi = _mm_unpackhi_epi8(o,z);
@ -75,60 +70,51 @@ VPMContentAnalysis::TemporalDiffMetric_SSE2()
_mm_add_epi64(_mm_unpackhi_epi32(sqsum_32,z), _mm_add_epi64(_mm_unpackhi_epi32(sqsum_32,z),
_mm_unpacklo_epi32(sqsum_32,z))); _mm_unpacklo_epi32(sqsum_32,z)));
imgBufO += _width * _skipNum; imgBufO += width_ * skip_num_;
imgBufP += _width * _skipNum; imgBufP += width_ * skip_num_;
numPixels += (width_end - _border); num_pixels += (width_end - border_);
} }
__m128i sad_final_128; __m128i sad_final_128;
__m128i sum_final_128; __m128i sum_final_128;
__m128i sqsum_final_128; __m128i sqsum_final_128;
// bring sums out of vector registers and into integer register // Bring sums out of vector registers and into integer register
// domain, summing them along the way // domain, summing them along the way.
_mm_store_si128 (&sad_final_128, sad_64); _mm_store_si128 (&sad_final_128, sad_64);
_mm_store_si128 (&sum_final_128, sum_64); _mm_store_si128 (&sum_final_128, sum_64);
_mm_store_si128 (&sqsum_final_128, sqsum_64); _mm_store_si128 (&sqsum_final_128, sqsum_64);
uint64_t *sad_final_64 = uint64_t *sad_final_64 = reinterpret_cast<uint64_t*>(&sad_final_128);
reinterpret_cast<uint64_t*>(&sad_final_128); uint64_t *sum_final_64 = reinterpret_cast<uint64_t*>(&sum_final_128);
uint64_t *sum_final_64 = uint64_t *sqsum_final_64 = reinterpret_cast<uint64_t*>(&sqsum_final_128);
reinterpret_cast<uint64_t*>(&sum_final_128);
uint64_t *sqsum_final_64 =
reinterpret_cast<uint64_t*>(&sqsum_final_128);
const uint32_t pixelSum = sum_final_64[0] + sum_final_64[1]; const uint32_t pixelSum = sum_final_64[0] + sum_final_64[1];
const uint64_t pixelSqSum = sqsum_final_64[0] + sqsum_final_64[1]; const uint64_t pixelSqSum = sqsum_final_64[0] + sqsum_final_64[1];
const uint32_t tempDiffSum = sad_final_64[0] + sad_final_64[1]; const uint32_t tempDiffSum = sad_final_64[0] + sad_final_64[1];
// default // Default.
_motionMagnitude = 0.0f; motion_magnitude_ = 0.0f;
if (tempDiffSum == 0) if (tempDiffSum == 0) return VPM_OK;
{
return VPM_OK;
}
// normalize over all pixels // Normalize over all pixels.
const float tempDiffAvg = (float)tempDiffSum / (float)(numPixels); const float tempDiffAvg = (float)tempDiffSum / (float)(num_pixels);
const float pixelSumAvg = (float)pixelSum / (float)(numPixels); const float pixelSumAvg = (float)pixelSum / (float)(num_pixels);
const float pixelSqSumAvg = (float)pixelSqSum / (float)(numPixels); const float pixelSqSumAvg = (float)pixelSqSum / (float)(num_pixels);
float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg); float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
if (contrast > 0.0) if (contrast > 0.0) {
{
contrast = sqrt(contrast); contrast = sqrt(contrast);
_motionMagnitude = tempDiffAvg/contrast; motion_magnitude_ = tempDiffAvg/contrast;
} }
return VPM_OK; return VPM_OK;
} }
int32_t int32_t VPMContentAnalysis::ComputeSpatialMetrics_SSE2() {
VPMContentAnalysis::ComputeSpatialMetrics_SSE2() const uint8_t* imgBuf = orig_frame_ + border_*width_;
{ const int32_t width_end = ((width_ - 2 * border_) & -16) + border_;
const uint8_t* imgBuf = _origFrame + _border*_width;
const int32_t width_end = ((_width - 2*_border) & -16) + _border;
__m128i se_32 = _mm_setzero_si128(); __m128i se_32 = _mm_setzero_si128();
__m128i sev_32 = _mm_setzero_si128(); __m128i sev_32 = _mm_setzero_si128();
@ -140,9 +126,8 @@ VPMContentAnalysis::ComputeSpatialMetrics_SSE2()
// height of 1080 lines, or about 67 macro blocks. If the 16 bit row // height of 1080 lines, or about 67 macro blocks. If the 16 bit row
// value is maxed out at 65529 for every row, 65529*1080 = 70777800, which // value is maxed out at 65529 for every row, 65529*1080 = 70777800, which
// will not roll over a 32 bit accumulator. // will not roll over a 32 bit accumulator.
// _skipNum is also used to reduce the number of rows // skip_num_ is also used to reduce the number of rows
for(int32_t i = 0; i < (_height - 2*_border); i += _skipNum) for (int32_t i = 0; i < (height_ - 2*border_); i += skip_num_) {
{
__m128i se_16 = _mm_setzero_si128(); __m128i se_16 = _mm_setzero_si128();
__m128i sev_16 = _mm_setzero_si128(); __m128i sev_16 = _mm_setzero_si128();
__m128i seh_16 = _mm_setzero_si128(); __m128i seh_16 = _mm_setzero_si128();
@ -155,15 +140,14 @@ VPMContentAnalysis::ComputeSpatialMetrics_SSE2()
// a point would be abs(0-255+255+255+255) which equals 1020. // a point would be abs(0-255+255+255+255) which equals 1020.
// 120*1020 = 122400. The probability of hitting this is quite low // 120*1020 = 122400. The probability of hitting this is quite low
// on well behaved content. A specially crafted image could roll over. // on well behaved content. A specially crafted image could roll over.
// _border could also be adjusted to concentrate on just the center of // border_ could also be adjusted to concentrate on just the center of
// the images for an HD capture in order to reduce the possiblity of // the images for an HD capture in order to reduce the possiblity of
// rollover. // rollover.
const uint8_t *lineTop = imgBuf - _width + _border; const uint8_t *lineTop = imgBuf - width_ + border_;
const uint8_t *lineCen = imgBuf + _border; const uint8_t *lineCen = imgBuf + border_;
const uint8_t *lineBot = imgBuf + _width + _border; const uint8_t *lineBot = imgBuf + width_ + border_;
for(int32_t j = 0; j < width_end - _border; j += 16) for (int32_t j = 0; j < width_end - border_; j += 16) {
{
const __m128i t = _mm_loadu_si128((__m128i*)(lineTop)); const __m128i t = _mm_loadu_si128((__m128i*)(lineTop));
const __m128i l = _mm_loadu_si128((__m128i*)(lineCen - 1)); const __m128i l = _mm_loadu_si128((__m128i*)(lineCen - 1));
const __m128i c = _mm_loadu_si128((__m128i*)(lineCen)); const __m128i c = _mm_loadu_si128((__m128i*)(lineCen));
@ -202,47 +186,35 @@ VPMContentAnalysis::ComputeSpatialMetrics_SSE2()
clo = _mm_slli_epi16(clo, 1); clo = _mm_slli_epi16(clo, 1);
chi = _mm_slli_epi16(chi, 1); chi = _mm_slli_epi16(chi, 1);
const __m128i setlo = _mm_subs_epi16(clo, const __m128i setlo = _mm_subs_epi16(clo, _mm_add_epi16(lrlo, tblo));
_mm_add_epi16(lrlo, tblo)); const __m128i sethi = _mm_subs_epi16(chi, _mm_add_epi16(lrhi, tbhi));
const __m128i sethi = _mm_subs_epi16(chi,
_mm_add_epi16(lrhi, tbhi));
// Add to 16 bit running sum // Add to 16 bit running sum
se_16 = _mm_add_epi16(se_16, se_16 = _mm_add_epi16(se_16, _mm_max_epi16(setlo,
_mm_max_epi16(setlo,
_mm_subs_epi16(z, setlo))); _mm_subs_epi16(z, setlo)));
se_16 = _mm_add_epi16(se_16, se_16 = _mm_add_epi16(se_16, _mm_max_epi16(sethi,
_mm_max_epi16(sethi,
_mm_subs_epi16(z, sethi))); _mm_subs_epi16(z, sethi)));
sev_16 = _mm_add_epi16(sev_16, sev_16 = _mm_add_epi16(sev_16, _mm_max_epi16(sevtlo,
_mm_max_epi16(sevtlo,
_mm_subs_epi16(z, sevtlo))); _mm_subs_epi16(z, sevtlo)));
sev_16 = _mm_add_epi16(sev_16, sev_16 = _mm_add_epi16(sev_16, _mm_max_epi16(sevthi,
_mm_max_epi16(sevthi,
_mm_subs_epi16(z, sevthi))); _mm_subs_epi16(z, sevthi)));
seh_16 = _mm_add_epi16(seh_16, seh_16 = _mm_add_epi16(seh_16, _mm_max_epi16(sehtlo,
_mm_max_epi16(sehtlo,
_mm_subs_epi16(z, sehtlo))); _mm_subs_epi16(z, sehtlo)));
seh_16 = _mm_add_epi16(seh_16, seh_16 = _mm_add_epi16(seh_16, _mm_max_epi16(sehthi,
_mm_max_epi16(sehthi,
_mm_subs_epi16(z, sehthi))); _mm_subs_epi16(z, sehthi)));
} }
// Add to 32 bit running sum as to not roll over. // Add to 32 bit running sum as to not roll over.
se_32 = _mm_add_epi32(se_32, se_32 = _mm_add_epi32(se_32, _mm_add_epi32(_mm_unpackhi_epi16(se_16,z),
_mm_add_epi32(_mm_unpackhi_epi16(se_16,z),
_mm_unpacklo_epi16(se_16,z))); _mm_unpacklo_epi16(se_16,z)));
sev_32 = _mm_add_epi32(sev_32, sev_32 = _mm_add_epi32(sev_32, _mm_add_epi32(_mm_unpackhi_epi16(sev_16,z),
_mm_add_epi32(_mm_unpackhi_epi16(sev_16,z),
_mm_unpacklo_epi16(sev_16,z))); _mm_unpacklo_epi16(sev_16,z)));
seh_32 = _mm_add_epi32(seh_32, seh_32 = _mm_add_epi32(seh_32, _mm_add_epi32(_mm_unpackhi_epi16(seh_16,z),
_mm_add_epi32(_mm_unpackhi_epi16(seh_16,z),
_mm_unpacklo_epi16(seh_16,z))); _mm_unpacklo_epi16(seh_16,z)));
msa_32 = _mm_add_epi32(msa_32, msa_32 = _mm_add_epi32(msa_32, _mm_add_epi32(_mm_unpackhi_epi16(msa_16,z),
_mm_add_epi32(_mm_unpackhi_epi16(msa_16,z),
_mm_unpacklo_epi16(msa_16,z))); _mm_unpacklo_epi16(msa_16,z)));
imgBuf += _width * _skipNum; imgBuf += width_ * skip_num_;
} }
__m128i se_128; __m128i se_128;
@ -250,49 +222,41 @@ VPMContentAnalysis::ComputeSpatialMetrics_SSE2()
__m128i seh_128; __m128i seh_128;
__m128i msa_128; __m128i msa_128;
// bring sums out of vector registers and into integer register // Bring sums out of vector registers and into integer register
// domain, summing them along the way // domain, summing them along the way.
_mm_store_si128 (&se_128, _mm_store_si128 (&se_128, _mm_add_epi64(_mm_unpackhi_epi32(se_32,z),
_mm_add_epi64(_mm_unpackhi_epi32(se_32,z),
_mm_unpacklo_epi32(se_32,z))); _mm_unpacklo_epi32(se_32,z)));
_mm_store_si128 (&sev_128, _mm_store_si128 (&sev_128, _mm_add_epi64(_mm_unpackhi_epi32(sev_32,z),
_mm_add_epi64(_mm_unpackhi_epi32(sev_32,z),
_mm_unpacklo_epi32(sev_32,z))); _mm_unpacklo_epi32(sev_32,z)));
_mm_store_si128 (&seh_128, _mm_store_si128 (&seh_128, _mm_add_epi64(_mm_unpackhi_epi32(seh_32,z),
_mm_add_epi64(_mm_unpackhi_epi32(seh_32,z),
_mm_unpacklo_epi32(seh_32,z))); _mm_unpacklo_epi32(seh_32,z)));
_mm_store_si128 (&msa_128, _mm_store_si128 (&msa_128, _mm_add_epi64(_mm_unpackhi_epi32(msa_32,z),
_mm_add_epi64(_mm_unpackhi_epi32(msa_32,z),
_mm_unpacklo_epi32(msa_32,z))); _mm_unpacklo_epi32(msa_32,z)));
uint64_t *se_64 = uint64_t *se_64 = reinterpret_cast<uint64_t*>(&se_128);
reinterpret_cast<uint64_t*>(&se_128); uint64_t *sev_64 = reinterpret_cast<uint64_t*>(&sev_128);
uint64_t *sev_64 = uint64_t *seh_64 = reinterpret_cast<uint64_t*>(&seh_128);
reinterpret_cast<uint64_t*>(&sev_128); uint64_t *msa_64 = reinterpret_cast<uint64_t*>(&msa_128);
uint64_t *seh_64 =
reinterpret_cast<uint64_t*>(&seh_128);
uint64_t *msa_64 =
reinterpret_cast<uint64_t*>(&msa_128);
const uint32_t spatialErrSum = se_64[0] + se_64[1]; const uint32_t spatialErrSum = se_64[0] + se_64[1];
const uint32_t spatialErrVSum = sev_64[0] + sev_64[1]; const uint32_t spatialErrVSum = sev_64[0] + sev_64[1];
const uint32_t spatialErrHSum = seh_64[0] + seh_64[1]; const uint32_t spatialErrHSum = seh_64[0] + seh_64[1];
const uint32_t pixelMSA = msa_64[0] + msa_64[1]; const uint32_t pixelMSA = msa_64[0] + msa_64[1];
// normalize over all pixels // Normalize over all pixels.
const float spatialErr = (float)(spatialErrSum >> 2); const float spatialErr = (float)(spatialErrSum >> 2);
const float spatialErrH = (float)(spatialErrHSum >> 1); const float spatialErrH = (float)(spatialErrHSum >> 1);
const float spatialErrV = (float)(spatialErrVSum >> 1); const float spatialErrV = (float)(spatialErrVSum >> 1);
const float norm = (float)pixelMSA; const float norm = (float)pixelMSA;
// 2X2: // 2X2:
_spatialPredErr = spatialErr / norm; spatial_pred_err_ = spatialErr / norm;
// 1X2: // 1X2:
_spatialPredErrH = spatialErrH / norm; spatial_pred_err_h_ = spatialErrH / norm;
// 2X1: // 2X1:
_spatialPredErrV = spatialErrV / norm; spatial_pred_err_v_ = spatialErrV / norm;
return VPM_OK; return VPM_OK;
} }

445
webrtc/modules/video_processing/main/source/deflickering.cc
View File

@ -20,254 +20,226 @@
namespace webrtc { namespace webrtc {
// Detection constants // Detection constants
enum { kFrequencyDeviation = 39 }; // (Q4) Maximum allowed deviation for detection // (Q4) Maximum allowed deviation for detection.
enum { kMinFrequencyToDetect = 32 }; // (Q4) Minimum frequency that can be detected enum { kFrequencyDeviation = 39 };
enum { kNumFlickerBeforeDetect = 2 }; // Number of flickers before we accept detection // (Q4) Minimum frequency that can be detected.
enum { kMeanValueScaling = 4 }; // (Q4) In power of 2 enum { kMinFrequencyToDetect = 32 };
enum { kZeroCrossingDeadzone = 10 }; // Deadzone region in terms of pixel values // Number of flickers before we accept detection
enum { kNumFlickerBeforeDetect = 2 };
// Deflickering constants enum { kmean_valueScaling = 4 }; // (Q4) In power of 2
// Dead-zone region in terms of pixel values
enum { kZeroCrossingDeadzone = 10 };
// Deflickering constants.
// Compute the quantiles over 1 / DownsamplingFactor of the image. // Compute the quantiles over 1 / DownsamplingFactor of the image.
enum { kDownsamplingFactor = 8 }; enum { kDownsamplingFactor = 8 };
enum { kLog2OfDownsamplingFactor = 3 }; enum { kLog2OfDownsamplingFactor = 3 };
// To generate in Matlab: // To generate in Matlab:
// >> probUW16 = round(2^11 * [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.97]); // >> probUW16 = round(2^11 *
// [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.97]);
// >> fprintf('%d, ', probUW16) // >> fprintf('%d, ', probUW16)
// Resolution reduced to avoid overflow when multiplying with the (potentially) large // Resolution reduced to avoid overflow when multiplying with the
// number of pixels. // (potentially) large number of pixels.
const uint16_t VPMDeflickering::_probUW16[kNumProbs] = const uint16_t VPMDeflickering::prob_uw16_[kNumProbs] = {102, 205, 410, 614,
{102, 205, 410, 614, 819, 1024, 1229, 1434, 1638, 1843, 1946, 1987}; // <Q11> 819, 1024, 1229, 1434, 1638, 1843, 1946, 1987}; // <Q11>
// To generate in Matlab: // To generate in Matlab:
// >> numQuants = 14; maxOnlyLength = 5; // >> numQuants = 14; maxOnlyLength = 5;
// >> weightUW16 = round(2^15 * [linspace(0.5, 1.0, numQuants - maxOnlyLength)]); // >> weightUW16 = round(2^15 *
// [linspace(0.5, 1.0, numQuants - maxOnlyLength)]);
// >> fprintf('%d, %d,\n ', weightUW16); // >> fprintf('%d, %d,\n ', weightUW16);
const uint16_t VPMDeflickering::_weightUW16[kNumQuants - kMaxOnlyLength] = const uint16_t VPMDeflickering::weight_uw16_[kNumQuants - kMaxOnlyLength] =
{16384, 18432, 20480, 22528, 24576, 26624, 28672, 30720, 32768}; // <Q15> {16384, 18432, 20480, 22528, 24576, 26624, 28672, 30720, 32768}; // <Q15>
VPMDeflickering::VPMDeflickering() : VPMDeflickering::VPMDeflickering()
_id(0) : id_(0) {
{
Reset(); Reset();
} }
VPMDeflickering::~VPMDeflickering() VPMDeflickering::~VPMDeflickering() {}
{
}
int32_t int32_t VPMDeflickering::ChangeUniqueId(const int32_t id) {
VPMDeflickering::ChangeUniqueId(const int32_t id) id_ = id;
{
_id = id;
return 0; return 0;
} }
void void VPMDeflickering::Reset() {
VPMDeflickering::Reset() mean_buffer_length_ = 0;
{ detection_state_ = 0;
_meanBufferLength = 0; frame_rate_ = 0;
_detectionState = 0;
_frameRate = 0;
memset(_meanBuffer, 0, sizeof(int32_t) * kMeanBufferLength); memset(mean_buffer_, 0, sizeof(int32_t) * kMeanBufferLength);
memset(_timestampBuffer, 0, sizeof(int32_t) * kMeanBufferLength); memset(timestamp_buffer_, 0, sizeof(int32_t) * kMeanBufferLength);
// Initialize the history with a uniformly distributed histogram // Initialize the history with a uniformly distributed histogram.
_quantHistUW8[0][0] = 0; quant_hist_uw8_[0][0] = 0;
_quantHistUW8[0][kNumQuants - 1] = 255; quant_hist_uw8_[0][kNumQuants - 1] = 255;
for (int32_t i = 0; i < kNumProbs; i++) for (int32_t i = 0; i < kNumProbs; i++) {
{ quant_hist_uw8_[0][i + 1] = static_cast<uint8_t>((WEBRTC_SPL_UMUL_16_16(
_quantHistUW8[0][i + 1] = static_cast<uint8_t>((WEBRTC_SPL_UMUL_16_16( prob_uw16_[i], 255) + (1 << 10)) >> 11); // Unsigned round. <Q0>
_probUW16[i], 255) + (1 << 10)) >> 11); // Unsigned round. <Q0>
} }
for (int32_t i = 1; i < kFrameHistorySize; i++) for (int32_t i = 1; i < kFrameHistory_size; i++) {
{ memcpy(quant_hist_uw8_[i], quant_hist_uw8_[0],
memcpy(_quantHistUW8[i], _quantHistUW8[0], sizeof(uint8_t) * kNumQuants); sizeof(uint8_t) * kNumQuants);
} }
} }
int32_t int32_t VPMDeflickering::ProcessFrame(I420VideoFrame* frame,
VPMDeflickering::ProcessFrame(I420VideoFrame* frame, VideoProcessingModule::FrameStats* stats) {
VideoProcessingModule::FrameStats* stats)
{
assert(frame); assert(frame);
uint32_t frameMemory; uint32_t frame_memory;
uint8_t quantUW8[kNumQuants]; uint8_t quant_uw8[kNumQuants];
uint8_t maxQuantUW8[kNumQuants]; uint8_t maxquant_uw8[kNumQuants];
uint8_t minQuantUW8[kNumQuants]; uint8_t minquant_uw8[kNumQuants];
uint16_t targetQuantUW16[kNumQuants]; uint16_t target_quant_uw16[kNumQuants];
uint16_t incrementUW16; uint16_t increment_uw16;
uint8_t mapUW8[256]; uint8_t map_uw8[256];
uint16_t tmpUW16; uint16_t tmp_uw16;
uint32_t tmpUW32; uint32_t tmp_uw32;
int width = frame->width(); int width = frame->width();
int height = frame->height(); int height = frame->height();
if (frame->IsZeroSize()) if (frame->IsZeroSize()) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Null frame pointer"); "Null frame pointer");
return VPM_GENERAL_ERROR; return VPM_GENERAL_ERROR;
} }
// Stricter height check due to subsampling size calculation below. // Stricter height check due to subsampling size calculation below.
if (height < 2) if (height < 2) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Invalid frame size"); "Invalid frame size");
return VPM_GENERAL_ERROR; return VPM_GENERAL_ERROR;
} }
if (!VideoProcessingModule::ValidFrameStats(*stats)) if (!VideoProcessingModule::ValidFrameStats(*stats)) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Invalid frame stats"); "Invalid frame stats");
return VPM_GENERAL_ERROR; return VPM_GENERAL_ERROR;
} }
if (PreDetection(frame->timestamp(), *stats) == -1) if (PreDetection(frame->timestamp(), *stats) == -1) return VPM_GENERAL_ERROR;
{
return VPM_GENERAL_ERROR;
}
// Flicker detection // Flicker detection
int32_t detFlicker = DetectFlicker(); int32_t det_flicker = DetectFlicker();
if (detFlicker < 0) if (det_flicker < 0) {
{ // Error
return VPM_GENERAL_ERROR; return VPM_GENERAL_ERROR;
} } else if (det_flicker != 1) {
else if (detFlicker != 1)
{
return 0; return 0;
} }
// Size of luminance component // Size of luminance component.
const uint32_t ySize = height * width; const uint32_t y_size = height * width;
const uint32_t ySubSize = width * (((height - 1) >> const uint32_t y_sub_size = width * (((height - 1) >>
kLog2OfDownsamplingFactor) + 1); kLog2OfDownsamplingFactor) + 1);
uint8_t* ySorted = new uint8_t[ySubSize]; uint8_t* y_sorted = new uint8_t[y_sub_size];
uint32_t sortRowIdx = 0; uint32_t sort_row_idx = 0;
for (int i = 0; i < height; i += kDownsamplingFactor) for (int i = 0; i < height; i += kDownsamplingFactor) {
{ memcpy(y_sorted + sort_row_idx * width,
memcpy(ySorted + sortRowIdx * width,
frame->buffer(kYPlane) + i * width, width); frame->buffer(kYPlane) + i * width, width);
sortRowIdx++; sort_row_idx++;
} }
webrtc::Sort(ySorted, ySubSize, webrtc::TYPE_UWord8); webrtc::Sort(y_sorted, y_sub_size, webrtc::TYPE_UWord8);
uint32_t probIdxUW32 = 0; uint32_t prob_idx_uw32 = 0;
quantUW8[0] = 0; quant_uw8[0] = 0;
quantUW8[kNumQuants - 1] = 255; quant_uw8[kNumQuants - 1] = 255;
// Ensure we won't get an overflow below. // Ensure we won't get an overflow below.
// In practice, the number of subsampled pixels will not become this large. // In practice, the number of subsampled pixels will not become this large.
if (ySubSize > (1 << 21) - 1) if (y_sub_size > (1 << 21) - 1) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Subsampled number of pixels too large"); "Subsampled number of pixels too large");
return -1; return -1;
} }
for (int32_t i = 0; i < kNumProbs; i++) for (int32_t i = 0; i < kNumProbs; i++) {
{ // <Q0>.
probIdxUW32 = WEBRTC_SPL_UMUL_32_16(ySubSize, _probUW16[i]) >> 11; // <Q0> prob_idx_uw32 = WEBRTC_SPL_UMUL_32_16(y_sub_size, prob_uw16_[i]) >> 11;
quantUW8[i + 1] = ySorted[probIdxUW32]; quant_uw8[i + 1] = y_sorted[prob_idx_uw32];
} }
delete [] ySorted; delete [] y_sorted;
ySorted = NULL; y_sorted = NULL;
// Shift history for new frame. // Shift history for new frame.
memmove(_quantHistUW8[1], _quantHistUW8[0], (kFrameHistorySize - 1) * kNumQuants * memmove(quant_hist_uw8_[1], quant_hist_uw8_[0],
sizeof(uint8_t)); (kFrameHistory_size - 1) * kNumQuants * sizeof(uint8_t));
// Store current frame in history. // Store current frame in history.
memcpy(_quantHistUW8[0], quantUW8, kNumQuants * sizeof(uint8_t)); memcpy(quant_hist_uw8_[0], quant_uw8, kNumQuants * sizeof(uint8_t));
// We use a frame memory equal to the ceiling of half the frame rate to ensure we // We use a frame memory equal to the ceiling of half the frame rate to
// capture an entire period of flicker. // ensure we capture an entire period of flicker.
frameMemory = (_frameRate + (1 << 5)) >> 5; // Unsigned ceiling. <Q0> frame_memory = (frame_rate_ + (1 << 5)) >> 5; // Unsigned ceiling. <Q0>
// _frameRate in Q4. // frame_rate_ in Q4.
if (frameMemory > kFrameHistorySize) if (frame_memory > kFrameHistory_size) {
{ frame_memory = kFrameHistory_size;
frameMemory = kFrameHistorySize;
} }
// Get maximum and minimum. // Get maximum and minimum.
for (int32_t i = 0; i < kNumQuants; i++) for (int32_t i = 0; i < kNumQuants; i++) {
{ maxquant_uw8[i] = 0;
maxQuantUW8[i] = 0; minquant_uw8[i] = 255;
minQuantUW8[i] = 255; for (uint32_t j = 0; j < frame_memory; j++) {
for (uint32_t j = 0; j < frameMemory; j++) if (quant_hist_uw8_[j][i] > maxquant_uw8[i]) {
{ maxquant_uw8[i] = quant_hist_uw8_[j][i];
if (_quantHistUW8[j][i] > maxQuantUW8[i])
{
maxQuantUW8[i] = _quantHistUW8[j][i];
} }
if (_quantHistUW8[j][i] < minQuantUW8[i]) if (quant_hist_uw8_[j][i] < minquant_uw8[i]) {
{ minquant_uw8[i] = quant_hist_uw8_[j][i];
minQuantUW8[i] = _quantHistUW8[j][i];
} }
} }
} }
// Get target quantiles. // Get target quantiles.
for (int32_t i = 0; i < kNumQuants - kMaxOnlyLength; i++) for (int32_t i = 0; i < kNumQuants - kMaxOnlyLength; i++) {
{ target_quant_uw16[i] = static_cast<uint16_t>((WEBRTC_SPL_UMUL_16_16(
targetQuantUW16[i] = static_cast<uint16_t>((WEBRTC_SPL_UMUL_16_16( weight_uw16_[i], maxquant_uw8[i]) + WEBRTC_SPL_UMUL_16_16((1 << 15) -
_weightUW16[i], maxQuantUW8[i]) + WEBRTC_SPL_UMUL_16_16((1 << 15) - weight_uw16_[i], minquant_uw8[i])) >> 8); // <Q7>
_weightUW16[i], minQuantUW8[i])) >> 8); // <Q7>
} }
for (int32_t i = kNumQuants - kMaxOnlyLength; i < kNumQuants; i++) for (int32_t i = kNumQuants - kMaxOnlyLength; i < kNumQuants; i++) {
{ target_quant_uw16[i] = ((uint16_t)maxquant_uw8[i]) << 7;
targetQuantUW16[i] = ((uint16_t)maxQuantUW8[i]) << 7;
} }
// Compute the map from input to output pixels. // Compute the map from input to output pixels.
uint16_t mapUW16; // <Q7> uint16_t mapUW16; // <Q7>
for (int32_t i = 1; i < kNumQuants; i++) for (int32_t i = 1; i < kNumQuants; i++) {
{ // As quant and targetQuant are limited to UWord8, it's safe to use Q7 here.
// As quant and targetQuant are limited to UWord8, we're safe to use Q7 here. tmp_uw32 = static_cast<uint32_t>(target_quant_uw16[i] -
tmpUW32 = static_cast<uint32_t>(targetQuantUW16[i] - target_quant_uw16[i - 1]);
targetQuantUW16[i - 1]); // <Q7> tmp_uw16 = static_cast<uint16_t>(quant_uw8[i] - quant_uw8[i - 1]); // <Q0>
tmpUW16 = static_cast<uint16_t>(quantUW8[i] - quantUW8[i - 1]); // <Q0>
if (tmpUW16 > 0) if (tmp_uw16 > 0) {
{ increment_uw16 = static_cast<uint16_t>(WebRtcSpl_DivU32U16(tmp_uw32,
incrementUW16 = static_cast<uint16_t>(WebRtcSpl_DivU32U16(tmpUW32, tmp_uw16)); // <Q7>
tmpUW16)); // <Q7> } else {
}
else
{
// The value is irrelevant; the loop below will only iterate once. // The value is irrelevant; the loop below will only iterate once.
incrementUW16 = 0; increment_uw16 = 0;
} }
mapUW16 = targetQuantUW16[i - 1]; mapUW16 = target_quant_uw16[i - 1];
for (uint32_t j = quantUW8[i - 1]; j < (uint32_t)(quantUW8[i] + 1); j++) for (uint32_t j = quant_uw8[i - 1]; j < (uint32_t)(quant_uw8[i] + 1); j++) {
{ // Unsigned round. <Q0>
mapUW8[j] = (uint8_t)((mapUW16 + (1 << 6)) >> 7); // Unsigned round. <Q0> map_uw8[j] = (uint8_t)((mapUW16 + (1 << 6)) >> 7);
mapUW16 += incrementUW16; mapUW16 += increment_uw16;
} }
} }
// Map to the output frame. // Map to the output frame.
uint8_t* buffer = frame->buffer(kYPlane); uint8_t* buffer = frame->buffer(kYPlane);
for (uint32_t i = 0; i < ySize; i++) for (uint32_t i = 0; i < y_size; i++) {
{ buffer[i] = map_uw8[buffer[i]];
buffer[i] = mapUW8[buffer[i]];
} }
// Frame was altered, so reset stats. // Frame was altered, so reset stats.
VideoProcessingModule::ClearFrameStats(stats); VideoProcessingModule::ClearFrameStats(stats);
return 0; return VPM_OK;
} }
/** /**
@ -282,133 +254,121 @@ VPMDeflickering::ProcessFrame(I420VideoFrame* frame,
zero.\n zero.\n
-1: Error -1: Error
*/ */
int32_t int32_t VPMDeflickering::PreDetection(const uint32_t timestamp,
VPMDeflickering::PreDetection(const uint32_t timestamp, const VideoProcessingModule::FrameStats& stats) {
const VideoProcessingModule::FrameStats& stats) int32_t mean_val; // Mean value of frame (Q4)
{ uint32_t frame_rate = 0;
int32_t meanVal; // Mean value of frame (Q4) int32_t meanBufferLength; // Temp variable.
uint32_t frameRate = 0;
int32_t meanBufferLength; // Temp variable
meanVal = ((stats.sum << kMeanValueScaling) / stats.numPixels); mean_val = ((stats.sum << kmean_valueScaling) / stats.num_pixels);
/* Update mean value buffer. // Update mean value buffer.
* This should be done even though we might end up in an unreliable detection. // This should be done even though we might end up in an unreliable detection.
*/ memmove(mean_buffer_ + 1, mean_buffer_,
memmove(_meanBuffer + 1, _meanBuffer, (kMeanBufferLength - 1) * sizeof(int32_t)); (kMeanBufferLength - 1) * sizeof(int32_t));
_meanBuffer[0] = meanVal; mean_buffer_[0] = mean_val;
/* Update timestamp buffer. // Update timestamp buffer.
* This should be done even though we might end up in an unreliable detection. // This should be done even though we might end up in an unreliable detection.
*/ memmove(timestamp_buffer_ + 1, timestamp_buffer_, (kMeanBufferLength - 1) *
memmove(_timestampBuffer + 1, _timestampBuffer, (kMeanBufferLength - 1) *
sizeof(uint32_t)); sizeof(uint32_t));
_timestampBuffer[0] = timestamp; timestamp_buffer_[0] = timestamp;
/* Compute current frame rate (Q4) */ /* Compute current frame rate (Q4) */
if (_timestampBuffer[kMeanBufferLength - 1] != 0) if (timestamp_buffer_[kMeanBufferLength - 1] != 0) {
{ frame_rate = ((90000 << 4) * (kMeanBufferLength - 1));
frameRate = ((90000 << 4) * (kMeanBufferLength - 1)); frame_rate /=
frameRate /= (_timestampBuffer[0] - _timestampBuffer[kMeanBufferLength - 1]); (timestamp_buffer_[0] - timestamp_buffer_[kMeanBufferLength - 1]);
}else if (_timestampBuffer[1] != 0) } else if (timestamp_buffer_[1] != 0) {
{ frame_rate = (90000 << 4) / (timestamp_buffer_[0] - timestamp_buffer_[1]);
frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]);
} }
/* Determine required size of mean value buffer (_meanBufferLength) */ /* Determine required size of mean value buffer (mean_buffer_length_) */
if (frameRate == 0) { if (frame_rate == 0) {
meanBufferLength = 1; meanBufferLength = 1;
} } else {
else { meanBufferLength =
meanBufferLength = (kNumFlickerBeforeDetect * frameRate) / kMinFrequencyToDetect; (kNumFlickerBeforeDetect * frame_rate) / kMinFrequencyToDetect;
} }
/* Sanity check of buffer length */ /* Sanity check of buffer length */
if (meanBufferLength >= kMeanBufferLength) if (meanBufferLength >= kMeanBufferLength) {
{
/* Too long buffer. The flickering frequency is too close to zero, which /* Too long buffer. The flickering frequency is too close to zero, which
* makes the estimation unreliable. * makes the estimation unreliable.
*/ */
_meanBufferLength = 0; mean_buffer_length_ = 0;
return 2; return 2;
} }
_meanBufferLength = meanBufferLength; mean_buffer_length_ = meanBufferLength;
if ((_timestampBuffer[_meanBufferLength - 1] != 0) && (_meanBufferLength != 1)) if ((timestamp_buffer_[mean_buffer_length_ - 1] != 0) &&
{ (mean_buffer_length_ != 1)) {
frameRate = ((90000 << 4) * (_meanBufferLength - 1)); frame_rate = ((90000 << 4) * (mean_buffer_length_ - 1));
frameRate /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]); frame_rate /=
}else if (_timestampBuffer[1] != 0) (timestamp_buffer_[0] - timestamp_buffer_[mean_buffer_length_ - 1]);
{ } else if (timestamp_buffer_[1] != 0) {
frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]); frame_rate = (90000 << 4) / (timestamp_buffer_[0] - timestamp_buffer_[1]);
} }
_frameRate = frameRate; frame_rate_ = frame_rate;
return 0; return VPM_OK;
} }
/** /**
This function detects flicker in the video stream. As a side effect the mean value This function detects flicker in the video stream. As a side effect the
buffer is updated with the new mean value. mean value buffer is updated with the new mean value.
\return 0: No flickering detected\n \return 0: No flickering detected\n
1: Flickering detected\n 1: Flickering detected\n
2: Detection not possible due to unreliable frequency interval 2: Detection not possible due to unreliable frequency interval
-1: Error -1: Error
*/ */
int32_t VPMDeflickering::DetectFlicker() int32_t VPMDeflickering::DetectFlicker() {
{
/* Local variables */
uint32_t i; uint32_t i;
int32_t freqEst; // (Q4) Frequency estimate to base detection upon int32_t freqEst; // (Q4) Frequency estimate to base detection upon
int32_t retVal = -1; int32_t ret_val = -1;
/* Sanity check for _meanBufferLength */ /* Sanity check for mean_buffer_length_ */
if (_meanBufferLength < 2) if (mean_buffer_length_ < 2) {
{
/* Not possible to estimate frequency */ /* Not possible to estimate frequency */
return(2); return(2);
} }
/* Count zero crossings with a dead zone to be robust against noise. // Count zero crossings with a dead zone to be robust against noise. If the
* If the noise std is 2 pixel this corresponds to about 95% confidence interval. // noise std is 2 pixel this corresponds to about 95% confidence interval.
*/ int32_t deadzone = (kZeroCrossingDeadzone << kmean_valueScaling); // Q4
int32_t deadzone = (kZeroCrossingDeadzone << kMeanValueScaling); // Q4 int32_t meanOfBuffer = 0; // Mean value of mean value buffer.
int32_t meanOfBuffer = 0; // Mean value of mean value buffer int32_t numZeros = 0; // Number of zeros that cross the dead-zone.
int32_t numZeros = 0; // Number of zeros that cross the deadzone int32_t cntState = 0; // State variable for zero crossing regions.
int32_t cntState = 0; // State variable for zero crossing regions int32_t cntStateOld = 0; // Previous state for zero crossing regions.
int32_t cntStateOld = 0; // Previous state variable for zero crossing regions
for (i = 0; i < _meanBufferLength; i++) for (i = 0; i < mean_buffer_length_; i++) {
{ meanOfBuffer += mean_buffer_[i];
meanOfBuffer += _meanBuffer[i];
} }
meanOfBuffer += (_meanBufferLength >> 1); // Rounding, not truncation meanOfBuffer += (mean_buffer_length_ >> 1); // Rounding, not truncation.
meanOfBuffer /= _meanBufferLength; meanOfBuffer /= mean_buffer_length_;
/* Count zero crossings */ // Count zero crossings.
cntStateOld = (_meanBuffer[0] >= (meanOfBuffer + deadzone)); cntStateOld = (mean_buffer_[0] >= (meanOfBuffer + deadzone));
cntStateOld -= (_meanBuffer[0] <= (meanOfBuffer - deadzone)); cntStateOld -= (mean_buffer_[0] <= (meanOfBuffer - deadzone));
for (i = 1; i < _meanBufferLength; i++) for (i = 1; i < mean_buffer_length_; i++) {
{ cntState = (mean_buffer_[i] >= (meanOfBuffer + deadzone));
cntState = (_meanBuffer[i] >= (meanOfBuffer + deadzone)); cntState -= (mean_buffer_[i] <= (meanOfBuffer - deadzone));
cntState -= (_meanBuffer[i] <= (meanOfBuffer - deadzone)); if (cntStateOld == 0) {
if (cntStateOld == 0)
{
cntStateOld = -cntState; cntStateOld = -cntState;
} }
if (((cntState + cntStateOld) == 0) && (cntState != 0)) if (((cntState + cntStateOld) == 0) && (cntState != 0)) {
{
numZeros++; numZeros++;
cntStateOld = cntState; cntStateOld = cntState;
} }
} }
/* END count zero crossings */ // END count zero crossings.
/* Frequency estimation according to: /* Frequency estimation according to:
* freqEst = numZeros * frameRate / 2 / _meanBufferLength; * freqEst = numZeros * frame_rate / 2 / mean_buffer_length_;
* *
* Resolution is set to Q4 * Resolution is set to Q4
*/ */
freqEst = ((numZeros * 90000) << 3); freqEst = ((numZeros * 90000) << 3);
freqEst /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]); freqEst /=
(timestamp_buffer_[0] - timestamp_buffer_[mean_buffer_length_ - 1]);
/* Translate frequency estimate to regions close to 100 and 120 Hz */ /* Translate frequency estimate to regions close to 100 and 120 Hz */
uint8_t freqState = 0; // Current translation state; uint8_t freqState = 0; // Current translation state;
@ -416,13 +376,11 @@ int32_t VPMDeflickering::DetectFlicker()
// (1) Within valid interval, // (1) Within valid interval,
// (2) Out of range // (2) Out of range
int32_t freqAlias = freqEst; int32_t freqAlias = freqEst;
if (freqEst > kMinFrequencyToDetect) if (freqEst > kMinFrequencyToDetect) {
{
uint8_t aliasState = 1; uint8_t aliasState = 1;
while(freqState == 0) while(freqState == 0) {
{
/* Increase frequency */ /* Increase frequency */
freqAlias += (aliasState * _frameRate); freqAlias += (aliasState * frame_rate_);
freqAlias += ((freqEst << 1) * (1 - (aliasState << 1))); freqAlias += ((freqEst << 1) * (1 - (aliasState << 1)));
/* Compute state */ /* Compute state */
freqState = (abs(freqAlias - (100 << 4)) <= kFrequencyDeviation); freqState = (abs(freqAlias - (100 << 4)) <= kFrequencyDeviation);
@ -434,17 +392,14 @@ int32_t VPMDeflickering::DetectFlicker()
} }
} }
/* Is frequency estimate within detection region? */ /* Is frequency estimate within detection region? */
if (freqState == 1) if (freqState == 1) {
{ ret_val = 1;
retVal = 1; } else if (freqState == 0) {
}else if (freqState == 0) ret_val = 2;
{ } else {
retVal = 2; ret_val = 0;
}else
{
retVal = 0;
} }
return retVal; return ret_val;
} }
} // namespace } // namespace webrtc

41
webrtc/modules/video_processing/main/source/deflickering.h
View File

@ -8,12 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCEdeflickering__H
* deflickering.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCEdeflickering__H
*/
#ifndef VPM_DEFLICKERING_H
#define VPM_DEFLICKERING_H
#include <string.h> // NULL #include <string.h> // NULL
@ -22,44 +18,43 @@
namespace webrtc { namespace webrtc {
class VPMDeflickering class VPMDeflickering {
{ public:
public:
VPMDeflickering(); VPMDeflickering();
~VPMDeflickering(); ~VPMDeflickering();
int32_t ChangeUniqueId(int32_t id); int32_t ChangeUniqueId(int32_t id);
void Reset(); void Reset();
int32_t ProcessFrame(I420VideoFrame* frame, int32_t ProcessFrame(I420VideoFrame* frame,
VideoProcessingModule::FrameStats* stats); VideoProcessingModule::FrameStats* stats);
private:
private:
int32_t PreDetection(uint32_t timestamp, int32_t PreDetection(uint32_t timestamp,
const VideoProcessingModule::FrameStats& stats); const VideoProcessingModule::FrameStats& stats);
int32_t DetectFlicker(); int32_t DetectFlicker();
enum { kMeanBufferLength = 32 }; enum { kMeanBufferLength = 32 };
enum { kFrameHistorySize = 15 }; enum { kFrameHistory_size = 15 };
enum { kNumProbs = 12 }; enum { kNumProbs = 12 };
enum { kNumQuants = kNumProbs + 2 }; enum { kNumQuants = kNumProbs + 2 };
enum { kMaxOnlyLength = 5 }; enum { kMaxOnlyLength = 5 };
int32_t _id; int32_t id_;
uint32_t _meanBufferLength; uint32_t mean_buffer_length_;
uint8_t _detectionState; // 0: No flickering uint8_t detection_state_; // 0: No flickering
// 1: Flickering detected // 1: Flickering detected
// 2: In flickering // 2: In flickering
int32_t _meanBuffer[kMeanBufferLength]; int32_t mean_buffer_[kMeanBufferLength];
uint32_t _timestampBuffer[kMeanBufferLength]; uint32_t timestamp_buffer_[kMeanBufferLength];
uint32_t _frameRate; uint32_t frame_rate_;
static const uint16_t _probUW16[kNumProbs]; static const uint16_t prob_uw16_[kNumProbs];
static const uint16_t _weightUW16[kNumQuants - kMaxOnlyLength]; static const uint16_t weight_uw16_[kNumQuants - kMaxOnlyLength];
uint8_t _quantHistUW8[kFrameHistorySize][kNumQuants]; uint8_t quant_hist_uw8_[kFrameHistory_size][kNumQuants];
}; };
} // namespace } // namespace webrtc
#endif // VPM_DEFLICKERING_H #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCEdeflickering__H

183
webrtc/modules/video_processing/main/source/denoising.cc
View File

@ -14,79 +14,71 @@
#include <string.h> #include <string.h>
namespace webrtc { namespace webrtc {
// Down-sampling in time (unit: number of frames)
enum { kSubsamplingTime = 0 };
// Sub-sampling in width (unit: power of 2.
enum { kSubsamplingWidth = 0 };
// Sub-sampling in height (unit: power of 2)
enum { kSubsamplingHeight = 0 };
// (Q8) De-noising filter parameter
enum { kDenoiseFiltParam = 179 };
// (Q8) 1 - filter parameter
enum { kDenoiseFiltParamRec = 77 };
// (Q8) De-noising threshold level
enum { kDenoiseThreshold = 19200 };
enum { kSubsamplingTime = 0 }; // Down-sampling in time (unit: number of frames) VPMDenoising::VPMDenoising()
enum { kSubsamplingWidth = 0 }; // Sub-sampling in width (unit: power of 2) : id_(0),
enum { kSubsamplingHeight = 0 }; // Sub-sampling in height (unit: power of 2) moment1_(NULL),
enum { kDenoiseFiltParam = 179 }; // (Q8) De-noising filter parameter moment2_(NULL) {
enum { kDenoiseFiltParamRec = 77 }; // (Q8) 1 - filter parameter
enum { kDenoiseThreshold = 19200 }; // (Q8) De-noising threshold level
VPMDenoising::VPMDenoising() :
_id(0),
_moment1(NULL),
_moment2(NULL)
{
Reset(); Reset();
} }
VPMDenoising::~VPMDenoising() VPMDenoising::~VPMDenoising() {
{ if (moment1_) {
if (_moment1) delete [] moment1_;
{ moment1_ = NULL;
delete [] _moment1; }
_moment1 = NULL;
}
if (_moment2) if (moment2_) {
{ delete [] moment2_;
delete [] _moment2; moment2_ = NULL;
_moment2 = NULL;
} }
} }
int32_t int32_t VPMDenoising::ChangeUniqueId(const int32_t id) {
VPMDenoising::ChangeUniqueId(const int32_t id) id_ = id;
{
_id = id;
return VPM_OK; return VPM_OK;
} }
void void VPMDenoising::Reset() {
VPMDenoising::Reset() frame_size_ = 0;
{ denoise_frame_cnt_ = 0;
_frameSize = 0;
_denoiseFrameCnt = 0;
if (_moment1) if (moment1_) {
{ delete [] moment1_;
delete [] _moment1; moment1_ = NULL;
_moment1 = NULL;
} }
if (_moment2) if (moment2_) {
{ delete [] moment2_;
delete [] _moment2; moment2_ = NULL;
_moment2 = NULL;
} }
} }
int32_t int32_t VPMDenoising::ProcessFrame(I420VideoFrame* frame) {
VPMDenoising::ProcessFrame(I420VideoFrame* frame)
{
assert(frame); assert(frame);
int32_t thevar; int32_t thevar;
int k; int k;
int jsub, ksub; int jsub, ksub;
int32_t diff0; int32_t diff0;
uint32_t tmpMoment1; uint32_t tmp_moment1;
uint32_t tmpMoment2; uint32_t tmp_moment2;
uint32_t tmp; uint32_t tmp;
int32_t numPixelsChanged = 0; int32_t num_pixels_changed = 0;
if (frame->IsZeroSize()) if (frame->IsZeroSize()) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"zero size frame"); "zero size frame");
return VPM_GENERAL_ERROR; return VPM_GENERAL_ERROR;
} }
@ -95,86 +87,73 @@ VPMDenoising::ProcessFrame(I420VideoFrame* frame)
int height = frame->height(); int height = frame->height();
/* Size of luminance component */ /* Size of luminance component */
const uint32_t ysize = height * width; const uint32_t y_size = height * width;
/* Initialization */ /* Initialization */
if (ysize != _frameSize) if (y_size != frame_size_) {
{ delete [] moment1_;
delete [] _moment1; moment1_ = NULL;
_moment1 = NULL;
delete [] _moment2; delete [] moment2_;
_moment2 = NULL; moment2_ = NULL;
} }
_frameSize = ysize; frame_size_ = y_size;
if (!_moment1) if (!moment1_) {
{ moment1_ = new uint32_t[y_size];
_moment1 = new uint32_t[ysize]; memset(moment1_, 0, sizeof(uint32_t)*y_size);
memset(_moment1, 0, sizeof(uint32_t)*ysize);
} }
if (!_moment2) if (!moment2_) {
{ moment2_ = new uint32_t[y_size];
_moment2 = new uint32_t[ysize]; memset(moment2_, 0, sizeof(uint32_t)*y_size);
memset(_moment2, 0, sizeof(uint32_t)*ysize);
} }
/* Apply de-noising on each pixel, but update variance sub-sampled */ /* Apply de-noising on each pixel, but update variance sub-sampled */
uint8_t* buffer = frame->buffer(kYPlane); uint8_t* buffer = frame->buffer(kYPlane);
for (int i = 0; i < height; i++) for (int i = 0; i < height; i++) { // Collect over height
{ // Collect over height
k = i * width; k = i * width;
ksub = ((i >> kSubsamplingHeight) << kSubsamplingHeight) * width; ksub = ((i >> kSubsamplingHeight) << kSubsamplingHeight) * width;
for (int j = 0; j < width; j++) for (int j = 0; j < width; j++) { // Collect over width
{ // Collect over width
jsub = ((j >> kSubsamplingWidth) << kSubsamplingWidth); jsub = ((j >> kSubsamplingWidth) << kSubsamplingWidth);
/* Update mean value for every pixel and every frame */ /* Update mean value for every pixel and every frame */
tmpMoment1 = _moment1[k + j]; tmp_moment1 = moment1_[k + j];
tmpMoment1 *= kDenoiseFiltParam; // Q16 tmp_moment1 *= kDenoiseFiltParam; // Q16
tmpMoment1 += ((kDenoiseFiltParamRec * tmp_moment1 += ((kDenoiseFiltParamRec * ((uint32_t)buffer[k + j])) << 8);
((uint32_t)buffer[k + j])) << 8); tmp_moment1 >>= 8; // Q8
tmpMoment1 >>= 8; // Q8 moment1_[k + j] = tmp_moment1;
_moment1[k + j] = tmpMoment1;
tmpMoment2 = _moment2[ksub + jsub]; tmp_moment2 = moment2_[ksub + jsub];
if ((ksub == k) && (jsub == j) && (_denoiseFrameCnt == 0)) if ((ksub == k) && (jsub == j) && (denoise_frame_cnt_ == 0)) {
{
tmp = ((uint32_t)buffer[k + j] * tmp = ((uint32_t)buffer[k + j] *
(uint32_t)buffer[k + j]); (uint32_t)buffer[k + j]);
tmpMoment2 *= kDenoiseFiltParam; // Q16 tmp_moment2 *= kDenoiseFiltParam; // Q16
tmpMoment2 += ((kDenoiseFiltParamRec * tmp)<<8); tmp_moment2 += ((kDenoiseFiltParamRec * tmp) << 8);
tmpMoment2 >>= 8; // Q8 tmp_moment2 >>= 8; // Q8
} }
_moment2[k + j] = tmpMoment2; moment2_[k + j] = tmp_moment2;
/* Current event = deviation from mean value */ /* Current event = deviation from mean value */
diff0 = ((int32_t)buffer[k + j] << 8) - _moment1[k + j]; diff0 = ((int32_t)buffer[k + j] << 8) - moment1_[k + j];
/* Recent events = variance (variations over time) */ /* Recent events = variance (variations over time) */
thevar = _moment2[k + j]; thevar = moment2_[k + j];
thevar -= ((_moment1[k + j] * _moment1[k + j]) >> 8); thevar -= ((moment1_[k + j] * moment1_[k + j]) >> 8);
/*************************************************************************** // De-noising criteria, i.e., when should we replace a pixel by its mean.
* De-noising criteria, i.e., when should we replace a pixel by its mean // 1) recent events are minor.
* // 2) current events are minor.
* 1) recent events are minor
* 2) current events are minor
***************************************************************************/
if ((thevar < kDenoiseThreshold) if ((thevar < kDenoiseThreshold)
&& ((diff0 * diff0 >> 8) < kDenoiseThreshold)) && ((diff0 * diff0 >> 8) < kDenoiseThreshold)) {
{ // Replace with mean // Replace with mean.
buffer[k + j] = (uint8_t)(_moment1[k + j] >> 8); buffer[k + j] = (uint8_t)(moment1_[k + j] >> 8);
numPixelsChanged++; num_pixels_changed++;
} }
} }
} }
/* Update frame counter */ denoise_frame_cnt_++;
_denoiseFrameCnt++; if (denoise_frame_cnt_ > kSubsamplingTime)
if (_denoiseFrameCnt > kSubsamplingTime) denoise_frame_cnt_ = 0;
{
_denoiseFrameCnt = 0;
}
return numPixelsChanged; return num_pixels_changed;
} }
} // namespace } // namespace

28
webrtc/modules/video_processing/main/source/denoising.h
View File

@ -8,20 +8,16 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_DENOISING_H_
* denoising.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_DENOISING_H_
*/
#ifndef VPM_DENOISING_H
#define VPM_DENOISING_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
class VPMDenoising class VPMDenoising {
{ public:
public:
VPMDenoising(); VPMDenoising();
~VPMDenoising(); ~VPMDenoising();
@ -31,16 +27,16 @@ public:
int32_t ProcessFrame(I420VideoFrame* frame); int32_t ProcessFrame(I420VideoFrame* frame);
private: private:
int32_t _id; int32_t id_;
uint32_t* _moment1; // (Q8) First order moment (mean) uint32_t* moment1_; // (Q8) First order moment (mean).
uint32_t* _moment2; // (Q8) Second order moment uint32_t* moment2_; // (Q8) Second order moment.
uint32_t _frameSize; // Size (# of pixels) of frame uint32_t frame_size_; // Size (# of pixels) of frame.
int _denoiseFrameCnt; // Counter for subsampling in time int denoise_frame_cnt_; // Counter for subsampling in time.
}; };
} // namespace } // namespace webrtc
#endif // VPM_DENOISING_H #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_DENOISING_H_

188
webrtc/modules/video_processing/main/source/frame_preprocessor.cc
View File

@ -13,177 +13,137 @@
namespace webrtc { namespace webrtc {
VPMFramePreprocessor::VPMFramePreprocessor(): VPMFramePreprocessor::VPMFramePreprocessor()
_id(0), : id_(0),
_contentMetrics(NULL), content_metrics_(NULL),
_maxFrameRate(0), max_frame_rate_(0),
_resampledFrame(), resampled_frame_(),
_enableCA(false), enable_ca_(false),
_frameCnt(0) frame_cnt_(0) {
{ spatial_resampler_ = new VPMSimpleSpatialResampler();
_spatialResampler = new VPMSimpleSpatialResampler(); ca_ = new VPMContentAnalysis(true);
_ca = new VPMContentAnalysis(true); vd_ = new VPMVideoDecimator();
_vd = new VPMVideoDecimator();
} }
VPMFramePreprocessor::~VPMFramePreprocessor() VPMFramePreprocessor::~VPMFramePreprocessor() {
{
Reset(); Reset();
delete _spatialResampler; delete spatial_resampler_;
delete _ca; delete ca_;
delete _vd; delete vd_;
} }
int32_t int32_t VPMFramePreprocessor::ChangeUniqueId(const int32_t id) {
VPMFramePreprocessor::ChangeUniqueId(const int32_t id) id_ = id;
{
_id = id;
return VPM_OK; return VPM_OK;
} }
void void VPMFramePreprocessor::Reset() {
VPMFramePreprocessor::Reset() ca_->Release();
{ vd_->Reset();
_ca->Release(); content_metrics_ = NULL;
_vd->Reset(); spatial_resampler_->Reset();
_contentMetrics = NULL; enable_ca_ = false;
_spatialResampler->Reset(); frame_cnt_ = 0;
_enableCA = false;
_frameCnt = 0;
} }
void void VPMFramePreprocessor::EnableTemporalDecimation(bool enable) {
VPMFramePreprocessor::EnableTemporalDecimation(bool enable) vd_->EnableTemporalDecimation(enable);
{
_vd->EnableTemporalDecimation(enable);
}
void
VPMFramePreprocessor::EnableContentAnalysis(bool enable)
{
_enableCA = enable;
} }
void void VPMFramePreprocessor::EnableContentAnalysis(bool enable) {
VPMFramePreprocessor::SetInputFrameResampleMode(VideoFrameResampling resamplingMode) enable_ca_ = enable;
{
_spatialResampler->SetInputFrameResampleMode(resamplingMode);
} }
void VPMFramePreprocessor::SetInputFrameResampleMode(
VideoFrameResampling resampling_mode) {
spatial_resampler_->SetInputFrameResampleMode(resampling_mode);
}
int32_t int32_t VPMFramePreprocessor::SetMaxFramerate(uint32_t max_frame_rate) {
VPMFramePreprocessor::SetMaxFrameRate(uint32_t maxFrameRate) if (max_frame_rate == 0) return VPM_PARAMETER_ERROR;
{
if (maxFrameRate == 0) // Max allowed frame_rate.
{ max_frame_rate_ = max_frame_rate;
return vd_->SetMaxFramerate(max_frame_rate);
}
int32_t VPMFramePreprocessor::SetTargetResolution(
uint32_t width, uint32_t height, uint32_t frame_rate) {
if ( (width == 0) || (height == 0) || (frame_rate == 0)) {
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
} }
//Max allowed frame rate int32_t ret_val = 0;
_maxFrameRate = maxFrameRate; ret_val = spatial_resampler_->SetTargetFrameSize(width, height);
return _vd->SetMaxFrameRate(maxFrameRate); if (ret_val < 0) return ret_val;
}
ret_val = vd_->SetTargetframe_rate(frame_rate);
int32_t if (ret_val < 0) return ret_val;
VPMFramePreprocessor::SetTargetResolution(uint32_t width, uint32_t height, uint32_t frameRate)
{
if ( (width == 0) || (height == 0) || (frameRate == 0))
{
return VPM_PARAMETER_ERROR;
}
int32_t retVal = 0;
retVal = _spatialResampler->SetTargetFrameSize(width, height);
if (retVal < 0)
{
return retVal;
}
retVal = _vd->SetTargetFrameRate(frameRate);
if (retVal < 0)
{
return retVal;
}
return VPM_OK; return VPM_OK;
} }
void void VPMFramePreprocessor::UpdateIncomingframe_rate() {
VPMFramePreprocessor::UpdateIncomingFrameRate() vd_->UpdateIncomingframe_rate();
{
_vd->UpdateIncomingFrameRate();
} }
uint32_t uint32_t VPMFramePreprocessor::Decimatedframe_rate() {
VPMFramePreprocessor::DecimatedFrameRate() return vd_->Decimatedframe_rate();
{
return _vd->DecimatedFrameRate();
} }
uint32_t uint32_t VPMFramePreprocessor::DecimatedWidth() const {
VPMFramePreprocessor::DecimatedWidth() const return spatial_resampler_->TargetWidth();
{
return _spatialResampler->TargetWidth();
} }
uint32_t uint32_t VPMFramePreprocessor::DecimatedHeight() const {
VPMFramePreprocessor::DecimatedHeight() const return spatial_resampler_->TargetHeight();
{
return _spatialResampler->TargetHeight();
} }
int32_t int32_t VPMFramePreprocessor::PreprocessFrame(const I420VideoFrame& frame,
VPMFramePreprocessor::PreprocessFrame(const I420VideoFrame& frame, I420VideoFrame** processed_frame) {
I420VideoFrame** processedFrame) if (frame.IsZeroSize()) {
{
if (frame.IsZeroSize())
{
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
} }
_vd->UpdateIncomingFrameRate(); vd_->UpdateIncomingframe_rate();
if (_vd->DropFrame()) if (vd_->DropFrame()) {
{ WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, id_,
WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, _id,
"Drop frame due to frame rate"); "Drop frame due to frame rate");
return 1; // drop 1 frame return 1; // drop 1 frame
} }
// Resizing incoming frame if needed. Otherwise, remains NULL. // Resizing incoming frame if needed. Otherwise, remains NULL.
// We are not allowed to resample the input frame (must make a copy of it). // We are not allowed to resample the input frame (must make a copy of it).
*processedFrame = NULL; *processed_frame = NULL;
if (_spatialResampler->ApplyResample(frame.width(), frame.height())) { if (spatial_resampler_->ApplyResample(frame.width(), frame.height())) {
int32_t ret = _spatialResampler->ResampleFrame(frame, &_resampledFrame); int32_t ret = spatial_resampler_->ResampleFrame(frame, &resampled_frame_);
if (ret != VPM_OK) if (ret != VPM_OK) return ret;
return ret; *processed_frame = &resampled_frame_;
*processedFrame = &_resampledFrame;
} }
// Perform content analysis on the frame to be encoded. // Perform content analysis on the frame to be encoded.
if (_enableCA) if (enable_ca_) {
{
// Compute new metrics every |kSkipFramesCA| frames, starting with // Compute new metrics every |kSkipFramesCA| frames, starting with
// the first frame. // the first frame.
if (_frameCnt % kSkipFrameCA == 0) { if (frame_cnt_ % kSkipFrameCA == 0) {
if (*processedFrame == NULL) { if (*processed_frame == NULL) {
_contentMetrics = _ca->ComputeContentMetrics(frame); content_metrics_ = ca_->ComputeContentMetrics(frame);
} else { } else {
_contentMetrics = _ca->ComputeContentMetrics(_resampledFrame); content_metrics_ = ca_->ComputeContentMetrics(resampled_frame_);
} }
} }
++_frameCnt; ++frame_cnt_;
} }
return VPM_OK; return VPM_OK;
} }
VideoContentMetrics* VPMFramePreprocessor::ContentMetrics() const {
VideoContentMetrics* return content_metrics_;
VPMFramePreprocessor::ContentMetrics() const
{
return _contentMetrics;
} }
} // namespace } // namespace

63
webrtc/modules/video_processing/main/source/frame_preprocessor.h
View File

@ -11,8 +11,8 @@
/* /*
* frame_preprocessor.h * frame_preprocessor.h
*/ */
#ifndef VPM_FRAME_PREPROCESSOR_H #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_FRAME_PREPROCESSOR_H
#define VPM_FRAME_PREPROCESSOR_H #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_FRAME_PREPROCESSOR_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/modules/video_processing/main/source/content_analysis.h" #include "webrtc/modules/video_processing/main/source/content_analysis.h"
@ -22,11 +22,8 @@
namespace webrtc { namespace webrtc {
class VPMFramePreprocessor {
class VPMFramePreprocessor public:
{
public:
VPMFramePreprocessor(); VPMFramePreprocessor();
~VPMFramePreprocessor(); ~VPMFramePreprocessor();
@ -34,53 +31,53 @@ public:
void Reset(); void Reset();
// Enable temporal decimation // Enable temporal decimation.
void EnableTemporalDecimation(bool enable); void EnableTemporalDecimation(bool enable);
void SetInputFrameResampleMode(VideoFrameResampling resamplingMode); void SetInputFrameResampleMode(VideoFrameResampling resampling_mode);
//Enable content analysis // Enable content analysis.
void EnableContentAnalysis(bool enable); void EnableContentAnalysis(bool enable);
//Set max frame rate // Set max frame rate.
int32_t SetMaxFrameRate(uint32_t maxFrameRate); int32_t SetMaxFramerate(uint32_t max_frame_rate);
//Set target resolution: frame rate and dimension // Set target resolution: frame rate and dimension.
int32_t SetTargetResolution(uint32_t width, uint32_t height, int32_t SetTargetResolution(uint32_t width, uint32_t height,
uint32_t frameRate); uint32_t frame_rate);
//Update incoming frame rate/dimension // Update incoming frame rate/dimension.
void UpdateIncomingFrameRate(); void UpdateIncomingframe_rate();
int32_t updateIncomingFrameSize(uint32_t width, uint32_t height); int32_t updateIncomingFrameSize(uint32_t width, uint32_t height);
//Set decimated values: frame rate/dimension // Set decimated values: frame rate/dimension.
uint32_t DecimatedFrameRate(); uint32_t Decimatedframe_rate();
uint32_t DecimatedWidth() const; uint32_t DecimatedWidth() const;
uint32_t DecimatedHeight() const; uint32_t DecimatedHeight() const;
//Preprocess output: // Preprocess output:
int32_t PreprocessFrame(const I420VideoFrame& frame, int32_t PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processedFrame); I420VideoFrame** processed_frame);
VideoContentMetrics* ContentMetrics() const; VideoContentMetrics* ContentMetrics() const;
private: private:
// The content does not change so much every frame, so to reduce complexity // The content does not change so much every frame, so to reduce complexity
// we can compute new content metrics every |kSkipFrameCA| frames. // we can compute new content metrics every |kSkipFrameCA| frames.
enum { kSkipFrameCA = 2 }; enum { kSkipFrameCA = 2 };
int32_t _id; int32_t id_;
VideoContentMetrics* _contentMetrics; VideoContentMetrics* content_metrics_;
uint32_t _maxFrameRate; uint32_t max_frame_rate_;
I420VideoFrame _resampledFrame; I420VideoFrame resampled_frame_;
VPMSpatialResampler* _spatialResampler; VPMSpatialResampler* spatial_resampler_;
VPMContentAnalysis* _ca; VPMContentAnalysis* ca_;
VPMVideoDecimator* _vd; VPMVideoDecimator* vd_;
bool _enableCA; bool enable_ca_;
int _frameCnt; int frame_cnt_;
}; // end of VPMFramePreprocessor class definition };
} // namespace } // namespace webrtc
#endif // VPM_FRAME_PREPROCESS_H #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_FRAME_PREPROCESSOR_H

104
webrtc/modules/video_processing/main/source/spatial_resampler.cc
View File

@ -14,109 +14,85 @@
namespace webrtc { namespace webrtc {
VPMSimpleSpatialResampler::VPMSimpleSpatialResampler() VPMSimpleSpatialResampler::VPMSimpleSpatialResampler()
: : resampling_mode_(kFastRescaling),
_resamplingMode(kFastRescaling), target_width_(0),
_targetWidth(0), target_height_(0),
_targetHeight(0), scaler_() {}
_scaler()
{
}
VPMSimpleSpatialResampler::~VPMSimpleSpatialResampler() VPMSimpleSpatialResampler::~VPMSimpleSpatialResampler() {}
{
//
}
int32_t int32_t VPMSimpleSpatialResampler::SetTargetFrameSize(int32_t width,
VPMSimpleSpatialResampler::SetTargetFrameSize(int32_t width, int32_t height) {
int32_t height) if (resampling_mode_ == kNoRescaling) return VPM_OK;
{
if (_resamplingMode == kNoRescaling) {
return VPM_OK;
}
if (width < 1 || height < 1) { if (width < 1 || height < 1) return VPM_PARAMETER_ERROR;
return VPM_PARAMETER_ERROR;
}
_targetWidth = width; target_width_ = width;
_targetHeight = height; target_height_ = height;
return VPM_OK; return VPM_OK;
} }
void void VPMSimpleSpatialResampler::SetInputFrameResampleMode(
VPMSimpleSpatialResampler::SetInputFrameResampleMode(VideoFrameResampling VideoFrameResampling resampling_mode) {
resamplingMode) resampling_mode_ = resampling_mode;
{
_resamplingMode = resamplingMode;
} }
void void VPMSimpleSpatialResampler::Reset() {
VPMSimpleSpatialResampler::Reset() resampling_mode_ = kFastRescaling;
{ target_width_ = 0;
_resamplingMode = kFastRescaling; target_height_ = 0;
_targetWidth = 0;
_targetHeight = 0;
} }
int32_t int32_t VPMSimpleSpatialResampler::ResampleFrame(const I420VideoFrame& inFrame,
VPMSimpleSpatialResampler::ResampleFrame(const I420VideoFrame& inFrame, I420VideoFrame* outFrame) {
I420VideoFrame* outFrame)
{
// Don't copy if frame remains as is. // Don't copy if frame remains as is.
if (_resamplingMode == kNoRescaling) if (resampling_mode_ == kNoRescaling)
return VPM_OK; return VPM_OK;
// Check if re-sampling is needed // Check if re-sampling is needed
else if ((inFrame.width() == _targetWidth) && else if ((inFrame.width() == target_width_) &&
(inFrame.height() == _targetHeight)) { (inFrame.height() == target_height_)) {
return VPM_OK; return VPM_OK;
} }
// Setting scaler // Setting scaler
// TODO(mikhal/marpan): Should we allow for setting the filter mode in // TODO(mikhal/marpan): Should we allow for setting the filter mode in
// _scale.Set() with |_resamplingMode|? // _scale.Set() with |resampling_mode_|?
int retVal = 0; int ret_val = 0;
retVal = _scaler.Set(inFrame.width(), inFrame.height(), ret_val = scaler_.Set(inFrame.width(), inFrame.height(),
_targetWidth, _targetHeight, kI420, kI420, kScaleBox); target_width_, target_height_, kI420, kI420, kScaleBox);
if (retVal < 0) if (ret_val < 0)
return retVal; return ret_val;
retVal = _scaler.Scale(inFrame, outFrame); ret_val = scaler_.Scale(inFrame, outFrame);
// Setting time parameters to the output frame. // Setting time parameters to the output frame.
// Timestamp will be reset in Scale call above, so we should set it after. // Timestamp will be reset in Scale call above, so we should set it after.
outFrame->set_timestamp(inFrame.timestamp()); outFrame->set_timestamp(inFrame.timestamp());
outFrame->set_render_time_ms(inFrame.render_time_ms()); outFrame->set_render_time_ms(inFrame.render_time_ms());
if (retVal == 0) if (ret_val == 0)
return VPM_OK; return VPM_OK;
else else
return VPM_SCALE_ERROR; return VPM_SCALE_ERROR;
} }
int32_t int32_t VPMSimpleSpatialResampler::TargetHeight() {
VPMSimpleSpatialResampler::TargetHeight() return target_height_;
{
return _targetHeight;
} }
int32_t int32_t VPMSimpleSpatialResampler::TargetWidth() {
VPMSimpleSpatialResampler::TargetWidth() return target_width_;
{
return _targetWidth;
} }
bool bool VPMSimpleSpatialResampler::ApplyResample(int32_t width,
VPMSimpleSpatialResampler::ApplyResample(int32_t width, int32_t height) {
int32_t height) if ((width == target_width_ && height == target_height_) ||
{ resampling_mode_ == kNoRescaling)
if ((width == _targetWidth && height == _targetHeight) ||
_resamplingMode == kNoRescaling)
return false; return false;
else else
return true; return true;
} }
} // namespace } // namespace webrtc

36
webrtc/modules/video_processing/main/source/spatial_resampler.h
View File

@ -8,12 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_SPATIAL_RESAMPLER_H
* spatial_resampler.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_SPATIAL_RESAMPLER_H
*/
#ifndef VPM_SPATIAL_RESAMPLER_H
#define VPM_SPATIAL_RESAMPLER_H
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
@ -25,13 +21,12 @@
namespace webrtc { namespace webrtc {
class VPMSpatialResampler class VPMSpatialResampler {
{ public:
public:
virtual ~VPMSpatialResampler() {}; virtual ~VPMSpatialResampler() {};
virtual int32_t SetTargetFrameSize(int32_t width, int32_t height) = 0; virtual int32_t SetTargetFrameSize(int32_t width, int32_t height) = 0;
virtual void SetInputFrameResampleMode(VideoFrameResampling virtual void SetInputFrameResampleMode(VideoFrameResampling
resamplingMode) = 0; resampling_mode) = 0;
virtual void Reset() = 0; virtual void Reset() = 0;
virtual int32_t ResampleFrame(const I420VideoFrame& inFrame, virtual int32_t ResampleFrame(const I420VideoFrame& inFrame,
I420VideoFrame* outFrame) = 0; I420VideoFrame* outFrame) = 0;
@ -40,13 +35,12 @@ public:
virtual bool ApplyResample(int32_t width, int32_t height) = 0; virtual bool ApplyResample(int32_t width, int32_t height) = 0;
}; };
class VPMSimpleSpatialResampler : public VPMSpatialResampler class VPMSimpleSpatialResampler : public VPMSpatialResampler {
{ public:
public:
VPMSimpleSpatialResampler(); VPMSimpleSpatialResampler();
~VPMSimpleSpatialResampler(); ~VPMSimpleSpatialResampler();
virtual int32_t SetTargetFrameSize(int32_t width, int32_t height); virtual int32_t SetTargetFrameSize(int32_t width, int32_t height);
virtual void SetInputFrameResampleMode(VideoFrameResampling resamplingMode); virtual void SetInputFrameResampleMode(VideoFrameResampling resampling_mode);
virtual void Reset(); virtual void Reset();
virtual int32_t ResampleFrame(const I420VideoFrame& inFrame, virtual int32_t ResampleFrame(const I420VideoFrame& inFrame,
I420VideoFrame* outFrame); I420VideoFrame* outFrame);
@ -54,14 +48,14 @@ public:
virtual int32_t TargetHeight(); virtual int32_t TargetHeight();
virtual bool ApplyResample(int32_t width, int32_t height); virtual bool ApplyResample(int32_t width, int32_t height);
private: private:
VideoFrameResampling _resamplingMode; VideoFrameResampling resampling_mode_;
int32_t _targetWidth; int32_t target_width_;
int32_t _targetHeight; int32_t target_height_;
Scaler _scaler; Scaler scaler_;
}; };
} // namespace } // namespace webrtc
#endif #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_SPATIAL_RESAMPLER_H

257
webrtc/modules/video_processing/main/source/video_decimator.cc
View File

@ -17,219 +17,156 @@
namespace webrtc { namespace webrtc {
VPMVideoDecimator::VPMVideoDecimator() VPMVideoDecimator::VPMVideoDecimator()
: : overshoot_modifier_(0),
_overShootModifier(0), drop_count_(0),
_dropCount(0), keep_count_(0),
_keepCount(0), target_frame_rate_(30),
_targetFrameRate(30), incoming_frame_rate_(0.0f),
_incomingFrameRate(0.0f), max_frame_rate_(30),
_maxFrameRate(30), incoming_frame_times_(),
_incomingFrameTimes(), enable_temporal_decimation_(true) {
_enableTemporalDecimation(true)
{
Reset(); Reset();
} }
VPMVideoDecimator::~VPMVideoDecimator() VPMVideoDecimator::~VPMVideoDecimator() {}
{
// void VPMVideoDecimator::Reset() {
overshoot_modifier_ = 0;
drop_count_ = 0;
keep_count_ = 0;
target_frame_rate_ = 30;
incoming_frame_rate_ = 0.0f;
max_frame_rate_ = 30;
memset(incoming_frame_times_, 0, sizeof(incoming_frame_times_));
enable_temporal_decimation_ = true;
} }
void void VPMVideoDecimator::EnableTemporalDecimation(bool enable) {
VPMVideoDecimator::Reset() enable_temporal_decimation_ = enable;
{
_overShootModifier = 0;
_dropCount = 0;
_keepCount = 0;
_targetFrameRate = 30;
_incomingFrameRate = 0.0f;
_maxFrameRate = 30;
memset(_incomingFrameTimes, 0, sizeof(_incomingFrameTimes));
_enableTemporalDecimation = true;
} }
void int32_t VPMVideoDecimator::SetMaxFramerate(uint32_t max_frame_rate) {
VPMVideoDecimator::EnableTemporalDecimation(bool enable) if (max_frame_rate == 0) return VPM_PARAMETER_ERROR;
{
_enableTemporalDecimation = enable; max_frame_rate_ = max_frame_rate;
if (target_frame_rate_ > max_frame_rate_)
target_frame_rate_ = max_frame_rate_;
return VPM_OK;
} }
int32_t
VPMVideoDecimator::SetMaxFrameRate(uint32_t maxFrameRate)
{
if (maxFrameRate == 0)
{
return VPM_PARAMETER_ERROR;
}
_maxFrameRate = maxFrameRate; int32_t VPMVideoDecimator::SetTargetframe_rate(uint32_t frame_rate) {
if (frame_rate == 0) return VPM_PARAMETER_ERROR;
if (_targetFrameRate > _maxFrameRate)
{
_targetFrameRate = _maxFrameRate;
if (frame_rate > max_frame_rate_) {
// Override.
target_frame_rate_ = max_frame_rate_;
} else {
target_frame_rate_ = frame_rate;
} }
return VPM_OK; return VPM_OK;
} }
int32_t bool VPMVideoDecimator::DropFrame() {
VPMVideoDecimator::SetTargetFrameRate(uint32_t frameRate) if (!enable_temporal_decimation_) return false;
{
if (frameRate == 0)
{
return VPM_PARAMETER_ERROR;
}
if (frameRate > _maxFrameRate)
{
//override
_targetFrameRate = _maxFrameRate;
}
else
{
_targetFrameRate = frameRate;
}
return VPM_OK;
}
bool if (incoming_frame_rate_ <= 0) return false;
VPMVideoDecimator::DropFrame()
{
if (!_enableTemporalDecimation)
{
return false;
}
if (_incomingFrameRate <= 0) const uint32_t incomingframe_rate =
{ static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
return false;
}
const uint32_t incomingFrameRate = static_cast<uint32_t>(_incomingFrameRate + 0.5f); if (target_frame_rate_ == 0) return true;
if (_targetFrameRate == 0)
{
return true;
}
bool drop = false; bool drop = false;
if (incomingFrameRate > _targetFrameRate) if (incomingframe_rate > target_frame_rate_) {
{ int32_t overshoot =
int32_t overshoot = _overShootModifier + (incomingFrameRate - _targetFrameRate); overshoot_modifier_ + (incomingframe_rate - target_frame_rate_);
if(overshoot < 0) if (overshoot < 0) {
{
overshoot = 0; overshoot = 0;
_overShootModifier = 0; overshoot_modifier_ = 0;
} }
if (overshoot && 2 * overshoot < (int32_t) incomingFrameRate) if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) {
{ if (drop_count_) { // Just got here so drop to be sure.
drop_count_ = 0;
if (_dropCount) // Just got here so drop to be sure.
{
_dropCount = 0;
return true; return true;
} }
const uint32_t dropVar = incomingFrameRate / overshoot; const uint32_t dropVar = incomingframe_rate / overshoot;
if (_keepCount >= dropVar) if (keep_count_ >= dropVar) {
{
drop = true; drop = true;
_overShootModifier = -((int32_t) incomingFrameRate % overshoot) / 3; overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3;
_keepCount = 1; keep_count_ = 1;
} else {
keep_count_++;
} }
else } else {
{ keep_count_ = 0;
const uint32_t dropVar = overshoot / target_frame_rate_;
_keepCount++; if (drop_count_ < dropVar) {
}
}
else
{
_keepCount = 0;
const uint32_t dropVar = overshoot / _targetFrameRate;
if (_dropCount < dropVar)
{
drop = true; drop = true;
_dropCount++; drop_count_++;
} } else {
else overshoot_modifier_ = overshoot % target_frame_rate_;
{
_overShootModifier = overshoot % _targetFrameRate;
drop = false; drop = false;
_dropCount = 0; drop_count_ = 0;
} }
} }
} }
return drop; return drop;
} }
uint32_t uint32_t VPMVideoDecimator::Decimatedframe_rate() {
VPMVideoDecimator::DecimatedFrameRate() ProcessIncomingframe_rate(TickTime::MillisecondTimestamp());
{ if (!enable_temporal_decimation_) {
ProcessIncomingFrameRate(TickTime::MillisecondTimestamp()); return static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
if (!_enableTemporalDecimation)
{
return static_cast<uint32_t>(_incomingFrameRate + 0.5f);
} }
return VD_MIN(_targetFrameRate, static_cast<uint32_t>(_incomingFrameRate + 0.5f)); return VD_MIN(target_frame_rate_,
static_cast<uint32_t>(incoming_frame_rate_ + 0.5f));
} }
uint32_t uint32_t VPMVideoDecimator::Inputframe_rate() {
VPMVideoDecimator::InputFrameRate() ProcessIncomingframe_rate(TickTime::MillisecondTimestamp());
{ return static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
ProcessIncomingFrameRate(TickTime::MillisecondTimestamp());
return static_cast<uint32_t>(_incomingFrameRate + 0.5f);
} }
void void VPMVideoDecimator::UpdateIncomingframe_rate() {
VPMVideoDecimator::UpdateIncomingFrameRate()
{
int64_t now = TickTime::MillisecondTimestamp(); int64_t now = TickTime::MillisecondTimestamp();
if(_incomingFrameTimes[0] == 0) if (incoming_frame_times_[0] == 0) {
{ // First no shift.
// first no shift } else {
} else // Shift.
{ for (int i = kFrameCountHistory_size - 2; i >= 0; i--) {
// shift incoming_frame_times_[i+1] = incoming_frame_times_[i];
for(int i = (kFrameCountHistorySize - 2); i >= 0 ; i--)
{
_incomingFrameTimes[i+1] = _incomingFrameTimes[i];
} }
} }
_incomingFrameTimes[0] = now; incoming_frame_times_[0] = now;
ProcessIncomingFrameRate(now); ProcessIncomingframe_rate(now);
} }
void void VPMVideoDecimator::ProcessIncomingframe_rate(int64_t now) {
VPMVideoDecimator::ProcessIncomingFrameRate(int64_t now)
{
int32_t num = 0; int32_t num = 0;
int32_t nrOfFrames = 0; int32_t nrOfFrames = 0;
for(num = 1; num < (kFrameCountHistorySize - 1); num++) for (num = 1; num < (kFrameCountHistory_size - 1); num++) {
{ // Don't use data older than 2sec.
if (_incomingFrameTimes[num] <= 0 || if (incoming_frame_times_[num] <= 0 ||
now - _incomingFrameTimes[num] > kFrameHistoryWindowMs) // don't use data older than 2sec now - incoming_frame_times_[num] > kFrameHistoryWindowMs) {
{
break; break;
} else } else {
{
nrOfFrames++; nrOfFrames++;
} }
} }
if (num > 1) if (num > 1) {
{ int64_t diff = now - incoming_frame_times_[num-1];
int64_t diff = now - _incomingFrameTimes[num-1]; incoming_frame_rate_ = 1.0;
_incomingFrameRate = 1.0; if (diff > 0) {
if(diff >0) incoming_frame_rate_ = nrOfFrames * 1000.0f / static_cast<float>(diff);
{
_incomingFrameRate = nrOfFrames * 1000.0f / static_cast<float>(diff);
} }
} } else {
else incoming_frame_rate_ = static_cast<float>(nrOfFrames);
{
_incomingFrameRate = static_cast<float>(nrOfFrames);
} }
} }
} // namespace } // namespace webrtc

55
webrtc/modules/video_processing/main/source/video_decimator.h
View File

@ -8,20 +8,16 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_VIDEO_DECIMATOR_H
* video_decimator.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_VIDEO_DECIMATOR_H
*/
#ifndef VPM_VIDEO_DECIMATOR_H
#define VPM_VIDEO_DECIMATOR_H
#include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/interface/module_common_types.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
class VPMVideoDecimator class VPMVideoDecimator {
{ public:
public:
VPMVideoDecimator(); VPMVideoDecimator();
~VPMVideoDecimator(); ~VPMVideoDecimator();
@ -29,37 +25,36 @@ public:
void EnableTemporalDecimation(bool enable); void EnableTemporalDecimation(bool enable);
int32_t SetMaxFrameRate(uint32_t maxFrameRate); int32_t SetMaxFramerate(uint32_t max_frame_rate);
int32_t SetTargetFrameRate(uint32_t frameRate); int32_t SetTargetframe_rate(uint32_t frame_rate);
bool DropFrame(); bool DropFrame();
void UpdateIncomingFrameRate(); void UpdateIncomingframe_rate();
// Get Decimated Frame Rate/Dimensions // Get Decimated Frame Rate/Dimensions.
uint32_t DecimatedFrameRate(); uint32_t Decimatedframe_rate();
//Get input frame rate // Get input frame rate.
uint32_t InputFrameRate(); uint32_t Inputframe_rate();
private: private:
void ProcessIncomingFrameRate(int64_t now); void ProcessIncomingframe_rate(int64_t now);
enum { kFrameCountHistorySize = 90}; enum { kFrameCountHistory_size = 90};
enum { kFrameHistoryWindowMs = 2000}; enum { kFrameHistoryWindowMs = 2000};
// Temporal decimation // Temporal decimation.
int32_t _overShootModifier; int32_t overshoot_modifier_;
uint32_t _dropCount; uint32_t drop_count_;
uint32_t _keepCount; uint32_t keep_count_;
uint32_t _targetFrameRate; uint32_t target_frame_rate_;
float _incomingFrameRate; float incoming_frame_rate_;
uint32_t _maxFrameRate; uint32_t max_frame_rate_;
int64_t _incomingFrameTimes[kFrameCountHistorySize]; int64_t incoming_frame_times_[kFrameCountHistory_size];
bool _enableTemporalDecimation; bool enable_temporal_decimation_;
}; };
} // namespace } // namespace webrtc
#endif #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_VIDEO_DECIMATOR_H

269
webrtc/modules/video_processing/main/source/video_processing_impl.cc
View File

@ -17,109 +17,78 @@
namespace webrtc { namespace webrtc {
namespace namespace {
{ void SetSubSampling(VideoProcessingModule::FrameStats* stats,
void
SetSubSampling(VideoProcessingModule::FrameStats* stats,
const int32_t width, const int32_t width,
const int32_t height) const int32_t height) {
{ if (width * height >= 640 * 480) {
if (width * height >= 640 * 480)
{
stats->subSamplWidth = 3; stats->subSamplWidth = 3;
stats->subSamplHeight = 3; stats->subSamplHeight = 3;
} } else if (width * height >= 352 * 288) {
else if (width * height >= 352 * 288)
{
stats->subSamplWidth = 2; stats->subSamplWidth = 2;
stats->subSamplHeight = 2; stats->subSamplHeight = 2;
} } else if (width * height >= 176 * 144) {
else if (width * height >= 176 * 144)
{
stats->subSamplWidth = 1; stats->subSamplWidth = 1;
stats->subSamplHeight = 1; stats->subSamplHeight = 1;
} } else {
else
{
stats->subSamplWidth = 0; stats->subSamplWidth = 0;
stats->subSamplHeight = 0; stats->subSamplHeight = 0;
} }
}
} }
} // namespace
VideoProcessingModule* VideoProcessingModule* VideoProcessingModule::Create(const int32_t id) {
VideoProcessingModule::Create(const int32_t id)
{
return new VideoProcessingModuleImpl(id); return new VideoProcessingModuleImpl(id);
} }
void void VideoProcessingModule::Destroy(VideoProcessingModule* module) {
VideoProcessingModule::Destroy(VideoProcessingModule* module)
{
if (module) if (module)
{
delete static_cast<VideoProcessingModuleImpl*>(module); delete static_cast<VideoProcessingModuleImpl*>(module);
}
} }
int32_t int32_t VideoProcessingModuleImpl::ChangeUniqueId(const int32_t id) {
VideoProcessingModuleImpl::ChangeUniqueId(const int32_t id) CriticalSectionScoped mutex(&mutex_);
{ id_ = id;
CriticalSectionScoped mutex(&_mutex); brightness_detection_.ChangeUniqueId(id);
_id = id; deflickering_.ChangeUniqueId(id);
_brightnessDetection.ChangeUniqueId(id); denoising_.ChangeUniqueId(id);
_deflickering.ChangeUniqueId(id); frame_pre_processor_.ChangeUniqueId(id);
_denoising.ChangeUniqueId(id);
_framePreProcessor.ChangeUniqueId(id);
return VPM_OK; return VPM_OK;
} }
int32_t int32_t VideoProcessingModuleImpl::Id() const {
VideoProcessingModuleImpl::Id() const CriticalSectionScoped mutex(&mutex_);
{ return id_;
CriticalSectionScoped mutex(&_mutex);
return _id;
} }
VideoProcessingModuleImpl::VideoProcessingModuleImpl(const int32_t id) : VideoProcessingModuleImpl::VideoProcessingModuleImpl(const int32_t id)
_id(id), : id_(id),
_mutex(*CriticalSectionWrapper::CreateCriticalSection()) mutex_(*CriticalSectionWrapper::CreateCriticalSection()) {
{ brightness_detection_.ChangeUniqueId(id);
_brightnessDetection.ChangeUniqueId(id); deflickering_.ChangeUniqueId(id);
_deflickering.ChangeUniqueId(id); denoising_.ChangeUniqueId(id);
_denoising.ChangeUniqueId(id); frame_pre_processor_.ChangeUniqueId(id);
_framePreProcessor.ChangeUniqueId(id); WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, id_,
WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, _id,
"Created"); "Created");
} }
VideoProcessingModuleImpl::~VideoProcessingModuleImpl() {
VideoProcessingModuleImpl::~VideoProcessingModuleImpl() WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, id_,
{
WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, _id,
"Destroyed"); "Destroyed");
delete &mutex_;
delete &_mutex;
} }
void void VideoProcessingModuleImpl::Reset() {
VideoProcessingModuleImpl::Reset() CriticalSectionScoped mutex(&mutex_);
{ deflickering_.Reset();
CriticalSectionScoped mutex(&_mutex); denoising_.Reset();
_deflickering.Reset(); brightness_detection_.Reset();
_denoising.Reset(); frame_pre_processor_.Reset();
_brightnessDetection.Reset();
_framePreProcessor.Reset();
} }
int32_t int32_t VideoProcessingModule::GetFrameStats(FrameStats* stats,
VideoProcessingModule::GetFrameStats(FrameStats* stats, const I420VideoFrame& frame) {
const I420VideoFrame& frame) if (frame.IsZeroSize()) {
{
if (frame.IsZeroSize())
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, -1, WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, -1,
"zero size frame"); "zero size frame");
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
@ -133,161 +102,119 @@ VideoProcessingModule::GetFrameStats(FrameStats* stats,
const uint8_t* buffer = frame.buffer(kYPlane); const uint8_t* buffer = frame.buffer(kYPlane);
// Compute histogram and sum of frame // Compute histogram and sum of frame
for (int i = 0; i < height; i += (1 << stats->subSamplHeight)) for (int i = 0; i < height; i += (1 << stats->subSamplHeight)) {
{
int k = i * width; int k = i * width;
for (int j = 0; j < width; j += (1 << stats->subSamplWidth)) for (int j = 0; j < width; j += (1 << stats->subSamplWidth)) {
{
stats->hist[buffer[k + j]]++; stats->hist[buffer[k + j]]++;
stats->sum += buffer[k + j]; stats->sum += buffer[k + j];
} }
} }
stats->numPixels = (width * height) / ((1 << stats->subSamplWidth) * stats->num_pixels = (width * height) / ((1 << stats->subSamplWidth) *
(1 << stats->subSamplHeight)); (1 << stats->subSamplHeight));
assert(stats->numPixels > 0); assert(stats->num_pixels > 0);
// Compute mean value of frame // Compute mean value of frame
stats->mean = stats->sum / stats->numPixels; stats->mean = stats->sum / stats->num_pixels;
return VPM_OK; return VPM_OK;
} }
bool bool VideoProcessingModule::ValidFrameStats(const FrameStats& stats) {
VideoProcessingModule::ValidFrameStats(const FrameStats& stats) if (stats.num_pixels == 0) return false;
{
if (stats.numPixels == 0)
{
return false;
}
return true; return true;
} }
void void VideoProcessingModule::ClearFrameStats(FrameStats* stats) {
VideoProcessingModule::ClearFrameStats(FrameStats* stats)
{
stats->mean = 0; stats->mean = 0;
stats->sum = 0; stats->sum = 0;
stats->numPixels = 0; stats->num_pixels = 0;
stats->subSamplWidth = 0; stats->subSamplWidth = 0;
stats->subSamplHeight = 0; stats->subSamplHeight = 0;
memset(stats->hist, 0, sizeof(stats->hist)); memset(stats->hist, 0, sizeof(stats->hist));
} }
int32_t int32_t VideoProcessingModule::ColorEnhancement(I420VideoFrame* frame) {
VideoProcessingModule::ColorEnhancement(I420VideoFrame* frame)
{
return VideoProcessing::ColorEnhancement(frame); return VideoProcessing::ColorEnhancement(frame);
} }
int32_t int32_t VideoProcessingModule::Brighten(I420VideoFrame* frame, int delta) {
VideoProcessingModule::Brighten(I420VideoFrame* frame, int delta)
{
return VideoProcessing::Brighten(frame, delta); return VideoProcessing::Brighten(frame, delta);
} }
int32_t int32_t VideoProcessingModuleImpl::Deflickering(I420VideoFrame* frame,
VideoProcessingModuleImpl::Deflickering(I420VideoFrame* frame, FrameStats* stats) {
FrameStats* stats) CriticalSectionScoped mutex(&mutex_);
{ return deflickering_.ProcessFrame(frame, stats);
CriticalSectionScoped mutex(&_mutex);
return _deflickering.ProcessFrame(frame, stats);
} }
int32_t int32_t VideoProcessingModuleImpl::Denoising(I420VideoFrame* frame) {
VideoProcessingModuleImpl::Denoising(I420VideoFrame* frame) CriticalSectionScoped mutex(&mutex_);
{ return denoising_.ProcessFrame(frame);
CriticalSectionScoped mutex(&_mutex);
return _denoising.ProcessFrame(frame);
} }
int32_t int32_t VideoProcessingModuleImpl::BrightnessDetection(
VideoProcessingModuleImpl::BrightnessDetection(const I420VideoFrame& frame, const I420VideoFrame& frame,
const FrameStats& stats) const FrameStats& stats) {
{ CriticalSectionScoped mutex(&mutex_);
CriticalSectionScoped mutex(&_mutex); return brightness_detection_.ProcessFrame(frame, stats);
return _brightnessDetection.ProcessFrame(frame, stats);
} }
void void VideoProcessingModuleImpl::EnableTemporalDecimation(bool enable) {
VideoProcessingModuleImpl::EnableTemporalDecimation(bool enable) CriticalSectionScoped mutex(&mutex_);
{ frame_pre_processor_.EnableTemporalDecimation(enable);
CriticalSectionScoped mutex(&_mutex);
_framePreProcessor.EnableTemporalDecimation(enable);
} }
void void VideoProcessingModuleImpl::SetInputFrameResampleMode(VideoFrameResampling
VideoProcessingModuleImpl::SetInputFrameResampleMode(VideoFrameResampling resampling_mode) {
resamplingMode) CriticalSectionScoped cs(&mutex_);
{ frame_pre_processor_.SetInputFrameResampleMode(resampling_mode);
CriticalSectionScoped cs(&_mutex);
_framePreProcessor.SetInputFrameResampleMode(resamplingMode);
} }
int32_t int32_t VideoProcessingModuleImpl::SetMaxFramerate(uint32_t max_frame_rate) {
VideoProcessingModuleImpl::SetMaxFrameRate(uint32_t maxFrameRate) CriticalSectionScoped cs(&mutex_);
{ return frame_pre_processor_.SetMaxFramerate(max_frame_rate);
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.SetMaxFrameRate(maxFrameRate);
} }
int32_t int32_t VideoProcessingModuleImpl::SetTargetResolution(uint32_t width,
VideoProcessingModuleImpl::SetTargetResolution(uint32_t width,
uint32_t height, uint32_t height,
uint32_t frameRate) uint32_t frame_rate) {
{ CriticalSectionScoped cs(&mutex_);
CriticalSectionScoped cs(&_mutex); return frame_pre_processor_.SetTargetResolution(width, height, frame_rate);
return _framePreProcessor.SetTargetResolution(width, height, frameRate);
} }
uint32_t VideoProcessingModuleImpl::Decimatedframe_rate() {
uint32_t CriticalSectionScoped cs(&mutex_);
VideoProcessingModuleImpl::DecimatedFrameRate() return frame_pre_processor_.Decimatedframe_rate();
{
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.DecimatedFrameRate();
} }
uint32_t VideoProcessingModuleImpl::DecimatedWidth() const {
uint32_t CriticalSectionScoped cs(&mutex_);
VideoProcessingModuleImpl::DecimatedWidth() const return frame_pre_processor_.DecimatedWidth();
{
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.DecimatedWidth();
} }
uint32_t uint32_t VideoProcessingModuleImpl::DecimatedHeight() const {
VideoProcessingModuleImpl::DecimatedHeight() const CriticalSectionScoped cs(&mutex_);
{ return frame_pre_processor_.DecimatedHeight();
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.DecimatedHeight();
} }
int32_t int32_t VideoProcessingModuleImpl::PreprocessFrame(
VideoProcessingModuleImpl::PreprocessFrame(const I420VideoFrame& frame, const I420VideoFrame& frame,
I420VideoFrame **processedFrame) I420VideoFrame **processed_frame) {
{ CriticalSectionScoped mutex(&mutex_);
CriticalSectionScoped mutex(&_mutex); return frame_pre_processor_.PreprocessFrame(frame, processed_frame);
return _framePreProcessor.PreprocessFrame(frame, processedFrame);
} }
VideoContentMetrics* VideoContentMetrics* VideoProcessingModuleImpl::ContentMetrics() const {
VideoProcessingModuleImpl::ContentMetrics() const CriticalSectionScoped mutex(&mutex_);
{ return frame_pre_processor_.ContentMetrics();
CriticalSectionScoped mutex(&_mutex);
return _framePreProcessor.ContentMetrics();
} }
void VideoProcessingModuleImpl::EnableContentAnalysis(bool enable) {
void CriticalSectionScoped mutex(&mutex_);
VideoProcessingModuleImpl::EnableContentAnalysis(bool enable) frame_pre_processor_.EnableContentAnalysis(enable);
{
CriticalSectionScoped mutex(&_mutex);
_framePreProcessor.EnableContentAnalysis(enable);
} }
} // namespace } // namespace webrtc

41
webrtc/modules/video_processing/main/source/video_processing_impl.h
View File

@ -22,10 +22,8 @@
namespace webrtc { namespace webrtc {
class CriticalSectionWrapper; class CriticalSectionWrapper;
class VideoProcessingModuleImpl : public VideoProcessingModule class VideoProcessingModuleImpl : public VideoProcessingModule {
{ public:
public:
VideoProcessingModuleImpl(int32_t id); VideoProcessingModuleImpl(int32_t id);
virtual ~VideoProcessingModuleImpl(); virtual ~VideoProcessingModuleImpl();
@ -43,46 +41,45 @@ public:
virtual int32_t BrightnessDetection(const I420VideoFrame& frame, virtual int32_t BrightnessDetection(const I420VideoFrame& frame,
const FrameStats& stats); const FrameStats& stats);
//Frame pre-processor functions // Frame pre-processor functions
//Enable temporal decimation // Enable temporal decimation
virtual void EnableTemporalDecimation(bool enable); virtual void EnableTemporalDecimation(bool enable);
virtual void SetInputFrameResampleMode(VideoFrameResampling resamplingMode); virtual void SetInputFrameResampleMode(VideoFrameResampling resampling_mode);
//Enable content analysis // Enable content analysis
virtual void EnableContentAnalysis(bool enable); virtual void EnableContentAnalysis(bool enable);
//Set max frame rate // Set max frame rate
virtual int32_t SetMaxFrameRate(uint32_t maxFrameRate); virtual int32_t SetMaxFramerate(uint32_t max_frame_rate);
// Set Target Resolution: frame rate and dimension // Set Target Resolution: frame rate and dimension
virtual int32_t SetTargetResolution(uint32_t width, virtual int32_t SetTargetResolution(uint32_t width,
uint32_t height, uint32_t height,
uint32_t frameRate); uint32_t frame_rate);
// Get decimated values: frame rate/dimension // Get decimated values: frame rate/dimension
virtual uint32_t DecimatedFrameRate(); virtual uint32_t Decimatedframe_rate();
virtual uint32_t DecimatedWidth() const; virtual uint32_t DecimatedWidth() const;
virtual uint32_t DecimatedHeight() const; virtual uint32_t DecimatedHeight() const;
// Preprocess: // Preprocess:
// Pre-process incoming frame: Sample when needed and compute content // Pre-process incoming frame: Sample when needed and compute content
// metrics when enabled. // metrics when enabled.
// If no resampling takes place - processedFrame is set to NULL. // If no resampling takes place - processed_frame is set to NULL.
virtual int32_t PreprocessFrame(const I420VideoFrame& frame, virtual int32_t PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processedFrame); I420VideoFrame** processed_frame);
virtual VideoContentMetrics* ContentMetrics() const; virtual VideoContentMetrics* ContentMetrics() const;
private: private:
int32_t _id; int32_t id_;
CriticalSectionWrapper& _mutex; CriticalSectionWrapper& mutex_;
VPMDeflickering deflickering_;
VPMDeflickering _deflickering; VPMDenoising denoising_;
VPMDenoising _denoising; VPMBrightnessDetection brightness_detection_;
VPMBrightnessDetection _brightnessDetection; VPMFramePreprocessor frame_pre_processor_;
VPMFramePreprocessor _framePreProcessor;
}; };
} // namespace } // namespace

56
webrtc/modules/video_processing/main/test/unit_test/brightness_detection_test.cc
View File

@ -19,24 +19,24 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
uint32_t frameNum = 0; uint32_t frameNum = 0;
int32_t brightnessWarning = 0; int32_t brightnessWarning = 0;
uint32_t warningCount = 0; uint32_t warningCount = 0;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_GE(brightnessWarning = _vpm->BrightnessDetection(_videoFrame, ASSERT_GE(brightnessWarning = vpm_->BrightnessDetection(video_frame_,
stats), 0); stats), 0);
if (brightnessWarning != VideoProcessingModule::kNoWarning) if (brightnessWarning != VideoProcessingModule::kNoWarning)
{ {
warningCount++; warningCount++;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Expect few warnings // Expect few warnings
float warningProportion = static_cast<float>(warningCount) / frameNum * 100; float warningProportion = static_cast<float>(warningCount) / frameNum * 100;
@ -44,21 +44,21 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
printf("Stock foreman: %.1f %%\n", warningProportion); printf("Stock foreman: %.1f %%\n", warningProportion);
EXPECT_LT(warningProportion, 10); EXPECT_LT(warningProportion, 10);
rewind(_sourceFile); rewind(source_file_);
frameNum = 0; frameNum = 0;
warningCount = 0; warningCount = 0;
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length && frame_length_ &&
frameNum < 300) frameNum < 300)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
uint8_t* frame = _videoFrame.buffer(kYPlane); uint8_t* frame = video_frame_.buffer(kYPlane);
uint32_t yTmp = 0; uint32_t yTmp = 0;
for (int yIdx = 0; yIdx < _width * _height; yIdx++) for (int yIdx = 0; yIdx < width_ * height_; yIdx++)
{ {
yTmp = frame[yIdx] << 1; yTmp = frame[yIdx] << 1;
if (yTmp > 255) if (yTmp > 255)
@ -69,8 +69,8 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
} }
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_GE(brightnessWarning = _vpm->BrightnessDetection(_videoFrame, ASSERT_GE(brightnessWarning = vpm_->BrightnessDetection(video_frame_,
stats), 0); stats), 0);
EXPECT_NE(VideoProcessingModule::kDarkWarning, brightnessWarning); EXPECT_NE(VideoProcessingModule::kDarkWarning, brightnessWarning);
if (brightnessWarning == VideoProcessingModule::kBrightWarning) if (brightnessWarning == VideoProcessingModule::kBrightWarning)
@ -78,35 +78,35 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
warningCount++; warningCount++;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Expect many brightness warnings // Expect many brightness warnings
warningProportion = static_cast<float>(warningCount) / frameNum * 100; warningProportion = static_cast<float>(warningCount) / frameNum * 100;
printf("Bright foreman: %.1f %%\n", warningProportion); printf("Bright foreman: %.1f %%\n", warningProportion);
EXPECT_GT(warningProportion, 95); EXPECT_GT(warningProportion, 95);
rewind(_sourceFile); rewind(source_file_);
frameNum = 0; frameNum = 0;
warningCount = 0; warningCount = 0;
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length && frameNum < 300) frame_length_ && frameNum < 300)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
uint8_t* y_plane = _videoFrame.buffer(kYPlane); uint8_t* y_plane = video_frame_.buffer(kYPlane);
int32_t yTmp = 0; int32_t yTmp = 0;
for (int yIdx = 0; yIdx < _width * _height; yIdx++) for (int yIdx = 0; yIdx < width_ * height_; yIdx++)
{ {
yTmp = y_plane[yIdx] >> 1; yTmp = y_plane[yIdx] >> 1;
y_plane[yIdx] = static_cast<uint8_t>(yTmp); y_plane[yIdx] = static_cast<uint8_t>(yTmp);
} }
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_GE(brightnessWarning = _vpm->BrightnessDetection(_videoFrame, ASSERT_GE(brightnessWarning = vpm_->BrightnessDetection(video_frame_,
stats), 0); stats), 0);
EXPECT_NE(VideoProcessingModule::kBrightWarning, brightnessWarning); EXPECT_NE(VideoProcessingModule::kBrightWarning, brightnessWarning);
if (brightnessWarning == VideoProcessingModule::kDarkWarning) if (brightnessWarning == VideoProcessingModule::kDarkWarning)
@ -114,7 +114,7 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
warningCount++; warningCount++;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Expect many darkness warnings // Expect many darkness warnings
warningProportion = static_cast<float>(warningCount) / frameNum * 100; warningProportion = static_cast<float>(warningCount) / frameNum * 100;

76
webrtc/modules/video_processing/main/test/unit_test/color_enhancement_test.cc
View File

@ -23,14 +23,14 @@ TEST_F(VideoProcessingModuleTest, ColorEnhancement)
{ {
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
// Use a shorter version of the Foreman clip for this test. // Use a shorter version of the Foreman clip for this test.
fclose(_sourceFile); fclose(source_file_);
const std::string video_file = const std::string video_file =
webrtc::test::ResourcePath("foreman_cif_short", "yuv"); webrtc::test::ResourcePath("foreman_cif_short", "yuv");
_sourceFile = fopen(video_file.c_str(), "rb"); source_file_ = fopen(video_file.c_str(), "rb");
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read source file: " + video_file + "\n"; "Cannot read source file: " + video_file + "\n";
std::string output_file = webrtc::test::OutputPath() + std::string output_file = webrtc::test::OutputPath() +
@ -39,27 +39,27 @@ TEST_F(VideoProcessingModuleTest, ColorEnhancement)
ASSERT_TRUE(modFile != NULL) << "Could not open output file.\n"; ASSERT_TRUE(modFile != NULL) << "Could not open output file.\n";
uint32_t frameNum = 0; uint32_t frameNum = 0;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
t0 = TickTime::Now(); t0 = TickTime::Now();
ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&_videoFrame)); ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&video_frame_));
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += t1 - t0; acc_ticks += t1 - t0;
if (PrintI420VideoFrame(_videoFrame, modFile) < 0) { if (PrintI420VideoFrame(video_frame_, modFile) < 0) {
return; return;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
printf("\nTime per frame: %d us \n", printf("\nTime per frame: %d us \n",
static_cast<int>(accTicks.Microseconds() / frameNum)); static_cast<int>(acc_ticks.Microseconds() / frameNum));
rewind(modFile); rewind(modFile);
printf("Comparing files...\n\n"); printf("Comparing files...\n\n");
@ -82,62 +82,62 @@ TEST_F(VideoProcessingModuleTest, ColorEnhancement)
ASSERT_EQ(refLen, testLen) << "File lengths differ."; ASSERT_EQ(refLen, testLen) << "File lengths differ.";
I420VideoFrame refVideoFrame; I420VideoFrame refVideoFrame;
refVideoFrame.CreateEmptyFrame(_width, _height, refVideoFrame.CreateEmptyFrame(width_, height_,
_width, _half_width, _half_width); width_, half_width_, half_width_);
// Compare frame-by-frame. // Compare frame-by-frame.
scoped_array<uint8_t> ref_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> ref_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, modFile) == while (fread(video_buffer.get(), 1, frame_length_, modFile) ==
_frame_length) frame_length_)
{ {
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
ASSERT_EQ(_frame_length, fread(ref_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(ref_buffer.get(), 1, frame_length_,
refFile)); refFile));
EXPECT_EQ(0, ConvertToI420(kI420, ref_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, ref_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &refVideoFrame)); 0, kRotateNone, &refVideoFrame));
EXPECT_EQ(0, memcmp(_videoFrame.buffer(kYPlane), EXPECT_EQ(0, memcmp(video_frame_.buffer(kYPlane),
refVideoFrame.buffer(kYPlane), refVideoFrame.buffer(kYPlane),
_size_y)); size_y_));
EXPECT_EQ(0, memcmp(_videoFrame.buffer(kUPlane), EXPECT_EQ(0, memcmp(video_frame_.buffer(kUPlane),
refVideoFrame.buffer(kUPlane), refVideoFrame.buffer(kUPlane),
_size_uv)); size_uv_));
EXPECT_EQ(0, memcmp(_videoFrame.buffer(kVPlane), EXPECT_EQ(0, memcmp(video_frame_.buffer(kVPlane),
refVideoFrame.buffer(kVPlane), refVideoFrame.buffer(kVPlane),
_size_uv)); size_uv_));
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Verify that all color pixels are enhanced, and no luminance values are // Verify that all color pixels are enhanced, and no luminance values are
// altered. // altered.
scoped_array<uint8_t> testFrame(new uint8_t[_frame_length]); scoped_array<uint8_t> testFrame(new uint8_t[frame_length_]);
// Use value 128 as probe value, since we know that this will be changed // Use value 128 as probe value, since we know that this will be changed
// in the enhancement. // in the enhancement.
memset(testFrame.get(), 128, _frame_length); memset(testFrame.get(), 128, frame_length_);
I420VideoFrame testVideoFrame; I420VideoFrame testVideoFrame;
testVideoFrame.CreateEmptyFrame(_width, _height, testVideoFrame.CreateEmptyFrame(width_, height_,
_width, _half_width, _half_width); width_, half_width_, half_width_);
EXPECT_EQ(0, ConvertToI420(kI420, testFrame.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, testFrame.get(), 0, 0,
_width, _height, 0, kRotateNone, width_, height_, 0, kRotateNone,
&testVideoFrame)); &testVideoFrame));
ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&testVideoFrame)); ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&testVideoFrame));
EXPECT_EQ(0, memcmp(testVideoFrame.buffer(kYPlane), testFrame.get(), EXPECT_EQ(0, memcmp(testVideoFrame.buffer(kYPlane), testFrame.get(),
_size_y)) size_y_))
<< "Function is modifying the luminance."; << "Function is modifying the luminance.";
EXPECT_NE(0, memcmp(testVideoFrame.buffer(kUPlane), EXPECT_NE(0, memcmp(testVideoFrame.buffer(kUPlane),
testFrame.get() + _size_y, _size_uv)) << testFrame.get() + size_y_, size_uv_)) <<
"Function is not modifying all chrominance pixels"; "Function is not modifying all chrominance pixels";
EXPECT_NE(0, memcmp(testVideoFrame.buffer(kVPlane), EXPECT_NE(0, memcmp(testVideoFrame.buffer(kVPlane),
testFrame.get() + _size_y + _size_uv, _size_uv)) << testFrame.get() + size_y_ + size_uv_, size_uv_)) <<
"Function is not modifying all chrominance pixels"; "Function is not modifying all chrominance pixels";
ASSERT_EQ(0, fclose(refFile)); ASSERT_EQ(0, fclose(refFile));

28
webrtc/modules/video_processing/main/test/unit_test/content_metrics_test.cc
View File

@ -15,32 +15,30 @@
namespace webrtc { namespace webrtc {
TEST_F(VideoProcessingModuleTest, ContentAnalysis) TEST_F(VideoProcessingModuleTest, ContentAnalysis) {
{ VPMContentAnalysis ca__c(false);
VPMContentAnalysis _ca_c(false); VPMContentAnalysis ca__sse(true);
VPMContentAnalysis _ca_sse(true);
VideoContentMetrics *_cM_c, *_cM_SSE; VideoContentMetrics *_cM_c, *_cM_SSE;
_ca_c.Initialize(_width,_height); ca__c.Initialize(width_,height_);
_ca_sse.Initialize(_width,_height); ca__sse.Initialize(width_,height_);
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) while (fread(video_buffer.get(), 1, frame_length_, source_file_)
== _frame_length) == frame_length_) {
{
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
_cM_c = _ca_c.ComputeContentMetrics(_videoFrame); _cM_c = ca__c.ComputeContentMetrics(video_frame_);
_cM_SSE = _ca_sse.ComputeContentMetrics(_videoFrame); _cM_SSE = ca__sse.ComputeContentMetrics(video_frame_);
ASSERT_EQ(_cM_c->spatial_pred_err, _cM_SSE->spatial_pred_err); ASSERT_EQ(_cM_c->spatial_pred_err, _cM_SSE->spatial_pred_err);
ASSERT_EQ(_cM_c->spatial_pred_err_v, _cM_SSE->spatial_pred_err_v); ASSERT_EQ(_cM_c->spatial_pred_err_v, _cM_SSE->spatial_pred_err_v);
ASSERT_EQ(_cM_c->spatial_pred_err_h, _cM_SSE->spatial_pred_err_h); ASSERT_EQ(_cM_c->spatial_pred_err_h, _cM_SSE->spatial_pred_err_h);
ASSERT_EQ(_cM_c->motion_magnitude, _cM_SSE->motion_magnitude); ASSERT_EQ(_cM_c->motion_magnitude, _cM_SSE->motion_magnitude);
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
} }
} // namespace webrtc } // namespace webrtc

56
webrtc/modules/video_processing/main/test/unit_test/deflickering_test.cc
View File

@ -23,17 +23,17 @@ TEST_F(VideoProcessingModuleTest, Deflickering)
{ {
enum { NumRuns = 30 }; enum { NumRuns = 30 };
uint32_t frameNum = 0; uint32_t frameNum = 0;
const uint32_t frameRate = 15; const uint32_t frame_rate = 15;
int64_t minRuntime = 0; int64_t min_runtime = 0;
int64_t avgRuntime = 0; int64_t avg_runtime = 0;
// Close automatically opened Foreman. // Close automatically opened Foreman.
fclose(_sourceFile); fclose(source_file_);
const std::string input_file = const std::string input_file =
webrtc::test::ResourcePath("deflicker_before_cif_short", "yuv"); webrtc::test::ResourcePath("deflicker_before_cif_short", "yuv");
_sourceFile = fopen(input_file.c_str(), "rb"); source_file_ = fopen(input_file.c_str(), "rb");
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read input file: " << input_file << "\n"; "Cannot read input file: " << input_file << "\n";
const std::string output_file = const std::string output_file =
@ -43,57 +43,57 @@ TEST_F(VideoProcessingModuleTest, Deflickering)
"Could not open output file: " << output_file << "\n"; "Could not open output file: " << output_file << "\n";
printf("\nRun time [us / frame]:\n"); printf("\nRun time [us / frame]:\n");
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
for (uint32_t runIdx = 0; runIdx < NumRuns; runIdx++) for (uint32_t run_idx = 0; run_idx < NumRuns; run_idx++)
{ {
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
uint32_t timeStamp = 1; uint32_t timeStamp = 1;
frameNum = 0; frameNum = 0;
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
frameNum++; frameNum++;
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
_videoFrame.set_timestamp(timeStamp); video_frame_.set_timestamp(timeStamp);
t0 = TickTime::Now(); t0 = TickTime::Now();
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_EQ(0, _vpm->Deflickering(&_videoFrame, &stats)); ASSERT_EQ(0, vpm_->Deflickering(&video_frame_, &stats));
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += (t1 - t0); acc_ticks += (t1 - t0);
if (runIdx == 0) if (run_idx == 0)
{ {
if (PrintI420VideoFrame(_videoFrame, deflickerFile) < 0) { if (PrintI420VideoFrame(video_frame_, deflickerFile) < 0) {
return; return;
} }
} }
timeStamp += (90000 / frameRate); timeStamp += (90000 / frame_rate);
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
printf("%u\n", static_cast<int>(accTicks.Microseconds() / frameNum)); printf("%u\n", static_cast<int>(acc_ticks.Microseconds() / frameNum));
if (accTicks.Microseconds() < minRuntime || runIdx == 0) if (acc_ticks.Microseconds() < min_runtime || run_idx == 0)
{ {
minRuntime = accTicks.Microseconds(); min_runtime = acc_ticks.Microseconds();
} }
avgRuntime += accTicks.Microseconds(); avg_runtime += acc_ticks.Microseconds();
rewind(_sourceFile); rewind(source_file_);
} }
ASSERT_EQ(0, fclose(deflickerFile)); ASSERT_EQ(0, fclose(deflickerFile));
// TODO(kjellander): Add verification of deflicker output file. // TODO(kjellander): Add verification of deflicker output file.
printf("\nAverage run time = %d us / frame\n", printf("\nAverage run time = %d us / frame\n",
static_cast<int>(avgRuntime / frameNum / NumRuns)); static_cast<int>(avg_runtime / frameNum / NumRuns));
printf("Min run time = %d us / frame\n\n", printf("Min run time = %d us / frame\n\n",
static_cast<int>(minRuntime / frameNum)); static_cast<int>(min_runtime / frameNum));
} }
} // namespace webrtc } // namespace webrtc

66
webrtc/modules/video_processing/main/test/unit_test/denoising_test.cc
View File

@ -25,8 +25,8 @@ TEST_F(VideoProcessingModuleTest, DISABLED_ON_ANDROID(Denoising))
enum { NumRuns = 10 }; enum { NumRuns = 10 };
uint32_t frameNum = 0; uint32_t frameNum = 0;
int64_t minRuntime = 0; int64_t min_runtime = 0;
int64_t avgRuntime = 0; int64_t avg_runtime = 0;
const std::string denoise_filename = const std::string denoise_filename =
webrtc::test::OutputPath() + "denoise_testfile.yuv"; webrtc::test::OutputPath() + "denoise_testfile.yuv";
@ -41,50 +41,50 @@ TEST_F(VideoProcessingModuleTest, DISABLED_ON_ANDROID(Denoising))
"Could not open noisy file: " << noise_filename << "\n"; "Could not open noisy file: " << noise_filename << "\n";
printf("\nRun time [us / frame]:\n"); printf("\nRun time [us / frame]:\n");
for (uint32_t runIdx = 0; runIdx < NumRuns; runIdx++) for (uint32_t run_idx = 0; run_idx < NumRuns; run_idx++)
{ {
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
int32_t modifiedPixels = 0; int32_t modifiedPixels = 0;
frameNum = 0; frameNum = 0;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
uint8_t* sourceBuffer = _videoFrame.buffer(kYPlane); uint8_t* sourceBuffer = video_frame_.buffer(kYPlane);
// Add noise to a part in video stream // Add noise to a part in video stream
// Random noise // Random noise
// TODO: investigate the effectiveness of this test. // TODO: investigate the effectiveness of this test.
for (int ir = 0; ir < _height; ir++) for (int ir = 0; ir < height_; ir++)
{ {
uint32_t ik = ir * _width; uint32_t ik = ir * width_;
for (int ic = 0; ic < _width; ic++) for (int ic = 0; ic < width_; ic++)
{ {
uint8_t r = rand() % 16; uint8_t r = rand() % 16;
r -= 8; r -= 8;
if (ir < _height / 4) if (ir < height_ / 4)
r = 0; r = 0;
if (ir >= 3 * _height / 4) if (ir >= 3 * height_ / 4)
r = 0; r = 0;
if (ic < _width / 4) if (ic < width_ / 4)
r = 0; r = 0;
if (ic >= 3 * _width / 4) if (ic >= 3 * width_ / 4)
r = 0; r = 0;
/*uint8_t pixelValue = 0; /*uint8_t pixelValue = 0;
if (ir >= _height / 2) if (ir >= height_ / 2)
{ // Region 3 or 4 { // Region 3 or 4
pixelValue = 170; pixelValue = 170;
} }
if (ic >= _width / 2) if (ic >= width_ / 2)
{ // Region 2 or 4 { // Region 2 or 4
pixelValue += 85; pixelValue += 85;
} }
@ -95,42 +95,42 @@ TEST_F(VideoProcessingModuleTest, DISABLED_ON_ANDROID(Denoising))
} }
} }
if (runIdx == 0) if (run_idx == 0)
{ {
if (PrintI420VideoFrame(_videoFrame, noiseFile) < 0) { if (PrintI420VideoFrame(video_frame_, noiseFile) < 0) {
return; return;
} }
} }
t0 = TickTime::Now(); t0 = TickTime::Now();
ASSERT_GE(modifiedPixels = _vpm->Denoising(&_videoFrame), 0); ASSERT_GE(modifiedPixels = vpm_->Denoising(&video_frame_), 0);
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += (t1 - t0); acc_ticks += (t1 - t0);
if (runIdx == 0) if (run_idx == 0)
{ {
if (PrintI420VideoFrame(_videoFrame, noiseFile) < 0) { if (PrintI420VideoFrame(video_frame_, noiseFile) < 0) {
return; return;
} }
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
printf("%u\n", static_cast<int>(accTicks.Microseconds() / frameNum)); printf("%u\n", static_cast<int>(acc_ticks.Microseconds() / frameNum));
if (accTicks.Microseconds() < minRuntime || runIdx == 0) if (acc_ticks.Microseconds() < min_runtime || run_idx == 0)
{ {
minRuntime = accTicks.Microseconds(); min_runtime = acc_ticks.Microseconds();
} }
avgRuntime += accTicks.Microseconds(); avg_runtime += acc_ticks.Microseconds();
rewind(_sourceFile); rewind(source_file_);
} }
ASSERT_EQ(0, fclose(denoiseFile)); ASSERT_EQ(0, fclose(denoiseFile));
ASSERT_EQ(0, fclose(noiseFile)); ASSERT_EQ(0, fclose(noiseFile));
printf("\nAverage run time = %d us / frame\n", printf("\nAverage run time = %d us / frame\n",
static_cast<int>(avgRuntime / frameNum / NumRuns)); static_cast<int>(avg_runtime / frameNum / NumRuns));
printf("Min run time = %d us / frame\n\n", printf("Min run time = %d us / frame\n\n",
static_cast<int>(minRuntime / frameNum)); static_cast<int>(min_runtime / frameNum));
} }
} // namespace webrtc } // namespace webrtc

358
webrtc/modules/video_processing/main/test/unit_test/video_processing_unittest.cc
View File

@ -18,297 +18,285 @@
namespace webrtc { namespace webrtc {
// The |sourceFrame| is scaled to |target_width|,|target_height|, using the // The |sourceFrame| is scaled to |targetwidth_|,|targetheight_|, using the
// filter mode set to |mode|. The |expected_psnr| is used to verify basic // filter mode set to |mode|. The |expected_psnr| is used to verify basic
// quality when the resampled frame is scaled back up/down to the // quality when the resampled frame is scaled back up/down to the
// original/source size. |expected_psnr| is set to be ~0.1/0.05dB lower than // original/source size. |expected_psnr| is set to be ~0.1/0.05dB lower than
// actual PSNR verified under the same conditions. // actual PSNR verified under the same conditions.
void TestSize(const I420VideoFrame& sourceFrame, int target_width, void TestSize(const I420VideoFrame& sourceFrame, int targetwidth_,
int target_height, int mode, double expected_psnr, int targetheight_, int mode, double expected_psnr,
VideoProcessingModule* vpm); VideoProcessingModule* vpm);
bool CompareFrames(const webrtc::I420VideoFrame& frame1, bool CompareFrames(const webrtc::I420VideoFrame& frame1,
const webrtc::I420VideoFrame& frame2); const webrtc::I420VideoFrame& frame2);
VideoProcessingModuleTest::VideoProcessingModuleTest() : VideoProcessingModuleTest::VideoProcessingModuleTest()
_vpm(NULL), : vpm_(NULL),
_sourceFile(NULL), source_file_(NULL),
_width(352), width_(352),
_half_width((_width + 1) / 2), half_width_((width_ + 1) / 2),
_height(288), height_(288),
_size_y(_width * _height), size_y_(width_ * height_),
_size_uv(_half_width * ((_height + 1) / 2)), size_uv_(half_width_ * ((height_ + 1) / 2)),
_frame_length(CalcBufferSize(kI420, _width, _height)) frame_length_(CalcBufferSize(kI420, width_, height_)) {}
{
}
void VideoProcessingModuleTest::SetUp() void VideoProcessingModuleTest::SetUp() {
{ vpm_ = VideoProcessingModule::Create(0);
_vpm = VideoProcessingModule::Create(0); ASSERT_TRUE(vpm_ != NULL);
ASSERT_TRUE(_vpm != NULL);
ASSERT_EQ(0, _videoFrame.CreateEmptyFrame(_width, _height, _width, ASSERT_EQ(0, video_frame_.CreateEmptyFrame(width_, height_, width_,
_half_width, _half_width)); half_width_, half_width_));
const std::string video_file = const std::string video_file =
webrtc::test::ResourcePath("foreman_cif", "yuv"); webrtc::test::ResourcePath("foreman_cif", "yuv");
_sourceFile = fopen(video_file.c_str(),"rb"); source_file_ = fopen(video_file.c_str(),"rb");
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read source file: " + video_file + "\n"; "Cannot read source file: " + video_file + "\n";
} }
void VideoProcessingModuleTest::TearDown() void VideoProcessingModuleTest::TearDown() {
{ if (source_file_ != NULL) {
if (_sourceFile != NULL) { ASSERT_EQ(0, fclose(source_file_));
ASSERT_EQ(0, fclose(_sourceFile));
} }
_sourceFile = NULL; source_file_ = NULL;
if (_vpm != NULL) { if (vpm_ != NULL) {
VideoProcessingModule::Destroy(_vpm); VideoProcessingModule::Destroy(vpm_);
} }
_vpm = NULL; vpm_ = NULL;
} }
TEST_F(VideoProcessingModuleTest, HandleNullBuffer) TEST_F(VideoProcessingModuleTest, HandleNullBuffer) {
{
// TODO(mikhal/stefan): Do we need this one? // TODO(mikhal/stefan): Do we need this one?
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
// Video frame with unallocated buffer. // Video frame with unallocated buffer.
I420VideoFrame videoFrame; I420VideoFrame videoFrame;
videoFrame.set_width(_width); videoFrame.set_width(width_);
videoFrame.set_height(_height); videoFrame.set_height(height_);
EXPECT_EQ(-3, _vpm->GetFrameStats(&stats, videoFrame)); EXPECT_EQ(-3, vpm_->GetFrameStats(&stats, videoFrame));
EXPECT_EQ(-1, _vpm->ColorEnhancement(&videoFrame)); EXPECT_EQ(-1, vpm_->ColorEnhancement(&videoFrame));
EXPECT_EQ(-1, _vpm->Deflickering(&videoFrame, &stats)); EXPECT_EQ(-1, vpm_->Deflickering(&videoFrame, &stats));
EXPECT_EQ(-1, _vpm->Denoising(&videoFrame)); EXPECT_EQ(-1, vpm_->Denoising(&videoFrame));
EXPECT_EQ(-3, _vpm->BrightnessDetection(videoFrame, stats)); EXPECT_EQ(-3, vpm_->BrightnessDetection(videoFrame, stats));
} }
TEST_F(VideoProcessingModuleTest, HandleBadStats) TEST_F(VideoProcessingModuleTest, HandleBadStats) {
{
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
EXPECT_EQ(-1, _vpm->Deflickering(&_videoFrame, &stats)); EXPECT_EQ(-1, vpm_->Deflickering(&video_frame_, &stats));
EXPECT_EQ(-3, _vpm->BrightnessDetection(_videoFrame, stats)); EXPECT_EQ(-3, vpm_->BrightnessDetection(video_frame_, stats));
} }
TEST_F(VideoProcessingModuleTest, HandleBadSize) TEST_F(VideoProcessingModuleTest, HandleBadSize) {
{
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
_videoFrame.ResetSize(); video_frame_.ResetSize();
_videoFrame.set_width(_width); video_frame_.set_width(width_);
_videoFrame.set_height(0); video_frame_.set_height(0);
EXPECT_EQ(-3, _vpm->GetFrameStats(&stats, _videoFrame)); EXPECT_EQ(-3, vpm_->GetFrameStats(&stats, video_frame_));
EXPECT_EQ(-1, _vpm->ColorEnhancement(&_videoFrame)); EXPECT_EQ(-1, vpm_->ColorEnhancement(&video_frame_));
EXPECT_EQ(-1, _vpm->Deflickering(&_videoFrame, &stats)); EXPECT_EQ(-1, vpm_->Deflickering(&video_frame_, &stats));
EXPECT_EQ(-1, _vpm->Denoising(&_videoFrame)); EXPECT_EQ(-1, vpm_->Denoising(&video_frame_));
EXPECT_EQ(-3, _vpm->BrightnessDetection(_videoFrame, stats)); EXPECT_EQ(-3, vpm_->BrightnessDetection(video_frame_, stats));
EXPECT_EQ(VPM_PARAMETER_ERROR, _vpm->SetTargetResolution(0,0,0)); EXPECT_EQ(VPM_PARAMETER_ERROR, vpm_->SetTargetResolution(0,0,0));
EXPECT_EQ(VPM_PARAMETER_ERROR, _vpm->SetMaxFrameRate(0)); EXPECT_EQ(VPM_PARAMETER_ERROR, vpm_->SetMaxFramerate(0));
I420VideoFrame *outFrame = NULL; I420VideoFrame *out_frame = NULL;
EXPECT_EQ(VPM_PARAMETER_ERROR, _vpm->PreprocessFrame(_videoFrame, EXPECT_EQ(VPM_PARAMETER_ERROR, vpm_->PreprocessFrame(video_frame_,
&outFrame)); &out_frame));
} }
TEST_F(VideoProcessingModuleTest, IdenticalResultsAfterReset) TEST_F(VideoProcessingModuleTest, IdenticalResultsAfterReset) {
{ I420VideoFrame video_frame2;
I420VideoFrame videoFrame2;
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
// Only testing non-static functions here. // Only testing non-static functions here.
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_EQ(0, videoFrame2.CopyFrame(_videoFrame)); ASSERT_EQ(0, video_frame2.CopyFrame(video_frame_));
ASSERT_EQ(0, _vpm->Deflickering(&_videoFrame, &stats)); ASSERT_EQ(0, vpm_->Deflickering(&video_frame_, &stats));
_vpm->Reset(); vpm_->Reset();
// Retrieve frame stats again in case Deflickering() has zeroed them. // Retrieve frame stats again in case Deflickering() has zeroed them.
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, videoFrame2)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame2));
ASSERT_EQ(0, _vpm->Deflickering(&videoFrame2, &stats)); ASSERT_EQ(0, vpm_->Deflickering(&video_frame2, &stats));
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
videoFrame2.CopyFrame(_videoFrame); video_frame2.CopyFrame(video_frame_);
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
ASSERT_GE(_vpm->Denoising(&_videoFrame), 0); ASSERT_GE(vpm_->Denoising(&video_frame_), 0);
_vpm->Reset(); vpm_->Reset();
ASSERT_GE(_vpm->Denoising(&videoFrame2), 0); ASSERT_GE(vpm_->Denoising(&video_frame2), 0);
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
videoFrame2.CopyFrame(_videoFrame); video_frame2.CopyFrame(video_frame_);
ASSERT_EQ(0, _vpm->BrightnessDetection(_videoFrame, stats)); ASSERT_EQ(0, vpm_->BrightnessDetection(video_frame_, stats));
_vpm->Reset(); vpm_->Reset();
ASSERT_EQ(0, _vpm->BrightnessDetection(videoFrame2, stats)); ASSERT_EQ(0, vpm_->BrightnessDetection(video_frame2, stats));
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
} }
TEST_F(VideoProcessingModuleTest, FrameStats) TEST_F(VideoProcessingModuleTest, FrameStats) {
{
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
EXPECT_FALSE(_vpm->ValidFrameStats(stats)); EXPECT_FALSE(vpm_->ValidFrameStats(stats));
EXPECT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); EXPECT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
EXPECT_TRUE(_vpm->ValidFrameStats(stats)); EXPECT_TRUE(vpm_->ValidFrameStats(stats));
printf("\nFrameStats\n"); printf("\nFrameStats\n");
printf("mean: %u\nnumPixels: %u\nsubSamplWidth: " printf("mean: %u\nnum_pixels: %u\nsubSamplWidth: "
"%u\nsumSamplHeight: %u\nsum: %u\n\n", "%u\nsumSamplHeight: %u\nsum: %u\n\n",
static_cast<unsigned int>(stats.mean), static_cast<unsigned int>(stats.mean),
static_cast<unsigned int>(stats.numPixels), static_cast<unsigned int>(stats.num_pixels),
static_cast<unsigned int>(stats.subSamplHeight), static_cast<unsigned int>(stats.subSamplHeight),
static_cast<unsigned int>(stats.subSamplWidth), static_cast<unsigned int>(stats.subSamplWidth),
static_cast<unsigned int>(stats.sum)); static_cast<unsigned int>(stats.sum));
_vpm->ClearFrameStats(&stats); vpm_->ClearFrameStats(&stats);
EXPECT_FALSE(_vpm->ValidFrameStats(stats)); EXPECT_FALSE(vpm_->ValidFrameStats(stats));
} }
TEST_F(VideoProcessingModuleTest, PreprocessorLogic) TEST_F(VideoProcessingModuleTest, PreprocessorLogic) {
{
// Disable temporal sampling (frame dropping). // Disable temporal sampling (frame dropping).
_vpm->EnableTemporalDecimation(false); vpm_->EnableTemporalDecimation(false);
int resolution = 100; int resolution = 100;
EXPECT_EQ(VPM_OK, _vpm->SetMaxFrameRate(30)); EXPECT_EQ(VPM_OK, vpm_->SetMaxFramerate(30));
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 15)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 15));
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 30)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 30));
// Disable spatial sampling. // Disable spatial sampling.
_vpm->SetInputFrameResampleMode(kNoRescaling); vpm_->SetInputFrameResampleMode(kNoRescaling);
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 30)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 30));
I420VideoFrame* outFrame = NULL; I420VideoFrame* out_frame = NULL;
// Set rescaling => output frame != NULL. // Set rescaling => output frame != NULL.
_vpm->SetInputFrameResampleMode(kFastRescaling); vpm_->SetInputFrameResampleMode(kFastRescaling);
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 30)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 30));
EXPECT_EQ(VPM_OK, _vpm->PreprocessFrame(_videoFrame, &outFrame)); EXPECT_EQ(VPM_OK, vpm_->PreprocessFrame(video_frame_, &out_frame));
EXPECT_FALSE(outFrame == NULL); EXPECT_FALSE(out_frame == NULL);
if (outFrame) { if (out_frame) {
EXPECT_EQ(resolution, outFrame->width()); EXPECT_EQ(resolution, out_frame->width());
EXPECT_EQ(resolution, outFrame->height()); EXPECT_EQ(resolution, out_frame->height());
} }
// No rescaling=> output frame = NULL. // No rescaling=> output frame = NULL.
_vpm->SetInputFrameResampleMode(kNoRescaling); vpm_->SetInputFrameResampleMode(kNoRescaling);
EXPECT_EQ(VPM_OK, _vpm->PreprocessFrame(_videoFrame, &outFrame)); EXPECT_EQ(VPM_OK, vpm_->PreprocessFrame(video_frame_, &out_frame));
EXPECT_TRUE(outFrame == NULL); EXPECT_TRUE(out_frame == NULL);
} }
TEST_F(VideoProcessingModuleTest, Resampler) TEST_F(VideoProcessingModuleTest, Resampler) {
{
enum { NumRuns = 1 }; enum { NumRuns = 1 };
int64_t minRuntime = 0; int64_t min_runtime = 0;
int64_t avgRuntime = 0; int64_t avg_runtime = 0;
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
rewind(_sourceFile); rewind(source_file_);
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read input file \n"; "Cannot read input file \n";
// CA not needed here // CA not needed here
_vpm->EnableContentAnalysis(false); vpm_->EnableContentAnalysis(false);
// no temporal decimation // no temporal decimation
_vpm->EnableTemporalDecimation(false); vpm_->EnableTemporalDecimation(false);
// Reading test frame // Reading test frame
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
for (uint32_t runIdx = 0; runIdx < NumRuns; runIdx++) for (uint32_t run_idx = 0; run_idx < NumRuns; run_idx++) {
{ // Initiate test timer.
// initiate test timer
t0 = TickTime::Now(); t0 = TickTime::Now();
// Init the sourceFrame with a timestamp. // Init the sourceFrame with a timestamp.
_videoFrame.set_render_time_ms(t0.MillisecondTimestamp()); video_frame_.set_render_time_ms(t0.MillisecondTimestamp());
_videoFrame.set_timestamp(t0.MillisecondTimestamp() * 90); video_frame_.set_timestamp(t0.MillisecondTimestamp() * 90);
// Test scaling to different sizes: source is of |width|/|height| = 352/288. // Test scaling to different sizes: source is of |width|/|height| = 352/288.
// Scaling mode in VPM is currently fixed to kScaleBox (mode = 3). // Scaling mode in VPM is currently fixed to kScaleBox (mode = 3).
TestSize(_videoFrame, 100, 50, 3, 24.0, _vpm); TestSize(video_frame_, 100, 50, 3, 24.0, vpm_);
TestSize(_videoFrame, 352/4, 288/4, 3, 25.2, _vpm); TestSize(video_frame_, 352/4, 288/4, 3, 25.2, vpm_);
TestSize(_videoFrame, 352/2, 288/2, 3, 28.1, _vpm); TestSize(video_frame_, 352/2, 288/2, 3, 28.1, vpm_);
TestSize(_videoFrame, 352, 288, 3, -1, _vpm); // no resampling. TestSize(video_frame_, 352, 288, 3, -1, vpm_); // no resampling.
TestSize(_videoFrame, 2*352, 2*288, 3, 32.2, _vpm); TestSize(video_frame_, 2*352, 2*288, 3, 32.2, vpm_);
TestSize(_videoFrame, 400, 256, 3, 31.3, _vpm); TestSize(video_frame_, 400, 256, 3, 31.3, vpm_);
TestSize(_videoFrame, 480, 640, 3, 32.15, _vpm); TestSize(video_frame_, 480, 640, 3, 32.15, vpm_);
TestSize(_videoFrame, 960, 720, 3, 32.2, _vpm); TestSize(video_frame_, 960, 720, 3, 32.2, vpm_);
TestSize(_videoFrame, 1280, 720, 3, 32.15, _vpm); TestSize(video_frame_, 1280, 720, 3, 32.15, vpm_);
// Upsampling to odd size. // Upsampling to odd size.
TestSize(_videoFrame, 501, 333, 3, 32.05, _vpm); TestSize(video_frame_, 501, 333, 3, 32.05, vpm_);
// Downsample to odd size. // Downsample to odd size.
TestSize(_videoFrame, 281, 175, 3, 29.3, _vpm); TestSize(video_frame_, 281, 175, 3, 29.3, vpm_);
// stop timer // stop timer
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += (t1 - t0); acc_ticks += (t1 - t0);
if (accTicks.Microseconds() < minRuntime || runIdx == 0) { if (acc_ticks.Microseconds() < min_runtime || run_idx == 0) {
minRuntime = accTicks.Microseconds(); min_runtime = acc_ticks.Microseconds();
} }
avgRuntime += accTicks.Microseconds(); avg_runtime += acc_ticks.Microseconds();
} }
printf("\nAverage run time = %d us / frame\n", printf("\nAverage run time = %d us / frame\n",
//static_cast<int>(avgRuntime / frameNum / NumRuns)); //static_cast<int>(avg_runtime / frameNum / NumRuns));
static_cast<int>(avgRuntime)); static_cast<int>(avg_runtime));
printf("Min run time = %d us / frame\n\n", printf("Min run time = %d us / frame\n\n",
//static_cast<int>(minRuntime / frameNum)); //static_cast<int>(min_runtime / frameNum));
static_cast<int>(minRuntime)); static_cast<int>(min_runtime));
} }
void TestSize(const I420VideoFrame& source_frame, int target_width, void TestSize(const I420VideoFrame& source_frame, int targetwidth_,
int target_height, int mode, double expected_psnr, int targetheight_, int mode, double expected_psnr,
VideoProcessingModule* vpm) { VideoProcessingModule* vpm) {
int source_width = source_frame.width(); int sourcewidth_ = source_frame.width();
int source_height = source_frame.height(); int sourceheight_ = source_frame.height();
I420VideoFrame* out_frame = NULL; I420VideoFrame* out_frame = NULL;
ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(target_width, target_height, 30)); ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(targetwidth_, targetheight_, 30));
ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(source_frame, &out_frame)); ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(source_frame, &out_frame));
if (out_frame) { if (out_frame) {
@ -321,19 +309,19 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
// (2) scale the resampled frame (|out_frame|) back to the original size and // (2) scale the resampled frame (|out_frame|) back to the original size and
// compute PSNR relative to |source_frame| (for automatic verification). // compute PSNR relative to |source_frame| (for automatic verification).
// (3) write out the processed frame for viewing. // (3) write out the processed frame for viewing.
if (target_width != static_cast<int>(source_width) || if (targetwidth_ != static_cast<int>(sourcewidth_) ||
target_height != static_cast<int>(source_height)) { targetheight_ != static_cast<int>(sourceheight_)) {
// Write the processed frame to file for visual inspection. // Write the processed frame to file for visual inspection.
std::ostringstream filename; std::ostringstream filename;
filename << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" << filename << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" <<
"from_" << source_width << "x" << source_height << "_to_" << "from_" << sourcewidth_ << "x" << sourceheight_ << "_to_" <<
target_width << "x" << target_height << "_30Hz_P420.yuv"; targetwidth_ << "x" << targetheight_ << "_30Hz_P420.yuv";
std::cout << "Watch " << filename.str() << " and verify that it is okay." std::cout << "Watch " << filename.str() << " and verify that it is okay."
<< std::endl; << std::endl;
FILE* stand_alone_file = fopen(filename.str().c_str(), "wb"); FILE* stand_alone_file = fopen(filename.str().c_str(), "wb");
if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) { if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) {
fprintf(stderr, "Failed to write frame for scaling to width/height: " fprintf(stderr, "Failed to write frame for scaling to width/height: "
" %d %d \n", target_width, target_height); " %d %d \n", targetwidth_, targetheight_);
return; return;
} }
fclose(stand_alone_file); fclose(stand_alone_file);
@ -342,8 +330,8 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
resampled_source_frame.CopyFrame(*out_frame); resampled_source_frame.CopyFrame(*out_frame);
// Scale |resampled_source_frame| back to original/source size. // Scale |resampled_source_frame| back to original/source size.
ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(source_width, ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(sourcewidth_,
source_height, sourceheight_,
30)); 30));
ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(resampled_source_frame, ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(resampled_source_frame,
&out_frame)); &out_frame));
@ -351,14 +339,14 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
// Write the processed frame to file for visual inspection. // Write the processed frame to file for visual inspection.
std::ostringstream filename2; std::ostringstream filename2;
filename2 << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" << filename2 << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" <<
"from_" << target_width << "x" << target_height << "_to_" << "from_" << targetwidth_ << "x" << targetheight_ << "_to_" <<
source_width << "x" << source_height << "_30Hz_P420.yuv"; sourcewidth_ << "x" << sourceheight_ << "_30Hz_P420.yuv";
std::cout << "Watch " << filename2.str() << " and verify that it is okay." std::cout << "Watch " << filename2.str() << " and verify that it is okay."
<< std::endl; << std::endl;
stand_alone_file = fopen(filename2.str().c_str(), "wb"); stand_alone_file = fopen(filename2.str().c_str(), "wb");
if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) { if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) {
fprintf(stderr, "Failed to write frame for scaling to width/height " fprintf(stderr, "Failed to write frame for scaling to width/height "
"%d %d \n", source_width, source_height); "%d %d \n", sourcewidth_, sourceheight_);
return; return;
} }
fclose(stand_alone_file); fclose(stand_alone_file);
@ -368,7 +356,7 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
EXPECT_GT(psnr, expected_psnr); EXPECT_GT(psnr, expected_psnr);
printf("PSNR: %f. PSNR is between source of size %d %d, and a modified " printf("PSNR: %f. PSNR is between source of size %d %d, and a modified "
"source which is scaled down/up to: %d %d, and back to source size \n", "source which is scaled down/up to: %d %d, and back to source size \n",
psnr, source_width, source_height, target_width, target_height); psnr, sourcewidth_, sourceheight_, targetwidth_, targetheight_);
} }
} }

29
webrtc/modules/video_processing/main/test/unit_test/video_processing_unittest.h
View File

@ -18,31 +18,28 @@
namespace webrtc { namespace webrtc {
class VideoProcessingModuleTest : public ::testing::Test class VideoProcessingModuleTest : public ::testing::Test {
{ protected:
protected:
VideoProcessingModuleTest(); VideoProcessingModuleTest();
virtual void SetUp(); virtual void SetUp();
virtual void TearDown(); virtual void TearDown();
static void SetUpTestCase() static void SetUpTestCase() {
{
Trace::CreateTrace(); Trace::CreateTrace();
std::string trace_file = webrtc::test::OutputPath() + "VPMTrace.txt"; std::string trace_file = webrtc::test::OutputPath() + "VPMTrace.txt";
ASSERT_EQ(0, Trace::SetTraceFile(trace_file.c_str())); ASSERT_EQ(0, Trace::SetTraceFile(trace_file.c_str()));
} }
static void TearDownTestCase() static void TearDownTestCase() {
{
Trace::ReturnTrace(); Trace::ReturnTrace();
} }
VideoProcessingModule* _vpm; VideoProcessingModule* vpm_;
FILE* _sourceFile; FILE* source_file_;
I420VideoFrame _videoFrame; I420VideoFrame video_frame_;
const int _width; const int width_;
const int _half_width; const int half_width_;
const int _height; const int height_;
const int _size_y; const int size_y_;
const int _size_uv; const int size_uv_;
const unsigned int _frame_length; const unsigned int frame_length_;
}; };
} // namespace webrtc } // namespace webrtc