generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reformatting VPM: First step - No functional changes.

Browse Source 
R=marpan@google.com, marpan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/2333004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4912 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
mikhal@webrtc.org 2013-10-03 16:42:41 +00:00
parent 26f78f7ecb
commit b43d8078a1
31 changed files with 2335 additions and 2809 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webrtc/modules/video_coding/main/test/quality_modes_test.cc
View File

@ -249,7 +249,7 @@ QualityModesTest::Perform(const CmdArgs& args)
VideoContentMetrics* contentMetrics = NULL; VideoContentMetrics* contentMetrics = NULL;
// setting user frame rate // setting user frame rate
_vpm->SetMaxFrameRate((uint32_t)(_nativeFrameRate+ 0.5f)); _vpm->SetMaxFramerate((uint32_t)(_nativeFrameRate+ 0.5f));
// for starters: keeping native values: // for starters: keeping native values:
_vpm->SetTargetResolution(_width, _height, _vpm->SetTargetResolution(_width, _height,
(uint32_t)(_frameRate+ 0.5f)); (uint32_t)(_frameRate+ 0.5f));

523
webrtc/modules/video_processing/main/interface/video_processing.h
View File

@ -24,281 +24,274 @@
#include "webrtc/modules/video_processing/main/interface/video_processing_defines.h" #include "webrtc/modules/video_processing/main/interface/video_processing_defines.h"
/** /**
The module is largely intended to process video streams, except functionality The module is largely intended to process video streams, except functionality
provided by static functions which operate independent of previous frames. It provided by static functions which operate independent of previous frames. It
is recommended, but not required that a unique instance be used for each is recommended, but not required that a unique instance be used for each
concurrently processed stream. Similarly, it is recommended to call Reset() concurrently processed stream. Similarly, it is recommended to call Reset()
before switching to a new stream, but this is not absolutely required. before switching to a new stream, but this is not absolutely required.
The module provides basic thread safety by permitting only a single function The module provides basic thread safety by permitting only a single function
to execute concurrently. to execute concurrently.
*/ */
namespace webrtc { namespace webrtc {
class VideoProcessingModule : public Module class VideoProcessingModule : public Module {
{ public:
public: /**
/** Structure to hold frame statistics. Populate it with GetFrameStats().
Structure to hold frame statistics. Populate it with GetFrameStats(). */
*/ struct FrameStats {
struct FrameStats FrameStats() :
{ mean(0),
FrameStats() : sum(0),
mean(0), num_pixels(0),
sum(0), subSamplWidth(0),
numPixels(0), subSamplHeight(0) {
subSamplWidth(0), memset(hist, 0, sizeof(hist));
subSamplHeight(0) }
{
memset(hist, 0, sizeof(hist));
}
uint32_t hist[256]; /**< Histogram of frame */
uint32_t mean; /**< Mean value of frame */
uint32_t sum; /**< Sum of frame */
uint32_t numPixels; /**< Number of pixels */
uint8_t subSamplWidth; /**< Subsampling rate of width in powers
of 2 */
uint8_t subSamplHeight; /**< Subsampling rate of height in powers
of 2 */
};
/**
Specifies the warning types returned by BrightnessDetection().
*/
enum BrightnessWarning
{
kNoWarning, /**< Frame has acceptable brightness */
kDarkWarning, /**< Frame is too dark */
kBrightWarning /**< Frame is too bright */
};
/*
Creates a VPM object.
\param[in] id
Unique identifier of this object.
\return Pointer to a VPM object.
*/
static VideoProcessingModule* Create(int32_t id);
/**
Destroys a VPM object.
\param[in] module
Pointer to the VPM object to destroy.
*/
static void Destroy(VideoProcessingModule* module);
/**
Not supported.
*/
virtual int32_t TimeUntilNextProcess() { return -1; }
/**
Not supported.
*/
virtual int32_t Process() { return -1; }
/**
Resets all processing components to their initial states. This should be
called whenever a new video stream is started.
*/
virtual void Reset() = 0;
/**
Retrieves statistics for the input frame. This function must be used to
prepare a FrameStats struct for use in certain VPM functions.
\param[out] stats
The frame statistics will be stored here on return.
\param[in] frame
Reference to the video frame.
\return 0 on success, -1 on failure.
*/
static int32_t GetFrameStats(FrameStats* stats,
const I420VideoFrame& frame);
/**
Checks the validity of a FrameStats struct. Currently, valid implies only
that is had changed from its initialized state.
\param[in] stats
Frame statistics.
\return True on valid stats, false on invalid stats.
*/
static bool ValidFrameStats(const FrameStats& stats);
/**
Returns a FrameStats struct to its intialized state.
\param[in,out] stats
Frame statistics.
*/
static void ClearFrameStats(FrameStats* stats);
/**
Enhances the color of an image through a constant mapping. Only the
chrominance is altered. Has a fixed-point implementation.
\param[in,out] frame
Pointer to the video frame.
*/
static int32_t ColorEnhancement(I420VideoFrame* frame);
/**
Increases/decreases the luminance value.
\param[in,out] frame
Pointer to the video frame.
\param[in] delta
The amount to change the chrominance value of every single pixel.
Can be < 0 also.
\return 0 on success, -1 on failure.
*/
static int32_t Brighten(I420VideoFrame* frame, int delta);
/**
Detects and removes camera flicker from a video stream. Every frame from
the stream must be passed in. A frame will only be altered if flicker has
been detected. Has a fixed-point implementation.
\param[in,out] frame
Pointer to the video frame.
\param[in,out] stats
Frame statistics provided by GetFrameStats(). On return the stats will
be reset to zero if the frame was altered. Call GetFrameStats() again
if the statistics for the altered frame are required.
\return 0 on success, -1 on failure.
*/
virtual int32_t Deflickering(I420VideoFrame* frame, FrameStats* stats) = 0;
/**
Denoises a video frame. Every frame from the stream should be passed in.
Has a fixed-point implementation.
\param[in,out] frame
Pointer to the video frame.
\return The number of modified pixels on success, -1 on failure.
*/
virtual int32_t Denoising(I420VideoFrame* frame) = 0;
/**
Detects if a video frame is excessively bright or dark. Returns a
warning if this is the case. Multiple frames should be passed in before
expecting a warning. Has a floating-point implementation.
\param[in] frame
Pointer to the video frame.
\param[in] stats
Frame statistics provided by GetFrameStats().
\return A member of BrightnessWarning on success, -1 on error
*/
virtual int32_t BrightnessDetection(const I420VideoFrame& frame,
const FrameStats& stats) = 0;
/**
The following functions refer to the pre-processor unit within VPM. The
pre-processor perfoms spatial/temporal decimation and content analysis on
the frames prior to encoding.
*/
/**
Enable/disable temporal decimation
\param[in] enable when true, temporal decimation is enabled
*/
virtual void EnableTemporalDecimation(bool enable) = 0;
/**
Set target resolution
\param[in] width
Target width
\param[in] height
Target height
\param[in] frameRate
Target frameRate
\return VPM_OK on success, a negative value on error (see error codes)
*/
virtual int32_t SetTargetResolution(uint32_t width,
uint32_t height,
uint32_t frameRate) = 0;
/**
Set max frame rate
\param[in] maxFrameRate: maximum frame rate (limited to native frame rate)
\return VPM_OK on success, a negative value on error (see error codes)
*/
virtual int32_t SetMaxFrameRate(uint32_t maxFrameRate) = 0;
/**
Get decimated(target) frame rate
*/
virtual uint32_t DecimatedFrameRate() = 0;
/**
Get decimated(target) frame width
*/
virtual uint32_t DecimatedWidth() const = 0;
/**
Get decimated(target) frame height
*/
virtual uint32_t DecimatedHeight() const = 0 ;
/**
Set the spatial resampling settings of the VPM: The resampler may either be
disabled or one of the following:
scaling to a close to target dimension followed by crop/pad
\param[in] resamplingMode
Set resampling mode (a member of VideoFrameResampling)
*/
virtual void SetInputFrameResampleMode(VideoFrameResampling
resamplingMode) = 0;
/**
Get Processed (decimated) frame
\param[in] frame pointer to the video frame.
\param[in] processedFrame pointer (double) to the processed frame. If no
processing is required, processedFrame will be NULL.
\return VPM_OK on success, a negative value on error (see error codes)
*/
virtual int32_t PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processedFrame) = 0;
/**
Return content metrics for the last processed frame
*/
virtual VideoContentMetrics* ContentMetrics() const = 0 ;
/**
Enable content analysis
*/
virtual void EnableContentAnalysis(bool enable) = 0;
uint32_t hist[256]; // FRame histogram.
uint32_t mean; // Frame Mean value.
uint32_t sum; // Sum of frame.
uint32_t num_pixels; // Number of pixels.
uint8_t subSamplWidth; // Subsampling rate of width in powers of 2.
uint8_t subSamplHeight; // Subsampling rate of height in powers of 2.
}; };
} // namespace /**
Specifies the warning types returned by BrightnessDetection().
*/
enum BrightnessWarning {
kNoWarning, // Frame has acceptable brightness.
kDarkWarning, // Frame is too dark.
kBrightWarning // Frame is too bright.
};
#endif /*
Creates a VPM object.
\param[in] id
Unique identifier of this object.
\return Pointer to a VPM object.
*/
static VideoProcessingModule* Create(int32_t id);
/**
Destroys a VPM object.
\param[in] module
Pointer to the VPM object to destroy.
*/
static void Destroy(VideoProcessingModule* module);
/**
Not supported.
*/
virtual int32_t TimeUntilNextProcess() { return -1; }
/**
Not supported.
*/
virtual int32_t Process() { return -1; }
/**
Resets all processing components to their initial states. This should be
called whenever a new video stream is started.
*/
virtual void Reset() = 0;
/**
Retrieves statistics for the input frame. This function must be used to
prepare a FrameStats struct for use in certain VPM functions.
\param[out] stats
The frame statistics will be stored here on return.
\param[in] frame
Reference to the video frame.
\return 0 on success, -1 on failure.
*/
static int32_t GetFrameStats(FrameStats* stats,
const I420VideoFrame& frame);
/**
Checks the validity of a FrameStats struct. Currently, valid implies only
that is had changed from its initialized state.
\param[in] stats
Frame statistics.
\return True on valid stats, false on invalid stats.
*/
static bool ValidFrameStats(const FrameStats& stats);
/**
Returns a FrameStats struct to its intialized state.
\param[in,out] stats
Frame statistics.
*/
static void ClearFrameStats(FrameStats* stats);
/**
Enhances the color of an image through a constant mapping. Only the
chrominance is altered. Has a fixed-point implementation.
\param[in,out] frame
Pointer to the video frame.
*/
static int32_t ColorEnhancement(I420VideoFrame* frame);
/**
Increases/decreases the luminance value.
\param[in,out] frame
Pointer to the video frame.
\param[in] delta
The amount to change the chrominance value of every single pixel.
Can be < 0 also.
\return 0 on success, -1 on failure.
*/
static int32_t Brighten(I420VideoFrame* frame, int delta);
/**
Detects and removes camera flicker from a video stream. Every frame from
the stream must be passed in. A frame will only be altered if flicker has
been detected. Has a fixed-point implementation.
\param[in,out] frame
Pointer to the video frame.
\param[in,out] stats
Frame statistics provided by GetFrameStats(). On return the stats will
be reset to zero if the frame was altered. Call GetFrameStats() again
if the statistics for the altered frame are required.
\return 0 on success, -1 on failure.
*/
virtual int32_t Deflickering(I420VideoFrame* frame, FrameStats* stats) = 0;
/**
Denoises a video frame. Every frame from the stream should be passed in.
Has a fixed-point implementation.
\param[in,out] frame
Pointer to the video frame.
\return The number of modified pixels on success, -1 on failure.
*/
virtual int32_t Denoising(I420VideoFrame* frame) = 0;
/**
Detects if a video frame is excessively bright or dark. Returns a
warning if this is the case. Multiple frames should be passed in before
expecting a warning. Has a floating-point implementation.
\param[in] frame
Pointer to the video frame.
\param[in] stats
Frame statistics provided by GetFrameStats().
\return A member of BrightnessWarning on success, -1 on error
*/
virtual int32_t BrightnessDetection(const I420VideoFrame& frame,
const FrameStats& stats) = 0;
/**
The following functions refer to the pre-processor unit within VPM. The
pre-processor perfoms spatial/temporal decimation and content analysis on
the frames prior to encoding.
*/
/**
Enable/disable temporal decimation
\param[in] enable when true, temporal decimation is enabled
*/
virtual void EnableTemporalDecimation(bool enable) = 0;
/**
Set target resolution
\param[in] width
Target width
\param[in] height
Target height
\param[in] frame_rate
Target frame_rate
\return VPM_OK on success, a negative value on error (see error codes)
*/
virtual int32_t SetTargetResolution(uint32_t width,
uint32_t height,
uint32_t frame_rate) = 0;
/**
Set max frame rate
\param[in] max_frame_rate: maximum frame rate (limited to native frame rate)
\return VPM_OK on success, a negative value on error (see error codes)
*/
virtual int32_t SetMaxFramerate(uint32_t max_frame_rate) = 0;
/**
Get decimated(target) frame rate
*/
virtual uint32_t Decimatedframe_rate() = 0;
/**
Get decimated(target) frame width
*/
virtual uint32_t DecimatedWidth() const = 0;
/**
Get decimated(target) frame height
*/
virtual uint32_t DecimatedHeight() const = 0 ;
/**
Set the spatial resampling settings of the VPM: The resampler may either be
disabled or one of the following:
scaling to a close to target dimension followed by crop/pad
\param[in] resampling_mode
Set resampling mode (a member of VideoFrameResampling)
*/
virtual void SetInputFrameResampleMode(VideoFrameResampling
resampling_mode) = 0;
/**
Get Processed (decimated) frame
\param[in] frame pointer to the video frame.
\param[in] processed_frame pointer (double) to the processed frame. If no
processing is required, processed_frame will be NULL.
\return VPM_OK on success, a negative value on error (see error codes)
*/
virtual int32_t PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processed_frame) = 0;
/**
Return content metrics for the last processed frame
*/
virtual VideoContentMetrics* ContentMetrics() const = 0 ;
/**
Enable content analysis
*/
virtual void EnableContentAnalysis(bool enable) = 0;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_INTERFACE_VIDEO_PROCESSING_H

16
webrtc/modules/video_processing/main/interface/video_processing_defines.h
View File

@ -29,15 +29,13 @@ namespace webrtc {
#define VPM_UNINITIALIZED -5 #define VPM_UNINITIALIZED -5
#define VPM_UNIMPLEMENTED -6 #define VPM_UNIMPLEMENTED -6
enum VideoFrameResampling enum VideoFrameResampling {
{ kNoRescaling, // Disables rescaling.
// TODO: Do we still need crop/pad? kFastRescaling, // Point filter.
kNoRescaling, // disables rescaling kBiLinear, // Bi-linear interpolation.
kFastRescaling, // point kBox, // Box inteprolation.
kBiLinear, // bi-linear interpolation
kBox, // Box inteprolation
}; };
} // namespace } // namespace webrtc
#endif #endif // WEBRTC_MODULES_INTERFACE_VIDEO_PROCESSING_DEFINES_H

14
webrtc/modules/video_processing/main/source/brighten.cc
View File

@ -31,19 +31,19 @@ int32_t Brighten(I420VideoFrame* frame, int delta) {
return VPM_PARAMETER_ERROR; return VPM_PARAMETER_ERROR;
} }
int numPixels = frame->width() * frame->height(); int num_pixels = frame->width() * frame->height();
int lookUp[256]; int look_up[256];
for (int i = 0; i < 256; i++) { for (int i = 0; i < 256; i++) {
int val = i + delta; int val = i + delta;
lookUp[i] = ((((val < 0) ? 0 : val) > 255) ? 255 : val); look_up[i] = ((((val < 0) ? 0 : val) > 255) ? 255 : val);
} }
uint8_t* tempPtr = frame->buffer(kYPlane); uint8_t* temp_ptr = frame->buffer(kYPlane);
for (int i = 0; i < numPixels; i++) { for (int i = 0; i < num_pixels; i++) {
*tempPtr = static_cast<uint8_t>(lookUp[*tempPtr]); *temp_ptr = static_cast<uint8_t>(look_up[*temp_ptr]);
tempPtr++; temp_ptr++;
} }
return VPM_OK; return VPM_OK;
} }

266
webrtc/modules/video_processing/main/source/brightness_detection.cc
View File

@ -17,180 +17,128 @@
namespace webrtc { namespace webrtc {
VPMBrightnessDetection::VPMBrightnessDetection() : VPMBrightnessDetection::VPMBrightnessDetection() :
_id(0) id_(0) {
{ Reset();
Reset();
} }
VPMBrightnessDetection::~VPMBrightnessDetection() VPMBrightnessDetection::~VPMBrightnessDetection() {}
{
int32_t VPMBrightnessDetection::ChangeUniqueId(const int32_t id) {
id_ = id;
return VPM_OK;
} }
int32_t void VPMBrightnessDetection::Reset() {
VPMBrightnessDetection::ChangeUniqueId(const int32_t id) frame_cnt_bright_ = 0;
{ frame_cnt_dark_ = 0;
_id = id;
return VPM_OK;
} }
void int32_t VPMBrightnessDetection::ProcessFrame(
VPMBrightnessDetection::Reset() const I420VideoFrame& frame,
{ const VideoProcessingModule::FrameStats& stats) {
_frameCntBright = 0; if (frame.IsZeroSize()) {
_frameCntDark = 0; WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
} "Null frame pointer");
return VPM_PARAMETER_ERROR;
}
int width = frame.width();
int height = frame.height();
int32_t if (!VideoProcessingModule::ValidFrameStats(stats)) {
VPMBrightnessDetection::ProcessFrame(const I420VideoFrame& frame, WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
const VideoProcessingModule::FrameStats& "Invalid frame stats");
stats) return VPM_PARAMETER_ERROR;
{ }
if (frame.IsZeroSize())
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Null frame pointer");
return VPM_PARAMETER_ERROR;
}
int width = frame.width();
int height = frame.height();
if (!VideoProcessingModule::ValidFrameStats(stats)) const uint8_t frame_cnt_alarm = 2;
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Invalid frame stats");
return VPM_PARAMETER_ERROR;
}
const uint8_t frameCntAlarm = 2; // Get proportion in lowest bins.
uint8_t low_th = 20;
float prop_low = 0;
for (uint32_t i = 0; i < low_th; i++) {
prop_low += stats.hist[i];
}
prop_low /= stats.num_pixels;
// Get proportion in lowest bins // Get proportion in highest bins.
uint8_t lowTh = 20; unsigned char high_th = 230;
float propLow = 0; float prop_high = 0;
for (uint32_t i = 0; i < lowTh; i++) for (uint32_t i = high_th; i < 256; i++) {
{ prop_high += stats.hist[i];
propLow += stats.hist[i]; }
} prop_high /= stats.num_pixels;
propLow /= stats.numPixels;
// Get proportion in highest bins if (prop_high < 0.4) {
unsigned char highTh = 230; if (stats.mean < 90 || stats.mean > 170) {
float propHigh = 0; // Standard deviation of Y
for (uint32_t i = highTh; i < 256; i++) const uint8_t* buffer = frame.buffer(kYPlane);
{ float std_y = 0;
propHigh += stats.hist[i]; for (int h = 0; h < height; h += (1 << stats.subSamplHeight)) {
} int row = h*width;
propHigh /= stats.numPixels; for (int w = 0; w < width; w += (1 << stats.subSamplWidth)) {
std_y += (buffer[w + row] - stats.mean) * (buffer[w + row] -
stats.mean);
}
}
std_y = sqrt(std_y / stats.num_pixels);
if(propHigh < 0.4) // Get percentiles.
{ uint32_t sum = 0;
if (stats.mean < 90 || stats.mean > 170) uint32_t median_y = 140;
{ uint32_t perc05 = 0;
// Standard deviation of Y uint32_t perc95 = 255;
const uint8_t* buffer = frame.buffer(kYPlane); float pos_perc05 = stats.num_pixels * 0.05f;
float stdY = 0; float pos_median = stats.num_pixels * 0.5f;
for (int h = 0; h < height; h += (1 << stats.subSamplHeight)) float posPerc95 = stats.num_pixels * 0.95f;
{ for (uint32_t i = 0; i < 256; i++) {
int row = h*width; sum += stats.hist[i];
for (int w = 0; w < width; w += (1 << stats.subSamplWidth)) if (sum < pos_perc05) perc05 = i; // 5th perc.
{ if (sum < pos_median) median_y = i; // 50th perc.
stdY += (buffer[w + row] - stats.mean) * (buffer[w + row] - if (sum < posPerc95)
stats.mean); perc95 = i; // 95th perc.
}
}
stdY = sqrt(stdY / stats.numPixels);
// Get percentiles
uint32_t sum = 0;
uint32_t medianY = 140;
uint32_t perc05 = 0;
uint32_t perc95 = 255;
float posPerc05 = stats.numPixels * 0.05f;
float posMedian = stats.numPixels * 0.5f;
float posPerc95 = stats.numPixels * 0.95f;
for (uint32_t i = 0; i < 256; i++)
{
sum += stats.hist[i];
if (sum < posPerc05)
{
perc05 = i; // 5th perc
}
if (sum < posMedian)
{
medianY = i; // 50th perc
}
if (sum < posPerc95)
{
perc95 = i; // 95th perc
}
else
{
break;
}
}
// Check if image is too dark
if ((stdY < 55) && (perc05 < 50))
{
if (medianY < 60 || stats.mean < 80 || perc95 < 130 ||
propLow > 0.20)
{
_frameCntDark++;
}
else
{
_frameCntDark = 0;
}
}
else
{
_frameCntDark = 0;
}
// Check if image is too bright
if ((stdY < 52) && (perc95 > 200) && (medianY > 160))
{
if (medianY > 185 || stats.mean > 185 || perc05 > 140 ||
propHigh > 0.25)
{
_frameCntBright++;
}
else
{
_frameCntBright = 0;
}
}
else
{
_frameCntBright = 0;
}
}
else else
{ break;
_frameCntDark = 0; }
_frameCntBright = 0;
// Check if image is too dark
if ((std_y < 55) && (perc05 < 50)) {
if (median_y < 60 || stats.mean < 80 || perc95 < 130 ||
prop_low > 0.20) {
frame_cnt_dark_++;
} else {
frame_cnt_dark_ = 0;
}
} else {
frame_cnt_dark_ = 0;
} }
} // Check if image is too bright
else if ((std_y < 52) && (perc95 > 200) && (median_y > 160)) {
{ if (median_y > 185 || stats.mean > 185 || perc05 > 140 ||
_frameCntBright++; prop_high > 0.25) {
_frameCntDark = 0; frame_cnt_bright_++;
} } else {
frame_cnt_bright_ = 0;
if (_frameCntDark > frameCntAlarm) }
{ } else {
return VideoProcessingModule::kDarkWarning; frame_cnt_bright_ = 0;
} }
else if (_frameCntBright > frameCntAlarm) } else {
{ frame_cnt_dark_ = 0;
return VideoProcessingModule::kBrightWarning; frame_cnt_bright_ = 0;
}
else
{
return VideoProcessingModule::kNoWarning;
} }
} else {
frame_cnt_bright_++;
frame_cnt_dark_ = 0;
}
if (frame_cnt_dark_ > frame_cnt_alarm) {
return VideoProcessingModule::kDarkWarning;
} else if (frame_cnt_bright_ > frame_cnt_alarm) {
return VideoProcessingModule::kBrightWarning;
} else {
return VideoProcessingModule::kNoWarning;
}
} }
} // namespace } // namespace webrtc

36
webrtc/modules/video_processing/main/source/brightness_detection.h
View File

@ -11,34 +11,30 @@
/* /*
* brightness_detection.h * brightness_detection.h
*/ */
#ifndef VPM_BRIGHTNESS_DETECTION_H #ifndef MODULES_VIDEO_PROCESSING_MAIN_SOURCE_BRIGHTNESS_DETECTION_H
#define VPM_BRIGHTNESS_DETECTION_H #define MODULES_VIDEO_PROCESSING_MAIN_SOURCE_BRIGHTNESS_DETECTION_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
class VPMBrightnessDetection class VPMBrightnessDetection {
{ public:
public: VPMBrightnessDetection();
VPMBrightnessDetection(); ~VPMBrightnessDetection();
~VPMBrightnessDetection(); int32_t ChangeUniqueId(int32_t id);
int32_t ChangeUniqueId(int32_t id); void Reset();
int32_t ProcessFrame(const I420VideoFrame& frame,
const VideoProcessingModule::FrameStats& stats);
void Reset(); private:
int32_t id_;
int32_t ProcessFrame(const I420VideoFrame& frame, uint32_t frame_cnt_bright_;
const VideoProcessingModule::FrameStats& stats); uint32_t frame_cnt_dark_;
private:
int32_t _id;
uint32_t _frameCntBright;
uint32_t _frameCntDark;
}; };
} // namespace } // namespace webrtc
#endif // VPM_BRIGHTNESS_DETECTION_H #endif // MODULES_VIDEO_PROCESSING_MAIN_SOURCE_BRIGHTNESS_DETECTION_H

72
webrtc/modules/video_processing/main/source/color_enhancement.cc
View File

@ -16,50 +16,42 @@
namespace webrtc { namespace webrtc {
namespace VideoProcessing namespace VideoProcessing {
{ int32_t ColorEnhancement(I420VideoFrame* frame) {
int32_t assert(frame);
ColorEnhancement(I420VideoFrame* frame) // Pointers to U and V color pixels.
{ uint8_t* ptr_u;
assert(frame); uint8_t* ptr_v;
// pointers to U and V color pixels uint8_t temp_chroma;
uint8_t* ptrU; if (frame->IsZeroSize()) {
uint8_t* ptrV; WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing,
uint8_t tempChroma; -1, "Null frame pointer");
return VPM_GENERAL_ERROR;
}
if (frame->IsZeroSize()) if (frame->width() == 0 || frame->height() == 0) {
{ WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing,
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, -1, "Invalid frame size");
-1, "Null frame pointer"); return VPM_GENERAL_ERROR;
return VPM_GENERAL_ERROR; }
}
if (frame->width() == 0 || frame->height() == 0) // set pointers to first U and V pixels (skip luminance)
{ ptr_u = frame->buffer(kUPlane);
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, ptr_v = frame->buffer(kVPlane);
-1, "Invalid frame size"); int size_uv = ((frame->width() + 1) / 2) * ((frame->height() + 1) / 2);
return VPM_GENERAL_ERROR;
}
// set pointers to first U and V pixels (skip luminance) // Loop through all chrominance pixels and modify color
ptrU = frame->buffer(kUPlane); for (int ix = 0; ix < size_uv; ix++) {
ptrV = frame->buffer(kVPlane); temp_chroma = colorTable[*ptr_u][*ptr_v];
int size_uv = ((frame->width() + 1) / 2) * ((frame->height() + 1) / 2); *ptr_v = colorTable[*ptr_v][*ptr_u];
*ptr_u = temp_chroma;
// loop through all chrominance pixels and modify color ptr_u++;
for (int ix = 0; ix < size_uv; ix++) ptr_v++;
{ }
tempChroma = colorTable[*ptrU][*ptrV]; return VPM_OK;
*ptrV = colorTable[*ptrV][*ptrU]; }
*ptrU = tempChroma;
// increment pointers
ptrU++;
ptrV++;
}
return VPM_OK;
}
} // namespace } // namespace VideoProcessing
} // namespace webrtc } // namespace webrtc

13
webrtc/modules/video_processing/main/source/color_enhancement.h
View File

@ -11,19 +11,18 @@
/* /*
* color_enhancement.h * color_enhancement.h
*/ */
#ifndef VPM_COLOR_ENHANCEMENT_H #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_COLOR_ENHANCEMENT_H
#define VPM_COLOR_ENHANCEMENT_H #define WEBRTC_MODULES_VIDEO_PROCESSING_COLOR_ENHANCEMENT_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
namespace VideoProcessing namespace VideoProcessing {
{ int32_t ColorEnhancement(I420VideoFrame* frame);
int32_t ColorEnhancement(I420VideoFrame* frame);
} }
} // namespace } // namespace webrtc
#endif // VPM_COLOR_ENHANCEMENT_H #endif // WEBRTC_MODULES_VIDEO_PROCESSING_COLOR_ENHANCEMENT_H

12
webrtc/modules/video_processing/main/source/color_enhancement_private.h
View File

@ -1,3 +1,13 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VPM_COLOR_ENHANCEMENT_PRIVATE_H #ifndef VPM_COLOR_ENHANCEMENT_PRIVATE_H
#define VPM_COLOR_ENHANCEMENT_PRIVATE_H #define VPM_COLOR_ENHANCEMENT_PRIVATE_H
@ -270,4 +280,4 @@ static const uint8_t colorTable[256][256] = {
} // namespace } // namespace
#endif // VPM_COLOR_ENHANCEMENT_PRIVATE_H #endif // VPM_COLOR_ENHANCEMENT_PRIVATE_H

425
webrtc/modules/video_processing/main/source/content_analysis.cc
View File

@ -17,229 +17,185 @@
namespace webrtc { namespace webrtc {
VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection): VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection)
_origFrame(NULL), : orig_frame_(NULL),
_prevFrame(NULL), prev_frame_(NULL),
_width(0), width_(0),
_height(0), height_(0),
_skipNum(1), skip_num_(1),
_border(8), border_(8),
_motionMagnitude(0.0f), motion_magnitude_(0.0f),
_spatialPredErr(0.0f), spatial_pred_err_(0.0f),
_spatialPredErrH(0.0f), spatial_pred_err_h_(0.0f),
_spatialPredErrV(0.0f), spatial_pred_err_v_(0.0f),
_firstFrame(true), first_frame_(true),
_CAInit(false), ca_Init_(false),
_cMetrics(NULL) content_metrics_(NULL) {
{ ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C;
ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C; TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C;
TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C;
if (runtime_cpu_detection) if (runtime_cpu_detection) {
{
#if defined(WEBRTC_ARCH_X86_FAMILY) #if defined(WEBRTC_ARCH_X86_FAMILY)
if (WebRtc_GetCPUInfo(kSSE2)) if (WebRtc_GetCPUInfo(kSSE2)) {
{ ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_SSE2;
ComputeSpatialMetrics = TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2;
&VPMContentAnalysis::ComputeSpatialMetrics_SSE2; }
TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2;
}
#endif #endif
} }
Release();
Release();
} }
VPMContentAnalysis::~VPMContentAnalysis() VPMContentAnalysis::~VPMContentAnalysis() {
{ Release();
Release();
} }
VideoContentMetrics* VideoContentMetrics* VPMContentAnalysis::ComputeContentMetrics(
VPMContentAnalysis::ComputeContentMetrics(const I420VideoFrame& inputFrame) const I420VideoFrame& inputFrame) {
{ if (inputFrame.IsZeroSize())
if (inputFrame.IsZeroSize()) return NULL;
{
return NULL;
}
// Init if needed (native dimension change) // Init if needed (native dimension change).
if (_width != inputFrame.width() || _height != inputFrame.height()) if (width_ != inputFrame.width() || height_ != inputFrame.height()) {
{ if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height()))
if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height())) return NULL;
{ }
return NULL; // Only interested in the Y plane.
} orig_frame_ = inputFrame.buffer(kYPlane);
}
// Only interested in the Y plane.
_origFrame = inputFrame.buffer(kYPlane);
// compute spatial metrics: 3 spatial prediction errors // Compute spatial metrics: 3 spatial prediction errors.
(this->*ComputeSpatialMetrics)(); (this->*ComputeSpatialMetrics)();
// compute motion metrics // Compute motion metrics
if (_firstFrame == false) if (first_frame_ == false)
ComputeMotionMetrics(); ComputeMotionMetrics();
// saving current frame as previous one: Y only // Saving current frame as previous one: Y only.
memcpy(_prevFrame, _origFrame, _width * _height); memcpy(prev_frame_, orig_frame_, width_ * height_);
_firstFrame = false; first_frame_ = false;
_CAInit = true; ca_Init_ = true;
return ContentMetrics(); return ContentMetrics();
} }
int32_t int32_t VPMContentAnalysis::Release() {
VPMContentAnalysis::Release() if (content_metrics_ != NULL) {
{ delete content_metrics_;
if (_cMetrics != NULL) content_metrics_ = NULL;
{ }
delete _cMetrics;
_cMetrics = NULL;
}
if (_prevFrame != NULL) if (prev_frame_ != NULL) {
{ delete [] prev_frame_;
delete [] _prevFrame; prev_frame_ = NULL;
_prevFrame = NULL; }
}
_width = 0; width_ = 0;
_height = 0; height_ = 0;
_firstFrame = true; first_frame_ = true;
return VPM_OK; return VPM_OK;
} }
int32_t int32_t VPMContentAnalysis::Initialize(int width, int height) {
VPMContentAnalysis::Initialize(int width, int height) width_ = width;
{ height_ = height;
_width = width; first_frame_ = true;
_height = height;
_firstFrame = true;
// skip parameter: # of skipped rows: for complexity reduction // skip parameter: # of skipped rows: for complexity reduction
// temporal also currently uses it for column reduction. // temporal also currently uses it for column reduction.
_skipNum = 1; skip_num_ = 1;
// use skipNum = 2 for 4CIF, WHD // use skipNum = 2 for 4CIF, WHD
if ( (_height >= 576) && (_width >= 704) ) if ( (height_ >= 576) && (width_ >= 704) ) {
{ skip_num_ = 2;
_skipNum = 2; }
} // use skipNum = 4 for FULLL_HD images
// use skipNum = 4 for FULLL_HD images if ( (height_ >= 1080) && (width_ >= 1920) ) {
if ( (_height >= 1080) && (_width >= 1920) ) skip_num_ = 4;
{ }
_skipNum = 4;
}
if (_cMetrics != NULL) if (content_metrics_ != NULL) {
{ delete content_metrics_;
delete _cMetrics; }
}
if (_prevFrame != NULL) if (prev_frame_ != NULL) {
{ delete [] prev_frame_;
delete [] _prevFrame; }
}
// Spatial Metrics don't work on a border of 8. Minimum processing // Spatial Metrics don't work on a border of 8. Minimum processing
// block size is 16 pixels. So make sure the width and height support this. // block size is 16 pixels. So make sure the width and height support this.
if (_width <= 32 || _height <= 32) if (width_ <= 32 || height_ <= 32) {
{ ca_Init_ = false;
_CAInit = false; return VPM_PARAMETER_ERROR;
return VPM_PARAMETER_ERROR; }
}
_cMetrics = new VideoContentMetrics(); content_metrics_ = new VideoContentMetrics();
if (_cMetrics == NULL) if (content_metrics_ == NULL) {
{ return VPM_MEMORY;
return VPM_MEMORY; }
}
_prevFrame = new uint8_t[_width * _height] ; // Y only prev_frame_ = new uint8_t[width_ * height_]; // Y only.
if (_prevFrame == NULL) if (prev_frame_ == NULL) return VPM_MEMORY;
{
return VPM_MEMORY;
}
return VPM_OK; return VPM_OK;
} }
// Compute motion metrics: magnitude over non-zero motion vectors, // Compute motion metrics: magnitude over non-zero motion vectors,
// and size of zero cluster // and size of zero cluster
int32_t int32_t VPMContentAnalysis::ComputeMotionMetrics() {
VPMContentAnalysis::ComputeMotionMetrics() // Motion metrics: only one is derived from normalized
{ // (MAD) temporal difference
(this->*TemporalDiffMetric)();
// Motion metrics: only one is derived from normalized return VPM_OK;
// (MAD) temporal difference
(this->*TemporalDiffMetric)();
return VPM_OK;
} }
// Normalized temporal difference (MAD): used as a motion level metric // Normalized temporal difference (MAD): used as a motion level metric
// Normalize MAD by spatial contrast: images with more contrast // Normalize MAD by spatial contrast: images with more contrast
// (pixel variance) likely have larger temporal difference // (pixel variance) likely have larger temporal difference
// To reduce complexity, we compute the metric for a reduced set of points. // To reduce complexity, we compute the metric for a reduced set of points.
int32_t int32_t VPMContentAnalysis::TemporalDiffMetric_C() {
VPMContentAnalysis::TemporalDiffMetric_C() // size of original frame
{ int sizei = height_;
// size of original frame int sizej = width_;
int sizei = _height; uint32_t tempDiffSum = 0;
int sizej = _width; uint32_t pixelSum = 0;
uint64_t pixelSqSum = 0;
uint32_t tempDiffSum = 0; uint32_t num_pixels = 0; // Counter for # of pixels.
uint32_t pixelSum = 0; const int width_end = ((width_ - 2*border_) & -16) + border_;
uint64_t pixelSqSum = 0;
uint32_t numPixels = 0; // counter for # of pixels for (int i = border_; i < sizei - border_; i += skip_num_) {
for (int j = border_; j < width_end; j++) {
num_pixels += 1;
int ssn = i * sizej + j;
const int width_end = ((_width - 2*_border) & -16) + _border; uint8_t currPixel = orig_frame_[ssn];
uint8_t prevPixel = prev_frame_[ssn];
for(int i = _border; i < sizei - _border; i += _skipNum) tempDiffSum += (uint32_t)abs((int16_t)(currPixel - prevPixel));
{ pixelSum += (uint32_t) currPixel;
for(int j = _border; j < width_end; j++) pixelSqSum += (uint64_t) (currPixel * currPixel);
{
numPixels += 1;
int ssn = i * sizej + j;
uint8_t currPixel = _origFrame[ssn];
uint8_t prevPixel = _prevFrame[ssn];
tempDiffSum += (uint32_t)
abs((int16_t)(currPixel - prevPixel));
pixelSum += (uint32_t) currPixel;
pixelSqSum += (uint64_t) (currPixel * currPixel);
}
} }
}
// default // Default.
_motionMagnitude = 0.0f; motion_magnitude_ = 0.0f;
if (tempDiffSum == 0) if (tempDiffSum == 0) return VPM_OK;
{
return VPM_OK;
}
// normalize over all pixels // Normalize over all pixels.
float const tempDiffAvg = (float)tempDiffSum / (float)(numPixels); float const tempDiffAvg = (float)tempDiffSum / (float)(num_pixels);
float const pixelSumAvg = (float)pixelSum / (float)(numPixels); float const pixelSumAvg = (float)pixelSum / (float)(num_pixels);
float const pixelSqSumAvg = (float)pixelSqSum / (float)(numPixels); float const pixelSqSumAvg = (float)pixelSqSum / (float)(num_pixels);
float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg); float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
if (contrast > 0.0)
{
contrast = sqrt(contrast);
_motionMagnitude = tempDiffAvg/contrast;
}
return VPM_OK;
if (contrast > 0.0) {
contrast = sqrt(contrast);
motion_magnitude_ = tempDiffAvg/contrast;
}
return VPM_OK;
} }
// Compute spatial metrics: // Compute spatial metrics:
@ -249,88 +205,71 @@ VPMContentAnalysis::TemporalDiffMetric_C()
// The metrics are a simple estimate of the up-sampling prediction error, // The metrics are a simple estimate of the up-sampling prediction error,
// estimated assuming sub-sampling for decimation (no filtering), // estimated assuming sub-sampling for decimation (no filtering),
// and up-sampling back up with simple bilinear interpolation. // and up-sampling back up with simple bilinear interpolation.
int32_t int32_t VPMContentAnalysis::ComputeSpatialMetrics_C() {
VPMContentAnalysis::ComputeSpatialMetrics_C() const int sizei = height_;
{ const int sizej = width_;
//size of original frame
const int sizei = _height;
const int sizej = _width;
// pixel mean square average: used to normalize the spatial metrics // Pixel mean square average: used to normalize the spatial metrics.
uint32_t pixelMSA = 0; uint32_t pixelMSA = 0;
uint32_t spatialErrSum = 0; uint32_t spatialErrSum = 0;
uint32_t spatialErrVSum = 0; uint32_t spatialErrVSum = 0;
uint32_t spatialErrHSum = 0; uint32_t spatialErrHSum = 0;
// make sure work section is a multiple of 16 // make sure work section is a multiple of 16
const int width_end = ((sizej - 2*_border) & -16) + _border; const int width_end = ((sizej - 2*border_) & -16) + border_;
for(int i = _border; i < sizei - _border; i += _skipNum) for (int i = border_; i < sizei - border_; i += skip_num_) {
{ for (int j = border_; j < width_end; j++) {
for(int j = _border; j < width_end; j++) int ssn1= i * sizej + j;
{ int ssn2 = (i + 1) * sizej + j; // bottom
int ssn3 = (i - 1) * sizej + j; // top
int ssn4 = i * sizej + j + 1; // right
int ssn5 = i * sizej + j - 1; // left
int ssn1= i * sizej + j; uint16_t refPixel1 = orig_frame_[ssn1] << 1;
int ssn2 = (i + 1) * sizej + j; // bottom uint16_t refPixel2 = orig_frame_[ssn1] << 2;
int ssn3 = (i - 1) * sizej + j; // top
int ssn4 = i * sizej + j + 1; // right
int ssn5 = i * sizej + j - 1; // left
uint16_t refPixel1 = _origFrame[ssn1] << 1; uint8_t bottPixel = orig_frame_[ssn2];
uint16_t refPixel2 = _origFrame[ssn1] << 2; uint8_t topPixel = orig_frame_[ssn3];
uint8_t rightPixel = orig_frame_[ssn4];
uint8_t leftPixel = orig_frame_[ssn5];
uint8_t bottPixel = _origFrame[ssn2]; spatialErrSum += (uint32_t) abs((int16_t)(refPixel2
uint8_t topPixel = _origFrame[ssn3]; - (uint16_t)(bottPixel + topPixel + leftPixel + rightPixel)));
uint8_t rightPixel = _origFrame[ssn4]; spatialErrVSum += (uint32_t) abs((int16_t)(refPixel1
uint8_t leftPixel = _origFrame[ssn5]; - (uint16_t)(bottPixel + topPixel)));
spatialErrHSum += (uint32_t) abs((int16_t)(refPixel1
spatialErrSum += (uint32_t) abs((int16_t)(refPixel2 - (uint16_t)(leftPixel + rightPixel)));
- (uint16_t)(bottPixel + topPixel pixelMSA += orig_frame_[ssn1];
+ leftPixel + rightPixel)));
spatialErrVSum += (uint32_t) abs((int16_t)(refPixel1
- (uint16_t)(bottPixel + topPixel)));
spatialErrHSum += (uint32_t) abs((int16_t)(refPixel1
- (uint16_t)(leftPixel + rightPixel)));
pixelMSA += _origFrame[ssn1];
}
} }
}
// normalize over all pixels // Normalize over all pixels.
const float spatialErr = (float)(spatialErrSum >> 2); const float spatialErr = (float)(spatialErrSum >> 2);
const float spatialErrH = (float)(spatialErrHSum >> 1); const float spatialErrH = (float)(spatialErrHSum >> 1);
const float spatialErrV = (float)(spatialErrVSum >> 1); const float spatialErrV = (float)(spatialErrVSum >> 1);
const float norm = (float)pixelMSA; const float norm = (float)pixelMSA;
// 2X2: // 2X2:
_spatialPredErr = spatialErr / norm; spatial_pred_err_ = spatialErr / norm;
// 1X2:
// 1X2: spatial_pred_err_h_ = spatialErrH / norm;
_spatialPredErrH = spatialErrH / norm; // 2X1:
spatial_pred_err_v_ = spatialErrV / norm;
// 2X1: return VPM_OK;
_spatialPredErrV = spatialErrV / norm;
return VPM_OK;
} }
VideoContentMetrics* VideoContentMetrics* VPMContentAnalysis::ContentMetrics() {
VPMContentAnalysis::ContentMetrics() if (ca_Init_ == false) return NULL;
{
if (_CAInit == false)
{
return NULL;
}
_cMetrics->spatial_pred_err = _spatialPredErr; content_metrics_->spatial_pred_err = spatial_pred_err_;
_cMetrics->spatial_pred_err_h = _spatialPredErrH; content_metrics_->spatial_pred_err_h = spatial_pred_err_h_;
_cMetrics->spatial_pred_err_v = _spatialPredErrV; content_metrics_->spatial_pred_err_v = spatial_pred_err_v_;
// Motion metric: normalized temporal difference (MAD) // Motion metric: normalized temporal difference (MAD).
_cMetrics->motion_magnitude = _motionMagnitude; content_metrics_->motion_magnitude = motion_magnitude_;
return _cMetrics;
return content_metrics_;
} }
} // namespace } // namespace webrtc

114
webrtc/modules/video_processing/main/source/content_analysis.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef VPM_CONTENT_ANALYSIS_H #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_CONTENT_ANALYSIS_H
#define VPM_CONTENT_ANALYSIS_H #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_CONTENT_ANALYSIS_H
#include "webrtc/common_video/interface/i420_video_frame.h" #include "webrtc/common_video/interface/i420_video_frame.h"
#include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/interface/module_common_types.h"
@ -18,75 +18,71 @@
namespace webrtc { namespace webrtc {
class VPMContentAnalysis class VPMContentAnalysis {
{ public:
public: // When |runtime_cpu_detection| is true, runtime selection of an optimized
// When |runtime_cpu_detection| is true, runtime selection of an optimized // code path is allowed.
// code path is allowed. explicit VPMContentAnalysis(bool runtime_cpu_detection);
VPMContentAnalysis(bool runtime_cpu_detection); ~VPMContentAnalysis();
~VPMContentAnalysis();
// Initialize ContentAnalysis - should be called prior to // Initialize ContentAnalysis - should be called prior to
// extractContentFeature // extractContentFeature
// Inputs: width, height // Inputs: width, height
// Return value: 0 if OK, negative value upon error // Return value: 0 if OK, negative value upon error
int32_t Initialize(int width, int height); int32_t Initialize(int width, int height);
// Extract content Feature - main function of ContentAnalysis // Extract content Feature - main function of ContentAnalysis
// Input: new frame // Input: new frame
// Return value: pointer to structure containing content Analysis // Return value: pointer to structure containing content Analysis
// metrics or NULL value upon error // metrics or NULL value upon error
VideoContentMetrics* ComputeContentMetrics(const I420VideoFrame& VideoContentMetrics* ComputeContentMetrics(const I420VideoFrame&
inputFrame); inputFrame);
// Release all allocated memory // Release all allocated memory
// Output: 0 if OK, negative value upon error // Output: 0 if OK, negative value upon error
int32_t Release(); int32_t Release();
private: private:
// return motion metrics
VideoContentMetrics* ContentMetrics();
// return motion metrics // Normalized temporal difference metric: for motion magnitude
VideoContentMetrics* ContentMetrics(); typedef int32_t (VPMContentAnalysis::*TemporalDiffMetricFunc)();
TemporalDiffMetricFunc TemporalDiffMetric;
int32_t TemporalDiffMetric_C();
// Normalized temporal difference metric: for motion magnitude // Motion metric method: call 2 metrics (magnitude and size)
typedef int32_t (VPMContentAnalysis::*TemporalDiffMetricFunc)(); int32_t ComputeMotionMetrics();
TemporalDiffMetricFunc TemporalDiffMetric;
int32_t TemporalDiffMetric_C();
// Motion metric method: call 2 metrics (magnitude and size) // Spatial metric method: computes the 3 frame-average spatial
int32_t ComputeMotionMetrics(); // prediction errors (1x2,2x1,2x2)
typedef int32_t (VPMContentAnalysis::*ComputeSpatialMetricsFunc)();
// Spatial metric method: computes the 3 frame-average spatial ComputeSpatialMetricsFunc ComputeSpatialMetrics;
// prediction errors (1x2,2x1,2x2) int32_t ComputeSpatialMetrics_C();
typedef int32_t (VPMContentAnalysis::*ComputeSpatialMetricsFunc)();
ComputeSpatialMetricsFunc ComputeSpatialMetrics;
int32_t ComputeSpatialMetrics_C();
#if defined(WEBRTC_ARCH_X86_FAMILY) #if defined(WEBRTC_ARCH_X86_FAMILY)
int32_t ComputeSpatialMetrics_SSE2(); int32_t ComputeSpatialMetrics_SSE2();
int32_t TemporalDiffMetric_SSE2(); int32_t TemporalDiffMetric_SSE2();
#endif #endif
const uint8_t* _origFrame; const uint8_t* orig_frame_;
uint8_t* _prevFrame; uint8_t* prev_frame_;
int _width; int width_;
int _height; int height_;
int _skipNum; int skip_num_;
int _border; int border_;
// Content Metrics: // Content Metrics: Stores the local average of the metrics.
// stores the local average of the metrics float motion_magnitude_; // motion class
float _motionMagnitude; // motion class float spatial_pred_err_; // spatial class
float _spatialPredErr; // spatial class float spatial_pred_err_h_; // spatial class
float _spatialPredErrH; // spatial class float spatial_pred_err_v_; // spatial class
float _spatialPredErrV; // spatial class bool first_frame_;
bool _firstFrame; bool ca_Init_;
bool _CAInit;
VideoContentMetrics* _cMetrics; VideoContentMetrics* content_metrics_;
};
}; // end of VPMContentAnalysis class definition } // namespace webrtc
} // namespace #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_CONTENT_ANALYSIS_H
#endif

432
webrtc/modules/video_processing/main/source/content_analysis_sse2.cc
View File

@ -15,284 +15,248 @@
namespace webrtc { namespace webrtc {
int32_t int32_t VPMContentAnalysis::TemporalDiffMetric_SSE2() {
VPMContentAnalysis::TemporalDiffMetric_SSE2() uint32_t num_pixels = 0; // counter for # of pixels
{ const uint8_t* imgBufO = orig_frame_ + border_*width_ + border_;
uint32_t numPixels = 0; // counter for # of pixels const uint8_t* imgBufP = prev_frame_ + border_*width_ + border_;
const uint8_t* imgBufO = _origFrame + _border*_width + _border; const int32_t width_end = ((width_ - 2*border_) & -16) + border_;
const uint8_t* imgBufP = _prevFrame + _border*_width + _border;
const int32_t width_end = ((_width - 2*_border) & -16) + _border; __m128i sad_64 = _mm_setzero_si128();
__m128i sum_64 = _mm_setzero_si128();
__m128i sqsum_64 = _mm_setzero_si128();
const __m128i z = _mm_setzero_si128();
__m128i sad_64 = _mm_setzero_si128(); for (uint16_t i = 0; i < (height_ - 2*border_); i += skip_num_) {
__m128i sum_64 = _mm_setzero_si128(); __m128i sqsum_32 = _mm_setzero_si128();
__m128i sqsum_64 = _mm_setzero_si128();
const __m128i z = _mm_setzero_si128();
for(uint16_t i = 0; i < (_height - 2*_border); i += _skipNum) const uint8_t *lineO = imgBufO;
{ const uint8_t *lineP = imgBufP;
__m128i sqsum_32 = _mm_setzero_si128();
const uint8_t *lineO = imgBufO; // Work on 16 pixels at a time. For HD content with a width of 1920
const uint8_t *lineP = imgBufP; // this loop will run ~67 times (depending on border). Maximum for
// abs(o-p) and sum(o) will be 255. _mm_sad_epu8 produces 2 64 bit
// results which are then accumulated. There is no chance of
// rollover for these two accumulators.
// o*o will have a maximum of 255*255 = 65025. This will roll over
// a 16 bit accumulator as 67*65025 > 65535, but will fit in a
// 32 bit accumulator.
for (uint16_t j = 0; j < width_end - border_; j += 16) {
const __m128i o = _mm_loadu_si128((__m128i*)(lineO));
const __m128i p = _mm_loadu_si128((__m128i*)(lineP));
// Work on 16 pixels at a time. For HD content with a width of 1920 lineO += 16;
// this loop will run ~67 times (depending on border). Maximum for lineP += 16;
// abs(o-p) and sum(o) will be 255. _mm_sad_epu8 produces 2 64 bit
// results which are then accumulated. There is no chance of
// rollover for these two accumulators.
// o*o will have a maximum of 255*255 = 65025. This will roll over
// a 16 bit accumulator as 67*65025 > 65535, but will fit in a
// 32 bit accumulator.
for(uint16_t j = 0; j < width_end - _border; j += 16)
{
const __m128i o = _mm_loadu_si128((__m128i*)(lineO));
const __m128i p = _mm_loadu_si128((__m128i*)(lineP));
lineO += 16; // Abs pixel difference between frames.
lineP += 16; sad_64 = _mm_add_epi64 (sad_64, _mm_sad_epu8(o, p));
// abs pixel difference between frames // sum of all pixels in frame
sad_64 = _mm_add_epi64 (sad_64, _mm_sad_epu8(o, p)); sum_64 = _mm_add_epi64 (sum_64, _mm_sad_epu8(o, z));
// sum of all pixels in frame // Squared sum of all pixels in frame.
sum_64 = _mm_add_epi64 (sum_64, _mm_sad_epu8(o, z)); const __m128i olo = _mm_unpacklo_epi8(o,z);
const __m128i ohi = _mm_unpackhi_epi8(o,z);
// squared sum of all pixels in frame const __m128i sqsum_32_lo = _mm_madd_epi16(olo, olo);
const __m128i olo = _mm_unpacklo_epi8(o,z); const __m128i sqsum_32_hi = _mm_madd_epi16(ohi, ohi);
const __m128i ohi = _mm_unpackhi_epi8(o,z);
const __m128i sqsum_32_lo = _mm_madd_epi16(olo, olo); sqsum_32 = _mm_add_epi32(sqsum_32, sqsum_32_lo);
const __m128i sqsum_32_hi = _mm_madd_epi16(ohi, ohi); sqsum_32 = _mm_add_epi32(sqsum_32, sqsum_32_hi);
sqsum_32 = _mm_add_epi32(sqsum_32, sqsum_32_lo);
sqsum_32 = _mm_add_epi32(sqsum_32, sqsum_32_hi);
}
// Add to 64 bit running sum as to not roll over.
sqsum_64 = _mm_add_epi64(sqsum_64,
_mm_add_epi64(_mm_unpackhi_epi32(sqsum_32,z),
_mm_unpacklo_epi32(sqsum_32,z)));
imgBufO += _width * _skipNum;
imgBufP += _width * _skipNum;
numPixels += (width_end - _border);
} }
__m128i sad_final_128; // Add to 64 bit running sum as to not roll over.
__m128i sum_final_128; sqsum_64 = _mm_add_epi64(sqsum_64,
__m128i sqsum_final_128; _mm_add_epi64(_mm_unpackhi_epi32(sqsum_32,z),
_mm_unpacklo_epi32(sqsum_32,z)));
// bring sums out of vector registers and into integer register imgBufO += width_ * skip_num_;
// domain, summing them along the way imgBufP += width_ * skip_num_;
_mm_store_si128 (&sad_final_128, sad_64); num_pixels += (width_end - border_);
_mm_store_si128 (&sum_final_128, sum_64); }
_mm_store_si128 (&sqsum_final_128, sqsum_64);
uint64_t *sad_final_64 = __m128i sad_final_128;
reinterpret_cast<uint64_t*>(&sad_final_128); __m128i sum_final_128;
uint64_t *sum_final_64 = __m128i sqsum_final_128;
reinterpret_cast<uint64_t*>(&sum_final_128);
uint64_t *sqsum_final_64 =
reinterpret_cast<uint64_t*>(&sqsum_final_128);
const uint32_t pixelSum = sum_final_64[0] + sum_final_64[1]; // Bring sums out of vector registers and into integer register
const uint64_t pixelSqSum = sqsum_final_64[0] + sqsum_final_64[1]; // domain, summing them along the way.
const uint32_t tempDiffSum = sad_final_64[0] + sad_final_64[1]; _mm_store_si128 (&sad_final_128, sad_64);
_mm_store_si128 (&sum_final_128, sum_64);
_mm_store_si128 (&sqsum_final_128, sqsum_64);
// default uint64_t *sad_final_64 = reinterpret_cast<uint64_t*>(&sad_final_128);
_motionMagnitude = 0.0f; uint64_t *sum_final_64 = reinterpret_cast<uint64_t*>(&sum_final_128);
uint64_t *sqsum_final_64 = reinterpret_cast<uint64_t*>(&sqsum_final_128);
if (tempDiffSum == 0) const uint32_t pixelSum = sum_final_64[0] + sum_final_64[1];
{ const uint64_t pixelSqSum = sqsum_final_64[0] + sqsum_final_64[1];
return VPM_OK; const uint32_t tempDiffSum = sad_final_64[0] + sad_final_64[1];
}
// normalize over all pixels // Default.
const float tempDiffAvg = (float)tempDiffSum / (float)(numPixels); motion_magnitude_ = 0.0f;
const float pixelSumAvg = (float)pixelSum / (float)(numPixels);
const float pixelSqSumAvg = (float)pixelSqSum / (float)(numPixels);
float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
if (contrast > 0.0) if (tempDiffSum == 0) return VPM_OK;
{
contrast = sqrt(contrast);
_motionMagnitude = tempDiffAvg/contrast;
}
return VPM_OK; // Normalize over all pixels.
const float tempDiffAvg = (float)tempDiffSum / (float)(num_pixels);
const float pixelSumAvg = (float)pixelSum / (float)(num_pixels);
const float pixelSqSumAvg = (float)pixelSqSum / (float)(num_pixels);
float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
if (contrast > 0.0) {
contrast = sqrt(contrast);
motion_magnitude_ = tempDiffAvg/contrast;
}
return VPM_OK;
} }
int32_t int32_t VPMContentAnalysis::ComputeSpatialMetrics_SSE2() {
VPMContentAnalysis::ComputeSpatialMetrics_SSE2() const uint8_t* imgBuf = orig_frame_ + border_*width_;
{ const int32_t width_end = ((width_ - 2 * border_) & -16) + border_;
const uint8_t* imgBuf = _origFrame + _border*_width;
const int32_t width_end = ((_width - 2*_border) & -16) + _border;
__m128i se_32 = _mm_setzero_si128(); __m128i se_32 = _mm_setzero_si128();
__m128i sev_32 = _mm_setzero_si128(); __m128i sev_32 = _mm_setzero_si128();
__m128i seh_32 = _mm_setzero_si128(); __m128i seh_32 = _mm_setzero_si128();
__m128i msa_32 = _mm_setzero_si128(); __m128i msa_32 = _mm_setzero_si128();
const __m128i z = _mm_setzero_si128(); const __m128i z = _mm_setzero_si128();
// Error is accumulated as a 32 bit value. Looking at HD content with a // Error is accumulated as a 32 bit value. Looking at HD content with a
// height of 1080 lines, or about 67 macro blocks. If the 16 bit row // height of 1080 lines, or about 67 macro blocks. If the 16 bit row
// value is maxed out at 65529 for every row, 65529*1080 = 70777800, which // value is maxed out at 65529 for every row, 65529*1080 = 70777800, which
// will not roll over a 32 bit accumulator. // will not roll over a 32 bit accumulator.
// _skipNum is also used to reduce the number of rows // skip_num_ is also used to reduce the number of rows
for(int32_t i = 0; i < (_height - 2*_border); i += _skipNum) for (int32_t i = 0; i < (height_ - 2*border_); i += skip_num_) {
{ __m128i se_16 = _mm_setzero_si128();
__m128i se_16 = _mm_setzero_si128(); __m128i sev_16 = _mm_setzero_si128();
__m128i sev_16 = _mm_setzero_si128(); __m128i seh_16 = _mm_setzero_si128();
__m128i seh_16 = _mm_setzero_si128(); __m128i msa_16 = _mm_setzero_si128();
__m128i msa_16 = _mm_setzero_si128();
// Row error is accumulated as a 16 bit value. There are 8 // Row error is accumulated as a 16 bit value. There are 8
// accumulators. Max value of a 16 bit number is 65529. Looking // accumulators. Max value of a 16 bit number is 65529. Looking
// at HD content, 1080p, has a width of 1920, 120 macro blocks. // at HD content, 1080p, has a width of 1920, 120 macro blocks.
// A mb at a time is processed at a time. Absolute max error at // A mb at a time is processed at a time. Absolute max error at
// a point would be abs(0-255+255+255+255) which equals 1020. // a point would be abs(0-255+255+255+255) which equals 1020.
// 120*1020 = 122400. The probability of hitting this is quite low // 120*1020 = 122400. The probability of hitting this is quite low
// on well behaved content. A specially crafted image could roll over. // on well behaved content. A specially crafted image could roll over.
// _border could also be adjusted to concentrate on just the center of // border_ could also be adjusted to concentrate on just the center of
// the images for an HD capture in order to reduce the possiblity of // the images for an HD capture in order to reduce the possiblity of
// rollover. // rollover.
const uint8_t *lineTop = imgBuf - _width + _border; const uint8_t *lineTop = imgBuf - width_ + border_;
const uint8_t *lineCen = imgBuf + _border; const uint8_t *lineCen = imgBuf + border_;
const uint8_t *lineBot = imgBuf + _width + _border; const uint8_t *lineBot = imgBuf + width_ + border_;
for(int32_t j = 0; j < width_end - _border; j += 16) for (int32_t j = 0; j < width_end - border_; j += 16) {
{ const __m128i t = _mm_loadu_si128((__m128i*)(lineTop));
const __m128i t = _mm_loadu_si128((__m128i*)(lineTop)); const __m128i l = _mm_loadu_si128((__m128i*)(lineCen - 1));
const __m128i l = _mm_loadu_si128((__m128i*)(lineCen - 1)); const __m128i c = _mm_loadu_si128((__m128i*)(lineCen));
const __m128i c = _mm_loadu_si128((__m128i*)(lineCen)); const __m128i r = _mm_loadu_si128((__m128i*)(lineCen + 1));
const __m128i r = _mm_loadu_si128((__m128i*)(lineCen + 1)); const __m128i b = _mm_loadu_si128((__m128i*)(lineBot));
const __m128i b = _mm_loadu_si128((__m128i*)(lineBot));
lineTop += 16; lineTop += 16;
lineCen += 16; lineCen += 16;
lineBot += 16; lineBot += 16;
// center pixel unpacked // center pixel unpacked
__m128i clo = _mm_unpacklo_epi8(c,z); __m128i clo = _mm_unpacklo_epi8(c,z);
__m128i chi = _mm_unpackhi_epi8(c,z); __m128i chi = _mm_unpackhi_epi8(c,z);
// left right pixels unpacked and added together // left right pixels unpacked and added together
const __m128i lrlo = _mm_add_epi16(_mm_unpacklo_epi8(l,z), const __m128i lrlo = _mm_add_epi16(_mm_unpacklo_epi8(l,z),
_mm_unpacklo_epi8(r,z)); _mm_unpacklo_epi8(r,z));
const __m128i lrhi = _mm_add_epi16(_mm_unpackhi_epi8(l,z), const __m128i lrhi = _mm_add_epi16(_mm_unpackhi_epi8(l,z),
_mm_unpackhi_epi8(r,z)); _mm_unpackhi_epi8(r,z));
// top & bottom pixels unpacked and added together // top & bottom pixels unpacked and added together
const __m128i tblo = _mm_add_epi16(_mm_unpacklo_epi8(t,z), const __m128i tblo = _mm_add_epi16(_mm_unpacklo_epi8(t,z),
_mm_unpacklo_epi8(b,z)); _mm_unpacklo_epi8(b,z));
const __m128i tbhi = _mm_add_epi16(_mm_unpackhi_epi8(t,z), const __m128i tbhi = _mm_add_epi16(_mm_unpackhi_epi8(t,z),
_mm_unpackhi_epi8(b,z)); _mm_unpackhi_epi8(b,z));
// running sum of all pixels // running sum of all pixels
msa_16 = _mm_add_epi16(msa_16, _mm_add_epi16(chi, clo)); msa_16 = _mm_add_epi16(msa_16, _mm_add_epi16(chi, clo));
clo = _mm_slli_epi16(clo, 1); clo = _mm_slli_epi16(clo, 1);
chi = _mm_slli_epi16(chi, 1); chi = _mm_slli_epi16(chi, 1);
const __m128i sevtlo = _mm_subs_epi16(clo, tblo); const __m128i sevtlo = _mm_subs_epi16(clo, tblo);
const __m128i sevthi = _mm_subs_epi16(chi, tbhi); const __m128i sevthi = _mm_subs_epi16(chi, tbhi);
const __m128i sehtlo = _mm_subs_epi16(clo, lrlo); const __m128i sehtlo = _mm_subs_epi16(clo, lrlo);
const __m128i sehthi = _mm_subs_epi16(chi, lrhi); const __m128i sehthi = _mm_subs_epi16(chi, lrhi);
clo = _mm_slli_epi16(clo, 1); clo = _mm_slli_epi16(clo, 1);
chi = _mm_slli_epi16(chi, 1); chi = _mm_slli_epi16(chi, 1);
const __m128i setlo = _mm_subs_epi16(clo, const __m128i setlo = _mm_subs_epi16(clo, _mm_add_epi16(lrlo, tblo));
_mm_add_epi16(lrlo, tblo)); const __m128i sethi = _mm_subs_epi16(chi, _mm_add_epi16(lrhi, tbhi));
const __m128i sethi = _mm_subs_epi16(chi,
_mm_add_epi16(lrhi, tbhi));
// Add to 16 bit running sum // Add to 16 bit running sum
se_16 = _mm_add_epi16(se_16, se_16 = _mm_add_epi16(se_16, _mm_max_epi16(setlo,
_mm_max_epi16(setlo, _mm_subs_epi16(z, setlo)));
_mm_subs_epi16(z, setlo))); se_16 = _mm_add_epi16(se_16, _mm_max_epi16(sethi,
se_16 = _mm_add_epi16(se_16, _mm_subs_epi16(z, sethi)));
_mm_max_epi16(sethi, sev_16 = _mm_add_epi16(sev_16, _mm_max_epi16(sevtlo,
_mm_subs_epi16(z, sethi))); _mm_subs_epi16(z, sevtlo)));
sev_16 = _mm_add_epi16(sev_16, sev_16 = _mm_add_epi16(sev_16, _mm_max_epi16(sevthi,
_mm_max_epi16(sevtlo, _mm_subs_epi16(z, sevthi)));
_mm_subs_epi16(z, sevtlo))); seh_16 = _mm_add_epi16(seh_16, _mm_max_epi16(sehtlo,
sev_16 = _mm_add_epi16(sev_16, _mm_subs_epi16(z, sehtlo)));
_mm_max_epi16(sevthi, seh_16 = _mm_add_epi16(seh_16, _mm_max_epi16(sehthi,
_mm_subs_epi16(z, sevthi))); _mm_subs_epi16(z, sehthi)));
seh_16 = _mm_add_epi16(seh_16,
_mm_max_epi16(sehtlo,
_mm_subs_epi16(z, sehtlo)));
seh_16 = _mm_add_epi16(seh_16,
_mm_max_epi16(sehthi,
_mm_subs_epi16(z, sehthi)));
}
// Add to 32 bit running sum as to not roll over.
se_32 = _mm_add_epi32(se_32,
_mm_add_epi32(_mm_unpackhi_epi16(se_16,z),
_mm_unpacklo_epi16(se_16,z)));
sev_32 = _mm_add_epi32(sev_32,
_mm_add_epi32(_mm_unpackhi_epi16(sev_16,z),
_mm_unpacklo_epi16(sev_16,z)));
seh_32 = _mm_add_epi32(seh_32,
_mm_add_epi32(_mm_unpackhi_epi16(seh_16,z),
_mm_unpacklo_epi16(seh_16,z)));
msa_32 = _mm_add_epi32(msa_32,
_mm_add_epi32(_mm_unpackhi_epi16(msa_16,z),
_mm_unpacklo_epi16(msa_16,z)));
imgBuf += _width * _skipNum;
} }
__m128i se_128; // Add to 32 bit running sum as to not roll over.
__m128i sev_128; se_32 = _mm_add_epi32(se_32, _mm_add_epi32(_mm_unpackhi_epi16(se_16,z),
__m128i seh_128; _mm_unpacklo_epi16(se_16,z)));
__m128i msa_128; sev_32 = _mm_add_epi32(sev_32, _mm_add_epi32(_mm_unpackhi_epi16(sev_16,z),
_mm_unpacklo_epi16(sev_16,z)));
seh_32 = _mm_add_epi32(seh_32, _mm_add_epi32(_mm_unpackhi_epi16(seh_16,z),
_mm_unpacklo_epi16(seh_16,z)));
msa_32 = _mm_add_epi32(msa_32, _mm_add_epi32(_mm_unpackhi_epi16(msa_16,z),
_mm_unpacklo_epi16(msa_16,z)));
// bring sums out of vector registers and into integer register imgBuf += width_ * skip_num_;
// domain, summing them along the way }
_mm_store_si128 (&se_128,
_mm_add_epi64(_mm_unpackhi_epi32(se_32,z),
_mm_unpacklo_epi32(se_32,z)));
_mm_store_si128 (&sev_128,
_mm_add_epi64(_mm_unpackhi_epi32(sev_32,z),
_mm_unpacklo_epi32(sev_32,z)));
_mm_store_si128 (&seh_128,
_mm_add_epi64(_mm_unpackhi_epi32(seh_32,z),
_mm_unpacklo_epi32(seh_32,z)));
_mm_store_si128 (&msa_128,
_mm_add_epi64(_mm_unpackhi_epi32(msa_32,z),
_mm_unpacklo_epi32(msa_32,z)));
uint64_t *se_64 = __m128i se_128;
reinterpret_cast<uint64_t*>(&se_128); __m128i sev_128;
uint64_t *sev_64 = __m128i seh_128;
reinterpret_cast<uint64_t*>(&sev_128); __m128i msa_128;
uint64_t *seh_64 =
reinterpret_cast<uint64_t*>(&seh_128);
uint64_t *msa_64 =
reinterpret_cast<uint64_t*>(&msa_128);
const uint32_t spatialErrSum = se_64[0] + se_64[1]; // Bring sums out of vector registers and into integer register
const uint32_t spatialErrVSum = sev_64[0] + sev_64[1]; // domain, summing them along the way.
const uint32_t spatialErrHSum = seh_64[0] + seh_64[1]; _mm_store_si128 (&se_128, _mm_add_epi64(_mm_unpackhi_epi32(se_32,z),
const uint32_t pixelMSA = msa_64[0] + msa_64[1]; _mm_unpacklo_epi32(se_32,z)));
_mm_store_si128 (&sev_128, _mm_add_epi64(_mm_unpackhi_epi32(sev_32,z),
_mm_unpacklo_epi32(sev_32,z)));
_mm_store_si128 (&seh_128, _mm_add_epi64(_mm_unpackhi_epi32(seh_32,z),
_mm_unpacklo_epi32(seh_32,z)));
_mm_store_si128 (&msa_128, _mm_add_epi64(_mm_unpackhi_epi32(msa_32,z),
_mm_unpacklo_epi32(msa_32,z)));
// normalize over all pixels uint64_t *se_64 = reinterpret_cast<uint64_t*>(&se_128);
const float spatialErr = (float)(spatialErrSum >> 2); uint64_t *sev_64 = reinterpret_cast<uint64_t*>(&sev_128);
const float spatialErrH = (float)(spatialErrHSum >> 1); uint64_t *seh_64 = reinterpret_cast<uint64_t*>(&seh_128);
const float spatialErrV = (float)(spatialErrVSum >> 1); uint64_t *msa_64 = reinterpret_cast<uint64_t*>(&msa_128);
const float norm = (float)pixelMSA;
// 2X2: const uint32_t spatialErrSum = se_64[0] + se_64[1];
_spatialPredErr = spatialErr / norm; const uint32_t spatialErrVSum = sev_64[0] + sev_64[1];
const uint32_t spatialErrHSum = seh_64[0] + seh_64[1];
const uint32_t pixelMSA = msa_64[0] + msa_64[1];
// 1X2: // Normalize over all pixels.
_spatialPredErrH = spatialErrH / norm; const float spatialErr = (float)(spatialErrSum >> 2);
const float spatialErrH = (float)(spatialErrHSum >> 1);
const float spatialErrV = (float)(spatialErrVSum >> 1);
const float norm = (float)pixelMSA;
// 2X1: // 2X2:
_spatialPredErrV = spatialErrV / norm; spatial_pred_err_ = spatialErr / norm;
// 1X2:
spatial_pred_err_h_ = spatialErrH / norm;
// 2X1:
spatial_pred_err_v_ = spatialErrV / norm;
return VPM_OK; return VPM_OK;
} }

687
webrtc/modules/video_processing/main/source/deflickering.cc
View File

@ -20,431 +20,386 @@
namespace webrtc { namespace webrtc {
// Detection constants // Detection constants
enum { kFrequencyDeviation = 39 }; // (Q4) Maximum allowed deviation for detection // (Q4) Maximum allowed deviation for detection.
enum { kMinFrequencyToDetect = 32 }; // (Q4) Minimum frequency that can be detected enum { kFrequencyDeviation = 39 };
enum { kNumFlickerBeforeDetect = 2 }; // Number of flickers before we accept detection // (Q4) Minimum frequency that can be detected.
enum { kMeanValueScaling = 4 }; // (Q4) In power of 2 enum { kMinFrequencyToDetect = 32 };
enum { kZeroCrossingDeadzone = 10 }; // Deadzone region in terms of pixel values // Number of flickers before we accept detection
enum { kNumFlickerBeforeDetect = 2 };
// Deflickering constants enum { kmean_valueScaling = 4 }; // (Q4) In power of 2
// Dead-zone region in terms of pixel values
enum { kZeroCrossingDeadzone = 10 };
// Deflickering constants.
// Compute the quantiles over 1 / DownsamplingFactor of the image. // Compute the quantiles over 1 / DownsamplingFactor of the image.
enum { kDownsamplingFactor = 8 }; enum { kDownsamplingFactor = 8 };
enum { kLog2OfDownsamplingFactor = 3 }; enum { kLog2OfDownsamplingFactor = 3 };
// To generate in Matlab: // To generate in Matlab:
// >> probUW16 = round(2^11 * [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.97]); // >> probUW16 = round(2^11 *
// [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.97]);
// >> fprintf('%d, ', probUW16) // >> fprintf('%d, ', probUW16)
// Resolution reduced to avoid overflow when multiplying with the (potentially) large // Resolution reduced to avoid overflow when multiplying with the
// number of pixels. // (potentially) large number of pixels.
const uint16_t VPMDeflickering::_probUW16[kNumProbs] = const uint16_t VPMDeflickering::prob_uw16_[kNumProbs] = {102, 205, 410, 614,
{102, 205, 410, 614, 819, 1024, 1229, 1434, 1638, 1843, 1946, 1987}; // <Q11> 819, 1024, 1229, 1434, 1638, 1843, 1946, 1987}; // <Q11>
// To generate in Matlab: // To generate in Matlab:
// >> numQuants = 14; maxOnlyLength = 5; // >> numQuants = 14; maxOnlyLength = 5;
// >> weightUW16 = round(2^15 * [linspace(0.5, 1.0, numQuants - maxOnlyLength)]); // >> weightUW16 = round(2^15 *
// [linspace(0.5, 1.0, numQuants - maxOnlyLength)]);
// >> fprintf('%d, %d,\n ', weightUW16); // >> fprintf('%d, %d,\n ', weightUW16);
const uint16_t VPMDeflickering::_weightUW16[kNumQuants - kMaxOnlyLength] = const uint16_t VPMDeflickering::weight_uw16_[kNumQuants - kMaxOnlyLength] =
{16384, 18432, 20480, 22528, 24576, 26624, 28672, 30720, 32768}; // <Q15> {16384, 18432, 20480, 22528, 24576, 26624, 28672, 30720, 32768}; // <Q15>
VPMDeflickering::VPMDeflickering() : VPMDeflickering::VPMDeflickering()
_id(0) : id_(0) {
{ Reset();
Reset();
} }
VPMDeflickering::~VPMDeflickering() VPMDeflickering::~VPMDeflickering() {}
{
int32_t VPMDeflickering::ChangeUniqueId(const int32_t id) {
id_ = id;
return 0;
} }
int32_t void VPMDeflickering::Reset() {
VPMDeflickering::ChangeUniqueId(const int32_t id) mean_buffer_length_ = 0;
{ detection_state_ = 0;
_id = id; frame_rate_ = 0;
memset(mean_buffer_, 0, sizeof(int32_t) * kMeanBufferLength);
memset(timestamp_buffer_, 0, sizeof(int32_t) * kMeanBufferLength);
// Initialize the history with a uniformly distributed histogram.
quant_hist_uw8_[0][0] = 0;
quant_hist_uw8_[0][kNumQuants - 1] = 255;
for (int32_t i = 0; i < kNumProbs; i++) {
quant_hist_uw8_[0][i + 1] = static_cast<uint8_t>((WEBRTC_SPL_UMUL_16_16(
prob_uw16_[i], 255) + (1 << 10)) >> 11); // Unsigned round. <Q0>
}
for (int32_t i = 1; i < kFrameHistory_size; i++) {
memcpy(quant_hist_uw8_[i], quant_hist_uw8_[0],
sizeof(uint8_t) * kNumQuants);
}
}
int32_t VPMDeflickering::ProcessFrame(I420VideoFrame* frame,
VideoProcessingModule::FrameStats* stats) {
assert(frame);
uint32_t frame_memory;
uint8_t quant_uw8[kNumQuants];
uint8_t maxquant_uw8[kNumQuants];
uint8_t minquant_uw8[kNumQuants];
uint16_t target_quant_uw16[kNumQuants];
uint16_t increment_uw16;
uint8_t map_uw8[256];
uint16_t tmp_uw16;
uint32_t tmp_uw32;
int width = frame->width();
int height = frame->height();
if (frame->IsZeroSize()) {
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
"Null frame pointer");
return VPM_GENERAL_ERROR;
}
// Stricter height check due to subsampling size calculation below.
if (height < 2) {
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
"Invalid frame size");
return VPM_GENERAL_ERROR;
}
if (!VideoProcessingModule::ValidFrameStats(*stats)) {
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
"Invalid frame stats");
return VPM_GENERAL_ERROR;
}
if (PreDetection(frame->timestamp(), *stats) == -1) return VPM_GENERAL_ERROR;
// Flicker detection
int32_t det_flicker = DetectFlicker();
if (det_flicker < 0) {
return VPM_GENERAL_ERROR;
} else if (det_flicker != 1) {
return 0; return 0;
} }
void // Size of luminance component.
VPMDeflickering::Reset() const uint32_t y_size = height * width;
{
_meanBufferLength = 0;
_detectionState = 0;
_frameRate = 0;
memset(_meanBuffer, 0, sizeof(int32_t) * kMeanBufferLength); const uint32_t y_sub_size = width * (((height - 1) >>
memset(_timestampBuffer, 0, sizeof(int32_t) * kMeanBufferLength); kLog2OfDownsamplingFactor) + 1);
uint8_t* y_sorted = new uint8_t[y_sub_size];
uint32_t sort_row_idx = 0;
for (int i = 0; i < height; i += kDownsamplingFactor) {
memcpy(y_sorted + sort_row_idx * width,
frame->buffer(kYPlane) + i * width, width);
sort_row_idx++;
}
// Initialize the history with a uniformly distributed histogram webrtc::Sort(y_sorted, y_sub_size, webrtc::TYPE_UWord8);
_quantHistUW8[0][0] = 0;
_quantHistUW8[0][kNumQuants - 1] = 255; uint32_t prob_idx_uw32 = 0;
for (int32_t i = 0; i < kNumProbs; i++) quant_uw8[0] = 0;
{ quant_uw8[kNumQuants - 1] = 255;
_quantHistUW8[0][i + 1] = static_cast<uint8_t>((WEBRTC_SPL_UMUL_16_16(
_probUW16[i], 255) + (1 << 10)) >> 11); // Unsigned round. <Q0> // Ensure we won't get an overflow below.
// In practice, the number of subsampled pixels will not become this large.
if (y_sub_size > (1 << 21) - 1) {
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
"Subsampled number of pixels too large");
return -1;
}
for (int32_t i = 0; i < kNumProbs; i++) {
// <Q0>.
prob_idx_uw32 = WEBRTC_SPL_UMUL_32_16(y_sub_size, prob_uw16_[i]) >> 11;
quant_uw8[i + 1] = y_sorted[prob_idx_uw32];
}
delete [] y_sorted;
y_sorted = NULL;
// Shift history for new frame.
memmove(quant_hist_uw8_[1], quant_hist_uw8_[0],
(kFrameHistory_size - 1) * kNumQuants * sizeof(uint8_t));
// Store current frame in history.
memcpy(quant_hist_uw8_[0], quant_uw8, kNumQuants * sizeof(uint8_t));
// We use a frame memory equal to the ceiling of half the frame rate to
// ensure we capture an entire period of flicker.
frame_memory = (frame_rate_ + (1 << 5)) >> 5; // Unsigned ceiling. <Q0>
// frame_rate_ in Q4.
if (frame_memory > kFrameHistory_size) {
frame_memory = kFrameHistory_size;
}
// Get maximum and minimum.
for (int32_t i = 0; i < kNumQuants; i++) {
maxquant_uw8[i] = 0;
minquant_uw8[i] = 255;
for (uint32_t j = 0; j < frame_memory; j++) {
if (quant_hist_uw8_[j][i] > maxquant_uw8[i]) {
maxquant_uw8[i] = quant_hist_uw8_[j][i];
}
if (quant_hist_uw8_[j][i] < minquant_uw8[i]) {
minquant_uw8[i] = quant_hist_uw8_[j][i];
}
} }
}
for (int32_t i = 1; i < kFrameHistorySize; i++)
{
memcpy(_quantHistUW8[i], _quantHistUW8[0], sizeof(uint8_t) * kNumQuants);
}
}
int32_t // Get target quantiles.
VPMDeflickering::ProcessFrame(I420VideoFrame* frame, for (int32_t i = 0; i < kNumQuants - kMaxOnlyLength; i++) {
VideoProcessingModule::FrameStats* stats) target_quant_uw16[i] = static_cast<uint16_t>((WEBRTC_SPL_UMUL_16_16(
{ weight_uw16_[i], maxquant_uw8[i]) + WEBRTC_SPL_UMUL_16_16((1 << 15) -
assert(frame); weight_uw16_[i], minquant_uw8[i])) >> 8); // <Q7>
uint32_t frameMemory; }
uint8_t quantUW8[kNumQuants];
uint8_t maxQuantUW8[kNumQuants];
uint8_t minQuantUW8[kNumQuants];
uint16_t targetQuantUW16[kNumQuants];
uint16_t incrementUW16;
uint8_t mapUW8[256];
uint16_t tmpUW16; for (int32_t i = kNumQuants - kMaxOnlyLength; i < kNumQuants; i++) {
uint32_t tmpUW32; target_quant_uw16[i] = ((uint16_t)maxquant_uw8[i]) << 7;
int width = frame->width(); }
int height = frame->height();
if (frame->IsZeroSize()) // Compute the map from input to output pixels.
{ uint16_t mapUW16; // <Q7>
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, for (int32_t i = 1; i < kNumQuants; i++) {
"Null frame pointer"); // As quant and targetQuant are limited to UWord8, it's safe to use Q7 here.
return VPM_GENERAL_ERROR; tmp_uw32 = static_cast<uint32_t>(target_quant_uw16[i] -
target_quant_uw16[i - 1]);
tmp_uw16 = static_cast<uint16_t>(quant_uw8[i] - quant_uw8[i - 1]); // <Q0>
if (tmp_uw16 > 0) {
increment_uw16 = static_cast<uint16_t>(WebRtcSpl_DivU32U16(tmp_uw32,
tmp_uw16)); // <Q7>
} else {
// The value is irrelevant; the loop below will only iterate once.
increment_uw16 = 0;
} }
// Stricter height check due to subsampling size calculation below. mapUW16 = target_quant_uw16[i - 1];
if (height < 2) for (uint32_t j = quant_uw8[i - 1]; j < (uint32_t)(quant_uw8[i] + 1); j++) {
{ // Unsigned round. <Q0>
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, map_uw8[j] = (uint8_t)((mapUW16 + (1 << 6)) >> 7);
"Invalid frame size"); mapUW16 += increment_uw16;
return VPM_GENERAL_ERROR;
} }
}
if (!VideoProcessingModule::ValidFrameStats(*stats)) // Map to the output frame.
{ uint8_t* buffer = frame->buffer(kYPlane);
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, for (uint32_t i = 0; i < y_size; i++) {
"Invalid frame stats"); buffer[i] = map_uw8[buffer[i]];
return VPM_GENERAL_ERROR; }
}
if (PreDetection(frame->timestamp(), *stats) == -1) // Frame was altered, so reset stats.
{ VideoProcessingModule::ClearFrameStats(stats);
return VPM_GENERAL_ERROR;
}
// Flicker detection return VPM_OK;
int32_t detFlicker = DetectFlicker();
if (detFlicker < 0)
{ // Error
return VPM_GENERAL_ERROR;
}
else if (detFlicker != 1)
{
return 0;
}
// Size of luminance component
const uint32_t ySize = height * width;
const uint32_t ySubSize = width * (((height - 1) >>
kLog2OfDownsamplingFactor) + 1);
uint8_t* ySorted = new uint8_t[ySubSize];
uint32_t sortRowIdx = 0;
for (int i = 0; i < height; i += kDownsamplingFactor)
{
memcpy(ySorted + sortRowIdx * width,
frame->buffer(kYPlane) + i * width, width);
sortRowIdx++;
}
webrtc::Sort(ySorted, ySubSize, webrtc::TYPE_UWord8);
uint32_t probIdxUW32 = 0;
quantUW8[0] = 0;
quantUW8[kNumQuants - 1] = 255;
// Ensure we won't get an overflow below.
// In practice, the number of subsampled pixels will not become this large.
if (ySubSize > (1 << 21) - 1)
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
"Subsampled number of pixels too large");
return -1;
}
for (int32_t i = 0; i < kNumProbs; i++)
{
probIdxUW32 = WEBRTC_SPL_UMUL_32_16(ySubSize, _probUW16[i]) >> 11; // <Q0>
quantUW8[i + 1] = ySorted[probIdxUW32];
}
delete [] ySorted;
ySorted = NULL;
// Shift history for new frame.
memmove(_quantHistUW8[1], _quantHistUW8[0], (kFrameHistorySize - 1) * kNumQuants *
sizeof(uint8_t));
// Store current frame in history.
memcpy(_quantHistUW8[0], quantUW8, kNumQuants * sizeof(uint8_t));
// We use a frame memory equal to the ceiling of half the frame rate to ensure we
// capture an entire period of flicker.
frameMemory = (_frameRate + (1 << 5)) >> 5; // Unsigned ceiling. <Q0>
// _frameRate in Q4.
if (frameMemory > kFrameHistorySize)
{
frameMemory = kFrameHistorySize;
}
// Get maximum and minimum.
for (int32_t i = 0; i < kNumQuants; i++)
{
maxQuantUW8[i] = 0;
minQuantUW8[i] = 255;
for (uint32_t j = 0; j < frameMemory; j++)
{
if (_quantHistUW8[j][i] > maxQuantUW8[i])
{
maxQuantUW8[i] = _quantHistUW8[j][i];
}
if (_quantHistUW8[j][i] < minQuantUW8[i])
{
minQuantUW8[i] = _quantHistUW8[j][i];
}
}
}
// Get target quantiles.
for (int32_t i = 0; i < kNumQuants - kMaxOnlyLength; i++)
{
targetQuantUW16[i] = static_cast<uint16_t>((WEBRTC_SPL_UMUL_16_16(
_weightUW16[i], maxQuantUW8[i]) + WEBRTC_SPL_UMUL_16_16((1 << 15) -
_weightUW16[i], minQuantUW8[i])) >> 8); // <Q7>
}
for (int32_t i = kNumQuants - kMaxOnlyLength; i < kNumQuants; i++)
{
targetQuantUW16[i] = ((uint16_t)maxQuantUW8[i]) << 7;
}
// Compute the map from input to output pixels.
uint16_t mapUW16; // <Q7>
for (int32_t i = 1; i < kNumQuants; i++)
{
// As quant and targetQuant are limited to UWord8, we're safe to use Q7 here.
tmpUW32 = static_cast<uint32_t>(targetQuantUW16[i] -
targetQuantUW16[i - 1]); // <Q7>
tmpUW16 = static_cast<uint16_t>(quantUW8[i] - quantUW8[i - 1]); // <Q0>
if (tmpUW16 > 0)
{
incrementUW16 = static_cast<uint16_t>(WebRtcSpl_DivU32U16(tmpUW32,
tmpUW16)); // <Q7>
}
else
{
// The value is irrelevant; the loop below will only iterate once.
incrementUW16 = 0;
}
mapUW16 = targetQuantUW16[i - 1];
for (uint32_t j = quantUW8[i - 1]; j < (uint32_t)(quantUW8[i] + 1); j++)
{
mapUW8[j] = (uint8_t)((mapUW16 + (1 << 6)) >> 7); // Unsigned round. <Q0>
mapUW16 += incrementUW16;
}
}
// Map to the output frame.
uint8_t* buffer = frame->buffer(kYPlane);
for (uint32_t i = 0; i < ySize; i++)
{
buffer[i] = mapUW8[buffer[i]];
}
// Frame was altered, so reset stats.
VideoProcessingModule::ClearFrameStats(stats);
return 0;
} }
/** /**
Performs some pre-detection operations. Must be called before Performs some pre-detection operations. Must be called before
DetectFlicker(). DetectFlicker().
\param[in] timestamp Timestamp of the current frame. \param[in] timestamp Timestamp of the current frame.
\param[in] stats Statistics of the current frame. \param[in] stats Statistics of the current frame.
\return 0: Success\n \return 0: Success\n
2: Detection not possible due to flickering frequency too close to 2: Detection not possible due to flickering frequency too close to
zero.\n zero.\n
-1: Error -1: Error
*/ */
int32_t int32_t VPMDeflickering::PreDetection(const uint32_t timestamp,
VPMDeflickering::PreDetection(const uint32_t timestamp, const VideoProcessingModule::FrameStats& stats) {
const VideoProcessingModule::FrameStats& stats) int32_t mean_val; // Mean value of frame (Q4)
{ uint32_t frame_rate = 0;
int32_t meanVal; // Mean value of frame (Q4) int32_t meanBufferLength; // Temp variable.
uint32_t frameRate = 0;
int32_t meanBufferLength; // Temp variable
meanVal = ((stats.sum << kMeanValueScaling) / stats.numPixels); mean_val = ((stats.sum << kmean_valueScaling) / stats.num_pixels);
/* Update mean value buffer. // Update mean value buffer.
* This should be done even though we might end up in an unreliable detection. // This should be done even though we might end up in an unreliable detection.
memmove(mean_buffer_ + 1, mean_buffer_,
(kMeanBufferLength - 1) * sizeof(int32_t));
mean_buffer_[0] = mean_val;
// Update timestamp buffer.
// This should be done even though we might end up in an unreliable detection.
memmove(timestamp_buffer_ + 1, timestamp_buffer_, (kMeanBufferLength - 1) *
sizeof(uint32_t));
timestamp_buffer_[0] = timestamp;
/* Compute current frame rate (Q4) */
if (timestamp_buffer_[kMeanBufferLength - 1] != 0) {
frame_rate = ((90000 << 4) * (kMeanBufferLength - 1));
frame_rate /=
(timestamp_buffer_[0] - timestamp_buffer_[kMeanBufferLength - 1]);
} else if (timestamp_buffer_[1] != 0) {
frame_rate = (90000 << 4) / (timestamp_buffer_[0] - timestamp_buffer_[1]);
}
/* Determine required size of mean value buffer (mean_buffer_length_) */
if (frame_rate == 0) {
meanBufferLength = 1;
} else {
meanBufferLength =
(kNumFlickerBeforeDetect * frame_rate) / kMinFrequencyToDetect;
}
/* Sanity check of buffer length */
if (meanBufferLength >= kMeanBufferLength) {
/* Too long buffer. The flickering frequency is too close to zero, which
* makes the estimation unreliable.
*/ */
memmove(_meanBuffer + 1, _meanBuffer, (kMeanBufferLength - 1) * sizeof(int32_t)); mean_buffer_length_ = 0;
_meanBuffer[0] = meanVal; return 2;
}
mean_buffer_length_ = meanBufferLength;
/* Update timestamp buffer. if ((timestamp_buffer_[mean_buffer_length_ - 1] != 0) &&
* This should be done even though we might end up in an unreliable detection. (mean_buffer_length_ != 1)) {
*/ frame_rate = ((90000 << 4) * (mean_buffer_length_ - 1));
memmove(_timestampBuffer + 1, _timestampBuffer, (kMeanBufferLength - 1) * frame_rate /=
sizeof(uint32_t)); (timestamp_buffer_[0] - timestamp_buffer_[mean_buffer_length_ - 1]);
_timestampBuffer[0] = timestamp; } else if (timestamp_buffer_[1] != 0) {
frame_rate = (90000 << 4) / (timestamp_buffer_[0] - timestamp_buffer_[1]);
}
frame_rate_ = frame_rate;
/* Compute current frame rate (Q4) */ return VPM_OK;
if (_timestampBuffer[kMeanBufferLength - 1] != 0)
{
frameRate = ((90000 << 4) * (kMeanBufferLength - 1));
frameRate /= (_timestampBuffer[0] - _timestampBuffer[kMeanBufferLength - 1]);
}else if (_timestampBuffer[1] != 0)
{
frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]);
}
/* Determine required size of mean value buffer (_meanBufferLength) */
if (frameRate == 0) {
meanBufferLength = 1;
}
else {
meanBufferLength = (kNumFlickerBeforeDetect * frameRate) / kMinFrequencyToDetect;
}
/* Sanity check of buffer length */
if (meanBufferLength >= kMeanBufferLength)
{
/* Too long buffer. The flickering frequency is too close to zero, which
* makes the estimation unreliable.
*/
_meanBufferLength = 0;
return 2;
}
_meanBufferLength = meanBufferLength;
if ((_timestampBuffer[_meanBufferLength - 1] != 0) && (_meanBufferLength != 1))
{
frameRate = ((90000 << 4) * (_meanBufferLength - 1));
frameRate /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]);
}else if (_timestampBuffer[1] != 0)
{
frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]);
}
_frameRate = frameRate;
return 0;
} }
/** /**
This function detects flicker in the video stream. As a side effect the mean value This function detects flicker in the video stream. As a side effect the
buffer is updated with the new mean value. mean value buffer is updated with the new mean value.
\return 0: No flickering detected\n \return 0: No flickering detected\n
1: Flickering detected\n 1: Flickering detected\n
2: Detection not possible due to unreliable frequency interval 2: Detection not possible due to unreliable frequency interval
-1: Error -1: Error
*/ */
int32_t VPMDeflickering::DetectFlicker() int32_t VPMDeflickering::DetectFlicker() {
{ uint32_t i;
/* Local variables */ int32_t freqEst; // (Q4) Frequency estimate to base detection upon
uint32_t i; int32_t ret_val = -1;
int32_t freqEst; // (Q4) Frequency estimate to base detection upon
int32_t retVal = -1;
/* Sanity check for _meanBufferLength */ /* Sanity check for mean_buffer_length_ */
if (_meanBufferLength < 2) if (mean_buffer_length_ < 2) {
{ /* Not possible to estimate frequency */
/* Not possible to estimate frequency */ return(2);
return(2); }
} // Count zero crossings with a dead zone to be robust against noise. If the
/* Count zero crossings with a dead zone to be robust against noise. // noise std is 2 pixel this corresponds to about 95% confidence interval.
* If the noise std is 2 pixel this corresponds to about 95% confidence interval. int32_t deadzone = (kZeroCrossingDeadzone << kmean_valueScaling); // Q4
*/ int32_t meanOfBuffer = 0; // Mean value of mean value buffer.
int32_t deadzone = (kZeroCrossingDeadzone << kMeanValueScaling); // Q4 int32_t numZeros = 0; // Number of zeros that cross the dead-zone.
int32_t meanOfBuffer = 0; // Mean value of mean value buffer int32_t cntState = 0; // State variable for zero crossing regions.
int32_t numZeros = 0; // Number of zeros that cross the deadzone int32_t cntStateOld = 0; // Previous state for zero crossing regions.
int32_t cntState = 0; // State variable for zero crossing regions
int32_t cntStateOld = 0; // Previous state variable for zero crossing regions
for (i = 0; i < _meanBufferLength; i++) for (i = 0; i < mean_buffer_length_; i++) {
{ meanOfBuffer += mean_buffer_[i];
meanOfBuffer += _meanBuffer[i]; }
} meanOfBuffer += (mean_buffer_length_ >> 1); // Rounding, not truncation.
meanOfBuffer += (_meanBufferLength >> 1); // Rounding, not truncation meanOfBuffer /= mean_buffer_length_;
meanOfBuffer /= _meanBufferLength;
/* Count zero crossings */ // Count zero crossings.
cntStateOld = (_meanBuffer[0] >= (meanOfBuffer + deadzone)); cntStateOld = (mean_buffer_[0] >= (meanOfBuffer + deadzone));
cntStateOld -= (_meanBuffer[0] <= (meanOfBuffer - deadzone)); cntStateOld -= (mean_buffer_[0] <= (meanOfBuffer - deadzone));
for (i = 1; i < _meanBufferLength; i++) for (i = 1; i < mean_buffer_length_; i++) {
{ cntState = (mean_buffer_[i] >= (meanOfBuffer + deadzone));
cntState = (_meanBuffer[i] >= (meanOfBuffer + deadzone)); cntState -= (mean_buffer_[i] <= (meanOfBuffer - deadzone));
cntState -= (_meanBuffer[i] <= (meanOfBuffer - deadzone)); if (cntStateOld == 0) {
if (cntStateOld == 0) cntStateOld = -cntState;
{
cntStateOld = -cntState;
}
if (((cntState + cntStateOld) == 0) && (cntState != 0))
{
numZeros++;
cntStateOld = cntState;
}
} }
/* END count zero crossings */ if (((cntState + cntStateOld) == 0) && (cntState != 0)) {
numZeros++;
cntStateOld = cntState;
}
}
// END count zero crossings.
/* Frequency estimation according to: /* Frequency estimation according to:
* freqEst = numZeros * frameRate / 2 / _meanBufferLength; * freqEst = numZeros * frame_rate / 2 / mean_buffer_length_;
* *
* Resolution is set to Q4 * Resolution is set to Q4
*/ */
freqEst = ((numZeros * 90000) << 3); freqEst = ((numZeros * 90000) << 3);
freqEst /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]); freqEst /=
(timestamp_buffer_[0] - timestamp_buffer_[mean_buffer_length_ - 1]);
/* Translate frequency estimate to regions close to 100 and 120 Hz */ /* Translate frequency estimate to regions close to 100 and 120 Hz */
uint8_t freqState = 0; // Current translation state; uint8_t freqState = 0; // Current translation state;
// (0) Not in interval, // (0) Not in interval,
// (1) Within valid interval, // (1) Within valid interval,
// (2) Out of range // (2) Out of range
int32_t freqAlias = freqEst; int32_t freqAlias = freqEst;
if (freqEst > kMinFrequencyToDetect) if (freqEst > kMinFrequencyToDetect) {
{ uint8_t aliasState = 1;
uint8_t aliasState = 1; while(freqState == 0) {
while(freqState == 0) /* Increase frequency */
{ freqAlias += (aliasState * frame_rate_);
/* Increase frequency */ freqAlias += ((freqEst << 1) * (1 - (aliasState << 1)));
freqAlias += (aliasState * _frameRate); /* Compute state */
freqAlias += ((freqEst << 1) * (1 - (aliasState << 1))); freqState = (abs(freqAlias - (100 << 4)) <= kFrequencyDeviation);
/* Compute state */ freqState += (abs(freqAlias - (120 << 4)) <= kFrequencyDeviation);
freqState = (abs(freqAlias - (100 << 4)) <= kFrequencyDeviation); freqState += 2 * (freqAlias > ((120 << 4) + kFrequencyDeviation));
freqState += (abs(freqAlias - (120 << 4)) <= kFrequencyDeviation); /* Switch alias state */
freqState += 2 * (freqAlias > ((120 << 4) + kFrequencyDeviation)); aliasState++;
/* Switch alias state */ aliasState &= 0x01;
aliasState++;
aliasState &= 0x01;
}
} }
/* Is frequency estimate within detection region? */ }
if (freqState == 1) /* Is frequency estimate within detection region? */
{ if (freqState == 1) {
retVal = 1; ret_val = 1;
}else if (freqState == 0) } else if (freqState == 0) {
{ ret_val = 2;
retVal = 2; } else {
}else ret_val = 0;
{ }
retVal = 0; return ret_val;
}
return retVal;
} }
} // namespace } // namespace webrtc

69
webrtc/modules/video_processing/main/source/deflickering.h
View File

@ -8,12 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCEdeflickering__H
* deflickering.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCEdeflickering__H
*/
#ifndef VPM_DEFLICKERING_H
#define VPM_DEFLICKERING_H
#include <string.h> // NULL #include <string.h> // NULL
@ -22,44 +18,43 @@
namespace webrtc { namespace webrtc {
class VPMDeflickering class VPMDeflickering {
{ public:
public: VPMDeflickering();
VPMDeflickering(); ~VPMDeflickering();
~VPMDeflickering();
int32_t ChangeUniqueId(int32_t id); int32_t ChangeUniqueId(int32_t id);
void Reset(); void Reset();
int32_t ProcessFrame(I420VideoFrame* frame,
VideoProcessingModule::FrameStats* stats);
int32_t ProcessFrame(I420VideoFrame* frame, private:
VideoProcessingModule::FrameStats* stats); int32_t PreDetection(uint32_t timestamp,
private: const VideoProcessingModule::FrameStats& stats);
int32_t PreDetection(uint32_t timestamp,
const VideoProcessingModule::FrameStats& stats);
int32_t DetectFlicker(); int32_t DetectFlicker();
enum { kMeanBufferLength = 32 }; enum { kMeanBufferLength = 32 };
enum { kFrameHistorySize = 15 }; enum { kFrameHistory_size = 15 };
enum { kNumProbs = 12 }; enum { kNumProbs = 12 };
enum { kNumQuants = kNumProbs + 2 }; enum { kNumQuants = kNumProbs + 2 };
enum { kMaxOnlyLength = 5 }; enum { kMaxOnlyLength = 5 };
int32_t _id; int32_t id_;
uint32_t _meanBufferLength; uint32_t mean_buffer_length_;
uint8_t _detectionState; // 0: No flickering uint8_t detection_state_; // 0: No flickering
// 1: Flickering detected // 1: Flickering detected
// 2: In flickering // 2: In flickering
int32_t _meanBuffer[kMeanBufferLength]; int32_t mean_buffer_[kMeanBufferLength];
uint32_t _timestampBuffer[kMeanBufferLength]; uint32_t timestamp_buffer_[kMeanBufferLength];
uint32_t _frameRate; uint32_t frame_rate_;
static const uint16_t _probUW16[kNumProbs]; static const uint16_t prob_uw16_[kNumProbs];
static const uint16_t _weightUW16[kNumQuants - kMaxOnlyLength]; static const uint16_t weight_uw16_[kNumQuants - kMaxOnlyLength];
uint8_t _quantHistUW8[kFrameHistorySize][kNumQuants]; uint8_t quant_hist_uw8_[kFrameHistory_size][kNumQuants];
}; };
} // namespace } // namespace webrtc
#endif // VPM_DEFLICKERING_H #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCEdeflickering__H

275
webrtc/modules/video_processing/main/source/denoising.cc
View File

@ -14,167 +14,146 @@
#include <string.h> #include <string.h>
namespace webrtc { namespace webrtc {
// Down-sampling in time (unit: number of frames)
enum { kSubsamplingTime = 0 };
// Sub-sampling in width (unit: power of 2.
enum { kSubsamplingWidth = 0 };
// Sub-sampling in height (unit: power of 2)
enum { kSubsamplingHeight = 0 };
// (Q8) De-noising filter parameter
enum { kDenoiseFiltParam = 179 };
// (Q8) 1 - filter parameter
enum { kDenoiseFiltParamRec = 77 };
// (Q8) De-noising threshold level
enum { kDenoiseThreshold = 19200 };
enum { kSubsamplingTime = 0 }; // Down-sampling in time (unit: number of frames) VPMDenoising::VPMDenoising()
enum { kSubsamplingWidth = 0 }; // Sub-sampling in width (unit: power of 2) : id_(0),
enum { kSubsamplingHeight = 0 }; // Sub-sampling in height (unit: power of 2) moment1_(NULL),
enum { kDenoiseFiltParam = 179 }; // (Q8) De-noising filter parameter moment2_(NULL) {
enum { kDenoiseFiltParamRec = 77 }; // (Q8) 1 - filter parameter Reset();
enum { kDenoiseThreshold = 19200 }; // (Q8) De-noising threshold level
VPMDenoising::VPMDenoising() :
_id(0),
_moment1(NULL),
_moment2(NULL)
{
Reset();
} }
VPMDenoising::~VPMDenoising() VPMDenoising::~VPMDenoising() {
{ if (moment1_) {
if (_moment1) delete [] moment1_;
{ moment1_ = NULL;
delete [] _moment1;
_moment1 = NULL;
}
if (_moment2)
{
delete [] _moment2;
_moment2 = NULL;
}
} }
int32_t if (moment2_) {
VPMDenoising::ChangeUniqueId(const int32_t id) delete [] moment2_;
{ moment2_ = NULL;
_id = id; }
return VPM_OK;
} }
void int32_t VPMDenoising::ChangeUniqueId(const int32_t id) {
VPMDenoising::Reset() id_ = id;
{ return VPM_OK;
_frameSize = 0;
_denoiseFrameCnt = 0;
if (_moment1)
{
delete [] _moment1;
_moment1 = NULL;
}
if (_moment2)
{
delete [] _moment2;
_moment2 = NULL;
}
} }
int32_t void VPMDenoising::Reset() {
VPMDenoising::ProcessFrame(I420VideoFrame* frame) frame_size_ = 0;
{ denoise_frame_cnt_ = 0;
assert(frame);
int32_t thevar;
int k;
int jsub, ksub;
int32_t diff0;
uint32_t tmpMoment1;
uint32_t tmpMoment2;
uint32_t tmp;
int32_t numPixelsChanged = 0;
if (frame->IsZeroSize()) if (moment1_) {
{ delete [] moment1_;
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, moment1_ = NULL;
"zero size frame"); }
return VPM_GENERAL_ERROR;
if (moment2_) {
delete [] moment2_;
moment2_ = NULL;
}
}
int32_t VPMDenoising::ProcessFrame(I420VideoFrame* frame) {
assert(frame);
int32_t thevar;
int k;
int jsub, ksub;
int32_t diff0;
uint32_t tmp_moment1;
uint32_t tmp_moment2;
uint32_t tmp;
int32_t num_pixels_changed = 0;
if (frame->IsZeroSize()) {
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, id_,
"zero size frame");
return VPM_GENERAL_ERROR;
}
int width = frame->width();
int height = frame->height();
/* Size of luminance component */
const uint32_t y_size = height * width;
/* Initialization */
if (y_size != frame_size_) {
delete [] moment1_;
moment1_ = NULL;
delete [] moment2_;
moment2_ = NULL;
}
frame_size_ = y_size;
if (!moment1_) {
moment1_ = new uint32_t[y_size];
memset(moment1_, 0, sizeof(uint32_t)*y_size);
}
if (!moment2_) {
moment2_ = new uint32_t[y_size];
memset(moment2_, 0, sizeof(uint32_t)*y_size);
}
/* Apply de-noising on each pixel, but update variance sub-sampled */
uint8_t* buffer = frame->buffer(kYPlane);
for (int i = 0; i < height; i++) { // Collect over height
k = i * width;
ksub = ((i >> kSubsamplingHeight) << kSubsamplingHeight) * width;
for (int j = 0; j < width; j++) { // Collect over width
jsub = ((j >> kSubsamplingWidth) << kSubsamplingWidth);
/* Update mean value for every pixel and every frame */
tmp_moment1 = moment1_[k + j];
tmp_moment1 *= kDenoiseFiltParam; // Q16
tmp_moment1 += ((kDenoiseFiltParamRec * ((uint32_t)buffer[k + j])) << 8);
tmp_moment1 >>= 8; // Q8
moment1_[k + j] = tmp_moment1;
tmp_moment2 = moment2_[ksub + jsub];
if ((ksub == k) && (jsub == j) && (denoise_frame_cnt_ == 0)) {
tmp = ((uint32_t)buffer[k + j] *
(uint32_t)buffer[k + j]);
tmp_moment2 *= kDenoiseFiltParam; // Q16
tmp_moment2 += ((kDenoiseFiltParamRec * tmp) << 8);
tmp_moment2 >>= 8; // Q8
}
moment2_[k + j] = tmp_moment2;
/* Current event = deviation from mean value */
diff0 = ((int32_t)buffer[k + j] << 8) - moment1_[k + j];
/* Recent events = variance (variations over time) */
thevar = moment2_[k + j];
thevar -= ((moment1_[k + j] * moment1_[k + j]) >> 8);
// De-noising criteria, i.e., when should we replace a pixel by its mean.
// 1) recent events are minor.
// 2) current events are minor.
if ((thevar < kDenoiseThreshold)
&& ((diff0 * diff0 >> 8) < kDenoiseThreshold)) {
// Replace with mean.
buffer[k + j] = (uint8_t)(moment1_[k + j] >> 8);
num_pixels_changed++;
}
} }
}
int width = frame->width(); denoise_frame_cnt_++;
int height = frame->height(); if (denoise_frame_cnt_ > kSubsamplingTime)
denoise_frame_cnt_ = 0;
/* Size of luminance component */ return num_pixels_changed;
const uint32_t ysize = height * width;
/* Initialization */
if (ysize != _frameSize)
{
delete [] _moment1;
_moment1 = NULL;
delete [] _moment2;
_moment2 = NULL;
}
_frameSize = ysize;
if (!_moment1)
{
_moment1 = new uint32_t[ysize];
memset(_moment1, 0, sizeof(uint32_t)*ysize);
}
if (!_moment2)
{
_moment2 = new uint32_t[ysize];
memset(_moment2, 0, sizeof(uint32_t)*ysize);
}
/* Apply de-noising on each pixel, but update variance sub-sampled */
uint8_t* buffer = frame->buffer(kYPlane);
for (int i = 0; i < height; i++)
{ // Collect over height
k = i * width;
ksub = ((i >> kSubsamplingHeight) << kSubsamplingHeight) * width;
for (int j = 0; j < width; j++)
{ // Collect over width
jsub = ((j >> kSubsamplingWidth) << kSubsamplingWidth);
/* Update mean value for every pixel and every frame */
tmpMoment1 = _moment1[k + j];
tmpMoment1 *= kDenoiseFiltParam; // Q16
tmpMoment1 += ((kDenoiseFiltParamRec *
((uint32_t)buffer[k + j])) << 8);
tmpMoment1 >>= 8; // Q8
_moment1[k + j] = tmpMoment1;
tmpMoment2 = _moment2[ksub + jsub];
if ((ksub == k) && (jsub == j) && (_denoiseFrameCnt == 0))
{
tmp = ((uint32_t)buffer[k + j] *
(uint32_t)buffer[k + j]);
tmpMoment2 *= kDenoiseFiltParam; // Q16
tmpMoment2 += ((kDenoiseFiltParamRec * tmp)<<8);
tmpMoment2 >>= 8; // Q8
}
_moment2[k + j] = tmpMoment2;
/* Current event = deviation from mean value */
diff0 = ((int32_t)buffer[k + j] << 8) - _moment1[k + j];
/* Recent events = variance (variations over time) */
thevar = _moment2[k + j];
thevar -= ((_moment1[k + j] * _moment1[k + j]) >> 8);
/***************************************************************************
* De-noising criteria, i.e., when should we replace a pixel by its mean
*
* 1) recent events are minor
* 2) current events are minor
***************************************************************************/
if ((thevar < kDenoiseThreshold)
&& ((diff0 * diff0 >> 8) < kDenoiseThreshold))
{ // Replace with mean
buffer[k + j] = (uint8_t)(_moment1[k + j] >> 8);
numPixelsChanged++;
}
}
}
/* Update frame counter */
_denoiseFrameCnt++;
if (_denoiseFrameCnt > kSubsamplingTime)
{
_denoiseFrameCnt = 0;
}
return numPixelsChanged;
} }
} // namespace } // namespace

40
webrtc/modules/video_processing/main/source/denoising.h
View File

@ -8,39 +8,35 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_DENOISING_H_
* denoising.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_DENOISING_H_
*/
#ifndef VPM_DENOISING_H
#define VPM_DENOISING_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
class VPMDenoising class VPMDenoising {
{ public:
public: VPMDenoising();
VPMDenoising(); ~VPMDenoising();
~VPMDenoising();
int32_t ChangeUniqueId(int32_t id); int32_t ChangeUniqueId(int32_t id);
void Reset(); void Reset();
int32_t ProcessFrame(I420VideoFrame* frame); int32_t ProcessFrame(I420VideoFrame* frame);
private: private:
int32_t _id; int32_t id_;
uint32_t* _moment1; // (Q8) First order moment (mean) uint32_t* moment1_; // (Q8) First order moment (mean).
uint32_t* _moment2; // (Q8) Second order moment uint32_t* moment2_; // (Q8) Second order moment.
uint32_t _frameSize; // Size (# of pixels) of frame uint32_t frame_size_; // Size (# of pixels) of frame.
int _denoiseFrameCnt; // Counter for subsampling in time int denoise_frame_cnt_; // Counter for subsampling in time.
}; };
} // namespace } // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_DENOISING_H_
#endif // VPM_DENOISING_H

276
webrtc/modules/video_processing/main/source/frame_preprocessor.cc
View File

@ -13,177 +13,137 @@
namespace webrtc { namespace webrtc {
VPMFramePreprocessor::VPMFramePreprocessor(): VPMFramePreprocessor::VPMFramePreprocessor()
_id(0), : id_(0),
_contentMetrics(NULL), content_metrics_(NULL),
_maxFrameRate(0), max_frame_rate_(0),
_resampledFrame(), resampled_frame_(),
_enableCA(false), enable_ca_(false),
_frameCnt(0) frame_cnt_(0) {
{ spatial_resampler_ = new VPMSimpleSpatialResampler();
_spatialResampler = new VPMSimpleSpatialResampler(); ca_ = new VPMContentAnalysis(true);
_ca = new VPMContentAnalysis(true); vd_ = new VPMVideoDecimator();
_vd = new VPMVideoDecimator();
} }
VPMFramePreprocessor::~VPMFramePreprocessor() VPMFramePreprocessor::~VPMFramePreprocessor() {
{ Reset();
Reset(); delete spatial_resampler_;
delete _spatialResampler; delete ca_;
delete _ca; delete vd_;
delete _vd;
} }
int32_t int32_t VPMFramePreprocessor::ChangeUniqueId(const int32_t id) {
VPMFramePreprocessor::ChangeUniqueId(const int32_t id) id_ = id;
{ return VPM_OK;
_id = id;
return VPM_OK;
} }
void void VPMFramePreprocessor::Reset() {
VPMFramePreprocessor::Reset() ca_->Release();
{ vd_->Reset();
_ca->Release(); content_metrics_ = NULL;
_vd->Reset(); spatial_resampler_->Reset();
_contentMetrics = NULL; enable_ca_ = false;
_spatialResampler->Reset(); frame_cnt_ = 0;
_enableCA = false;
_frameCnt = 0;
}
void
VPMFramePreprocessor::EnableTemporalDecimation(bool enable)
{
_vd->EnableTemporalDecimation(enable);
}
void
VPMFramePreprocessor::EnableContentAnalysis(bool enable)
{
_enableCA = enable;
} }
void
VPMFramePreprocessor::SetInputFrameResampleMode(VideoFrameResampling resamplingMode) void VPMFramePreprocessor::EnableTemporalDecimation(bool enable) {
{ vd_->EnableTemporalDecimation(enable);
_spatialResampler->SetInputFrameResampleMode(resamplingMode);
} }
void VPMFramePreprocessor::EnableContentAnalysis(bool enable) {
int32_t enable_ca_ = enable;
VPMFramePreprocessor::SetMaxFrameRate(uint32_t maxFrameRate) }
{
if (maxFrameRate == 0) void VPMFramePreprocessor::SetInputFrameResampleMode(
{ VideoFrameResampling resampling_mode) {
return VPM_PARAMETER_ERROR; spatial_resampler_->SetInputFrameResampleMode(resampling_mode);
}
int32_t VPMFramePreprocessor::SetMaxFramerate(uint32_t max_frame_rate) {
if (max_frame_rate == 0) return VPM_PARAMETER_ERROR;
// Max allowed frame_rate.
max_frame_rate_ = max_frame_rate;
return vd_->SetMaxFramerate(max_frame_rate);
}
int32_t VPMFramePreprocessor::SetTargetResolution(
uint32_t width, uint32_t height, uint32_t frame_rate) {
if ( (width == 0) || (height == 0) || (frame_rate == 0)) {
return VPM_PARAMETER_ERROR;
}
int32_t ret_val = 0;
ret_val = spatial_resampler_->SetTargetFrameSize(width, height);
if (ret_val < 0) return ret_val;
ret_val = vd_->SetTargetframe_rate(frame_rate);
if (ret_val < 0) return ret_val;
return VPM_OK;
}
void VPMFramePreprocessor::UpdateIncomingframe_rate() {
vd_->UpdateIncomingframe_rate();
}
uint32_t VPMFramePreprocessor::Decimatedframe_rate() {
return vd_->Decimatedframe_rate();
}
uint32_t VPMFramePreprocessor::DecimatedWidth() const {
return spatial_resampler_->TargetWidth();
}
uint32_t VPMFramePreprocessor::DecimatedHeight() const {
return spatial_resampler_->TargetHeight();
}
int32_t VPMFramePreprocessor::PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processed_frame) {
if (frame.IsZeroSize()) {
return VPM_PARAMETER_ERROR;
}
vd_->UpdateIncomingframe_rate();
if (vd_->DropFrame()) {
WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, id_,
"Drop frame due to frame rate");
return 1; // drop 1 frame
}
// Resizing incoming frame if needed. Otherwise, remains NULL.
// We are not allowed to resample the input frame (must make a copy of it).
*processed_frame = NULL;
if (spatial_resampler_->ApplyResample(frame.width(), frame.height())) {
int32_t ret = spatial_resampler_->ResampleFrame(frame, &resampled_frame_);
if (ret != VPM_OK) return ret;
*processed_frame = &resampled_frame_;
}
// Perform content analysis on the frame to be encoded.
if (enable_ca_) {
// Compute new metrics every |kSkipFramesCA| frames, starting with
// the first frame.
if (frame_cnt_ % kSkipFrameCA == 0) {
if (*processed_frame == NULL) {
content_metrics_ = ca_->ComputeContentMetrics(frame);
} else {
content_metrics_ = ca_->ComputeContentMetrics(resampled_frame_);
}
} }
//Max allowed frame rate ++frame_cnt_;
_maxFrameRate = maxFrameRate; }
return VPM_OK;
return _vd->SetMaxFrameRate(maxFrameRate);
}
int32_t
VPMFramePreprocessor::SetTargetResolution(uint32_t width, uint32_t height, uint32_t frameRate)
{
if ( (width == 0) || (height == 0) || (frameRate == 0))
{
return VPM_PARAMETER_ERROR;
}
int32_t retVal = 0;
retVal = _spatialResampler->SetTargetFrameSize(width, height);
if (retVal < 0)
{
return retVal;
}
retVal = _vd->SetTargetFrameRate(frameRate);
if (retVal < 0)
{
return retVal;
}
return VPM_OK;
} }
void VideoContentMetrics* VPMFramePreprocessor::ContentMetrics() const {
VPMFramePreprocessor::UpdateIncomingFrameRate() return content_metrics_;
{
_vd->UpdateIncomingFrameRate();
}
uint32_t
VPMFramePreprocessor::DecimatedFrameRate()
{
return _vd->DecimatedFrameRate();
}
uint32_t
VPMFramePreprocessor::DecimatedWidth() const
{
return _spatialResampler->TargetWidth();
}
uint32_t
VPMFramePreprocessor::DecimatedHeight() const
{
return _spatialResampler->TargetHeight();
}
int32_t
VPMFramePreprocessor::PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processedFrame)
{
if (frame.IsZeroSize())
{
return VPM_PARAMETER_ERROR;
}
_vd->UpdateIncomingFrameRate();
if (_vd->DropFrame())
{
WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, _id,
"Drop frame due to frame rate");
return 1; // drop 1 frame
}
// Resizing incoming frame if needed. Otherwise, remains NULL.
// We are not allowed to resample the input frame (must make a copy of it).
*processedFrame = NULL;
if (_spatialResampler->ApplyResample(frame.width(), frame.height())) {
int32_t ret = _spatialResampler->ResampleFrame(frame, &_resampledFrame);
if (ret != VPM_OK)
return ret;
*processedFrame = &_resampledFrame;
}
// Perform content analysis on the frame to be encoded.
if (_enableCA)
{
// Compute new metrics every |kSkipFramesCA| frames, starting with
// the first frame.
if (_frameCnt % kSkipFrameCA == 0) {
if (*processedFrame == NULL) {
_contentMetrics = _ca->ComputeContentMetrics(frame);
} else {
_contentMetrics = _ca->ComputeContentMetrics(_resampledFrame);
}
}
++_frameCnt;
}
return VPM_OK;
}
VideoContentMetrics*
VPMFramePreprocessor::ContentMetrics() const
{
return _contentMetrics;
} }
} // namespace } // namespace

93
webrtc/modules/video_processing/main/source/frame_preprocessor.h
View File

@ -11,8 +11,8 @@
/* /*
* frame_preprocessor.h * frame_preprocessor.h
*/ */
#ifndef VPM_FRAME_PREPROCESSOR_H #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_FRAME_PREPROCESSOR_H
#define VPM_FRAME_PREPROCESSOR_H #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_FRAME_PREPROCESSOR_H
#include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h"
#include "webrtc/modules/video_processing/main/source/content_analysis.h" #include "webrtc/modules/video_processing/main/source/content_analysis.h"
@ -22,65 +22,62 @@
namespace webrtc { namespace webrtc {
class VPMFramePreprocessor {
public:
VPMFramePreprocessor();
~VPMFramePreprocessor();
class VPMFramePreprocessor int32_t ChangeUniqueId(const int32_t id);
{
public:
VPMFramePreprocessor(); void Reset();
~VPMFramePreprocessor();
int32_t ChangeUniqueId(const int32_t id); // Enable temporal decimation.
void EnableTemporalDecimation(bool enable);
void Reset(); void SetInputFrameResampleMode(VideoFrameResampling resampling_mode);
// Enable temporal decimation // Enable content analysis.
void EnableTemporalDecimation(bool enable); void EnableContentAnalysis(bool enable);
void SetInputFrameResampleMode(VideoFrameResampling resamplingMode); // Set max frame rate.
int32_t SetMaxFramerate(uint32_t max_frame_rate);
//Enable content analysis // Set target resolution: frame rate and dimension.
void EnableContentAnalysis(bool enable); int32_t SetTargetResolution(uint32_t width, uint32_t height,
uint32_t frame_rate);
//Set max frame rate // Update incoming frame rate/dimension.
int32_t SetMaxFrameRate(uint32_t maxFrameRate); void UpdateIncomingframe_rate();
//Set target resolution: frame rate and dimension int32_t updateIncomingFrameSize(uint32_t width, uint32_t height);
int32_t SetTargetResolution(uint32_t width, uint32_t height,
uint32_t frameRate);
//Update incoming frame rate/dimension // Set decimated values: frame rate/dimension.
void UpdateIncomingFrameRate(); uint32_t Decimatedframe_rate();
uint32_t DecimatedWidth() const;
uint32_t DecimatedHeight() const;
int32_t updateIncomingFrameSize(uint32_t width, uint32_t height); // Preprocess output:
int32_t PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processed_frame);
VideoContentMetrics* ContentMetrics() const;
//Set decimated values: frame rate/dimension private:
uint32_t DecimatedFrameRate(); // The content does not change so much every frame, so to reduce complexity
uint32_t DecimatedWidth() const; // we can compute new content metrics every |kSkipFrameCA| frames.
uint32_t DecimatedHeight() const; enum { kSkipFrameCA = 2 };
//Preprocess output: int32_t id_;
int32_t PreprocessFrame(const I420VideoFrame& frame, VideoContentMetrics* content_metrics_;
I420VideoFrame** processedFrame); uint32_t max_frame_rate_;
VideoContentMetrics* ContentMetrics() const; I420VideoFrame resampled_frame_;
VPMSpatialResampler* spatial_resampler_;
VPMContentAnalysis* ca_;
VPMVideoDecimator* vd_;
bool enable_ca_;
int frame_cnt_;
private: };
// The content does not change so much every frame, so to reduce complexity
// we can compute new content metrics every |kSkipFrameCA| frames.
enum { kSkipFrameCA = 2 };
int32_t _id; } // namespace webrtc
VideoContentMetrics* _contentMetrics;
uint32_t _maxFrameRate;
I420VideoFrame _resampledFrame;
VPMSpatialResampler* _spatialResampler;
VPMContentAnalysis* _ca;
VPMVideoDecimator* _vd;
bool _enableCA;
int _frameCnt;
}; // end of VPMFramePreprocessor class definition
} // namespace #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_FRAME_PREPROCESSOR_H
#endif // VPM_FRAME_PREPROCESS_H

104
webrtc/modules/video_processing/main/source/spatial_resampler.cc
View File

@ -14,109 +14,85 @@
namespace webrtc { namespace webrtc {
VPMSimpleSpatialResampler::VPMSimpleSpatialResampler() VPMSimpleSpatialResampler::VPMSimpleSpatialResampler()
: : resampling_mode_(kFastRescaling),
_resamplingMode(kFastRescaling), target_width_(0),
_targetWidth(0), target_height_(0),
_targetHeight(0), scaler_() {}
_scaler()
{
}
VPMSimpleSpatialResampler::~VPMSimpleSpatialResampler() VPMSimpleSpatialResampler::~VPMSimpleSpatialResampler() {}
{
//
}
int32_t int32_t VPMSimpleSpatialResampler::SetTargetFrameSize(int32_t width,
VPMSimpleSpatialResampler::SetTargetFrameSize(int32_t width, int32_t height) {
int32_t height) if (resampling_mode_ == kNoRescaling) return VPM_OK;
{
if (_resamplingMode == kNoRescaling) {
return VPM_OK;
}
if (width < 1 || height < 1) { if (width < 1 || height < 1) return VPM_PARAMETER_ERROR;
return VPM_PARAMETER_ERROR;
}
_targetWidth = width; target_width_ = width;
_targetHeight = height; target_height_ = height;
return VPM_OK; return VPM_OK;
} }
void void VPMSimpleSpatialResampler::SetInputFrameResampleMode(
VPMSimpleSpatialResampler::SetInputFrameResampleMode(VideoFrameResampling VideoFrameResampling resampling_mode) {
resamplingMode) resampling_mode_ = resampling_mode;
{
_resamplingMode = resamplingMode;
} }
void void VPMSimpleSpatialResampler::Reset() {
VPMSimpleSpatialResampler::Reset() resampling_mode_ = kFastRescaling;
{ target_width_ = 0;
_resamplingMode = kFastRescaling; target_height_ = 0;
_targetWidth = 0;
_targetHeight = 0;
} }
int32_t int32_t VPMSimpleSpatialResampler::ResampleFrame(const I420VideoFrame& inFrame,
VPMSimpleSpatialResampler::ResampleFrame(const I420VideoFrame& inFrame, I420VideoFrame* outFrame) {
I420VideoFrame* outFrame)
{
// Don't copy if frame remains as is. // Don't copy if frame remains as is.
if (_resamplingMode == kNoRescaling) if (resampling_mode_ == kNoRescaling)
return VPM_OK; return VPM_OK;
// Check if re-sampling is needed // Check if re-sampling is needed
else if ((inFrame.width() == _targetWidth) && else if ((inFrame.width() == target_width_) &&
(inFrame.height() == _targetHeight)) { (inFrame.height() == target_height_)) {
return VPM_OK; return VPM_OK;
} }
// Setting scaler // Setting scaler
// TODO(mikhal/marpan): Should we allow for setting the filter mode in // TODO(mikhal/marpan): Should we allow for setting the filter mode in
// _scale.Set() with |_resamplingMode|? // _scale.Set() with |resampling_mode_|?
int retVal = 0; int ret_val = 0;
retVal = _scaler.Set(inFrame.width(), inFrame.height(), ret_val = scaler_.Set(inFrame.width(), inFrame.height(),
_targetWidth, _targetHeight, kI420, kI420, kScaleBox); target_width_, target_height_, kI420, kI420, kScaleBox);
if (retVal < 0) if (ret_val < 0)
return retVal; return ret_val;
retVal = _scaler.Scale(inFrame, outFrame); ret_val = scaler_.Scale(inFrame, outFrame);
// Setting time parameters to the output frame. // Setting time parameters to the output frame.
// Timestamp will be reset in Scale call above, so we should set it after. // Timestamp will be reset in Scale call above, so we should set it after.
outFrame->set_timestamp(inFrame.timestamp()); outFrame->set_timestamp(inFrame.timestamp());
outFrame->set_render_time_ms(inFrame.render_time_ms()); outFrame->set_render_time_ms(inFrame.render_time_ms());
if (retVal == 0) if (ret_val == 0)
return VPM_OK; return VPM_OK;
else else
return VPM_SCALE_ERROR; return VPM_SCALE_ERROR;
} }
int32_t int32_t VPMSimpleSpatialResampler::TargetHeight() {
VPMSimpleSpatialResampler::TargetHeight() return target_height_;
{
return _targetHeight;
} }
int32_t int32_t VPMSimpleSpatialResampler::TargetWidth() {
VPMSimpleSpatialResampler::TargetWidth() return target_width_;
{
return _targetWidth;
} }
bool bool VPMSimpleSpatialResampler::ApplyResample(int32_t width,
VPMSimpleSpatialResampler::ApplyResample(int32_t width, int32_t height) {
int32_t height) if ((width == target_width_ && height == target_height_) ||
{ resampling_mode_ == kNoRescaling)
if ((width == _targetWidth && height == _targetHeight) ||
_resamplingMode == kNoRescaling)
return false; return false;
else else
return true; return true;
} }
} // namespace } // namespace webrtc

36
webrtc/modules/video_processing/main/source/spatial_resampler.h
View File

@ -8,12 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_SPATIAL_RESAMPLER_H
* spatial_resampler.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_SPATIAL_RESAMPLER_H
*/
#ifndef VPM_SPATIAL_RESAMPLER_H
#define VPM_SPATIAL_RESAMPLER_H
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
@ -25,13 +21,12 @@
namespace webrtc { namespace webrtc {
class VPMSpatialResampler class VPMSpatialResampler {
{ public:
public:
virtual ~VPMSpatialResampler() {}; virtual ~VPMSpatialResampler() {};
virtual int32_t SetTargetFrameSize(int32_t width, int32_t height) = 0; virtual int32_t SetTargetFrameSize(int32_t width, int32_t height) = 0;
virtual void SetInputFrameResampleMode(VideoFrameResampling virtual void SetInputFrameResampleMode(VideoFrameResampling
resamplingMode) = 0; resampling_mode) = 0;
virtual void Reset() = 0; virtual void Reset() = 0;
virtual int32_t ResampleFrame(const I420VideoFrame& inFrame, virtual int32_t ResampleFrame(const I420VideoFrame& inFrame,
I420VideoFrame* outFrame) = 0; I420VideoFrame* outFrame) = 0;
@ -40,13 +35,12 @@ public:
virtual bool ApplyResample(int32_t width, int32_t height) = 0; virtual bool ApplyResample(int32_t width, int32_t height) = 0;
}; };
class VPMSimpleSpatialResampler : public VPMSpatialResampler class VPMSimpleSpatialResampler : public VPMSpatialResampler {
{ public:
public:
VPMSimpleSpatialResampler(); VPMSimpleSpatialResampler();
~VPMSimpleSpatialResampler(); ~VPMSimpleSpatialResampler();
virtual int32_t SetTargetFrameSize(int32_t width, int32_t height); virtual int32_t SetTargetFrameSize(int32_t width, int32_t height);
virtual void SetInputFrameResampleMode(VideoFrameResampling resamplingMode); virtual void SetInputFrameResampleMode(VideoFrameResampling resampling_mode);
virtual void Reset(); virtual void Reset();
virtual int32_t ResampleFrame(const I420VideoFrame& inFrame, virtual int32_t ResampleFrame(const I420VideoFrame& inFrame,
I420VideoFrame* outFrame); I420VideoFrame* outFrame);
@ -54,14 +48,14 @@ public:
virtual int32_t TargetHeight(); virtual int32_t TargetHeight();
virtual bool ApplyResample(int32_t width, int32_t height); virtual bool ApplyResample(int32_t width, int32_t height);
private: private:
VideoFrameResampling _resamplingMode; VideoFrameResampling resampling_mode_;
int32_t _targetWidth; int32_t target_width_;
int32_t _targetHeight; int32_t target_height_;
Scaler _scaler; Scaler scaler_;
}; };
} // namespace } // namespace webrtc
#endif #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_SPATIAL_RESAMPLER_H

311
webrtc/modules/video_processing/main/source/video_decimator.cc
View File

@ -17,219 +17,156 @@
namespace webrtc { namespace webrtc {
VPMVideoDecimator::VPMVideoDecimator() VPMVideoDecimator::VPMVideoDecimator()
: : overshoot_modifier_(0),
_overShootModifier(0), drop_count_(0),
_dropCount(0), keep_count_(0),
_keepCount(0), target_frame_rate_(30),
_targetFrameRate(30), incoming_frame_rate_(0.0f),
_incomingFrameRate(0.0f), max_frame_rate_(30),
_maxFrameRate(30), incoming_frame_times_(),
_incomingFrameTimes(), enable_temporal_decimation_(true) {
_enableTemporalDecimation(true) Reset();
{
Reset();
} }
VPMVideoDecimator::~VPMVideoDecimator() VPMVideoDecimator::~VPMVideoDecimator() {}
{
// void VPMVideoDecimator::Reset() {
overshoot_modifier_ = 0;
drop_count_ = 0;
keep_count_ = 0;
target_frame_rate_ = 30;
incoming_frame_rate_ = 0.0f;
max_frame_rate_ = 30;
memset(incoming_frame_times_, 0, sizeof(incoming_frame_times_));
enable_temporal_decimation_ = true;
} }
void void VPMVideoDecimator::EnableTemporalDecimation(bool enable) {
VPMVideoDecimator::Reset() enable_temporal_decimation_ = enable;
{
_overShootModifier = 0;
_dropCount = 0;
_keepCount = 0;
_targetFrameRate = 30;
_incomingFrameRate = 0.0f;
_maxFrameRate = 30;
memset(_incomingFrameTimes, 0, sizeof(_incomingFrameTimes));
_enableTemporalDecimation = true;
} }
void int32_t VPMVideoDecimator::SetMaxFramerate(uint32_t max_frame_rate) {
VPMVideoDecimator::EnableTemporalDecimation(bool enable) if (max_frame_rate == 0) return VPM_PARAMETER_ERROR;
{
_enableTemporalDecimation = enable;
}
int32_t
VPMVideoDecimator::SetMaxFrameRate(uint32_t maxFrameRate)
{
if (maxFrameRate == 0)
{
return VPM_PARAMETER_ERROR;
}
_maxFrameRate = maxFrameRate; max_frame_rate_ = max_frame_rate;
if (_targetFrameRate > _maxFrameRate)
{
_targetFrameRate = _maxFrameRate;
} if (target_frame_rate_ > max_frame_rate_)
return VPM_OK; target_frame_rate_ = max_frame_rate_;
return VPM_OK;
} }
int32_t int32_t VPMVideoDecimator::SetTargetframe_rate(uint32_t frame_rate) {
VPMVideoDecimator::SetTargetFrameRate(uint32_t frameRate) if (frame_rate == 0) return VPM_PARAMETER_ERROR;
{
if (frameRate == 0) if (frame_rate > max_frame_rate_) {
{ // Override.
return VPM_PARAMETER_ERROR; target_frame_rate_ = max_frame_rate_;
} } else {
if (frameRate > _maxFrameRate) target_frame_rate_ = frame_rate;
{ }
//override return VPM_OK;
_targetFrameRate = _maxFrameRate;
}
else
{
_targetFrameRate = frameRate;
}
return VPM_OK;
} }
bool bool VPMVideoDecimator::DropFrame() {
VPMVideoDecimator::DropFrame() if (!enable_temporal_decimation_) return false;
{
if (!_enableTemporalDecimation) if (incoming_frame_rate_ <= 0) return false;
{
return false; const uint32_t incomingframe_rate =
static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
if (target_frame_rate_ == 0) return true;
bool drop = false;
if (incomingframe_rate > target_frame_rate_) {
int32_t overshoot =
overshoot_modifier_ + (incomingframe_rate - target_frame_rate_);
if (overshoot < 0) {
overshoot = 0;
overshoot_modifier_ = 0;
} }
if (_incomingFrameRate <= 0) if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) {
{ if (drop_count_) { // Just got here so drop to be sure.
return false; drop_count_ = 0;
return true;
}
const uint32_t dropVar = incomingframe_rate / overshoot;
if (keep_count_ >= dropVar) {
drop = true;
overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3;
keep_count_ = 1;
} else {
keep_count_++;
}
} else {
keep_count_ = 0;
const uint32_t dropVar = overshoot / target_frame_rate_;
if (drop_count_ < dropVar) {
drop = true;
drop_count_++;
} else {
overshoot_modifier_ = overshoot % target_frame_rate_;
drop = false;
drop_count_ = 0;
}
} }
}
const uint32_t incomingFrameRate = static_cast<uint32_t>(_incomingFrameRate + 0.5f); return drop;
if (_targetFrameRate == 0)
{
return true;
}
bool drop = false;
if (incomingFrameRate > _targetFrameRate)
{
int32_t overshoot = _overShootModifier + (incomingFrameRate - _targetFrameRate);
if(overshoot < 0)
{
overshoot = 0;
_overShootModifier = 0;
}
if (overshoot && 2 * overshoot < (int32_t) incomingFrameRate)
{
if (_dropCount) // Just got here so drop to be sure.
{
_dropCount = 0;
return true;
}
const uint32_t dropVar = incomingFrameRate / overshoot;
if (_keepCount >= dropVar)
{
drop = true;
_overShootModifier = -((int32_t) incomingFrameRate % overshoot) / 3;
_keepCount = 1;
}
else
{
_keepCount++;
}
}
else
{
_keepCount = 0;
const uint32_t dropVar = overshoot / _targetFrameRate;
if (_dropCount < dropVar)
{
drop = true;
_dropCount++;
}
else
{
_overShootModifier = overshoot % _targetFrameRate;
drop = false;
_dropCount = 0;
}
}
}
return drop;
} }
uint32_t uint32_t VPMVideoDecimator::Decimatedframe_rate() {
VPMVideoDecimator::DecimatedFrameRate() ProcessIncomingframe_rate(TickTime::MillisecondTimestamp());
{ if (!enable_temporal_decimation_) {
ProcessIncomingFrameRate(TickTime::MillisecondTimestamp()); return static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
if (!_enableTemporalDecimation) }
{ return VD_MIN(target_frame_rate_,
return static_cast<uint32_t>(_incomingFrameRate + 0.5f); static_cast<uint32_t>(incoming_frame_rate_ + 0.5f));
}
return VD_MIN(_targetFrameRate, static_cast<uint32_t>(_incomingFrameRate + 0.5f));
} }
uint32_t uint32_t VPMVideoDecimator::Inputframe_rate() {
VPMVideoDecimator::InputFrameRate() ProcessIncomingframe_rate(TickTime::MillisecondTimestamp());
{ return static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
ProcessIncomingFrameRate(TickTime::MillisecondTimestamp());
return static_cast<uint32_t>(_incomingFrameRate + 0.5f);
} }
void void VPMVideoDecimator::UpdateIncomingframe_rate() {
VPMVideoDecimator::UpdateIncomingFrameRate() int64_t now = TickTime::MillisecondTimestamp();
{ if (incoming_frame_times_[0] == 0) {
int64_t now = TickTime::MillisecondTimestamp(); // First no shift.
if(_incomingFrameTimes[0] == 0) } else {
{ // Shift.
// first no shift for (int i = kFrameCountHistory_size - 2; i >= 0; i--) {
} else incoming_frame_times_[i+1] = incoming_frame_times_[i];
{
// shift
for(int i = (kFrameCountHistorySize - 2); i >= 0 ; i--)
{
_incomingFrameTimes[i+1] = _incomingFrameTimes[i];
}
} }
_incomingFrameTimes[0] = now; }
ProcessIncomingFrameRate(now); incoming_frame_times_[0] = now;
ProcessIncomingframe_rate(now);
} }
void void VPMVideoDecimator::ProcessIncomingframe_rate(int64_t now) {
VPMVideoDecimator::ProcessIncomingFrameRate(int64_t now) int32_t num = 0;
{ int32_t nrOfFrames = 0;
int32_t num = 0; for (num = 1; num < (kFrameCountHistory_size - 1); num++) {
int32_t nrOfFrames = 0; // Don't use data older than 2sec.
for(num = 1; num < (kFrameCountHistorySize - 1); num++) if (incoming_frame_times_[num] <= 0 ||
{ now - incoming_frame_times_[num] > kFrameHistoryWindowMs) {
if (_incomingFrameTimes[num] <= 0 || break;
now - _incomingFrameTimes[num] > kFrameHistoryWindowMs) // don't use data older than 2sec } else {
{ nrOfFrames++;
break;
} else
{
nrOfFrames++;
}
} }
if (num > 1) }
{ if (num > 1) {
int64_t diff = now - _incomingFrameTimes[num-1]; int64_t diff = now - incoming_frame_times_[num-1];
_incomingFrameRate = 1.0; incoming_frame_rate_ = 1.0;
if(diff >0) if (diff > 0) {
{ incoming_frame_rate_ = nrOfFrames * 1000.0f / static_cast<float>(diff);
_incomingFrameRate = nrOfFrames * 1000.0f / static_cast<float>(diff);
}
}
else
{
_incomingFrameRate = static_cast<float>(nrOfFrames);
} }
} else {
incoming_frame_rate_ = static_cast<float>(nrOfFrames);
}
} }
} // namespace } // namespace webrtc

73
webrtc/modules/video_processing/main/source/video_decimator.h
View File

@ -8,58 +8,53 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
/* #ifndef WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_VIDEO_DECIMATOR_H
* video_decimator.h #define WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_VIDEO_DECIMATOR_H
*/
#ifndef VPM_VIDEO_DECIMATOR_H
#define VPM_VIDEO_DECIMATOR_H
#include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/interface/module_common_types.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
namespace webrtc { namespace webrtc {
class VPMVideoDecimator class VPMVideoDecimator {
{ public:
public: VPMVideoDecimator();
VPMVideoDecimator(); ~VPMVideoDecimator();
~VPMVideoDecimator();
void Reset();
void EnableTemporalDecimation(bool enable);
int32_t SetMaxFrameRate(uint32_t maxFrameRate);
int32_t SetTargetFrameRate(uint32_t frameRate);
bool DropFrame(); void Reset();
void UpdateIncomingFrameRate();
// Get Decimated Frame Rate/Dimensions void EnableTemporalDecimation(bool enable);
uint32_t DecimatedFrameRate();
//Get input frame rate int32_t SetMaxFramerate(uint32_t max_frame_rate);
uint32_t InputFrameRate(); int32_t SetTargetframe_rate(uint32_t frame_rate);
private: bool DropFrame();
void ProcessIncomingFrameRate(int64_t now);
enum { kFrameCountHistorySize = 90}; void UpdateIncomingframe_rate();
enum { kFrameHistoryWindowMs = 2000};
// Temporal decimation // Get Decimated Frame Rate/Dimensions.
int32_t _overShootModifier; uint32_t Decimatedframe_rate();
uint32_t _dropCount;
uint32_t _keepCount;
uint32_t _targetFrameRate;
float _incomingFrameRate;
uint32_t _maxFrameRate;
int64_t _incomingFrameTimes[kFrameCountHistorySize];
bool _enableTemporalDecimation;
// Get input frame rate.
uint32_t Inputframe_rate();
private:
void ProcessIncomingframe_rate(int64_t now);
enum { kFrameCountHistory_size = 90};
enum { kFrameHistoryWindowMs = 2000};
// Temporal decimation.
int32_t overshoot_modifier_;
uint32_t drop_count_;
uint32_t keep_count_;
uint32_t target_frame_rate_;
float incoming_frame_rate_;
uint32_t max_frame_rate_;
int64_t incoming_frame_times_[kFrameCountHistory_size];
bool enable_temporal_decimation_;
}; };
} // namespace } // namespace webrtc
#endif #endif // WEBRTC_MODULES_VIDEO_PROCESSING_MAIN_SOURCE_VIDEO_DECIMATOR_H

467
webrtc/modules/video_processing/main/source/video_processing_impl.cc
View File

@ -17,277 +17,204 @@
namespace webrtc { namespace webrtc {
namespace namespace {
{ void SetSubSampling(VideoProcessingModule::FrameStats* stats,
void const int32_t width,
SetSubSampling(VideoProcessingModule::FrameStats* stats, const int32_t height) {
const int32_t width, if (width * height >= 640 * 480) {
const int32_t height) stats->subSamplWidth = 3;
{ stats->subSamplHeight = 3;
if (width * height >= 640 * 480) } else if (width * height >= 352 * 288) {
{ stats->subSamplWidth = 2;
stats->subSamplWidth = 3; stats->subSamplHeight = 2;
stats->subSamplHeight = 3; } else if (width * height >= 176 * 144) {
} stats->subSamplWidth = 1;
else if (width * height >= 352 * 288) stats->subSamplHeight = 1;
{ } else {
stats->subSamplWidth = 2;
stats->subSamplHeight = 2;
}
else if (width * height >= 176 * 144)
{
stats->subSamplWidth = 1;
stats->subSamplHeight = 1;
}
else
{
stats->subSamplWidth = 0;
stats->subSamplHeight = 0;
}
}
}
VideoProcessingModule*
VideoProcessingModule::Create(const int32_t id)
{
return new VideoProcessingModuleImpl(id);
}
void
VideoProcessingModule::Destroy(VideoProcessingModule* module)
{
if (module)
{
delete static_cast<VideoProcessingModuleImpl*>(module);
}
}
int32_t
VideoProcessingModuleImpl::ChangeUniqueId(const int32_t id)
{
CriticalSectionScoped mutex(&_mutex);
_id = id;
_brightnessDetection.ChangeUniqueId(id);
_deflickering.ChangeUniqueId(id);
_denoising.ChangeUniqueId(id);
_framePreProcessor.ChangeUniqueId(id);
return VPM_OK;
}
int32_t
VideoProcessingModuleImpl::Id() const
{
CriticalSectionScoped mutex(&_mutex);
return _id;
}
VideoProcessingModuleImpl::VideoProcessingModuleImpl(const int32_t id) :
_id(id),
_mutex(*CriticalSectionWrapper::CreateCriticalSection())
{
_brightnessDetection.ChangeUniqueId(id);
_deflickering.ChangeUniqueId(id);
_denoising.ChangeUniqueId(id);
_framePreProcessor.ChangeUniqueId(id);
WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, _id,
"Created");
}
VideoProcessingModuleImpl::~VideoProcessingModuleImpl()
{
WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, _id,
"Destroyed");
delete &_mutex;
}
void
VideoProcessingModuleImpl::Reset()
{
CriticalSectionScoped mutex(&_mutex);
_deflickering.Reset();
_denoising.Reset();
_brightnessDetection.Reset();
_framePreProcessor.Reset();
}
int32_t
VideoProcessingModule::GetFrameStats(FrameStats* stats,
const I420VideoFrame& frame)
{
if (frame.IsZeroSize())
{
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, -1,
"zero size frame");
return VPM_PARAMETER_ERROR;
}
int width = frame.width();
int height = frame.height();
ClearFrameStats(stats); // The histogram needs to be zeroed out.
SetSubSampling(stats, width, height);
const uint8_t* buffer = frame.buffer(kYPlane);
// Compute histogram and sum of frame
for (int i = 0; i < height; i += (1 << stats->subSamplHeight))
{
int k = i * width;
for (int j = 0; j < width; j += (1 << stats->subSamplWidth))
{
stats->hist[buffer[k + j]]++;
stats->sum += buffer[k + j];
}
}
stats->numPixels = (width * height) / ((1 << stats->subSamplWidth) *
(1 << stats->subSamplHeight));
assert(stats->numPixels > 0);
// Compute mean value of frame
stats->mean = stats->sum / stats->numPixels;
return VPM_OK;
}
bool
VideoProcessingModule::ValidFrameStats(const FrameStats& stats)
{
if (stats.numPixels == 0)
{
return false;
}
return true;
}
void
VideoProcessingModule::ClearFrameStats(FrameStats* stats)
{
stats->mean = 0;
stats->sum = 0;
stats->numPixels = 0;
stats->subSamplWidth = 0; stats->subSamplWidth = 0;
stats->subSamplHeight = 0; stats->subSamplHeight = 0;
memset(stats->hist, 0, sizeof(stats->hist)); }
} }
int32_t
VideoProcessingModule::ColorEnhancement(I420VideoFrame* frame)
{
return VideoProcessing::ColorEnhancement(frame);
}
int32_t
VideoProcessingModule::Brighten(I420VideoFrame* frame, int delta)
{
return VideoProcessing::Brighten(frame, delta);
}
int32_t
VideoProcessingModuleImpl::Deflickering(I420VideoFrame* frame,
FrameStats* stats)
{
CriticalSectionScoped mutex(&_mutex);
return _deflickering.ProcessFrame(frame, stats);
}
int32_t
VideoProcessingModuleImpl::Denoising(I420VideoFrame* frame)
{
CriticalSectionScoped mutex(&_mutex);
return _denoising.ProcessFrame(frame);
}
int32_t
VideoProcessingModuleImpl::BrightnessDetection(const I420VideoFrame& frame,
const FrameStats& stats)
{
CriticalSectionScoped mutex(&_mutex);
return _brightnessDetection.ProcessFrame(frame, stats);
}
void
VideoProcessingModuleImpl::EnableTemporalDecimation(bool enable)
{
CriticalSectionScoped mutex(&_mutex);
_framePreProcessor.EnableTemporalDecimation(enable);
}
void
VideoProcessingModuleImpl::SetInputFrameResampleMode(VideoFrameResampling
resamplingMode)
{
CriticalSectionScoped cs(&_mutex);
_framePreProcessor.SetInputFrameResampleMode(resamplingMode);
}
int32_t
VideoProcessingModuleImpl::SetMaxFrameRate(uint32_t maxFrameRate)
{
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.SetMaxFrameRate(maxFrameRate);
}
int32_t
VideoProcessingModuleImpl::SetTargetResolution(uint32_t width,
uint32_t height,
uint32_t frameRate)
{
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.SetTargetResolution(width, height, frameRate);
}
uint32_t
VideoProcessingModuleImpl::DecimatedFrameRate()
{
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.DecimatedFrameRate();
}
uint32_t
VideoProcessingModuleImpl::DecimatedWidth() const
{
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.DecimatedWidth();
}
uint32_t
VideoProcessingModuleImpl::DecimatedHeight() const
{
CriticalSectionScoped cs(&_mutex);
return _framePreProcessor.DecimatedHeight();
}
int32_t
VideoProcessingModuleImpl::PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame **processedFrame)
{
CriticalSectionScoped mutex(&_mutex);
return _framePreProcessor.PreprocessFrame(frame, processedFrame);
}
VideoContentMetrics*
VideoProcessingModuleImpl::ContentMetrics() const
{
CriticalSectionScoped mutex(&_mutex);
return _framePreProcessor.ContentMetrics();
}
void
VideoProcessingModuleImpl::EnableContentAnalysis(bool enable)
{
CriticalSectionScoped mutex(&_mutex);
_framePreProcessor.EnableContentAnalysis(enable);
}
} // namespace } // namespace
VideoProcessingModule* VideoProcessingModule::Create(const int32_t id) {
return new VideoProcessingModuleImpl(id);
}
void VideoProcessingModule::Destroy(VideoProcessingModule* module) {
if (module)
delete static_cast<VideoProcessingModuleImpl*>(module);
}
int32_t VideoProcessingModuleImpl::ChangeUniqueId(const int32_t id) {
CriticalSectionScoped mutex(&mutex_);
id_ = id;
brightness_detection_.ChangeUniqueId(id);
deflickering_.ChangeUniqueId(id);
denoising_.ChangeUniqueId(id);
frame_pre_processor_.ChangeUniqueId(id);
return VPM_OK;
}
int32_t VideoProcessingModuleImpl::Id() const {
CriticalSectionScoped mutex(&mutex_);
return id_;
}
VideoProcessingModuleImpl::VideoProcessingModuleImpl(const int32_t id)
: id_(id),
mutex_(*CriticalSectionWrapper::CreateCriticalSection()) {
brightness_detection_.ChangeUniqueId(id);
deflickering_.ChangeUniqueId(id);
denoising_.ChangeUniqueId(id);
frame_pre_processor_.ChangeUniqueId(id);
WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, id_,
"Created");
}
VideoProcessingModuleImpl::~VideoProcessingModuleImpl() {
WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceVideoPreocessing, id_,
"Destroyed");
delete &mutex_;
}
void VideoProcessingModuleImpl::Reset() {
CriticalSectionScoped mutex(&mutex_);
deflickering_.Reset();
denoising_.Reset();
brightness_detection_.Reset();
frame_pre_processor_.Reset();
}
int32_t VideoProcessingModule::GetFrameStats(FrameStats* stats,
const I420VideoFrame& frame) {
if (frame.IsZeroSize()) {
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, -1,
"zero size frame");
return VPM_PARAMETER_ERROR;
}
int width = frame.width();
int height = frame.height();
ClearFrameStats(stats); // The histogram needs to be zeroed out.
SetSubSampling(stats, width, height);
const uint8_t* buffer = frame.buffer(kYPlane);
// Compute histogram and sum of frame
for (int i = 0; i < height; i += (1 << stats->subSamplHeight)) {
int k = i * width;
for (int j = 0; j < width; j += (1 << stats->subSamplWidth)) {
stats->hist[buffer[k + j]]++;
stats->sum += buffer[k + j];
}
}
stats->num_pixels = (width * height) / ((1 << stats->subSamplWidth) *
(1 << stats->subSamplHeight));
assert(stats->num_pixels > 0);
// Compute mean value of frame
stats->mean = stats->sum / stats->num_pixels;
return VPM_OK;
}
bool VideoProcessingModule::ValidFrameStats(const FrameStats& stats) {
if (stats.num_pixels == 0) return false;
return true;
}
void VideoProcessingModule::ClearFrameStats(FrameStats* stats) {
stats->mean = 0;
stats->sum = 0;
stats->num_pixels = 0;
stats->subSamplWidth = 0;
stats->subSamplHeight = 0;
memset(stats->hist, 0, sizeof(stats->hist));
}
int32_t VideoProcessingModule::ColorEnhancement(I420VideoFrame* frame) {
return VideoProcessing::ColorEnhancement(frame);
}
int32_t VideoProcessingModule::Brighten(I420VideoFrame* frame, int delta) {
return VideoProcessing::Brighten(frame, delta);
}
int32_t VideoProcessingModuleImpl::Deflickering(I420VideoFrame* frame,
FrameStats* stats) {
CriticalSectionScoped mutex(&mutex_);
return deflickering_.ProcessFrame(frame, stats);
}
int32_t VideoProcessingModuleImpl::Denoising(I420VideoFrame* frame) {
CriticalSectionScoped mutex(&mutex_);
return denoising_.ProcessFrame(frame);
}
int32_t VideoProcessingModuleImpl::BrightnessDetection(
const I420VideoFrame& frame,
const FrameStats& stats) {
CriticalSectionScoped mutex(&mutex_);
return brightness_detection_.ProcessFrame(frame, stats);
}
void VideoProcessingModuleImpl::EnableTemporalDecimation(bool enable) {
CriticalSectionScoped mutex(&mutex_);
frame_pre_processor_.EnableTemporalDecimation(enable);
}
void VideoProcessingModuleImpl::SetInputFrameResampleMode(VideoFrameResampling
resampling_mode) {
CriticalSectionScoped cs(&mutex_);
frame_pre_processor_.SetInputFrameResampleMode(resampling_mode);
}
int32_t VideoProcessingModuleImpl::SetMaxFramerate(uint32_t max_frame_rate) {
CriticalSectionScoped cs(&mutex_);
return frame_pre_processor_.SetMaxFramerate(max_frame_rate);
}
int32_t VideoProcessingModuleImpl::SetTargetResolution(uint32_t width,
uint32_t height,
uint32_t frame_rate) {
CriticalSectionScoped cs(&mutex_);
return frame_pre_processor_.SetTargetResolution(width, height, frame_rate);
}
uint32_t VideoProcessingModuleImpl::Decimatedframe_rate() {
CriticalSectionScoped cs(&mutex_);
return frame_pre_processor_.Decimatedframe_rate();
}
uint32_t VideoProcessingModuleImpl::DecimatedWidth() const {
CriticalSectionScoped cs(&mutex_);
return frame_pre_processor_.DecimatedWidth();
}
uint32_t VideoProcessingModuleImpl::DecimatedHeight() const {
CriticalSectionScoped cs(&mutex_);
return frame_pre_processor_.DecimatedHeight();
}
int32_t VideoProcessingModuleImpl::PreprocessFrame(
const I420VideoFrame& frame,
I420VideoFrame **processed_frame) {
CriticalSectionScoped mutex(&mutex_);
return frame_pre_processor_.PreprocessFrame(frame, processed_frame);
}
VideoContentMetrics* VideoProcessingModuleImpl::ContentMetrics() const {
CriticalSectionScoped mutex(&mutex_);
return frame_pre_processor_.ContentMetrics();
}
void VideoProcessingModuleImpl::EnableContentAnalysis(bool enable) {
CriticalSectionScoped mutex(&mutex_);
frame_pre_processor_.EnableContentAnalysis(enable);
}
} // namespace webrtc

85
webrtc/modules/video_processing/main/source/video_processing_impl.h
View File

@ -22,67 +22,64 @@
namespace webrtc { namespace webrtc {
class CriticalSectionWrapper; class CriticalSectionWrapper;
class VideoProcessingModuleImpl : public VideoProcessingModule class VideoProcessingModuleImpl : public VideoProcessingModule {
{ public:
public: VideoProcessingModuleImpl(int32_t id);
VideoProcessingModuleImpl(int32_t id); virtual ~VideoProcessingModuleImpl();
virtual ~VideoProcessingModuleImpl(); int32_t Id() const;
int32_t Id() const; virtual int32_t ChangeUniqueId(const int32_t id);
virtual int32_t ChangeUniqueId(const int32_t id); virtual void Reset();
virtual void Reset(); virtual int32_t Deflickering(I420VideoFrame* frame, FrameStats* stats);
virtual int32_t Deflickering(I420VideoFrame* frame, FrameStats* stats); virtual int32_t Denoising(I420VideoFrame* frame);
virtual int32_t Denoising(I420VideoFrame* frame); virtual int32_t BrightnessDetection(const I420VideoFrame& frame,
const FrameStats& stats);
virtual int32_t BrightnessDetection(const I420VideoFrame& frame, // Frame pre-processor functions
const FrameStats& stats);
//Frame pre-processor functions // Enable temporal decimation
virtual void EnableTemporalDecimation(bool enable);
//Enable temporal decimation virtual void SetInputFrameResampleMode(VideoFrameResampling resampling_mode);
virtual void EnableTemporalDecimation(bool enable);
virtual void SetInputFrameResampleMode(VideoFrameResampling resamplingMode); // Enable content analysis
virtual void EnableContentAnalysis(bool enable);
//Enable content analysis // Set max frame rate
virtual void EnableContentAnalysis(bool enable); virtual int32_t SetMaxFramerate(uint32_t max_frame_rate);
//Set max frame rate // Set Target Resolution: frame rate and dimension
virtual int32_t SetMaxFrameRate(uint32_t maxFrameRate); virtual int32_t SetTargetResolution(uint32_t width,
uint32_t height,
// Set Target Resolution: frame rate and dimension uint32_t frame_rate);
virtual int32_t SetTargetResolution(uint32_t width,
uint32_t height,
uint32_t frameRate);
// Get decimated values: frame rate/dimension // Get decimated values: frame rate/dimension
virtual uint32_t DecimatedFrameRate(); virtual uint32_t Decimatedframe_rate();
virtual uint32_t DecimatedWidth() const; virtual uint32_t DecimatedWidth() const;
virtual uint32_t DecimatedHeight() const; virtual uint32_t DecimatedHeight() const;
// Preprocess: // Preprocess:
// Pre-process incoming frame: Sample when needed and compute content // Pre-process incoming frame: Sample when needed and compute content
// metrics when enabled. // metrics when enabled.
// If no resampling takes place - processedFrame is set to NULL. // If no resampling takes place - processed_frame is set to NULL.
virtual int32_t PreprocessFrame(const I420VideoFrame& frame, virtual int32_t PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processedFrame); I420VideoFrame** processed_frame);
virtual VideoContentMetrics* ContentMetrics() const; virtual VideoContentMetrics* ContentMetrics() const;
private: private:
int32_t _id; int32_t id_;
CriticalSectionWrapper& _mutex; CriticalSectionWrapper& mutex_;
VPMDeflickering deflickering_;
VPMDeflickering _deflickering; VPMDenoising denoising_;
VPMDenoising _denoising; VPMBrightnessDetection brightness_detection_;
VPMBrightnessDetection _brightnessDetection; VPMFramePreprocessor frame_pre_processor_;
VPMFramePreprocessor _framePreProcessor;
}; };
} // namespace } // namespace

56
webrtc/modules/video_processing/main/test/unit_test/brightness_detection_test.cc
View File

@ -19,24 +19,24 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
uint32_t frameNum = 0; uint32_t frameNum = 0;
int32_t brightnessWarning = 0; int32_t brightnessWarning = 0;
uint32_t warningCount = 0; uint32_t warningCount = 0;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_GE(brightnessWarning = _vpm->BrightnessDetection(_videoFrame, ASSERT_GE(brightnessWarning = vpm_->BrightnessDetection(video_frame_,
stats), 0); stats), 0);
if (brightnessWarning != VideoProcessingModule::kNoWarning) if (brightnessWarning != VideoProcessingModule::kNoWarning)
{ {
warningCount++; warningCount++;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Expect few warnings // Expect few warnings
float warningProportion = static_cast<float>(warningCount) / frameNum * 100; float warningProportion = static_cast<float>(warningCount) / frameNum * 100;
@ -44,21 +44,21 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
printf("Stock foreman: %.1f %%\n", warningProportion); printf("Stock foreman: %.1f %%\n", warningProportion);
EXPECT_LT(warningProportion, 10); EXPECT_LT(warningProportion, 10);
rewind(_sourceFile); rewind(source_file_);
frameNum = 0; frameNum = 0;
warningCount = 0; warningCount = 0;
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length && frame_length_ &&
frameNum < 300) frameNum < 300)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
uint8_t* frame = _videoFrame.buffer(kYPlane); uint8_t* frame = video_frame_.buffer(kYPlane);
uint32_t yTmp = 0; uint32_t yTmp = 0;
for (int yIdx = 0; yIdx < _width * _height; yIdx++) for (int yIdx = 0; yIdx < width_ * height_; yIdx++)
{ {
yTmp = frame[yIdx] << 1; yTmp = frame[yIdx] << 1;
if (yTmp > 255) if (yTmp > 255)
@ -69,8 +69,8 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
} }
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_GE(brightnessWarning = _vpm->BrightnessDetection(_videoFrame, ASSERT_GE(brightnessWarning = vpm_->BrightnessDetection(video_frame_,
stats), 0); stats), 0);
EXPECT_NE(VideoProcessingModule::kDarkWarning, brightnessWarning); EXPECT_NE(VideoProcessingModule::kDarkWarning, brightnessWarning);
if (brightnessWarning == VideoProcessingModule::kBrightWarning) if (brightnessWarning == VideoProcessingModule::kBrightWarning)
@ -78,35 +78,35 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
warningCount++; warningCount++;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Expect many brightness warnings // Expect many brightness warnings
warningProportion = static_cast<float>(warningCount) / frameNum * 100; warningProportion = static_cast<float>(warningCount) / frameNum * 100;
printf("Bright foreman: %.1f %%\n", warningProportion); printf("Bright foreman: %.1f %%\n", warningProportion);
EXPECT_GT(warningProportion, 95); EXPECT_GT(warningProportion, 95);
rewind(_sourceFile); rewind(source_file_);
frameNum = 0; frameNum = 0;
warningCount = 0; warningCount = 0;
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length && frameNum < 300) frame_length_ && frameNum < 300)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
uint8_t* y_plane = _videoFrame.buffer(kYPlane); uint8_t* y_plane = video_frame_.buffer(kYPlane);
int32_t yTmp = 0; int32_t yTmp = 0;
for (int yIdx = 0; yIdx < _width * _height; yIdx++) for (int yIdx = 0; yIdx < width_ * height_; yIdx++)
{ {
yTmp = y_plane[yIdx] >> 1; yTmp = y_plane[yIdx] >> 1;
y_plane[yIdx] = static_cast<uint8_t>(yTmp); y_plane[yIdx] = static_cast<uint8_t>(yTmp);
} }
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_GE(brightnessWarning = _vpm->BrightnessDetection(_videoFrame, ASSERT_GE(brightnessWarning = vpm_->BrightnessDetection(video_frame_,
stats), 0); stats), 0);
EXPECT_NE(VideoProcessingModule::kBrightWarning, brightnessWarning); EXPECT_NE(VideoProcessingModule::kBrightWarning, brightnessWarning);
if (brightnessWarning == VideoProcessingModule::kDarkWarning) if (brightnessWarning == VideoProcessingModule::kDarkWarning)
@ -114,7 +114,7 @@ TEST_F(VideoProcessingModuleTest, BrightnessDetection)
warningCount++; warningCount++;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Expect many darkness warnings // Expect many darkness warnings
warningProportion = static_cast<float>(warningCount) / frameNum * 100; warningProportion = static_cast<float>(warningCount) / frameNum * 100;

76
webrtc/modules/video_processing/main/test/unit_test/color_enhancement_test.cc
View File

@ -23,14 +23,14 @@ TEST_F(VideoProcessingModuleTest, ColorEnhancement)
{ {
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
// Use a shorter version of the Foreman clip for this test. // Use a shorter version of the Foreman clip for this test.
fclose(_sourceFile); fclose(source_file_);
const std::string video_file = const std::string video_file =
webrtc::test::ResourcePath("foreman_cif_short", "yuv"); webrtc::test::ResourcePath("foreman_cif_short", "yuv");
_sourceFile = fopen(video_file.c_str(), "rb"); source_file_ = fopen(video_file.c_str(), "rb");
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read source file: " + video_file + "\n"; "Cannot read source file: " + video_file + "\n";
std::string output_file = webrtc::test::OutputPath() + std::string output_file = webrtc::test::OutputPath() +
@ -39,27 +39,27 @@ TEST_F(VideoProcessingModuleTest, ColorEnhancement)
ASSERT_TRUE(modFile != NULL) << "Could not open output file.\n"; ASSERT_TRUE(modFile != NULL) << "Could not open output file.\n";
uint32_t frameNum = 0; uint32_t frameNum = 0;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
t0 = TickTime::Now(); t0 = TickTime::Now();
ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&_videoFrame)); ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&video_frame_));
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += t1 - t0; acc_ticks += t1 - t0;
if (PrintI420VideoFrame(_videoFrame, modFile) < 0) { if (PrintI420VideoFrame(video_frame_, modFile) < 0) {
return; return;
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
printf("\nTime per frame: %d us \n", printf("\nTime per frame: %d us \n",
static_cast<int>(accTicks.Microseconds() / frameNum)); static_cast<int>(acc_ticks.Microseconds() / frameNum));
rewind(modFile); rewind(modFile);
printf("Comparing files...\n\n"); printf("Comparing files...\n\n");
@ -82,62 +82,62 @@ TEST_F(VideoProcessingModuleTest, ColorEnhancement)
ASSERT_EQ(refLen, testLen) << "File lengths differ."; ASSERT_EQ(refLen, testLen) << "File lengths differ.";
I420VideoFrame refVideoFrame; I420VideoFrame refVideoFrame;
refVideoFrame.CreateEmptyFrame(_width, _height, refVideoFrame.CreateEmptyFrame(width_, height_,
_width, _half_width, _half_width); width_, half_width_, half_width_);
// Compare frame-by-frame. // Compare frame-by-frame.
scoped_array<uint8_t> ref_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> ref_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, modFile) == while (fread(video_buffer.get(), 1, frame_length_, modFile) ==
_frame_length) frame_length_)
{ {
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
ASSERT_EQ(_frame_length, fread(ref_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(ref_buffer.get(), 1, frame_length_,
refFile)); refFile));
EXPECT_EQ(0, ConvertToI420(kI420, ref_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, ref_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &refVideoFrame)); 0, kRotateNone, &refVideoFrame));
EXPECT_EQ(0, memcmp(_videoFrame.buffer(kYPlane), EXPECT_EQ(0, memcmp(video_frame_.buffer(kYPlane),
refVideoFrame.buffer(kYPlane), refVideoFrame.buffer(kYPlane),
_size_y)); size_y_));
EXPECT_EQ(0, memcmp(_videoFrame.buffer(kUPlane), EXPECT_EQ(0, memcmp(video_frame_.buffer(kUPlane),
refVideoFrame.buffer(kUPlane), refVideoFrame.buffer(kUPlane),
_size_uv)); size_uv_));
EXPECT_EQ(0, memcmp(_videoFrame.buffer(kVPlane), EXPECT_EQ(0, memcmp(video_frame_.buffer(kVPlane),
refVideoFrame.buffer(kVPlane), refVideoFrame.buffer(kVPlane),
_size_uv)); size_uv_));
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
// Verify that all color pixels are enhanced, and no luminance values are // Verify that all color pixels are enhanced, and no luminance values are
// altered. // altered.
scoped_array<uint8_t> testFrame(new uint8_t[_frame_length]); scoped_array<uint8_t> testFrame(new uint8_t[frame_length_]);
// Use value 128 as probe value, since we know that this will be changed // Use value 128 as probe value, since we know that this will be changed
// in the enhancement. // in the enhancement.
memset(testFrame.get(), 128, _frame_length); memset(testFrame.get(), 128, frame_length_);
I420VideoFrame testVideoFrame; I420VideoFrame testVideoFrame;
testVideoFrame.CreateEmptyFrame(_width, _height, testVideoFrame.CreateEmptyFrame(width_, height_,
_width, _half_width, _half_width); width_, half_width_, half_width_);
EXPECT_EQ(0, ConvertToI420(kI420, testFrame.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, testFrame.get(), 0, 0,
_width, _height, 0, kRotateNone, width_, height_, 0, kRotateNone,
&testVideoFrame)); &testVideoFrame));
ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&testVideoFrame)); ASSERT_EQ(0, VideoProcessingModule::ColorEnhancement(&testVideoFrame));
EXPECT_EQ(0, memcmp(testVideoFrame.buffer(kYPlane), testFrame.get(), EXPECT_EQ(0, memcmp(testVideoFrame.buffer(kYPlane), testFrame.get(),
_size_y)) size_y_))
<< "Function is modifying the luminance."; << "Function is modifying the luminance.";
EXPECT_NE(0, memcmp(testVideoFrame.buffer(kUPlane), EXPECT_NE(0, memcmp(testVideoFrame.buffer(kUPlane),
testFrame.get() + _size_y, _size_uv)) << testFrame.get() + size_y_, size_uv_)) <<
"Function is not modifying all chrominance pixels"; "Function is not modifying all chrominance pixels";
EXPECT_NE(0, memcmp(testVideoFrame.buffer(kVPlane), EXPECT_NE(0, memcmp(testVideoFrame.buffer(kVPlane),
testFrame.get() + _size_y + _size_uv, _size_uv)) << testFrame.get() + size_y_ + size_uv_, size_uv_)) <<
"Function is not modifying all chrominance pixels"; "Function is not modifying all chrominance pixels";
ASSERT_EQ(0, fclose(refFile)); ASSERT_EQ(0, fclose(refFile));

44
webrtc/modules/video_processing/main/test/unit_test/content_metrics_test.cc
View File

@ -15,32 +15,30 @@
namespace webrtc { namespace webrtc {
TEST_F(VideoProcessingModuleTest, ContentAnalysis) TEST_F(VideoProcessingModuleTest, ContentAnalysis) {
{ VPMContentAnalysis ca__c(false);
VPMContentAnalysis _ca_c(false); VPMContentAnalysis ca__sse(true);
VPMContentAnalysis _ca_sse(true); VideoContentMetrics *_cM_c, *_cM_SSE;
VideoContentMetrics *_cM_c, *_cM_SSE;
_ca_c.Initialize(_width,_height); ca__c.Initialize(width_,height_);
_ca_sse.Initialize(_width,_height); ca__sse.Initialize(width_,height_);
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) while (fread(video_buffer.get(), 1, frame_length_, source_file_)
== _frame_length) == frame_length_) {
{ // Using ConvertToI420 to add stride to the image.
// Using ConvertToI420 to add stride to the image. EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, width_, height_,
_width, _height, 0, kRotateNone, &video_frame_));
0, kRotateNone, &_videoFrame)); _cM_c = ca__c.ComputeContentMetrics(video_frame_);
_cM_c = _ca_c.ComputeContentMetrics(_videoFrame); _cM_SSE = ca__sse.ComputeContentMetrics(video_frame_);
_cM_SSE = _ca_sse.ComputeContentMetrics(_videoFrame);
ASSERT_EQ(_cM_c->spatial_pred_err, _cM_SSE->spatial_pred_err); ASSERT_EQ(_cM_c->spatial_pred_err, _cM_SSE->spatial_pred_err);
ASSERT_EQ(_cM_c->spatial_pred_err_v, _cM_SSE->spatial_pred_err_v); ASSERT_EQ(_cM_c->spatial_pred_err_v, _cM_SSE->spatial_pred_err_v);
ASSERT_EQ(_cM_c->spatial_pred_err_h, _cM_SSE->spatial_pred_err_h); ASSERT_EQ(_cM_c->spatial_pred_err_h, _cM_SSE->spatial_pred_err_h);
ASSERT_EQ(_cM_c->motion_magnitude, _cM_SSE->motion_magnitude); ASSERT_EQ(_cM_c->motion_magnitude, _cM_SSE->motion_magnitude);
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
} }
} // namespace webrtc } // namespace webrtc

56
webrtc/modules/video_processing/main/test/unit_test/deflickering_test.cc
View File

@ -23,17 +23,17 @@ TEST_F(VideoProcessingModuleTest, Deflickering)
{ {
enum { NumRuns = 30 }; enum { NumRuns = 30 };
uint32_t frameNum = 0; uint32_t frameNum = 0;
const uint32_t frameRate = 15; const uint32_t frame_rate = 15;
int64_t minRuntime = 0; int64_t min_runtime = 0;
int64_t avgRuntime = 0; int64_t avg_runtime = 0;
// Close automatically opened Foreman. // Close automatically opened Foreman.
fclose(_sourceFile); fclose(source_file_);
const std::string input_file = const std::string input_file =
webrtc::test::ResourcePath("deflicker_before_cif_short", "yuv"); webrtc::test::ResourcePath("deflicker_before_cif_short", "yuv");
_sourceFile = fopen(input_file.c_str(), "rb"); source_file_ = fopen(input_file.c_str(), "rb");
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read input file: " << input_file << "\n"; "Cannot read input file: " << input_file << "\n";
const std::string output_file = const std::string output_file =
@ -43,57 +43,57 @@ TEST_F(VideoProcessingModuleTest, Deflickering)
"Could not open output file: " << output_file << "\n"; "Could not open output file: " << output_file << "\n";
printf("\nRun time [us / frame]:\n"); printf("\nRun time [us / frame]:\n");
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
for (uint32_t runIdx = 0; runIdx < NumRuns; runIdx++) for (uint32_t run_idx = 0; run_idx < NumRuns; run_idx++)
{ {
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
uint32_t timeStamp = 1; uint32_t timeStamp = 1;
frameNum = 0; frameNum = 0;
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
frameNum++; frameNum++;
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
_videoFrame.set_timestamp(timeStamp); video_frame_.set_timestamp(timeStamp);
t0 = TickTime::Now(); t0 = TickTime::Now();
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_EQ(0, _vpm->Deflickering(&_videoFrame, &stats)); ASSERT_EQ(0, vpm_->Deflickering(&video_frame_, &stats));
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += (t1 - t0); acc_ticks += (t1 - t0);
if (runIdx == 0) if (run_idx == 0)
{ {
if (PrintI420VideoFrame(_videoFrame, deflickerFile) < 0) { if (PrintI420VideoFrame(video_frame_, deflickerFile) < 0) {
return; return;
} }
} }
timeStamp += (90000 / frameRate); timeStamp += (90000 / frame_rate);
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
printf("%u\n", static_cast<int>(accTicks.Microseconds() / frameNum)); printf("%u\n", static_cast<int>(acc_ticks.Microseconds() / frameNum));
if (accTicks.Microseconds() < minRuntime || runIdx == 0) if (acc_ticks.Microseconds() < min_runtime || run_idx == 0)
{ {
minRuntime = accTicks.Microseconds(); min_runtime = acc_ticks.Microseconds();
} }
avgRuntime += accTicks.Microseconds(); avg_runtime += acc_ticks.Microseconds();
rewind(_sourceFile); rewind(source_file_);
} }
ASSERT_EQ(0, fclose(deflickerFile)); ASSERT_EQ(0, fclose(deflickerFile));
// TODO(kjellander): Add verification of deflicker output file. // TODO(kjellander): Add verification of deflicker output file.
printf("\nAverage run time = %d us / frame\n", printf("\nAverage run time = %d us / frame\n",
static_cast<int>(avgRuntime / frameNum / NumRuns)); static_cast<int>(avg_runtime / frameNum / NumRuns));
printf("Min run time = %d us / frame\n\n", printf("Min run time = %d us / frame\n\n",
static_cast<int>(minRuntime / frameNum)); static_cast<int>(min_runtime / frameNum));
} }
} // namespace webrtc } // namespace webrtc

66
webrtc/modules/video_processing/main/test/unit_test/denoising_test.cc
View File

@ -25,8 +25,8 @@ TEST_F(VideoProcessingModuleTest, DISABLED_ON_ANDROID(Denoising))
enum { NumRuns = 10 }; enum { NumRuns = 10 };
uint32_t frameNum = 0; uint32_t frameNum = 0;
int64_t minRuntime = 0; int64_t min_runtime = 0;
int64_t avgRuntime = 0; int64_t avg_runtime = 0;
const std::string denoise_filename = const std::string denoise_filename =
webrtc::test::OutputPath() + "denoise_testfile.yuv"; webrtc::test::OutputPath() + "denoise_testfile.yuv";
@ -41,50 +41,50 @@ TEST_F(VideoProcessingModuleTest, DISABLED_ON_ANDROID(Denoising))
"Could not open noisy file: " << noise_filename << "\n"; "Could not open noisy file: " << noise_filename << "\n";
printf("\nRun time [us / frame]:\n"); printf("\nRun time [us / frame]:\n");
for (uint32_t runIdx = 0; runIdx < NumRuns; runIdx++) for (uint32_t run_idx = 0; run_idx < NumRuns; run_idx++)
{ {
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
int32_t modifiedPixels = 0; int32_t modifiedPixels = 0;
frameNum = 0; frameNum = 0;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, _frame_length, _sourceFile) == while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
_frame_length) frame_length_)
{ {
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
frameNum++; frameNum++;
uint8_t* sourceBuffer = _videoFrame.buffer(kYPlane); uint8_t* sourceBuffer = video_frame_.buffer(kYPlane);
// Add noise to a part in video stream // Add noise to a part in video stream
// Random noise // Random noise
// TODO: investigate the effectiveness of this test. // TODO: investigate the effectiveness of this test.
for (int ir = 0; ir < _height; ir++) for (int ir = 0; ir < height_; ir++)
{ {
uint32_t ik = ir * _width; uint32_t ik = ir * width_;
for (int ic = 0; ic < _width; ic++) for (int ic = 0; ic < width_; ic++)
{ {
uint8_t r = rand() % 16; uint8_t r = rand() % 16;
r -= 8; r -= 8;
if (ir < _height / 4) if (ir < height_ / 4)
r = 0; r = 0;
if (ir >= 3 * _height / 4) if (ir >= 3 * height_ / 4)
r = 0; r = 0;
if (ic < _width / 4) if (ic < width_ / 4)
r = 0; r = 0;
if (ic >= 3 * _width / 4) if (ic >= 3 * width_ / 4)
r = 0; r = 0;
/*uint8_t pixelValue = 0; /*uint8_t pixelValue = 0;
if (ir >= _height / 2) if (ir >= height_ / 2)
{ // Region 3 or 4 { // Region 3 or 4
pixelValue = 170; pixelValue = 170;
} }
if (ic >= _width / 2) if (ic >= width_ / 2)
{ // Region 2 or 4 { // Region 2 or 4
pixelValue += 85; pixelValue += 85;
} }
@ -95,42 +95,42 @@ TEST_F(VideoProcessingModuleTest, DISABLED_ON_ANDROID(Denoising))
} }
} }
if (runIdx == 0) if (run_idx == 0)
{ {
if (PrintI420VideoFrame(_videoFrame, noiseFile) < 0) { if (PrintI420VideoFrame(video_frame_, noiseFile) < 0) {
return; return;
} }
} }
t0 = TickTime::Now(); t0 = TickTime::Now();
ASSERT_GE(modifiedPixels = _vpm->Denoising(&_videoFrame), 0); ASSERT_GE(modifiedPixels = vpm_->Denoising(&video_frame_), 0);
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += (t1 - t0); acc_ticks += (t1 - t0);
if (runIdx == 0) if (run_idx == 0)
{ {
if (PrintI420VideoFrame(_videoFrame, noiseFile) < 0) { if (PrintI420VideoFrame(video_frame_, noiseFile) < 0) {
return; return;
} }
} }
} }
ASSERT_NE(0, feof(_sourceFile)) << "Error reading source file"; ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
printf("%u\n", static_cast<int>(accTicks.Microseconds() / frameNum)); printf("%u\n", static_cast<int>(acc_ticks.Microseconds() / frameNum));
if (accTicks.Microseconds() < minRuntime || runIdx == 0) if (acc_ticks.Microseconds() < min_runtime || run_idx == 0)
{ {
minRuntime = accTicks.Microseconds(); min_runtime = acc_ticks.Microseconds();
} }
avgRuntime += accTicks.Microseconds(); avg_runtime += acc_ticks.Microseconds();
rewind(_sourceFile); rewind(source_file_);
} }
ASSERT_EQ(0, fclose(denoiseFile)); ASSERT_EQ(0, fclose(denoiseFile));
ASSERT_EQ(0, fclose(noiseFile)); ASSERT_EQ(0, fclose(noiseFile));
printf("\nAverage run time = %d us / frame\n", printf("\nAverage run time = %d us / frame\n",
static_cast<int>(avgRuntime / frameNum / NumRuns)); static_cast<int>(avg_runtime / frameNum / NumRuns));
printf("Min run time = %d us / frame\n\n", printf("Min run time = %d us / frame\n\n",
static_cast<int>(minRuntime / frameNum)); static_cast<int>(min_runtime / frameNum));
} }
} // namespace webrtc } // namespace webrtc

358
webrtc/modules/video_processing/main/test/unit_test/video_processing_unittest.cc
View File

@ -18,297 +18,285 @@
namespace webrtc { namespace webrtc {
// The |sourceFrame| is scaled to |target_width|,|target_height|, using the // The |sourceFrame| is scaled to |targetwidth_|,|targetheight_|, using the
// filter mode set to |mode|. The |expected_psnr| is used to verify basic // filter mode set to |mode|. The |expected_psnr| is used to verify basic
// quality when the resampled frame is scaled back up/down to the // quality when the resampled frame is scaled back up/down to the
// original/source size. |expected_psnr| is set to be ~0.1/0.05dB lower than // original/source size. |expected_psnr| is set to be ~0.1/0.05dB lower than
// actual PSNR verified under the same conditions. // actual PSNR verified under the same conditions.
void TestSize(const I420VideoFrame& sourceFrame, int target_width, void TestSize(const I420VideoFrame& sourceFrame, int targetwidth_,
int target_height, int mode, double expected_psnr, int targetheight_, int mode, double expected_psnr,
VideoProcessingModule* vpm); VideoProcessingModule* vpm);
bool CompareFrames(const webrtc::I420VideoFrame& frame1, bool CompareFrames(const webrtc::I420VideoFrame& frame1,
const webrtc::I420VideoFrame& frame2); const webrtc::I420VideoFrame& frame2);
VideoProcessingModuleTest::VideoProcessingModuleTest() : VideoProcessingModuleTest::VideoProcessingModuleTest()
_vpm(NULL), : vpm_(NULL),
_sourceFile(NULL), source_file_(NULL),
_width(352), width_(352),
_half_width((_width + 1) / 2), half_width_((width_ + 1) / 2),
_height(288), height_(288),
_size_y(_width * _height), size_y_(width_ * height_),
_size_uv(_half_width * ((_height + 1) / 2)), size_uv_(half_width_ * ((height_ + 1) / 2)),
_frame_length(CalcBufferSize(kI420, _width, _height)) frame_length_(CalcBufferSize(kI420, width_, height_)) {}
{
}
void VideoProcessingModuleTest::SetUp() void VideoProcessingModuleTest::SetUp() {
{ vpm_ = VideoProcessingModule::Create(0);
_vpm = VideoProcessingModule::Create(0); ASSERT_TRUE(vpm_ != NULL);
ASSERT_TRUE(_vpm != NULL);
ASSERT_EQ(0, _videoFrame.CreateEmptyFrame(_width, _height, _width, ASSERT_EQ(0, video_frame_.CreateEmptyFrame(width_, height_, width_,
_half_width, _half_width)); half_width_, half_width_));
const std::string video_file = const std::string video_file =
webrtc::test::ResourcePath("foreman_cif", "yuv"); webrtc::test::ResourcePath("foreman_cif", "yuv");
_sourceFile = fopen(video_file.c_str(),"rb"); source_file_ = fopen(video_file.c_str(),"rb");
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read source file: " + video_file + "\n"; "Cannot read source file: " + video_file + "\n";
} }
void VideoProcessingModuleTest::TearDown() void VideoProcessingModuleTest::TearDown() {
{ if (source_file_ != NULL) {
if (_sourceFile != NULL) { ASSERT_EQ(0, fclose(source_file_));
ASSERT_EQ(0, fclose(_sourceFile));
} }
_sourceFile = NULL; source_file_ = NULL;
if (_vpm != NULL) { if (vpm_ != NULL) {
VideoProcessingModule::Destroy(_vpm); VideoProcessingModule::Destroy(vpm_);
} }
_vpm = NULL; vpm_ = NULL;
} }
TEST_F(VideoProcessingModuleTest, HandleNullBuffer) TEST_F(VideoProcessingModuleTest, HandleNullBuffer) {
{
// TODO(mikhal/stefan): Do we need this one? // TODO(mikhal/stefan): Do we need this one?
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
// Video frame with unallocated buffer. // Video frame with unallocated buffer.
I420VideoFrame videoFrame; I420VideoFrame videoFrame;
videoFrame.set_width(_width); videoFrame.set_width(width_);
videoFrame.set_height(_height); videoFrame.set_height(height_);
EXPECT_EQ(-3, _vpm->GetFrameStats(&stats, videoFrame)); EXPECT_EQ(-3, vpm_->GetFrameStats(&stats, videoFrame));
EXPECT_EQ(-1, _vpm->ColorEnhancement(&videoFrame)); EXPECT_EQ(-1, vpm_->ColorEnhancement(&videoFrame));
EXPECT_EQ(-1, _vpm->Deflickering(&videoFrame, &stats)); EXPECT_EQ(-1, vpm_->Deflickering(&videoFrame, &stats));
EXPECT_EQ(-1, _vpm->Denoising(&videoFrame)); EXPECT_EQ(-1, vpm_->Denoising(&videoFrame));
EXPECT_EQ(-3, _vpm->BrightnessDetection(videoFrame, stats)); EXPECT_EQ(-3, vpm_->BrightnessDetection(videoFrame, stats));
} }
TEST_F(VideoProcessingModuleTest, HandleBadStats) TEST_F(VideoProcessingModuleTest, HandleBadStats) {
{
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
EXPECT_EQ(-1, _vpm->Deflickering(&_videoFrame, &stats)); EXPECT_EQ(-1, vpm_->Deflickering(&video_frame_, &stats));
EXPECT_EQ(-3, _vpm->BrightnessDetection(_videoFrame, stats)); EXPECT_EQ(-3, vpm_->BrightnessDetection(video_frame_, stats));
} }
TEST_F(VideoProcessingModuleTest, HandleBadSize) TEST_F(VideoProcessingModuleTest, HandleBadSize) {
{
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
_videoFrame.ResetSize(); video_frame_.ResetSize();
_videoFrame.set_width(_width); video_frame_.set_width(width_);
_videoFrame.set_height(0); video_frame_.set_height(0);
EXPECT_EQ(-3, _vpm->GetFrameStats(&stats, _videoFrame)); EXPECT_EQ(-3, vpm_->GetFrameStats(&stats, video_frame_));
EXPECT_EQ(-1, _vpm->ColorEnhancement(&_videoFrame)); EXPECT_EQ(-1, vpm_->ColorEnhancement(&video_frame_));
EXPECT_EQ(-1, _vpm->Deflickering(&_videoFrame, &stats)); EXPECT_EQ(-1, vpm_->Deflickering(&video_frame_, &stats));
EXPECT_EQ(-1, _vpm->Denoising(&_videoFrame)); EXPECT_EQ(-1, vpm_->Denoising(&video_frame_));
EXPECT_EQ(-3, _vpm->BrightnessDetection(_videoFrame, stats)); EXPECT_EQ(-3, vpm_->BrightnessDetection(video_frame_, stats));
EXPECT_EQ(VPM_PARAMETER_ERROR, _vpm->SetTargetResolution(0,0,0)); EXPECT_EQ(VPM_PARAMETER_ERROR, vpm_->SetTargetResolution(0,0,0));
EXPECT_EQ(VPM_PARAMETER_ERROR, _vpm->SetMaxFrameRate(0)); EXPECT_EQ(VPM_PARAMETER_ERROR, vpm_->SetMaxFramerate(0));
I420VideoFrame *outFrame = NULL; I420VideoFrame *out_frame = NULL;
EXPECT_EQ(VPM_PARAMETER_ERROR, _vpm->PreprocessFrame(_videoFrame, EXPECT_EQ(VPM_PARAMETER_ERROR, vpm_->PreprocessFrame(video_frame_,
&outFrame)); &out_frame));
} }
TEST_F(VideoProcessingModuleTest, IdenticalResultsAfterReset) TEST_F(VideoProcessingModuleTest, IdenticalResultsAfterReset) {
{ I420VideoFrame video_frame2;
I420VideoFrame videoFrame2;
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
// Only testing non-static functions here. // Only testing non-static functions here.
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
ASSERT_EQ(0, videoFrame2.CopyFrame(_videoFrame)); ASSERT_EQ(0, video_frame2.CopyFrame(video_frame_));
ASSERT_EQ(0, _vpm->Deflickering(&_videoFrame, &stats)); ASSERT_EQ(0, vpm_->Deflickering(&video_frame_, &stats));
_vpm->Reset(); vpm_->Reset();
// Retrieve frame stats again in case Deflickering() has zeroed them. // Retrieve frame stats again in case Deflickering() has zeroed them.
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, videoFrame2)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame2));
ASSERT_EQ(0, _vpm->Deflickering(&videoFrame2, &stats)); ASSERT_EQ(0, vpm_->Deflickering(&video_frame2, &stats));
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
videoFrame2.CopyFrame(_videoFrame); video_frame2.CopyFrame(video_frame_);
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
ASSERT_GE(_vpm->Denoising(&_videoFrame), 0); ASSERT_GE(vpm_->Denoising(&video_frame_), 0);
_vpm->Reset(); vpm_->Reset();
ASSERT_GE(_vpm->Denoising(&videoFrame2), 0); ASSERT_GE(vpm_->Denoising(&video_frame2), 0);
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
ASSERT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); ASSERT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
videoFrame2.CopyFrame(_videoFrame); video_frame2.CopyFrame(video_frame_);
ASSERT_EQ(0, _vpm->BrightnessDetection(_videoFrame, stats)); ASSERT_EQ(0, vpm_->BrightnessDetection(video_frame_, stats));
_vpm->Reset(); vpm_->Reset();
ASSERT_EQ(0, _vpm->BrightnessDetection(videoFrame2, stats)); ASSERT_EQ(0, vpm_->BrightnessDetection(video_frame2, stats));
EXPECT_TRUE(CompareFrames(_videoFrame, videoFrame2)); EXPECT_TRUE(CompareFrames(video_frame_, video_frame2));
} }
TEST_F(VideoProcessingModuleTest, FrameStats) TEST_F(VideoProcessingModuleTest, FrameStats) {
{
VideoProcessingModule::FrameStats stats; VideoProcessingModule::FrameStats stats;
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
EXPECT_FALSE(_vpm->ValidFrameStats(stats)); EXPECT_FALSE(vpm_->ValidFrameStats(stats));
EXPECT_EQ(0, _vpm->GetFrameStats(&stats, _videoFrame)); EXPECT_EQ(0, vpm_->GetFrameStats(&stats, video_frame_));
EXPECT_TRUE(_vpm->ValidFrameStats(stats)); EXPECT_TRUE(vpm_->ValidFrameStats(stats));
printf("\nFrameStats\n"); printf("\nFrameStats\n");
printf("mean: %u\nnumPixels: %u\nsubSamplWidth: " printf("mean: %u\nnum_pixels: %u\nsubSamplWidth: "
"%u\nsumSamplHeight: %u\nsum: %u\n\n", "%u\nsumSamplHeight: %u\nsum: %u\n\n",
static_cast<unsigned int>(stats.mean), static_cast<unsigned int>(stats.mean),
static_cast<unsigned int>(stats.numPixels), static_cast<unsigned int>(stats.num_pixels),
static_cast<unsigned int>(stats.subSamplHeight), static_cast<unsigned int>(stats.subSamplHeight),
static_cast<unsigned int>(stats.subSamplWidth), static_cast<unsigned int>(stats.subSamplWidth),
static_cast<unsigned int>(stats.sum)); static_cast<unsigned int>(stats.sum));
_vpm->ClearFrameStats(&stats); vpm_->ClearFrameStats(&stats);
EXPECT_FALSE(_vpm->ValidFrameStats(stats)); EXPECT_FALSE(vpm_->ValidFrameStats(stats));
} }
TEST_F(VideoProcessingModuleTest, PreprocessorLogic) TEST_F(VideoProcessingModuleTest, PreprocessorLogic) {
{
// Disable temporal sampling (frame dropping). // Disable temporal sampling (frame dropping).
_vpm->EnableTemporalDecimation(false); vpm_->EnableTemporalDecimation(false);
int resolution = 100; int resolution = 100;
EXPECT_EQ(VPM_OK, _vpm->SetMaxFrameRate(30)); EXPECT_EQ(VPM_OK, vpm_->SetMaxFramerate(30));
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 15)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 15));
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 30)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 30));
// Disable spatial sampling. // Disable spatial sampling.
_vpm->SetInputFrameResampleMode(kNoRescaling); vpm_->SetInputFrameResampleMode(kNoRescaling);
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 30)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 30));
I420VideoFrame* outFrame = NULL; I420VideoFrame* out_frame = NULL;
// Set rescaling => output frame != NULL. // Set rescaling => output frame != NULL.
_vpm->SetInputFrameResampleMode(kFastRescaling); vpm_->SetInputFrameResampleMode(kFastRescaling);
EXPECT_EQ(VPM_OK, _vpm->SetTargetResolution(resolution, resolution, 30)); EXPECT_EQ(VPM_OK, vpm_->SetTargetResolution(resolution, resolution, 30));
EXPECT_EQ(VPM_OK, _vpm->PreprocessFrame(_videoFrame, &outFrame)); EXPECT_EQ(VPM_OK, vpm_->PreprocessFrame(video_frame_, &out_frame));
EXPECT_FALSE(outFrame == NULL); EXPECT_FALSE(out_frame == NULL);
if (outFrame) { if (out_frame) {
EXPECT_EQ(resolution, outFrame->width()); EXPECT_EQ(resolution, out_frame->width());
EXPECT_EQ(resolution, outFrame->height()); EXPECT_EQ(resolution, out_frame->height());
} }
// No rescaling=> output frame = NULL. // No rescaling=> output frame = NULL.
_vpm->SetInputFrameResampleMode(kNoRescaling); vpm_->SetInputFrameResampleMode(kNoRescaling);
EXPECT_EQ(VPM_OK, _vpm->PreprocessFrame(_videoFrame, &outFrame)); EXPECT_EQ(VPM_OK, vpm_->PreprocessFrame(video_frame_, &out_frame));
EXPECT_TRUE(outFrame == NULL); EXPECT_TRUE(out_frame == NULL);
} }
TEST_F(VideoProcessingModuleTest, Resampler) TEST_F(VideoProcessingModuleTest, Resampler) {
{
enum { NumRuns = 1 }; enum { NumRuns = 1 };
int64_t minRuntime = 0; int64_t min_runtime = 0;
int64_t avgRuntime = 0; int64_t avg_runtime = 0;
TickTime t0; TickTime t0;
TickTime t1; TickTime t1;
TickInterval accTicks; TickInterval acc_ticks;
rewind(_sourceFile); rewind(source_file_);
ASSERT_TRUE(_sourceFile != NULL) << ASSERT_TRUE(source_file_ != NULL) <<
"Cannot read input file \n"; "Cannot read input file \n";
// CA not needed here // CA not needed here
_vpm->EnableContentAnalysis(false); vpm_->EnableContentAnalysis(false);
// no temporal decimation // no temporal decimation
_vpm->EnableTemporalDecimation(false); vpm_->EnableTemporalDecimation(false);
// Reading test frame // Reading test frame
scoped_array<uint8_t> video_buffer(new uint8_t[_frame_length]); scoped_array<uint8_t> video_buffer(new uint8_t[frame_length_]);
ASSERT_EQ(_frame_length, fread(video_buffer.get(), 1, _frame_length, ASSERT_EQ(frame_length_, fread(video_buffer.get(), 1, frame_length_,
_sourceFile)); source_file_));
// Using ConvertToI420 to add stride to the image. // Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0,
_width, _height, width_, height_,
0, kRotateNone, &_videoFrame)); 0, kRotateNone, &video_frame_));
for (uint32_t runIdx = 0; runIdx < NumRuns; runIdx++) for (uint32_t run_idx = 0; run_idx < NumRuns; run_idx++) {
{ // Initiate test timer.
// initiate test timer
t0 = TickTime::Now(); t0 = TickTime::Now();
// Init the sourceFrame with a timestamp. // Init the sourceFrame with a timestamp.
_videoFrame.set_render_time_ms(t0.MillisecondTimestamp()); video_frame_.set_render_time_ms(t0.MillisecondTimestamp());
_videoFrame.set_timestamp(t0.MillisecondTimestamp() * 90); video_frame_.set_timestamp(t0.MillisecondTimestamp() * 90);
// Test scaling to different sizes: source is of |width|/|height| = 352/288. // Test scaling to different sizes: source is of |width|/|height| = 352/288.
// Scaling mode in VPM is currently fixed to kScaleBox (mode = 3). // Scaling mode in VPM is currently fixed to kScaleBox (mode = 3).
TestSize(_videoFrame, 100, 50, 3, 24.0, _vpm); TestSize(video_frame_, 100, 50, 3, 24.0, vpm_);
TestSize(_videoFrame, 352/4, 288/4, 3, 25.2, _vpm); TestSize(video_frame_, 352/4, 288/4, 3, 25.2, vpm_);
TestSize(_videoFrame, 352/2, 288/2, 3, 28.1, _vpm); TestSize(video_frame_, 352/2, 288/2, 3, 28.1, vpm_);
TestSize(_videoFrame, 352, 288, 3, -1, _vpm); // no resampling. TestSize(video_frame_, 352, 288, 3, -1, vpm_); // no resampling.
TestSize(_videoFrame, 2*352, 2*288, 3, 32.2, _vpm); TestSize(video_frame_, 2*352, 2*288, 3, 32.2, vpm_);
TestSize(_videoFrame, 400, 256, 3, 31.3, _vpm); TestSize(video_frame_, 400, 256, 3, 31.3, vpm_);
TestSize(_videoFrame, 480, 640, 3, 32.15, _vpm); TestSize(video_frame_, 480, 640, 3, 32.15, vpm_);
TestSize(_videoFrame, 960, 720, 3, 32.2, _vpm); TestSize(video_frame_, 960, 720, 3, 32.2, vpm_);
TestSize(_videoFrame, 1280, 720, 3, 32.15, _vpm); TestSize(video_frame_, 1280, 720, 3, 32.15, vpm_);
// Upsampling to odd size. // Upsampling to odd size.
TestSize(_videoFrame, 501, 333, 3, 32.05, _vpm); TestSize(video_frame_, 501, 333, 3, 32.05, vpm_);
// Downsample to odd size. // Downsample to odd size.
TestSize(_videoFrame, 281, 175, 3, 29.3, _vpm); TestSize(video_frame_, 281, 175, 3, 29.3, vpm_);
// stop timer // stop timer
t1 = TickTime::Now(); t1 = TickTime::Now();
accTicks += (t1 - t0); acc_ticks += (t1 - t0);
if (accTicks.Microseconds() < minRuntime || runIdx == 0) { if (acc_ticks.Microseconds() < min_runtime || run_idx == 0) {
minRuntime = accTicks.Microseconds(); min_runtime = acc_ticks.Microseconds();
} }
avgRuntime += accTicks.Microseconds(); avg_runtime += acc_ticks.Microseconds();
} }
printf("\nAverage run time = %d us / frame\n", printf("\nAverage run time = %d us / frame\n",
//static_cast<int>(avgRuntime / frameNum / NumRuns)); //static_cast<int>(avg_runtime / frameNum / NumRuns));
static_cast<int>(avgRuntime)); static_cast<int>(avg_runtime));
printf("Min run time = %d us / frame\n\n", printf("Min run time = %d us / frame\n\n",
//static_cast<int>(minRuntime / frameNum)); //static_cast<int>(min_runtime / frameNum));
static_cast<int>(minRuntime)); static_cast<int>(min_runtime));
} }
void TestSize(const I420VideoFrame& source_frame, int target_width, void TestSize(const I420VideoFrame& source_frame, int targetwidth_,
int target_height, int mode, double expected_psnr, int targetheight_, int mode, double expected_psnr,
VideoProcessingModule* vpm) { VideoProcessingModule* vpm) {
int source_width = source_frame.width(); int sourcewidth_ = source_frame.width();
int source_height = source_frame.height(); int sourceheight_ = source_frame.height();
I420VideoFrame* out_frame = NULL; I420VideoFrame* out_frame = NULL;
ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(target_width, target_height, 30)); ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(targetwidth_, targetheight_, 30));
ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(source_frame, &out_frame)); ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(source_frame, &out_frame));
if (out_frame) { if (out_frame) {
@ -321,19 +309,19 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
// (2) scale the resampled frame (|out_frame|) back to the original size and // (2) scale the resampled frame (|out_frame|) back to the original size and
// compute PSNR relative to |source_frame| (for automatic verification). // compute PSNR relative to |source_frame| (for automatic verification).
// (3) write out the processed frame for viewing. // (3) write out the processed frame for viewing.
if (target_width != static_cast<int>(source_width) || if (targetwidth_ != static_cast<int>(sourcewidth_) ||
target_height != static_cast<int>(source_height)) { targetheight_ != static_cast<int>(sourceheight_)) {
// Write the processed frame to file for visual inspection. // Write the processed frame to file for visual inspection.
std::ostringstream filename; std::ostringstream filename;
filename << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" << filename << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" <<
"from_" << source_width << "x" << source_height << "_to_" << "from_" << sourcewidth_ << "x" << sourceheight_ << "_to_" <<
target_width << "x" << target_height << "_30Hz_P420.yuv"; targetwidth_ << "x" << targetheight_ << "_30Hz_P420.yuv";
std::cout << "Watch " << filename.str() << " and verify that it is okay." std::cout << "Watch " << filename.str() << " and verify that it is okay."
<< std::endl; << std::endl;
FILE* stand_alone_file = fopen(filename.str().c_str(), "wb"); FILE* stand_alone_file = fopen(filename.str().c_str(), "wb");
if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) { if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) {
fprintf(stderr, "Failed to write frame for scaling to width/height: " fprintf(stderr, "Failed to write frame for scaling to width/height: "
" %d %d \n", target_width, target_height); " %d %d \n", targetwidth_, targetheight_);
return; return;
} }
fclose(stand_alone_file); fclose(stand_alone_file);
@ -342,8 +330,8 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
resampled_source_frame.CopyFrame(*out_frame); resampled_source_frame.CopyFrame(*out_frame);
// Scale |resampled_source_frame| back to original/source size. // Scale |resampled_source_frame| back to original/source size.
ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(source_width, ASSERT_EQ(VPM_OK, vpm->SetTargetResolution(sourcewidth_,
source_height, sourceheight_,
30)); 30));
ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(resampled_source_frame, ASSERT_EQ(VPM_OK, vpm->PreprocessFrame(resampled_source_frame,
&out_frame)); &out_frame));
@ -351,14 +339,14 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
// Write the processed frame to file for visual inspection. // Write the processed frame to file for visual inspection.
std::ostringstream filename2; std::ostringstream filename2;
filename2 << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" << filename2 << webrtc::test::OutputPath() << "Resampler_"<< mode << "_" <<
"from_" << target_width << "x" << target_height << "_to_" << "from_" << targetwidth_ << "x" << targetheight_ << "_to_" <<
source_width << "x" << source_height << "_30Hz_P420.yuv"; sourcewidth_ << "x" << sourceheight_ << "_30Hz_P420.yuv";
std::cout << "Watch " << filename2.str() << " and verify that it is okay." std::cout << "Watch " << filename2.str() << " and verify that it is okay."
<< std::endl; << std::endl;
stand_alone_file = fopen(filename2.str().c_str(), "wb"); stand_alone_file = fopen(filename2.str().c_str(), "wb");
if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) { if (PrintI420VideoFrame(*out_frame, stand_alone_file) < 0) {
fprintf(stderr, "Failed to write frame for scaling to width/height " fprintf(stderr, "Failed to write frame for scaling to width/height "
"%d %d \n", source_width, source_height); "%d %d \n", sourcewidth_, sourceheight_);
return; return;
} }
fclose(stand_alone_file); fclose(stand_alone_file);
@ -368,7 +356,7 @@ void TestSize(const I420VideoFrame& source_frame, int target_width,
EXPECT_GT(psnr, expected_psnr); EXPECT_GT(psnr, expected_psnr);
printf("PSNR: %f. PSNR is between source of size %d %d, and a modified " printf("PSNR: %f. PSNR is between source of size %d %d, and a modified "
"source which is scaled down/up to: %d %d, and back to source size \n", "source which is scaled down/up to: %d %d, and back to source size \n",
psnr, source_width, source_height, target_width, target_height); psnr, sourcewidth_, sourceheight_, targetwidth_, targetheight_);
} }
} }

47
webrtc/modules/video_processing/main/test/unit_test/video_processing_unittest.h
View File

@ -18,31 +18,28 @@
namespace webrtc { namespace webrtc {
class VideoProcessingModuleTest : public ::testing::Test class VideoProcessingModuleTest : public ::testing::Test {
{ protected:
protected: VideoProcessingModuleTest();
VideoProcessingModuleTest(); virtual void SetUp();
virtual void SetUp(); virtual void TearDown();
virtual void TearDown(); static void SetUpTestCase() {
static void SetUpTestCase() Trace::CreateTrace();
{ std::string trace_file = webrtc::test::OutputPath() + "VPMTrace.txt";
Trace::CreateTrace(); ASSERT_EQ(0, Trace::SetTraceFile(trace_file.c_str()));
std::string trace_file = webrtc::test::OutputPath() + "VPMTrace.txt"; }
ASSERT_EQ(0, Trace::SetTraceFile(trace_file.c_str())); static void TearDownTestCase() {
} Trace::ReturnTrace();
static void TearDownTestCase() }
{ VideoProcessingModule* vpm_;
Trace::ReturnTrace(); FILE* source_file_;
} I420VideoFrame video_frame_;
VideoProcessingModule* _vpm; const int width_;
FILE* _sourceFile; const int half_width_;
I420VideoFrame _videoFrame; const int height_;
const int _width; const int size_y_;
const int _half_width; const int size_uv_;
const int _height; const unsigned int frame_length_;
const int _size_y;
const int _size_uv;
const unsigned int _frame_length;
}; };
} // namespace webrtc } // namespace webrtc