446 lines
14 KiB
C++
446 lines
14 KiB
C++
|
/*
|
||
|
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
|
||
|
*
|
||
|
* Use of this source code is governed by a BSD-style license
|
||
|
* that can be found in the LICENSE file in the root of the source
|
||
|
* tree. An additional intellectual property rights grant can be found
|
||
|
* in the file PATENTS. All contributing project authors may
|
||
|
* be found in the AUTHORS file in the root of the source tree.
|
||
|
*/
|
||
|
|
||
|
|
||
|
#include <math.h>
|
||
|
#include <stdlib.h>
|
||
|
|
||
|
#include "deflickering.h"
|
||
|
#include "trace.h"
|
||
|
#include "signal_processing_library.h"
|
||
|
#include "sort.h"
|
||
|
|
||
|
namespace webrtc {
|
||
|
|
||
|
// Detection constants
|
||
|
enum { kFrequencyDeviation = 39 }; // (Q4) Maximum allowed deviation for detection
|
||
|
enum { kMinFrequencyToDetect = 32 }; // (Q4) Minimum frequency that can be detected
|
||
|
enum { kNumFlickerBeforeDetect = 2 }; // Number of flickers before we accept detection
|
||
|
enum { kMeanValueScaling = 4 }; // (Q4) In power of 2
|
||
|
enum { kZeroCrossingDeadzone = 10 }; // Deadzone region in terms of pixel values
|
||
|
|
||
|
// Deflickering constants
|
||
|
// Compute the quantiles over 1 / DownsamplingFactor of the image.
|
||
|
enum { kDownsamplingFactor = 8 };
|
||
|
enum { kLog2OfDownsamplingFactor = 3 };
|
||
|
|
||
|
// To generate in Matlab:
|
||
|
// >> probUW16 = round(2^11 * [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.97]);
|
||
|
// >> fprintf('%d, ', probUW16)
|
||
|
// Resolution reduced to avoid overflow when multiplying with the (potentially) large
|
||
|
// number of pixels.
|
||
|
const WebRtc_UWord16 VPMDeflickering::_probUW16[kNumProbs] =
|
||
|
{102, 205, 410, 614, 819, 1024, 1229, 1434, 1638, 1843, 1946, 1987}; // <Q11>
|
||
|
|
||
|
// To generate in Matlab:
|
||
|
// >> numQuants = 14; maxOnlyLength = 5;
|
||
|
// >> weightUW16 = round(2^15 * [linspace(0.5, 1.0, numQuants - maxOnlyLength)]);
|
||
|
// >> fprintf('%d, %d,\n ', weightUW16);
|
||
|
const WebRtc_UWord16 VPMDeflickering::_weightUW16[kNumQuants - kMaxOnlyLength] =
|
||
|
{16384, 18432, 20480, 22528, 24576, 26624, 28672, 30720, 32768}; // <Q15>
|
||
|
|
||
|
VPMDeflickering::VPMDeflickering() :
|
||
|
_id(0)
|
||
|
{
|
||
|
Reset();
|
||
|
}
|
||
|
|
||
|
VPMDeflickering::~VPMDeflickering()
|
||
|
{
|
||
|
}
|
||
|
|
||
|
WebRtc_Word32
|
||
|
VPMDeflickering::ChangeUniqueId(const WebRtc_Word32 id)
|
||
|
{
|
||
|
_id = id;
|
||
|
return 0;
|
||
|
}
|
||
|
|
||
|
void
|
||
|
VPMDeflickering::Reset()
|
||
|
{
|
||
|
_meanBufferLength = 0;
|
||
|
_detectionState = 0;
|
||
|
_frameRate = 0;
|
||
|
|
||
|
memset(_meanBuffer, 0, sizeof(WebRtc_Word32) * kMeanBufferLength);
|
||
|
memset(_timestampBuffer, 0, sizeof(WebRtc_Word32) * kMeanBufferLength);
|
||
|
|
||
|
// Initialize the history with a uniformly distributed histogram
|
||
|
_quantHistUW8[0][0] = 0;
|
||
|
_quantHistUW8[0][kNumQuants - 1] = 255;
|
||
|
for (WebRtc_Word32 i = 0; i < kNumProbs; i++)
|
||
|
{
|
||
|
_quantHistUW8[0][i + 1] = static_cast<WebRtc_UWord8>((WEBRTC_SPL_UMUL_16_16(
|
||
|
_probUW16[i], 255) + (1 << 10)) >> 11); // Unsigned round. <Q0>
|
||
|
}
|
||
|
|
||
|
for (WebRtc_Word32 i = 1; i < kFrameHistorySize; i++)
|
||
|
{
|
||
|
memcpy(_quantHistUW8[i], _quantHistUW8[0], sizeof(WebRtc_UWord8) * kNumQuants);
|
||
|
}
|
||
|
}
|
||
|
|
||
|
WebRtc_Word32
|
||
|
VPMDeflickering::ProcessFrame(WebRtc_UWord8* frame,
|
||
|
const WebRtc_UWord32 width,
|
||
|
const WebRtc_UWord32 height,
|
||
|
const WebRtc_UWord32 timestamp,
|
||
|
VideoProcessingModule::FrameStats& stats)
|
||
|
{
|
||
|
WebRtc_UWord32 frameMemory;
|
||
|
WebRtc_UWord8 quantUW8[kNumQuants];
|
||
|
WebRtc_UWord8 maxQuantUW8[kNumQuants];
|
||
|
WebRtc_UWord8 minQuantUW8[kNumQuants];
|
||
|
WebRtc_UWord16 targetQuantUW16[kNumQuants];
|
||
|
WebRtc_UWord16 incrementUW16;
|
||
|
WebRtc_UWord8 mapUW8[256];
|
||
|
|
||
|
WebRtc_UWord16 tmpUW16;
|
||
|
WebRtc_UWord32 tmpUW32;
|
||
|
|
||
|
if (frame == NULL)
|
||
|
{
|
||
|
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, "Null frame pointer");
|
||
|
return VPM_GENERAL_ERROR;
|
||
|
}
|
||
|
|
||
|
// Stricter height check due to subsampling size calculation below.
|
||
|
if (width == 0 || height < 2)
|
||
|
{
|
||
|
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, "Invalid frame size");
|
||
|
return VPM_GENERAL_ERROR;
|
||
|
}
|
||
|
|
||
|
if (!VideoProcessingModule::ValidFrameStats(stats))
|
||
|
{
|
||
|
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, "Invalid frame stats");
|
||
|
return VPM_GENERAL_ERROR;
|
||
|
}
|
||
|
|
||
|
if (PreDetection(timestamp, stats) == -1)
|
||
|
{
|
||
|
return VPM_GENERAL_ERROR;
|
||
|
}
|
||
|
|
||
|
// Flicker detection
|
||
|
WebRtc_Word32 detFlicker = DetectFlicker();
|
||
|
if (detFlicker < 0)
|
||
|
{ // Error
|
||
|
return VPM_GENERAL_ERROR;
|
||
|
}
|
||
|
else if (detFlicker != 1)
|
||
|
{
|
||
|
return 0;
|
||
|
}
|
||
|
|
||
|
// Size of luminance component
|
||
|
const WebRtc_UWord32 ySize = height * width;
|
||
|
|
||
|
const WebRtc_UWord32 ySubSize = width * (((height - 1) >>
|
||
|
kLog2OfDownsamplingFactor) + 1);
|
||
|
WebRtc_UWord8* ySorted = new WebRtc_UWord8[ySubSize];
|
||
|
WebRtc_UWord32 sortRowIdx = 0;
|
||
|
for (WebRtc_UWord32 i = 0; i < height; i += kDownsamplingFactor)
|
||
|
{
|
||
|
memcpy(ySorted + sortRowIdx * width, frame + i * width, width);
|
||
|
sortRowIdx++;
|
||
|
}
|
||
|
|
||
|
webrtc::Sort(ySorted, ySubSize, webrtc::TYPE_UWord8);
|
||
|
|
||
|
WebRtc_UWord32 probIdxUW32 = 0;
|
||
|
quantUW8[0] = 0;
|
||
|
quantUW8[kNumQuants - 1] = 255;
|
||
|
|
||
|
// Ensure we won't get an overflow below.
|
||
|
// In practice, the number of subsampled pixels will not become this large.
|
||
|
if (ySubSize > (1 << 21) - 1)
|
||
|
{
|
||
|
WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id,
|
||
|
"Subsampled number of pixels too large");
|
||
|
return -1;
|
||
|
}
|
||
|
|
||
|
for (WebRtc_Word32 i = 0; i < kNumProbs; i++)
|
||
|
{
|
||
|
probIdxUW32 = WEBRTC_SPL_UMUL_32_16(ySubSize, _probUW16[i]) >> 11; // <Q0>
|
||
|
quantUW8[i + 1] = ySorted[probIdxUW32];
|
||
|
}
|
||
|
|
||
|
delete [] ySorted;
|
||
|
ySorted = NULL;
|
||
|
|
||
|
// Shift history for new frame.
|
||
|
memmove(_quantHistUW8[1], _quantHistUW8[0], (kFrameHistorySize - 1) * kNumQuants *
|
||
|
sizeof(WebRtc_UWord8));
|
||
|
// Store current frame in history.
|
||
|
memcpy(_quantHistUW8[0], quantUW8, kNumQuants * sizeof(WebRtc_UWord8));
|
||
|
|
||
|
// We use a frame memory equal to the ceiling of half the frame rate to ensure we
|
||
|
// capture an entire period of flicker.
|
||
|
frameMemory = (_frameRate + (1 << 5)) >> 5; // Unsigned ceiling. <Q0>
|
||
|
// _frameRate in Q4.
|
||
|
if (frameMemory > kFrameHistorySize)
|
||
|
{
|
||
|
frameMemory = kFrameHistorySize;
|
||
|
}
|
||
|
|
||
|
// Get maximum and minimum.
|
||
|
for (WebRtc_Word32 i = 0; i < kNumQuants; i++)
|
||
|
{
|
||
|
maxQuantUW8[i] = 0;
|
||
|
minQuantUW8[i] = 255;
|
||
|
for (WebRtc_UWord32 j = 0; j < frameMemory; j++)
|
||
|
{
|
||
|
if (_quantHistUW8[j][i] > maxQuantUW8[i])
|
||
|
{
|
||
|
maxQuantUW8[i] = _quantHistUW8[j][i];
|
||
|
}
|
||
|
|
||
|
if (_quantHistUW8[j][i] < minQuantUW8[i])
|
||
|
{
|
||
|
minQuantUW8[i] = _quantHistUW8[j][i];
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
// Get target quantiles.
|
||
|
for (WebRtc_Word32 i = 0; i < kNumQuants - kMaxOnlyLength; i++)
|
||
|
{
|
||
|
targetQuantUW16[i] = static_cast<WebRtc_UWord16>((WEBRTC_SPL_UMUL_16_16(
|
||
|
_weightUW16[i], maxQuantUW8[i]) + WEBRTC_SPL_UMUL_16_16((1 << 15) -
|
||
|
_weightUW16[i], minQuantUW8[i])) >> 8); // <Q7>
|
||
|
}
|
||
|
|
||
|
for (WebRtc_Word32 i = kNumQuants - kMaxOnlyLength; i < kNumQuants; i++)
|
||
|
{
|
||
|
targetQuantUW16[i] = ((WebRtc_UWord16)maxQuantUW8[i]) << 7;
|
||
|
}
|
||
|
|
||
|
// Compute the map from input to output pixels.
|
||
|
WebRtc_UWord16 mapUW16; // <Q7>
|
||
|
for (WebRtc_Word32 i = 1; i < kNumQuants; i++)
|
||
|
{
|
||
|
// As quant and targetQuant are limited to UWord8, we're safe to use Q7 here.
|
||
|
tmpUW32 = static_cast<WebRtc_UWord32>(targetQuantUW16[i] -
|
||
|
targetQuantUW16[i - 1]); // <Q7>
|
||
|
tmpUW16 = static_cast<WebRtc_UWord16>(quantUW8[i] - quantUW8[i - 1]); // <Q0>
|
||
|
|
||
|
if (tmpUW16 > 0)
|
||
|
{
|
||
|
incrementUW16 = static_cast<WebRtc_UWord16>(WebRtcSpl_DivU32U16(tmpUW32,
|
||
|
tmpUW16)); // <Q7>
|
||
|
}
|
||
|
else
|
||
|
{
|
||
|
// The value is irrelevant; the loop below will only iterate once.
|
||
|
incrementUW16 = 0;
|
||
|
}
|
||
|
|
||
|
mapUW16 = targetQuantUW16[i - 1];
|
||
|
for (WebRtc_UWord32 j = quantUW8[i - 1]; j < (WebRtc_UWord32)(quantUW8[i] + 1); j++)
|
||
|
{
|
||
|
mapUW8[j] = (WebRtc_UWord8)((mapUW16 + (1 << 6)) >> 7); // Unsigned round. <Q0>
|
||
|
mapUW16 += incrementUW16;
|
||
|
}
|
||
|
}
|
||
|
|
||
|
// Map to the output frame.
|
||
|
for (WebRtc_UWord32 i = 0; i < ySize; i++)
|
||
|
{
|
||
|
frame[i] = mapUW8[frame[i]];
|
||
|
}
|
||
|
|
||
|
// Frame was altered, so reset stats.
|
||
|
VideoProcessingModule::ClearFrameStats(stats);
|
||
|
|
||
|
return 0;
|
||
|
}
|
||
|
|
||
|
/**
|
||
|
Performs some pre-detection operations. Must be called before
|
||
|
DetectFlicker().
|
||
|
|
||
|
\param[in] timestamp Timestamp of the current frame.
|
||
|
\param[in] stats Statistics of the current frame.
|
||
|
|
||
|
\return 0: Success\n
|
||
|
2: Detection not possible due to flickering frequency too close to
|
||
|
zero.\n
|
||
|
-1: Error
|
||
|
*/
|
||
|
WebRtc_Word32
|
||
|
VPMDeflickering::PreDetection(const WebRtc_UWord32 timestamp,
|
||
|
const VideoProcessingModule::FrameStats& stats)
|
||
|
{
|
||
|
WebRtc_Word32 meanVal; // Mean value of frame (Q4)
|
||
|
WebRtc_UWord32 frameRate = 0;
|
||
|
WebRtc_Word32 meanBufferLength; // Temp variable
|
||
|
|
||
|
meanVal = ((stats.sum << kMeanValueScaling) / stats.numPixels);
|
||
|
/* Update mean value buffer.
|
||
|
* This should be done even though we might end up in an unreliable detection.
|
||
|
*/
|
||
|
memmove(_meanBuffer + 1, _meanBuffer, (kMeanBufferLength - 1) * sizeof(WebRtc_Word32));
|
||
|
_meanBuffer[0] = meanVal;
|
||
|
|
||
|
/* Update timestamp buffer.
|
||
|
* This should be done even though we might end up in an unreliable detection.
|
||
|
*/
|
||
|
memmove(_timestampBuffer + 1, _timestampBuffer, (kMeanBufferLength - 1) *
|
||
|
sizeof(WebRtc_UWord32));
|
||
|
_timestampBuffer[0] = timestamp;
|
||
|
|
||
|
/* Compute current frame rate (Q4) */
|
||
|
if (_timestampBuffer[kMeanBufferLength - 1] != 0)
|
||
|
{
|
||
|
frameRate = ((90000 << 4) * (kMeanBufferLength - 1));
|
||
|
frameRate /= (_timestampBuffer[0] - _timestampBuffer[kMeanBufferLength - 1]);
|
||
|
}else if (_timestampBuffer[1] != 0)
|
||
|
{
|
||
|
frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]);
|
||
|
}
|
||
|
|
||
|
/* Determine required size of mean value buffer (_meanBufferLength) */
|
||
|
if (frameRate == 0) {
|
||
|
meanBufferLength = 1;
|
||
|
}
|
||
|
else {
|
||
|
meanBufferLength = (kNumFlickerBeforeDetect * frameRate) / kMinFrequencyToDetect;
|
||
|
}
|
||
|
/* Sanity check of buffer length */
|
||
|
if (meanBufferLength >= kMeanBufferLength)
|
||
|
{
|
||
|
/* Too long buffer. The flickering frequency is too close to zero, which
|
||
|
* makes the estimation unreliable.
|
||
|
*/
|
||
|
_meanBufferLength = 0;
|
||
|
return 2;
|
||
|
}
|
||
|
_meanBufferLength = meanBufferLength;
|
||
|
|
||
|
if ((_timestampBuffer[_meanBufferLength - 1] != 0) && (_meanBufferLength != 1))
|
||
|
{
|
||
|
frameRate = ((90000 << 4) * (_meanBufferLength - 1));
|
||
|
frameRate /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]);
|
||
|
}else if (_timestampBuffer[1] != 0)
|
||
|
{
|
||
|
frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]);
|
||
|
}
|
||
|
_frameRate = frameRate;
|
||
|
|
||
|
return 0;
|
||
|
}
|
||
|
|
||
|
/**
|
||
|
This function detects flicker in the video stream. As a side effect the mean value
|
||
|
buffer is updated with the new mean value.
|
||
|
|
||
|
\return 0: No flickering detected\n
|
||
|
1: Flickering detected\n
|
||
|
2: Detection not possible due to unreliable frequency interval
|
||
|
-1: Error
|
||
|
*/
|
||
|
WebRtc_Word32 VPMDeflickering::DetectFlicker()
|
||
|
{
|
||
|
/* Local variables */
|
||
|
WebRtc_UWord32 i;
|
||
|
WebRtc_Word32 freqEst; // (Q4) Frequency estimate to base detection upon
|
||
|
WebRtc_Word32 retVal = -1;
|
||
|
|
||
|
/* Sanity check for _meanBufferLength */
|
||
|
if (_meanBufferLength < 2)
|
||
|
{
|
||
|
/* Not possible to estimate frequency */
|
||
|
return(2);
|
||
|
}
|
||
|
/* Count zero crossings with a dead zone to be robust against noise.
|
||
|
* If the noise std is 2 pixel this corresponds to about 95% confidence interval.
|
||
|
*/
|
||
|
WebRtc_Word32 deadzone = (kZeroCrossingDeadzone << kMeanValueScaling); // Q4
|
||
|
WebRtc_Word32 meanOfBuffer = 0; // Mean value of mean value buffer
|
||
|
WebRtc_Word32 numZeros = 0; // Number of zeros that cross the deadzone
|
||
|
WebRtc_Word32 cntState = 0; // State variable for zero crossing regions
|
||
|
WebRtc_Word32 cntStateOld = 0; // Previous state variable for zero crossing regions
|
||
|
|
||
|
for (i = 0; i < _meanBufferLength; i++)
|
||
|
{
|
||
|
meanOfBuffer += _meanBuffer[i];
|
||
|
}
|
||
|
meanOfBuffer += (_meanBufferLength >> 1); // Rounding, not truncation
|
||
|
meanOfBuffer /= _meanBufferLength;
|
||
|
|
||
|
/* Count zero crossings */
|
||
|
cntStateOld = (_meanBuffer[0] >= (meanOfBuffer + deadzone));
|
||
|
cntStateOld -= (_meanBuffer[0] <= (meanOfBuffer - deadzone));
|
||
|
for (i = 1; i < _meanBufferLength; i++)
|
||
|
{
|
||
|
cntState = (_meanBuffer[i] >= (meanOfBuffer + deadzone));
|
||
|
cntState -= (_meanBuffer[i] <= (meanOfBuffer - deadzone));
|
||
|
if (cntStateOld == 0)
|
||
|
{
|
||
|
cntStateOld = -cntState;
|
||
|
}
|
||
|
if (((cntState + cntStateOld) == 0) && (cntState != 0))
|
||
|
{
|
||
|
numZeros++;
|
||
|
cntStateOld = cntState;
|
||
|
}
|
||
|
}
|
||
|
/* END count zero crossings */
|
||
|
|
||
|
/* Frequency estimation according to:
|
||
|
* freqEst = numZeros * frameRate / 2 / _meanBufferLength;
|
||
|
*
|
||
|
* Resolution is set to Q4
|
||
|
*/
|
||
|
freqEst = ((numZeros * 90000) << 3);
|
||
|
freqEst /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]);
|
||
|
|
||
|
/* Translate frequency estimate to regions close to 100 and 120 Hz */
|
||
|
WebRtc_UWord8 freqState = 0; // Current translation state;
|
||
|
// (0) Not in interval,
|
||
|
// (1) Within valid interval,
|
||
|
// (2) Out of range
|
||
|
WebRtc_Word32 freqAlias = freqEst;
|
||
|
if (freqEst > kMinFrequencyToDetect)
|
||
|
{
|
||
|
WebRtc_UWord8 aliasState = 1;
|
||
|
while(freqState == 0)
|
||
|
{
|
||
|
/* Increase frequency */
|
||
|
freqAlias += (aliasState * _frameRate);
|
||
|
freqAlias += ((freqEst << 1) * (1 - (aliasState << 1)));
|
||
|
/* Compute state */
|
||
|
freqState = (abs(freqAlias - (100 << 4)) <= kFrequencyDeviation);
|
||
|
freqState += (abs(freqAlias - (120 << 4)) <= kFrequencyDeviation);
|
||
|
freqState += 2 * (freqAlias > ((120 << 4) + kFrequencyDeviation));
|
||
|
/* Switch alias state */
|
||
|
aliasState++;
|
||
|
aliasState &= 0x01;
|
||
|
}
|
||
|
}
|
||
|
/* Is frequency estimate within detection region? */
|
||
|
if (freqState == 1)
|
||
|
{
|
||
|
retVal = 1;
|
||
|
}else if (freqState == 0)
|
||
|
{
|
||
|
retVal = 2;
|
||
|
}else
|
||
|
{
|
||
|
retVal = 0;
|
||
|
}
|
||
|
return retVal;
|
||
|
}
|
||
|
|
||
|
} //namespace
|