/* * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include #include #include "deflickering.h" #include "trace.h" #include "signal_processing_library.h" #include "sort.h" namespace webrtc { // Detection constants enum { kFrequencyDeviation = 39 }; // (Q4) Maximum allowed deviation for detection enum { kMinFrequencyToDetect = 32 }; // (Q4) Minimum frequency that can be detected enum { kNumFlickerBeforeDetect = 2 }; // Number of flickers before we accept detection enum { kMeanValueScaling = 4 }; // (Q4) In power of 2 enum { kZeroCrossingDeadzone = 10 }; // Deadzone region in terms of pixel values // Deflickering constants // Compute the quantiles over 1 / DownsamplingFactor of the image. enum { kDownsamplingFactor = 8 }; enum { kLog2OfDownsamplingFactor = 3 }; // To generate in Matlab: // >> probUW16 = round(2^11 * [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.97]); // >> fprintf('%d, ', probUW16) // Resolution reduced to avoid overflow when multiplying with the (potentially) large // number of pixels. const WebRtc_UWord16 VPMDeflickering::_probUW16[kNumProbs] = {102, 205, 410, 614, 819, 1024, 1229, 1434, 1638, 1843, 1946, 1987}; // // To generate in Matlab: // >> numQuants = 14; maxOnlyLength = 5; // >> weightUW16 = round(2^15 * [linspace(0.5, 1.0, numQuants - maxOnlyLength)]); // >> fprintf('%d, %d,\n ', weightUW16); const WebRtc_UWord16 VPMDeflickering::_weightUW16[kNumQuants - kMaxOnlyLength] = {16384, 18432, 20480, 22528, 24576, 26624, 28672, 30720, 32768}; // VPMDeflickering::VPMDeflickering() : _id(0) { Reset(); } VPMDeflickering::~VPMDeflickering() { } WebRtc_Word32 VPMDeflickering::ChangeUniqueId(const WebRtc_Word32 id) { _id = id; return 0; } void VPMDeflickering::Reset() { _meanBufferLength = 0; _detectionState = 0; _frameRate = 0; memset(_meanBuffer, 0, sizeof(WebRtc_Word32) * kMeanBufferLength); memset(_timestampBuffer, 0, sizeof(WebRtc_Word32) * kMeanBufferLength); // Initialize the history with a uniformly distributed histogram _quantHistUW8[0][0] = 0; _quantHistUW8[0][kNumQuants - 1] = 255; for (WebRtc_Word32 i = 0; i < kNumProbs; i++) { _quantHistUW8[0][i + 1] = static_cast((WEBRTC_SPL_UMUL_16_16( _probUW16[i], 255) + (1 << 10)) >> 11); // Unsigned round. } for (WebRtc_Word32 i = 1; i < kFrameHistorySize; i++) { memcpy(_quantHistUW8[i], _quantHistUW8[0], sizeof(WebRtc_UWord8) * kNumQuants); } } WebRtc_Word32 VPMDeflickering::ProcessFrame(WebRtc_UWord8* frame, const WebRtc_UWord32 width, const WebRtc_UWord32 height, const WebRtc_UWord32 timestamp, VideoProcessingModule::FrameStats& stats) { WebRtc_UWord32 frameMemory; WebRtc_UWord8 quantUW8[kNumQuants]; WebRtc_UWord8 maxQuantUW8[kNumQuants]; WebRtc_UWord8 minQuantUW8[kNumQuants]; WebRtc_UWord16 targetQuantUW16[kNumQuants]; WebRtc_UWord16 incrementUW16; WebRtc_UWord8 mapUW8[256]; WebRtc_UWord16 tmpUW16; WebRtc_UWord32 tmpUW32; if (frame == NULL) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, "Null frame pointer"); return VPM_GENERAL_ERROR; } // Stricter height check due to subsampling size calculation below. if (width == 0 || height < 2) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, "Invalid frame size"); return VPM_GENERAL_ERROR; } if (!VideoProcessingModule::ValidFrameStats(stats)) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, "Invalid frame stats"); return VPM_GENERAL_ERROR; } if (PreDetection(timestamp, stats) == -1) { return VPM_GENERAL_ERROR; } // Flicker detection WebRtc_Word32 detFlicker = DetectFlicker(); if (detFlicker < 0) { // Error return VPM_GENERAL_ERROR; } else if (detFlicker != 1) { return 0; } // Size of luminance component const WebRtc_UWord32 ySize = height * width; const WebRtc_UWord32 ySubSize = width * (((height - 1) >> kLog2OfDownsamplingFactor) + 1); WebRtc_UWord8* ySorted = new WebRtc_UWord8[ySubSize]; WebRtc_UWord32 sortRowIdx = 0; for (WebRtc_UWord32 i = 0; i < height; i += kDownsamplingFactor) { memcpy(ySorted + sortRowIdx * width, frame + i * width, width); sortRowIdx++; } webrtc::Sort(ySorted, ySubSize, webrtc::TYPE_UWord8); WebRtc_UWord32 probIdxUW32 = 0; quantUW8[0] = 0; quantUW8[kNumQuants - 1] = 255; // Ensure we won't get an overflow below. // In practice, the number of subsampled pixels will not become this large. if (ySubSize > (1 << 21) - 1) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoPreocessing, _id, "Subsampled number of pixels too large"); return -1; } for (WebRtc_Word32 i = 0; i < kNumProbs; i++) { probIdxUW32 = WEBRTC_SPL_UMUL_32_16(ySubSize, _probUW16[i]) >> 11; // quantUW8[i + 1] = ySorted[probIdxUW32]; } delete [] ySorted; ySorted = NULL; // Shift history for new frame. memmove(_quantHistUW8[1], _quantHistUW8[0], (kFrameHistorySize - 1) * kNumQuants * sizeof(WebRtc_UWord8)); // Store current frame in history. memcpy(_quantHistUW8[0], quantUW8, kNumQuants * sizeof(WebRtc_UWord8)); // We use a frame memory equal to the ceiling of half the frame rate to ensure we // capture an entire period of flicker. frameMemory = (_frameRate + (1 << 5)) >> 5; // Unsigned ceiling. // _frameRate in Q4. if (frameMemory > kFrameHistorySize) { frameMemory = kFrameHistorySize; } // Get maximum and minimum. for (WebRtc_Word32 i = 0; i < kNumQuants; i++) { maxQuantUW8[i] = 0; minQuantUW8[i] = 255; for (WebRtc_UWord32 j = 0; j < frameMemory; j++) { if (_quantHistUW8[j][i] > maxQuantUW8[i]) { maxQuantUW8[i] = _quantHistUW8[j][i]; } if (_quantHistUW8[j][i] < minQuantUW8[i]) { minQuantUW8[i] = _quantHistUW8[j][i]; } } } // Get target quantiles. for (WebRtc_Word32 i = 0; i < kNumQuants - kMaxOnlyLength; i++) { targetQuantUW16[i] = static_cast((WEBRTC_SPL_UMUL_16_16( _weightUW16[i], maxQuantUW8[i]) + WEBRTC_SPL_UMUL_16_16((1 << 15) - _weightUW16[i], minQuantUW8[i])) >> 8); // } for (WebRtc_Word32 i = kNumQuants - kMaxOnlyLength; i < kNumQuants; i++) { targetQuantUW16[i] = ((WebRtc_UWord16)maxQuantUW8[i]) << 7; } // Compute the map from input to output pixels. WebRtc_UWord16 mapUW16; // for (WebRtc_Word32 i = 1; i < kNumQuants; i++) { // As quant and targetQuant are limited to UWord8, we're safe to use Q7 here. tmpUW32 = static_cast(targetQuantUW16[i] - targetQuantUW16[i - 1]); // tmpUW16 = static_cast(quantUW8[i] - quantUW8[i - 1]); // if (tmpUW16 > 0) { incrementUW16 = static_cast(WebRtcSpl_DivU32U16(tmpUW32, tmpUW16)); // } else { // The value is irrelevant; the loop below will only iterate once. incrementUW16 = 0; } mapUW16 = targetQuantUW16[i - 1]; for (WebRtc_UWord32 j = quantUW8[i - 1]; j < (WebRtc_UWord32)(quantUW8[i] + 1); j++) { mapUW8[j] = (WebRtc_UWord8)((mapUW16 + (1 << 6)) >> 7); // Unsigned round. mapUW16 += incrementUW16; } } // Map to the output frame. for (WebRtc_UWord32 i = 0; i < ySize; i++) { frame[i] = mapUW8[frame[i]]; } // Frame was altered, so reset stats. VideoProcessingModule::ClearFrameStats(stats); return 0; } /** Performs some pre-detection operations. Must be called before DetectFlicker(). \param[in] timestamp Timestamp of the current frame. \param[in] stats Statistics of the current frame. \return 0: Success\n 2: Detection not possible due to flickering frequency too close to zero.\n -1: Error */ WebRtc_Word32 VPMDeflickering::PreDetection(const WebRtc_UWord32 timestamp, const VideoProcessingModule::FrameStats& stats) { WebRtc_Word32 meanVal; // Mean value of frame (Q4) WebRtc_UWord32 frameRate = 0; WebRtc_Word32 meanBufferLength; // Temp variable meanVal = ((stats.sum << kMeanValueScaling) / stats.numPixels); /* Update mean value buffer. * This should be done even though we might end up in an unreliable detection. */ memmove(_meanBuffer + 1, _meanBuffer, (kMeanBufferLength - 1) * sizeof(WebRtc_Word32)); _meanBuffer[0] = meanVal; /* Update timestamp buffer. * This should be done even though we might end up in an unreliable detection. */ memmove(_timestampBuffer + 1, _timestampBuffer, (kMeanBufferLength - 1) * sizeof(WebRtc_UWord32)); _timestampBuffer[0] = timestamp; /* Compute current frame rate (Q4) */ if (_timestampBuffer[kMeanBufferLength - 1] != 0) { frameRate = ((90000 << 4) * (kMeanBufferLength - 1)); frameRate /= (_timestampBuffer[0] - _timestampBuffer[kMeanBufferLength - 1]); }else if (_timestampBuffer[1] != 0) { frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]); } /* Determine required size of mean value buffer (_meanBufferLength) */ if (frameRate == 0) { meanBufferLength = 1; } else { meanBufferLength = (kNumFlickerBeforeDetect * frameRate) / kMinFrequencyToDetect; } /* Sanity check of buffer length */ if (meanBufferLength >= kMeanBufferLength) { /* Too long buffer. The flickering frequency is too close to zero, which * makes the estimation unreliable. */ _meanBufferLength = 0; return 2; } _meanBufferLength = meanBufferLength; if ((_timestampBuffer[_meanBufferLength - 1] != 0) && (_meanBufferLength != 1)) { frameRate = ((90000 << 4) * (_meanBufferLength - 1)); frameRate /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]); }else if (_timestampBuffer[1] != 0) { frameRate = (90000 << 4) / (_timestampBuffer[0] - _timestampBuffer[1]); } _frameRate = frameRate; return 0; } /** This function detects flicker in the video stream. As a side effect the mean value buffer is updated with the new mean value. \return 0: No flickering detected\n 1: Flickering detected\n 2: Detection not possible due to unreliable frequency interval -1: Error */ WebRtc_Word32 VPMDeflickering::DetectFlicker() { /* Local variables */ WebRtc_UWord32 i; WebRtc_Word32 freqEst; // (Q4) Frequency estimate to base detection upon WebRtc_Word32 retVal = -1; /* Sanity check for _meanBufferLength */ if (_meanBufferLength < 2) { /* Not possible to estimate frequency */ return(2); } /* Count zero crossings with a dead zone to be robust against noise. * If the noise std is 2 pixel this corresponds to about 95% confidence interval. */ WebRtc_Word32 deadzone = (kZeroCrossingDeadzone << kMeanValueScaling); // Q4 WebRtc_Word32 meanOfBuffer = 0; // Mean value of mean value buffer WebRtc_Word32 numZeros = 0; // Number of zeros that cross the deadzone WebRtc_Word32 cntState = 0; // State variable for zero crossing regions WebRtc_Word32 cntStateOld = 0; // Previous state variable for zero crossing regions for (i = 0; i < _meanBufferLength; i++) { meanOfBuffer += _meanBuffer[i]; } meanOfBuffer += (_meanBufferLength >> 1); // Rounding, not truncation meanOfBuffer /= _meanBufferLength; /* Count zero crossings */ cntStateOld = (_meanBuffer[0] >= (meanOfBuffer + deadzone)); cntStateOld -= (_meanBuffer[0] <= (meanOfBuffer - deadzone)); for (i = 1; i < _meanBufferLength; i++) { cntState = (_meanBuffer[i] >= (meanOfBuffer + deadzone)); cntState -= (_meanBuffer[i] <= (meanOfBuffer - deadzone)); if (cntStateOld == 0) { cntStateOld = -cntState; } if (((cntState + cntStateOld) == 0) && (cntState != 0)) { numZeros++; cntStateOld = cntState; } } /* END count zero crossings */ /* Frequency estimation according to: * freqEst = numZeros * frameRate / 2 / _meanBufferLength; * * Resolution is set to Q4 */ freqEst = ((numZeros * 90000) << 3); freqEst /= (_timestampBuffer[0] - _timestampBuffer[_meanBufferLength - 1]); /* Translate frequency estimate to regions close to 100 and 120 Hz */ WebRtc_UWord8 freqState = 0; // Current translation state; // (0) Not in interval, // (1) Within valid interval, // (2) Out of range WebRtc_Word32 freqAlias = freqEst; if (freqEst > kMinFrequencyToDetect) { WebRtc_UWord8 aliasState = 1; while(freqState == 0) { /* Increase frequency */ freqAlias += (aliasState * _frameRate); freqAlias += ((freqEst << 1) * (1 - (aliasState << 1))); /* Compute state */ freqState = (abs(freqAlias - (100 << 4)) <= kFrequencyDeviation); freqState += (abs(freqAlias - (120 << 4)) <= kFrequencyDeviation); freqState += 2 * (freqAlias > ((120 << 4) + kFrequencyDeviation)); /* Switch alias state */ aliasState++; aliasState &= 0x01; } } /* Is frequency estimate within detection region? */ if (freqState == 1) { retVal = 1; }else if (freqState == 0) { retVal = 2; }else { retVal = 0; } return retVal; } } //namespace