From 8db245b6a1754849ebd305f93dc011275b2d56ed Mon Sep 17 00:00:00 2001 From: Marco Paniconi Date: Wed, 6 Aug 2014 17:19:29 -0700 Subject: [PATCH] Add an adaptive denoising mode. On key frame, will always start with normal denoising mode, but based on a computed noise metric (normalized mse on source diff) may switch to aggressive mode (and back down again). Change-Id: I20330b2dcf3056287be37223302b2cab5fc103eb --- examples/vpx_temporal_svc_encoder.c | 3 +- vp8/common/onyx.h | 3 +- vp8/encoder/denoising.c | 18 +++- vp8/encoder/denoising.h | 12 ++- vp8/encoder/onyx_if.c | 128 ++++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 5 deletions(-) diff --git a/examples/vpx_temporal_svc_encoder.c b/examples/vpx_temporal_svc_encoder.c index be3e7b2f1..4ec18482e 100644 --- a/examples/vpx_temporal_svc_encoder.c +++ b/examples/vpx_temporal_svc_encoder.c @@ -37,7 +37,8 @@ enum denoiserState { kDenoiserOff, kDenoiserOnYOnly, kDenoiserOnYUV, - kDenoiserOnYUVAggressive // Aggressive mode not implemented currently. + kDenoiserOnYUVAggressive, + kDenoiserOnAdaptive }; static int mode_to_num_layers[12] = {1, 2, 2, 3, 3, 3, 3, 5, 2, 3, 3, 3}; diff --git a/vp8/common/onyx.h b/vp8/common/onyx.h index a46fbfbbd..b05ad146c 100644 --- a/vp8/common/onyx.h +++ b/vp8/common/onyx.h @@ -108,7 +108,8 @@ extern "C" * For temporal denoiser: noise_sensitivity = 0 means off, * noise_sensitivity = 1 means temporal denoiser on for Y channel only, * noise_sensitivity = 2 means temporal denoiser on for all channels. - * noise_sensitivity >= 3 means aggressive denoising mode. + * noise_sensitivity = 3 means aggressive denoising mode. + * noise_sensitivity >= 4 means adaptive denoising mode. * Temporal denoiser is enabled via the configuration option: * CONFIG_TEMPORAL_DENOISING. * For spatial denoiser: noise_sensitivity controls the amount of diff --git a/vp8/encoder/denoising.c b/vp8/encoder/denoising.c index 75401fc2b..4621d13e2 100644 --- a/vp8/encoder/denoising.c +++ b/vp8/encoder/denoising.c @@ -341,8 +341,10 @@ void vp8_denoiser_set_parameters(VP8_DENOISER *denoiser, int mode) { denoiser->denoiser_mode = kDenoiserOnYOnly; } else if (mode == 2) { denoiser->denoiser_mode = kDenoiserOnYUV; - } else { + } else if (mode == 3) { denoiser->denoiser_mode = kDenoiserOnYUVAggressive; + } else { + denoiser->denoiser_mode = kDenoiserOnAdaptive; } if (denoiser->denoiser_mode != kDenoiserOnYUVAggressive) { denoiser->denoise_pars.scale_sse_thresh = 1; @@ -397,9 +399,23 @@ int vp8_denoiser_allocate(VP8_DENOISER *denoiser, int width, int height, vpx_memset(denoiser->yv12_mc_running_avg.buffer_alloc, 0, denoiser->yv12_mc_running_avg.frame_size); + if (vp8_yv12_alloc_frame_buffer(&denoiser->yv12_last_source, width, + height, VP8BORDERINPIXELS) < 0) { + vp8_denoiser_free(denoiser); + return 1; + } + vpx_memset(denoiser->yv12_last_source.buffer_alloc, 0, + denoiser->yv12_last_source.frame_size); + denoiser->denoise_state = vpx_calloc((num_mb_rows * num_mb_cols), 1); vpx_memset(denoiser->denoise_state, 0, (num_mb_rows * num_mb_cols)); vp8_denoiser_set_parameters(denoiser, mode); + denoiser->nmse_source_diff = 0; + denoiser->nmse_source_diff_count = 0; + // TODO(marpan): Adjust thresholds, including effect on resolution. + denoiser->threshold_aggressive_mode = 40; + if (width * height > 640 * 480) + denoiser->threshold_aggressive_mode = 180; return 0; } diff --git a/vp8/encoder/denoising.h b/vp8/encoder/denoising.h index 89832d3c2..e1844f18a 100644 --- a/vp8/encoder/denoising.h +++ b/vp8/encoder/denoising.h @@ -24,7 +24,7 @@ extern "C" { #define SUM_DIFF_THRESHOLD_UV (96) // (8 * 8 * 1.5) #define SUM_DIFF_THRESHOLD_HIGH_UV (8 * 8 * 2) -#define SUM_DIFF_FROM_AVG_THRESH_UV (8 * 8 * 4) +#define SUM_DIFF_FROM_AVG_THRESH_UV (8 * 8 * 8) #define MOTION_MAGNITUDE_THRESHOLD_UV (8*3) enum vp8_denoiser_decision @@ -43,7 +43,8 @@ enum vp8_denoiser_mode { kDenoiserOff, kDenoiserOnYOnly, kDenoiserOnYUV, - kDenoiserOnYUVAggressive + kDenoiserOnYUVAggressive, + kDenoiserOnAdaptive }; typedef struct { @@ -72,9 +73,14 @@ typedef struct vp8_denoiser { YV12_BUFFER_CONFIG yv12_running_avg[MAX_REF_FRAMES]; YV12_BUFFER_CONFIG yv12_mc_running_avg; + // TODO(marpan): Should remove yv12_last_source and use vp8_lookahead_peak. + YV12_BUFFER_CONFIG yv12_last_source; unsigned char* denoise_state; int num_mb_cols; int denoiser_mode; + int threshold_aggressive_mode; + int nmse_source_diff; + int nmse_source_diff_count; denoise_params denoise_pars; } VP8_DENOISER; @@ -83,6 +89,8 @@ int vp8_denoiser_allocate(VP8_DENOISER *denoiser, int width, int height, void vp8_denoiser_free(VP8_DENOISER *denoiser); +void vp8_denoiser_set_parameters(VP8_DENOISER *denoiser, int mode); + void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser, MACROBLOCK *x, unsigned int best_sse, diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 7140f2f1b..da223f127 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -3299,12 +3299,119 @@ static void update_reference_frames(VP8_COMP *cpi) &cpi->denoiser.yv12_running_avg[LAST_FRAME]); } } + if (cpi->oxcf.noise_sensitivity == 4) + vp8_yv12_copy_frame(cpi->Source, &cpi->denoiser.yv12_last_source); } #endif } +static void process_denoiser_mode_change(VP8_COMP *cpi) { + const VP8_COMMON *const cm = &cpi->common; + int i, j; + int total = 0; + int num_blocks = 0; + // Number of blocks skipped along row/column in computing the + // nmse (normalized mean square error) of source. + int skip = 2; + // Only select blocks for computing nmse that have been encoded + // as ZERO LAST min_consec_zero_last frames in a row. + int min_consec_zero_last = 10; + // Decision is tested for changing the denoising mode every + // num_mode_change times this function is called. Note that this + // function called every 8 frames, so (8 * num_mode_change) is number + // of frames where denoising mode change is tested for switch. + int num_mode_change = 15; + // Framerate factor, to compensate for larger mse at lower framerates. + // TODO(marpan): Adjust this factor, + int fac_framerate = cpi->output_framerate < 25.0f ? 80 : 100; + int tot_num_blocks = cm->mb_rows * cm->mb_cols; + int ystride = cpi->Source->y_stride; + unsigned char *src = cpi->Source->y_buffer; + unsigned char *dst = cpi->denoiser.yv12_last_source.y_buffer; + static const unsigned char const_source[16] = { + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128}; + + // Loop through the Y plane, every skip blocks along rows and columns, + // summing the normalized mean square error, only for blocks that have + // been encoded as ZEROMV LAST at least min_consec_zero_last least frames in + // a row and have small sum difference between current and previous frame. + // Normalization here is by the contrast of the current frame block. + for (i = 0; i < cm->Height; i += 16 * skip) { + int block_index_row = (i >> 4) * cm->mb_cols; + for (j = 0; j < cm->Width; j += 16 * skip) { + int index = block_index_row + (j >> 4); + if (cpi->consec_zero_last[index] >= min_consec_zero_last) { + unsigned int sse; + const unsigned int mse = vp8_mse16x16(src + j, + ystride, + dst + j, + ystride, + &sse); + const unsigned int var = vp8_variance16x16(src + j, + ystride, + dst + j, + ystride, + &sse); + // Only consider this block as valid for noise measurement + // if the sum_diff average of the current and previous frame + // is small (to avoid effects from lighting change). + if ((mse - var) < 256) { + const unsigned int act = vp8_variance16x16(src + j, + ystride, + const_source, + 0, + &sse); + if (act > 0) + total += mse / act; + num_blocks++; + } + } + } + src += 16 * skip * ystride; + dst += 16 * skip * ystride; + } + total = total * fac_framerate / 100; + + // Only consider this frame as valid sample if we have computed nmse over + // at least ~1/16 blocks, and Total > 0 (Total == 0 can happen if the + // application inputs duplicate frames, or contrast is all zero). + if (total > 0 && + (num_blocks > (tot_num_blocks >> 4))) { + // Update the recursive mean square source_diff. + if (cpi->denoiser.nmse_source_diff_count == 0) + // First sample in new interval. + cpi->denoiser.nmse_source_diff = total; + else + // For subsequent samples, use average with weight ~1/4 for new sample. + cpi->denoiser.nmse_source_diff = (int)((total >> 2) + + 3 * (cpi->denoiser.nmse_source_diff >> 2)); + cpi->denoiser.nmse_source_diff_count++; + } + // Check for changing the denoiser mode, when we have obtained #samples = + // num_mode_change. + if (cpi->denoiser.nmse_source_diff_count == num_mode_change) { + // Check for going up: from normal to aggressive mode. + if ((cpi->denoiser.denoiser_mode = kDenoiserOnYUV) && + (cpi->denoiser.nmse_source_diff > + cpi->denoiser.threshold_aggressive_mode)) { + vp8_denoiser_set_parameters(&cpi->denoiser, kDenoiserOnYUVAggressive); + } else { + // Check for going down: from aggressive to normal mode. + if ((cpi->denoiser.denoiser_mode = kDenoiserOnYUVAggressive) && + (cpi->denoiser.nmse_source_diff < + cpi->denoiser.threshold_aggressive_mode)) { + vp8_denoiser_set_parameters(&cpi->denoiser, kDenoiserOnYUV); + } + } + // Reset metric and counter for next interval. + cpi->denoiser.nmse_source_diff = 0; + cpi->denoiser.nmse_source_diff_count = 0; + } +} + void vp8_loopfilter_frame(VP8_COMP *cpi, VP8_COMMON *cm) { const FRAME_TYPE frame_type = cm->frame_type; @@ -3461,6 +3568,12 @@ static void encode_frame_to_data_rate { /* Key frame from VFW/auto-keyframe/first frame */ cm->frame_type = KEY_FRAME; +#if CONFIG_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity == 4) { + // For adaptive mode, reset denoiser to normal mode on key frame. + vp8_denoiser_set_parameters(&cpi->denoiser, kDenoiserOnYUV); + } +#endif } #if CONFIG_MULTI_RES_ENCODING @@ -4462,6 +4575,21 @@ static void encode_frame_to_data_rate cm->frame_to_show = &cm->yv12_fb[cm->new_fb_idx]; +#if CONFIG_TEMPORAL_DENOISING + // For the adaptive denoising mode (noise_sensitivity == 4), sample the mse + // of source diff (between current and previous frame), and determine if we + // should switch the denoiser mode. Sampling refers to computing the mse for + // a sub-sample of the frame (i.e., skip x blocks along row/column), and + // only for blocks in that set that have used ZEROMV LAST, along with some + // constraint on the sum diff between blocks. This process is called every + // ~8 frames, to further reduce complexity. + if (cpi->oxcf.noise_sensitivity == 4 && + cpi->frames_since_key % 8 == 0 && + cm->frame_type != KEY_FRAME) { + process_denoiser_mode_change(cpi); + } +#endif + #if CONFIG_MULTITHREAD if (cpi->b_multi_threaded) {