generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "vp9 SVC: Fix the denoiser frame buffer management."

Browse Source
This commit is contained in:
Jerome Jiang
2017-08-11 00:54:35 +00:00
committed by Gerrit Code Review
parent 15193ce51f 0f8ebddec4
commit d48be6ad73
3 changed files with 110 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
vp9/encoder/vp9_denoiser.c
View File

@@ -190,11 +190,12 @@ static VP9_DENOISER_DECISION perform_motion_compensation(
VP9_COMMON *const cm, VP9_DENOISER *denoiser, MACROBLOCK *mb, BLOCK_SIZE bs,
int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx,
int motion_magnitude, int is_skin, int *zeromv_filter, int consec_zeromv,
int num_spatial_layers, int width) {
int num_spatial_layers, int width, int lst_fb_idx, int gld_fb_idx,
int use_svc) {
const int sse_diff = (ctx->newmv_sse == UINT_MAX)
? 0
: ((int)ctx->zeromv_sse - (int)ctx->newmv_sse);
MV_REFERENCE_FRAME frame;
int frame;
MACROBLOCKD *filter_mbd = &mb->e_mbd;
MODE_INFO *mi = filter_mbd->mi[0];
MODE_INFO saved_mi;
@@ -202,8 +203,10 @@ static VP9_DENOISER_DECISION perform_motion_compensation(
struct buf_2d saved_dst[MAX_MB_PLANE];
struct buf_2d saved_pre[MAX_MB_PLANE];
RefBuffer *saved_block_refs[2];
MV_REFERENCE_FRAME saved_frame;
frame = ctx->best_reference_frame;
saved_mi = *mi;
if (is_skin && (motion_magnitude > 0 || consec_zeromv < 4)) return COPY_BLOCK;
@@ -246,6 +249,15 @@ static VP9_DENOISER_DECISION perform_motion_compensation(
}
}
saved_frame = frame;
// When using SVC, we need to map REF_FRAME to the frame buffer index.
if (use_svc) {
if (frame == LAST_FRAME)
frame = lst_fb_idx + 1;
else if (frame == GOLDEN_FRAME)
frame = gld_fb_idx + 1;
}
if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
// Restore everything to its original state
*mi = saved_mi;
@@ -292,7 +304,7 @@ static VP9_DENOISER_DECISION perform_motion_compensation(
denoiser->mc_running_avg_y.uv_stride, mi_row, mi_col);
filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
set_ref_ptrs(cm, filter_mbd, frame, NONE);
set_ref_ptrs(cm, filter_mbd, saved_frame, NONE);
vp9_build_inter_predictors_sby(filter_mbd, mi_row, mi_col, bs);
// Restore everything to its original state
@@ -370,7 +382,8 @@ void vp9_denoiser_denoise(VP9_COMP *cpi, MACROBLOCK *mb, int mi_row, int mi_col,
decision = perform_motion_compensation(
&cpi->common, denoiser, mb, bs, increase_denoising, mi_row, mi_col, ctx,
motion_magnitude, is_skin, &zeromv_filter, consec_zeromv,
cpi->svc.number_spatial_layers, cpi->Source->y_width);
cpi->svc.number_spatial_layers, cpi->Source->y_width, cpi->lst_fb_idx,
cpi->gld_fb_idx, cpi->use_svc);
if (decision == FILTER_BLOCK) {
decision = vp9_denoiser_filter(src.buf, src.stride, mc_avg_start,
@@ -417,23 +430,11 @@ static void swap_frame_buffer(YV12_BUFFER_CONFIG *const dest,
src->y_buffer = tmp_buf;
}
void vp9_denoise_init_svc(VP9_COMP *cpi) {
// For fixed pattern SVC, on base temporal layer. Note we only denoise
// higher spatial layer for SVC.
if (cpi->svc.temporal_layering_mode != VP9E_TEMPORAL_LAYERING_MODE_BYPASS &&
cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 &&
cpi->svc.temporal_layer_id == 0) {
VP9_DENOISER *denoiser = &cpi->denoiser;
copy_frame(&denoiser->running_avg_y[LAST_FRAME],
&denoiser->running_avg_y[GOLDEN_FRAME]);
}
}
void vp9_denoiser_update_frame_info(
VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
int resized, int svc_base_is_key, int svc_fixed_pattern,
int temporal_layer_id) {
int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
int svc_base_is_key) {
// Copy source into denoised reference buffers on KEY_FRAME or
// if the just encoded frame was resized. For SVC, copy source if the base
// spatial layer was key frame.
@@ -441,48 +442,42 @@ void vp9_denoiser_update_frame_info(
svc_base_is_key) {
int i;
// Start at 1 so as not to overwrite the INTRA_FRAME
for (i = 1; i < denoiser->num_ref_frames; ++i)
copy_frame(&denoiser->running_avg_y[i], &src);
for (i = 1; i < denoiser->num_ref_frames; ++i) {
if (denoiser->running_avg_y[i].buffer_alloc != NULL)
copy_frame(&denoiser->running_avg_y[i], &src);
}
denoiser->reset = 0;
return;
}
// If more than one refresh occurs, must copy frame buffer.
if (refresh_golden_frame + refresh_last_frame + refresh_alt_ref_frame > 1) {
if (refresh_golden_frame) {
copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame) > 1) {
if (refresh_alt_ref_frame) {
copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
// For fixed pattern SVC: update denoised last_frame if alt_ref is
// refreshed, only for non-zero temporal layer.
if (refresh_last_frame ||
(refresh_alt_ref_frame && svc_fixed_pattern && temporal_layer_id > 0)) {
copy_frame(&denoiser->running_avg_y[LAST_FRAME],
if (refresh_golden_frame) {
copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
if (refresh_last_frame) {
copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
} else {
if (refresh_alt_ref_frame) {
swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
if (refresh_golden_frame) {
swap_frame_buffer(&denoiser->running_avg_y[GOLDEN_FRAME],
swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
// For fixed pattern SVC: update denoised last_frame if alt_ref is
// refreshed, only for non-zero temporal layer.
if (refresh_last_frame ||
(refresh_alt_ref_frame && svc_fixed_pattern && temporal_layer_id > 0)) {
swap_frame_buffer(&denoiser->running_avg_y[LAST_FRAME],
if (refresh_last_frame) {
swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
}
// For fixed pattern SVC we need to keep track of denoised last_frame for base
// temporal layer (since alt_ref refresh may update denoised last_frame on
// the upper/middle temporal layers).We do this by copying the current
// denoised last into the denoised golden_frame, for temporal_layer_id = 0.
// For the fixed pattern SVC golden is always spatial reference and is never
// used for denoising, so we can use it to keep track of denoised last_frame.
if (svc_fixed_pattern && temporal_layer_id == 0) {
copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
&denoiser->running_avg_y[LAST_FRAME]);
}
}
void vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
@@ -509,21 +504,62 @@ void vp9_denoiser_update_frame_stats(MODE_INFO *mi, unsigned int sse,
}
}
static int vp9_denoiser_realloc_svc_helper(VP9_COMMON *cm,
VP9_DENOISER *denoiser, int fb_idx) {
int fail = 0;
if (denoiser->running_avg_y[fb_idx].buffer_alloc == NULL) {
fail =
vpx_alloc_frame_buffer(&denoiser->running_avg_y[fb_idx], cm->width,
cm->height, cm->subsampling_x, cm->subsampling_y,
#if CONFIG_VP9_HIGHBITDEPTH
cm->use_highbitdepth,
#endif
VP9_ENC_BORDER_IN_PIXELS, 0);
if (fail) {
vp9_denoiser_free(denoiser);
return 1;
}
}
return 0;
}
int vp9_denoiser_realloc_svc(VP9_COMMON *cm, VP9_DENOISER *denoiser,
int refresh_alt, int refresh_gld, int refresh_lst,
int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) {
int fail = 0;
if (refresh_alt) {
// Increase the frame buffer index by 1 to map it to the buffer index in the
// denoiser.
fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, alt_fb_idx + 1);
if (fail) return 1;
}
if (refresh_gld) {
fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, gld_fb_idx + 1);
if (fail) return 1;
}
if (refresh_lst) {
fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, lst_fb_idx + 1);
if (fail) return 1;
}
return 0;
}
int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
int width, int height, int ssx, int ssy,
#if CONFIG_VP9_HIGHBITDEPTH
int use_highbitdepth,
#endif
int border) {
int i, fail;
int i, fail, init_num_ref_frames;
const int legacy_byte_alignment = 0;
assert(denoiser != NULL);
denoiser->num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
denoiser->num_ref_frames = use_svc ? SVC_REF_FRAMES : NONSVC_REF_FRAMES;
init_num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
CHECK_MEM_ERROR(
cm, denoiser->running_avg_y,
vpx_calloc(denoiser->num_ref_frames, sizeof(denoiser->running_avg_y[0])));
for (i = 0; i < denoiser->num_ref_frames; ++i) {
for (i = 0; i < init_num_ref_frames; ++i) {
fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
ssx, ssy,
#if CONFIG_VP9_HIGHBITDEPTH
@@ -594,7 +630,8 @@ void vp9_denoiser_set_noise_level(VP9_DENOISER *denoiser, int noise_level) {
denoiser->prev_denoising_level = denoiser->denoising_level;
}
// Scale/increase the partition threshold for denoiser speed-up.
// Scale/increase the partition threshold
// for denoiser speed-up.
int64_t vp9_scale_part_thresh(int64_t threshold, VP9_DENOISER_LEVEL noise_level,
int content_state, int temporal_layer_id) {
if ((content_state == kLowSadLowSumdiff) ||
@@ -609,7 +646,8 @@ int64_t vp9_scale_part_thresh(int64_t threshold, VP9_DENOISER_LEVEL noise_level,
}
}
// Scale/increase the ac skip threshold for denoiser speed-up.
// Scale/increase the ac skip threshold for
// denoiser speed-up.
int64_t vp9_scale_acskip_thresh(int64_t threshold,
VP9_DENOISER_LEVEL noise_level, int abs_sumdiff,
int temporal_layer_id) {

14
vp9/encoder/vp9_denoiser.h
View File

@@ -25,6 +25,10 @@ extern "C" {
// need to allocate for it, and hence we need MAX_REF_FRAME - 1
#define NONSVC_REF_FRAMES MAX_REF_FRAMES - 1
// Number of frame buffers when SVC is used. [0] for current denoised buffer and
// [1..8] for REF_FRAMES
#define SVC_REF_FRAMES 9
typedef enum vp9_denoiser_decision {
COPY_BLOCK,
FILTER_BLOCK,
@@ -63,13 +67,11 @@ typedef struct {
struct VP9_COMP;
void vp9_denoise_init_svc(struct VP9_COMP *cpi);
void vp9_denoiser_update_frame_info(
VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
int resized, int svc_base_is_key, int svc_fixed_pattern,
int temporal_layer_id);
int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
int svc_base_is_key);
void vp9_denoiser_denoise(struct VP9_COMP *cpi, MACROBLOCK *mb, int mi_row,
int mi_col, BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx,
@@ -81,6 +83,10 @@ void vp9_denoiser_update_frame_stats(MODE_INFO *mi, unsigned int sse,
PREDICTION_MODE mode,
PICK_MODE_CONTEXT *ctx);
int vp9_denoiser_realloc_svc(VP9_COMMON *cm, VP9_DENOISER *denoiser,
int refresh_alt, int refresh_gld, int refresh_lst,
int alt_fb_idx, int gld_fb_idx, int lst_fb_idx);
int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
int width, int height, int ssx, int ssy,
#if CONFIG_VP9_HIGHBITDEPTH

24
vp9/encoder/vp9_encoder.c
View File

@@ -2833,21 +2833,29 @@ void vp9_update_reference_frames(VP9_COMP *cpi) {
if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
cpi->denoiser.denoising_level > kDenLowLow) {
int svc_base_is_key = 0;
int svc_fixed_pattern = 0;
if (cpi->use_svc) {
int realloc_fail = 0;
int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
cpi->svc.temporal_layer_id,
cpi->svc.number_temporal_layers);
LAYER_CONTEXT *lc = &cpi->svc.layer_context[layer];
svc_base_is_key = lc->is_key_frame;
svc_fixed_pattern = (cpi->svc.temporal_layering_mode !=
VP9E_TEMPORAL_LAYERING_MODE_BYPASS);
// Check if we need to allocate extra buffers in the denoiser for
// refreshed frames.
realloc_fail = vp9_denoiser_realloc_svc(
cm, &cpi->denoiser, cpi->refresh_alt_ref_frame,
cpi->refresh_golden_frame, cpi->refresh_last_frame, cpi->alt_fb_idx,
cpi->gld_fb_idx, cpi->lst_fb_idx);
if (realloc_fail)
vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
"Failed to re-allocate denoiser for SVC");
}
vp9_denoiser_update_frame_info(
&cpi->denoiser, *cpi->Source, cpi->common.frame_type,
cpi->refresh_alt_ref_frame, cpi->refresh_golden_frame,
cpi->refresh_last_frame, cpi->resize_pending, svc_base_is_key,
svc_fixed_pattern, cpi->svc.temporal_layer_id);
cpi->refresh_last_frame, cpi->alt_fb_idx, cpi->gld_fb_idx,
cpi->lst_fb_idx, cpi->resize_pending, svc_base_is_key);
}
#endif
if (is_one_pass_cbr_svc(cpi)) {
@@ -3481,12 +3489,6 @@ static void encode_without_recode_loop(VP9_COMP *cpi, size_t *size,
vp9_update_noise_estimate(cpi);
#if CONFIG_VP9_TEMPORAL_DENOISING
if (cpi->oxcf.noise_sensitivity > 0 && cpi->use_svc &&
cpi->denoiser.denoising_level > kDenLowLow)
vp9_denoise_init_svc(cpi);
#endif
// Scene detection is always used for VBR mode or screen-content case.
// For other cases (e.g., CBR mode) use it for 5 <= speed < 8 for now
// (need to check encoding time cost for doing this for speed 8).