generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

set_var_thresh_from_histogram: prevent negative variance

Browse Source 
For 8-bit the subtrahend is small enough to fit into uint32_t.

For 10/12-bit apply:
63a37d16f Prevent negative variance

previously:
47b9a0912 Resolve -Wshorten-64-to-32 in highbd variance.
c0241664a Resolve -Wshorten-64-to-32 in variance.

Change-Id: I181c85f0b9a03da37c2e8b89482d48aa3dbc0aee
This commit is contained in:
James Zern
2017-07-22 13:01:49 -07:00
parent 8c7a60e04d
commit 8836e46ffd
1 changed files with 16 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
vp9/encoder/vp9_encodeframe.c
View File

@@ -4231,6 +4231,17 @@ static void encode_nonrd_sb_row(VP9_COMP *cpi, ThreadData *td,
}
// end RTC play code
static INLINE uint32_t variance(const diff *const d) {
return d->sse - (uint32_t)(((int64_t)d->sum * d->sum) >> 8);
}
#if CONFIG_VP9_HIGHBITDEPTH
static INLINE uint32_t variance_highbd(diff *const d) {
const int64_t var = (int64_t)d->sse - (((int64_t)d->sum * d->sum) >> 8);
return (var >= 0) ? (uint32_t)var : 0;
}
#endif // CONFIG_VP9_HIGHBITDEPTH
static int set_var_thresh_from_histogram(VP9_COMP *cpi) {
const SPEED_FEATURES *const sf = &cpi->sf;
const VP9_COMMON *const cm = &cpi->common;
@@ -4260,14 +4271,17 @@ static int set_var_thresh_from_histogram(VP9_COMP *cpi) {
case VPX_BITS_8:
vpx_highbd_8_get16x16var(src, src_stride, last_src, last_stride,
&var16->sse, &var16->sum);
var16->var = variance(var16);
break;
case VPX_BITS_10:
vpx_highbd_10_get16x16var(src, src_stride, last_src, last_stride,
&var16->sse, &var16->sum);
var16->var = variance_highbd(var16);
break;
case VPX_BITS_12:
vpx_highbd_12_get16x16var(src, src_stride, last_src, last_stride,
&var16->sse, &var16->sum);
var16->var = variance_highbd(var16);
break;
default:
assert(0 &&
@@ -4278,12 +4292,13 @@ static int set_var_thresh_from_histogram(VP9_COMP *cpi) {
} else {
vpx_get16x16var(src, src_stride, last_src, last_stride, &var16->sse,
&var16->sum);
var16->var = variance(var16);
}
#else
vpx_get16x16var(src, src_stride, last_src, last_stride, &var16->sse,
&var16->sum);
var16->var = variance(var16);
#endif // CONFIG_VP9_HIGHBITDEPTH
var16->var = var16->sse - (((uint32_t)var16->sum * var16->sum) >> 8);
if (var16->var >= VAR_HIST_MAX_BG_VAR)
hist[VAR_HIST_BINS - 1]++;