generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "Prevent negative variance"

Browse Source
This commit is contained in:
Yaowu Xu
2016-06-29 20:53:36 +00:00
committed by Gerrit Code Review
parent f14c323b4c 63a37d16f3
commit c02a4beed8
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
vpx_dsp/x86/highbd_variance_sse2.c
View File

@@ -143,24 +143,28 @@ uint32_t vpx_highbd_10_variance##w##x##h##_sse2( \
const uint8_t *src8, int src_stride, \ const uint8_t *src8, int src_stride, \
const uint8_t *ref8, int ref_stride, uint32_t *sse) { \ const uint8_t *ref8, int ref_stride, uint32_t *sse) { \
int sum; \ int sum; \
int64_t var; \
uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ uint16_t *src = CONVERT_TO_SHORTPTR(src8); \
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \
highbd_10_variance_sse2( \ highbd_10_variance_sse2( \
src, src_stride, ref, ref_stride, w, h, sse, &sum, \ src, src_stride, ref, ref_stride, w, h, sse, &sum, \
vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \
return *sse - (((int64_t)sum * sum) >> shift); \ var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \
return (var >= 0) ? (uint32_t)var : 0; \
} \ } \
\ \
uint32_t vpx_highbd_12_variance##w##x##h##_sse2( \ uint32_t vpx_highbd_12_variance##w##x##h##_sse2( \
const uint8_t *src8, int src_stride, \ const uint8_t *src8, int src_stride, \
const uint8_t *ref8, int ref_stride, uint32_t *sse) { \ const uint8_t *ref8, int ref_stride, uint32_t *sse) { \
int sum; \ int sum; \
int64_t var; \
uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ uint16_t *src = CONVERT_TO_SHORTPTR(src8); \
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \
highbd_12_variance_sse2( \ highbd_12_variance_sse2( \
src, src_stride, ref, ref_stride, w, h, sse, &sum, \ src, src_stride, ref, ref_stride, w, h, sse, &sum, \
vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \
return *sse - (((int64_t)sum * sum) >> shift); \ var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \
return (var >= 0) ? (uint32_t)var : 0; \
} }
VAR_FN(64, 64, 16, 12); VAR_FN(64, 64, 16, 12);