generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix an integer overflow issue in restoration

Browse Source 
https://bugs.chromium.org/p/webm/issues/detail?id=1306

Change-Id: Icd11d373ff08954121c097728e4c7791791e223f
This commit is contained in:
Debargha Mukherjee
2016-10-04 11:50:00 -07:00
parent 438b1dcb72
commit 1a16a987ee
1 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
av1/common/restoration.c
View File

@@ -218,15 +218,15 @@ static void loop_bilateral_filter_tile(uint8_t *data, int tile_idx, int width,
for (i = 0; i < (v_end - v_start); ++i) { for (i = 0; i < (v_end - v_start); ++i) {
for (j = 0; j < (h_end - h_start); ++j) { for (j = 0; j < (h_end - h_start); ++j) {
int x, y; int x, y, wt;
int flsum = 0, wtsum = 0, wt; int64_t flsum = 0, wtsum = 0;
uint8_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride; uint8_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride;
for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) { for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) { for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] * wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
(int)wr_lut_[data_p2[x] - data_p[j]]; (int)wr_lut_[data_p2[x] - data_p[j]];
wtsum += wt; wtsum += (int64_t)wt;
flsum += wt * data_p2[x]; flsum += (int64_t)wt * data_p2[x];
} }
data_p2 += stride; data_p2 += stride;
} }
@@ -392,15 +392,15 @@ static void loop_bilateral_filter_tile_highbd(uint16_t *data, int tile_idx,
for (i = 0; i < (v_end - v_start); ++i) { for (i = 0; i < (v_end - v_start); ++i) {
for (j = 0; j < (h_end - h_start); ++j) { for (j = 0; j < (h_end - h_start); ++j) {
int x, y; int x, y, wt;
int flsum = 0, wtsum = 0, wt; int64_t flsum = 0, wtsum = 0;
uint16_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride; uint16_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride;
for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) { for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) { for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] * wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
(int)wr_lut_[data_p2[x] - data_p[j]]; (int)wr_lut_[data_p2[x] - data_p[j]];
wtsum += wt; wtsum += (int64_t)wt;
flsum += wt * data_p2[x]; flsum += (int64_t)wt * data_p2[x];
} }
data_p2 += stride; data_p2 += stride;
} }