generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix bilateral filter asan error for highbitdepth

Browse Source 
BUG=webm:1334

Change-Id: I5886eec0a22a8cc056e1bdb493d2faf183816656
This commit is contained in:
Debargha Mukherjee
2016-11-03 15:14:27 -07:00
parent d65708a375
commit c57924cb9e
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
av1/common/restoration.c
View File

@@ -369,6 +369,7 @@ static void loop_bilateral_filter_tile_highbd(uint16_t *data, int tile_idx,
const int tile_height = rst->tile_height >> rst->subsampling_y;
int i, j, subtile_idx;
int h_start, h_end, v_start, v_end;
const int shift = bit_depth - 8;
for (subtile_idx = 0; subtile_idx < BILATERAL_SUBTILES; ++subtile_idx) {
uint16_t *data_p, *tmpdata_p;
@@ -398,7 +399,7 @@ static void loop_bilateral_filter_tile_highbd(uint16_t *data, int tile_idx,
for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
(int)wr_lut_[data_p2[x] - data_p[j]];
(int)wr_lut_[(data_p2[x] >> shift) - (data_p[j] >> shift)];
wtsum += (int64_t)wt;
flsum += (int64_t)wt * data_p2[x];
}