generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix intermediate height in convolve

Browse Source 
intermediate_height for horizontal filtering must be at least 8
pixels to be able to do vertical filtering correctly. Currently
it can be less for small block and y_step_q4 sizes.

Change-Id: I2ee28b0591b2041c2fa9844d0ae2ff8a1a59cc21
This commit is contained in:
Tero Rintaluoma 2013-07-05 13:53:36 +03:00
parent ef0ca2deaa
commit 18303b1263
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vp9/common/vp9_convolve.c
View File

@ -217,12 +217,13 @@ static void convolve_c(const uint8_t *src, int src_stride,
* h == 64, taps == 8.
*/
uint8_t temp[64 * 135];
int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
assert(w <= 64);
assert(h <= 64);
assert(taps <= 8);
assert(y_step_q4 <= 32);
assert(x_step_q4 <= 32);
if (intermediate_height < h)
intermediate_height = h;
@ -246,12 +247,13 @@ static void convolve_avg_c(const uint8_t *src, int src_stride,
* h == 64, taps == 8.
*/
uint8_t temp[64 * 135];
int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
assert(w <= 64);
assert(h <= 64);
assert(taps <= 8);
assert(y_step_q4 <= 32);
assert(x_step_q4 <= 32);
if (intermediate_height < h)
intermediate_height = h;