From c23acbaed40101c677dfcfbbfe0d2c230a8e8f44 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 5 Mar 2012 16:01:19 -0800 Subject: [PATCH] Don't use ff_cropTbl[] for IDCT. Results of IDCT can by far outreach the range of ff_cropTbl[], leading to overreads and potentially crashes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/dsputil.c | 70 ++++++++++------------ libavcodec/h264idct_template.c | 32 +++++----- libavcodec/rv34dsp.c | 15 ++--- libavcodec/simple_idct.c | 19 +++--- libavcodec/simple_idct_template.c | 34 +++++------ libavcodec/svq3.c | 9 ++- libavcodec/vc1dsp.c | 97 +++++++++++++------------------ libavcodec/vp3dsp.c | 68 +++++++++++----------- libavcodec/vp8dsp.c | 18 +++--- 9 files changed, 161 insertions(+), 201 deletions(-) diff --git a/libavcodec/dsputil.c b/libavcodec/dsputil.c index bd10054234..29c5976596 100644 --- a/libavcodec/dsputil.c +++ b/libavcodec/dsputil.c @@ -366,18 +366,17 @@ void ff_put_pixels_clamped_c(const DCTELEM *block, uint8_t *restrict pixels, int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<8;i++) { - pixels[0] = cm[block[0]]; - pixels[1] = cm[block[1]]; - pixels[2] = cm[block[2]]; - pixels[3] = cm[block[3]]; - pixels[4] = cm[block[4]]; - pixels[5] = cm[block[5]]; - pixels[6] = cm[block[6]]; - pixels[7] = cm[block[7]]; + pixels[0] = av_clip_uint8(block[0]); + pixels[1] = av_clip_uint8(block[1]); + pixels[2] = av_clip_uint8(block[2]); + pixels[3] = av_clip_uint8(block[3]); + pixels[4] = av_clip_uint8(block[4]); + pixels[5] = av_clip_uint8(block[5]); + pixels[6] = av_clip_uint8(block[6]); + pixels[7] = av_clip_uint8(block[7]); pixels += line_size; block += 8; @@ -388,14 +387,13 @@ static void put_pixels_clamped4_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<4;i++) { - pixels[0] = cm[block[0]]; - pixels[1] = cm[block[1]]; - pixels[2] = cm[block[2]]; - pixels[3] = cm[block[3]]; + pixels[0] = av_clip_uint8(block[0]); + pixels[1] = av_clip_uint8(block[1]); + pixels[2] = av_clip_uint8(block[2]); + pixels[3] = av_clip_uint8(block[3]); pixels += line_size; block += 8; @@ -406,12 +404,11 @@ static void put_pixels_clamped2_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<2;i++) { - pixels[0] = cm[block[0]]; - pixels[1] = cm[block[1]]; + pixels[0] = av_clip_uint8(block[0]); + pixels[1] = av_clip_uint8(block[1]); pixels += line_size; block += 8; @@ -443,18 +440,17 @@ void ff_add_pixels_clamped_c(const DCTELEM *block, uint8_t *restrict pixels, int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<8;i++) { - pixels[0] = cm[pixels[0] + block[0]]; - pixels[1] = cm[pixels[1] + block[1]]; - pixels[2] = cm[pixels[2] + block[2]]; - pixels[3] = cm[pixels[3] + block[3]]; - pixels[4] = cm[pixels[4] + block[4]]; - pixels[5] = cm[pixels[5] + block[5]]; - pixels[6] = cm[pixels[6] + block[6]]; - pixels[7] = cm[pixels[7] + block[7]]; + pixels[0] = av_clip_uint8(pixels[0] + block[0]); + pixels[1] = av_clip_uint8(pixels[1] + block[1]); + pixels[2] = av_clip_uint8(pixels[2] + block[2]); + pixels[3] = av_clip_uint8(pixels[3] + block[3]); + pixels[4] = av_clip_uint8(pixels[4] + block[4]); + pixels[5] = av_clip_uint8(pixels[5] + block[5]); + pixels[6] = av_clip_uint8(pixels[6] + block[6]); + pixels[7] = av_clip_uint8(pixels[7] + block[7]); pixels += line_size; block += 8; } @@ -464,14 +460,13 @@ static void add_pixels_clamped4_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<4;i++) { - pixels[0] = cm[pixels[0] + block[0]]; - pixels[1] = cm[pixels[1] + block[1]]; - pixels[2] = cm[pixels[2] + block[2]]; - pixels[3] = cm[pixels[3] + block[3]]; + pixels[0] = av_clip_uint8(pixels[0] + block[0]); + pixels[1] = av_clip_uint8(pixels[1] + block[1]); + pixels[2] = av_clip_uint8(pixels[2] + block[2]); + pixels[3] = av_clip_uint8(pixels[3] + block[3]); pixels += line_size; block += 8; } @@ -481,12 +476,11 @@ static void add_pixels_clamped2_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<2;i++) { - pixels[0] = cm[pixels[0] + block[0]]; - pixels[1] = cm[pixels[1] + block[1]]; + pixels[0] = av_clip_uint8(pixels[0] + block[0]); + pixels[1] = av_clip_uint8(pixels[1] + block[1]); pixels += line_size; block += 8; } @@ -2733,15 +2727,11 @@ static void ff_jref_idct2_add(uint8_t *dest, int line_size, DCTELEM *block) static void ff_jref_idct1_put(uint8_t *dest, int line_size, DCTELEM *block) { - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; - - dest[0] = cm[(block[0] + 4)>>3]; + dest[0] = av_clip_uint8((block[0] + 4)>>3); } static void ff_jref_idct1_add(uint8_t *dest, int line_size, DCTELEM *block) { - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; - - dest[0] = cm[dest[0] + ((block[0] + 4)>>3)]; + dest[0] = av_clip_uint8(dest[0] + ((block[0] + 4)>>3)); } static void just_return(void *mem av_unused, int stride av_unused, int h av_unused) { return; } diff --git a/libavcodec/h264idct_template.c b/libavcodec/h264idct_template.c index eba850ac6f..e476f89a6f 100644 --- a/libavcodec/h264idct_template.c +++ b/libavcodec/h264idct_template.c @@ -49,7 +49,6 @@ static const uint8_t scan8[16*3]={ void FUNCC(ff_h264_idct_add)(uint8_t *_dst, DCTELEM *_block, int stride) { int i; - INIT_CLIP pixel *dst = (pixel*)_dst; dctcoef *block = (dctcoef*)_block; stride /= sizeof(pixel); @@ -74,16 +73,15 @@ void FUNCC(ff_h264_idct_add)(uint8_t *_dst, DCTELEM *_block, int stride) const int z2= (block[1 + 4*i]>>1) - block[3 + 4*i]; const int z3= block[1 + 4*i] + (block[3 + 4*i]>>1); - dst[i + 0*stride]= CLIP(dst[i + 0*stride] + ((z0 + z3) >> 6)); - dst[i + 1*stride]= CLIP(dst[i + 1*stride] + ((z1 + z2) >> 6)); - dst[i + 2*stride]= CLIP(dst[i + 2*stride] + ((z1 - z2) >> 6)); - dst[i + 3*stride]= CLIP(dst[i + 3*stride] + ((z0 - z3) >> 6)); + dst[i + 0*stride]= av_clip_pixel(dst[i + 0*stride] + ((z0 + z3) >> 6)); + dst[i + 1*stride]= av_clip_pixel(dst[i + 1*stride] + ((z1 + z2) >> 6)); + dst[i + 2*stride]= av_clip_pixel(dst[i + 2*stride] + ((z1 - z2) >> 6)); + dst[i + 3*stride]= av_clip_pixel(dst[i + 3*stride] + ((z0 - z3) >> 6)); } } void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ int i; - INIT_CLIP pixel *dst = (pixel*)_dst; dctcoef *block = (dctcoef*)_block; stride /= sizeof(pixel); @@ -143,14 +141,14 @@ void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ const int b5 = (a3>>2) - a5; const int b7 = a7 - (a1>>2); - dst[i + 0*stride] = CLIP( dst[i + 0*stride] + ((b0 + b7) >> 6) ); - dst[i + 1*stride] = CLIP( dst[i + 1*stride] + ((b2 + b5) >> 6) ); - dst[i + 2*stride] = CLIP( dst[i + 2*stride] + ((b4 + b3) >> 6) ); - dst[i + 3*stride] = CLIP( dst[i + 3*stride] + ((b6 + b1) >> 6) ); - dst[i + 4*stride] = CLIP( dst[i + 4*stride] + ((b6 - b1) >> 6) ); - dst[i + 5*stride] = CLIP( dst[i + 5*stride] + ((b4 - b3) >> 6) ); - dst[i + 6*stride] = CLIP( dst[i + 6*stride] + ((b2 - b5) >> 6) ); - dst[i + 7*stride] = CLIP( dst[i + 7*stride] + ((b0 - b7) >> 6) ); + dst[i + 0*stride] = av_clip_pixel( dst[i + 0*stride] + ((b0 + b7) >> 6) ); + dst[i + 1*stride] = av_clip_pixel( dst[i + 1*stride] + ((b2 + b5) >> 6) ); + dst[i + 2*stride] = av_clip_pixel( dst[i + 2*stride] + ((b4 + b3) >> 6) ); + dst[i + 3*stride] = av_clip_pixel( dst[i + 3*stride] + ((b6 + b1) >> 6) ); + dst[i + 4*stride] = av_clip_pixel( dst[i + 4*stride] + ((b6 - b1) >> 6) ); + dst[i + 5*stride] = av_clip_pixel( dst[i + 5*stride] + ((b4 - b3) >> 6) ); + dst[i + 6*stride] = av_clip_pixel( dst[i + 6*stride] + ((b2 - b5) >> 6) ); + dst[i + 7*stride] = av_clip_pixel( dst[i + 7*stride] + ((b0 - b7) >> 6) ); } } @@ -158,13 +156,12 @@ void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ int i, j; int dc = (((dctcoef*)block)[0] + 32) >> 6; - INIT_CLIP pixel *dst = (pixel*)_dst; stride /= sizeof(pixel); for( j = 0; j < 4; j++ ) { for( i = 0; i < 4; i++ ) - dst[i] = CLIP( dst[i] + dc ); + dst[i] = av_clip_pixel( dst[i] + dc ); dst += stride; } } @@ -172,13 +169,12 @@ void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ void FUNCC(ff_h264_idct8_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ int i, j; int dc = (((dctcoef*)block)[0] + 32) >> 6; - INIT_CLIP pixel *dst = (pixel*)_dst; stride /= sizeof(pixel); for( j = 0; j < 8; j++ ) { for( i = 0; i < 8; i++ ) - dst[i] = CLIP( dst[i] + dc ); + dst[i] = av_clip_pixel( dst[i] + dc ); dst += stride; } } diff --git a/libavcodec/rv34dsp.c b/libavcodec/rv34dsp.c index ce5be93894..4145c4dd85 100644 --- a/libavcodec/rv34dsp.c +++ b/libavcodec/rv34dsp.c @@ -55,7 +55,6 @@ static av_always_inline void rv34_row_transform(int temp[16], DCTELEM *block) */ static void rv34_idct_add_c(uint8_t *dst, ptrdiff_t stride, DCTELEM *block){ int temp[16]; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; int i; rv34_row_transform(temp, block); @@ -67,10 +66,10 @@ static void rv34_idct_add_c(uint8_t *dst, ptrdiff_t stride, DCTELEM *block){ const int z2 = 7* temp[4*1+i] - 17*temp[4*3+i]; const int z3 = 17* temp[4*1+i] + 7*temp[4*3+i]; - dst[0] = cm[ dst[0] + ( (z0 + z3) >> 10 ) ]; - dst[1] = cm[ dst[1] + ( (z1 + z2) >> 10 ) ]; - dst[2] = cm[ dst[2] + ( (z1 - z2) >> 10 ) ]; - dst[3] = cm[ dst[3] + ( (z0 - z3) >> 10 ) ]; + dst[0] = av_clip_uint8( dst[0] + ( (z0 + z3) >> 10 ) ); + dst[1] = av_clip_uint8( dst[1] + ( (z1 + z2) >> 10 ) ); + dst[2] = av_clip_uint8( dst[2] + ( (z1 - z2) >> 10 ) ); + dst[3] = av_clip_uint8( dst[3] + ( (z0 - z3) >> 10 ) ); dst += stride; } @@ -103,15 +102,13 @@ static void rv34_inv_transform_noround_c(DCTELEM *block){ static void rv34_idct_dc_add_c(uint8_t *dst, ptrdiff_t stride, int dc) { - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; int i, j; - cm += (13*13*dc + 0x200) >> 10; - + dc = (13*13*dc + 0x200) >> 10; for (i = 0; i < 4; i++) { for (j = 0; j < 4; j++) - dst[j] = cm[ dst[j] ]; + dst[j] = av_clip_uint8( dst[j] + dc ); dst += stride; } diff --git a/libavcodec/simple_idct.c b/libavcodec/simple_idct.c index 0c75261079..5812a87705 100644 --- a/libavcodec/simple_idct.c +++ b/libavcodec/simple_idct.c @@ -53,7 +53,6 @@ static inline void idct4col_put(uint8_t *dest, int line_size, const DCTELEM *col) { int c0, c1, c2, c3, a0, a1, a2, a3; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = col[8*0]; a1 = col[8*2]; @@ -63,13 +62,13 @@ static inline void idct4col_put(uint8_t *dest, int line_size, const DCTELEM *col c2 = ((a0 - a2) << (CN_SHIFT - 1)) + (1 << (C_SHIFT - 1)); c1 = a1 * C1 + a3 * C2; c3 = a1 * C2 - a3 * C1; - dest[0] = cm[(c0 + c1) >> C_SHIFT]; + dest[0] = av_clip_uint8((c0 + c1) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c2 + c3) >> C_SHIFT]; + dest[0] = av_clip_uint8((c2 + c3) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c2 - c3) >> C_SHIFT]; + dest[0] = av_clip_uint8((c2 - c3) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c0 - c1) >> C_SHIFT]; + dest[0] = av_clip_uint8((c0 - c1) >> C_SHIFT); } #define BF(k) \ @@ -133,7 +132,6 @@ void ff_simple_idct248_put(uint8_t *dest, int line_size, DCTELEM *block) static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col) { int c0, c1, c2, c3, a0, a1, a2, a3; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = col[8*0]; a1 = col[8*1]; @@ -143,13 +141,13 @@ static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col c2 = (a0 - a2)*C3 + (1 << (C_SHIFT - 1)); c1 = a1 * C1 + a3 * C2; c3 = a1 * C2 - a3 * C1; - dest[0] = cm[dest[0] + ((c0 + c1) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c0 + c1) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c2 + c3) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c2 + c3) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c2 - c3) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c2 - c3) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c0 - c1) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c0 - c1) >> C_SHIFT)); } #define RN_SHIFT 15 @@ -161,7 +159,6 @@ static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col static inline void idct4row(DCTELEM *row) { int c0, c1, c2, c3, a0, a1, a2, a3; - //const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = row[0]; a1 = row[1]; diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index fdec3aab2b..3c855e3825 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -224,50 +224,48 @@ static inline void FUNC(idctSparseColPut)(pixel *dest, int line_size, DCTELEM *col) { int a0, a1, a2, a3, b0, b1, b2, b3; - INIT_CLIP; IDCT_COLS; - dest[0] = CLIP((a0 + b0) >> COL_SHIFT); + dest[0] = av_clip_pixel((a0 + b0) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a1 + b1) >> COL_SHIFT); + dest[0] = av_clip_pixel((a1 + b1) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a2 + b2) >> COL_SHIFT); + dest[0] = av_clip_pixel((a2 + b2) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a3 + b3) >> COL_SHIFT); + dest[0] = av_clip_pixel((a3 + b3) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a3 - b3) >> COL_SHIFT); + dest[0] = av_clip_pixel((a3 - b3) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a2 - b2) >> COL_SHIFT); + dest[0] = av_clip_pixel((a2 - b2) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a1 - b1) >> COL_SHIFT); + dest[0] = av_clip_pixel((a1 - b1) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a0 - b0) >> COL_SHIFT); + dest[0] = av_clip_pixel((a0 - b0) >> COL_SHIFT); } static inline void FUNC(idctSparseColAdd)(pixel *dest, int line_size, DCTELEM *col) { int a0, a1, a2, a3, b0, b1, b2, b3; - INIT_CLIP; IDCT_COLS; - dest[0] = CLIP(dest[0] + ((a0 + b0) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a0 + b0) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a1 + b1) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a1 + b1) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a2 + b2) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a2 + b2) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a3 + b3) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a3 + b3) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a3 - b3) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a3 - b3) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a2 - b2) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a2 - b2) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a1 - b1) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a1 - b1) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a0 - b0) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a0 - b0) >> COL_SHIFT)); } static inline void FUNC(idctSparseCol)(DCTELEM *col) diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index e157061f64..caafc49969 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -173,7 +173,6 @@ void ff_svq3_add_idct_c(uint8_t *dst, DCTELEM *block, int stride, int qp, { const int qmul = svq3_dequant_coeff[qp]; int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; if (dc) { dc = 13*13*((dc == 1) ? 1538*block[0] : ((qmul*(block[0] >> 3)) / 2)); @@ -199,10 +198,10 @@ void ff_svq3_add_idct_c(uint8_t *dst, DCTELEM *block, int stride, int qp, const int z3 = 17* block[i + 4*1] + 7*block[i + 4*3]; const int rr = (dc + 0x80000); - dst[i + stride*0] = cm[ dst[i + stride*0] + (((z0 + z3)*qmul + rr) >> 20) ]; - dst[i + stride*1] = cm[ dst[i + stride*1] + (((z1 + z2)*qmul + rr) >> 20) ]; - dst[i + stride*2] = cm[ dst[i + stride*2] + (((z1 - z2)*qmul + rr) >> 20) ]; - dst[i + stride*3] = cm[ dst[i + stride*3] + (((z0 - z3)*qmul + rr) >> 20) ]; + dst[i + stride*0] = av_clip_uint8( dst[i + stride*0] + (((z0 + z3)*qmul + rr) >> 20) ); + dst[i + stride*1] = av_clip_uint8( dst[i + stride*1] + (((z1 + z2)*qmul + rr) >> 20) ); + dst[i + stride*2] = av_clip_uint8( dst[i + stride*2] + (((z1 - z2)*qmul + rr) >> 20) ); + dst[i + stride*3] = av_clip_uint8( dst[i + stride*3] + (((z0 - z3)*qmul + rr) >> 20) ); } } diff --git a/libavcodec/vc1dsp.c b/libavcodec/vc1dsp.c index 9bd107cdd9..b40824b86a 100644 --- a/libavcodec/vc1dsp.c +++ b/libavcodec/vc1dsp.c @@ -139,8 +139,6 @@ static void vc1_h_s_overlap_c(DCTELEM *left, DCTELEM *right) * @see 8.6 */ static av_always_inline int vc1_filter_line(uint8_t* src, int stride, int pq){ - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; - int a0 = (2*(src[-2*stride] - src[ 1*stride]) - 5*(src[-1*stride] - src[ 0*stride]) + 4) >> 3; int a0_sign = a0 >> 31; /* Store sign */ a0 = (a0 ^ a0_sign) - a0_sign; /* a0 = FFABS(a0); */ @@ -163,8 +161,8 @@ static av_always_inline int vc1_filter_line(uint8_t* src, int stride, int pq){ else{ d = FFMIN(d, clip); d = (d ^ d_sign) - d_sign; /* Restore sign */ - src[-1*stride] = cm[src[-1*stride] - d]; - src[ 0*stride] = cm[src[ 0*stride] + d]; + src[-1*stride] = av_clip_uint8(src[-1*stride] - d); + src[ 0*stride] = av_clip_uint8(src[ 0*stride] + d); } return 1; } @@ -234,19 +232,17 @@ static void vc1_inv_trans_8x8_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = (3 * dc + 1) >> 1; dc = (3 * dc + 16) >> 5; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 8; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; - dest[4] = cm[dest[4]]; - dest[5] = cm[dest[5]]; - dest[6] = cm[dest[6]]; - dest[7] = cm[dest[7]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); + dest[4] = av_clip_uint8(dest[4] + dc); + dest[5] = av_clip_uint8(dest[5] + dc); + dest[6] = av_clip_uint8(dest[6] + dc); + dest[7] = av_clip_uint8(dest[7] + dc); dest += linesize; } } @@ -326,19 +322,17 @@ static void vc1_inv_trans_8x4_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = ( 3 * dc + 1) >> 1; dc = (17 * dc + 64) >> 7; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 4; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; - dest[4] = cm[dest[4]]; - dest[5] = cm[dest[5]]; - dest[6] = cm[dest[6]]; - dest[7] = cm[dest[7]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); + dest[4] = av_clip_uint8(dest[4] + dc); + dest[5] = av_clip_uint8(dest[5] + dc); + dest[6] = av_clip_uint8(dest[6] + dc); + dest[7] = av_clip_uint8(dest[7] + dc); dest += linesize; } } @@ -348,7 +342,6 @@ static void vc1_inv_trans_8x4_c(uint8_t *dest, int linesize, DCTELEM *block) int i; register int t1,t2,t3,t4,t5,t6,t7,t8; DCTELEM *src, *dst; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; src = block; dst = block; @@ -388,10 +381,10 @@ static void vc1_inv_trans_8x4_c(uint8_t *dest, int linesize, DCTELEM *block) t3 = 22 * src[ 8] + 10 * src[24]; t4 = 22 * src[24] - 10 * src[ 8]; - dest[0*linesize] = cm[dest[0*linesize] + ((t1 + t3) >> 7)]; - dest[1*linesize] = cm[dest[1*linesize] + ((t2 - t4) >> 7)]; - dest[2*linesize] = cm[dest[2*linesize] + ((t2 + t4) >> 7)]; - dest[3*linesize] = cm[dest[3*linesize] + ((t1 - t3) >> 7)]; + dest[0*linesize] = av_clip_uint8(dest[0*linesize] + ((t1 + t3) >> 7)); + dest[1*linesize] = av_clip_uint8(dest[1*linesize] + ((t2 - t4) >> 7)); + dest[2*linesize] = av_clip_uint8(dest[2*linesize] + ((t2 + t4) >> 7)); + dest[3*linesize] = av_clip_uint8(dest[3*linesize] + ((t1 - t3) >> 7)); src ++; dest++; @@ -404,15 +397,13 @@ static void vc1_inv_trans_4x8_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = (17 * dc + 4) >> 3; dc = (12 * dc + 64) >> 7; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 8; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); dest += linesize; } } @@ -422,7 +413,6 @@ static void vc1_inv_trans_4x8_c(uint8_t *dest, int linesize, DCTELEM *block) int i; register int t1,t2,t3,t4,t5,t6,t7,t8; DCTELEM *src, *dst; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; src = block; dst = block; @@ -458,14 +448,14 @@ static void vc1_inv_trans_4x8_c(uint8_t *dest, int linesize, DCTELEM *block) t3 = 9 * src[ 8] - 16 * src[24] + 4 * src[40] + 15 * src[56]; t4 = 4 * src[ 8] - 9 * src[24] + 15 * src[40] - 16 * src[56]; - dest[0*linesize] = cm[dest[0*linesize] + ((t5 + t1) >> 7)]; - dest[1*linesize] = cm[dest[1*linesize] + ((t6 + t2) >> 7)]; - dest[2*linesize] = cm[dest[2*linesize] + ((t7 + t3) >> 7)]; - dest[3*linesize] = cm[dest[3*linesize] + ((t8 + t4) >> 7)]; - dest[4*linesize] = cm[dest[4*linesize] + ((t8 - t4 + 1) >> 7)]; - dest[5*linesize] = cm[dest[5*linesize] + ((t7 - t3 + 1) >> 7)]; - dest[6*linesize] = cm[dest[6*linesize] + ((t6 - t2 + 1) >> 7)]; - dest[7*linesize] = cm[dest[7*linesize] + ((t5 - t1 + 1) >> 7)]; + dest[0*linesize] = av_clip_uint8(dest[0*linesize] + ((t5 + t1) >> 7)); + dest[1*linesize] = av_clip_uint8(dest[1*linesize] + ((t6 + t2) >> 7)); + dest[2*linesize] = av_clip_uint8(dest[2*linesize] + ((t7 + t3) >> 7)); + dest[3*linesize] = av_clip_uint8(dest[3*linesize] + ((t8 + t4) >> 7)); + dest[4*linesize] = av_clip_uint8(dest[4*linesize] + ((t8 - t4 + 1) >> 7)); + dest[5*linesize] = av_clip_uint8(dest[5*linesize] + ((t7 - t3 + 1) >> 7)); + dest[6*linesize] = av_clip_uint8(dest[6*linesize] + ((t6 - t2 + 1) >> 7)); + dest[7*linesize] = av_clip_uint8(dest[7*linesize] + ((t5 - t1 + 1) >> 7)); src ++; dest++; @@ -478,15 +468,13 @@ static void vc1_inv_trans_4x4_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = (17 * dc + 4) >> 3; dc = (17 * dc + 64) >> 7; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 4; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); dest += linesize; } } @@ -496,7 +484,6 @@ static void vc1_inv_trans_4x4_c(uint8_t *dest, int linesize, DCTELEM *block) int i; register int t1,t2,t3,t4; DCTELEM *src, *dst; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; src = block; dst = block; @@ -522,10 +509,10 @@ static void vc1_inv_trans_4x4_c(uint8_t *dest, int linesize, DCTELEM *block) t3 = 22 * src[ 8] + 10 * src[24]; t4 = 22 * src[24] - 10 * src[ 8]; - dest[0*linesize] = cm[dest[0*linesize] + ((t1 + t3) >> 7)]; - dest[1*linesize] = cm[dest[1*linesize] + ((t2 - t4) >> 7)]; - dest[2*linesize] = cm[dest[2*linesize] + ((t2 + t4) >> 7)]; - dest[3*linesize] = cm[dest[3*linesize] + ((t1 - t3) >> 7)]; + dest[0*linesize] = av_clip_uint8(dest[0*linesize] + ((t1 + t3) >> 7)); + dest[1*linesize] = av_clip_uint8(dest[1*linesize] + ((t2 - t4) >> 7)); + dest[2*linesize] = av_clip_uint8(dest[2*linesize] + ((t2 + t4) >> 7)); + dest[3*linesize] = av_clip_uint8(dest[3*linesize] + ((t1 - t3) >> 7)); src ++; dest++; diff --git a/libavcodec/vp3dsp.c b/libavcodec/vp3dsp.c index baa22a5519..438ae76b57 100644 --- a/libavcodec/vp3dsp.c +++ b/libavcodec/vp3dsp.c @@ -41,7 +41,6 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int type) { int16_t *ip = input; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; int A, B, C, D, Ad, Bd, Cd, Dd, E, F, G, H; int Ed, Gd, Add, Bdd, Fd, Hd; @@ -147,29 +146,29 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int ip[5*8] = (Fd + Bdd ) >> 4; ip[6*8] = (Fd - Bdd ) >> 4; }else if(type==1){ - dst[0*stride] = cm[(Gd + Cd ) >> 4]; - dst[7*stride] = cm[(Gd - Cd ) >> 4]; + dst[0*stride] = av_clip_uint8((Gd + Cd ) >> 4); + dst[7*stride] = av_clip_uint8((Gd - Cd ) >> 4); - dst[1*stride] = cm[(Add + Hd ) >> 4]; - dst[2*stride] = cm[(Add - Hd ) >> 4]; + dst[1*stride] = av_clip_uint8((Add + Hd ) >> 4); + dst[2*stride] = av_clip_uint8((Add - Hd ) >> 4); - dst[3*stride] = cm[(Ed + Dd ) >> 4]; - dst[4*stride] = cm[(Ed - Dd ) >> 4]; + dst[3*stride] = av_clip_uint8((Ed + Dd ) >> 4); + dst[4*stride] = av_clip_uint8((Ed - Dd ) >> 4); - dst[5*stride] = cm[(Fd + Bdd ) >> 4]; - dst[6*stride] = cm[(Fd - Bdd ) >> 4]; + dst[5*stride] = av_clip_uint8((Fd + Bdd ) >> 4); + dst[6*stride] = av_clip_uint8((Fd - Bdd ) >> 4); }else{ - dst[0*stride] = cm[dst[0*stride] + ((Gd + Cd ) >> 4)]; - dst[7*stride] = cm[dst[7*stride] + ((Gd - Cd ) >> 4)]; + dst[0*stride] = av_clip_uint8(dst[0*stride] + ((Gd + Cd ) >> 4)); + dst[7*stride] = av_clip_uint8(dst[7*stride] + ((Gd - Cd ) >> 4)); - dst[1*stride] = cm[dst[1*stride] + ((Add + Hd ) >> 4)]; - dst[2*stride] = cm[dst[2*stride] + ((Add - Hd ) >> 4)]; + dst[1*stride] = av_clip_uint8(dst[1*stride] + ((Add + Hd ) >> 4)); + dst[2*stride] = av_clip_uint8(dst[2*stride] + ((Add - Hd ) >> 4)); - dst[3*stride] = cm[dst[3*stride] + ((Ed + Dd ) >> 4)]; - dst[4*stride] = cm[dst[4*stride] + ((Ed - Dd ) >> 4)]; + dst[3*stride] = av_clip_uint8(dst[3*stride] + ((Ed + Dd ) >> 4)); + dst[4*stride] = av_clip_uint8(dst[4*stride] + ((Ed - Dd ) >> 4)); - dst[5*stride] = cm[dst[5*stride] + ((Fd + Bdd ) >> 4)]; - dst[6*stride] = cm[dst[6*stride] + ((Fd - Bdd ) >> 4)]; + dst[5*stride] = av_clip_uint8(dst[5*stride] + ((Fd + Bdd ) >> 4)); + dst[6*stride] = av_clip_uint8(dst[6*stride] + ((Fd - Bdd ) >> 4)); } } else { @@ -190,18 +189,18 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int dst[4*stride]= dst[5*stride]= dst[6*stride]= - dst[7*stride]= cm[128 + ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20)]; + dst[7*stride]= av_clip_uint8(128 + ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20)); }else{ if(ip[0*8]){ int v= ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20); - dst[0*stride] = cm[dst[0*stride] + v]; - dst[1*stride] = cm[dst[1*stride] + v]; - dst[2*stride] = cm[dst[2*stride] + v]; - dst[3*stride] = cm[dst[3*stride] + v]; - dst[4*stride] = cm[dst[4*stride] + v]; - dst[5*stride] = cm[dst[5*stride] + v]; - dst[6*stride] = cm[dst[6*stride] + v]; - dst[7*stride] = cm[dst[7*stride] + v]; + dst[0*stride] = av_clip_uint8(dst[0*stride] + v); + dst[1*stride] = av_clip_uint8(dst[1*stride] + v); + dst[2*stride] = av_clip_uint8(dst[2*stride] + v); + dst[3*stride] = av_clip_uint8(dst[3*stride] + v); + dst[4*stride] = av_clip_uint8(dst[4*stride] + v); + dst[5*stride] = av_clip_uint8(dst[5*stride] + v); + dst[6*stride] = av_clip_uint8(dst[6*stride] + v); + dst[7*stride] = av_clip_uint8(dst[7*stride] + v); } } } @@ -225,17 +224,16 @@ void ff_vp3_idct_add_c(uint8_t *dest/*align 8*/, int line_size, DCTELEM *block/* void ff_vp3_idct_dc_add_c(uint8_t *dest/*align 8*/, int line_size, const DCTELEM *block/*align 16*/){ int i, dc = (block[0] + 15) >> 5; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 8; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; - dest[4] = cm[dest[4]]; - dest[5] = cm[dest[5]]; - dest[6] = cm[dest[6]]; - dest[7] = cm[dest[7]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); + dest[4] = av_clip_uint8(dest[4] + dc); + dest[5] = av_clip_uint8(dest[5] + dc); + dest[6] = av_clip_uint8(dest[6] + dc); + dest[7] = av_clip_uint8(dest[7] + dc); dest += line_size; } } diff --git a/libavcodec/vp8dsp.c b/libavcodec/vp8dsp.c index 86dc42ed37..f6c944328a 100644 --- a/libavcodec/vp8dsp.c +++ b/libavcodec/vp8dsp.c @@ -80,7 +80,6 @@ static void vp8_luma_dc_wht_dc_c(DCTELEM block[4][4][16], DCTELEM dc[16]) static void vp8_idct_add_c(uint8_t *dst, DCTELEM block[16], ptrdiff_t stride) { int i, t0, t1, t2, t3; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; DCTELEM tmp[16]; for (i = 0; i < 4; i++) { @@ -105,10 +104,10 @@ static void vp8_idct_add_c(uint8_t *dst, DCTELEM block[16], ptrdiff_t stride) t2 = MUL_35468(tmp[1*4+i]) - MUL_20091(tmp[3*4+i]); t3 = MUL_20091(tmp[1*4+i]) + MUL_35468(tmp[3*4+i]); - dst[0] = cm[dst[0] + ((t0 + t3 + 4) >> 3)]; - dst[1] = cm[dst[1] + ((t1 + t2 + 4) >> 3)]; - dst[2] = cm[dst[2] + ((t1 - t2 + 4) >> 3)]; - dst[3] = cm[dst[3] + ((t0 - t3 + 4) >> 3)]; + dst[0] = av_clip_uint8(dst[0] + ((t0 + t3 + 4) >> 3)); + dst[1] = av_clip_uint8(dst[1] + ((t1 + t2 + 4) >> 3)); + dst[2] = av_clip_uint8(dst[2] + ((t1 - t2 + 4) >> 3)); + dst[3] = av_clip_uint8(dst[3] + ((t0 - t3 + 4) >> 3)); dst += stride; } } @@ -116,14 +115,13 @@ static void vp8_idct_add_c(uint8_t *dst, DCTELEM block[16], ptrdiff_t stride) static void vp8_idct_dc_add_c(uint8_t *dst, DCTELEM block[16], ptrdiff_t stride) { int i, dc = (block[0] + 4) >> 3; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP + dc; block[0] = 0; for (i = 0; i < 4; i++) { - dst[0] = cm[dst[0]]; - dst[1] = cm[dst[1]]; - dst[2] = cm[dst[2]]; - dst[3] = cm[dst[3]]; + dst[0] = av_clip_uint8(dst[0] + dc); + dst[1] = av_clip_uint8(dst[1] + dc); + dst[2] = av_clip_uint8(dst[2] + dc); + dst[3] = av_clip_uint8(dst[3] + dc); dst += stride; } }