diff --git a/vp9/common/vp9_idct.c b/vp9/common/vp9_idct.c index dc2e03946..c6a39f85c 100644 --- a/vp9/common/vp9_idct.c +++ b/vp9/common/vp9_idct.c @@ -203,17 +203,6 @@ void vp9_iht16x16_add(TX_TYPE tx_type, const tran_low_t *input, uint8_t *dest, #if CONFIG_VP9_HIGHBITDEPTH -// 12 signal input bits + 7 forward transform amplify bits + 1 bit -// for contingency in rounding and quantizing -#define VALID_IHT_MAGNITUDE_RANGE (1 << 20) - -static INLINE int detect_invalid_iht_input(const tran_low_t *input, int size) { - int i; - for (i = 0; i < size; ++i) - if (abs(input[i]) >= VALID_IHT_MAGNITUDE_RANGE) return 1; - return 0; -} - void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, int stride, int tx_type, int bd) { const highbd_transform_2d IHT_4[] = { @@ -229,13 +218,6 @@ void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, tran_low_t *outptr = out; tran_low_t temp_in[4], temp_out[4]; - if (detect_invalid_iht_input(input, 16)) { -#if CONFIG_COEFFICIENT_RANGE_CHECKING - assert(0 && "invalid highbd iht input"); -#endif // CONFIG_COEFFICIENT_RANGE_CHECKING - return; - } - // Inverse transform row vectors. for (i = 0; i < 4; ++i) { IHT_4[tx_type].rows(input, outptr, bd); @@ -270,13 +252,6 @@ void vp9_highbd_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest8, const highbd_transform_2d ht = HIGH_IHT_8[tx_type]; uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); - if (detect_invalid_iht_input(input, 64)) { -#if CONFIG_COEFFICIENT_RANGE_CHECKING - assert(0 && "invalid highbd iht input"); -#endif // CONFIG_COEFFICIENT_RANGE_CHECKING - return; - } - // Inverse transform row vectors. for (i = 0; i < 8; ++i) { ht.rows(input, outptr, bd); @@ -311,13 +286,6 @@ void vp9_highbd_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest8, const highbd_transform_2d ht = HIGH_IHT_16[tx_type]; uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); - if (detect_invalid_iht_input(input, 256)) { -#if CONFIG_COEFFICIENT_RANGE_CHECKING - assert(0 && "invalid highbd iht input"); -#endif // CONFIG_COEFFICIENT_RANGE_CHECKING - return; - } - // Rows for (i = 0; i < 16; ++i) { ht.rows(input, outptr, bd); diff --git a/vpx_dsp/inv_txfm.c b/vpx_dsp/inv_txfm.c index 1526663d8..46ddd1da0 100644 --- a/vpx_dsp/inv_txfm.c +++ b/vpx_dsp/inv_txfm.c @@ -9,6 +9,7 @@ */ #include +#include #include #include "./vpx_dsp_rtcd.h" @@ -1252,6 +1253,19 @@ void vpx_idct32x32_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { } #if CONFIG_VP9_HIGHBITDEPTH + +// 12 signal input bits + 7 2D forward transform amplify bits + 5 1D inverse +// transform amplify bits + 1 bit for contingency in rounding and quantizing +#define HIGHBD_VALID_TXFM_MAGNITUDE_RANGE (1 << 25) + +static INLINE int detect_invalid_highbd_input(const tran_low_t *input, + int size) { + int i; + for (i = 0; i < size; ++i) + if (abs(input[i]) >= HIGHBD_VALID_TXFM_MAGNITUDE_RANGE) return 1; + return 0; +} + void vpx_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, int stride, int bd) { /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds, @@ -1347,6 +1361,15 @@ void vpx_highbd_idct4_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_low_t step[4]; tran_high_t temp1, temp2; (void)bd; + + if (detect_invalid_highbd_input(input, 4)) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + assert(0 && "invalid highbd txfm input"); +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + memset(output, 0, sizeof(*output) * 4); + return; + } + // stage 1 temp1 = (input[0] + input[2]) * cospi_16_64; temp2 = (input[0] - input[2]) * cospi_16_64; @@ -1413,6 +1436,15 @@ void vpx_highbd_idct4x4_1_add_c(const tran_low_t *input, uint8_t *dest8, void vpx_highbd_idct8_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_low_t step1[8], step2[8]; tran_high_t temp1, temp2; + + if (detect_invalid_highbd_input(input, 8)) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + assert(0 && "invalid highbd txfm input"); +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + memset(output, 0, sizeof(*output) * 8); + return; + } + // stage 1 step1[0] = input[0]; step1[2] = input[4]; @@ -1498,13 +1530,20 @@ void vpx_highbd_idct8x8_1_add_c(const tran_low_t *input, uint8_t *dest8, void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; - tran_low_t x0 = input[0]; tran_low_t x1 = input[1]; tran_low_t x2 = input[2]; tran_low_t x3 = input[3]; (void)bd; + if (detect_invalid_highbd_input(input, 4)) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + assert(0 && "invalid highbd txfm input"); +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + memset(output, 0, sizeof(*output) * 4); + return; + } + if (!(x0 | x1 | x2 | x3)) { memset(output, 0, 4 * sizeof(*output)); return; @@ -1536,7 +1575,6 @@ void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) { void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; - tran_low_t x0 = input[7]; tran_low_t x1 = input[0]; tran_low_t x2 = input[5]; @@ -1547,6 +1585,14 @@ void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_low_t x7 = input[6]; (void)bd; + if (detect_invalid_highbd_input(input, 8)) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + assert(0 && "invalid highbd txfm input"); +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + memset(output, 0, sizeof(*output) * 8); + return; + } + if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) { memset(output, 0, 8 * sizeof(*output)); return; @@ -1642,6 +1688,14 @@ void vpx_highbd_idct16_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_high_t temp1, temp2; (void)bd; + if (detect_invalid_highbd_input(input, 16)) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + assert(0 && "invalid highbd txfm input"); +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + memset(output, 0, sizeof(*output) * 16); + return; + } + // stage 1 step1[0] = input[0 / 2]; step1[1] = input[16 / 2]; @@ -1832,7 +1886,6 @@ void vpx_highbd_idct16x16_256_add_c(const tran_low_t *input, uint8_t *dest8, void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8; tran_high_t s9, s10, s11, s12, s13, s14, s15; - tran_low_t x0 = input[15]; tran_low_t x1 = input[0]; tran_low_t x2 = input[13]; @@ -1851,6 +1904,14 @@ void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) { tran_low_t x15 = input[14]; (void)bd; + if (detect_invalid_highbd_input(input, 16)) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + assert(0 && "invalid highbd txfm input"); +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + memset(output, 0, sizeof(*output) * 16); + return; + } + if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 | x13 | x14 | x15)) { memset(output, 0, 16 * sizeof(*output)); @@ -2048,6 +2109,14 @@ static void highbd_idct32_c(const tran_low_t *input, tran_low_t *output, tran_high_t temp1, temp2; (void)bd; + if (detect_invalid_highbd_input(input, 32)) { +#if CONFIG_COEFFICIENT_RANGE_CHECKING + assert(0 && "invalid highbd txfm input"); +#endif // CONFIG_COEFFICIENT_RANGE_CHECKING + memset(output, 0, sizeof(*output) * 32); + return; + } + // stage 1 step1[0] = input[0]; step1[1] = input[16];