From dbbbd4430411ba8931de18a8e802934de511c22c Mon Sep 17 00:00:00 2001 From: Linfeng Zhang Date: Tue, 26 Sep 2017 12:33:40 -0700 Subject: [PATCH] fix signed integer overflow of idct Exposed by fuzz test in high bitdepth. The bug is introduced in commit 64653fa. BUG=webm:1466 Change-Id: Idd77d5c6a60efb9241471611ce1aba0646cb6ff5 --- test/invalid_file_test.cc | 3 + test/test-data.mk | 2 + test/test-data.sha1 | 2 + vpx_dsp/arm/idct16x16_1_add_neon.c | 3 +- vpx_dsp/arm/idct32x32_1_add_neon.c | 3 +- vpx_dsp/arm/idct4x4_1_add_neon.c | 3 +- vpx_dsp/arm/idct8x8_1_add_neon.c | 3 +- vpx_dsp/inv_txfm.c | 140 +++++++++++++++-------------- vpx_dsp/x86/inv_txfm_sse2.c | 11 ++- 9 files changed, 94 insertions(+), 76 deletions(-) diff --git a/test/invalid_file_test.cc b/test/invalid_file_test.cc index 8f7cd107a..79220b0f6 100644 --- a/test/invalid_file_test.cc +++ b/test/invalid_file_test.cc @@ -134,6 +134,9 @@ const DecodeParam kVP9InvalidFileTests[] = { { 1, "invalid-vp90-02-v2.webm" }, #if CONFIG_VP9_HIGHBITDEPTH { 1, "invalid-vp90-2-00-quantizer-00.webm.ivf.s5861_r01-05_b6-.v2.ivf" }, + { 1, + "invalid-vp90-2-21-resize_inter_320x180_5_3-4.webm.ivf.s45551_r01-05_b6-." + "ivf" }, #endif { 1, "invalid-vp90-03-v3.webm" }, { 1, "invalid-vp90-2-00-quantizer-11.webm.ivf.s52984_r01-05_b6-.ivf" }, diff --git a/test/test-data.mk b/test/test-data.mk index 1656372b3..67a2568bd 100644 --- a/test/test-data.mk +++ b/test/test-data.mk @@ -774,6 +774,8 @@ LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp90-2-12-droppable_1.ivf.s367 LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp90-2-12-droppable_1.ivf.s3676_r01-05_b6-.ivf.res LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp90-2-12-droppable_1.ivf.s73804_r01-05_b6-.ivf LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp90-2-12-droppable_1.ivf.s73804_r01-05_b6-.ivf.res +LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp90-2-21-resize_inter_320x180_5_3-4.webm.ivf.s45551_r01-05_b6-.ivf +LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp90-2-21-resize_inter_320x180_5_3-4.webm.ivf.s45551_r01-05_b6-.ivf.res LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp91-2-mixedrefcsp-444to420.ivf LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp91-2-mixedrefcsp-444to420.ivf.res LIBVPX_TEST_DATA-$(CONFIG_VP9_DECODER) += invalid-vp90-2-07-frame_parallel-1.webm diff --git a/test/test-data.sha1 b/test/test-data.sha1 index 7948f9e3f..57d6502a9 100644 --- a/test/test-data.sha1 +++ b/test/test-data.sha1 @@ -6,6 +6,8 @@ b87815bf86020c592ccc7a846ba2e28ec8043902 *hantro_odd.yuv 456d1493e52d32a5c30edf44a27debc1fa6b253a *invalid-vp90-2-00-quantizer-11.webm.ivf.s52984_r01-05_b6-.ivf.res c123d1f9f02fb4143abb5e271916e3a3080de8f6 *invalid-vp90-2-00-quantizer-11.webm.ivf.s52984_r01-05_b6-z.ivf 456d1493e52d32a5c30edf44a27debc1fa6b253a *invalid-vp90-2-00-quantizer-11.webm.ivf.s52984_r01-05_b6-z.ivf.res +efafb92b7567bc04c3f1432ea6c268c1c31affd5 *invalid-vp90-2-21-resize_inter_320x180_5_3-4.webm.ivf.s45551_r01-05_b6-.ivf +5d9474c0309b7ca09a182d888f73b37a8fe1362c *invalid-vp90-2-21-resize_inter_320x180_5_3-4.webm.ivf.s45551_r01-05_b6-.ivf.res fe346136b9b8c1e6f6084cc106485706915795e4 *invalid-vp90-01-v3.webm 5d9474c0309b7ca09a182d888f73b37a8fe1362c *invalid-vp90-01-v3.webm.res d78e2fceba5ac942246503ec8366f879c4775ca5 *invalid-vp90-02-v2.webm diff --git a/vpx_dsp/arm/idct16x16_1_add_neon.c b/vpx_dsp/arm/idct16x16_1_add_neon.c index 968bc5cc3..bf5192a68 100644 --- a/vpx_dsp/arm/idct16x16_1_add_neon.c +++ b/vpx_dsp/arm/idct16x16_1_add_neon.c @@ -32,7 +32,8 @@ static INLINE void idct16x16_1_add_neg_kernel(uint8_t **dest, const int stride, void vpx_idct16x16_1_add_neon(const tran_low_t *input, uint8_t *dest, int stride) { - const int16_t out0 = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + const int16_t out0 = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64)); const int16_t a1 = ROUND_POWER_OF_TWO(out1, 6); diff --git a/vpx_dsp/arm/idct32x32_1_add_neon.c b/vpx_dsp/arm/idct32x32_1_add_neon.c index 604d82abd..8920b9336 100644 --- a/vpx_dsp/arm/idct32x32_1_add_neon.c +++ b/vpx_dsp/arm/idct32x32_1_add_neon.c @@ -39,7 +39,8 @@ static INLINE void idct32x32_1_add_neg_kernel(uint8_t **dest, const int stride, void vpx_idct32x32_1_add_neon(const tran_low_t *input, uint8_t *dest, int stride) { int i; - const int16_t out0 = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + const int16_t out0 = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64)); const int16_t a1 = ROUND_POWER_OF_TWO(out1, 6); diff --git a/vpx_dsp/arm/idct4x4_1_add_neon.c b/vpx_dsp/arm/idct4x4_1_add_neon.c index 21d21b033..a14b89543 100644 --- a/vpx_dsp/arm/idct4x4_1_add_neon.c +++ b/vpx_dsp/arm/idct4x4_1_add_neon.c @@ -32,7 +32,8 @@ static INLINE void idct4x4_1_add_kernel(uint8_t **dest, const int stride, void vpx_idct4x4_1_add_neon(const tran_low_t *input, uint8_t *dest, int stride) { - const int16_t out0 = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + const int16_t out0 = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64)); const int16_t a1 = ROUND_POWER_OF_TWO(out1, 4); const int16x8_t dc = vdupq_n_s16(a1); diff --git a/vpx_dsp/arm/idct8x8_1_add_neon.c b/vpx_dsp/arm/idct8x8_1_add_neon.c index 7bcce913b..ce9b45958 100644 --- a/vpx_dsp/arm/idct8x8_1_add_neon.c +++ b/vpx_dsp/arm/idct8x8_1_add_neon.c @@ -36,7 +36,8 @@ static INLINE void idct8x8_1_add_neg_kernel(uint8_t **dest, const int stride, void vpx_idct8x8_1_add_neon(const tran_low_t *input, uint8_t *dest, int stride) { - const int16_t out0 = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + const int16_t out0 = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); const int16_t out1 = WRAPLOW(dct_const_round_shift(out0 * cospi_16_64)); const int16_t a1 = ROUND_POWER_OF_TWO(out1, 5); diff --git a/vpx_dsp/inv_txfm.c b/vpx_dsp/inv_txfm.c index 2320261ad..0194aa1e1 100644 --- a/vpx_dsp/inv_txfm.c +++ b/vpx_dsp/inv_txfm.c @@ -131,16 +131,16 @@ void iadst4_c(const tran_low_t *input, tran_low_t *output) { } void idct4_c(const tran_low_t *input, tran_low_t *output) { - tran_low_t step[4]; + int16_t step[4]; tran_high_t temp1, temp2; // stage 1 - temp1 = (input[0] + input[2]) * cospi_16_64; - temp2 = (input[0] - input[2]) * cospi_16_64; + temp1 = ((int16_t)input[0] + (int16_t)input[2]) * cospi_16_64; + temp2 = ((int16_t)input[0] - (int16_t)input[2]) * cospi_16_64; step[0] = WRAPLOW(dct_const_round_shift(temp1)); step[1] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[1] * cospi_24_64 - input[3] * cospi_8_64; - temp2 = input[1] * cospi_8_64 + input[3] * cospi_24_64; + temp1 = (int16_t)input[1] * cospi_24_64 - (int16_t)input[3] * cospi_8_64; + temp2 = (int16_t)input[1] * cospi_8_64 + (int16_t)input[3] * cospi_24_64; step[2] = WRAPLOW(dct_const_round_shift(temp1)); step[3] = WRAPLOW(dct_const_round_shift(temp2)); @@ -178,7 +178,8 @@ void vpx_idct4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride) { void vpx_idct4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { int i; tran_high_t a1; - tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + tran_low_t out = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); a1 = ROUND_POWER_OF_TWO(out, 4); @@ -268,20 +269,20 @@ void iadst8_c(const tran_low_t *input, tran_low_t *output) { } void idct8_c(const tran_low_t *input, tran_low_t *output) { - tran_low_t step1[8], step2[8]; + int16_t step1[8], step2[8]; tran_high_t temp1, temp2; // stage 1 - step1[0] = input[0]; - step1[2] = input[4]; - step1[1] = input[2]; - step1[3] = input[6]; - temp1 = input[1] * cospi_28_64 - input[7] * cospi_4_64; - temp2 = input[1] * cospi_4_64 + input[7] * cospi_28_64; + step1[0] = (int16_t)input[0]; + step1[2] = (int16_t)input[4]; + step1[1] = (int16_t)input[2]; + step1[3] = (int16_t)input[6]; + temp1 = (int16_t)input[1] * cospi_28_64 - (int16_t)input[7] * cospi_4_64; + temp2 = (int16_t)input[1] * cospi_4_64 + (int16_t)input[7] * cospi_28_64; step1[4] = WRAPLOW(dct_const_round_shift(temp1)); step1[7] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[5] * cospi_12_64 - input[3] * cospi_20_64; - temp2 = input[5] * cospi_20_64 + input[3] * cospi_12_64; + temp1 = (int16_t)input[5] * cospi_12_64 - (int16_t)input[3] * cospi_20_64; + temp2 = (int16_t)input[5] * cospi_20_64 + (int16_t)input[3] * cospi_12_64; step1[5] = WRAPLOW(dct_const_round_shift(temp1)); step1[6] = WRAPLOW(dct_const_round_shift(temp2)); @@ -374,7 +375,8 @@ void vpx_idct8x8_12_add_c(const tran_low_t *input, uint8_t *dest, int stride) { void vpx_idct8x8_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { int i, j; tran_high_t a1; - tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + tran_low_t out = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); a1 = ROUND_POWER_OF_TWO(out, 5); @@ -553,26 +555,26 @@ void iadst16_c(const tran_low_t *input, tran_low_t *output) { } void idct16_c(const tran_low_t *input, tran_low_t *output) { - tran_low_t step1[16], step2[16]; + int16_t step1[16], step2[16]; tran_high_t temp1, temp2; // stage 1 - step1[0] = input[0 / 2]; - step1[1] = input[16 / 2]; - step1[2] = input[8 / 2]; - step1[3] = input[24 / 2]; - step1[4] = input[4 / 2]; - step1[5] = input[20 / 2]; - step1[6] = input[12 / 2]; - step1[7] = input[28 / 2]; - step1[8] = input[2 / 2]; - step1[9] = input[18 / 2]; - step1[10] = input[10 / 2]; - step1[11] = input[26 / 2]; - step1[12] = input[6 / 2]; - step1[13] = input[22 / 2]; - step1[14] = input[14 / 2]; - step1[15] = input[30 / 2]; + step1[0] = (int16_t)input[0 / 2]; + step1[1] = (int16_t)input[16 / 2]; + step1[2] = (int16_t)input[8 / 2]; + step1[3] = (int16_t)input[24 / 2]; + step1[4] = (int16_t)input[4 / 2]; + step1[5] = (int16_t)input[20 / 2]; + step1[6] = (int16_t)input[12 / 2]; + step1[7] = (int16_t)input[28 / 2]; + step1[8] = (int16_t)input[2 / 2]; + step1[9] = (int16_t)input[18 / 2]; + step1[10] = (int16_t)input[10 / 2]; + step1[11] = (int16_t)input[26 / 2]; + step1[12] = (int16_t)input[6 / 2]; + step1[13] = (int16_t)input[22 / 2]; + step1[14] = (int16_t)input[14 / 2]; + step1[15] = (int16_t)input[30 / 2]; // stage 2 step2[0] = step1[0]; @@ -797,7 +799,8 @@ void vpx_idct16x16_10_add_c(const tran_low_t *input, uint8_t *dest, void vpx_idct16x16_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { int i, j; tran_high_t a1; - tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + tran_low_t out = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); a1 = ROUND_POWER_OF_TWO(out, 6); @@ -808,64 +811,64 @@ void vpx_idct16x16_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { } void idct32_c(const tran_low_t *input, tran_low_t *output) { - tran_low_t step1[32], step2[32]; + int16_t step1[32], step2[32]; tran_high_t temp1, temp2; // stage 1 - step1[0] = input[0]; - step1[1] = input[16]; - step1[2] = input[8]; - step1[3] = input[24]; - step1[4] = input[4]; - step1[5] = input[20]; - step1[6] = input[12]; - step1[7] = input[28]; - step1[8] = input[2]; - step1[9] = input[18]; - step1[10] = input[10]; - step1[11] = input[26]; - step1[12] = input[6]; - step1[13] = input[22]; - step1[14] = input[14]; - step1[15] = input[30]; + step1[0] = (int16_t)input[0]; + step1[1] = (int16_t)input[16]; + step1[2] = (int16_t)input[8]; + step1[3] = (int16_t)input[24]; + step1[4] = (int16_t)input[4]; + step1[5] = (int16_t)input[20]; + step1[6] = (int16_t)input[12]; + step1[7] = (int16_t)input[28]; + step1[8] = (int16_t)input[2]; + step1[9] = (int16_t)input[18]; + step1[10] = (int16_t)input[10]; + step1[11] = (int16_t)input[26]; + step1[12] = (int16_t)input[6]; + step1[13] = (int16_t)input[22]; + step1[14] = (int16_t)input[14]; + step1[15] = (int16_t)input[30]; - temp1 = input[1] * cospi_31_64 - input[31] * cospi_1_64; - temp2 = input[1] * cospi_1_64 + input[31] * cospi_31_64; + temp1 = (int16_t)input[1] * cospi_31_64 - (int16_t)input[31] * cospi_1_64; + temp2 = (int16_t)input[1] * cospi_1_64 + (int16_t)input[31] * cospi_31_64; step1[16] = WRAPLOW(dct_const_round_shift(temp1)); step1[31] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[17] * cospi_15_64 - input[15] * cospi_17_64; - temp2 = input[17] * cospi_17_64 + input[15] * cospi_15_64; + temp1 = (int16_t)input[17] * cospi_15_64 - (int16_t)input[15] * cospi_17_64; + temp2 = (int16_t)input[17] * cospi_17_64 + (int16_t)input[15] * cospi_15_64; step1[17] = WRAPLOW(dct_const_round_shift(temp1)); step1[30] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[9] * cospi_23_64 - input[23] * cospi_9_64; - temp2 = input[9] * cospi_9_64 + input[23] * cospi_23_64; + temp1 = (int16_t)input[9] * cospi_23_64 - (int16_t)input[23] * cospi_9_64; + temp2 = (int16_t)input[9] * cospi_9_64 + (int16_t)input[23] * cospi_23_64; step1[18] = WRAPLOW(dct_const_round_shift(temp1)); step1[29] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[25] * cospi_7_64 - input[7] * cospi_25_64; - temp2 = input[25] * cospi_25_64 + input[7] * cospi_7_64; + temp1 = (int16_t)input[25] * cospi_7_64 - (int16_t)input[7] * cospi_25_64; + temp2 = (int16_t)input[25] * cospi_25_64 + (int16_t)input[7] * cospi_7_64; step1[19] = WRAPLOW(dct_const_round_shift(temp1)); step1[28] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[5] * cospi_27_64 - input[27] * cospi_5_64; - temp2 = input[5] * cospi_5_64 + input[27] * cospi_27_64; + temp1 = (int16_t)input[5] * cospi_27_64 - (int16_t)input[27] * cospi_5_64; + temp2 = (int16_t)input[5] * cospi_5_64 + (int16_t)input[27] * cospi_27_64; step1[20] = WRAPLOW(dct_const_round_shift(temp1)); step1[27] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[21] * cospi_11_64 - input[11] * cospi_21_64; - temp2 = input[21] * cospi_21_64 + input[11] * cospi_11_64; + temp1 = (int16_t)input[21] * cospi_11_64 - (int16_t)input[11] * cospi_21_64; + temp2 = (int16_t)input[21] * cospi_21_64 + (int16_t)input[11] * cospi_11_64; step1[21] = WRAPLOW(dct_const_round_shift(temp1)); step1[26] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[13] * cospi_19_64 - input[19] * cospi_13_64; - temp2 = input[13] * cospi_13_64 + input[19] * cospi_19_64; + temp1 = (int16_t)input[13] * cospi_19_64 - (int16_t)input[19] * cospi_13_64; + temp2 = (int16_t)input[13] * cospi_13_64 + (int16_t)input[19] * cospi_19_64; step1[22] = WRAPLOW(dct_const_round_shift(temp1)); step1[25] = WRAPLOW(dct_const_round_shift(temp2)); - temp1 = input[29] * cospi_3_64 - input[3] * cospi_29_64; - temp2 = input[29] * cospi_29_64 + input[3] * cospi_3_64; + temp1 = (int16_t)input[29] * cospi_3_64 - (int16_t)input[3] * cospi_29_64; + temp2 = (int16_t)input[29] * cospi_29_64 + (int16_t)input[3] * cospi_3_64; step1[23] = WRAPLOW(dct_const_round_shift(temp1)); step1[24] = WRAPLOW(dct_const_round_shift(temp2)); @@ -1260,7 +1263,8 @@ void vpx_idct32x32_34_add_c(const tran_low_t *input, uint8_t *dest, void vpx_idct32x32_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) { int i, j; tran_high_t a1; - tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + tran_low_t out = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); a1 = ROUND_POWER_OF_TWO(out, 6); diff --git a/vpx_dsp/x86/inv_txfm_sse2.c b/vpx_dsp/x86/inv_txfm_sse2.c index 64fe50be1..f6e56b6f9 100644 --- a/vpx_dsp/x86/inv_txfm_sse2.c +++ b/vpx_dsp/x86/inv_txfm_sse2.c @@ -51,7 +51,7 @@ void vpx_idct4x4_1_add_sse2(const tran_low_t *input, uint8_t *dest, int a; __m128i dc_value, d[2]; - a = (int)dct_const_round_shift(input[0] * cospi_16_64); + a = (int)dct_const_round_shift((int16_t)input[0] * cospi_16_64); a = (int)dct_const_round_shift(a * cospi_16_64); a = ROUND_POWER_OF_TWO(a, 4); @@ -210,7 +210,8 @@ void vpx_idct8x8_1_add_sse2(const tran_low_t *input, uint8_t *dest, int stride) { __m128i dc_value; tran_high_t a1; - tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + tran_low_t out = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); a1 = ROUND_POWER_OF_TWO(out, 5); @@ -547,7 +548,8 @@ void vpx_idct16x16_1_add_sse2(const tran_low_t *input, uint8_t *dest, __m128i dc_value; int i; tran_high_t a1; - tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + tran_low_t out = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); a1 = ROUND_POWER_OF_TWO(out, 6); @@ -1334,7 +1336,8 @@ void vpx_idct32x32_1_add_sse2(const tran_low_t *input, uint8_t *dest, __m128i dc_value; int j; tran_high_t a1; - tran_low_t out = WRAPLOW(dct_const_round_shift(input[0] * cospi_16_64)); + tran_low_t out = + WRAPLOW(dct_const_round_shift((int16_t)input[0] * cospi_16_64)); out = WRAPLOW(dct_const_round_shift(out * cospi_16_64)); a1 = ROUND_POWER_OF_TWO(out, 6);