generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move highbd txfm input range check from 2d iht transform to 1d idct/iadst

Browse Source 
This change will make the highbd txfm input range check more comprehensive

The 25-bit highbd input range is composed by
12 signal input bits + 7 bits for 2D forward transform amplification + 5 bits for
1D inverse transform amplification + 1 bit for contingency in rounding and quantizing

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1286
BUG=https://bugs.chromium.org/p/chromium/issues/detail?id=651625

Change-Id: I04c0796edd7653f8d463fba5dc418132986131e7
This commit is contained in:
Angie Chiang 2016-09-30 16:53:20 -07:00
parent 50b9c467da
commit 5b073c695b
2 changed files with 72 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
vp9/common/vp9_idct.c
View File

@ -203,17 +203,6 @@ void vp9_iht16x16_add(TX_TYPE tx_type, const tran_low_t *input, uint8_t *dest,
#if CONFIG_VP9_HIGHBITDEPTH #if CONFIG_VP9_HIGHBITDEPTH
// 12 signal input bits + 7 forward transform amplify bits + 1 bit
// for contingency in rounding and quantizing
#define VALID_IHT_MAGNITUDE_RANGE (1 << 20)
static INLINE int detect_invalid_iht_input(const tran_low_t *input, int size) {
int i;
for (i = 0; i < size; ++i)
if (abs(input[i]) >= VALID_IHT_MAGNITUDE_RANGE) return 1;
return 0;
}
void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
int stride, int tx_type, int bd) { int stride, int tx_type, int bd) {
const highbd_transform_2d IHT_4[] = { const highbd_transform_2d IHT_4[] = {
@ -229,13 +218,6 @@ void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
tran_low_t *outptr = out; tran_low_t *outptr = out;
tran_low_t temp_in[4], temp_out[4]; tran_low_t temp_in[4], temp_out[4];
if (detect_invalid_iht_input(input, 16)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd iht input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
return;
}
// Inverse transform row vectors. // Inverse transform row vectors.
for (i = 0; i < 4; ++i) { for (i = 0; i < 4; ++i) {
IHT_4[tx_type].rows(input, outptr, bd); IHT_4[tx_type].rows(input, outptr, bd);
@ -270,13 +252,6 @@ void vp9_highbd_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest8,
const highbd_transform_2d ht = HIGH_IHT_8[tx_type]; const highbd_transform_2d ht = HIGH_IHT_8[tx_type];
uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
if (detect_invalid_iht_input(input, 64)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd iht input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
return;
}
// Inverse transform row vectors. // Inverse transform row vectors.
for (i = 0; i < 8; ++i) { for (i = 0; i < 8; ++i) {
ht.rows(input, outptr, bd); ht.rows(input, outptr, bd);
@ -311,13 +286,6 @@ void vp9_highbd_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest8,
const highbd_transform_2d ht = HIGH_IHT_16[tx_type]; const highbd_transform_2d ht = HIGH_IHT_16[tx_type];
uint16_t *dest = CONVERT_TO_SHORTPTR(dest8); uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
if (detect_invalid_iht_input(input, 256)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd iht input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
return;
}
// Rows // Rows
for (i = 0; i < 16; ++i) { for (i = 0; i < 16; ++i) {
ht.rows(input, outptr, bd); ht.rows(input, outptr, bd);

75
vpx_dsp/inv_txfm.c
View File

@ -9,6 +9,7 @@
*/ */
#include <math.h> #include <math.h>
#include <stdlib.h>
#include <string.h> #include <string.h>
#include "./vpx_dsp_rtcd.h" #include "./vpx_dsp_rtcd.h"
@ -1252,6 +1253,19 @@ void vpx_idct32x32_1_add_c(const tran_low_t *input, uint8_t *dest, int stride) {
} }
#if CONFIG_VP9_HIGHBITDEPTH #if CONFIG_VP9_HIGHBITDEPTH
// 12 signal input bits + 7 2D forward transform amplify bits + 5 1D inverse
// transform amplify bits + 1 bit for contingency in rounding and quantizing
#define HIGHBD_VALID_TXFM_MAGNITUDE_RANGE (1 << 25)
static INLINE int detect_invalid_highbd_input(const tran_low_t *input,
int size) {
int i;
for (i = 0; i < size; ++i)
if (abs(input[i]) >= HIGHBD_VALID_TXFM_MAGNITUDE_RANGE) return 1;
return 0;
}
void vpx_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8, void vpx_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
int stride, int bd) { int stride, int bd) {
/* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds, /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds,
@ -1347,6 +1361,15 @@ void vpx_highbd_idct4_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_low_t step[4]; tran_low_t step[4];
tran_high_t temp1, temp2; tran_high_t temp1, temp2;
(void)bd; (void)bd;
if (detect_invalid_highbd_input(input, 4)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd txfm input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
memset(output, 0, sizeof(*output) * 4);
return;
}
// stage 1 // stage 1
temp1 = (input[0] + input[2]) * cospi_16_64; temp1 = (input[0] + input[2]) * cospi_16_64;
temp2 = (input[0] - input[2]) * cospi_16_64; temp2 = (input[0] - input[2]) * cospi_16_64;
@ -1413,6 +1436,15 @@ void vpx_highbd_idct4x4_1_add_c(const tran_low_t *input, uint8_t *dest8,
void vpx_highbd_idct8_c(const tran_low_t *input, tran_low_t *output, int bd) { void vpx_highbd_idct8_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_low_t step1[8], step2[8]; tran_low_t step1[8], step2[8];
tran_high_t temp1, temp2; tran_high_t temp1, temp2;
if (detect_invalid_highbd_input(input, 8)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd txfm input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
memset(output, 0, sizeof(*output) * 8);
return;
}
// stage 1 // stage 1
step1[0] = input[0]; step1[0] = input[0];
step1[2] = input[4]; step1[2] = input[4];
@ -1498,13 +1530,20 @@ void vpx_highbd_idct8x8_1_add_c(const tran_low_t *input, uint8_t *dest8,
void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) { void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;
tran_low_t x0 = input[0]; tran_low_t x0 = input[0];
tran_low_t x1 = input[1]; tran_low_t x1 = input[1];
tran_low_t x2 = input[2]; tran_low_t x2 = input[2];
tran_low_t x3 = input[3]; tran_low_t x3 = input[3];
(void)bd; (void)bd;
if (detect_invalid_highbd_input(input, 4)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd txfm input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
memset(output, 0, sizeof(*output) * 4);
return;
}
if (!(x0 | x1 | x2 | x3)) { if (!(x0 | x1 | x2 | x3)) {
memset(output, 0, 4 * sizeof(*output)); memset(output, 0, 4 * sizeof(*output));
return; return;
@ -1536,7 +1575,6 @@ void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) {
void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) { void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_high_t s0, s1, s2, s3, s4, s5, s6, s7; tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;
tran_low_t x0 = input[7]; tran_low_t x0 = input[7];
tran_low_t x1 = input[0]; tran_low_t x1 = input[0];
tran_low_t x2 = input[5]; tran_low_t x2 = input[5];
@ -1547,6 +1585,14 @@ void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_low_t x7 = input[6]; tran_low_t x7 = input[6];
(void)bd; (void)bd;
if (detect_invalid_highbd_input(input, 8)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd txfm input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
memset(output, 0, sizeof(*output) * 8);
return;
}
if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) { if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) {
memset(output, 0, 8 * sizeof(*output)); memset(output, 0, 8 * sizeof(*output));
return; return;
@ -1642,6 +1688,14 @@ void vpx_highbd_idct16_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_high_t temp1, temp2; tran_high_t temp1, temp2;
(void)bd; (void)bd;
if (detect_invalid_highbd_input(input, 16)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd txfm input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
memset(output, 0, sizeof(*output) * 16);
return;
}
// stage 1 // stage 1
step1[0] = input[0 / 2]; step1[0] = input[0 / 2];
step1[1] = input[16 / 2]; step1[1] = input[16 / 2];
@ -1832,7 +1886,6 @@ void vpx_highbd_idct16x16_256_add_c(const tran_low_t *input, uint8_t *dest8,
void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) { void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8; tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
tran_high_t s9, s10, s11, s12, s13, s14, s15; tran_high_t s9, s10, s11, s12, s13, s14, s15;
tran_low_t x0 = input[15]; tran_low_t x0 = input[15];
tran_low_t x1 = input[0]; tran_low_t x1 = input[0];
tran_low_t x2 = input[13]; tran_low_t x2 = input[13];
@ -1851,6 +1904,14 @@ void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) {
tran_low_t x15 = input[14]; tran_low_t x15 = input[14];
(void)bd; (void)bd;
if (detect_invalid_highbd_input(input, 16)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd txfm input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
memset(output, 0, sizeof(*output) * 16);
return;
}
if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 | if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 |
x13 | x14 | x15)) { x13 | x14 | x15)) {
memset(output, 0, 16 * sizeof(*output)); memset(output, 0, 16 * sizeof(*output));
@ -2048,6 +2109,14 @@ static void highbd_idct32_c(const tran_low_t *input, tran_low_t *output,
tran_high_t temp1, temp2; tran_high_t temp1, temp2;
(void)bd; (void)bd;
if (detect_invalid_highbd_input(input, 32)) {
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(0 && "invalid highbd txfm input");
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
memset(output, 0, sizeof(*output) * 32);
return;
}
// stage 1 // stage 1
step1[0] = input[0]; step1[0] = input[0];
step1[1] = input[16]; step1[1] = input[16];