generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes in 64x64 transform

Browse Source 
Extends quantizer range and some other fixes.

Change-Id: Ia9adf7848e772783365d6501efd07585bff80c15
stdhd: +0.166%, turns positive a little
derf: -0.036% (very few blocks use the 64x64 mode)
This commit is contained in:
Deb Mukherjee 2014-11-03 10:07:22 -08:00
parent 8bdf4cebb9
commit a137e3c3ba
7 changed files with 63 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
vp9/common/vp9_entropy.c
View File

@ -21,8 +21,12 @@ const vp9_prob vp9_cat3_prob[] = { 173, 148, 140 };
const vp9_prob vp9_cat4_prob[] = { 176, 155, 140, 135 };
const vp9_prob vp9_cat5_prob[] = { 180, 157, 141, 134, 130 };
const vp9_prob vp9_cat6_prob[] = {
#if TX_64X64
255,
#endif
254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129
};
#if CONFIG_VP9_HIGHBITDEPTH
const vp9_prob vp9_cat1_prob_high10[] = { 159 };
const vp9_prob vp9_cat2_prob_high10[] = { 165, 145 };
@ -30,19 +34,26 @@ const vp9_prob vp9_cat3_prob_high10[] = { 173, 148, 140 };
const vp9_prob vp9_cat4_prob_high10[] = { 176, 155, 140, 135 };
const vp9_prob vp9_cat5_prob_high10[] = { 180, 157, 141, 134, 130 };
const vp9_prob vp9_cat6_prob_high10[] = {
#if TX_64X64
255,
#endif
255, 255, 254, 254, 254, 252, 249, 243,
230, 196, 177, 153, 140, 133, 130, 129
};
const vp9_prob vp9_cat1_prob_high12[] = { 159 };
const vp9_prob vp9_cat2_prob_high12[] = { 165, 145 };
const vp9_prob vp9_cat3_prob_high12[] = { 173, 148, 140 };
const vp9_prob vp9_cat4_prob_high12[] = { 176, 155, 140, 135 };
const vp9_prob vp9_cat5_prob_high12[] = { 180, 157, 141, 134, 130 };
const vp9_prob vp9_cat6_prob_high12[] = {
#if TX_64X64
255,
#endif
255, 255, 255, 255, 254, 254, 254, 252, 249,
243, 230, 196, 177, 153, 140, 133, 130, 129
};
#endif
#endif // CONFIG_VP9_HIGHBITDEPTH
const uint8_t vp9_coefband_trans_8x8plus[MAX_NUM_COEFS] = {
0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4,

46
vp9/common/vp9_entropy.h
View File

@ -50,13 +50,35 @@ DECLARE_ALIGNED(16, extern const uint8_t, vp9_pt_energy_class[ENTROPY_TOKENS]);
#define CAT5_MIN_VAL 35
#define CAT6_MIN_VAL 67
#if CONFIG_TX64X64
#define DCT_MAX_VALUE 32768
#define NUM_CAT6_BITS 15
#else
#define DCT_MAX_VALUE 16384
#define NUM_CAT6_BITS 14
#endif // CONFIG_TX64X64
#if CONFIG_VP9_HIGHBITDEPTH
#if CONFIG_TX64X64
#define DCT_MAX_VALUE_HIGH10 131072
#define DCT_MAX_VALUE_HIGH12 524288
#define NUM_CAT6_BITS_HIGH10 17
#define NUM_CAT6_BITS_HIGH12 19
#else
#define DCT_MAX_VALUE_HIGH10 65536
#define DCT_MAX_VALUE_HIGH12 262144
#define NUM_CAT6_BITS_HIGH10 16
#define NUM_CAT6_BITS_HIGH12 18
#endif // CONFIG_TX64X64
#endif // CONFIG_VP9_HIGHBITDEPTH
// Extra bit probabilities.
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat1_prob[1]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat2_prob[2]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat3_prob[3]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat4_prob[4]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat5_prob[5]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob[14]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob[NUM_CAT6_BITS]);
#if CONFIG_VP9_HIGHBITDEPTH
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat1_prob_high10[1]);
@ -64,13 +86,15 @@ DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat2_prob_high10[2]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat3_prob_high10[3]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat4_prob_high10[4]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat5_prob_high10[5]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob_high10[16]);
DECLARE_ALIGNED(16, extern const uint8_t,
vp9_cat6_prob_high10[NUM_CAT6_BITS_HIGH10]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat1_prob_high12[1]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat2_prob_high12[2]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat3_prob_high12[3]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat4_prob_high12[4]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat5_prob_high12[5]);
DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob_high12[18]);
DECLARE_ALIGNED(16, extern const uint8_t,
vp9_cat6_prob_high12[NUM_CAT6_BITS_HIGH12]);
#endif // CONFIG_VP9_HIGHBITDEPTH
#define EOB_MODEL_TOKEN 3
@ -90,22 +114,6 @@ extern const vp9_extra_bit vp9_extra_bits_high10[ENTROPY_TOKENS];
extern const vp9_extra_bit vp9_extra_bits_high12[ENTROPY_TOKENS];
#endif // CONFIG_VP9_HIGHBITDEPTH
#if CONFIG_TX64X64
#define DCT_MAX_VALUE 32768
#else
#define DCT_MAX_VALUE 16384
#endif // CONFIG_TX64X64
#if CONFIG_VP9_HIGHBITDEPTH
#if CONFIG_TX64X64
#define DCT_MAX_VALUE_HIGH10 131072
#define DCT_MAX_VALUE_HIGH12 524288
#else
#define DCT_MAX_VALUE_HIGH10 65536
#define DCT_MAX_VALUE_HIGH12 262144
#endif // CONFIG_TX64X64
#endif // CONFIG_VP9_HIGHBITDEPTH
/* Coefficients are predicted via a 3-dimensional probability table. */
#define REF_TYPES 2 // intra=0, inter=1

6
vp9/common/vp9_entropymode.c
View File

@ -266,8 +266,8 @@ static const vp9_prob default_single_ref_p[REF_CONTEXTS][2] = {
static const struct tx_probs default_tx_probs = {
#if CONFIG_TX64X64
{ { 3, 3, 136, 37 },
{ 3, 5, 52, 13 } },
{ { 1, 3, 136, 48 },
{ 1, 5, 52, 24 } },
#endif
{ { 3, 136, 37 },
@ -298,7 +298,7 @@ void tx_counts_to_branch_counts_64x64(const unsigned int *tx_count_64x64p,
ct_64x64p[3][0] = tx_count_64x64p[TX_32X32];
ct_64x64p[3][1] = tx_count_64x64p[TX_64X64];
}
#endif
#endif // CONFIG_TX64X64
void tx_counts_to_branch_counts_32x32(const unsigned int *tx_count_32x32p,
unsigned int (*ct_32x32p)[2]) {

2
vp9/common/vp9_scan.c
View File

@ -2075,7 +2075,7 @@ DECLARE_ALIGNED(16, static const int16_t,
4026, 4089, 3901, 3964, 3838, 3901, 3964, 4027,
3775, 3838, 4027, 4090, 3902, 3965, 3965, 4028,
3839, 3902, 4028, 4091, 3966, 4029, 3903, 3966,
4029, 4092, 3967, 4030, 4030, 4093, 4031, 4094,
4029, 4092, 3967, 4030, 4030, 4093, 4031, 4094, 0, 0,
};
#endif // CONFIG_TX64X64

10
vp9/decoder/vp9_detokenize.c
View File

@ -171,20 +171,22 @@ static int decode_coefs(VP9_COMMON *cm, const MACROBLOCKD *xd, PLANE_TYPE type,
#if CONFIG_VP9_HIGHBITDEPTH
switch (cm->bit_depth) {
case VPX_BITS_8:
val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
val = CAT6_MIN_VAL + read_coeff(cat6_prob, NUM_CAT6_BITS, r);
break;
case VPX_BITS_10:
val = CAT6_MIN_VAL + read_coeff(cat6_prob, 16, r);
val = CAT6_MIN_VAL +
read_coeff(cat6_prob, NUM_CAT6_BITS_HIGH10, r);
break;
case VPX_BITS_12:
val = CAT6_MIN_VAL + read_coeff(cat6_prob, 18, r);
val = CAT6_MIN_VAL +
read_coeff(cat6_prob, NUM_CAT6_BITS_HIGH12, r);
break;
default:
assert(0);
return -1;
}
#else
val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
val = CAT6_MIN_VAL + read_coeff(cat6_prob, NUM_CAT6_BITS, r);
#endif
break;
}

2
vp9/encoder/vp9_encodemb.c
View File

@ -702,7 +702,9 @@ static void encode_block(int plane, int block, BLOCK_SIZE plane_bsize,
dst = &pd->dst.buf[4 * j * pd->dst.stride + 4 * i];
a = &ctx->ta[plane][i];
l = &ctx->tl[plane][j];
#if CONFIG_TX64X64
if (plane) assert(tx_size != TX_64X64);
#endif
// TODO(jingning): per transformed block zero forcing only enabled for
// luma component. will integrate chroma components as well.

22
vp9/encoder/vp9_tokenize.c
View File

@ -67,21 +67,21 @@ const vp9_tree_index vp9_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)] = {
-CATEGORY5_TOKEN, -CATEGORY6_TOKEN // 7 = CAT_FIVE
};
static vp9_tree_index cat1[2], cat2[4], cat3[6], cat4[8], cat5[10], cat6[28];
static vp9_tree_index cat1[2], cat2[4], cat3[6], cat4[8], cat5[10],
cat6[NUM_CAT6_BITS * 2];
#if CONFIG_VP9_HIGHBITDEPTH
static vp9_tree_index cat1_high10[2];
static vp9_tree_index cat2_high10[4];
static vp9_tree_index cat3_high10[6];
static vp9_tree_index cat4_high10[8];
static vp9_tree_index cat5_high10[10];
static vp9_tree_index cat6_high10[32];
static vp9_tree_index cat6_high10[NUM_CAT6_BITS_HIGH10 * 2];
static vp9_tree_index cat1_high12[2];
static vp9_tree_index cat2_high12[4];
static vp9_tree_index cat3_high12[6];
static vp9_tree_index cat4_high12[8];
static vp9_tree_index cat5_high12[10];
static vp9_tree_index cat6_high12[36];
static vp9_tree_index cat6_high12[NUM_CAT6_BITS_HIGH12 * 2];
#endif
static void init_bit_tree(vp9_tree_index *p, int n) {
@ -101,20 +101,20 @@ static void init_bit_trees() {
init_bit_tree(cat3, 3);
init_bit_tree(cat4, 4);
init_bit_tree(cat5, 5);
init_bit_tree(cat6, 14);
init_bit_tree(cat6, NUM_CAT6_BITS);
#if CONFIG_VP9_HIGHBITDEPTH
init_bit_tree(cat1_high10, 1);
init_bit_tree(cat2_high10, 2);
init_bit_tree(cat3_high10, 3);
init_bit_tree(cat4_high10, 4);
init_bit_tree(cat5_high10, 5);
init_bit_tree(cat6_high10, 16);
init_bit_tree(cat6_high10, NUM_CAT6_BITS_HIGH10);
init_bit_tree(cat1_high12, 1);
init_bit_tree(cat2_high12, 2);
init_bit_tree(cat3_high12, 3);
init_bit_tree(cat4_high12, 4);
init_bit_tree(cat5_high12, 5);
init_bit_tree(cat6_high12, 18);
init_bit_tree(cat6_high12, NUM_CAT6_BITS_HIGH12);
#endif
}
@ -129,7 +129,7 @@ const vp9_extra_bit vp9_extra_bits[ENTROPY_TOKENS] = {
{cat3, vp9_cat3_prob, 3, CAT3_MIN_VAL}, // CATEGORY3_TOKEN
{cat4, vp9_cat4_prob, 4, CAT4_MIN_VAL}, // CATEGORY4_TOKEN
{cat5, vp9_cat5_prob, 5, CAT5_MIN_VAL}, // CATEGORY5_TOKEN
{cat6, vp9_cat6_prob, 14, CAT6_MIN_VAL}, // CATEGORY6_TOKEN
{cat6, vp9_cat6_prob, NUM_CAT6_BITS, CAT6_MIN_VAL}, // CATEGORY6_TOKEN
{0, 0, 0, 0} // EOB_TOKEN
};
@ -145,7 +145,8 @@ const vp9_extra_bit vp9_extra_bits_high10[ENTROPY_TOKENS] = {
{cat3_high10, vp9_cat3_prob_high10, 3, CAT3_MIN_VAL}, // CATEGORY3_TOKEN
{cat4_high10, vp9_cat4_prob_high10, 4, CAT4_MIN_VAL}, // CATEGORY4_TOKEN
{cat5_high10, vp9_cat5_prob_high10, 5, CAT5_MIN_VAL}, // CATEGORY5_TOKEN
{cat6_high10, vp9_cat6_prob_high10, 16, CAT6_MIN_VAL}, // CATEGORY6_TOKEN
{cat6_high10, vp9_cat6_prob_high10, NUM_CAT6_BITS_HIGH10, CAT6_MIN_VAL},
// CATEGORY6_TOKEN
{0, 0, 0, 0} // EOB_TOKEN
};
const vp9_extra_bit vp9_extra_bits_high12[ENTROPY_TOKENS] = {
@ -159,7 +160,8 @@ const vp9_extra_bit vp9_extra_bits_high12[ENTROPY_TOKENS] = {
{cat3_high12, vp9_cat3_prob_high12, 3, CAT3_MIN_VAL}, // CATEGORY3_TOKEN
{cat4_high12, vp9_cat4_prob_high12, 4, CAT4_MIN_VAL}, // CATEGORY4_TOKEN
{cat5_high12, vp9_cat5_prob_high12, 5, CAT5_MIN_VAL}, // CATEGORY5_TOKEN
{cat6_high12, vp9_cat6_prob_high12, 18, CAT6_MIN_VAL}, // CATEGORY6_TOKEN
{cat6_high12, vp9_cat6_prob_high12, NUM_CAT6_BITS_HIGH12, CAT6_MIN_VAL},
// CATEGORY6_TOKEN
{0, 0, 0, 0} // EOB_TOKEN
};
#endif