generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use transform-size selection for i8x8_pred also.

Browse Source 
Change-Id: Iecb282fc89f9b5145ef31c5eda294ad42bc32a5d
This commit is contained in:
Ronald S. Bultje
2012-10-10 17:18:22 -07:00
parent a7333b0a5b
commit f39b0f192f
7 changed files with 220 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
vp8/decoder/decodemv.c
View File

@@ -172,10 +172,10 @@ static void vp8_kfread_modes(VP8D_COMP *pbi,
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
if (cm->txfm_mode == TX_MODE_SELECT && m->mbmi.mb_skip_coeff == 0 && if (cm->txfm_mode == TX_MODE_SELECT && m->mbmi.mb_skip_coeff == 0 &&
m->mbmi.mode <= TM_PRED) { m->mbmi.mode <= I8X8_PRED) {
// FIXME(rbultje) code ternary symbol once all experiments are merged // FIXME(rbultje) code ternary symbol once all experiments are merged
m->mbmi.txfm_size = vp8_read(bc, cm->prob_tx[0]); m->mbmi.txfm_size = vp8_read(bc, cm->prob_tx[0]);
if (m->mbmi.txfm_size != TX_4X4) if (m->mbmi.txfm_size != TX_4X4 && m->mbmi.mode != I8X8_PRED)
m->mbmi.txfm_size += vp8_read(bc, cm->prob_tx[1]); m->mbmi.txfm_size += vp8_read(bc, cm->prob_tx[1]);
} else } else
#endif #endif
@@ -1281,11 +1281,11 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
if (cm->txfm_mode == TX_MODE_SELECT && mbmi->mb_skip_coeff == 0 && if (cm->txfm_mode == TX_MODE_SELECT && mbmi->mb_skip_coeff == 0 &&
((mbmi->ref_frame == INTRA_FRAME && mbmi->mode <= TM_PRED) || ((mbmi->ref_frame == INTRA_FRAME && mbmi->mode <= I8X8_PRED) ||
(mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) { (mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) {
// FIXME(rbultje) code ternary symbol once all experiments are merged // FIXME(rbultje) code ternary symbol once all experiments are merged
mbmi->txfm_size = vp8_read(bc, cm->prob_tx[0]); mbmi->txfm_size = vp8_read(bc, cm->prob_tx[0]);
if (mbmi->txfm_size != TX_4X4) if (mbmi->txfm_size != TX_4X4 && mbmi->mode != I8X8_PRED)
mbmi->txfm_size += vp8_read(bc, cm->prob_tx[1]); mbmi->txfm_size += vp8_read(bc, cm->prob_tx[1]);
} else } else
#endif #endif

36
vp8/decoder/decodframe.c
View File

@@ -364,10 +364,8 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
if (mode == I8X8_PRED) { if (mode == I8X8_PRED) {
for (i = 0; i < 4; i++) { for (i = 0; i < 4; i++) {
int ib = vp8_i8x8_block[i]; int ib = vp8_i8x8_block[i];
#if !CONFIG_HYBRIDTRANSFORM8X8
const int iblock[4] = {0, 1, 4, 5}; const int iblock[4] = {0, 1, 4, 5};
int j; int j;
#endif
int i8x8mode; int i8x8mode;
BLOCKD *b; BLOCKD *b;
@@ -381,9 +379,10 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
b = &xd->block[ib]; b = &xd->block[ib];
i8x8mode = b->bmi.as_mode.first; i8x8mode = b->bmi.as_mode.first;
RECON_INVOKE(RTCD_VTABLE(recon), intra8x8_predict) RECON_INVOKE(RTCD_VTABLE(recon), intra8x8_predict)(b, i8x8mode,
(b, i8x8mode, b->predictor); b->predictor);
if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
#if CONFIG_HYBRIDTRANSFORM8X8 #if CONFIG_HYBRIDTRANSFORM8X8
vp8_ht_dequant_idct_add_8x8_c(b->bmi.as_mode.tx_type, vp8_ht_dequant_idct_add_8x8_c(b->bmi.as_mode.tx_type,
q, dq, pre, dst, 16, stride); q, dq, pre, dst, 16, stride);
@@ -392,19 +391,28 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
vp8_dequant_idct_add_8x8_c(q, dq, pre, dst, 16, stride); vp8_dequant_idct_add_8x8_c(q, dq, pre, dst, 16, stride);
q += 64; q += 64;
#endif #endif
} else {
for (j = 0; j < 4; j++) {
b = &xd->block[ib + iblock[j]];
vp8_dequant_idct_add_c(b->qcoeff, b->dequant, b->predictor,
*(b->base_dst) + b->dst, 16, b->dst_stride);
}
}
b = &xd->block[16 + i]; b = &xd->block[16 + i];
RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict) RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict)(b, i8x8mode,
(b, i8x8mode, b->predictor); b->predictor);
DEQUANT_INVOKE(&pbi->dequant, idct_add) DEQUANT_INVOKE(&pbi->dequant, idct_add)(b->qcoeff, b->dequant,
(b->qcoeff, b->dequant, b->predictor, b->predictor,
*(b->base_dst) + b->dst, 8, b->dst_stride); *(b->base_dst) + b->dst, 8,
b->dst_stride);
b = &xd->block[20 + i]; b = &xd->block[20 + i];
RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict) RECON_INVOKE(RTCD_VTABLE(recon), intra_uv4x4_predict)(b, i8x8mode,
(b, i8x8mode, b->predictor); b->predictor);
DEQUANT_INVOKE(&pbi->dequant, idct_add) DEQUANT_INVOKE(&pbi->dequant, idct_add)(b->qcoeff, b->dequant,
(b->qcoeff, b->dequant, b->predictor, b->predictor,
*(b->base_dst) + b->dst, 8, b->dst_stride); *(b->base_dst) + b->dst, 8,
b->dst_stride);
} }
} else if (mode == B_PRED) { } else if (mode == B_PRED) {
for (i = 0; i < 16; i++) { for (i = 0; i < 16; i++) {

49
vp8/encoder/bitstream.c
View File

@@ -239,6 +239,18 @@ static void update_mbintra_mode_probs(VP8_COMP *cpi) {
} }
} }
static __inline int get_prob(int num, int den) {
int p;
if (den <= 0)
return 128;
p = (num * 255 + (den >> 1)) / den;
if (p > 255)
return 255;
else if (p < 1)
return 1;
return p;
}
void update_skip_probs(VP8_COMP *cpi) { void update_skip_probs(VP8_COMP *cpi) {
VP8_COMMON *const pc = & cpi->common; VP8_COMMON *const pc = & cpi->common;
int prob_skip_false[3] = {0, 0, 0}; int prob_skip_false[3] = {0, 0, 0};
@@ -1289,7 +1301,7 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) {
} }
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
if (((rf == INTRA_FRAME && mode <= TM_PRED) || if (((rf == INTRA_FRAME && mode <= I8X8_PRED) ||
(rf != INTRA_FRAME && mode != SPLITMV)) && (rf != INTRA_FRAME && mode != SPLITMV)) &&
pc->txfm_mode == TX_MODE_SELECT && pc->txfm_mode == TX_MODE_SELECT &&
!((pc->mb_no_coeff_skip && mi->mb_skip_coeff) || !((pc->mb_no_coeff_skip && mi->mb_skip_coeff) ||
@@ -1298,7 +1310,7 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) {
TX_SIZE sz = mi->txfm_size; TX_SIZE sz = mi->txfm_size;
// FIXME(rbultje) code ternary symbol once all experiments are merged // FIXME(rbultje) code ternary symbol once all experiments are merged
vp8_write(w, sz != TX_4X4, pc->prob_tx[0]); vp8_write(w, sz != TX_4X4, pc->prob_tx[0]);
if (sz != TX_4X4) if (sz != TX_4X4 && mode != I8X8_PRED)
vp8_write(w, sz != TX_8X8, pc->prob_tx[1]); vp8_write(w, sz != TX_8X8, pc->prob_tx[1]);
} }
#endif #endif
@@ -1467,14 +1479,14 @@ static void write_kfmodes(VP8_COMP *cpi) {
write_uv_mode(bc, m->mbmi.uv_mode, c->kf_uv_mode_prob[ym]); write_uv_mode(bc, m->mbmi.uv_mode, c->kf_uv_mode_prob[ym]);
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
if (ym <= TM_PRED && c->txfm_mode == TX_MODE_SELECT && if (ym <= I8X8_PRED && c->txfm_mode == TX_MODE_SELECT &&
!((c->mb_no_coeff_skip && m->mbmi.mb_skip_coeff) || !((c->mb_no_coeff_skip && m->mbmi.mb_skip_coeff) ||
(segfeature_active(xd, segment_id, SEG_LVL_EOB) && (segfeature_active(xd, segment_id, SEG_LVL_EOB) &&
get_segdata(xd, segment_id, SEG_LVL_EOB) == 0))) { get_segdata(xd, segment_id, SEG_LVL_EOB) == 0))) {
TX_SIZE sz = m->mbmi.txfm_size; TX_SIZE sz = m->mbmi.txfm_size;
// FIXME(rbultje) code ternary symbol once all experiments are merged // FIXME(rbultje) code ternary symbol once all experiments are merged
vp8_write(bc, sz != TX_4X4, c->prob_tx[0]); vp8_write(bc, sz != TX_4X4, c->prob_tx[0]);
if (sz != TX_4X4) if (sz != TX_4X4 && ym <= TM_PRED)
vp8_write(bc, sz != TX_8X8, c->prob_tx[1]); vp8_write(bc, sz != TX_8X8, c->prob_tx[1]);
} }
#endif #endif
@@ -2626,32 +2638,13 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size)
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
{ {
int cnt = cpi->txfm_count[0] + cpi->txfm_count[1] + cpi->txfm_count[2]; if (pc->txfm_mode == TX_MODE_SELECT) {
if (cnt && pc->txfm_mode == TX_MODE_SELECT) { pc->prob_tx[0] = get_prob(cpi->txfm_count[0] + cpi->txfm_count_8x8p[0],
int prob = (255 * (cpi->txfm_count[1] + cpi->txfm_count[2]) + (cnt >> 1)) / cnt; cpi->txfm_count[0] + cpi->txfm_count[1] + cpi->txfm_count[2] +
if (prob <= 1) { cpi->txfm_count_8x8p[0] + cpi->txfm_count_8x8p[1]);
pc->prob_tx[0] = 1; pc->prob_tx[1] = get_prob(cpi->txfm_count[1], cpi->txfm_count[1] + cpi->txfm_count[2]);
} else if (prob >= 255) {
pc->prob_tx[0] = 255;
} else {
pc->prob_tx[0] = prob;
}
pc->prob_tx[0] = 256 - pc->prob_tx[0];
} else { } else {
pc->prob_tx[0] = 128; pc->prob_tx[0] = 128;
}
cnt -= cpi->txfm_count[0];
if (cnt && pc->txfm_mode == TX_MODE_SELECT) {
int prob = (255 * cpi->txfm_count[2] + (cnt >> 1)) / cnt;
if (prob <= 1) {
pc->prob_tx[1] = 1;
} else if (prob >= 255) {
pc->prob_tx[1] = 255;
} else {
pc->prob_tx[1] = prob;
}
pc->prob_tx[1] = 256 - pc->prob_tx[1];
} else {
pc->prob_tx[1] = 128; pc->prob_tx[1] = 128;
} }
vp8_write_literal(bc, pc->txfm_mode, 2); vp8_write_literal(bc, pc->txfm_mode, 2);

10
vp8/encoder/encodeframe.c
View File

@@ -1396,6 +1396,7 @@ static void encode_frame_internal(VP8_COMP *cpi) {
vpx_memset(cpi->comp_pred_count, 0, sizeof(cpi->comp_pred_count)); vpx_memset(cpi->comp_pred_count, 0, sizeof(cpi->comp_pred_count));
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
vpx_memset(cpi->txfm_count, 0, sizeof(cpi->txfm_count)); vpx_memset(cpi->txfm_count, 0, sizeof(cpi->txfm_count));
vpx_memset(cpi->txfm_count_8x8p, 0, sizeof(cpi->txfm_count_8x8p));
vpx_memset(cpi->rd_tx_select_diff, 0, sizeof(cpi->rd_tx_select_diff)); vpx_memset(cpi->rd_tx_select_diff, 0, sizeof(cpi->rd_tx_select_diff));
#endif #endif
{ {
@@ -1598,13 +1599,14 @@ void vp8_encode_frame(VP8_COMP *cpi) {
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
if (cpi->common.txfm_mode == TX_MODE_SELECT) { if (cpi->common.txfm_mode == TX_MODE_SELECT) {
const int count4x4 = cpi->txfm_count[TX_4X4]; const int count4x4 = cpi->txfm_count[TX_4X4] + cpi->txfm_count_8x8p[TX_4X4];
const int count8x8 = cpi->txfm_count[TX_8X8]; const int count8x8 = cpi->txfm_count[TX_8X8];
const int count8x8_8x8p = cpi->txfm_count_8x8p[TX_8X8];
const int count16x16 = cpi->txfm_count[TX_16X16]; const int count16x16 = cpi->txfm_count[TX_16X16];
if (count4x4 == 0 && count16x16 == 0) { if (count4x4 == 0 && count16x16 == 0) {
cpi->common.txfm_mode = ALLOW_8X8; cpi->common.txfm_mode = ALLOW_8X8;
} else if (count8x8 == 0 && count16x16 == 0) { } else if (count8x8 == 0 && count16x16 == 0 && count8x8_8x8p == 0) {
cpi->common.txfm_mode = ONLY_4X4; cpi->common.txfm_mode = ONLY_4X4;
} else if (count8x8 == 0 && count4x4 == 0) { } else if (count8x8 == 0 && count4x4 == 0) {
cpi->common.txfm_mode = ALLOW_16X16; cpi->common.txfm_mode = ALLOW_16X16;
@@ -1946,6 +1948,8 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
get_segdata(&x->e_mbd, segment_id, SEG_LVL_EOB) == 0))) { get_segdata(&x->e_mbd, segment_id, SEG_LVL_EOB) == 0))) {
if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED) { if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED) {
cpi->txfm_count[mbmi->txfm_size]++; cpi->txfm_count[mbmi->txfm_size]++;
} else if (mbmi->mode == I8X8_PRED) {
cpi->txfm_count_8x8p[mbmi->txfm_size]++;
} }
} else } else
#endif #endif
@@ -2138,6 +2142,8 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x,
if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED && if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED &&
mbmi->mode != SPLITMV) { mbmi->mode != SPLITMV) {
cpi->txfm_count[mbmi->txfm_size]++; cpi->txfm_count[mbmi->txfm_size]++;
} else if (mbmi->mode == I8X8_PRED) {
cpi->txfm_count_8x8p[mbmi->txfm_size]++;
} }
} else } else
#endif #endif

17
vp8/encoder/encodeintra.c
View File

@@ -238,7 +238,8 @@ void vp8_encode_intra16x16mbuv(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd, void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
MACROBLOCK *x, int ib) { MACROBLOCK *x, int ib) {
BLOCKD *b = &x->e_mbd.block[ib]; MACROBLOCKD *xd = &x->e_mbd;
BLOCKD *b = &xd->block[ib];
BLOCK *be = &x->block[ib]; BLOCK *be = &x->block[ib];
const int iblock[4] = {0, 1, 4, 5}; const int iblock[4] = {0, 1, 4, 5};
int i; int i;
@@ -255,8 +256,7 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
} }
#endif #endif
{ if (x->e_mbd.mode_info_context->mbmi.txfm_size == TX_8X8) {
MACROBLOCKD *xd = &x->e_mbd;
int idx = (ib & 0x02) ? (ib + 2) : ib; int idx = (ib & 0x02) ? (ib + 2) : ib;
// generate residual blocks // generate residual blocks
@@ -274,6 +274,16 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
x->quantize_b_8x8(x->block + idx, xd->block + idx); x->quantize_b_8x8(x->block + idx, xd->block + idx);
vp8_idct_idct8(xd->block[idx].dqcoeff, xd->block[ib].diff, 32); vp8_idct_idct8(xd->block[idx].dqcoeff, xd->block[ib].diff, 32);
#endif #endif
} else {
for (i = 0; i < 4; i++) {
b = &xd->block[ib + iblock[i]];
be = &x->block[ib + iblock[i]];
ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 16);
x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
x->quantize_b(be, b);
vp8_inverse_transform_b(IF_RTCD(&rtcd->common->idct), b, 32);
}
}
// reconstruct submacroblock // reconstruct submacroblock
for (i = 0; i < 4; i++) { for (i = 0; i < 4; i++) {
@@ -282,7 +292,6 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
b->dst_stride); b->dst_stride);
} }
} }
}
void vp8_encode_intra8x8mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) { void vp8_encode_intra8x8mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
int i, ib; int i, ib;

3
vp8/encoder/onyx_int.h
View File

@@ -470,7 +470,8 @@ typedef struct VP8_COMP {
int single_pred_count[COMP_PRED_CONTEXTS]; int single_pred_count[COMP_PRED_CONTEXTS];
#if CONFIG_TX_SELECT #if CONFIG_TX_SELECT
// FIXME contextualize // FIXME contextualize
int txfm_count[TX_SIZE_MAX + 1]; int txfm_count[TX_SIZE_MAX];
int txfm_count_8x8p[TX_SIZE_MAX - 1];
int64_t rd_tx_select_diff[NB_TXFM_MODES]; int64_t rd_tx_select_diff[NB_TXFM_MODES];
int rd_tx_select_threshes[4][NB_TXFM_MODES]; int rd_tx_select_threshes[4][NB_TXFM_MODES];
#endif #endif

135
vp8/encoder/rdopt.c
View File

@@ -1565,6 +1565,7 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
vp8_subtract_4b_c(be, b, 16); vp8_subtract_4b_c(be, b, 16);
if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
#if CONFIG_HYBRIDTRANSFORM8X8 #if CONFIG_HYBRIDTRANSFORM8X8
txfm_map(b, pred_mode_conv(mode)); txfm_map(b, pred_mode_conv(mode));
vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32, vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
@@ -1573,12 +1574,11 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
#else #else
x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32); x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
#endif #endif
x->quantize_b_8x8(x->block + idx, xd->block + idx); x->quantize_b_8x8(x->block + idx, xd->block + idx);
// compute quantization mse of 8x8 block // compute quantization mse of 8x8 block
distortion = vp8_block_error_c((x->block + idx)->coeff, distortion = vp8_block_error_c((x->block + idx)->coeff,
(xd->block + idx)->dqcoeff, 64)>>2; (xd->block + idx)->dqcoeff, 64);
ta0 = *(a + vp8_block2above_8x8[idx]); ta0 = *(a + vp8_block2above_8x8[idx]);
tl0 = *(l + vp8_block2left_8x8 [idx]); tl0 = *(l + vp8_block2left_8x8 [idx]);
@@ -1588,7 +1588,40 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
rate += rate_t; rate += rate_t;
ta1 = ta0; ta1 = ta0;
tl1 = tl0; tl1 = tl0;
} else {
x->vp8_short_fdct8x4(be->src_diff, be->coeff, 32);
x->vp8_short_fdct8x4((be + 4)->src_diff, (be + 4)->coeff, 32);
x->quantize_b_pair(x->block + ib, x->block + ib + 1,
xd->block + ib, xd->block + ib + 1);
x->quantize_b_pair(x->block + ib + 4, x->block + ib + 5,
xd->block + ib + 4, xd->block + ib + 5);
distortion = vp8_block_error_c((x->block + ib)->coeff,
(xd->block + ib)->dqcoeff, 16);
distortion += vp8_block_error_c((x->block + ib + 1)->coeff,
(xd->block + ib + 1)->dqcoeff, 16);
distortion += vp8_block_error_c((x->block + ib + 4)->coeff,
(xd->block + ib + 4)->dqcoeff, 16);
distortion += vp8_block_error_c((x->block + ib + 5)->coeff,
(xd->block + ib + 5)->dqcoeff, 16);
ta0 = *(a + vp8_block2above[ib]);
ta1 = *(a + vp8_block2above[ib + 1]);
tl0 = *(l + vp8_block2above[ib]);
tl1 = *(l + vp8_block2above[ib + 4]);
rate_t = cost_coeffs(x, xd->block + ib, PLANE_TYPE_Y_WITH_DC,
&ta0, &tl0, TX_4X4);
rate_t += cost_coeffs(x, xd->block + ib + 1, PLANE_TYPE_Y_WITH_DC,
&ta1, &tl0, TX_4X4);
rate_t += cost_coeffs(x, xd->block + ib + 4, PLANE_TYPE_Y_WITH_DC,
&ta0, &tl1, TX_4X4);
rate_t += cost_coeffs(x, xd->block + ib + 5, PLANE_TYPE_Y_WITH_DC,
&ta1, &tl1, TX_4X4);
rate += rate_t;
}
distortion >>= 2;
this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion); this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
if (this_rd < best_rd) { if (this_rd < best_rd) {
*bestrate = rate; *bestrate = rate;
@@ -1617,17 +1650,18 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
#endif #endif
vp8_encode_intra8x8(IF_RTCD(&cpi->rtcd), x, ib); vp8_encode_intra8x8(IF_RTCD(&cpi->rtcd), x, ib);
#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
*(a + vp8_block2above_8x8[idx]) = besta0; *(a + vp8_block2above_8x8[idx]) = besta0;
*(a + vp8_block2above_8x8[idx] + 1) = besta1; *(a + vp8_block2above_8x8[idx] + 1) = besta1;
*(l + vp8_block2left_8x8 [idx]) = bestl0; *(l + vp8_block2left_8x8 [idx]) = bestl0;
*(l + vp8_block2left_8x8 [idx] + 1) = bestl1; *(l + vp8_block2left_8x8 [idx] + 1) = bestl1;
#else } else {
*(a + vp8_block2above[ib]) = besta0; *(a + vp8_block2above[ib]) = besta0;
*(a + vp8_block2above[ib + 1]) = besta1; *(a + vp8_block2above[ib + 1]) = besta1;
*(l + vp8_block2above[ib]) = bestl0; *(l + vp8_block2above[ib]) = bestl0;
*(l + vp8_block2above[ib + 4]) = bestl1; *(l + vp8_block2above[ib + 4]) = bestl1;
#endif }
return best_rd; return best_rd;
} }
@@ -3395,7 +3429,7 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
int other_cost = 0; int other_cost = 0;
int compmode_cost = 0; int compmode_cost = 0;
int mode_excluded = 0; int mode_excluded = 0;
int64_t txfm_cache[NB_TXFM_MODES]; int64_t txfm_cache[NB_TXFM_MODES] = { 0 };
// These variables hold are rolling total cost and distortion for this mode // These variables hold are rolling total cost and distortion for this mode
rate2 = 0; rate2 = 0;
@@ -3579,12 +3613,51 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
} }
break; break;
case I8X8_PRED: { case I8X8_PRED: {
int64_t tmp_rd; #if CONFIG_TX_SELECT
mbmi->txfm_size = TX_8X8; // FIXME wrong in case of hybridtransform8x8 int cost0 = vp8_cost_bit(cm->prob_tx[0], 0);
tmp_rd = rd_pick_intra8x8mby_modes(cpi, x, &rate, &rate_y, int cost1 = vp8_cost_bit(cm->prob_tx[0], 1);
&distortion, best_yrd); int64_t tmp_rd_4x4s, tmp_rd_8x8s;
rate2 += rate; #endif
distortion2 += distortion; int64_t tmp_rd_4x4, tmp_rd_8x8, tmp_rd;
int r4x4, tok4x4, d4x4, r8x8, tok8x8, d8x8;
mbmi->txfm_size = TX_4X4;
tmp_rd_4x4 = rd_pick_intra8x8mby_modes(cpi, x, &r4x4, &tok4x4,
&d4x4, best_yrd);
mode8x8[0][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.first;
mode8x8[0][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.first;
mode8x8[0][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.first;
mode8x8[0][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.first;
#if CONFIG_COMP_INTRA_PRED
mode8x8[1][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.second;
mode8x8[1][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.second;
mode8x8[1][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.second;
mode8x8[1][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.second;
#endif
mbmi->txfm_size = TX_8X8;
tmp_rd_8x8 = rd_pick_intra8x8mby_modes(cpi, x, &r8x8, &tok8x8,
&d8x8, best_yrd);
txfm_cache[ONLY_4X4] = tmp_rd_4x4;
txfm_cache[ALLOW_8X8] = tmp_rd_8x8;
#if CONFIG_TX16X16
txfm_cache[ALLOW_16X16] = tmp_rd_8x8;
#endif
#if CONFIG_TX_SELECT
tmp_rd_4x4s = tmp_rd_4x4 + RDCOST(x->rdmult, x->rddiv, cost0, 0);
tmp_rd_8x8s = tmp_rd_8x8 + RDCOST(x->rdmult, x->rddiv, cost1, 0);
txfm_cache[TX_MODE_SELECT] = tmp_rd_4x4s < tmp_rd_8x8s ? tmp_rd_4x4s : tmp_rd_8x8s;
if (cm->txfm_mode == TX_MODE_SELECT) {
if (tmp_rd_4x4s < tmp_rd_8x8s) {
rate = r4x4 + cost0;
rate_y = tok4x4 + cost0;
distortion = d4x4;
mbmi->txfm_size = TX_4X4;
tmp_rd = tmp_rd_4x4s;
} else {
rate = r8x8 + cost1;
rate_y = tok8x8 + cost1;
distortion = d8x8;
mbmi->txfm_size = TX_8X8;
tmp_rd = tmp_rd_8x8s;
mode8x8[0][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.first; mode8x8[0][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.first;
mode8x8[0][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.first; mode8x8[0][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.first;
@@ -3596,6 +3669,36 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
mode8x8[1][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.second; mode8x8[1][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.second;
mode8x8[1][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.second; mode8x8[1][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.second;
#endif #endif
}
} else
#endif
if (cm->txfm_mode == ONLY_4X4) {
rate = r4x4;
rate_y = tok4x4;
distortion = d4x4;
mbmi->txfm_size = TX_4X4;
tmp_rd = tmp_rd_4x4;
} else {
rate = r8x8;
rate_y = tok8x8;
distortion = d8x8;
mbmi->txfm_size = TX_8X8;
tmp_rd = tmp_rd_8x8;
mode8x8[0][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.first;
mode8x8[0][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.first;
mode8x8[0][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.first;
mode8x8[0][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.first;
#if CONFIG_COMP_INTRA_PRED
mode8x8[1][0] = x->e_mbd.mode_info_context->bmi[0].as_mode.second;
mode8x8[1][1] = x->e_mbd.mode_info_context->bmi[2].as_mode.second;
mode8x8[1][2] = x->e_mbd.mode_info_context->bmi[8].as_mode.second;
mode8x8[1][3] = x->e_mbd.mode_info_context->bmi[10].as_mode.second;
#endif
}
rate2 += rate;
distortion2 += distortion;
/* TODO: uv rate maybe over-estimated here since there is UV intra /* TODO: uv rate maybe over-estimated here since there is UV intra
mode coded in I8X8_PRED prediction */ mode coded in I8X8_PRED prediction */
@@ -4026,8 +4129,7 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
if (!mode_excluded && this_rd != INT64_MAX) { if (!mode_excluded && this_rd != INT64_MAX) {
for (i = 0; i < NB_TXFM_MODES; i++) { for (i = 0; i < NB_TXFM_MODES; i++) {
int64_t adj_rd; int64_t adj_rd;
if (this_mode != B_PRED && this_mode != I8X8_PRED && if (this_mode != B_PRED && this_mode != SPLITMV) {
this_mode != SPLITMV) {
adj_rd = this_rd + txfm_cache[i] - txfm_cache[cm->txfm_mode]; adj_rd = this_rd + txfm_cache[i] - txfm_cache[cm->txfm_mode];
} else { } else {
adj_rd = this_rd; adj_rd = this_rd;
@@ -4264,6 +4366,8 @@ void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x,
mbmi->mode_rdopt = I8X8_PRED; mbmi->mode_rdopt = I8X8_PRED;
#endif #endif
// FIXME(rbultje) support transform-size selection
mbmi->txfm_size = (cm->txfm_mode == ONLY_4X4) ? TX_4X4 : TX_8X8;
error8x8 = rd_pick_intra8x8mby_modes(cpi, x, &rate8x8, &rate8x8_tokenonly, error8x8 = rd_pick_intra8x8mby_modes(cpi, x, &rate8x8, &rate8x8_tokenonly,
&dist8x8, error16x16); &dist8x8, error16x16);
mode8x8[0][0]= xd->mode_info_context->bmi[0].as_mode.first; mode8x8[0][0]= xd->mode_info_context->bmi[0].as_mode.first;
@@ -4365,8 +4469,9 @@ void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x,
sizeof(x->mb_context[xd->mb_index].txfm_rd_diff)); sizeof(x->mb_context[xd->mb_index].txfm_rd_diff));
#endif #endif
} else { } else {
// FIXME(rbultje) support transform-size selection
mbmi->mode = I8X8_PRED; mbmi->mode = I8X8_PRED;
mbmi->txfm_size = TX_8X8; mbmi->txfm_size = (cm->txfm_mode == ONLY_4X4) ? TX_4X4 : TX_8X8;
set_i8x8_block_modes(x, mode8x8); set_i8x8_block_modes(x, mode8x8);
rate = rate8x8 + rateuv; rate = rate8x8 + rateuv;
dist = dist8x8 + (distuv >> 2); dist = dist8x8 + (distuv >> 2);