generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added code to prevent I8X8_PRED mode for MBs using 8x8 transform

Browse Source 
This fixed a conflict introduced by the change of adding 8x8 intra
prediction modes. The 8x8 intra prediction mode code assumed the
use of 4x4 transform, and causes encoder crashes when the codec is
configured with --enable-t8x8.

Change-Id: I00cc94df63e9725377ffba9eb51be6b77fe3fcf9
This commit is contained in:
Yaowu Xu 2012-01-19 16:37:32 -08:00
parent be9af16e16
commit 5aab0c3fb7
1 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
vp8/encoder/rdopt.c
View File

@ -2206,7 +2206,9 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
#if CONFIG_T8X8
// No 4x4 modes if segment flagged as 8x8
else if ( ( get_seg_tx_type( xd, segment_id ) == TX_8X8 ) &&
( (this_mode == B_PRED) || (this_mode == SPLITMV) ) )
( (this_mode == B_PRED)
||(this_mode == I8X8_PRED)
|| (this_mode == SPLITMV) ) )
{
continue;
}
@ -3015,13 +3017,24 @@ void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_)
&rate16x16, &rate16x16_tokenonly,
&dist16x16);
mode16x16 = x->e_mbd.mode_info_context->mbmi.mode;
error8x8 = rd_pick_intra8x8mby_modes(cpi, x,
&rate8x8, &rate8x8_tokenonly,
&dist8x8, error16x16);
mode8x8[0]= x->e_mbd.mode_info_context->bmi[0].as_mode;
mode8x8[1]= x->e_mbd.mode_info_context->bmi[2].as_mode;
mode8x8[2]= x->e_mbd.mode_info_context->bmi[8].as_mode;
mode8x8[3]= x->e_mbd.mode_info_context->bmi[10].as_mode;
#if CONFIG_T8X8
if ( get_seg_tx_type( xd,
xd->mode_info_context->mbmi.segment_id ) == TX_8X8)
{
error8x8 = INT_MAX;
}
else
#else
{
error8x8 = rd_pick_intra8x8mby_modes(cpi, x,
&rate8x8, &rate8x8_tokenonly,
&dist8x8, error16x16);
mode8x8[0]= x->e_mbd.mode_info_context->bmi[0].as_mode;
mode8x8[1]= x->e_mbd.mode_info_context->bmi[2].as_mode;
mode8x8[2]= x->e_mbd.mode_info_context->bmi[8].as_mode;
mode8x8[3]= x->e_mbd.mode_info_context->bmi[10].as_mode;
}
#endif
#if CONFIG_T8X8
if ( get_seg_tx_type( xd,