Separating 8x8DCT use from hybridtransform8x8

With this change, even if hybridtransform8x8 experiment is off,
8x8 dct is used for the I8x8 mode. However note that the gains
observed with the hybridtransform8x8 experiment will now be less,
since part of the gain is now merged in.

Change-Id: I9afb3880906fd0a1368a374041fc08efcf060c54

vp8/common/blockd.h

@@ -89,10 +89,6 @@ typedef enum
 #endif
 } INTERPOLATIONFILTERTYPE;
 
-#if 0//CONFIG_SWITCHABLE_INTERP
-#define VP8_SWITCHABLE_FILTERS 2 /* number of switchable filters */
-#endif
-
 typedef enum
 {
   DC_PRED,            /* average of above and left pixels */
@@ -479,6 +475,40 @@ static void txfm_map(BLOCKD *b, B_PREDICTION_MODE bmode) {
       break;
   }
 }
+
+static TX_TYPE get_tx_type(MACROBLOCKD *xd, const BLOCKD *b) {
+  TX_TYPE tx_type = DCT_DCT;
+  int ib = (b - xd->block);
+  if (ib >= 16) return tx_type;
+#if CONFIG_HYBRIDTRANSFORM16X16
+  if (xd->mode_info_context->mbmi.txfm_size == TX_16X16) {
+    if (xd->mode_info_context->mbmi.mode < I8X8_PRED &&
+        xd->q_index < ACTIVE_HT16)
+      tx_type = b->bmi.as_mode.tx_type;
+    return tx_type;
+  }
+#endif
+#if CONFIG_HYBRIDTRANSFORM8X8
+  if (xd->mode_info_context->mbmi.txfm_size  == TX_8X8) {
+    BLOCKD *bb;
+    ib = (ib & 8) + ((ib & 4) >> 1);
+    bb = xd->block + ib;
+    if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
+      tx_type = bb->bmi.as_mode.tx_type;
+    return tx_type;
+  }
+#endif
+#if CONFIG_HYBRIDTRANSFORM
+  if (xd->mode_info_context->mbmi.txfm_size  == TX_4X4) {
+    if (xd->mode_info_context->mbmi.mode == B_PRED &&
+        xd->q_index < ACTIVE_HT) {
+      tx_type = b->bmi.as_mode.tx_type;
+    }
+    return tx_type;
+  }
+#endif
+  return tx_type;
+}
 #endif
 
 extern void vp8_build_block_doffsets(MACROBLOCKD *xd);

vp8/common/entropy.h

@@ -62,11 +62,7 @@ extern vp8_extra_bit_struct vp8_extra_bits[12];    /* indexed by token value */
 /* Outside dimension.  0 = Y no DC, 1 = Y2, 2 = UV, 3 = Y with DC */
 #define BLOCK_TYPES 4
 
-#if CONFIG_HYBRIDTRANSFORM8X8
 #define BLOCK_TYPES_8X8 4
-#else
-#define BLOCK_TYPES_8X8 3
-#endif
 
 #define BLOCK_TYPES_16X16 4
 

vp8/decoder/decodframe.c

@@ -243,9 +243,6 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
     else
 #endif
       if (pbi->common.txfm_mode == ALLOW_8X8 &&
-#if !CONFIG_HYBRIDTRANSFORM8X8
-          xd->mode_info_context->mbmi.mode != I8X8_PRED &&
-#endif
         xd->mode_info_context->mbmi.mode != B_PRED)
       xd->mode_info_context->mbmi.txfm_size = TX_8X8;
     else
@@ -261,9 +258,6 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
     else
 #endif
       if (pbi->common.txfm_mode == ALLOW_8X8 &&
-#if !CONFIG_HYBRIDTRANSFORM8X8
-        xd->mode_info_context->mbmi.mode != I8X8_PRED &&
-#endif
         xd->mode_info_context->mbmi.mode != B_PRED &&
         xd->mode_info_context->mbmi.mode != SPLITMV)
       xd->mode_info_context->mbmi.txfm_size = TX_8X8;
@@ -622,11 +616,8 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
 #if CONFIG_SUPERBLOCKS
     if (!xd->mode_info_context->mbmi.encoded_as_sb) {
 #endif
-      if ((tx_type == TX_8X8
+      if ((tx_type == TX_8X8 &&
-#if CONFIG_HYBRIDTRANSFORM8X8
+           xd->mode_info_context->mbmi.mode != I8X8_PRED)
-           && xd->mode_info_context->mbmi.mode != I8X8_PRED
-#endif
-           )
 #if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
           || tx_type == TX_16X16
 #endif

vp8/encoder/encodeframe.c

@@ -1833,9 +1833,6 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
   else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
-#if !CONFIG_HYBRIDTRANSFORM8X8
-      && mbmi->mode != I8X8_PRED
-#endif
       && mbmi->mode != B_PRED) {
     mbmi->txfm_size = TX_8X8;
     cpi->t8x8_count++;
@@ -1852,8 +1849,9 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
   else
     vp8_encode_intra16x16mby(IF_RTCD(&cpi->rtcd), x);
 
-  if (mbmi->mode != I8X8_PRED)
+  if (mbmi->mode != I8X8_PRED) {
     vp8_encode_intra16x16mbuv(IF_RTCD(&cpi->rtcd), x);
+  }
 
   if (output_enabled) {
     // Tokenize
@@ -1932,9 +1930,6 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x,
   } else
 #endif
   if (cpi->common.txfm_mode == ALLOW_8X8
-#if !CONFIG_HYBRIDTRANSFORM8X8
-      && mbmi->mode != I8X8_PRED
-#endif
       && mbmi->mode != B_PRED
       && mbmi->mode != SPLITMV) {
     mbmi->txfm_size = TX_8X8;
@@ -2115,9 +2110,6 @@ void vp8cx_encode_inter_superblock(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t,
 
   /* test code: set transform size based on mode selection */
   if (cpi->common.txfm_mode == ALLOW_8X8
-#if !CONFIG_HYBRIDTRANSFORM8X8
-      && x->e_mbd.mode_info_context->mbmi.mode != I8X8_PRED
-#endif
       && x->e_mbd.mode_info_context->mbmi.mode != B_PRED
       && x->e_mbd.mode_info_context->mbmi.mode != SPLITMV) {
     x->e_mbd.mode_info_context->mbmi.txfm_size = TX_8X8;

vp8/encoder/encodeintra.c

@@ -22,10 +22,6 @@
 #include "encodeintra.h"
 
 
-#ifdef ENC_DEBUG
-extern int enc_debug;
-#endif
-
 #if CONFIG_RUNTIME_CPU_DETECT
 #define IF_RTCD(x) (x)
 #else
@@ -208,37 +204,6 @@ void vp8_encode_intra16x16mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
   else
     vp8_inverse_transform_mby(IF_RTCD(&rtcd->common->idct), &x->e_mbd);
 
-#ifdef ENC_DEBUG
-  if (enc_debug) {
-    int i;
-    printf("Intra qcoeff:\n");
-    printf("%d %d:\n", x->e_mbd.mb_to_left_edge, x->e_mbd.mb_to_top_edge);
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.qcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("Intra dqcoeff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.dqcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("Intra diff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.diff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("Intra predictor:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.predictor[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("eobs:\n");
-    for (i = 0; i < 25; i++)
-      printf("%d ", x->e_mbd.block[i].eob);
-    printf("\n");
-  }
-#endif
-
   RECON_INVOKE(&rtcd->common->recon, recon_mby)
       (IF_RTCD(&rtcd->common->recon), &x->e_mbd);
 
@@ -270,37 +235,6 @@ void vp8_encode_intra16x16mbuv(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
   else
     vp8_quantize_mbuv(x);
 
-#ifdef ENC_DEBUG
-  if (enc_debug) {
-    int i;
-    printf("vp8_encode_intra16x16mbuv\n");
-    printf("%d %d:\n", x->e_mbd.mb_to_left_edge, x->e_mbd.mb_to_top_edge);
-    printf("qcoeff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.qcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("dqcoeff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.dqcoeff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("diff:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.diff[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("predictor:\n");
-    for (i = 0; i < 400; i++) {
-      printf("%3d ", x->e_mbd.predictor[i]);
-      if (i % 16 == 15) printf("\n");
-    }
-    printf("eobs:\n");
-    for (i = 0; i < 25; i++)
-      printf("%d ", x->e_mbd.block[i].eob);
-    printf("\n");
-  }
-#endif
   if (x->optimize) {
     if (tx_type == TX_8X8)
       vp8_optimize_mbuv_8x8(x, rtcd);
@@ -335,7 +269,6 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
   }
 #endif
 
-#if CONFIG_HYBRIDTRANSFORM8X8
   {
     MACROBLOCKD *xd = &x->e_mbd;
     int idx = (ib & 0x02) ? (ib + 2) : ib;
@@ -343,12 +276,18 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
     // generate residual blocks
     vp8_subtract_4b_c(be, b, 16);
 
+#if CONFIG_HYBRIDTRANSFORM8X8
     txfm_map(b, pred_mode_conv(b->bmi.as_mode.first));
     vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
               b->bmi.as_mode.tx_type, 8);
     x->quantize_b_8x8(x->block + idx, xd->block + idx);
     vp8_ihtllm_c(xd->block[idx].dqcoeff, xd->block[ib].diff, 32,
                  b->bmi.as_mode.tx_type, 8);
+#else
+    x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
+    x->quantize_b_8x8(x->block + idx, xd->block + idx);
+    vp8_idct_idct8(xd->block[idx].dqcoeff, xd->block[ib].diff, 32);
+#endif
 
     // reconstruct submacroblock
     for (i = 0; i < 4; i++) {
@@ -357,19 +296,6 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
                     b->dst_stride);
     }
   }
-#else
-  for (i = 0; i < 4; i++) {
-    b = &x->e_mbd.block[ib + iblock[i]];
-    be = &x->block[ib + iblock[i]];
-    ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 16);
-    x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
-    x->quantize_b(be, b);
-    vp8_inverse_transform_b(IF_RTCD(&rtcd->common->idct), b, 32);
-    RECON_INVOKE(&rtcd->common->recon, recon)(b->predictor,
-                                              b->diff, *(b->base_dst) + b->dst,
-                                              b->dst_stride);
-  }
-#endif
 }
 
 extern const int vp8_i8x8_block[4];
@@ -380,7 +306,6 @@ void vp8_encode_intra8x8mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
     ib = vp8_i8x8_block[i];
     vp8_encode_intra8x8(rtcd, x, ib);
   }
-
 }
 
 void vp8_encode_intra_uv4x4(const VP8_ENCODER_RTCD *rtcd,

vp8/encoder/rdopt.c

@@ -1332,11 +1332,9 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
   DECLARE_ALIGNED_ARRAY(16, unsigned char,  best_predictor, 16 * 8);
   DECLARE_ALIGNED_ARRAY(16, short, best_dqcoeff, 16 * 4);
 
-#if CONFIG_HYBRIDTRANSFORM8X8
   // perform transformation of dimension 8x8
   // note the input and output index mapping
   int idx = (ib & 0x02) ? (ib + 2) : ib;
-#endif
 
   for (mode = DC_PRED; mode <= TM_PRED; mode++) {
 #if CONFIG_COMP_INTRA_PRED
@@ -1367,52 +1365,25 @@ static int64_t rd_pick_intra8x8block(VP8_COMP *cpi, MACROBLOCK *x, int ib,
       txfm_map(b, pred_mode_conv(mode));
       vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
                 b->bmi.as_mode.tx_type, 8);
+
+#else
+      x->vp8_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
+#endif
+
       x->quantize_b_8x8(x->block + idx, xd->block + idx);
 
       // compute quantization mse of 8x8 block
       distortion = vp8_block_error_c((x->block + idx)->coeff,
                                      (xd->block + idx)->dqcoeff, 64)>>2;
-
       ta0 = *(a + vp8_block2above_8x8[idx]);
       tl0 = *(l + vp8_block2left_8x8 [idx]);
 
       rate_t = cost_coeffs(x, xd->block + idx, PLANE_TYPE_Y_WITH_DC,
                            &ta0, &tl0, TX_8X8);
+
       rate += rate_t;
       ta1 = ta0;
       tl1 = tl0;
-#else
-      x->vp8_short_fdct8x4(be->src_diff, be->coeff, 32);
-      x->vp8_short_fdct8x4(be->src_diff + 64, be->coeff + 64, 32);
-
-      x->quantize_b_pair(x->block + ib, x->block + ib + 1,
-                         xd->block + ib, xd->block + ib + 1);
-      x->quantize_b_pair(x->block + ib + 4, x->block + ib + 5,
-                         xd->block + ib + 4, xd->block + ib + 5);
-
-      distortion = ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                   ((x->block + ib)->coeff, (xd->block + ib)->dqcoeff, 16) >> 2;
-      distortion += ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                    ((x->block + ib + 1)->coeff, (xd->block + ib + 1)->dqcoeff, 16) >> 2;
-      distortion += ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                    ((x->block + ib + 4)->coeff, (xd->block + ib + 4)->dqcoeff, 16) >> 2;
-      distortion += ENCODEMB_INVOKE(IF_RTCD(&cpi->rtcd.encodemb), berr)
-                    ((x->block + ib + 5)->coeff, (xd->block + ib + 5)->dqcoeff, 16) >> 2;
-
-      ta0 = *(a + vp8_block2above[ib]);
-      ta1 = *(a + vp8_block2above[ib + 1]);
-      tl0 = *(l + vp8_block2above[ib]);
-      tl1 = *(l + vp8_block2above[ib + 4]);
-      rate_t = cost_coeffs(x, xd->block + ib, PLANE_TYPE_Y_WITH_DC,
-                           &ta0, &tl0, TX_4X4);
-      rate_t += cost_coeffs(x, xd->block + ib + 1, PLANE_TYPE_Y_WITH_DC,
-                            &ta1, &tl0, TX_4X4);
-      rate_t += cost_coeffs(x, xd->block + ib + 4, PLANE_TYPE_Y_WITH_DC,
-                            &ta0, &tl1, TX_4X4);
-      rate_t += cost_coeffs(x, xd->block + ib + 5, PLANE_TYPE_Y_WITH_DC,
-                            &ta1, &tl1, TX_4X4);
-      rate += rate_t;
-#endif
 
       this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
       if (this_rd < best_rd) {

vp8/encoder/tokenize.c

@@ -55,12 +55,6 @@ const TOKENVALUE *vp8_dct_value_tokens_ptr;
 static int dct_value_cost[DCT_MAX_VALUE * 2];
 const int *vp8_dct_value_cost_ptr;
 
-#ifdef ENC_DEBUG
-extern int mb_row_debug;
-extern int mb_col_debug;
-extern int enc_debug;
-#endif
-
 static void fill_value_tokens() {
 
   TOKENVALUE *const t = dct_value_tokens + DCT_MAX_VALUE;
@@ -218,12 +212,6 @@ static void tokenize2nd_order_b_8x8
 
     t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0));
 
-#ifdef ENC_DEBUG
-    if (t->skip_eob_node && vp8_coef_encodings[x].Len == 1)
-      printf("Trouble 2 x=%d Len=%d skip=%d eob=%d c=%d band=%d type=%d: [%d %d %d]\n",
-             x, vp8_coef_encodings[x].Len, t->skip_eob_node, eob, c, band, type,
-             cpi->count, mb_row_debug, mb_col_debug);
-#endif
     if (!dry_run)
       ++cpi->coef_counts_8x8       [type] [band] [pt] [x];
   } while (pt = vp8_prev_token_class[x], ++t, c < eob  &&  ++c < seg_eob);