Merge "Incorporate flexible tx type and tx partition in RD scheme" into nextgenv2

vp10/common/blockd.h

@@ -223,6 +223,7 @@ typedef struct macroblockd {
 
   // dimension in the unit of 8x8 block of the current block
   int16_t n8_w, n8_h;
+  TX_SIZE max_tx_size;
 #endif
 
 #if CONFIG_VP9_HIGHBITDEPTH

vp10/decoder/decodeframe.c

@@ -869,6 +869,11 @@ static MB_MODE_INFO *set_offsets(VP10_COMMON *const cm, MACROBLOCKD *const xd,
 
   set_skip_context(xd, mi_row, mi_col);
 
+
+#if CONFIG_VAR_TX
+  xd->max_tx_size = max_txsize_lookup[bsize];
+#endif
+
   // Distance of Mb to the various image edges. These are specified to 8th pel
   // as they are always compared to values that are in 1/8th pel units
   set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

vp10/encoder/encodeframe.c

@@ -199,6 +199,7 @@ static void set_offsets(VP10_COMP *cpi, const TileInfo *const tile,
 #if CONFIG_VAR_TX
   xd->above_txfm_context = cm->above_txfm_context + mi_col;
   xd->left_txfm_context = xd->left_txfm_context_buffer + (mi_row & 0x07);
+  xd->max_tx_size = max_txsize_lookup[bsize];
 #endif
 
   mbmi = &xd->mi[0]->mbmi;

vp10/encoder/rdopt.c

@@ -1650,8 +1650,6 @@ static void tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
   *bsse += (int64_t)tmp_sse * 16;
 
   if (p->eobs[block] > 0) {
-    // TODO(jingning): integrate multiple transform type experiment
-    TX_TYPE tx_type = DCT_DCT;
     switch (tx_size) {
       case TX_32X32:
         vp10_inv_txfm_add_32x32(dqcoeff, rec_buffer, 32, p->eobs[block],
@@ -1878,10 +1876,6 @@ static void inter_block_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
   *sse = 0;
   *skippable = 1;
 
-#if CONFIG_EXT_TX
-  xd->mi[0]->mbmi.tx_type = DCT_DCT;
-#endif
-
   if (is_cost_valid) {
     const struct macroblockd_plane *const pd = &xd->plane[0];
     const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
@@ -1936,6 +1930,99 @@ static void inter_block_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
   }
 }
 
+#if CONFIG_EXT_TX
+static void select_tx_type_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
+                               int *rate, int64_t *distortion, int *skippable,
+                               int64_t *sse, BLOCK_SIZE bsize,
+                               int64_t ref_best_rd) {
+  const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
+  const VP10_COMMON *const cm = &cpi->common;
+  MACROBLOCKD *const xd = &x->e_mbd;
+  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
+  int64_t rd = INT64_MAX;
+  int64_t best_rd = INT64_MAX;
+  TX_TYPE tx_type, best_tx_type = DCT_DCT;
+  int ext_tx_set;
+  const int is_inter = is_inter_block(mbmi);
+  vpx_prob skip_prob = vp10_get_skip_prob(cm, xd);
+  int s0 = vp10_cost_bit(skip_prob, 0);
+  int s1 = vp10_cost_bit(skip_prob, 1);
+
+  *distortion = INT64_MAX;
+  *rate       = INT_MAX;
+  *skippable  = 0;
+  *sse        = INT64_MAX;
+
+  for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
+    int this_rate = 0;
+    int this_skip = 1;
+    int64_t this_dist = 0;
+    int64_t this_sse  = 0;
+
+    ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter);
+    if (is_inter) {
+      if (!ext_tx_used_inter[ext_tx_set][tx_type])
+        continue;
+    } else {
+      if (!ext_tx_used_intra[ext_tx_set][tx_type])
+        continue;
+    }
+
+    mbmi->tx_type = tx_type;
+
+    if (ext_tx_set == 1 &&
+        mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
+        best_tx_type == DCT_DCT) {
+      tx_type = IDTX - 1;
+      break;
+    }
+
+    inter_block_yrd(cpi, x, &this_rate, &this_dist, &this_skip, &this_sse,
+                    bsize, ref_best_rd);
+
+    if (get_ext_tx_types(max_tx_size, bsize, is_inter) > 1 &&
+        !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+        this_rate != INT_MAX) {
+      if (is_inter) {
+        if (ext_tx_set > 0)
+          this_rate += cpi->inter_tx_type_costs[ext_tx_set]
+                                       [mbmi->tx_size][mbmi->tx_type];
+      } else {
+        if (ext_tx_set > 0)
+          this_rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
+                                       [mbmi->mode][mbmi->tx_type];
+      }
+    }
+
+    if (this_rate == INT_MAX)
+      continue;
+
+    if (this_skip)
+      rd = RDCOST(x->rdmult, x->rddiv, s1, this_sse);
+    else
+      rd = RDCOST(x->rdmult, x->rddiv, this_rate + s0, this_dist);
+
+    if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && !this_skip)
+      rd = VPXMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, this_sse));
+
+    if (rd <
+        (is_inter && best_tx_type == DCT_DCT ? ext_tx_th : 1) *
+        best_rd) {
+      best_rd = rd;
+      *distortion = this_dist;
+      *rate       = this_rate;
+      *skippable  = this_skip;
+      *sse        = this_sse;
+      best_tx_type = mbmi->tx_type;
+    }
+  }
+
+  mbmi->tx_type = best_tx_type;
+  inter_block_yrd(cpi, x, rate, distortion, skippable, sse,
+                  bsize, ref_best_rd);
+}
+#endif
+
 static void tx_block_rd(const VP10_COMP *cpi, MACROBLOCK *x,
                         int blk_row, int blk_col, int plane, int block,
                         TX_SIZE tx_size, BLOCK_SIZE plane_bsize,
@@ -3704,8 +3791,13 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,
     vp10_subtract_plane(x, bsize, 0);
 #if CONFIG_VAR_TX
     if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) {
+#if CONFIG_EXT_TX
+      select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
+                         bsize, ref_best_rd);
+#else
       inter_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
                       bsize, ref_best_rd);
+#endif
     } else {
       super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
                       bsize, ref_best_rd);