Make RECT_TX(>=8x8) work with VAR_TX
Bitstream syntax: For a rectangular inter block, 'rect_tx' flag is sent to indicate if the biggest rect tx is used. If no, continue to decode regular recursive tx partition. Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0
This commit is contained in:
		@@ -857,6 +857,12 @@ static const aom_prob
 | 
			
		||||
      },
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
 | 
			
		||||
// the probability of (0) using recursive square tx partition vs.
 | 
			
		||||
// (1) biggest rect tx for 4X8-8X4/8X16-16X8/16X32-32X16 blocks
 | 
			
		||||
static const aom_prob default_rect_tx_prob[TX_SIZES - 1] = { 192, 192, 192 };
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
 | 
			
		||||
 | 
			
		||||
int av1_get_palette_color_context(const uint8_t *color_map, int cols, int r,
 | 
			
		||||
                                  int c, int n, int *color_order) {
 | 
			
		||||
  int i, j, max, max_idx, temp;
 | 
			
		||||
@@ -1298,6 +1304,9 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
 | 
			
		||||
  av1_copy(fc->tx_size_probs, default_tx_size_prob);
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
  av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  av1_copy(fc->rect_tx_prob, default_rect_tx_prob);
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
#endif
 | 
			
		||||
  av1_copy(fc->skip_probs, default_skip_probs);
 | 
			
		||||
#if CONFIG_REF_MV
 | 
			
		||||
@@ -1451,6 +1460,15 @@ void av1_adapt_inter_frame_probs(AV1_COMMON *cm) {
 | 
			
		||||
  }
 | 
			
		||||
#endif  // CONFIG_EXT_INTER
 | 
			
		||||
 | 
			
		||||
#if CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  if (cm->tx_mode == TX_MODE_SELECT) {
 | 
			
		||||
    for (i = 0; i < TX_SIZES - 1; ++i) {
 | 
			
		||||
      fc->rect_tx_prob[i] =
 | 
			
		||||
          av1_mode_mv_merge_probs(pre_fc->rect_tx_prob[i], counts->rect_tx[i]);
 | 
			
		||||
    }
 | 
			
		||||
  }
 | 
			
		||||
#endif  // CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
 | 
			
		||||
  for (i = 0; i < BLOCK_SIZE_GROUPS; i++)
 | 
			
		||||
    aom_tree_merge_probs(av1_intra_mode_tree, pre_fc->y_mode_prob[i],
 | 
			
		||||
                         counts->y_mode[i], fc->y_mode_prob[i]);
 | 
			
		||||
@@ -1478,10 +1496,11 @@ void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
  if (cm->tx_mode == TX_MODE_SELECT)
 | 
			
		||||
  if (cm->tx_mode == TX_MODE_SELECT) {
 | 
			
		||||
    for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
 | 
			
		||||
      fc->txfm_partition_prob[i] = av1_mode_mv_merge_probs(
 | 
			
		||||
          pre_fc->txfm_partition_prob[i], counts->txfm_partition[i]);
 | 
			
		||||
  }
 | 
			
		||||
#endif
 | 
			
		||||
 | 
			
		||||
  for (i = 0; i < SKIP_CONTEXTS; ++i)
 | 
			
		||||
 
 | 
			
		||||
@@ -96,6 +96,10 @@ typedef struct frame_contexts {
 | 
			
		||||
  aom_prob tx_size_probs[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES - 1];
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
  aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  // TODO(yuec) make this flag harmonize with the original syntax
 | 
			
		||||
  aom_prob rect_tx_prob[TX_SIZES - 1];
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
#endif
 | 
			
		||||
  aom_prob skip_probs[SKIP_CONTEXTS];
 | 
			
		||||
#if CONFIG_REF_MV
 | 
			
		||||
@@ -179,6 +183,9 @@ typedef struct FRAME_COUNTS {
 | 
			
		||||
  unsigned int tx_size[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
  unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  unsigned int rect_tx[TX_SIZES - 1][2];
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
#endif
 | 
			
		||||
  unsigned int skip[SKIP_CONTEXTS][2];
 | 
			
		||||
#if CONFIG_REF_MV
 | 
			
		||||
 
 | 
			
		||||
@@ -1246,12 +1246,18 @@ void av1_filter_block_plane_non420(AV1_COMMON *cm,
 | 
			
		||||
        tx_size_mask = 0;
 | 
			
		||||
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
      if (is_inter_block(mbmi) && !mbmi->skip)
 | 
			
		||||
        tx_size =
 | 
			
		||||
            (plane->plane_type == PLANE_TYPE_UV)
 | 
			
		||||
                ? uv_txsize_lookup[sb_type][mbmi->inter_tx_size
 | 
			
		||||
                                                [blk_row][blk_col]][ss_x][ss_y]
 | 
			
		||||
                : mbmi->inter_tx_size[blk_row][blk_col];
 | 
			
		||||
      if (is_inter_block(mbmi) && !mbmi->skip) {
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
        TX_SIZE mb_tx_size = is_rect_tx(mbmi->tx_size)
 | 
			
		||||
                                 ? mbmi->tx_size
 | 
			
		||||
                                 : mbmi->inter_tx_size[blk_row][blk_col];
 | 
			
		||||
#else
 | 
			
		||||
        TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col];
 | 
			
		||||
#endif
 | 
			
		||||
        tx_size = (plane->plane_type == PLANE_TYPE_UV)
 | 
			
		||||
                      ? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y]
 | 
			
		||||
                      : mb_tx_size;
 | 
			
		||||
      }
 | 
			
		||||
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
      tx_size_r =
 | 
			
		||||
 
 | 
			
		||||
@@ -1308,10 +1308,9 @@ static void decode_block(AV1Decoder *const pbi, MACROBLOCKD *const xd,
 | 
			
		||||
        const int step = num_4x4_blocks_txsize_lookup[max_tx_size];
 | 
			
		||||
        int block = 0;
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
        const TX_SIZE tx_size =
 | 
			
		||||
            plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
 | 
			
		||||
 | 
			
		||||
        if (tx_size >= TX_SIZES) {  // rect txsize is used
 | 
			
		||||
        if (is_rect_tx(mbmi->tx_size)) {
 | 
			
		||||
          const TX_SIZE tx_size =
 | 
			
		||||
              plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
 | 
			
		||||
          const int stepr = num_4x4_blocks_high_txsize_lookup[tx_size];
 | 
			
		||||
          const int stepc = num_4x4_blocks_wide_txsize_lookup[tx_size];
 | 
			
		||||
          const int max_blocks_wide =
 | 
			
		||||
@@ -3491,6 +3490,12 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
 | 
			
		||||
    av1_diff_update_prob(&r, &fc->txfm_partition_prob[k]);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  if (cm->tx_mode == TX_MODE_SELECT) {
 | 
			
		||||
    for (i = 1; i < TX_SIZES - 1; ++i)
 | 
			
		||||
      av1_diff_update_prob(&r, &fc->rect_tx_prob[i]);
 | 
			
		||||
  }
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
#endif
 | 
			
		||||
 | 
			
		||||
  for (k = 0; k < SKIP_CONTEXTS; ++k)
 | 
			
		||||
 
 | 
			
		||||
@@ -1632,13 +1632,34 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
 | 
			
		||||
      const int width = num_4x4_blocks_wide_lookup[bsize];
 | 
			
		||||
      const int height = num_4x4_blocks_high_lookup[bsize];
 | 
			
		||||
      int idx, idy;
 | 
			
		||||
      for (idy = 0; idy < height; idy += bs)
 | 
			
		||||
        for (idx = 0; idx < width; idx += bs)
 | 
			
		||||
          read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
 | 
			
		||||
                             r);
 | 
			
		||||
      int tx_size_cat = inter_tx_size_cat_lookup[bsize];
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
      int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
 | 
			
		||||
      int use_rect_tx = 0;
 | 
			
		||||
 | 
			
		||||
      if (is_rect_tx_allowed) {
 | 
			
		||||
        use_rect_tx = aom_read(r, cm->fc->rect_tx_prob[tx_size_cat]);
 | 
			
		||||
        if (xd->counts) {
 | 
			
		||||
          ++xd->counts->rect_tx[tx_size_cat][use_rect_tx];
 | 
			
		||||
        }
 | 
			
		||||
      }
 | 
			
		||||
 | 
			
		||||
      if (use_rect_tx) {
 | 
			
		||||
        mbmi->tx_size = max_txsize_rect_lookup[bsize];
 | 
			
		||||
        set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
 | 
			
		||||
      } else {
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
        for (idy = 0; idy < height; idy += bs)
 | 
			
		||||
          for (idx = 0; idx < width; idx += bs)
 | 
			
		||||
            read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
 | 
			
		||||
                               r);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
      }
 | 
			
		||||
#endif
 | 
			
		||||
      if (xd->counts) {
 | 
			
		||||
        const int ctx = get_tx_size_context(xd);
 | 
			
		||||
        ++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][mbmi->tx_size];
 | 
			
		||||
        ++xd->counts
 | 
			
		||||
              ->tx_size[tx_size_cat][ctx][txsize_sqr_up_map[mbmi->tx_size]];
 | 
			
		||||
      }
 | 
			
		||||
    } else {
 | 
			
		||||
      if (inter_block)
 | 
			
		||||
 
 | 
			
		||||
@@ -1121,9 +1121,25 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
 | 
			
		||||
      const int width = num_4x4_blocks_wide_lookup[bsize];
 | 
			
		||||
      const int height = num_4x4_blocks_high_lookup[bsize];
 | 
			
		||||
      int idx, idy;
 | 
			
		||||
      for (idy = 0; idy < height; idy += bs)
 | 
			
		||||
        for (idx = 0; idx < width; idx += bs)
 | 
			
		||||
          write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
 | 
			
		||||
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
      if (is_rect_tx_allowed(mbmi)) {
 | 
			
		||||
        int tx_size_cat = inter_tx_size_cat_lookup[bsize];
 | 
			
		||||
 | 
			
		||||
        aom_write(w, is_rect_tx(mbmi->tx_size),
 | 
			
		||||
                  cm->fc->rect_tx_prob[tx_size_cat]);
 | 
			
		||||
      }
 | 
			
		||||
 | 
			
		||||
      if (is_rect_tx(mbmi->tx_size)) {
 | 
			
		||||
        set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
 | 
			
		||||
      } else {
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
        for (idy = 0; idy < height; idy += bs)
 | 
			
		||||
          for (idx = 0; idx < width; idx += bs)
 | 
			
		||||
            write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
      }
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
    } else {
 | 
			
		||||
      set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
 | 
			
		||||
      write_selected_tx_size(cm, xd, w);
 | 
			
		||||
@@ -1650,7 +1666,7 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile,
 | 
			
		||||
      TX_SIZE tx_size =
 | 
			
		||||
          plane ? get_uv_tx_size(mbmi, &xd->plane[plane]) : mbmi->tx_size;
 | 
			
		||||
 | 
			
		||||
      if (is_inter_block(mbmi) && tx_size < TX_SIZES) {
 | 
			
		||||
      if (is_inter_block(mbmi) && !is_rect_tx(tx_size)) {
 | 
			
		||||
#else
 | 
			
		||||
      if (is_inter_block(mbmi)) {
 | 
			
		||||
#endif
 | 
			
		||||
@@ -3271,6 +3287,13 @@ static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) {
 | 
			
		||||
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
  update_txfm_partition_probs(cm, header_bc, counts);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  if (cm->tx_mode == TX_MODE_SELECT) {
 | 
			
		||||
    for (i = 1; i < TX_SIZES - 1; ++i)
 | 
			
		||||
      av1_cond_prob_diff_update(header_bc, &fc->rect_tx_prob[i],
 | 
			
		||||
                                counts->rect_tx[i]);
 | 
			
		||||
  }
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
#endif
 | 
			
		||||
 | 
			
		||||
  update_skip_probs(cm, header_bc, counts);
 | 
			
		||||
 
 | 
			
		||||
@@ -5031,7 +5031,7 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
 | 
			
		||||
    av1_encode_sb(x, AOMMAX(bsize, BLOCK_8X8));
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
    if (mbmi->tx_size >= TX_SIZES)
 | 
			
		||||
    if (is_rect_tx(mbmi->tx_size))
 | 
			
		||||
      av1_tokenize_sb(cpi, td, t, !output_enabled, AOMMAX(bsize, BLOCK_8X8));
 | 
			
		||||
    else
 | 
			
		||||
#endif
 | 
			
		||||
@@ -5054,8 +5054,17 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
 | 
			
		||||
      assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
      if (is_inter)
 | 
			
		||||
        tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
      if (is_rect_tx_allowed(mbmi)) {
 | 
			
		||||
        td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
 | 
			
		||||
      }
 | 
			
		||||
      if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
        if (is_inter)
 | 
			
		||||
          tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
      }
 | 
			
		||||
#endif
 | 
			
		||||
#endif
 | 
			
		||||
      ++td->counts->tx_size[tx_size_cat][tx_size_ctx][coded_tx_size];
 | 
			
		||||
    } else {
 | 
			
		||||
@@ -5118,6 +5127,11 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
 | 
			
		||||
      is_inter_block(mbmi) && !(mbmi->skip || seg_skip)) {
 | 
			
		||||
    if (!output_enabled)
 | 
			
		||||
      tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
    if (is_rect_tx(mbmi->tx_size)) {
 | 
			
		||||
      set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
 | 
			
		||||
    }
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  } else {
 | 
			
		||||
    TX_SIZE tx_size;
 | 
			
		||||
    // The new intra coding scheme requires no change of transform size
 | 
			
		||||
 
 | 
			
		||||
@@ -791,7 +791,7 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
 | 
			
		||||
#endif
 | 
			
		||||
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
  // Assert not magic number (uninitialised).
 | 
			
		||||
  // Assert not magic number (uninitialized).
 | 
			
		||||
  assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);
 | 
			
		||||
 | 
			
		||||
  if (x->blk_skip[plane][(blk_row << bwl) + blk_col] == 0) {
 | 
			
		||||
@@ -976,9 +976,6 @@ void av1_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
 | 
			
		||||
    int idx, idy;
 | 
			
		||||
    int block = 0;
 | 
			
		||||
    int step = num_4x4_blocks_txsize_lookup[max_tx_size];
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
    const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
 | 
			
		||||
#endif
 | 
			
		||||
    av1_get_entropy_contexts(bsize, TX_4X4, pd, ctx.ta[plane], ctx.tl[plane]);
 | 
			
		||||
#else
 | 
			
		||||
    const struct macroblockd_plane *const pd = &xd->plane[plane];
 | 
			
		||||
@@ -991,7 +988,7 @@ void av1_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
 | 
			
		||||
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
    if (tx_size >= TX_SIZES) {
 | 
			
		||||
    if (is_rect_tx(mbmi->tx_size)) {
 | 
			
		||||
      av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block,
 | 
			
		||||
                                             &arg);
 | 
			
		||||
    } else {
 | 
			
		||||
 
 | 
			
		||||
@@ -2814,6 +2814,61 @@ static int64_t rd_pick_intra_sby_mode(AV1_COMP *cpi, MACROBLOCK *x, int *rate,
 | 
			
		||||
  return best_rd;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// Return value 0: early termination triggered, no valid rd cost available;
 | 
			
		||||
//              1: rd cost values are valid.
 | 
			
		||||
static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
 | 
			
		||||
                            int64_t *distortion, int *skippable, int64_t *sse,
 | 
			
		||||
                            BLOCK_SIZE bsize, int64_t ref_best_rd) {
 | 
			
		||||
  MACROBLOCKD *const xd = &x->e_mbd;
 | 
			
		||||
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
 | 
			
		||||
  const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
 | 
			
		||||
  int plane;
 | 
			
		||||
  int pnrate = 0, pnskip = 1;
 | 
			
		||||
  int64_t pndist = 0, pnsse = 0;
 | 
			
		||||
  int is_cost_valid = 1;
 | 
			
		||||
 | 
			
		||||
  if (ref_best_rd < 0) is_cost_valid = 0;
 | 
			
		||||
 | 
			
		||||
  if (is_inter_block(mbmi) && is_cost_valid) {
 | 
			
		||||
    int plane;
 | 
			
		||||
    for (plane = 1; plane < MAX_MB_PLANE; ++plane)
 | 
			
		||||
      av1_subtract_plane(x, bsize, plane);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  *rate = 0;
 | 
			
		||||
  *distortion = 0;
 | 
			
		||||
  *sse = 0;
 | 
			
		||||
  *skippable = 1;
 | 
			
		||||
 | 
			
		||||
  for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
 | 
			
		||||
    txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
 | 
			
		||||
                     plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
 | 
			
		||||
    if (pnrate == INT_MAX) {
 | 
			
		||||
      is_cost_valid = 0;
 | 
			
		||||
      break;
 | 
			
		||||
    }
 | 
			
		||||
    *rate += pnrate;
 | 
			
		||||
    *distortion += pndist;
 | 
			
		||||
    *sse += pnsse;
 | 
			
		||||
    *skippable &= pnskip;
 | 
			
		||||
    if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
 | 
			
		||||
        RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
 | 
			
		||||
      is_cost_valid = 0;
 | 
			
		||||
      break;
 | 
			
		||||
    }
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  if (!is_cost_valid) {
 | 
			
		||||
    // reset cost value
 | 
			
		||||
    *rate = INT_MAX;
 | 
			
		||||
    *distortion = INT64_MAX;
 | 
			
		||||
    *sse = INT64_MAX;
 | 
			
		||||
    *skippable = 0;
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  return is_cost_valid;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
#if CONFIG_VAR_TX
 | 
			
		||||
void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
 | 
			
		||||
                       int blk_row, int blk_col, int plane, int block,
 | 
			
		||||
@@ -3196,6 +3251,61 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
 | 
			
		||||
 | 
			
		||||
  mbmi->tx_type = tx_type;
 | 
			
		||||
  inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  if (is_rect_tx_allowed(mbmi)) {
 | 
			
		||||
    int rate_rect_tx, skippable_rect_tx = 0;
 | 
			
		||||
    int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
 | 
			
		||||
    int tx_size_cat = inter_tx_size_cat_lookup[bsize];
 | 
			
		||||
    TX_SIZE tx_size = max_txsize_rect_lookup[bsize];
 | 
			
		||||
    TX_SIZE var_tx_size = mbmi->tx_size;
 | 
			
		||||
 | 
			
		||||
    txfm_rd_in_plane(x, cpi, &rate_rect_tx, &dist_rect_tx, &skippable_rect_tx,
 | 
			
		||||
                     &sse_rect_tx, ref_best_rd, 0, bsize, tx_size,
 | 
			
		||||
                     cpi->sf.use_fast_coef_costing);
 | 
			
		||||
 | 
			
		||||
    if (*rate != INT_MAX) {
 | 
			
		||||
      *rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 0);
 | 
			
		||||
      if (*skippable) {
 | 
			
		||||
        rd = RDCOST(x->rdmult, x->rddiv, s1, *sse);
 | 
			
		||||
      } else {
 | 
			
		||||
        rd = RDCOST(x->rdmult, x->rddiv, *rate + s0, *dist);
 | 
			
		||||
        if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
 | 
			
		||||
            !(*skippable))
 | 
			
		||||
          rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, *sse));
 | 
			
		||||
      }
 | 
			
		||||
    } else {
 | 
			
		||||
      rd = INT64_MAX;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    if (rate_rect_tx != INT_MAX) {
 | 
			
		||||
      rate_rect_tx += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 1);
 | 
			
		||||
      if (skippable_rect_tx) {
 | 
			
		||||
        rd_rect_tx = RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx);
 | 
			
		||||
      } else {
 | 
			
		||||
        rd_rect_tx =
 | 
			
		||||
            RDCOST(x->rdmult, x->rddiv, rate_rect_tx + s0, dist_rect_tx);
 | 
			
		||||
        if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
 | 
			
		||||
            !(skippable_rect_tx))
 | 
			
		||||
          rd_rect_tx =
 | 
			
		||||
              AOMMIN(rd_rect_tx, RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx));
 | 
			
		||||
      }
 | 
			
		||||
    } else {
 | 
			
		||||
      rd_rect_tx = INT64_MAX;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    if (rd_rect_tx < rd) {
 | 
			
		||||
      *rate = rate_rect_tx;
 | 
			
		||||
      *dist = dist_rect_tx;
 | 
			
		||||
      *sse = sse_rect_tx;
 | 
			
		||||
      *skippable = skippable_rect_tx;
 | 
			
		||||
      if (!xd->lossless[mbmi->segment_id]) x->blk_skip[0][0] = *skippable;
 | 
			
		||||
      mbmi->tx_size = tx_size;
 | 
			
		||||
      mbmi->inter_tx_size[0][0] = mbmi->tx_size;
 | 
			
		||||
    } else {
 | 
			
		||||
      mbmi->tx_size = var_tx_size;
 | 
			
		||||
    }
 | 
			
		||||
  }
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
 | 
			
		||||
  if (*rate == INT_MAX) return INT64_MAX;
 | 
			
		||||
 | 
			
		||||
@@ -3409,17 +3519,24 @@ static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
 | 
			
		||||
 | 
			
		||||
  if (ref_best_rd < 0) is_cost_valid = 0;
 | 
			
		||||
 | 
			
		||||
  *rate = 0;
 | 
			
		||||
  *distortion = 0;
 | 
			
		||||
  *sse = 0;
 | 
			
		||||
  *skippable = 1;
 | 
			
		||||
 | 
			
		||||
#if CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
  if (is_rect_tx(mbmi->tx_size)) {
 | 
			
		||||
    return super_block_uvrd(cpi, x, rate, distortion, skippable, sse, bsize,
 | 
			
		||||
                            ref_best_rd);
 | 
			
		||||
  }
 | 
			
		||||
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 | 
			
		||||
 | 
			
		||||
  if (is_inter_block(mbmi) && is_cost_valid) {
 | 
			
		||||
    int plane;
 | 
			
		||||
    for (plane = 1; plane < MAX_MB_PLANE; ++plane)
 | 
			
		||||
      av1_subtract_plane(x, bsize, plane);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  *rate = 0;
 | 
			
		||||
  *distortion = 0;
 | 
			
		||||
  *sse = 0;
 | 
			
		||||
  *skippable = 1;
 | 
			
		||||
 | 
			
		||||
  for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
 | 
			
		||||
    const struct macroblockd_plane *const pd = &xd->plane[plane];
 | 
			
		||||
    const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
 | 
			
		||||
@@ -3477,61 +3594,6 @@ static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
 | 
			
		||||
}
 | 
			
		||||
#endif  // CONFIG_VAR_TX
 | 
			
		||||
 | 
			
		||||
// Return value 0: early termination triggered, no valid rd cost available;
 | 
			
		||||
//              1: rd cost values are valid.
 | 
			
		||||
static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
 | 
			
		||||
                            int64_t *distortion, int *skippable, int64_t *sse,
 | 
			
		||||
                            BLOCK_SIZE bsize, int64_t ref_best_rd) {
 | 
			
		||||
  MACROBLOCKD *const xd = &x->e_mbd;
 | 
			
		||||
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
 | 
			
		||||
  const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
 | 
			
		||||
  int plane;
 | 
			
		||||
  int pnrate = 0, pnskip = 1;
 | 
			
		||||
  int64_t pndist = 0, pnsse = 0;
 | 
			
		||||
  int is_cost_valid = 1;
 | 
			
		||||
 | 
			
		||||
  if (ref_best_rd < 0) is_cost_valid = 0;
 | 
			
		||||
 | 
			
		||||
  if (is_inter_block(mbmi) && is_cost_valid) {
 | 
			
		||||
    int plane;
 | 
			
		||||
    for (plane = 1; plane < MAX_MB_PLANE; ++plane)
 | 
			
		||||
      av1_subtract_plane(x, bsize, plane);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  *rate = 0;
 | 
			
		||||
  *distortion = 0;
 | 
			
		||||
  *sse = 0;
 | 
			
		||||
  *skippable = 1;
 | 
			
		||||
 | 
			
		||||
  for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
 | 
			
		||||
    txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
 | 
			
		||||
                     plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
 | 
			
		||||
    if (pnrate == INT_MAX) {
 | 
			
		||||
      is_cost_valid = 0;
 | 
			
		||||
      break;
 | 
			
		||||
    }
 | 
			
		||||
    *rate += pnrate;
 | 
			
		||||
    *distortion += pndist;
 | 
			
		||||
    *sse += pnsse;
 | 
			
		||||
    *skippable &= pnskip;
 | 
			
		||||
    if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
 | 
			
		||||
        RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
 | 
			
		||||
      is_cost_valid = 0;
 | 
			
		||||
      break;
 | 
			
		||||
    }
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  if (!is_cost_valid) {
 | 
			
		||||
    // reset cost value
 | 
			
		||||
    *rate = INT_MAX;
 | 
			
		||||
    *distortion = INT64_MAX;
 | 
			
		||||
    *sse = INT64_MAX;
 | 
			
		||||
    *skippable = 0;
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  return is_cost_valid;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
static void rd_pick_palette_intra_sbuv(
 | 
			
		||||
    AV1_COMP *cpi, MACROBLOCK *x, int dc_mode_cost,
 | 
			
		||||
    PALETTE_MODE_INFO *palette_mode_info, uint8_t *best_palette_color_map,
 | 
			
		||||
 
 | 
			
		||||
		Reference in New Issue
	
	Block a user