diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 1d08cfab4..8649b7db6 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h @@ -189,6 +189,7 @@ typedef struct { // TODO(jingning): This effectively assigned a separate entry for each // 8x8 block. Apparently it takes much more space than needed. TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE]; + TX_SIZE min_tx_size; #endif int8_t skip; int8_t segment_id; diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h index 464314eb4..9bf672de9 100644 --- a/av1/common/onyxc_int.h +++ b/av1/common/onyxc_int.h @@ -731,6 +731,11 @@ static INLINE void av1_zero_left_context(MACROBLOCKD *const xd) { } #if CONFIG_VAR_TX +static INLINE TX_SIZE get_min_tx_size(const TX_SIZE tx_size) { + if (tx_size >= TX_SIZES_ALL) assert(0); + return txsize_sqr_map[tx_size]; +} + static INLINE void set_txfm_ctx(TXFM_CONTEXT *txfm_ctx, uint8_t txs, int len) { int i; for (i = 0; i < len; ++i) txfm_ctx[i] = txs; diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index 50583f549..f302ad239 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c @@ -315,6 +315,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx) inter_tx_size[idy][idx] = tx_size; mbmi->tx_size = tx_size; + mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size)); if (counts) ++counts->txfm_partition[ctx][0]; txfm_partition_update(xd->above_txfm_context + tx_col, xd->left_txfm_context + tx_row, tx_size); @@ -333,6 +334,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, if (tx_size == TX_8X8) { inter_tx_size[0][0] = TX_4X4; mbmi->tx_size = TX_4X4; + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); txfm_partition_update(xd->above_txfm_context + tx_col, xd->left_txfm_context + tx_row, TX_4X4); return; @@ -352,6 +354,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx) inter_tx_size[idy][idx] = tx_size; mbmi->tx_size = tx_size; + mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size)); if (counts) ++counts->txfm_partition[ctx][0]; txfm_partition_update(xd->above_txfm_context + tx_col, xd->left_txfm_context + tx_row, tx_size); @@ -672,7 +675,11 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd, #endif aom_reader *r) { const int inter_block = is_inter_block(mbmi); +#if CONFIG_VAR_TX + const TX_SIZE tx_size = inter_block ? mbmi->min_tx_size : mbmi->tx_size; +#else const TX_SIZE tx_size = mbmi->tx_size; +#endif if (!FIXED_TX_TYPE) { #if CONFIG_EXT_TX if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block) > 1 && @@ -1856,6 +1863,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi, set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd); } else { #endif // CONFIG_EXT_TX && CONFIG_RECT_TX + mbmi->min_tx_size = TX_SIZES_ALL; for (idy = 0; idy < height; idy += bh) for (idx = 0; idx < width; idx += bw) read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, @@ -1877,7 +1885,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi, for (idx = 0; idx < width; ++idx) mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size; } - + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd); } #else diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 7c78edaf1..e631989fc 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c @@ -1129,7 +1129,11 @@ static void write_tx_type(const AV1_COMMON *const cm, #endif aom_writer *w) { const int is_inter = is_inter_block(mbmi); +#if CONFIG_VAR_TX + const TX_SIZE tx_size = is_inter ? mbmi->min_tx_size : mbmi->tx_size; +#else const TX_SIZE tx_size = mbmi->tx_size; +#endif if (!FIXED_TX_TYPE) { #if CONFIG_EXT_TX const BLOCK_SIZE bsize = mbmi->sb_type; @@ -1184,13 +1188,6 @@ static void write_tx_type(const AV1_COMMON *const cm, &ext_tx_encodings[mbmi->tx_type]); #endif } - } else { - if (!mbmi->skip) { -#if CONFIG_SUPERTX - if (!supertx_enabled) -#endif // CONFIG_SUPERTX - assert(mbmi->tx_type == DCT_DCT); - } } #endif // CONFIG_EXT_TX } @@ -1898,6 +1895,7 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile, (*tok)++; } } + #if CONFIG_RD_DEBUG for (plane = 0; plane < MAX_MB_PLANE; ++plane) { if (m->mbmi.txb_coeff_cost[plane] != txb_coeff_cost[plane]) { diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index 6f82e83e3..9de8695ac 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c @@ -1300,6 +1300,9 @@ static void update_state_supertx(const AV1_COMP *const cpi, ThreadData *td, for (i = 0; i < 1; ++i) memcpy(x->blk_skip[i], ctx->blk_skip[i], sizeof(uint8_t) * ctx->num_4x4_blk); + + if (!is_inter_block(mbmi) || mbmi->skip) + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); #endif // CONFIG_VAR_TX #if CONFIG_VAR_TX @@ -1502,6 +1505,7 @@ static void update_supertx_param(ThreadData *td, PICK_MODE_CONTEXT *ctx, for (i = 0; i < 1; ++i) memcpy(ctx->blk_skip[i], x->blk_skip[i], sizeof(uint8_t) * ctx->num_4x4_blk); + ctx->mic.mbmi.min_tx_size = get_min_tx_size(supertx_size); #endif // CONFIG_VAR_TX ctx->mic.mbmi.tx_size = supertx_size; ctx->skip = x->skip; @@ -5195,6 +5199,9 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, } } #endif // CONFIG_PALETTE +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif av1_tokenize_sb(cpi, td, t, dry_run, AOMMAX(bsize, BLOCK_8X8), rate); } else { int ref; @@ -5222,6 +5229,7 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, av1_encode_sb((AV1_COMMON *)cm, x, AOMMAX(bsize, BLOCK_8X8)); #if CONFIG_VAR_TX + if (mbmi->skip) mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); #if CONFIG_EXT_TX && CONFIG_RECT_TX if (is_rect_tx(mbmi->tx_size)) av1_tokenize_sb(cpi, td, t, dry_run, AOMMAX(bsize, BLOCK_8X8), rate); @@ -5235,29 +5243,35 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, } if (!dry_run) { + const int is_inter = is_inter_block(mbmi); +#if CONFIG_VAR_TX + TX_SIZE tx_size = + is_inter && !mbmi->skip ? mbmi->min_tx_size : mbmi->tx_size; +#else + TX_SIZE tx_size = mbmi->tx_size; +#endif if (cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8 && !(is_inter_block(mbmi) && (mbmi->skip || seg_skip))) { - const int is_inter = is_inter_block(mbmi); const int tx_size_ctx = get_tx_size_context(xd); const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] : intra_tx_size_cat_lookup[bsize]; - const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size]; + const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size]; const int depth = tx_size_to_depth(coded_tx_size); #if CONFIG_EXT_TX && CONFIG_RECT_TX - assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi))); + assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi))); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX #if CONFIG_VAR_TX #if CONFIG_EXT_TX && CONFIG_RECT_TX if (is_rect_tx_allowed(xd, mbmi)) { - td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++; + td->counts->rect_tx[tx_size_cat][is_rect_tx(tx_size)]++; } - if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) { + if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(tx_size)) { #endif // CONFIG_EXT_TX && CONFIG_RECT_TX if (is_inter) { tx_partition_count_update(cm, x, bsize, mi_row, mi_col, td->counts); } else { ++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth]; - if (mbmi->tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count; + if (tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count; } #if CONFIG_EXT_TX && CONFIG_RECT_TX } @@ -5268,57 +5282,57 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, #endif } else { int i, j; - TX_SIZE tx_size; + TX_SIZE intra_tx_size; // The new intra coding scheme requires no change of transform size if (is_inter_block(&mi->mbmi)) { if (xd->lossless[mbmi->segment_id]) { - tx_size = TX_4X4; + intra_tx_size = TX_4X4; } else { - tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1); + intra_tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1); } #if CONFIG_EXT_TX && CONFIG_RECT_TX ++td->counts->tx_size_implied[max_txsize_lookup[bsize]] - [txsize_sqr_up_map[mbmi->tx_size]]; + [txsize_sqr_up_map[tx_size]]; #endif // CONFIG_EXT_TX && CONFIG_RECT_TX } else { - tx_size = (bsize >= BLOCK_8X8) ? mbmi->tx_size : TX_4X4; + intra_tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4; } for (j = 0; j < mi_height; j++) for (i = 0; i < mi_width; i++) if (mi_col + i < cm->mi_cols && mi_row + j < cm->mi_rows) - mi_8x8[mis * j + i]->mbmi.tx_size = tx_size; + mi_8x8[mis * j + i]->mbmi.tx_size = intra_tx_size; #if CONFIG_VAR_TX - if (mbmi->tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count; + mbmi->min_tx_size = get_min_tx_size(intra_tx_size); + if (intra_tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count; #endif } - ++td->counts->tx_size_totals[txsize_sqr_map[mbmi->tx_size]]; + ++td->counts->tx_size_totals[txsize_sqr_map[tx_size]]; ++td->counts ->tx_size_totals[txsize_sqr_map[get_uv_tx_size(mbmi, &xd->plane[1])]]; #if CONFIG_EXT_TX - if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter_block(mbmi)) > 1 && + if (get_ext_tx_types(tx_size, bsize, is_inter_block(mbmi)) > 1 && cm->base_qindex > 0 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { - int eset = get_ext_tx_set(mbmi->tx_size, bsize, is_inter_block(mbmi)); + int eset = get_ext_tx_set(tx_size, bsize, is_inter_block(mbmi)); if (eset > 0) { if (is_inter_block(mbmi)) { - ++td->counts->inter_ext_tx[eset][txsize_sqr_map[mbmi->tx_size]] + ++td->counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]] [mbmi->tx_type]; } else { - ++td->counts->intra_ext_tx[eset][mbmi->tx_size][mbmi->mode] - [mbmi->tx_type]; + ++td->counts->intra_ext_tx[eset][tx_size][mbmi->mode][mbmi->tx_type]; } } } #else - if (mbmi->tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip && + if (tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { if (is_inter_block(mbmi)) { - ++td->counts->inter_ext_tx[mbmi->tx_size][mbmi->tx_type]; + ++td->counts->inter_ext_tx[tx_size][mbmi->tx_type]; } else { - ++td->counts->intra_ext_tx[mbmi->tx_size] + ++td->counts->intra_ext_tx[tx_size] [intra_mode_to_tx_type_context[mbmi->mode]] [mbmi->tx_type]; } @@ -5339,7 +5353,7 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX } else { - TX_SIZE tx_size; + TX_SIZE tx_size = mbmi->tx_size; // The new intra coding scheme requires no change of transform size if (is_inter_block(mbmi)) #if CONFIG_EXT_TX && CONFIG_RECT_TX @@ -5355,7 +5369,7 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td, max_txsize_lookup[bsize]); #endif else - tx_size = (bsize >= BLOCK_8X8) ? mbmi->tx_size : TX_4X4; + tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4; mbmi->tx_size = tx_size; set_txfm_ctxs(tx_size, xd->n8_w, xd->n8_h, xd); } diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index d23a7ba30..792654a2d 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c @@ -1536,7 +1536,9 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, *sse = INT64_MAX; mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); - +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif #if CONFIG_EXT_TX ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter); #endif // CONFIG_EXT_TX @@ -1665,6 +1667,9 @@ static void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, mbmi->tx_size = TX_4X4; mbmi->tx_type = DCT_DCT; +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(TX_4X4); +#endif txfm_rd_in_plane(x, cpi, rate, distortion, skip, sse, ref_best_rd, 0, bs, mbmi->tx_size, cpi->sf.use_fast_coef_costing); @@ -1716,6 +1721,10 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi, mbmi->tx_size = best_tx; mbmi->tx_type = best_tx_type; +#if CONFIG_VAR_TX + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif + #if !CONFIG_EXT_TX if (mbmi->tx_size >= TX_32X32) assert(mbmi->tx_type == DCT_DCT); #endif @@ -3246,8 +3255,12 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x, int s0 = av1_cost_bit(skip_prob, 0); int s1 = av1_cost_bit(skip_prob, 1); int64_t rd; + int row, col; + const int max_blocks_high = max_block_high(xd, bsize, 0); + const int max_blocks_wide = max_block_wide(xd, bsize, 0); mbmi->tx_type = tx_type; + mbmi->min_tx_size = TX_SIZES_ALL; inter_block_yrd(cpi, x, rd_stats, bsize, ref_best_rd); #if CONFIG_EXT_TX && CONFIG_RECT_TX if (is_rect_tx_allowed(xd, mbmi)) { @@ -3304,31 +3317,32 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x, if (rd_stats->rate == INT_MAX) return INT64_MAX; + for (row = 0; row < max_blocks_high / 2; ++row) + for (col = 0; col < max_blocks_wide / 2; ++col) + mbmi->min_tx_size = AOMMIN( + mbmi->min_tx_size, get_min_tx_size(mbmi->inter_tx_size[row][col])); + #if CONFIG_EXT_TX - if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter) > 1 && + if (get_ext_tx_types(mbmi->min_tx_size, bsize, is_inter) > 1 && !xd->lossless[xd->mi[0]->mbmi.segment_id]) { - int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bsize, is_inter); + int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter); if (is_inter) { if (ext_tx_set > 0) rd_stats->rate += - cpi->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[mbmi->tx_size]] + cpi->inter_tx_type_costs[ext_tx_set] + [txsize_sqr_map[mbmi->min_tx_size]] [mbmi->tx_type]; } else { if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) - rd_stats->rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size] - [mbmi->mode][mbmi->tx_type]; + rd_stats->rate += + cpi->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size][mbmi->mode] + [mbmi->tx_type]; } } #else // CONFIG_EXT_TX - if (mbmi->tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id]) { - if (is_inter) - rd_stats->rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type]; - else - rd_stats->rate += - cpi->intra_tx_type_costs[mbmi->tx_size] - [intra_mode_to_tx_type_context[mbmi->mode]] - [mbmi->tx_type]; - } + if (mbmi->min_tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id]) + rd_stats->rate += + cpi->inter_tx_type_costs[mbmi->min_tx_size][mbmi->tx_type]; #endif // CONFIG_EXT_TX if (rd_stats->skip) @@ -3355,6 +3369,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, const int is_inter = is_inter_block(mbmi); TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE]; TX_SIZE best_tx = max_txsize_lookup[bsize]; + TX_SIZE best_min_tx_size = TX_SIZES_ALL; uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 4]; const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4); int idx, idy; @@ -3388,7 +3403,6 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue; } #else // CONFIG_EXT_TX - if (max_tx_size >= TX_32X32 && tx_type != DCT_DCT) continue; if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE && !do_tx_type_search(tx_type, prune)) continue; @@ -3397,6 +3411,9 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, tx_type != get_default_tx_type(0, xd, 0, max_tx_size)) continue; + if (xd->lossless[mbmi->segment_id]) + if (tx_type != DCT_DCT) continue; + rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, ref_best_rd, tx_type); @@ -3405,6 +3422,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, *rd_stats = this_rd_stats; best_tx_type = mbmi->tx_type; best_tx = mbmi->tx_size; + best_min_tx_size = mbmi->min_tx_size; memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4); for (idy = 0; idy < xd->n8_h; ++idy) for (idx = 0; idx < xd->n8_w; ++idx) @@ -3417,6 +3435,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, for (idx = 0; idx < xd->n8_w; ++idx) mbmi->inter_tx_size[idy][idx] = best_tx_size[idy][idx]; mbmi->tx_size = best_tx; + mbmi->min_tx_size = best_min_tx_size; #if CONFIG_RD_DEBUG // record plane y's transform block coefficient cost mbmi->txb_coeff_cost[0] = rd_stats->txb_coeff_cost[0]; @@ -9271,6 +9290,8 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, for (i = 0; i < MAX_MB_PLANE; ++i) memcpy(ctx->blk_skip[i], x->blk_skip[i], sizeof(uint8_t) * ctx->num_4x4_blk); + + best_mbmode.min_tx_size = mbmi->min_tx_size; #endif rd_cost->rate += (rate_y + rate_uv - best_rate_y - best_rate_uv); rd_cost->dist = dist_y + dist_uv; @@ -10078,6 +10099,7 @@ void av1_rd_pick_inter_mode_sub8x8(const struct AV1_COMP *cpi, #if CONFIG_VAR_TX mbmi->inter_tx_size[0][0] = mbmi->tx_size; + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); #endif if (ref_frame == INTRA_FRAME) {