From 75f45814bacd4826e96cb0133fd5271173489b13 Mon Sep 17 00:00:00 2001
From: Angie Chiang <angiebird@google.com>
Date: Thu, 3 Nov 2016 16:01:03 -0700
Subject: [PATCH 1/2] Separate coefficient cost of U/V planes in
 write_modes_b()

Change-Id: Ie3082db5b0fead8c322b2aeede4eff7cd723ea12
---
 av1/common/blockd.h     |  3 +--
 av1/encoder/bitstream.c | 21 +++++++--------------
 2 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 53bca13df..1d08cfab4 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -251,8 +251,7 @@ typedef struct {
   int current_q_index;
 #endif
 #if CONFIG_RD_DEBUG
-  int64_t txb_cost_y;
-  int64_t txb_cost_uv;
+  int64_t txb_coeff_cost[MAX_MB_PLANE];
   int mi_row;
   int mi_col;
 #endif
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 568852f5e..fa6b905dd 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1756,8 +1756,7 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile,
   int plane;
   int bh, bw;
 #if CONFIG_RD_DEBUG
-  int64_t txb_cost_y = 0;
-  int64_t txb_cost_uv = 0;
+  int64_t txb_coeff_cost[MAX_MB_PLANE] = { 0 };
 #endif
 #if CONFIG_RANS
   (void)tok;
@@ -1903,10 +1902,7 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile,
 #endif  // CONFIG_VAR_TX
 
 #if CONFIG_RD_DEBUG
-      if (plane == 0)
-        txb_cost_y += token_stats.cost;
-      else
-        txb_cost_uv += token_stats.cost;
+      txb_coeff_cost[plane] += token_stats.cost;
 #else
       (void)token_stats;
 #endif
@@ -1916,14 +1912,11 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile,
     }
   }
 #if CONFIG_RD_DEBUG
-  if (m->mbmi.txb_cost_y != txb_cost_y) {
-    dump_mode_info(m);
-    assert(0);
-  }
-
-  if (m->mbmi.txb_cost_uv != txb_cost_uv) {
-    dump_mode_info(m);
-    assert(0);
+  for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
+    if (m->mbmi.txb_coeff_cost[plane] != txb_coeff_cost[plane]) {
+      dump_mode_info(m);
+      assert(0);
+    }
   }
 #endif
 }

From 628d7c915ba4160f80e0435523f7ac396ac7e311 Mon Sep 17 00:00:00 2001
From: Angie Chiang <angiebird@google.com>
Date: Thu, 3 Nov 2016 16:24:56 -0700
Subject: [PATCH 2/2] Record YUV planes' txfm block coeff cost in
 handle_inter_mode()

Change-Id: Ia37f170d8fd961d78a751d84b9525ab7e973b81a
---
 av1/encoder/rdopt.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index b337ec2a6..d23a7ba30 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3417,6 +3417,10 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
     for (idx = 0; idx < xd->n8_w; ++idx)
       mbmi->inter_tx_size[idy][idx] = best_tx_size[idy][idx];
   mbmi->tx_size = best_tx;
+#if CONFIG_RD_DEBUG
+  // record plane y's transform block coefficient cost
+  mbmi->txb_coeff_cost[0] = rd_stats->txb_coeff_cost[0];
+#endif
   memcpy(x->blk_skip[0], best_blk_skip, sizeof(best_blk_skip[0]) * n4);
 }
 
@@ -3537,10 +3541,7 @@ static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x,
       break;
     }
 
-    rd_stats->rate += pn_rd_stats.rate;
-    rd_stats->dist += pn_rd_stats.dist;
-    rd_stats->sse += pn_rd_stats.sse;
-    rd_stats->skip &= pn_rd_stats.skip;
+    av1_merge_rd_stats(rd_stats, &pn_rd_stats);
 
     this_rd =
         AOMMIN(RDCOST(x->rdmult, x->rddiv, rd_stats->rate, rd_stats->dist),
@@ -7524,6 +7525,11 @@ static int64_t handle_inter_mode(
 #if CONFIG_VAR_TX
       is_cost_valid_uv =
           inter_block_uvrd(cpi, x, &rd_stats_uv, bsize, ref_best_rd - rdcosty);
+#if CONFIG_RD_DEBUG
+      // record uv planes' transform block coefficient cost
+      mbmi->txb_coeff_cost[1] = rd_stats_uv.txb_coeff_cost[1];
+      mbmi->txb_coeff_cost[2] = rd_stats_uv.txb_coeff_cost[2];
+#endif
       *rate_uv = rd_stats_uv.rate;
       distortion_uv = rd_stats_uv.dist;
       skippable_uv = rd_stats_uv.skip;