From c62859fe76fe5b9a14c4ae7e2252d31b63344b6b Mon Sep 17 00:00:00 2001
From: Alex Converse <aconverse@google.com>
Date: Wed, 14 Oct 2015 15:57:25 -0700
Subject: [PATCH] Write tokens with rans

Change-Id: I3009ec1cf54a36c96b59d8203bc01f4fe82145a0
---
 vp10/common/ans.h         |  71 ++++++++++
 vp10/common/entropy.c     | 269 ++++++++++++++++++++++++++++++++++++++
 vp10/common/entropy.h     |   1 +
 vp10/decoder/detokenize.c |  92 +++++++------
 vp10/encoder/bitstream.c  |  23 ++--
 5 files changed, 401 insertions(+), 55 deletions(-)

diff --git a/vp10/common/ans.h b/vp10/common/ans.h
index 4d169308a..b36cef3e0 100644
--- a/vp10/common/ans.h
+++ b/vp10/common/ans.h
@@ -109,6 +109,77 @@ static inline int rabs_read(struct AnsDecoder *ans, AnsP8 p0) {
   return val;
 }
 
+struct rans_sym {
+  AnsP8 prob;
+  AnsP8 cum_prob;  // not-inclusive
+};
+
+struct rans_dec_sym {
+  uint8_t val;
+  AnsP8 prob;
+  AnsP8 cum_prob;  // not-inclusive
+};
+
+static inline void rans_build_dec_tab(const AnsP8 token_probs[], struct rans_dec_sym dec_tab[]) {
+  int val = 0;
+  int cum_prob = 0;
+  int sym_end = token_probs[0];
+  int i;
+  for (i = 0; i < 256; ++i) {
+    if (i == sym_end) {
+      ++val;
+      cum_prob = sym_end;
+      sym_end += token_probs[val];
+    }
+    dec_tab[i].val = val;
+    dec_tab[i].prob = token_probs[val];
+    dec_tab[i].cum_prob = cum_prob;
+  }
+}
+
+#define DBG_RANS 0
+// rANS with normalization
+// sym->prob takes the place of l_s from the paper
+// ans_p8_precision is m
+static inline void rans_stream_encode(struct AnsCoder *ans, const struct rans_sym *const sym) {
+  const AnsP8 p = sym->prob;
+//  const unsigned int s0 = ans->state;
+  while (ans->state >= l_base / ans_p8_precision * io_base * p) {
+    ans->buf[ans->buf_offset++] = ans->state % io_base;
+    ans->state /= io_base;
+  }
+#if DBG_RANS
+  unsigned state0 = ans->state;
+#endif
+  ans->state = (ans->state / p) * ans_p8_precision + ans->state % p + sym->cum_prob;
+#if DBG_RANS
+  fprintf(stderr, "C(val = [%02x %02x], %x) = %x\n", sym->cum_prob, sym->prob, state0, ans->state);
+#endif
+}
+
+static inline int rans_stream_decode(struct AnsDecoder *ans,
+                                     const struct rans_dec_sym tab[]) {
+  unsigned rem;
+  unsigned quo;
+  int val;
+//  const unsigned int s0 = ans->state;
+  if (ans->state < l_base) {
+    ans->state = ans->state * io_base + ans->buf[--ans->buf_offset];
+  }
+#if DBG_RANS
+  unsigned state0 = ans->state;
+#endif
+  quo = ans->state / ans_p8_precision;
+  rem = ans->state % ans_p8_precision;
+  val = tab[rem].val;
+
+  ans->state = quo * tab[rem].prob + rem - tab[rem].cum_prob;
+#if DBG_RANS
+  fprintf(stderr, "D(%x) = (%d = [%02x %02x], %x)\n", state0, val, tab[rem].cum_prob, tab[rem].prob, ans->state);
+#endif
+  return val;
+}
+
 static inline int ans_read_init(struct AnsDecoder *const ans,
                                  const uint8_t *const buf,
                                  int offset) {
diff --git a/vp10/common/entropy.c b/vp10/common/entropy.c
index 56dd73a4a..275388274 100644
--- a/vp10/common/entropy.c
+++ b/vp10/common/entropy.c
@@ -406,6 +406,275 @@ const vpx_prob vp10_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES] = {
   {255, 246, 247, 255, 239, 255, 253, 255},
 };
 
+// Model obtained from a 2-sided zero-centerd distribuition derived
+// from a Pareto distribution. The cdf of the distribution is:
+// cdf(x) = 0.5 + 0.5 * sgn(x) * [1 - {alpha/(alpha + |x|)} ^ beta]
+//
+// For a given beta and a given probablity of the 1-node, the alpha
+// is first solved, and then the {alpha, beta} pair is used to generate
+// the probabilities for the rest of the nodes.
+//
+// beta = 8
+// Values for tokens ONE_TOKEN through CATEGORY6_TOKEN included here.
+// ZERO_TOKEN and EOB_TOKEN are coded as flags outside this coder.
+const vpx_prob vp10_pareto8_token_probs[COEFF_PROB_MODELS][ENTROPY_TOKENS - 2] =
+    {{1, 1, 1, 1, 1, 4, 8, 14, 26, 199},
+     {2, 2, 2, 2, 4, 7, 14, 26, 42, 155},
+     {2, 3, 3, 3, 6, 11, 20, 35, 51, 122},
+     {4, 4, 4, 5, 7, 14, 25, 41, 56, 96},
+     {4, 5, 5, 5, 9, 17, 30, 46, 58, 77},
+     {5, 6, 6, 6, 11, 20, 34, 50, 57, 61},
+     {7, 7, 7, 6, 12, 22, 37, 53, 56, 49},
+     {8, 8, 7, 7, 14, 25, 40, 54, 53, 40},
+     {9, 9, 8, 8, 15, 27, 43, 55, 50, 32},
+     {10, 10, 8, 9, 16, 29, 45, 56, 47, 26},
+     {11, 10, 11, 9, 18, 31, 47, 55, 43, 21},
+     {12, 11, 11, 10, 19, 32, 48, 55, 40, 18},
+     {13, 11, 12, 11, 20, 34, 49, 54, 37, 15},
+     {14, 13, 12, 12, 21, 35, 50, 53, 34, 12},
+     {15, 14, 13, 12, 22, 37, 51, 51, 31, 10},
+     {16, 15, 14, 13, 23, 38, 51, 50, 28, 8},
+     {17, 16, 15, 14, 24, 39, 51, 48, 26, 6},
+     {18, 17, 15, 14, 25, 40, 51, 46, 23, 7},
+     {19, 17, 16, 15, 26, 41, 51, 45, 21, 5},
+     {20, 18, 17, 15, 27, 41, 51, 43, 19, 5},
+     {21, 19, 16, 16, 28, 42, 51, 41, 18, 4},
+     {22, 20, 18, 16, 28, 43, 51, 39, 16, 3},
+     {23, 21, 19, 17, 29, 43, 50, 36, 15, 3},
+     {24, 21, 19, 17, 30, 44, 49, 36, 13, 3},
+     {25, 22, 20, 18, 30, 44, 49, 34, 12, 2},
+     {26, 23, 20, 18, 31, 44, 48, 33, 11, 2},
+     {27, 24, 21, 19, 31, 45, 47, 31, 10, 1},
+     {28, 25, 22, 19, 32, 45, 46, 29, 9, 1},
+     {29, 25, 22, 19, 32, 45, 46, 28, 8, 2},
+     {30, 26, 23, 20, 33, 45, 45, 27, 6, 1},
+     {31, 27, 23, 20, 33, 45, 44, 25, 7, 1},
+     {32, 28, 24, 21, 33, 45, 43, 24, 5, 1},
+     {33, 28, 23, 21, 34, 45, 42, 23, 6, 1},
+     {34, 29, 25, 21, 34, 45, 41, 22, 4, 1},
+     {35, 30, 25, 22, 34, 44, 40, 20, 5, 1},
+     {36, 30, 26, 22, 35, 44, 39, 19, 4, 1},
+     {37, 31, 26, 22, 35, 44, 38, 18, 4, 1},
+     {38, 32, 27, 22, 35, 44, 37, 16, 4, 1},
+     {39, 32, 27, 23, 35, 43, 36, 16, 3, 2},
+     {40, 33, 28, 23, 35, 43, 35, 16, 2, 1},
+     {41, 34, 28, 23, 35, 43, 34, 15, 2, 1},
+     {42, 34, 28, 23, 36, 42, 33, 14, 2, 2},
+     {43, 35, 29, 24, 36, 42, 32, 13, 1, 1},
+     {44, 36, 29, 24, 36, 42, 32, 11, 1, 1},
+     {45, 36, 29, 24, 36, 41, 31, 12, 1, 1},
+     {46, 37, 30, 24, 36, 41, 30, 10, 1, 1},
+     {47, 38, 30, 23, 36, 40, 29, 11, 1, 1},
+     {48, 38, 30, 24, 36, 40, 28, 10, 1, 1},
+     {49, 39, 31, 25, 36, 39, 27, 8, 1, 1},
+     {50, 39, 31, 25, 36, 39, 26, 8, 1, 1},
+     {51, 40, 31, 25, 36, 38, 26, 7, 1, 1},
+     {52, 40, 32, 25, 35, 38, 25, 7, 1, 1},
+     {53, 41, 32, 25, 34, 37, 24, 8, 1, 1},
+     {54, 42, 32, 25, 35, 37, 23, 6, 1, 1},
+     {55, 42, 32, 25, 35, 36, 22, 7, 1, 1},
+     {56, 43, 33, 25, 35, 36, 22, 4, 1, 1},
+     {57, 43, 33, 25, 35, 35, 21, 5, 1, 1},
+     {58, 44, 33, 25, 35, 34, 19, 6, 1, 1},
+     {59, 44, 33, 25, 35, 34, 20, 4, 1, 1},
+     {60, 45, 34, 25, 34, 33, 19, 4, 1, 1},
+     {61, 45, 34, 25, 34, 33, 18, 4, 1, 1},
+     {62, 46, 34, 23, 34, 32, 18, 5, 1, 1},
+     {63, 46, 34, 25, 34, 32, 17, 3, 1, 1},
+     {64, 47, 34, 25, 34, 31, 17, 2, 1, 1},
+     {65, 47, 34, 25, 33, 31, 16, 3, 1, 1},
+     {66, 47, 35, 25, 33, 30, 16, 2, 1, 1},
+     {67, 48, 35, 25, 33, 30, 15, 1, 1, 1},
+     {68, 48, 35, 25, 33, 29, 14, 2, 1, 1},
+     {69, 49, 35, 25, 32, 28, 14, 2, 1, 1},
+     {70, 49, 35, 25, 32, 28, 13, 2, 1, 1},
+     {71, 50, 35, 25, 32, 27, 13, 1, 1, 1},
+     {72, 50, 35, 25, 31, 27, 13, 1, 1, 1},
+     {73, 50, 35, 25, 31, 26, 12, 2, 1, 1},
+     {74, 51, 34, 25, 31, 26, 12, 1, 1, 1},
+     {75, 51, 35, 25, 31, 25, 11, 1, 1, 1},
+     {76, 52, 36, 25, 30, 25, 9, 1, 1, 1},
+     {77, 52, 36, 25, 30, 24, 9, 1, 1, 1},
+     {78, 52, 36, 25, 30, 24, 8, 1, 1, 1},
+     {79, 53, 36, 25, 29, 21, 10, 1, 1, 1},
+     {80, 53, 36, 24, 29, 23, 7, 2, 1, 1},
+     {81, 53, 36, 24, 29, 22, 7, 2, 1, 1},
+     {82, 54, 36, 24, 26, 22, 9, 1, 1, 1},
+     {83, 54, 36, 24, 28, 21, 7, 1, 1, 1},
+     {84, 54, 36, 24, 28, 21, 6, 1, 1, 1},
+     {85, 55, 36, 24, 27, 18, 8, 1, 1, 1},
+     {86, 55, 36, 24, 27, 20, 5, 1, 1, 1},
+     {87, 55, 36, 24, 27, 19, 5, 1, 1, 1},
+     {88, 55, 36, 23, 26, 19, 6, 1, 1, 1},
+     {89, 56, 36, 20, 26, 19, 7, 1, 1, 1},
+     {90, 56, 36, 23, 26, 18, 4, 1, 1, 1},
+     {91, 56, 36, 23, 25, 18, 4, 1, 1, 1},
+     {92, 56, 36, 23, 25, 17, 4, 1, 1, 1},
+     {93, 57, 35, 23, 25, 17, 3, 1, 1, 1},
+     {94, 57, 35, 23, 24, 14, 6, 1, 1, 1},
+     {95, 57, 35, 22, 24, 16, 4, 1, 1, 1},
+     {96, 57, 35, 22, 24, 16, 3, 1, 1, 1},
+     {97, 58, 35, 22, 23, 15, 3, 1, 1, 1},
+     {98, 58, 35, 22, 23, 15, 2, 1, 1, 1},
+     {99, 58, 31, 22, 23, 15, 5, 1, 1, 1},
+     {100, 58, 35, 22, 22, 14, 2, 1, 1, 1},
+     {101, 58, 35, 21, 22, 14, 2, 1, 1, 1},
+     {102, 59, 35, 21, 22, 13, 1, 1, 1, 1},
+     {103, 59, 35, 21, 21, 13, 1, 1, 1, 1},
+     {104, 59, 34, 21, 21, 13, 1, 1, 1, 1},
+     {105, 59, 34, 21, 21, 9, 4, 1, 1, 1},
+     {106, 59, 34, 20, 20, 12, 2, 1, 1, 1},
+     {107, 59, 34, 20, 20, 12, 1, 1, 1, 1},
+     {108, 59, 34, 19, 20, 12, 1, 1, 1, 1},
+     {109, 59, 34, 20, 19, 11, 1, 1, 1, 1},
+     {110, 60, 34, 20, 19, 9, 1, 1, 1, 1},
+     {111, 60, 33, 18, 19, 11, 1, 1, 1, 1},
+     {112, 60, 33, 19, 18, 8, 3, 1, 1, 1},
+     {113, 60, 33, 19, 18, 7, 3, 1, 1, 1},
+     {114, 60, 33, 19, 18, 8, 1, 1, 1, 1},
+     {115, 60, 33, 19, 17, 8, 1, 1, 1, 1},
+     {116, 60, 33, 18, 17, 8, 1, 1, 1, 1},
+     {117, 60, 32, 18, 17, 8, 1, 1, 1, 1},
+     {118, 60, 32, 16, 17, 9, 1, 1, 1, 1},
+     {119, 60, 32, 18, 16, 7, 1, 1, 1, 1},
+     {120, 60, 32, 18, 16, 6, 1, 1, 1, 1},
+     {121, 60, 32, 17, 16, 6, 1, 1, 1, 1},
+     {122, 60, 31, 17, 14, 8, 1, 1, 1, 1},
+     {123, 60, 31, 17, 13, 8, 1, 1, 1, 1},
+     {124, 60, 31, 17, 15, 4, 2, 1, 1, 1},
+     {125, 60, 31, 16, 14, 5, 2, 1, 1, 1},
+     {126, 60, 31, 16, 14, 5, 1, 1, 1, 1},
+     {127, 60, 30, 14, 14, 7, 1, 1, 1, 1},
+     {128, 60, 30, 16, 14, 4, 1, 1, 1, 1},
+     {129, 60, 30, 16, 13, 4, 1, 1, 1, 1},
+     {130, 60, 30, 15, 13, 4, 1, 1, 1, 1},
+     {131, 60, 29, 15, 13, 4, 1, 1, 1, 1},
+     {132, 60, 29, 15, 13, 3, 1, 1, 1, 1},
+     {133, 60, 29, 15, 9, 6, 1, 1, 1, 1},
+     {134, 60, 29, 15, 12, 2, 1, 1, 1, 1},
+     {135, 60, 29, 14, 12, 2, 1, 1, 1, 1},
+     {136, 60, 28, 14, 12, 2, 1, 1, 1, 1},
+     {137, 60, 28, 14, 11, 2, 1, 1, 1, 1},
+     {138, 60, 28, 14, 11, 1, 1, 1, 1, 1},
+     {139, 60, 28, 14, 6, 5, 1, 1, 1, 1},
+     {140, 60, 27, 13, 11, 1, 1, 1, 1, 1},
+     {141, 59, 27, 13, 10, 2, 1, 1, 1, 1},
+     {142, 59, 27, 13, 10, 1, 1, 1, 1, 1},
+     {143, 59, 27, 13, 9, 1, 1, 1, 1, 1},
+     {144, 59, 26, 12, 10, 1, 1, 1, 1, 1},
+     {145, 59, 26, 12, 6, 4, 1, 1, 1, 1},
+     {146, 59, 26, 12, 5, 4, 1, 1, 1, 1},
+     {147, 59, 25, 12, 5, 4, 1, 1, 1, 1},
+     {148, 58, 24, 12, 9, 1, 1, 1, 1, 1},
+     {149, 58, 25, 10, 9, 1, 1, 1, 1, 1},
+     {150, 58, 25, 11, 7, 1, 1, 1, 1, 1},
+     {151, 58, 24, 11, 7, 1, 1, 1, 1, 1},
+     {152, 58, 24, 11, 6, 1, 1, 1, 1, 1},
+     {153, 58, 24, 11, 5, 1, 1, 1, 1, 1},
+     {154, 57, 24, 6, 8, 3, 1, 1, 1, 1},
+     {155, 57, 23, 10, 4, 3, 1, 1, 1, 1},
+     {156, 57, 23, 10, 3, 3, 1, 1, 1, 1},
+     {157, 57, 23, 10, 4, 1, 1, 1, 1, 1},
+     {158, 56, 20, 10, 7, 1, 1, 1, 1, 1},
+     {159, 56, 19, 10, 7, 1, 1, 1, 1, 1},
+     {160, 56, 22, 9, 4, 1, 1, 1, 1, 1},
+     {161, 56, 22, 9, 3, 1, 1, 1, 1, 1},
+     {162, 55, 21, 9, 4, 1, 1, 1, 1, 1},
+     {163, 55, 21, 9, 3, 1, 1, 1, 1, 1},
+     {164, 55, 21, 9, 2, 1, 1, 1, 1, 1},
+     {165, 55, 20, 5, 6, 1, 1, 1, 1, 1},
+     {166, 54, 20, 8, 2, 2, 1, 1, 1, 1},
+     {167, 54, 20, 8, 1, 2, 1, 1, 1, 1},
+     {168, 54, 20, 8, 1, 1, 1, 1, 1, 1},
+     {169, 53, 19, 8, 1, 2, 1, 1, 1, 1},
+     {170, 53, 19, 8, 1, 1, 1, 1, 1, 1},
+     {171, 53, 19, 3, 5, 1, 1, 1, 1, 1},
+     {172, 52, 18, 7, 2, 1, 1, 1, 1, 1},
+     {173, 52, 18, 7, 1, 1, 1, 1, 1, 1},
+     {174, 52, 18, 6, 1, 1, 1, 1, 1, 1},
+     {175, 51, 18, 6, 1, 1, 1, 1, 1, 1},
+     {176, 51, 16, 7, 1, 1, 1, 1, 1, 1},
+     {177, 51, 17, 2, 4, 1, 1, 1, 1, 1},
+     {178, 50, 17, 2, 4, 1, 1, 1, 1, 1},
+     {179, 50, 16, 2, 4, 1, 1, 1, 1, 1},
+     {180, 50, 16, 4, 1, 1, 1, 1, 1, 1},
+     {181, 49, 16, 4, 1, 1, 1, 1, 1, 1},
+     {182, 49, 13, 6, 1, 1, 1, 1, 1, 1},
+     {183, 48, 15, 4, 1, 1, 1, 1, 1, 1},
+     {184, 48, 15, 3, 1, 1, 1, 1, 1, 1},
+     {185, 47, 15, 3, 1, 1, 1, 1, 1, 1},
+     {186, 47, 14, 3, 1, 1, 1, 1, 1, 1},
+     {187, 47, 14, 1, 2, 1, 1, 1, 1, 1},
+     {188, 46, 14, 1, 2, 1, 1, 1, 1, 1},
+     {189, 46, 8, 5, 3, 1, 1, 1, 1, 1},
+     {190, 45, 13, 2, 1, 1, 1, 1, 1, 1},
+     {191, 45, 13, 1, 1, 1, 1, 1, 1, 1},
+     {192, 44, 13, 1, 1, 1, 1, 1, 1, 1},
+     {193, 44, 12, 1, 1, 1, 1, 1, 1, 1},
+     {194, 43, 12, 1, 1, 1, 1, 1, 1, 1},
+     {195, 43, 11, 1, 1, 1, 1, 1, 1, 1},
+     {196, 41, 12, 1, 1, 1, 1, 1, 1, 1},
+     {197, 42, 7, 4, 1, 1, 1, 1, 1, 1},
+     {198, 41, 10, 1, 1, 1, 1, 1, 1, 1},
+     {199, 41, 9, 1, 1, 1, 1, 1, 1, 1},
+     {200, 40, 8, 1, 2, 1, 1, 1, 1, 1},
+     {201, 40, 7, 1, 2, 1, 1, 1, 1, 1},
+     {202, 39, 8, 1, 1, 1, 1, 1, 1, 1},
+     {203, 39, 7, 1, 1, 1, 1, 1, 1, 1},
+     {204, 38, 7, 1, 1, 1, 1, 1, 1, 1},
+     {205, 38, 6, 1, 1, 1, 1, 1, 1, 1},
+     {206, 37, 4, 3, 1, 1, 1, 1, 1, 1},
+     {207, 37, 5, 1, 1, 1, 1, 1, 1, 1},
+     {208, 36, 5, 1, 1, 1, 1, 1, 1, 1},
+     {209, 35, 5, 1, 1, 1, 1, 1, 1, 1},
+     {210, 35, 4, 1, 1, 1, 1, 1, 1, 1},
+     {211, 30, 8, 1, 1, 1, 1, 1, 1, 1},
+     {212, 34, 3, 1, 1, 1, 1, 1, 1, 1},
+     {213, 33, 3, 1, 1, 1, 1, 1, 1, 1},
+     {214, 32, 3, 1, 1, 1, 1, 1, 1, 1},
+     {215, 32, 2, 1, 1, 1, 1, 1, 1, 1},
+     {216, 31, 1, 2, 1, 1, 1, 1, 1, 1},
+     {217, 30, 1, 2, 1, 1, 1, 1, 1, 1},
+     {218, 29, 1, 2, 1, 1, 1, 1, 1, 1},
+     {219, 29, 1, 1, 1, 1, 1, 1, 1, 1},
+     {219, 29, 1, 1, 1, 1, 1, 1, 1, 1},
+     {221, 27, 1, 1, 1, 1, 1, 1, 1, 1},
+     {222, 26, 1, 1, 1, 1, 1, 1, 1, 1},
+     {221, 27, 1, 1, 1, 1, 1, 1, 1, 1},
+     {224, 24, 1, 1, 1, 1, 1, 1, 1, 1},
+     {225, 23, 1, 1, 1, 1, 1, 1, 1, 1},
+     {226, 22, 1, 1, 1, 1, 1, 1, 1, 1},
+     {227, 21, 1, 1, 1, 1, 1, 1, 1, 1},
+     {228, 20, 1, 1, 1, 1, 1, 1, 1, 1},
+     {229, 16, 4, 1, 1, 1, 1, 1, 1, 1},
+     {230, 18, 1, 1, 1, 1, 1, 1, 1, 1},
+     {231, 17, 1, 1, 1, 1, 1, 1, 1, 1},
+     {232, 16, 1, 1, 1, 1, 1, 1, 1, 1},
+     {233, 15, 1, 1, 1, 1, 1, 1, 1, 1},
+     {234, 14, 1, 1, 1, 1, 1, 1, 1, 1},
+     {235, 13, 1, 1, 1, 1, 1, 1, 1, 1},
+     {236, 12, 1, 1, 1, 1, 1, 1, 1, 1},
+     {237, 11, 1, 1, 1, 1, 1, 1, 1, 1},
+     {238, 10, 1, 1, 1, 1, 1, 1, 1, 1},
+     {239, 9, 1, 1, 1, 1, 1, 1, 1, 1},
+     {240, 8, 1, 1, 1, 1, 1, 1, 1, 1},
+     {241, 6, 2, 1, 1, 1, 1, 1, 1, 1},
+     {242, 6, 1, 1, 1, 1, 1, 1, 1, 1},
+     {236, 12, 1, 1, 1, 1, 1, 1, 1, 1},
+     {244, 4, 1, 1, 1, 1, 1, 1, 1, 1},
+     {245, 3, 1, 1, 1, 1, 1, 1, 1, 1},
+     {246, 2, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+     {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}  // Bogus row -- delete me
+    };
 static const vp10_coeff_probs_model default_coef_probs_4x4[PLANE_TYPES] = {
   {  // Y plane
     {  // Intra
diff --git a/vp10/common/entropy.h b/vp10/common/entropy.h
index fba7020a5..c8ae6ffb9 100644
--- a/vp10/common/entropy.h
+++ b/vp10/common/entropy.h
@@ -162,6 +162,7 @@ static INLINE const uint8_t *get_band_translate(TX_SIZE tx_size) {
 #define MODEL_NODES (ENTROPY_NODES - UNCONSTRAINED_NODES)
 extern const vpx_tree_index vp10_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)];
 extern const vpx_prob vp10_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES];
+extern const vpx_prob vp10_pareto8_token_probs[COEFF_PROB_MODELS][ENTROPY_TOKENS - 2];
 
 typedef vpx_prob vp10_coeff_probs_model[REF_TYPES][COEF_BANDS]
                                       [COEFF_CONTEXTS][UNCONSTRAINED_NODES];
diff --git a/vp10/decoder/detokenize.c b/vp10/decoder/detokenize.c
index ee797b40f..6f0ce8b7d 100644
--- a/vp10/decoder/detokenize.c
+++ b/vp10/decoder/detokenize.c
@@ -85,6 +85,7 @@ static int decode_coefs(const MACROBLOCKD *const xd,
   const uint8_t *cat4_prob;
   const uint8_t *cat5_prob;
   const uint8_t *cat6_prob;
+  struct rans_dec_sym dec_tab[256];
 
   if (counts) {
     coef_counts = counts->coef[tx_size][type][ref];
@@ -148,56 +149,53 @@ static int decode_coefs(const MACROBLOCKD *const xd,
       prob = coef_probs[band][ctx];
     }
 
-    if (!rabs_read(ans, prob[ONE_CONTEXT_NODE])) {
-      INCREMENT_COUNT(ONE_TOKEN);
-      token = ONE_TOKEN;
-      val = 1;
-    } else {
-      INCREMENT_COUNT(TWO_TOKEN);
-      token = rabs_read_tree(ans, vp10_coef_con_tree,
-                             vp10_pareto8_full[prob[PIVOT_NODE] - 1]);
-      switch (token) {
-        case TWO_TOKEN:
-        case THREE_TOKEN:
-        case FOUR_TOKEN:
-          val = token;
-          break;
-        case CATEGORY1_TOKEN:
-          val = CAT1_MIN_VAL + read_coeff(cat1_prob, 1, ans);
-          break;
-        case CATEGORY2_TOKEN:
-          val = CAT2_MIN_VAL + read_coeff(cat2_prob, 2, ans);
-          break;
-        case CATEGORY3_TOKEN:
-          val = CAT3_MIN_VAL + read_coeff(cat3_prob, 3, ans);
-          break;
-        case CATEGORY4_TOKEN:
-          val = CAT4_MIN_VAL + read_coeff(cat4_prob, 4, ans);
-          break;
-        case CATEGORY5_TOKEN:
-          val = CAT5_MIN_VAL + read_coeff(cat5_prob, 5, ans);
-          break;
-        case CATEGORY6_TOKEN:
+    // TODO: precompute dec_tab
+    unsigned state = ans->state;
+    rans_build_dec_tab(vp10_pareto8_token_probs[prob[PIVOT_NODE] - 1], dec_tab);
+    token = ONE_TOKEN + rans_stream_decode(ans, dec_tab);
+    INCREMENT_COUNT(ONE_TOKEN + (token > ONE_TOKEN));
+    switch (token) {
+      case ONE_TOKEN:
+      case TWO_TOKEN:
+      case THREE_TOKEN:
+      case FOUR_TOKEN:
+        val = token;
+        break;
+      case CATEGORY1_TOKEN:
+        val = CAT1_MIN_VAL + read_coeff(cat1_prob, 1, ans);
+        break;
+      case CATEGORY2_TOKEN:
+        val = CAT2_MIN_VAL + read_coeff(cat2_prob, 2, ans);
+        break;
+      case CATEGORY3_TOKEN:
+        val = CAT3_MIN_VAL + read_coeff(cat3_prob, 3, ans);
+        break;
+      case CATEGORY4_TOKEN:
+        val = CAT4_MIN_VAL + read_coeff(cat4_prob, 4, ans);
+        break;
+      case CATEGORY5_TOKEN:
+        val = CAT5_MIN_VAL + read_coeff(cat5_prob, 5, ans);
+        break;
+      case CATEGORY6_TOKEN:
 #if CONFIG_VP9_HIGHBITDEPTH
-          switch (xd->bd) {
-            case VPX_BITS_8:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, ans);
-              break;
-            case VPX_BITS_10:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 16, ans);
-              break;
-            case VPX_BITS_12:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 18, ans);
-              break;
-            default:
-              assert(0);
-              return -1;
-          }
+        switch (xd->bd) {
+          case VPX_BITS_8:
+            val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, ans);
+            break;
+          case VPX_BITS_10:
+            val = CAT6_MIN_VAL + read_coeff(cat6_prob, 16, ans);
+            break;
+          case VPX_BITS_12:
+            val = CAT6_MIN_VAL + read_coeff(cat6_prob, 18, ans);
+            break;
+          default:
+            assert(0);
+            return -1;
+        }
 #else
-          val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, ans);
+        val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, ans);
 #endif
-          break;
-      }
+        break;
     }
     v = (val * dqv) >> dq_shift;
 #if CONFIG_COEFFICIENT_RANGE_CHECKING
diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c
index d6203df21..8a52a8f76 100644
--- a/vp10/encoder/bitstream.c
+++ b/vp10/encoder/bitstream.c
@@ -242,15 +242,22 @@ static void pack_mb_tokens_ans(struct AnsCoder *const ans,
     // is split into two treed writes.  The first treed write takes care of the
     // unconstrained nodes.  The second treed write takes care of the
     // constrained nodes.
-    if (t >= TWO_TOKEN && t < EOB_TOKEN) {
-      int len = UNCONSTRAINED_NODES - p->skip_eob_node;
-      int bits = v >> (n - len);
-      vp10_write_tree_r(ans, vp10_coef_con_tree,
-                     vp10_pareto8_full[p->context_tree[PIVOT_NODE] - 1],
-                     v, n - len, 0);
-      vp10_write_tree_r(ans, vp10_coef_tree, p->context_tree, bits, len, i);
-    } else {
+    if (t == EOB_TOKEN || t == ZERO_TOKEN) {
       vp10_write_tree_r(ans, vp10_coef_tree, p->context_tree, v, n, i);
+    } else {
+      struct rans_sym s;
+      int j;
+      int len = 2 - p->skip_eob_node;  // Write <EOBF>?<ZEROF>
+      int bits = v >> (n - len);
+      const vpx_prob *token_probs =
+          vp10_pareto8_token_probs[p->context_tree[PIVOT_NODE] - 1];
+      s.cum_prob = 0;
+      for (j = ONE_TOKEN; j < t; ++j) {
+        s.cum_prob += token_probs[j - ONE_TOKEN];
+      }
+      s.prob = token_probs[t - ONE_TOKEN];
+      rans_stream_encode(ans, &s);
+      vp10_write_tree_r(ans, vp10_coef_tree, p->context_tree, bits, len, i);
     }
   }
   }