PRecompute token tab

Change-Id: I85e172352dacf7a990d8d9296e071f9d603fcb7b

vp10/common/ans.h

@@ -0,0 +1,201 @@
+#ifndef VP10_COMMON_ANS_H_
+#define VP10_COMMON_ANS_H_
+// An implementation of Asymmetric Numeral Systems
+// http://arxiv.org/abs/1311.2540v2
+
+#include <stdint.h>
+#include "vpx_ports/mem_ops.h"
+
+#define ANS_DIVIDE_BY_MULTIPLY 0
+#if ANS_DIVIDE_BY_MULTIPLY
+#include "divide.h"
+#define ANS_INIT_DIVIDE init_fastdiv()
+#define ANS_DIVREM(quotient, remainder, dividend, divisor) \
+  do { \
+    quotient = FASTDIV(dividend, divisor); \
+    remainder = dividend - quotient * divisor; \
+  } while (0)
+#else
+#define ANS_INIT_DIVIDE
+#define ANS_DIVREM(quotient, remainder, dividend, divisor) \
+  do { \
+    quotient = dividend / divisor; \
+    remainder = dividend % divisor; \
+  } while (0)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+struct AnsCoder {
+  uint8_t *buf;
+  int buf_offset;
+  uint32_t state;
+};
+
+struct AnsDecoder {
+  const uint8_t *buf;
+  int buf_offset;
+  uint32_t state;
+};
+
+typedef uint8_t AnsP8;
+#define ans_p8_precision 256
+#define ans_p8_shift 8
+#define l_base (ans_p8_precision * 4)  // l_base % precision must be 0
+#define io_base 256
+// Range I = { l_base, l_base + 1, ..., l_base * io_base - 1 }
+
+static inline void ans_write_init(struct AnsCoder *const ans,
+                                  uint8_t *const buf) {
+  ans->buf = buf;
+  ans->buf_offset = 0;
+  ans->state = l_base;
+}
+
+static inline int ans_write_end(struct AnsCoder *const ans) {
+  mem_put_le24(ans->buf + ans->buf_offset, ans->state);
+  return ans->buf_offset + 3;
+}
+
+// rABS with normalization
+// p or p0 takes the place of l_s from the paper
+// ans_p8_precision is m
+static inline void rabs_write(struct AnsCoder *ans, int val, AnsP8 p0) {
+  const AnsP8 p = ans_p8_precision - p0;
+  const unsigned l_s = val ? p : p0;
+  unsigned quot, rem;
+  if ((!val && ans->state >= l_base / ans_p8_precision * io_base * p0) ||
+      (val && ans->state >= l_base / ans_p8_precision * io_base * p)) {
+    ans->buf[ans->buf_offset++] = ans->state % io_base;
+    ans->state /= io_base;
+  }
+  ANS_DIVREM(quot, rem, ans->state, l_s);
+  ans->state = quot * ans_p8_precision + rem + (val ? 0 : p);
+}
+
+#define ANS_IMPL1 0
+#define UNPREDICTABLE(x) x
+static inline int rabs_read(struct AnsDecoder *ans, AnsP8 p0) {
+  int val;
+#if ANS_IMPL1
+  unsigned l_s;
+#else
+  unsigned quot, rem, x, xn;
+#endif
+  const AnsP8 p = ans_p8_precision - p0;
+  if (ans->state < l_base) {
+    ans->state = ans->state * io_base + ans->buf[--ans->buf_offset];
+  }
+#if ANS_IMPL1
+  val = ans->state % ans_p8_precision < p;
+  l_s = val ? p : p0;
+  ans->state = (ans->state / ans_p8_precision) * l_s +
+               ans->state % ans_p8_precision - (!val * p);
+#else
+  x = ans->state;
+  quot = x / ans_p8_precision;
+  rem = x % ans_p8_precision;
+  xn = quot * p;
+  val = rem < p;
+  if (UNPREDICTABLE(val)) {
+  ans->state = xn + rem;
+  } else {
+  //ans->state = quot * p0 + rem - p;
+  ans->state = x - xn - p;
+  }
+#endif
+  return val;
+}
+
+struct rans_sym {
+  AnsP8 prob;
+  AnsP8 cum_prob;  // not-inclusive
+};
+
+struct rans_dec_sym {
+  uint8_t val;
+  AnsP8 prob;
+  AnsP8 cum_prob;  // not-inclusive
+};
+
+static inline void rans_build_dec_tab(const AnsP8 token_probs[], struct rans_dec_sym dec_tab[]) {
+  int val = 0;
+  int cum_prob = 0;
+  int sym_end = token_probs[0];
+  int i;
+  for (i = 0; i < 256; ++i) {
+    if (i == sym_end) {
+      ++val;
+      cum_prob = sym_end;
+      sym_end += token_probs[val];
+    }
+    dec_tab[i].val = val;
+    dec_tab[i].prob = token_probs[val];
+    dec_tab[i].cum_prob = cum_prob;
+  }
+}
+
+#define DBG_RANS 0
+// rANS with normalization
+// sym->prob takes the place of l_s from the paper
+// ans_p8_precision is m
+static inline void rans_stream_encode(struct AnsCoder *ans, const struct rans_sym *const sym) {
+  const AnsP8 p = sym->prob;
+//  const unsigned int s0 = ans->state;
+  while (ans->state >= l_base / ans_p8_precision * io_base * p) {
+    ans->buf[ans->buf_offset++] = ans->state % io_base;
+    ans->state /= io_base;
+  }
+#if DBG_RANS
+  unsigned state0 = ans->state;
+#endif
+  ans->state = (ans->state / p) * ans_p8_precision + ans->state % p + sym->cum_prob;
+#if DBG_RANS
+  fprintf(stderr, "C(val = [%02x %02x], %x) = %x\n", sym->cum_prob, sym->prob, state0, ans->state);
+#endif
+}
+
+static inline int rans_stream_decode(struct AnsDecoder *ans,
+                                     const struct rans_dec_sym tab[]) {
+  unsigned rem;
+  unsigned quo;
+  int val;
+//  const unsigned int s0 = ans->state;
+  if (ans->state < l_base) {
+    ans->state = ans->state * io_base + ans->buf[--ans->buf_offset];
+  }
+#if DBG_RANS
+  unsigned state0 = ans->state;
+#endif
+  quo = ans->state / ans_p8_precision;
+  rem = ans->state % ans_p8_precision;
+  val = tab[rem].val;
+
+  ans->state = quo * tab[rem].prob + rem - tab[rem].cum_prob;
+#if DBG_RANS
+  fprintf(stderr, "D(%x) = (%d = [%02x %02x], %x)\n", state0, val, tab[rem].cum_prob, tab[rem].prob, ans->state);
+#endif
+  return val;
+}
+
+static inline int ans_read_init(struct AnsDecoder *const ans,
+                                 const uint8_t *const buf,
+                                 int offset) {
+  if (offset < 3)
+    return 1;
+  ans->buf = buf;
+  ans->buf_offset = offset - 3;
+  ans->state = mem_get_le24(buf + offset - 3);
+  return 0;
+}
+
+static inline int ans_read_end(struct AnsDecoder *const ans) {
+  return ans->state == l_base;
+}
+#undef ANS_DIVREM
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
+#endif  // VP10_COMMON_ANS_H_

vp10/common/entropy.c

@@ -406,6 +406,284 @@ const vpx_prob vp10_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES] = {
   {255, 246, 247, 255, 239, 255, 253, 255},
 };
 
+// Model obtained from a 2-sided zero-centerd distribuition derived
+// from a Pareto distribution. The cdf of the distribution is:
+// cdf(x) = 0.5 + 0.5 * sgn(x) * [1 - {alpha/(alpha + |x|)} ^ beta]
+//
+// For a given beta and a given probablity of the 1-node, the alpha
+// is first solved, and then the {alpha, beta} pair is used to generate
+// the probabilities for the rest of the nodes.
+//
+// beta = 8
+// Values for tokens ONE_TOKEN through CATEGORY6_TOKEN included here.
+// ZERO_TOKEN and EOB_TOKEN are coded as flags outside this coder.
+const vpx_prob
+    vp10_pareto8_token_probs[COEFF_PROB_MODELS - 1][ENTROPY_TOKENS - 2] = {
+        {1, 1, 1, 1, 1, 4, 8, 14, 26, 199},
+        {2, 2, 2, 2, 4, 7, 14, 26, 42, 155},
+        {2, 3, 3, 3, 6, 11, 20, 35, 51, 122},
+        {4, 4, 4, 5, 7, 14, 25, 41, 56, 96},
+        {4, 5, 5, 5, 9, 17, 30, 46, 58, 77},
+        {5, 6, 6, 6, 11, 20, 34, 50, 57, 61},
+        {7, 7, 7, 6, 12, 22, 37, 53, 56, 49},
+        {8, 8, 7, 7, 14, 25, 40, 54, 53, 40},
+        {9, 9, 8, 8, 15, 27, 43, 55, 50, 32},
+        {10, 10, 8, 9, 16, 29, 45, 56, 47, 26},
+        {11, 10, 11, 9, 18, 31, 47, 55, 43, 21},
+        {12, 11, 11, 10, 19, 32, 48, 55, 40, 18},
+        {13, 11, 12, 11, 20, 34, 49, 54, 37, 15},
+        {14, 13, 12, 12, 21, 35, 50, 53, 34, 12},
+        {15, 14, 13, 12, 22, 37, 51, 51, 31, 10},
+        {16, 15, 14, 13, 23, 38, 51, 50, 28, 8},
+        {17, 16, 15, 14, 24, 39, 51, 48, 26, 6},
+        {18, 17, 15, 14, 25, 40, 51, 46, 23, 7},
+        {19, 17, 16, 15, 26, 41, 51, 45, 21, 5},
+        {20, 18, 17, 15, 27, 41, 51, 43, 19, 5},
+        {21, 19, 16, 16, 28, 42, 51, 41, 18, 4},
+        {22, 20, 18, 16, 28, 43, 51, 39, 16, 3},
+        {23, 21, 19, 17, 29, 43, 50, 36, 15, 3},
+        {24, 21, 19, 17, 30, 44, 49, 36, 13, 3},
+        {25, 22, 20, 18, 30, 44, 49, 34, 12, 2},
+        {26, 23, 20, 18, 31, 44, 48, 33, 11, 2},
+        {27, 24, 21, 19, 31, 45, 47, 31, 10, 1},
+        {28, 25, 22, 19, 32, 45, 46, 29, 9, 1},
+        {29, 25, 22, 19, 32, 45, 46, 28, 8, 2},
+        {30, 26, 23, 20, 33, 45, 45, 27, 6, 1},
+        {31, 27, 23, 20, 33, 45, 44, 25, 7, 1},
+        {32, 28, 24, 21, 33, 45, 43, 24, 5, 1},
+        {33, 28, 23, 21, 34, 45, 42, 23, 6, 1},
+        {34, 29, 25, 21, 34, 45, 41, 22, 4, 1},
+        {35, 30, 25, 22, 34, 44, 40, 20, 5, 1},
+        {36, 30, 26, 22, 35, 44, 39, 19, 4, 1},
+        {37, 31, 26, 22, 35, 44, 38, 18, 4, 1},
+        {38, 32, 27, 22, 35, 44, 37, 16, 4, 1},
+        {39, 32, 27, 23, 35, 43, 36, 16, 3, 2},
+        {40, 33, 28, 23, 35, 43, 35, 16, 2, 1},
+        {41, 34, 28, 23, 35, 43, 34, 15, 2, 1},
+        {42, 34, 28, 23, 36, 42, 33, 14, 2, 2},
+        {43, 35, 29, 24, 36, 42, 32, 13, 1, 1},
+        {44, 36, 29, 24, 36, 42, 32, 11, 1, 1},
+        {45, 36, 29, 24, 36, 41, 31, 12, 1, 1},
+        {46, 37, 30, 24, 36, 41, 30, 10, 1, 1},
+        {47, 38, 30, 23, 36, 40, 29, 11, 1, 1},
+        {48, 38, 30, 24, 36, 40, 28, 10, 1, 1},
+        {49, 39, 31, 25, 36, 39, 27, 8, 1, 1},
+        {50, 39, 31, 25, 36, 39, 26, 8, 1, 1},
+        {51, 40, 31, 25, 36, 38, 26, 7, 1, 1},
+        {52, 40, 32, 25, 35, 38, 25, 7, 1, 1},
+        {53, 41, 32, 25, 34, 37, 24, 8, 1, 1},
+        {54, 42, 32, 25, 35, 37, 23, 6, 1, 1},
+        {55, 42, 32, 25, 35, 36, 22, 7, 1, 1},
+        {56, 43, 33, 25, 35, 36, 22, 4, 1, 1},
+        {57, 43, 33, 25, 35, 35, 21, 5, 1, 1},
+        {58, 44, 33, 25, 35, 34, 19, 6, 1, 1},
+        {59, 44, 33, 25, 35, 34, 20, 4, 1, 1},
+        {60, 45, 34, 25, 34, 33, 19, 4, 1, 1},
+        {61, 45, 34, 25, 34, 33, 18, 4, 1, 1},
+        {62, 46, 34, 23, 34, 32, 18, 5, 1, 1},
+        {63, 46, 34, 25, 34, 32, 17, 3, 1, 1},
+        {64, 47, 34, 25, 34, 31, 17, 2, 1, 1},
+        {65, 47, 34, 25, 33, 31, 16, 3, 1, 1},
+        {66, 47, 35, 25, 33, 30, 16, 2, 1, 1},
+        {67, 48, 35, 25, 33, 30, 15, 1, 1, 1},
+        {68, 48, 35, 25, 33, 29, 14, 2, 1, 1},
+        {69, 49, 35, 25, 32, 28, 14, 2, 1, 1},
+        {70, 49, 35, 25, 32, 28, 13, 2, 1, 1},
+        {71, 50, 35, 25, 32, 27, 13, 1, 1, 1},
+        {72, 50, 35, 25, 31, 27, 13, 1, 1, 1},
+        {73, 50, 35, 25, 31, 26, 12, 2, 1, 1},
+        {74, 51, 34, 25, 31, 26, 12, 1, 1, 1},
+        {75, 51, 35, 25, 31, 25, 11, 1, 1, 1},
+        {76, 52, 36, 25, 30, 25, 9, 1, 1, 1},
+        {77, 52, 36, 25, 30, 24, 9, 1, 1, 1},
+        {78, 52, 36, 25, 30, 24, 8, 1, 1, 1},
+        {79, 53, 36, 25, 29, 21, 10, 1, 1, 1},
+        {80, 53, 36, 24, 29, 23, 7, 2, 1, 1},
+        {81, 53, 36, 24, 29, 22, 7, 2, 1, 1},
+        {82, 54, 36, 24, 26, 22, 9, 1, 1, 1},
+        {83, 54, 36, 24, 28, 21, 7, 1, 1, 1},
+        {84, 54, 36, 24, 28, 21, 6, 1, 1, 1},
+        {85, 55, 36, 24, 27, 18, 8, 1, 1, 1},
+        {86, 55, 36, 24, 27, 20, 5, 1, 1, 1},
+        {87, 55, 36, 24, 27, 19, 5, 1, 1, 1},
+        {88, 55, 36, 23, 26, 19, 6, 1, 1, 1},
+        {89, 56, 36, 20, 26, 19, 7, 1, 1, 1},
+        {90, 56, 36, 23, 26, 18, 4, 1, 1, 1},
+        {91, 56, 36, 23, 25, 18, 4, 1, 1, 1},
+        {92, 56, 36, 23, 25, 17, 4, 1, 1, 1},
+        {93, 57, 35, 23, 25, 17, 3, 1, 1, 1},
+        {94, 57, 35, 23, 24, 14, 6, 1, 1, 1},
+        {95, 57, 35, 22, 24, 16, 4, 1, 1, 1},
+        {96, 57, 35, 22, 24, 16, 3, 1, 1, 1},
+        {97, 58, 35, 22, 23, 15, 3, 1, 1, 1},
+        {98, 58, 35, 22, 23, 15, 2, 1, 1, 1},
+        {99, 58, 31, 22, 23, 15, 5, 1, 1, 1},
+        {100, 58, 35, 22, 22, 14, 2, 1, 1, 1},
+        {101, 58, 35, 21, 22, 14, 2, 1, 1, 1},
+        {102, 59, 35, 21, 22, 13, 1, 1, 1, 1},
+        {103, 59, 35, 21, 21, 13, 1, 1, 1, 1},
+        {104, 59, 34, 21, 21, 13, 1, 1, 1, 1},
+        {105, 59, 34, 21, 21, 9, 4, 1, 1, 1},
+        {106, 59, 34, 20, 20, 12, 2, 1, 1, 1},
+        {107, 59, 34, 20, 20, 12, 1, 1, 1, 1},
+        {108, 59, 34, 19, 20, 12, 1, 1, 1, 1},
+        {109, 59, 34, 20, 19, 11, 1, 1, 1, 1},
+        {110, 60, 34, 20, 19, 9, 1, 1, 1, 1},
+        {111, 60, 33, 18, 19, 11, 1, 1, 1, 1},
+        {112, 60, 33, 19, 18, 8, 3, 1, 1, 1},
+        {113, 60, 33, 19, 18, 7, 3, 1, 1, 1},
+        {114, 60, 33, 19, 18, 8, 1, 1, 1, 1},
+        {115, 60, 33, 19, 17, 8, 1, 1, 1, 1},
+        {116, 60, 33, 18, 17, 8, 1, 1, 1, 1},
+        {117, 60, 32, 18, 17, 8, 1, 1, 1, 1},
+        {118, 60, 32, 16, 17, 9, 1, 1, 1, 1},
+        {119, 60, 32, 18, 16, 7, 1, 1, 1, 1},
+        {120, 60, 32, 18, 16, 6, 1, 1, 1, 1},
+        {121, 60, 32, 17, 16, 6, 1, 1, 1, 1},
+        {122, 60, 31, 17, 14, 8, 1, 1, 1, 1},
+        {123, 60, 31, 17, 13, 8, 1, 1, 1, 1},
+        {124, 60, 31, 17, 15, 4, 2, 1, 1, 1},
+        {125, 60, 31, 16, 14, 5, 2, 1, 1, 1},
+        {126, 60, 31, 16, 14, 5, 1, 1, 1, 1},
+        {127, 60, 30, 14, 14, 7, 1, 1, 1, 1},
+        {128, 60, 30, 16, 14, 4, 1, 1, 1, 1},
+        {129, 60, 30, 16, 13, 4, 1, 1, 1, 1},
+        {130, 60, 30, 15, 13, 4, 1, 1, 1, 1},
+        {131, 60, 29, 15, 13, 4, 1, 1, 1, 1},
+        {132, 60, 29, 15, 13, 3, 1, 1, 1, 1},
+        {133, 60, 29, 15, 9, 6, 1, 1, 1, 1},
+        {134, 60, 29, 15, 12, 2, 1, 1, 1, 1},
+        {135, 60, 29, 14, 12, 2, 1, 1, 1, 1},
+        {136, 60, 28, 14, 12, 2, 1, 1, 1, 1},
+        {137, 60, 28, 14, 11, 2, 1, 1, 1, 1},
+        {138, 60, 28, 14, 11, 1, 1, 1, 1, 1},
+        {139, 60, 28, 14, 6, 5, 1, 1, 1, 1},
+        {140, 60, 27, 13, 11, 1, 1, 1, 1, 1},
+        {141, 59, 27, 13, 10, 2, 1, 1, 1, 1},
+        {142, 59, 27, 13, 10, 1, 1, 1, 1, 1},
+        {143, 59, 27, 13, 9, 1, 1, 1, 1, 1},
+        {144, 59, 26, 12, 10, 1, 1, 1, 1, 1},
+        {145, 59, 26, 12, 6, 4, 1, 1, 1, 1},
+        {146, 59, 26, 12, 5, 4, 1, 1, 1, 1},
+        {147, 59, 25, 12, 5, 4, 1, 1, 1, 1},
+        {148, 58, 24, 12, 9, 1, 1, 1, 1, 1},
+        {149, 58, 25, 10, 9, 1, 1, 1, 1, 1},
+        {150, 58, 25, 11, 7, 1, 1, 1, 1, 1},
+        {151, 58, 24, 11, 7, 1, 1, 1, 1, 1},
+        {152, 58, 24, 11, 6, 1, 1, 1, 1, 1},
+        {153, 58, 24, 11, 5, 1, 1, 1, 1, 1},
+        {154, 57, 24, 6, 8, 3, 1, 1, 1, 1},
+        {155, 57, 23, 10, 4, 3, 1, 1, 1, 1},
+        {156, 57, 23, 10, 3, 3, 1, 1, 1, 1},
+        {157, 57, 23, 10, 4, 1, 1, 1, 1, 1},
+        {158, 56, 20, 10, 7, 1, 1, 1, 1, 1},
+        {159, 56, 19, 10, 7, 1, 1, 1, 1, 1},
+        {160, 56, 22, 9, 4, 1, 1, 1, 1, 1},
+        {161, 56, 22, 9, 3, 1, 1, 1, 1, 1},
+        {162, 55, 21, 9, 4, 1, 1, 1, 1, 1},
+        {163, 55, 21, 9, 3, 1, 1, 1, 1, 1},
+        {164, 55, 21, 9, 2, 1, 1, 1, 1, 1},
+        {165, 55, 20, 5, 6, 1, 1, 1, 1, 1},
+        {166, 54, 20, 8, 2, 2, 1, 1, 1, 1},
+        {167, 54, 20, 8, 1, 2, 1, 1, 1, 1},
+        {168, 54, 20, 8, 1, 1, 1, 1, 1, 1},
+        {169, 53, 19, 8, 1, 2, 1, 1, 1, 1},
+        {170, 53, 19, 8, 1, 1, 1, 1, 1, 1},
+        {171, 53, 19, 3, 5, 1, 1, 1, 1, 1},
+        {172, 52, 18, 7, 2, 1, 1, 1, 1, 1},
+        {173, 52, 18, 7, 1, 1, 1, 1, 1, 1},
+        {174, 52, 18, 6, 1, 1, 1, 1, 1, 1},
+        {175, 51, 18, 6, 1, 1, 1, 1, 1, 1},
+        {176, 51, 16, 7, 1, 1, 1, 1, 1, 1},
+        {177, 51, 17, 2, 4, 1, 1, 1, 1, 1},
+        {178, 50, 17, 2, 4, 1, 1, 1, 1, 1},
+        {179, 50, 16, 2, 4, 1, 1, 1, 1, 1},
+        {180, 50, 16, 4, 1, 1, 1, 1, 1, 1},
+        {181, 49, 16, 4, 1, 1, 1, 1, 1, 1},
+        {182, 49, 13, 6, 1, 1, 1, 1, 1, 1},
+        {183, 48, 15, 4, 1, 1, 1, 1, 1, 1},
+        {184, 48, 15, 3, 1, 1, 1, 1, 1, 1},
+        {185, 47, 15, 3, 1, 1, 1, 1, 1, 1},
+        {186, 47, 14, 3, 1, 1, 1, 1, 1, 1},
+        {187, 47, 14, 1, 2, 1, 1, 1, 1, 1},
+        {188, 46, 14, 1, 2, 1, 1, 1, 1, 1},
+        {189, 46, 8, 5, 3, 1, 1, 1, 1, 1},
+        {190, 45, 13, 2, 1, 1, 1, 1, 1, 1},
+        {191, 45, 13, 1, 1, 1, 1, 1, 1, 1},
+        {192, 44, 13, 1, 1, 1, 1, 1, 1, 1},
+        {193, 44, 12, 1, 1, 1, 1, 1, 1, 1},
+        {194, 43, 12, 1, 1, 1, 1, 1, 1, 1},
+        {195, 43, 11, 1, 1, 1, 1, 1, 1, 1},
+        {196, 41, 12, 1, 1, 1, 1, 1, 1, 1},
+        {197, 42, 7, 4, 1, 1, 1, 1, 1, 1},
+        {198, 41, 10, 1, 1, 1, 1, 1, 1, 1},
+        {199, 41, 9, 1, 1, 1, 1, 1, 1, 1},
+        {200, 40, 8, 1, 2, 1, 1, 1, 1, 1},
+        {201, 40, 7, 1, 2, 1, 1, 1, 1, 1},
+        {202, 39, 8, 1, 1, 1, 1, 1, 1, 1},
+        {203, 39, 7, 1, 1, 1, 1, 1, 1, 1},
+        {204, 38, 7, 1, 1, 1, 1, 1, 1, 1},
+        {205, 38, 6, 1, 1, 1, 1, 1, 1, 1},
+        {206, 37, 4, 3, 1, 1, 1, 1, 1, 1},
+        {207, 37, 5, 1, 1, 1, 1, 1, 1, 1},
+        {208, 36, 5, 1, 1, 1, 1, 1, 1, 1},
+        {209, 35, 5, 1, 1, 1, 1, 1, 1, 1},
+        {210, 35, 4, 1, 1, 1, 1, 1, 1, 1},
+        {211, 30, 8, 1, 1, 1, 1, 1, 1, 1},
+        {212, 34, 3, 1, 1, 1, 1, 1, 1, 1},
+        {213, 33, 3, 1, 1, 1, 1, 1, 1, 1},
+        {214, 32, 3, 1, 1, 1, 1, 1, 1, 1},
+        {215, 32, 2, 1, 1, 1, 1, 1, 1, 1},
+        {216, 31, 1, 2, 1, 1, 1, 1, 1, 1},
+        {217, 30, 1, 2, 1, 1, 1, 1, 1, 1},
+        {218, 29, 1, 2, 1, 1, 1, 1, 1, 1},
+        {219, 29, 1, 1, 1, 1, 1, 1, 1, 1},
+        {219, 29, 1, 1, 1, 1, 1, 1, 1, 1},
+        {221, 27, 1, 1, 1, 1, 1, 1, 1, 1},
+        {222, 26, 1, 1, 1, 1, 1, 1, 1, 1},
+        {221, 27, 1, 1, 1, 1, 1, 1, 1, 1},
+        {224, 24, 1, 1, 1, 1, 1, 1, 1, 1},
+        {225, 23, 1, 1, 1, 1, 1, 1, 1, 1},
+        {226, 22, 1, 1, 1, 1, 1, 1, 1, 1},
+        {227, 21, 1, 1, 1, 1, 1, 1, 1, 1},
+        {228, 20, 1, 1, 1, 1, 1, 1, 1, 1},
+        {229, 16, 4, 1, 1, 1, 1, 1, 1, 1},
+        {230, 18, 1, 1, 1, 1, 1, 1, 1, 1},
+        {231, 17, 1, 1, 1, 1, 1, 1, 1, 1},
+        {232, 16, 1, 1, 1, 1, 1, 1, 1, 1},
+        {233, 15, 1, 1, 1, 1, 1, 1, 1, 1},
+        {234, 14, 1, 1, 1, 1, 1, 1, 1, 1},
+        {235, 13, 1, 1, 1, 1, 1, 1, 1, 1},
+        {236, 12, 1, 1, 1, 1, 1, 1, 1, 1},
+        {237, 11, 1, 1, 1, 1, 1, 1, 1, 1},
+        {238, 10, 1, 1, 1, 1, 1, 1, 1, 1},
+        {239, 9, 1, 1, 1, 1, 1, 1, 1, 1},
+        {240, 8, 1, 1, 1, 1, 1, 1, 1, 1},
+        {241, 6, 2, 1, 1, 1, 1, 1, 1, 1},
+        {242, 6, 1, 1, 1, 1, 1, 1, 1, 1},
+        {236, 12, 1, 1, 1, 1, 1, 1, 1, 1},
+        {244, 4, 1, 1, 1, 1, 1, 1, 1, 1},
+        {245, 3, 1, 1, 1, 1, 1, 1, 1, 1},
+        {246, 2, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+        {247, 1, 1, 1, 1, 1, 1, 1, 1, 1}};
+
+void vp10_build_pareto8_dec_tab(
+    const vpx_prob token_probs[COEFF_PROB_MODELS - 1][ENTROPY_TOKENS - 2],
+    struct rans_dec_sym dec_tab[COEFF_PROB_MODELS - 1][256]) {
+  int p;
+  for (p = 0; p < COEFF_PROB_MODELS - 1; ++p) {
+    rans_build_dec_tab(token_probs[p], dec_tab[p]);
+  }
+}
+
 static const vp10_coeff_probs_model default_coef_probs_4x4[PLANE_TYPES] = {
   {  // Y plane
     {  // Intra

vp10/common/entropy.h

@@ -14,6 +14,7 @@
 #include "vpx/vpx_integer.h"
 #include "vpx_dsp/prob.h"
 
+#include "vp10/common/ans.h"
 #include "vp10/common/common.h"
 #include "vp10/common/enums.h"
 
@@ -162,6 +163,11 @@ static INLINE const uint8_t *get_band_translate(TX_SIZE tx_size) {
 #define MODEL_NODES (ENTROPY_NODES - UNCONSTRAINED_NODES)
 extern const vpx_tree_index vp10_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)];
 extern const vpx_prob vp10_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES];
+extern const vpx_prob vp10_pareto8_token_probs[COEFF_PROB_MODELS - 1][ENTROPY_TOKENS - 2];
+
+void vp10_build_pareto8_dec_tab(
+    const vpx_prob token_probs[COEFF_PROB_MODELS - 1][ENTROPY_TOKENS - 2],
+    struct rans_dec_sym dec_tab[COEFF_PROB_MODELS - 1][256]);
 
 typedef vpx_prob vp10_coeff_probs_model[REF_TYPES][COEF_BANDS]
                                       [COEFF_CONTEXTS][UNCONSTRAINED_NODES];

vp10/decoder/decodeframe.c

@@ -327,7 +327,8 @@ static void inverse_transform_block_intra(MACROBLOCKD* xd, int plane,
 }
 
 static void predict_and_reconstruct_intra_block(MACROBLOCKD *const xd,
-                                                vpx_reader *r,
+                        const struct rans_dec_sym (*const token_tab)[256],
+                                                struct AnsDecoder *const r,
                                                 MB_MODE_INFO *const mbmi,
                                                 int plane,
                                                 int row, int col,
@@ -350,14 +351,16 @@ static void predict_and_reconstruct_intra_block(MACROBLOCKD *const xd,
   if (!mbmi->skip) {
     TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx);
     const scan_order *sc = get_scan(tx_size, tx_type);
-    const int eob = vp10_decode_block_tokens(xd, plane, sc, col, row, tx_size,
+    const int eob = vp10_decode_block_tokens(xd, token_tab, plane, sc, col, row, tx_size,
                                              r, mbmi->segment_id);
     inverse_transform_block_intra(xd, plane, tx_type, tx_size,
                                   dst, pd->dst.stride, eob);
   }
 }
 
-static int reconstruct_inter_block(MACROBLOCKD *const xd, vpx_reader *r,
+static int reconstruct_inter_block(MACROBLOCKD *const xd,
+                        const struct rans_dec_sym (*const token_tab)[256],
+                                   struct AnsDecoder *const r,
                                    MB_MODE_INFO *const mbmi, int plane,
                                    int row, int col, TX_SIZE tx_size) {
   struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -365,7 +368,7 @@ static int reconstruct_inter_block(MACROBLOCKD *const xd, vpx_reader *r,
   int block_idx = (row << 1) + col;
   TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx);
   const scan_order *sc = get_scan(tx_size, tx_type);
-  const int eob = vp10_decode_block_tokens(xd, plane, sc, col, row, tx_size, r,
+  const int eob = vp10_decode_block_tokens(xd, token_tab, plane, sc, col, row, tx_size, r,
                                           mbmi->segment_id);
 
   inverse_transform_block_inter(xd, plane, tx_size,
@@ -780,7 +783,8 @@ static MB_MODE_INFO *set_offsets(VP10_COMMON *const cm, MACROBLOCKD *const xd,
 
 static void decode_block(VP10Decoder *const pbi, MACROBLOCKD *const xd,
                          int mi_row, int mi_col,
-                         vpx_reader *r, BLOCK_SIZE bsize,
+                         vpx_reader *r, struct AnsDecoder *const tok,
+                         BLOCK_SIZE bsize,
                          int bwl, int bhl) {
   VP10_COMMON *const cm = &pbi->common;
   const int less8x8 = bsize < BLOCK_8X8;
@@ -824,7 +828,7 @@ static void decode_block(VP10Decoder *const pbi, MACROBLOCKD *const xd,
 
       for (row = 0; row < max_blocks_high; row += step)
         for (col = 0; col < max_blocks_wide; col += step)
-          predict_and_reconstruct_intra_block(xd, r, mbmi, plane,
+          predict_and_reconstruct_intra_block(xd, pbi->token_tab, tok, mbmi, plane,
                                               row, col, tx_size);
     }
   } else {
@@ -852,7 +856,7 @@ static void decode_block(VP10Decoder *const pbi, MACROBLOCKD *const xd,
 
         for (row = 0; row < max_blocks_high; row += step)
           for (col = 0; col < max_blocks_wide; col += step)
-            eobtotal += reconstruct_inter_block(xd, r, mbmi, plane, row, col,
+            eobtotal += reconstruct_inter_block(xd, pbi->token_tab, tok, mbmi, plane, row, col,
                                                 tx_size);
       }
 
@@ -916,7 +920,8 @@ static PARTITION_TYPE read_partition(MACROBLOCKD *xd, int mi_row, int mi_col,
 // TODO(slavarnway): eliminate bsize and subsize in future commits
 static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
                              int mi_row, int mi_col,
-                             vpx_reader* r, BLOCK_SIZE bsize, int n4x4_l2) {
+                             vpx_reader* r, struct AnsDecoder *const tok,
+                             BLOCK_SIZE bsize, int n4x4_l2) {
   VP10_COMMON *const cm = &pbi->common;
   const int n8x8_l2 = n4x4_l2 - 1;
   const int num_8x8_wh = 1 << n8x8_l2;
@@ -936,29 +941,29 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
     // calculate bmode block dimensions (log 2)
     xd->bmode_blocks_wl = 1 >> !!(partition & PARTITION_VERT);
     xd->bmode_blocks_hl = 1 >> !!(partition & PARTITION_HORZ);
-    decode_block(pbi, xd, mi_row, mi_col, r, subsize, 1, 1);
+    decode_block(pbi, xd, mi_row, mi_col, r, tok, subsize, 1, 1);
   } else {
     switch (partition) {
       case PARTITION_NONE:
-        decode_block(pbi, xd, mi_row, mi_col, r, subsize, n4x4_l2, n4x4_l2);
+        decode_block(pbi, xd, mi_row, mi_col, r, tok, subsize, n4x4_l2, n4x4_l2);
         break;
       case PARTITION_HORZ:
-        decode_block(pbi, xd, mi_row, mi_col, r, subsize, n4x4_l2, n8x8_l2);
+        decode_block(pbi, xd, mi_row, mi_col, r, tok, subsize, n4x4_l2, n8x8_l2);
         if (has_rows)
-          decode_block(pbi, xd, mi_row + hbs, mi_col, r, subsize, n4x4_l2,
+          decode_block(pbi, xd, mi_row + hbs, mi_col, r, tok, subsize, n4x4_l2,
                        n8x8_l2);
         break;
       case PARTITION_VERT:
-        decode_block(pbi, xd, mi_row, mi_col, r, subsize, n8x8_l2, n4x4_l2);
+        decode_block(pbi, xd, mi_row, mi_col, r, tok, subsize, n8x8_l2, n4x4_l2);
         if (has_cols)
-          decode_block(pbi, xd, mi_row, mi_col + hbs, r, subsize, n8x8_l2,
+          decode_block(pbi, xd, mi_row, mi_col + hbs, r, tok, subsize, n8x8_l2,
                        n4x4_l2);
         break;
       case PARTITION_SPLIT:
-        decode_partition(pbi, xd, mi_row, mi_col, r, subsize, n8x8_l2);
+        decode_partition(pbi, xd, mi_row, mi_col, r, tok, subsize, n8x8_l2);
-        decode_partition(pbi, xd, mi_row, mi_col + hbs, r, subsize, n8x8_l2);
+        decode_partition(pbi, xd, mi_row, mi_col + hbs, r, tok, subsize, n8x8_l2);
-        decode_partition(pbi, xd, mi_row + hbs, mi_col, r, subsize, n8x8_l2);
+        decode_partition(pbi, xd, mi_row + hbs, mi_col, r, tok, subsize, n8x8_l2);
-        decode_partition(pbi, xd, mi_row + hbs, mi_col + hbs, r, subsize,
+        decode_partition(pbi, xd, mi_row + hbs, mi_col + hbs, r, tok, subsize,
                          n8x8_l2);
         break;
       default:
@@ -972,9 +977,9 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
     dec_update_partition_context(xd, mi_row, mi_col, subsize, num_8x8_wh);
 }
 
-static void setup_token_decoder(const uint8_t *data,
+static void setup_bool_decoder(const uint8_t *data,
                                const uint8_t *data_end,
-                                size_t read_size,
+                               const size_t read_size,
                                struct vpx_internal_error_info *error_info,
                                vpx_reader *r,
                                vpx_decrypt_cb decrypt_cb,
@@ -991,6 +996,27 @@ static void setup_token_decoder(const uint8_t *data,
                        "Failed to allocate bool decoder %d", 1);
 }
 
+static void setup_token_decoder(const uint8_t *data,
+                                const uint8_t *data_end,
+                                const size_t read_size,
+                                struct vpx_internal_error_info *error_info,
+                                struct AnsDecoder *const ans,
+                                vpx_decrypt_cb decrypt_cb,
+                                void *decrypt_state) {
+  (void) decrypt_cb;
+  (void) decrypt_state;
+  // Validate the calculated partition length. If the buffer
+  // described by the partition can't be fully read, then restrict
+  // it to the portion that can be (for EC mode) or throw an error.
+  if (!read_is_valid(data, read_size, data_end))
+    vpx_internal_error(error_info, VPX_CODEC_CORRUPT_FRAME,
+                       "Truncated packet or corrupt tile length");
+
+  if (ans_read_init(ans, data, (int)read_size))  //FIXME
+    vpx_internal_error(error_info, VPX_CODEC_MEM_ERROR,
+                       "Failed to allocate token decoder %d", 1);
+}
+
 static void read_coef_probs_common(vp10_coeff_probs_model *coef_probs,
                                    vpx_reader *r) {
   int i, j, k, l, m;
@@ -1449,6 +1475,8 @@ static const uint8_t *decode_tiles(VP10Decoder *pbi,
   for (tile_row = 0; tile_row < tile_rows; ++tile_row) {
     for (tile_col = 0; tile_col < tile_cols; ++tile_col) {
       const TileBuffer *const buf = &tile_buffers[tile_row][tile_col];
+      size_t token_offset;
+
       tile_data = pbi->tile_data + tile_cols * tile_row + tile_col;
       tile_data->cm = cm;
       tile_data->xd = pbi->mb;
@@ -1457,9 +1485,20 @@ static const uint8_t *decode_tiles(VP10Decoder *pbi,
                              NULL : &cm->counts;
       vp10_zero(tile_data->dqcoeff);
       vp10_tile_init(&tile_data->xd.tile, tile_data->cm, tile_row, tile_col);
-      setup_token_decoder(buf->data, data_end, buf->size, &cm->error,
+      if (buf->size < 4 || !read_is_valid(buf->data, buf->size, data_end))
+        vpx_internal_error(&cm->error, VPX_CODEC_CORRUPT_FRAME,
+                           "Truncated packet or corrupt tile length");
+      token_offset = mem_get_be32(buf->data);
+      if (token_offset > buf->size - 4)
+        vpx_internal_error(&cm->error, VPX_CODEC_CORRUPT_FRAME,
+                           "Truncated packet or corrupt tile length");
+      setup_bool_decoder(buf->data + 4, data_end, token_offset, &cm->error,
                          &tile_data->bit_reader, pbi->decrypt_cb,
                          pbi->decrypt_state);
+      setup_token_decoder(buf->data + (4 + token_offset), data_end,
+                          buf->size - (4 + token_offset), &cm->error,
+                          &tile_data->token_ans, pbi->decrypt_cb,
+                          pbi->decrypt_state);
       vp10_init_macroblockd(cm, &tile_data->xd, tile_data->dqcoeff);
     }
   }
@@ -1479,7 +1518,8 @@ static const uint8_t *decode_tiles(VP10Decoder *pbi,
         for (mi_col = tile.mi_col_start; mi_col < tile.mi_col_end;
              mi_col += MI_BLOCK_SIZE) {
           decode_partition(pbi, &tile_data->xd, mi_row,
-                           mi_col, &tile_data->bit_reader, BLOCK_64X64, 4);
+                           mi_col, &tile_data->bit_reader,
+                           &tile_data->token_ans, BLOCK_64X64, 4);
         }
         pbi->mb.corrupted |= tile_data->xd.corrupted;
         if (pbi->mb.corrupted)
@@ -1529,7 +1569,8 @@ static const uint8_t *decode_tiles(VP10Decoder *pbi,
 
   if (pbi->frame_parallel_decode)
     vp10_frameworker_broadcast(pbi->cur_buf, INT_MAX);
-  return vpx_reader_find_end(&tile_data->bit_reader);
+  //return vpx_reader_find_end(&tile_data->token_ans);
+  return data_end;
 }
 
 static int tile_worker_hook(TileWorkerData *const tile_data,
@@ -1553,6 +1594,7 @@ static int tile_worker_hook(TileWorkerData *const tile_data,
          mi_col += MI_BLOCK_SIZE) {
       decode_partition(tile_data->pbi, &tile_data->xd,
                        mi_row, mi_col, &tile_data->bit_reader,
+                       &tile_data->token_ans,
                        BLOCK_64X64, 4);
     }
   }
@@ -1583,6 +1625,7 @@ static const uint8_t *decode_tiles_mt(VP10Decoder *pbi,
   assert(tile_cols <= (1 << 6));
   assert(tile_rows == 1);
   (void)tile_rows;
+  abort();  // Tile parsing broken
 
   // TODO(jzern): See if we can remove the restriction of passing in max
   // threads to the decoder.
@@ -1678,7 +1721,7 @@ static const uint8_t *decode_tiles_mt(VP10Decoder *pbi,
       vp10_zero(tile_data->dqcoeff);
       vp10_tile_init(tile, cm, 0, buf->col);
       vp10_tile_init(&tile_data->xd.tile, cm, 0, buf->col);
-      setup_token_decoder(buf->data, data_end, buf->size, &cm->error,
+      setup_bool_decoder(buf->data, data_end, buf->size, &cm->error,
                          &tile_data->bit_reader, pbi->decrypt_cb,
                          pbi->decrypt_state);
       vp10_init_macroblockd(cm, &tile_data->xd, tile_data->dqcoeff);

vp10/decoder/decoder.c

@@ -115,6 +115,7 @@ VP10Decoder *vp10_decoder_create(BufferPool *const pool) {
   cm->setup_mi = vp10_dec_setup_mi;
 
   vp10_loop_filter_init(cm);
+  vp10_build_pareto8_dec_tab(vp10_pareto8_token_probs, pbi->token_tab);
 
   cm->error.setjmp = 0;
 

vp10/decoder/decoder.h

@@ -18,6 +18,8 @@
 #include "vpx_scale/yv12config.h"
 #include "vpx_util/vpx_thread.h"
 
+#include "vp10/common/ans.h"
+#include "vp10/common/entropy.h"
 #include "vp10/common/thread_common.h"
 #include "vp10/common/onyxc_int.h"
 #include "vp10/common/ppflags.h"
@@ -31,6 +33,7 @@ extern "C" {
 typedef struct TileData {
   VP10_COMMON *cm;
   vpx_reader bit_reader;
+  struct AnsDecoder token_ans;
   DECLARE_ALIGNED(16, MACROBLOCKD, xd);
   /* dqcoeff are shared by all the planes. So planes must be decoded serially */
   DECLARE_ALIGNED(16, tran_low_t, dqcoeff[32 * 32]);
@@ -39,6 +42,7 @@ typedef struct TileData {
 typedef struct TileWorkerData {
   struct VP10Decoder *pbi;
   vpx_reader bit_reader;
+  struct AnsDecoder token_ans;
   FRAME_COUNTS counts;
   DECLARE_ALIGNED(16, MACROBLOCKD, xd);
   /* dqcoeff are shared by all the planes. So planes must be decoded serially */
@@ -80,6 +84,7 @@ typedef struct VP10Decoder {
   int inv_tile_order;
   int need_resync;  // wait for key/intra-only frame.
   int hold_ref_buf;  // hold the reference buffer.
+  struct rans_dec_sym token_tab[COEFF_PROB_MODELS - 1][256];
 } VP10Decoder;
 
 int vp10_receive_compressed_data(struct VP10Decoder *pbi,

vp10/decoder/detokenize.c

@@ -11,6 +11,7 @@
 #include "vpx_mem/vpx_mem.h"
 #include "vpx_ports/mem.h"
 
+#include "vp10/common/ans.h"
 #include "vp10/common/blockd.h"
 #include "vp10/common/common.h"
 #include "vp10/common/entropy.h"
@@ -38,18 +39,32 @@
        ++coef_counts[band][ctx][token];                     \
   } while (0)
 
-static INLINE int read_coeff(const vpx_prob *probs, int n, vpx_reader *r) {
+
+static INLINE int rabs_read_tree(struct AnsDecoder *const ans,
+                                 const vpx_tree_index *const tree,
+                                 const vpx_prob *const probs) {
+  vpx_tree_index i = 0;
+
+  while ((i = tree[i + rabs_read(ans, probs[i >> 1])]) > 0)
+    continue;
+
+  return -i;
+}
+
+static INLINE int read_coeff(const vpx_prob *const probs, int n,
+                             struct AnsDecoder *const ans) {
   int i, val = 0;
   for (i = 0; i < n; ++i)
-    val = (val << 1) | vpx_read(r, probs[i]);
+    val = (val << 1) | rabs_read(ans, probs[i]);
   return val;
 }
 
-static int decode_coefs(const MACROBLOCKD *xd,
+static int decode_coefs(const MACROBLOCKD *const xd,
+                        const struct rans_dec_sym (*const token_tab)[256],
                         PLANE_TYPE type,
                         tran_low_t *dqcoeff, TX_SIZE tx_size, const int16_t *dq,
                         int ctx, const int16_t *scan, const int16_t *nb,
-                        vpx_reader *r) {
+                        struct AnsDecoder *const ans) {
   FRAME_COUNTS *counts = xd->counts;
   const int max_eob = 16 << (tx_size << 1);
   const FRAME_CONTEXT *const fc = xd->fc;
@@ -117,12 +132,12 @@ static int decode_coefs(const MACROBLOCKD *xd,
     prob = coef_probs[band][ctx];
     if (counts)
       ++eob_branch_count[band][ctx];
-    if (!vpx_read(r, prob[EOB_CONTEXT_NODE])) {
+    if (!rabs_read(ans, prob[EOB_CONTEXT_NODE])) {
       INCREMENT_COUNT(EOB_MODEL_TOKEN);
       break;
     }
 
-    while (!vpx_read(r, prob[ZERO_CONTEXT_NODE])) {
+    while (!rabs_read(ans, prob[ZERO_CONTEXT_NODE])) {
       INCREMENT_COUNT(ZERO_TOKEN);
       dqv = dq[1];
       token_cache[scan[c]] = 0;
@@ -134,67 +149,62 @@ static int decode_coefs(const MACROBLOCKD *xd,
       prob = coef_probs[band][ctx];
     }
 
-    if (!vpx_read(r, prob[ONE_CONTEXT_NODE])) {
+    token = ONE_TOKEN +
-      INCREMENT_COUNT(ONE_TOKEN);
+        rans_stream_decode(ans, token_tab[prob[PIVOT_NODE] - 1]);
-      token = ONE_TOKEN;
+    INCREMENT_COUNT(ONE_TOKEN + (token > ONE_TOKEN));
-      val = 1;
-    } else {
-      INCREMENT_COUNT(TWO_TOKEN);
-      token = vpx_read_tree(r, vp10_coef_con_tree,
-                            vp10_pareto8_full[prob[PIVOT_NODE] - 1]);
     switch (token) {
+      case ONE_TOKEN:
       case TWO_TOKEN:
       case THREE_TOKEN:
       case FOUR_TOKEN:
         val = token;
         break;
       case CATEGORY1_TOKEN:
-          val = CAT1_MIN_VAL + read_coeff(cat1_prob, 1, r);
+        val = CAT1_MIN_VAL + read_coeff(cat1_prob, 1, ans);
         break;
       case CATEGORY2_TOKEN:
-          val = CAT2_MIN_VAL + read_coeff(cat2_prob, 2, r);
+        val = CAT2_MIN_VAL + read_coeff(cat2_prob, 2, ans);
         break;
       case CATEGORY3_TOKEN:
-          val = CAT3_MIN_VAL + read_coeff(cat3_prob, 3, r);
+        val = CAT3_MIN_VAL + read_coeff(cat3_prob, 3, ans);
         break;
       case CATEGORY4_TOKEN:
-          val = CAT4_MIN_VAL + read_coeff(cat4_prob, 4, r);
+        val = CAT4_MIN_VAL + read_coeff(cat4_prob, 4, ans);
         break;
       case CATEGORY5_TOKEN:
-          val = CAT5_MIN_VAL + read_coeff(cat5_prob, 5, r);
+        val = CAT5_MIN_VAL + read_coeff(cat5_prob, 5, ans);
         break;
       case CATEGORY6_TOKEN:
 #if CONFIG_VP9_HIGHBITDEPTH
         switch (xd->bd) {
           case VPX_BITS_8:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
+            val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, ans);
             break;
           case VPX_BITS_10:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 16, r);
+            val = CAT6_MIN_VAL + read_coeff(cat6_prob, 16, ans);
             break;
           case VPX_BITS_12:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 18, r);
+            val = CAT6_MIN_VAL + read_coeff(cat6_prob, 18, ans);
             break;
           default:
             assert(0);
             return -1;
         }
 #else
-          val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
+        val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, ans);
 #endif
         break;
     }
-    }
     v = (val * dqv) >> dq_shift;
 #if CONFIG_COEFFICIENT_RANGE_CHECKING
 #if CONFIG_VP9_HIGHBITDEPTH
-    dqcoeff[scan[c]] = highbd_check_range((vpx_read_bit(r) ? -v : v),
+    dqcoeff[scan[c]] = highbd_check_range((rabs_read(ans, 128) ? -v : v),
                                           xd->bd);
 #else
-    dqcoeff[scan[c]] = check_range(vpx_read_bit(r) ? -v : v);
+    dqcoeff[scan[c]] = check_range(rabs_read(ans, 128) ? -v : v);
 #endif  // CONFIG_VP9_HIGHBITDEPTH
 #else
-    dqcoeff[scan[c]] = vpx_read_bit(r) ? -v : v;
+    dqcoeff[scan[c]] = rabs_read(ans, 128) ? -v : v;
 #endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
     token_cache[scan[c]] = vp10_pt_energy_class[token];
     ++c;
@@ -250,16 +260,18 @@ void dec_set_contexts(const MACROBLOCKD *xd, struct macroblockd_plane *pd,
   }
 }
 
-int vp10_decode_block_tokens(MACROBLOCKD *xd,
+int vp10_decode_block_tokens(MACROBLOCKD *const xd,
+                             const struct rans_dec_sym (*const token_tab)[256],
                              int plane, const scan_order *sc,
                              int x, int y,
-                            TX_SIZE tx_size, vpx_reader *r,
+                             TX_SIZE tx_size,
+                             struct AnsDecoder *const r,
                              int seg_id) {
   struct macroblockd_plane *const pd = &xd->plane[plane];
   const int16_t *const dequant = pd->seg_dequant[seg_id];
   const int ctx = get_entropy_context(tx_size, pd->above_context + x,
                                                pd->left_context + y);
-  const int eob = decode_coefs(xd, pd->plane_type,
+  const int eob = decode_coefs(xd, token_tab, pd->plane_type,
                                pd->dqcoeff, tx_size,
                                dequant, ctx, sc->scan, sc->neighbors, r);
   dec_set_contexts(xd, pd, tx_size, eob > 0, x, y);

vp10/decoder/detokenize.h

@@ -12,18 +12,22 @@
 #ifndef VP10_DECODER_DETOKENIZE_H_
 #define VP10_DECODER_DETOKENIZE_H_
 
-#include "vpx_dsp/bitreader.h"
 #include "vp10/decoder/decoder.h"
+#include "vp10/common/ans.h"
 #include "vp10/common/scan.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int vp10_decode_block_tokens(MACROBLOCKD *xd,
+struct AnsDecoder;
+
+int vp10_decode_block_tokens(MACROBLOCKD *const xd,
+                             const struct rans_dec_sym (*const token_tab)[256],
                              int plane, const scan_order *sc,
                              int x, int y,
-                            TX_SIZE tx_size, vpx_reader *r,
+                             TX_SIZE tx_size,
+                             struct AnsDecoder *const r,
                              int seg_id);
 
 #ifdef __cplusplus

vp10/encoder/bitstream.c

@@ -9,7 +9,6 @@
  */
 
 #include <assert.h>
-#include <stdio.h>
 #include <limits.h>
 
 #include "vpx/vpx_encoder.h"
@@ -193,6 +192,76 @@ static void pack_mb_tokens(vpx_writer *w,
   *tp = p + (p->token == EOSB_TOKEN);
 }
 
+// This function serializes the tokens backwards both in token order and
+// bit order in each token.
+static void pack_mb_tokens_ans(struct AnsCoder *const ans,
+                               const TOKENEXTRA *const start,
+                               const TOKENEXTRA *const stop,
+                               vpx_bit_depth_t bit_depth) {
+  const TOKENEXTRA *p;
+
+  for (p = stop - 1; p >= start; --p) {
+    const int t = p->token;
+    //TODO: remove EOSB_TOKEN
+    if (t != EOSB_TOKEN) {
+    const struct vp10_token *const a = &vp10_coef_encodings[t];
+    int i = 0;
+    int v = a->value;
+    int n = a->len;
+#if CONFIG_VP9_HIGHBITDEPTH
+    const vp10_extra_bit *b;
+    if (bit_depth == VPX_BITS_12)
+      b = &vp10_extra_bits_high12[t];
+    else if (bit_depth == VPX_BITS_10)
+      b = &vp10_extra_bits_high10[t];
+    else
+      b = &vp10_extra_bits[t];
+#else
+    const vp10_extra_bit *const b = &vp10_extra_bits[t];
+    (void) bit_depth;
+#endif  // CONFIG_VP9_HIGHBITDEPTH
+
+    // TODO: Replace this with serializing whole tokens at a time
+
+    // Write extra bits first
+    if (b->base_val) {
+      const int e = p->extra, l = b->len;
+      rabs_write(ans, e & 1, 128);
+      if (l) {
+        vp10_write_tree_r(ans, b->tree, b->prob, e >> 1, l, 0);
+      }
+    }
+
+    /* skip one or two nodes */
+    if (p->skip_eob_node) {
+      n -= p->skip_eob_node;
+      i = 2 * p->skip_eob_node;
+    }
+
+    // If we have a token that's in the constrained set, the coefficient tree
+    // is split into two treed writes.  The first treed write takes care of the
+    // unconstrained nodes.  The second treed write takes care of the
+    // constrained nodes.
+    if (t == EOB_TOKEN || t == ZERO_TOKEN) {
+      vp10_write_tree_r(ans, vp10_coef_tree, p->context_tree, v, n, i);
+    } else {
+      struct rans_sym s;
+      int j;
+      int len = 2 - p->skip_eob_node;  // Write <EOBF>?<ZEROF>
+      int bits = v >> (n - len);
+      const vpx_prob *token_probs =
+          vp10_pareto8_token_probs[p->context_tree[PIVOT_NODE] - 1];
+      s.cum_prob = 0;
+      for (j = ONE_TOKEN; j < t; ++j) {
+        s.cum_prob += token_probs[j - ONE_TOKEN];
+      }
+      s.prob = token_probs[t - ONE_TOKEN];
+      rans_stream_encode(ans, &s);
+      vp10_write_tree_r(ans, vp10_coef_tree, p->context_tree, bits, len, i);
+    }
+  }
+  }
+}
 static void write_segment_id(vpx_writer *w, const struct segmentation *seg,
                              int segment_id) {
   if (seg->enabled && seg->update_map)
@@ -376,8 +445,7 @@ static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd,
 }
 
 static void write_modes_b(VP10_COMP *cpi, const TileInfo *const tile,
-                          vpx_writer *w, TOKENEXTRA **tok,
+                          vpx_writer *w,
-                          const TOKENEXTRA *const tok_end,
                           int mi_row, int mi_col) {
   const VP10_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
@@ -398,9 +466,6 @@ static void write_modes_b(VP10_COMP *cpi, const TileInfo *const tile,
   } else {
     pack_inter_mode_mvs(cpi, m, w);
   }
-
-  assert(*tok < tok_end);
-  pack_mb_tokens(w, tok, tok_end, cm->bit_depth);
 }
 
 static void write_partition(const VP10_COMMON *const cm,
@@ -427,7 +492,6 @@ static void write_partition(const VP10_COMMON *const cm,
 
 static void write_modes_sb(VP10_COMP *cpi,
                            const TileInfo *const tile, vpx_writer *w,
-                           TOKENEXTRA **tok, const TOKENEXTRA *const tok_end,
                            int mi_row, int mi_col, BLOCK_SIZE bsize) {
   const VP10_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
@@ -447,30 +511,27 @@ static void write_modes_sb(VP10_COMP *cpi,
   write_partition(cm, xd, bs, mi_row, mi_col, partition, bsize, w);
   subsize = get_subsize(bsize, partition);
   if (subsize < BLOCK_8X8) {
-    write_modes_b(cpi, tile, w, tok, tok_end, mi_row, mi_col);
+    write_modes_b(cpi, tile, w, mi_row, mi_col);
   } else {
     switch (partition) {
       case PARTITION_NONE:
-        write_modes_b(cpi, tile, w, tok, tok_end, mi_row, mi_col);
+        write_modes_b(cpi, tile, w, mi_row, mi_col);
         break;
       case PARTITION_HORZ:
-        write_modes_b(cpi, tile, w, tok, tok_end, mi_row, mi_col);
+        write_modes_b(cpi, tile, w, mi_row, mi_col);
         if (mi_row + bs < cm->mi_rows)
-          write_modes_b(cpi, tile, w, tok, tok_end, mi_row + bs, mi_col);
+          write_modes_b(cpi, tile, w, mi_row + bs, mi_col);
         break;
       case PARTITION_VERT:
-        write_modes_b(cpi, tile, w, tok, tok_end, mi_row, mi_col);
+        write_modes_b(cpi, tile, w, mi_row, mi_col);
         if (mi_col + bs < cm->mi_cols)
-          write_modes_b(cpi, tile, w, tok, tok_end, mi_row, mi_col + bs);
+          write_modes_b(cpi, tile, w, mi_row, mi_col + bs);
         break;
       case PARTITION_SPLIT:
-        write_modes_sb(cpi, tile, w, tok, tok_end, mi_row, mi_col, subsize);
+        write_modes_sb(cpi, tile, w, mi_row, mi_col, subsize);
-        write_modes_sb(cpi, tile, w, tok, tok_end, mi_row, mi_col + bs,
+        write_modes_sb(cpi, tile, w, mi_row, mi_col + bs, subsize);
-                       subsize);
+        write_modes_sb(cpi, tile, w, mi_row + bs, mi_col, subsize);
-        write_modes_sb(cpi, tile, w, tok, tok_end, mi_row + bs, mi_col,
+        write_modes_sb(cpi, tile, w, mi_row + bs, mi_col + bs, subsize);
-                       subsize);
-        write_modes_sb(cpi, tile, w, tok, tok_end, mi_row + bs, mi_col + bs,
-                       subsize);
         break;
       default:
         assert(0);
@@ -484,8 +545,7 @@ static void write_modes_sb(VP10_COMP *cpi,
 }
 
 static void write_modes(VP10_COMP *cpi,
-                        const TileInfo *const tile, vpx_writer *w,
+                        const TileInfo *const tile, vpx_writer *w) {
-                        TOKENEXTRA **tok, const TOKENEXTRA *const tok_end) {
   const VP10_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
   int mi_row, mi_col;
@@ -497,7 +557,7 @@ static void write_modes(VP10_COMP *cpi,
     vp10_zero(xd->left_seg_context);
     for (mi_col = tile->mi_col_start; mi_col < tile->mi_col_end;
          mi_col += MI_BLOCK_SIZE)
-      write_modes_sb(cpi, tile, w, tok, tok_end, mi_row, mi_col,
+      write_modes_sb(cpi, tile, w, mi_row, mi_col,
                      BLOCK_64X64);
   }
 }
@@ -935,7 +995,8 @@ static int get_refresh_mask(VP10_COMP *cpi) {
 
 static size_t encode_tiles(VP10_COMP *cpi, uint8_t *data_ptr) {
   VP10_COMMON *const cm = &cpi->common;
-  vpx_writer residual_bc;
+  vpx_writer mode_bc;
+  struct AnsCoder token_ans;
   int tile_row, tile_col;
   TOKENEXTRA *tok_end;
   size_t total_size = 0;
@@ -948,27 +1009,32 @@ static size_t encode_tiles(VP10_COMP *cpi, uint8_t *data_ptr) {
   for (tile_row = 0; tile_row < tile_rows; tile_row++) {
     for (tile_col = 0; tile_col < tile_cols; tile_col++) {
       int tile_idx = tile_row * tile_cols + tile_col;
+      int put_tile_size = tile_col < tile_cols - 1 || tile_row < tile_rows - 1;
+      uint8_t *const mode_data_start =
+          data_ptr + total_size + (put_tile_size ? 8 : 4);
+      int token_section_size;
       TOKENEXTRA *tok = cpi->tile_tok[tile_row][tile_col];
 
       tok_end = cpi->tile_tok[tile_row][tile_col] +
           cpi->tok_count[tile_row][tile_col];
 
-      if (tile_col < tile_cols - 1 || tile_row < tile_rows - 1)
+      vpx_start_encode(&mode_bc, mode_data_start);
-        vpx_start_encode(&residual_bc, data_ptr + total_size + 4);
-      else
-        vpx_start_encode(&residual_bc, data_ptr + total_size);
 
-      write_modes(cpi, &cpi->tile_data[tile_idx].tile_info,
+      write_modes(cpi, &cpi->tile_data[tile_idx].tile_info, &mode_bc);
-                  &residual_bc, &tok, tok_end);
+      vpx_stop_encode(&mode_bc);
-      assert(tok == tok_end);
+      ans_write_init(&token_ans, mode_data_start + mode_bc.pos);
-      vpx_stop_encode(&residual_bc);
+      pack_mb_tokens_ans(&token_ans, tok, tok_end, cm->bit_depth);
-      if (tile_col < tile_cols - 1 || tile_row < tile_rows - 1) {
+      token_section_size = ans_write_end(&token_ans);
+      if (put_tile_size) {
         // size of this tile
-        mem_put_be32(data_ptr + total_size, residual_bc.pos);
+        mem_put_be32(data_ptr + total_size,
+                     4 + mode_bc.pos + token_section_size);
         total_size += 4;
       }
+      // put where the token section begins
+      mem_put_be32(data_ptr + total_size, mode_bc.pos);
 
-      total_size += residual_bc.pos;
+      total_size += 4 + mode_bc.pos + token_section_size;
     }
   }
 

vp10/encoder/treewriter.h

@@ -12,6 +12,7 @@
 #define VP10_ENCODER_TREEWRITER_H_
 
 #include "vpx_dsp/bitwriter.h"
+#include "vp10/common/ans.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -28,6 +29,29 @@ struct vp10_token {
 
 void vp10_tokens_from_tree(struct vp10_token*, const vpx_tree_index *);
 
+// TODO: CHECK MAX REVERSIBLE TREE SIZE, security concerns
+#define VP10_TOKEN_SCRATCH_LEN 32
+static INLINE void vp10_write_tree_r(struct AnsCoder *const ans,
+                                     const vpx_tree_index *const tree,
+                                     const vpx_prob *const probs,
+                                     int bits, int len,
+                                     vpx_tree_index tidx) {
+  int i;
+  struct { uint8_t bit; vpx_prob prob; } scratch[VP10_TOKEN_SCRATCH_LEN];
+  // assert(len < VP10_TOKEN_SCRATCH_LEN);
+
+  for (i = len - 1; i >= 0; --i) {
+    const int bit = (bits >> i) & 1;
+    scratch[i].bit = bit;
+    scratch[i].prob = probs[tidx >> 1];
+    tidx = tree[tidx + bit];
+  }
+  for (i = 0; i < len; ++i) {
+    rabs_write(ans, scratch[i].bit, scratch[i].prob);
+  }
+}
+#undef VP10_TOKEN_SCRATCH_LEN
+
 static INLINE void vp10_write_tree(vpx_writer *w, const vpx_tree_index *tree,
                                   const vpx_prob *probs, int bits, int len,
                                   vpx_tree_index i) {