diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
index e32240157..895928b90 100644
--- a/vp9/common/vp9_alloccommon.c
+++ b/vp9/common/vp9_alloccommon.c
@@ -88,9 +88,8 @@ int vp9_alloc_frame_buffers(VP9_COMMON *oci, int width, int height) {
   }
 
   oci->new_fb_idx = 0;
-  oci->lst_fb_idx = 1;
-  oci->gld_fb_idx = 2;
-  oci->alt_fb_idx = 3;
+  for (i = 0; i < 3; i++)
+    oci->active_ref_idx[i] = i + 1;
 
   oci->fb_idx_ref_cnt[0] = 1;
   oci->fb_idx_ref_cnt[1] = 1;
diff --git a/vp9/common/vp9_onyxc_int.h b/vp9/common/vp9_onyxc_int.h
index 8fbfafdc2..33ac3f236 100644
--- a/vp9/common/vp9_onyxc_int.h
+++ b/vp9/common/vp9_onyxc_int.h
@@ -37,7 +37,8 @@ void vp9_initialize_common(void);
 
 #define QINDEX_RANGE (MAXQ + 1)
 
-#define NUM_YV12_BUFFERS 4
+#define NUM_REF_FRAMES 3
+#define NUM_YV12_BUFFERS (NUM_REF_FRAMES + 1)
 
 #define COMP_PRED_CONTEXTS   2
 
@@ -142,8 +143,9 @@ typedef struct VP9Common {
   YV12_BUFFER_CONFIG *frame_to_show;
 
   YV12_BUFFER_CONFIG yv12_fb[NUM_YV12_BUFFERS];
-  int fb_idx_ref_cnt[NUM_YV12_BUFFERS];
-  int new_fb_idx, lst_fb_idx, gld_fb_idx, alt_fb_idx;
+  int fb_idx_ref_cnt[NUM_YV12_BUFFERS]; /* reference counts */
+  int active_ref_idx[3]; /* each frame can reference 3 buffers */
+  int new_fb_idx;
 
   YV12_BUFFER_CONFIG post_proc_buffer;
   YV12_BUFFER_CONFIG temp_scale_frame;
@@ -202,10 +204,6 @@ typedef struct VP9Common {
   int last_sharpness_level;
   int sharpness_level;
 
-  int refresh_last_frame;       /* Two state 0 = NO, 1 = YES */
-  int refresh_golden_frame;     /* Two state 0 = NO, 1 = YES */
-  int refresh_alt_ref_frame;     /* Two state 0 = NO, 1 = YES */
-
   int refresh_entropy_probs;    /* Two state 0 = NO, 1 = YES */
 
   int ref_frame_sign_bias[MAX_REF_FRAMES];    /* Two state 0, 1 */
diff --git a/vp9/decoder/vp9_decodemv.c b/vp9/decoder/vp9_decodemv.c
index c6c3d1576..eef9e6e60 100644
--- a/vp9/decoder/vp9_decodemv.c
+++ b/vp9/decoder/vp9_decodemv.c
@@ -730,12 +730,7 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
       int ref_fb_idx;
 
       /* Select the appropriate reference frame for this MB */
-      if (ref_frame == LAST_FRAME)
-        ref_fb_idx = cm->lst_fb_idx;
-      else if (ref_frame == GOLDEN_FRAME)
-        ref_fb_idx = cm->gld_fb_idx;
-      else
-        ref_fb_idx = cm->alt_fb_idx;
+      ref_fb_idx = cm->active_ref_idx[ref_frame - 1];
 
       recon_y_stride = cm->yv12_fb[ref_fb_idx].y_stride  ;
       recon_uv_stride = cm->yv12_fb[ref_fb_idx].uv_stride;
@@ -818,13 +813,7 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
       if (mbmi->second_ref_frame > 0) {
         int second_ref_fb_idx;
         /* Select the appropriate reference frame for this MB */
-        if (mbmi->second_ref_frame == LAST_FRAME)
-          second_ref_fb_idx = cm->lst_fb_idx;
-        else if (mbmi->second_ref_frame ==
-          GOLDEN_FRAME)
-          second_ref_fb_idx = cm->gld_fb_idx;
-        else
-          second_ref_fb_idx = cm->alt_fb_idx;
+        second_ref_fb_idx = cm->active_ref_idx[mbmi->second_ref_frame - 1];
 
         xd->second_pre.y_buffer =
           cm->yv12_fb[second_ref_fb_idx].y_buffer + recon_yoffset;
diff --git a/vp9/decoder/vp9_decodframe.c b/vp9/decoder/vp9_decodframe.c
index 648a84ead..9e77d202e 100644
--- a/vp9/decoder/vp9_decodframe.c
+++ b/vp9/decoder/vp9_decodframe.c
@@ -1091,12 +1091,7 @@ static void set_refs(VP9D_COMP *pbi, int block_size,
     int ref_fb_idx, ref_yoffset, ref_uvoffset, ref_y_stride, ref_uv_stride;
 
     /* Select the appropriate reference frame for this MB */
-    if (mbmi->ref_frame == LAST_FRAME)
-      ref_fb_idx = cm->lst_fb_idx;
-    else if (mbmi->ref_frame == GOLDEN_FRAME)
-      ref_fb_idx = cm->gld_fb_idx;
-    else
-      ref_fb_idx = cm->alt_fb_idx;
+    ref_fb_idx = cm->active_ref_idx[mbmi->ref_frame - 1];
 
     ref_y_stride = cm->yv12_fb[ref_fb_idx].y_stride;
     ref_yoffset = mb_row * 16 * ref_y_stride + 16 * mb_col;
@@ -1113,12 +1108,7 @@ static void set_refs(VP9D_COMP *pbi, int block_size,
       int second_ref_fb_idx;
 
       /* Select the appropriate reference frame for this MB */
-      if (mbmi->second_ref_frame == LAST_FRAME)
-        second_ref_fb_idx = cm->lst_fb_idx;
-      else if (mbmi->second_ref_frame == GOLDEN_FRAME)
-        second_ref_fb_idx = cm->gld_fb_idx;
-      else
-        second_ref_fb_idx = cm->alt_fb_idx;
+      second_ref_fb_idx = cm->active_ref_idx[mbmi->second_ref_frame - 1];
 
       xd->second_pre.y_buffer =
           cm->yv12_fb[second_ref_fb_idx].y_buffer + ref_yoffset;
@@ -1288,8 +1278,7 @@ static void init_frame(VP9D_COMP *pbi) {
     vpx_memset(xd->mode_lf_deltas, 0, sizeof(xd->mode_lf_deltas));
 
     /* All buffers are implicitly updated on key frames. */
-    pc->refresh_golden_frame = 1;
-    pc->refresh_alt_ref_frame = 1;
+    pbi->refresh_frame_flags = (1 << NUM_REF_FRAMES) - 1;
 
     /* Note that Golden and Altref modes cannot be used on a key frame so
      * ref_frame_sign_bias[] is undefined and meaningless
@@ -1645,10 +1634,10 @@ int vp9_decode_frame(VP9D_COMP *pbi, const unsigned char **p_data_end) {
    */
   if (pc->frame_type != KEY_FRAME) {
     /* Should the GF or ARF be updated from the current frame */
-    pc->refresh_golden_frame = vp9_read_bit(&header_bc);
-    pc->refresh_alt_ref_frame = vp9_read_bit(&header_bc);
+    pbi->refresh_frame_flags = vp9_read_literal(&header_bc, NUM_REF_FRAMES);
 
-    if (pc->refresh_alt_ref_frame) {
+    /* TODO(jkoleszar): What's the right thing to do here with more refs? */
+    if (pbi->refresh_frame_flags & 0x4) {
       vpx_memcpy(&pc->fc, &pc->lfc_a, sizeof(pc->fc));
     } else {
       vpx_memcpy(&pc->fc, &pc->lfc, sizeof(pc->fc));
@@ -1677,9 +1666,6 @@ int vp9_decode_frame(VP9D_COMP *pbi, const unsigned char **p_data_end) {
     vpx_memcpy(&pc->lfc, &pc->fc, sizeof(pc->fc));
   }
 
-  pc->refresh_last_frame = (pc->frame_type == KEY_FRAME)
-                           || vp9_read_bit(&header_bc);
-
   // Read inter mode probability context updates
   if (pc->frame_type != KEY_FRAME) {
     int i, j;
@@ -1722,12 +1708,10 @@ int vp9_decode_frame(VP9D_COMP *pbi, const unsigned char **p_data_end) {
 
   if (0) {
     FILE *z = fopen("decodestats.stt", "a");
-    fprintf(z, "%6d F:%d,G:%d,A:%d,L:%d,Q:%d\n",
+    fprintf(z, "%6d F:%d,R:%d,Q:%d\n",
             pc->current_video_frame,
             pc->frame_type,
-            pc->refresh_golden_frame,
-            pc->refresh_alt_ref_frame,
-            pc->refresh_last_frame,
+            pbi->refresh_frame_flags,
             pc->base_qindex);
     fclose(z);
   }
@@ -1779,8 +1763,11 @@ int vp9_decode_frame(VP9D_COMP *pbi, const unsigned char **p_data_end) {
 
   read_coef_probs(pbi, &header_bc);
 
-  vpx_memcpy(&xd->pre, &pc->yv12_fb[pc->lst_fb_idx], sizeof(YV12_BUFFER_CONFIG));
-  vpx_memcpy(&xd->dst, &pc->yv12_fb[pc->new_fb_idx], sizeof(YV12_BUFFER_CONFIG));
+  /* Initialize xd pointers. Any reference should do for xd->pre, so use 0. */
+  vpx_memcpy(&xd->pre, &pc->yv12_fb[pc->active_ref_idx[0]],
+             sizeof(YV12_BUFFER_CONFIG));
+  vpx_memcpy(&xd->dst, &pc->yv12_fb[pc->new_fb_idx],
+             sizeof(YV12_BUFFER_CONFIG));
 
   // Create the segmentation map structure and set to 0
   if (!pc->last_frame_seg_map)
@@ -1832,13 +1819,9 @@ int vp9_decode_frame(VP9D_COMP *pbi, const unsigned char **p_data_end) {
     vp9_update_mode_context(&pbi->common);
   }
 
-  /* If this was a kf or Gf note the Q used */
-  if ((pc->frame_type == KEY_FRAME) ||
-      pc->refresh_golden_frame || pc->refresh_alt_ref_frame) {
-    pc->last_kf_gf_q = pc->base_qindex;
-  }
   if (pc->refresh_entropy_probs) {
-    if (pc->refresh_alt_ref_frame)
+    /* TODO(jkoleszar): What's the right thing to do here with more refs? */
+    if (pbi->refresh_frame_flags & 0x4)
       vpx_memcpy(&pc->lfc_a, &pc->fc, sizeof(pc->fc));
     else
       vpx_memcpy(&pc->lfc, &pc->fc, sizeof(pc->fc));
diff --git a/vp9/decoder/vp9_onyxd_if.c b/vp9/decoder/vp9_onyxd_if.c
index 0c38cfdaf..1ab6fb23c 100644
--- a/vp9/decoder/vp9_onyxd_if.c
+++ b/vp9/decoder/vp9_onyxd_if.c
@@ -165,12 +165,17 @@ vpx_codec_err_t vp9_get_reference_dec(VP9D_PTR ptr, VP9_REFFRAME ref_frame_flag,
   VP9_COMMON *cm = &pbi->common;
   int ref_fb_idx;
 
+  /* TODO(jkoleszar): The decoder doesn't have any real knowledge of what the
+   * encoder is using the frame buffers for. This is just a stub to keep the
+   * vpxenc --test-decode functionality working, and will be replaced in a
+   * later commit that adds VP9-specific controls for this functionality.
+   */
   if (ref_frame_flag == VP9_LAST_FLAG)
-    ref_fb_idx = cm->lst_fb_idx;
+    ref_fb_idx = pbi->common.active_ref_idx[0];
   else if (ref_frame_flag == VP9_GOLD_FLAG)
-    ref_fb_idx = cm->gld_fb_idx;
+    ref_fb_idx = pbi->common.active_ref_idx[1];
   else if (ref_frame_flag == VP9_ALT_FLAG)
-    ref_fb_idx = cm->alt_fb_idx;
+    ref_fb_idx = pbi->common.active_ref_idx[2];
   else {
     vpx_internal_error(&pbi->common.error, VPX_CODEC_ERROR,
                        "Invalid reference frame");
@@ -197,12 +202,17 @@ vpx_codec_err_t vp9_set_reference_dec(VP9D_PTR ptr, VP9_REFFRAME ref_frame_flag,
   int *ref_fb_ptr = NULL;
   int free_fb;
 
+  /* TODO(jkoleszar): The decoder doesn't have any real knowledge of what the
+   * encoder is using the frame buffers for. This is just a stub to keep the
+   * vpxenc --test-decode functionality working, and will be replaced in a
+   * later commit that adds VP9-specific controls for this functionality.
+   */
   if (ref_frame_flag == VP9_LAST_FLAG)
-    ref_fb_ptr = &cm->lst_fb_idx;
+    ref_fb_ptr = &pbi->common.active_ref_idx[0];
   else if (ref_frame_flag == VP9_GOLD_FLAG)
-    ref_fb_ptr = &cm->gld_fb_idx;
+    ref_fb_ptr = &pbi->common.active_ref_idx[1];
   else if (ref_frame_flag == VP9_ALT_FLAG)
-    ref_fb_ptr = &cm->alt_fb_idx;
+    ref_fb_ptr = &pbi->common.active_ref_idx[2];
   else {
     vpx_internal_error(&pbi->common.error, VPX_CODEC_ERROR,
                        "Invalid reference frame");
@@ -231,29 +241,21 @@ vpx_codec_err_t vp9_set_reference_dec(VP9D_PTR ptr, VP9_REFFRAME ref_frame_flag,
 }
 
 
-/* If any buffer copy / swapping is signalled it should be done here. */
-static int swap_frame_buffers(VP9_COMMON *cm) {
-  int err = 0;
+/* If any buffer updating is signalled it should be done here. */
+static void swap_frame_buffers(VP9D_COMP *pbi) {
+  int ref_index = 0, mask;
 
-  /* The alternate reference frame or golden frame can be updated
-   * using the new frame.
-   */
-  if (cm->refresh_golden_frame)
-    ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->gld_fb_idx, cm->new_fb_idx);
+  for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) {
+    if (mask & 1) {
+      ref_cnt_fb(pbi->common.fb_idx_ref_cnt,
+                 &pbi->common.active_ref_idx[ref_index],
+                 pbi->common.new_fb_idx);
+    }
+    ++ref_index;
+  }
 
-  if (cm->refresh_alt_ref_frame)
-    ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->alt_fb_idx, cm->new_fb_idx);
-
-  if (cm->refresh_last_frame) {
-    ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->lst_fb_idx, cm->new_fb_idx);
-
-    cm->frame_to_show = &cm->yv12_fb[cm->lst_fb_idx];
-  } else
-    cm->frame_to_show = &cm->yv12_fb[cm->new_fb_idx];
-
-  cm->fb_idx_ref_cnt[cm->new_fb_idx]--;
-
-  return err;
+  pbi->common.frame_to_show = &pbi->common.yv12_fb[pbi->common.new_fb_idx];
+  pbi->common.fb_idx_ref_cnt[pbi->common.new_fb_idx]--;
 }
 
 int vp9_receive_compressed_data(VP9D_PTR ptr, unsigned long size,
@@ -281,8 +283,12 @@ int vp9_receive_compressed_data(VP9D_PTR ptr, unsigned long size,
      * We do not know if the missing frame(s) was supposed to update
      * any of the reference buffers, but we act conservative and
      * mark only the last buffer as corrupted.
+     *
+     * TODO(jkoleszar): Error concealment is undefined and non-normative
+     * at this point, but if it becomes so, [0] may not always be the correct
+     * thing to do here.
      */
-    cm->yv12_fb[cm->lst_fb_idx].corrupted = 1;
+    cm->yv12_fb[cm->active_ref_idx[0]].corrupted = 1;
   }
 
   cm->new_fb_idx = get_free_fb(cm);
@@ -293,8 +299,12 @@ int vp9_receive_compressed_data(VP9D_PTR ptr, unsigned long size,
     /* We do not know if the missing frame(s) was supposed to update
      * any of the reference buffers, but we act conservative and
      * mark only the last buffer as corrupted.
+     *
+     * TODO(jkoleszar): Error concealment is undefined and non-normative
+     * at this point, but if it becomes so, [0] may not always be the correct
+     * thing to do here.
      */
-    cm->yv12_fb[cm->lst_fb_idx].corrupted = 1;
+    cm->yv12_fb[cm->active_ref_idx[0]].corrupted = 1;
 
     if (cm->fb_idx_ref_cnt[cm->new_fb_idx] > 0)
       cm->fb_idx_ref_cnt[cm->new_fb_idx]--;
@@ -314,11 +324,7 @@ int vp9_receive_compressed_data(VP9D_PTR ptr, unsigned long size,
   }
 
   {
-    if (swap_frame_buffers(cm)) {
-      pbi->common.error.error_code = VPX_CODEC_ERROR;
-      pbi->common.error.setjmp = 0;
-      return -1;
-    }
+    swap_frame_buffers(pbi);
 
 #if WRITE_RECON_BUFFER == 2
     if (cm->show_frame)
diff --git a/vp9/decoder/vp9_onyxd_int.h b/vp9/decoder/vp9_onyxd_int.h
index 64975468d..e04b9f5e4 100644
--- a/vp9/decoder/vp9_onyxd_int.h
+++ b/vp9/decoder/vp9_onyxd_int.h
@@ -76,6 +76,7 @@ typedef struct VP9Decompressor {
   vp9_dequant_idct_add_y_block_fn_t    idct_add_y_block;
   vp9_dequant_idct_add_uv_block_fn_t   idct_add_uv_block;
 
+  int refresh_frame_flags;
   vp9_prob prob_skip_false;
 
   int decoded_key_frame;
diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index 148975205..af8fa495f 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -1773,8 +1773,9 @@ void vp9_pack_bitstream(VP9_COMP *cpi, unsigned char *dest,
   // When there is a key frame all reference buffers are updated using the new key frame
   if (pc->frame_type != KEY_FRAME) {
     // Should the GF or ARF be updated using the transmitted frame or buffer
-    vp9_write_bit(&header_bc, pc->refresh_golden_frame);
-    vp9_write_bit(&header_bc, pc->refresh_alt_ref_frame);
+    vp9_write_bit(&header_bc, cpi->refresh_alt_ref_frame);
+    vp9_write_bit(&header_bc, cpi->refresh_golden_frame);
+    vp9_write_bit(&header_bc, cpi->refresh_last_frame);
 
     // Indicate reference frame sign bias for Golden and ARF frames (always 0 for last frame buffer)
     vp9_write_bit(&header_bc, pc->ref_frame_sign_bias[GOLDEN_FRAME]);
@@ -1820,9 +1821,6 @@ void vp9_pack_bitstream(VP9_COMP *cpi, unsigned char *dest,
 
   vp9_write_bit(&header_bc, pc->refresh_entropy_probs);
 
-  if (pc->frame_type != KEY_FRAME)
-    vp9_write_bit(&header_bc, pc->refresh_last_frame);
-
 #ifdef ENTROPY_STATS
   if (pc->frame_type == INTER_FRAME)
     active_section = 0;
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 3f5133062..0d33edc06 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -640,7 +640,7 @@ static void set_offsets(VP9_COMP *cpi,
   const int src_uv_stride = x->src.uv_stride;
   const int src_yoffset = 16 * mb_row * src_y_stride + 16 * mb_col;
   const int src_uvoffset = 8 * mb_row * src_uv_stride + 8 * mb_col;
-  const int ref_fb_idx = cm->lst_fb_idx;
+  const int ref_fb_idx = cm->active_ref_idx[cpi->lst_fb_idx];
   const int ref_y_stride = cm->yv12_fb[ref_fb_idx].y_stride;
   const int ref_uv_stride = cm->yv12_fb[ref_fb_idx].uv_stride;
   const int idx_map = mb_row * cm->mb_cols + mb_col;
@@ -1205,7 +1205,7 @@ static void init_encode_frame_mb_context(VP9_COMP *cpi) {
 
   // Copy data over into macro block data structures.
   x->src = *cpi->Source;
-  xd->pre = cm->yv12_fb[cm->lst_fb_idx];
+  xd->pre = cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]];
   xd->dst = cm->yv12_fb[cm->new_fb_idx];
 
   // set up frame for intra coded blocks
@@ -1526,9 +1526,9 @@ void vp9_encode_frame(VP9_COMP *cpi) {
      */
     if (cpi->common.frame_type == KEY_FRAME)
       frame_type = 0;
-    else if (cpi->is_src_frame_alt_ref && cpi->common.refresh_golden_frame)
+    else if (cpi->is_src_frame_alt_ref && cpi->refresh_golden_frame)
       frame_type = 3;
-    else if (cpi->common.refresh_golden_frame || cpi->common.refresh_alt_ref_frame)
+    else if (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)
       frame_type = 1;
     else
       frame_type = 2;
@@ -2086,11 +2086,11 @@ static void encode_macroblock(VP9_COMP *cpi, TOKENEXTRA **t,
     assert(cm->frame_type != KEY_FRAME);
 
     if (mbmi->ref_frame == LAST_FRAME)
-      ref_fb_idx = cpi->common.lst_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->lst_fb_idx];
     else if (mbmi->ref_frame == GOLDEN_FRAME)
-      ref_fb_idx = cpi->common.gld_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->gld_fb_idx];
     else
-      ref_fb_idx = cpi->common.alt_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->alt_fb_idx];
 
     xd->pre.y_buffer = cpi->common.yv12_fb[ref_fb_idx].y_buffer + recon_yoffset;
     xd->pre.u_buffer = cpi->common.yv12_fb[ref_fb_idx].u_buffer + recon_uvoffset;
@@ -2100,11 +2100,11 @@ static void encode_macroblock(VP9_COMP *cpi, TOKENEXTRA **t,
       int second_ref_fb_idx;
 
       if (mbmi->second_ref_frame == LAST_FRAME)
-        second_ref_fb_idx = cpi->common.lst_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->lst_fb_idx];
       else if (mbmi->second_ref_frame == GOLDEN_FRAME)
-        second_ref_fb_idx = cpi->common.gld_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->gld_fb_idx];
       else
-        second_ref_fb_idx = cpi->common.alt_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->alt_fb_idx];
 
       xd->second_pre.y_buffer = cpi->common.yv12_fb[second_ref_fb_idx].y_buffer +
                                 recon_yoffset;
@@ -2326,11 +2326,11 @@ static void encode_superblock32(VP9_COMP *cpi, TOKENEXTRA **t,
     assert(cm->frame_type != KEY_FRAME);
 
     if (xd->mode_info_context->mbmi.ref_frame == LAST_FRAME)
-      ref_fb_idx = cpi->common.lst_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->lst_fb_idx];
     else if (xd->mode_info_context->mbmi.ref_frame == GOLDEN_FRAME)
-      ref_fb_idx = cpi->common.gld_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->gld_fb_idx];
     else
-      ref_fb_idx = cpi->common.alt_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->alt_fb_idx];
 
     xd->pre.y_buffer = cpi->common.yv12_fb[ref_fb_idx].y_buffer + recon_yoffset;
     xd->pre.u_buffer = cpi->common.yv12_fb[ref_fb_idx].u_buffer + recon_uvoffset;
@@ -2340,11 +2340,11 @@ static void encode_superblock32(VP9_COMP *cpi, TOKENEXTRA **t,
       int second_ref_fb_idx;
 
       if (xd->mode_info_context->mbmi.second_ref_frame == LAST_FRAME)
-        second_ref_fb_idx = cpi->common.lst_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->lst_fb_idx];
       else if (xd->mode_info_context->mbmi.second_ref_frame == GOLDEN_FRAME)
-        second_ref_fb_idx = cpi->common.gld_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->gld_fb_idx];
       else
-        second_ref_fb_idx = cpi->common.alt_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->alt_fb_idx];
 
       xd->second_pre.y_buffer = cpi->common.yv12_fb[second_ref_fb_idx].y_buffer +
                                     recon_yoffset;
@@ -2557,11 +2557,11 @@ static void encode_superblock64(VP9_COMP *cpi, TOKENEXTRA **t,
     assert(cm->frame_type != KEY_FRAME);
 
     if (xd->mode_info_context->mbmi.ref_frame == LAST_FRAME)
-      ref_fb_idx = cpi->common.lst_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->lst_fb_idx];
     else if (xd->mode_info_context->mbmi.ref_frame == GOLDEN_FRAME)
-      ref_fb_idx = cpi->common.gld_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->gld_fb_idx];
     else
-      ref_fb_idx = cpi->common.alt_fb_idx;
+      ref_fb_idx = cpi->common.active_ref_idx[cpi->alt_fb_idx];
 
     xd->pre.y_buffer =
         cpi->common.yv12_fb[ref_fb_idx].y_buffer + recon_yoffset;
@@ -2574,11 +2574,11 @@ static void encode_superblock64(VP9_COMP *cpi, TOKENEXTRA **t,
       int second_ref_fb_idx;
 
       if (xd->mode_info_context->mbmi.second_ref_frame == LAST_FRAME)
-        second_ref_fb_idx = cpi->common.lst_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->lst_fb_idx];
       else if (xd->mode_info_context->mbmi.second_ref_frame == GOLDEN_FRAME)
-        second_ref_fb_idx = cpi->common.gld_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->gld_fb_idx];
       else
-        second_ref_fb_idx = cpi->common.alt_fb_idx;
+        second_ref_fb_idx = cpi->common.active_ref_idx[cpi->alt_fb_idx];
 
       xd->second_pre.y_buffer =
           cpi->common.yv12_fb[second_ref_fb_idx].y_buffer + recon_yoffset;
diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c
index db981754e..0a407dfdb 100644
--- a/vp9/encoder/vp9_firstpass.c
+++ b/vp9/encoder/vp9_firstpass.c
@@ -435,9 +435,11 @@ void vp9_first_pass(VP9_COMP *cpi) {
   MACROBLOCKD *const xd = &x->e_mbd;
 
   int recon_yoffset, recon_uvoffset;
-  YV12_BUFFER_CONFIG *lst_yv12 = &cm->yv12_fb[cm->lst_fb_idx];
+  YV12_BUFFER_CONFIG *lst_yv12 =
+      &cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]];
   YV12_BUFFER_CONFIG *new_yv12 = &cm->yv12_fb[cm->new_fb_idx];
-  YV12_BUFFER_CONFIG *gld_yv12 = &cm->yv12_fb[cm->gld_fb_idx];
+  YV12_BUFFER_CONFIG *gld_yv12 =
+      &cm->yv12_fb[cm->active_ref_idx[cpi->gld_fb_idx]];
   int recon_y_stride = lst_yv12->y_stride;
   int recon_uv_stride = lst_yv12->uv_stride;
   int64_t intra_error = 0;
diff --git a/vp9/encoder/vp9_mbgraph.c b/vp9/encoder/vp9_mbgraph.c
index 0ff60c8b0..218a47a8e 100644
--- a/vp9/encoder/vp9_mbgraph.c
+++ b/vp9/encoder/vp9_mbgraph.c
@@ -433,7 +433,8 @@ void vp9_update_mbgraph_stats
 ) {
   VP9_COMMON *const cm = &cpi->common;
   int i, n_frames = vp9_lookahead_depth(cpi->lookahead);
-  YV12_BUFFER_CONFIG *golden_ref = &cm->yv12_fb[cm->gld_fb_idx];
+  YV12_BUFFER_CONFIG *golden_ref =
+      &cm->yv12_fb[cm->active_ref_idx[cpi->gld_fb_idx]];
 
   // we need to look ahead beyond where the ARF transitions into
   // being a GF - so exit if we don't look ahead beyond that
diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c
index ee6c46f00..2b9b662a4 100644
--- a/vp9/encoder/vp9_onyx_if.c
+++ b/vp9/encoder/vp9_onyx_if.c
@@ -236,12 +236,12 @@ static void update_base_skip_probs(VP9_COMP *cpi) {
   if (cm->frame_type != KEY_FRAME) {
     vp9_update_skip_probs(cpi);
 
-    if (cm->refresh_alt_ref_frame) {
+    if (cpi->refresh_alt_ref_frame) {
       int k;
       for (k = 0; k < MBSKIP_CONTEXTS; ++k)
         cpi->last_skip_false_probs[2][k] = cm->mbskip_pred_probs[k];
       cpi->last_skip_probs_q[2] = cm->base_qindex;
-    } else if (cpi->common.refresh_golden_frame) {
+    } else if (cpi->refresh_golden_frame) {
       int k;
       for (k = 0; k < MBSKIP_CONTEXTS; ++k)
         cpi->last_skip_false_probs[1][k] = cm->mbskip_pred_probs[k];
@@ -408,10 +408,8 @@ static void init_seg_features(VP9_COMP *cpi) {
 
     // Clear down the segment features.
     vp9_clearall_segfeatures(xd);
-  }
-
-  // If this is an alt ref frame
-  else if (cm->refresh_alt_ref_frame) {
+  } else if (cpi->refresh_alt_ref_frame) {
+    // If this is an alt ref frame
     // Clear down the global segmentation map
     vpx_memset(cpi->segmentation_map, 0, (cm->mb_rows * cm->mb_cols));
     xd->update_mb_segmentation_map = 0;
@@ -1028,12 +1026,14 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
     sf->improved_dct = 0;
   }
 
-  if (cpi->sf.search_method == NSTEP) {
-    vp9_init3smotion_compensation(&cpi->mb,
-                                  cm->yv12_fb[cm->lst_fb_idx].y_stride);
-  } else if (cpi->sf.search_method == DIAMOND) {
-    vp9_init_dsmotion_compensation(&cpi->mb,
-                                   cm->yv12_fb[cm->lst_fb_idx].y_stride);
+  {
+    int y_stride = cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]].y_stride;
+
+    if (cpi->sf.search_method == NSTEP) {
+      vp9_init3smotion_compensation(&cpi->mb, y_stride);
+    } else if (cpi->sf.search_method == DIAMOND) {
+      vp9_init_dsmotion_compensation(&cpi->mb, y_stride);
+    }
   }
 
   cpi->mb.vp9_short_fdct16x16 = vp9_short_fdct16x16;
@@ -1304,6 +1304,10 @@ static void init_config(VP9_PTR ptr, VP9_CONFIG *oxcf) {
 
   cpi->static_mb_pct = 0;
 
+  cpi->lst_fb_idx = 0;
+  cpi->gld_fb_idx = 1;
+  cpi->alt_fb_idx = 2;
+
 #if VP9_TEMPORAL_ALT_REF
   {
     int i;
@@ -1385,8 +1389,8 @@ void vp9_change_config(VP9_PTR ptr, VP9_CONFIG *oxcf) {
 
   // cpi->use_golden_frame_only = 0;
   // cpi->use_last_frame_only = 0;
-  cm->refresh_golden_frame = 0;
-  cm->refresh_last_frame = 1;
+  cpi->refresh_golden_frame = 0;
+  cpi->refresh_last_frame = 1;
   cm->refresh_entropy_probs = 1;
 
   setup_features(cpi);
@@ -1492,10 +1496,10 @@ void vp9_change_config(VP9_PTR ptr, VP9_CONFIG *oxcf) {
   }
 
   if (((cm->Width + 15) & 0xfffffff0) !=
-      cm->yv12_fb[cm->lst_fb_idx].y_width ||
+      cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]].y_width ||
       ((cm->Height + 15) & 0xfffffff0) !=
-      cm->yv12_fb[cm->lst_fb_idx].y_height ||
-      cm->yv12_fb[cm->lst_fb_idx].y_width == 0) {
+      cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]].y_height ||
+      cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]].y_width == 0) {
     alloc_raw_frame_buffers(cpi);
     vp9_alloc_compressor_data(cpi);
   }
@@ -1693,7 +1697,7 @@ VP9_PTR vp9_create_compressor(VP9_CONFIG *oxcf) {
 
   cpi->source_alt_ref_pending = FALSE;
   cpi->source_alt_ref_active = FALSE;
-  cpi->common.refresh_alt_ref_frame = 0;
+  cpi->refresh_alt_ref_frame = 0;
 
   cpi->b_calculate_psnr = CONFIG_INTERNAL_STATS;
 #if CONFIG_INTERNAL_STATS
@@ -1908,7 +1912,8 @@ void vp9_remove_compressor(VP9_PTR *ptr) {
       print_mode_contexts(&cpi->common);
 #endif
       if (cpi->b_calculate_psnr) {
-        YV12_BUFFER_CONFIG *lst_yv12 = &cpi->common.yv12_fb[cpi->common.lst_fb_idx];
+        YV12_BUFFER_CONFIG *lst_yv12 =
+            &cpi->common.yv12_fb[cpi->common.active_ref_idx[cpi->lst_fb_idx]];
         double samples = 3.0 / 2 * cpi->count * lst_yv12->y_width * lst_yv12->y_height;
         double total_psnr = vp9_mse2psnr(samples, 255.0, cpi->total_sq_error);
         double total_psnr2 = vp9_mse2psnr(samples, 255.0, cpi->total_sq_error2);
@@ -2230,18 +2235,18 @@ int vp9_update_reference(VP9_PTR ptr, int ref_frame_flags) {
   if (ref_frame_flags > 7)
     return -1;
 
-  cpi->common.refresh_golden_frame = 0;
-  cpi->common.refresh_alt_ref_frame = 0;
-  cpi->common.refresh_last_frame   = 0;
+  cpi->refresh_golden_frame = 0;
+  cpi->refresh_alt_ref_frame = 0;
+  cpi->refresh_last_frame   = 0;
 
   if (ref_frame_flags & VP9_LAST_FLAG)
-    cpi->common.refresh_last_frame = 1;
+    cpi->refresh_last_frame = 1;
 
   if (ref_frame_flags & VP9_GOLD_FLAG)
-    cpi->common.refresh_golden_frame = 1;
+    cpi->refresh_golden_frame = 1;
 
   if (ref_frame_flags & VP9_ALT_FLAG)
-    cpi->common.refresh_alt_ref_frame = 1;
+    cpi->refresh_alt_ref_frame = 1;
 
   return 0;
 }
@@ -2253,11 +2258,11 @@ int vp9_get_reference_enc(VP9_PTR ptr, VP9_REFFRAME ref_frame_flag,
   int ref_fb_idx;
 
   if (ref_frame_flag == VP9_LAST_FLAG)
-    ref_fb_idx = cm->lst_fb_idx;
+    ref_fb_idx = cm->active_ref_idx[cpi->lst_fb_idx];
   else if (ref_frame_flag == VP9_GOLD_FLAG)
-    ref_fb_idx = cm->gld_fb_idx;
+    ref_fb_idx = cm->active_ref_idx[cpi->gld_fb_idx];
   else if (ref_frame_flag == VP9_ALT_FLAG)
-    ref_fb_idx = cm->alt_fb_idx;
+    ref_fb_idx = cm->active_ref_idx[cpi->alt_fb_idx];
   else
     return -1;
 
@@ -2274,11 +2279,11 @@ int vp9_set_reference_enc(VP9_PTR ptr, VP9_REFFRAME ref_frame_flag,
   int ref_fb_idx;
 
   if (ref_frame_flag == VP9_LAST_FLAG)
-    ref_fb_idx = cm->lst_fb_idx;
+    ref_fb_idx = cm->active_ref_idx[cpi->lst_fb_idx];
   else if (ref_frame_flag == VP9_GOLD_FLAG)
-    ref_fb_idx = cm->gld_fb_idx;
+    ref_fb_idx = cm->active_ref_idx[cpi->gld_fb_idx];
   else if (ref_frame_flag == VP9_ALT_FLAG)
-    ref_fb_idx = cm->alt_fb_idx;
+    ref_fb_idx = cm->active_ref_idx[cpi->alt_fb_idx];
   else
     return -1;
 
@@ -2374,13 +2379,13 @@ static void update_golden_frame_stats(VP9_COMP *cpi) {
   VP9_COMMON *cm = &cpi->common;
 
   // Update the Golden frame usage counts.
-  if (cm->refresh_golden_frame) {
+  if (cpi->refresh_golden_frame) {
     // Update data structure that monitors level of reference to last GF
     vpx_memset(cpi->gf_active_flags, 1, (cm->mb_rows * cm->mb_cols));
     cpi->gf_active_count = cm->mb_rows * cm->mb_cols;
 
     // this frame refreshes means next frames don't unless specified by user
-    cm->refresh_golden_frame = 0;
+    cpi->refresh_golden_frame = 0;
     cpi->common.frames_since_golden = 0;
 
     // if ( cm->frame_type == KEY_FRAME )
@@ -2402,7 +2407,7 @@ static void update_golden_frame_stats(VP9_COMP *cpi) {
     // ******** Fixed Q test code only ************
     // If we are going to use the ALT reference for the next group of frames set a flag to say so.
     if (cpi->oxcf.fixed_q >= 0 &&
-        cpi->oxcf.play_alternate && !cpi->common.refresh_alt_ref_frame) {
+        cpi->oxcf.play_alternate && !cpi->refresh_alt_ref_frame) {
       cpi->source_alt_ref_pending = TRUE;
       cpi->frames_till_gf_update_due = cpi->baseline_gf_interval;
     }
@@ -2414,7 +2419,7 @@ static void update_golden_frame_stats(VP9_COMP *cpi) {
     if (cpi->frames_till_gf_update_due > 0)
       cpi->frames_till_gf_update_due--;
 
-  } else if (!cpi->common.refresh_alt_ref_frame) {
+  } else if (!cpi->refresh_alt_ref_frame) {
     // Decrement count down till next gf
     if (cpi->frames_till_gf_update_due > 0)
       cpi->frames_till_gf_update_due--;
@@ -2535,8 +2540,8 @@ static int recode_loop_test(VP9_COMP *cpi,
   if ((cpi->sf.recode_loop == 1) ||
       ((cpi->sf.recode_loop == 2) &&
        ((cm->frame_type == KEY_FRAME) ||
-        cm->refresh_golden_frame ||
-        cm->refresh_alt_ref_frame))) {
+        cpi->refresh_golden_frame ||
+        cpi->refresh_alt_ref_frame))) {
     // General over and under shoot tests
     if (((cpi->projected_frame_size > high_limit) && (q < maxq)) ||
         ((cpi->projected_frame_size < low_limit) && (q > minq))) {
@@ -2563,25 +2568,32 @@ static int recode_loop_test(VP9_COMP *cpi,
   return force_recode;
 }
 
-static void update_reference_frames(VP9_COMMON *cm) {
+static void update_reference_frames(VP9_COMP * const cpi) {
+  VP9_COMMON * const cm = &cpi->common;
+
   // At this point the new frame has been encoded.
   // If any buffer copy / swapping is signaled it should be done here.
 
   if (cm->frame_type == KEY_FRAME) {
-    ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->gld_fb_idx, cm->new_fb_idx);
-    ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->alt_fb_idx, cm->new_fb_idx);
+    ref_cnt_fb(cm->fb_idx_ref_cnt,
+               &cm->active_ref_idx[cpi->gld_fb_idx], cm->new_fb_idx);
+    ref_cnt_fb(cm->fb_idx_ref_cnt,
+               &cm->active_ref_idx[cpi->alt_fb_idx], cm->new_fb_idx);
   } else { /* For non key frames */
-    if (cm->refresh_alt_ref_frame) {
-      ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->alt_fb_idx, cm->new_fb_idx);
+    if (cpi->refresh_alt_ref_frame) {
+      ref_cnt_fb(cm->fb_idx_ref_cnt,
+                 &cm->active_ref_idx[cpi->alt_fb_idx], cm->new_fb_idx);
     }
 
-    if (cm->refresh_golden_frame) {
-      ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->gld_fb_idx, cm->new_fb_idx);
+    if (cpi->refresh_golden_frame) {
+      ref_cnt_fb(cm->fb_idx_ref_cnt,
+                 &cm->active_ref_idx[cpi->gld_fb_idx], cm->new_fb_idx);
     }
   }
 
-  if (cm->refresh_last_frame) {
-    ref_cnt_fb(cm->fb_idx_ref_cnt, &cm->lst_fb_idx, cm->new_fb_idx);
+  if (cpi->refresh_last_frame) {
+    ref_cnt_fb(cm->fb_idx_ref_cnt,
+               &cm->active_ref_idx[cpi->lst_fb_idx], cm->new_fb_idx);
   }
 }
 
@@ -2731,7 +2743,7 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
 
   // For an alt ref frame in 2 pass we skip the call to the second
   // pass function that sets the target bandwidth so must set it here
-  if (cpi->common.refresh_alt_ref_frame) {
+  if (cpi->refresh_alt_ref_frame) {
     cpi->per_frame_bandwidth = cpi->twopass.gf_bits;                           // Per frame bit target for the alt ref frame
     // per second target bitrate
     cpi->target_bandwidth = (int)(cpi->twopass.gf_bits *
@@ -2844,9 +2856,7 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
       if (cpi->active_best_quality < cpi->best_quality)
         cpi->active_best_quality = cpi->best_quality;
     }
-  }
-
-  else if (cm->refresh_golden_frame || cpi->common.refresh_alt_ref_frame) {
+  } else if (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame) {
     int high = 2000;
     int low = 400;
 
@@ -2926,7 +2936,8 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
   // Set highest allowed value for Zbin over quant
   if (cm->frame_type == KEY_FRAME)
     zbin_oq_high = 0; // ZBIN_OQ_MAX/16
-  else if (cm->refresh_alt_ref_frame || (cm->refresh_golden_frame && !cpi->source_alt_ref_active))
+  else if (cpi->refresh_alt_ref_frame
+           || (cpi->refresh_golden_frame && !cpi->source_alt_ref_active))
     zbin_oq_high = 16;
   else
     zbin_oq_high = ZBIN_OQ_MAX;
@@ -3035,12 +3046,12 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
           cm->mbskip_pred_probs[k] = cpi->base_skip_false_prob[Q][k];
 
         if (cm->frame_type != KEY_FRAME) {
-          if (cpi->common.refresh_alt_ref_frame) {
+          if (cpi->refresh_alt_ref_frame) {
             for (k = 0; k < MBSKIP_CONTEXTS; k++) {
               if (cpi->last_skip_false_probs[2][k] != 0)
                 cm->mbskip_pred_probs[k] = cpi->last_skip_false_probs[2][k];
             }
-          } else if (cpi->common.refresh_golden_frame) {
+          } else if (cpi->refresh_golden_frame) {
             for (k = 0; k < MBSKIP_CONTEXTS; k++) {
               if (cpi->last_skip_false_probs[1][k] != 0)
                 cm->mbskip_pred_probs[k] = cpi->last_skip_false_probs[1][k];
@@ -3360,12 +3371,18 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
   vp9_update_gf_useage_maps(cpi, cm, &cpi->mb);
 
   if (cm->frame_type == KEY_FRAME)
-    cm->refresh_last_frame = 1;
+    cpi->refresh_last_frame = 1;
 
 #if 0
   {
     FILE *f = fopen("gfactive.stt", "a");
-    fprintf(f, "%8d %8d %8d %8d %8d\n", cm->current_video_frame, (100 * cpi->gf_active_count) / (cpi->common.mb_rows * cpi->common.mb_cols), cpi->this_iiratio, cpi->next_iiratio, cm->refresh_golden_frame);
+    fprintf(f, "%8d %8d %8d %8d %8d\n",
+            cm->current_video_frame,
+            (100 * cpi->gf_active_count)
+              / (cpi->common.mb_rows * cpi->common.mb_cols),
+            cpi->this_iiratio,
+            cpi->next_iiratio,
+            cpi->refresh_golden_frame);
     fclose(f);
   }
 #endif
@@ -3392,7 +3409,7 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
     update_reference_segmentation_map(cpi);
   }
 
-  update_reference_frames(cm);
+  update_reference_frames(cpi);
   vp9_copy(cpi->common.fc.coef_counts_4x4, cpi->coef_counts_4x4);
   vp9_copy(cpi->common.fc.hybrid_coef_counts_4x4,
            cpi->hybrid_coef_counts_4x4);
@@ -3450,8 +3467,8 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
   if ((cm->base_qindex < cpi->last_boosted_qindex) ||
       ((cpi->static_mb_pct < 100) &&
        ((cm->frame_type == KEY_FRAME) ||
-        cm->refresh_alt_ref_frame ||
-        (cm->refresh_golden_frame && !cpi->is_src_frame_alt_ref)))) {
+        cpi->refresh_alt_ref_frame ||
+        (cpi->refresh_golden_frame && !cpi->is_src_frame_alt_ref)))) {
     cpi->last_boosted_qindex = cm->base_qindex;
   }
 
@@ -3464,7 +3481,8 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
     cpi->avg_frame_qindex = (2 + 3 * cpi->avg_frame_qindex + cm->base_qindex) >> 2;
 
   // Keep a record from which we can calculate the average Q excluding GF updates and key frames
-  if ((cm->frame_type != KEY_FRAME) && !cm->refresh_golden_frame && !cm->refresh_alt_ref_frame) {
+  if ((cm->frame_type != KEY_FRAME)
+      && !cpi->refresh_golden_frame && !cpi->refresh_alt_ref_frame) {
     cpi->ni_frames++;
     cpi->tot_q += vp9_convert_qindex_to_q(Q);
     cpi->avg_q = cpi->tot_q / (double)cpi->ni_frames;
@@ -3506,7 +3524,7 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
 
     if (cpi->twopass.kf_group_bits < 0)
       cpi->twopass.kf_group_bits = 0;
-  } else if (cm->refresh_golden_frame || cm->refresh_alt_ref_frame) {
+  } else if (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame) {
     cpi->twopass.gf_group_bits += cpi->this_frame_target - cpi->projected_frame_size;
 
     if (cpi->twopass.gf_group_bits < 0)
@@ -3547,7 +3565,7 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
               vp9_convert_qindex_to_q(cpi->cq_target_quality),
               cpi->zbin_over_quant,
               // cpi->avg_frame_qindex, cpi->zbin_over_quant,
-              cm->refresh_golden_frame, cm->refresh_alt_ref_frame,
+              cpi->refresh_golden_frame, cpi->refresh_alt_ref_frame,
               cm->frame_type, cpi->gfu_boost,
               cpi->twopass.est_max_qcorrection_factor,
               (int)cpi->twopass.bits_left,
@@ -3577,7 +3595,7 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
               vp9_convert_qindex_to_q(cpi->cq_target_quality),
               cpi->zbin_over_quant,
               // cpi->avg_frame_qindex, cpi->zbin_over_quant,
-              cm->refresh_golden_frame, cm->refresh_alt_ref_frame,
+              cpi->refresh_golden_frame, cpi->refresh_alt_ref_frame,
               cm->frame_type, cpi->gfu_boost,
               cpi->twopass.est_max_qcorrection_factor,
               (int)cpi->twopass.bits_left,
@@ -3593,8 +3611,8 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
 
       fprintf(fmodes, "%6d:%1d:%1d:%1d ",
               cpi->common.current_video_frame,
-              cm->frame_type, cm->refresh_golden_frame,
-              cm->refresh_alt_ref_frame);
+              cm->frame_type, cpi->refresh_golden_frame,
+              cpi->refresh_alt_ref_frame);
 
       for (i = 0; i < MAX_MODES; i++)
         fprintf(fmodes, "%5d ", cpi->mode_chosen_counts[i]);
@@ -3613,33 +3631,34 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
 #endif
 
   // If this was a kf or Gf note the Q
-  if ((cm->frame_type == KEY_FRAME) || cm->refresh_golden_frame || cm->refresh_alt_ref_frame)
+  if ((cm->frame_type == KEY_FRAME)
+      || cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)
     cm->last_kf_gf_q = cm->base_qindex;
 
-  if (cm->refresh_golden_frame == 1)
+  if (cpi->refresh_golden_frame == 1)
     cm->frame_flags = cm->frame_flags | FRAMEFLAGS_GOLDEN;
   else
     cm->frame_flags = cm->frame_flags&~FRAMEFLAGS_GOLDEN;
 
-  if (cm->refresh_alt_ref_frame == 1)
+  if (cpi->refresh_alt_ref_frame == 1)
     cm->frame_flags = cm->frame_flags | FRAMEFLAGS_ALTREF;
   else
     cm->frame_flags = cm->frame_flags&~FRAMEFLAGS_ALTREF;
 
 
-  if (cm->refresh_last_frame & cm->refresh_golden_frame) // both refreshed
+  if (cpi->refresh_last_frame & cpi->refresh_golden_frame)
     cpi->gold_is_last = 1;
-  else if (cm->refresh_last_frame ^ cm->refresh_golden_frame) // 1 refreshed but not the other
+  else if (cpi->refresh_last_frame ^ cpi->refresh_golden_frame)
     cpi->gold_is_last = 0;
 
-  if (cm->refresh_last_frame & cm->refresh_alt_ref_frame) // both refreshed
+  if (cpi->refresh_last_frame & cpi->refresh_alt_ref_frame)
     cpi->alt_is_last = 1;
-  else if (cm->refresh_last_frame ^ cm->refresh_alt_ref_frame) // 1 refreshed but not the other
+  else if (cpi->refresh_last_frame ^ cpi->refresh_alt_ref_frame)
     cpi->alt_is_last = 0;
 
-  if (cm->refresh_alt_ref_frame & cm->refresh_golden_frame) // both refreshed
+  if (cpi->refresh_alt_ref_frame & cpi->refresh_golden_frame)
     cpi->gold_is_alt = 1;
-  else if (cm->refresh_alt_ref_frame ^ cm->refresh_golden_frame) // 1 refreshed but not the other
+  else if (cpi->refresh_alt_ref_frame ^ cpi->refresh_golden_frame)
     cpi->gold_is_alt = 0;
 
   cpi->ref_frame_flags = VP9_ALT_FLAG | VP9_GOLD_FLAG | VP9_LAST_FLAG;
@@ -3653,7 +3672,8 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
   if (cpi->gold_is_alt)
     cpi->ref_frame_flags &= ~VP9_ALT_FLAG;
 
-  if (cpi->oxcf.play_alternate && cm->refresh_alt_ref_frame && (cm->frame_type != KEY_FRAME))
+  if (cpi->oxcf.play_alternate && cpi->refresh_alt_ref_frame
+      && (cm->frame_type != KEY_FRAME))
     // Update the alternate reference frame stats as appropriate.
     update_alt_ref_frame_stats(cpi);
   else
@@ -3692,8 +3712,9 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
     FILE *recon_file;
     sprintf(filename, "enc%04d.yuv", (int) cm->current_video_frame);
     recon_file = fopen(filename, "wb");
-    fwrite(cm->yv12_fb[cm->lst_fb_idx].buffer_alloc,
-           cm->yv12_fb[cm->lst_fb_idx].frame_size, 1, recon_file);
+    fwrite(cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]].buffer_alloc,
+           cm->yv12_fb[cm->active_ref_idx[cpi->lst_fb_idx]].frame_size,
+           1, recon_file);
     fclose(recon_file);
   }
 #endif
@@ -3713,13 +3734,13 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi,
 static void Pass2Encode(VP9_COMP *cpi, unsigned long *size,
                         unsigned char *dest, unsigned int *frame_flags) {
 
-  if (!cpi->common.refresh_alt_ref_frame)
+  if (!cpi->refresh_alt_ref_frame)
     vp9_second_pass(cpi);
 
   encode_frame_to_data_rate(cpi, size, dest, frame_flags);
   cpi->twopass.bits_left -= 8 * *size;
 
-  if (!cpi->common.refresh_alt_ref_frame) {
+  if (!cpi->refresh_alt_ref_frame) {
     double lower_bounds_min_rate = FRAME_OVERHEAD_BITS * cpi->oxcf.frame_rate;
     double two_pass_min_rate = (double)(cpi->oxcf.target_bandwidth
                                         * cpi->oxcf.two_pass_vbrmin_section / 100);
@@ -3756,8 +3777,8 @@ static int frame_is_reference(const VP9_COMP *cpi) {
   const VP9_COMMON *cm = &cpi->common;
   const MACROBLOCKD *xd = &cpi->mb.e_mbd;
 
-  return cm->frame_type == KEY_FRAME || cm->refresh_last_frame
-         || cm->refresh_golden_frame || cm->refresh_alt_ref_frame
+  return cm->frame_type == KEY_FRAME || cpi->refresh_last_frame
+         || cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame
          || cm->refresh_entropy_probs
          || xd->mode_ref_lf_delta_update
          || xd->update_mb_segmentation_map || xd->update_mb_segmentation_data;
@@ -3793,9 +3814,9 @@ int vp9_get_compressed_data(VP9_PTR ptr, unsigned int *frame_flags,
         force_src_buffer = &cpi->alt_ref_buffer;
       }
       cm->frames_till_alt_ref_frame = cpi->frames_till_gf_update_due;
-      cm->refresh_alt_ref_frame = 1;
-      cm->refresh_golden_frame = 0;
-      cm->refresh_last_frame = 0;
+      cpi->refresh_alt_ref_frame = 1;
+      cpi->refresh_golden_frame = 0;
+      cpi->refresh_last_frame = 0;
       cm->show_frame = 0;
       cpi->source_alt_ref_pending = FALSE;   // Clear Pending altf Ref flag.
       cpi->is_src_frame_alt_ref = 0;
@@ -3836,7 +3857,7 @@ int vp9_get_compressed_data(VP9_PTR ptr, unsigned int *frame_flags,
   }
 
   // adjust frame rates based on timestamps given
-  if (!cm->refresh_alt_ref_frame) {
+  if (!cpi->refresh_alt_ref_frame) {
     int64_t this_duration;
     int step = 0;
 
@@ -3892,13 +3913,13 @@ int vp9_get_compressed_data(VP9_PTR ptr, unsigned int *frame_flags,
 
 #if 0
 
-  if (cm->refresh_alt_ref_frame) {
-    // cm->refresh_golden_frame = 1;
-    cm->refresh_golden_frame = 0;
-    cm->refresh_last_frame = 0;
+  if (cpi->refresh_alt_ref_frame) {
+    // cpi->refresh_golden_frame = 1;
+    cpi->refresh_golden_frame = 0;
+    cpi->refresh_last_frame = 0;
   } else {
-    cm->refresh_golden_frame = 0;
-    cm->refresh_last_frame = 1;
+    cpi->refresh_golden_frame = 0;
+    cpi->refresh_last_frame = 1;
   }
 
 #endif
@@ -3918,7 +3939,7 @@ int vp9_get_compressed_data(VP9_PTR ptr, unsigned int *frame_flags,
   }
 
   if (cm->refresh_entropy_probs) {
-    if (cm->refresh_alt_ref_frame)
+    if (cpi->refresh_alt_ref_frame)
       vpx_memcpy(&cm->lfc_a, &cm->fc, sizeof(cm->fc));
     else
       vpx_memcpy(&cm->lfc, &cm->fc, sizeof(cm->fc));
@@ -3930,9 +3951,9 @@ int vp9_get_compressed_data(VP9_PTR ptr, unsigned int *frame_flags,
 
     // return to normal state
     cm->refresh_entropy_probs = 1;
-    cm->refresh_alt_ref_frame = 0;
-    cm->refresh_golden_frame = 0;
-    cm->refresh_last_frame = 1;
+    cpi->refresh_alt_ref_frame = 0;
+    cpi->refresh_golden_frame = 0;
+    cpi->refresh_last_frame = 1;
     cm->frame_type = INTER_FRAME;
 
   }
@@ -4055,7 +4076,7 @@ int vp9_get_preview_raw_frame(VP9_PTR comp, YV12_BUFFER_CONFIG *dest,
                               vp9_ppflags_t *flags) {
   VP9_COMP *cpi = (VP9_COMP *) comp;
 
-  if (cpi->common.refresh_alt_ref_frame)
+  if (cpi->refresh_alt_ref_frame)
     return -1;
   else {
     int ret;
diff --git a/vp9/encoder/vp9_onyx_int.h b/vp9/encoder/vp9_onyx_int.h
index 74a58b430..0ab0ab1df 100644
--- a/vp9/encoder/vp9_onyx_int.h
+++ b/vp9/encoder/vp9_onyx_int.h
@@ -357,7 +357,12 @@ typedef struct VP9_COMP {
   int alt_is_last;  // Alt reference frame same as last ( short circuit altref search)
   int gold_is_alt;  // don't do both alt and gold search ( just do gold).
 
-  // int refresh_alt_ref_frame;
+  int lst_fb_idx;
+  int gld_fb_idx;
+  int alt_fb_idx;
+  int refresh_last_frame;
+  int refresh_golden_frame;
+  int refresh_alt_ref_frame;
   YV12_BUFFER_CONFIG last_frame_uf;
 
   TOKENEXTRA *tok;
diff --git a/vp9/encoder/vp9_ratectrl.c b/vp9/encoder/vp9_ratectrl.c
index f663b56c9..0acd6513d 100644
--- a/vp9/encoder/vp9_ratectrl.c
+++ b/vp9/encoder/vp9_ratectrl.c
@@ -258,8 +258,8 @@ void vp9_setup_key_frame(VP9_COMP *cpi) {
   // interval before next GF
   cpi->frames_till_gf_update_due = cpi->baseline_gf_interval;
 
-  cpi->common.refresh_golden_frame = TRUE;
-  cpi->common.refresh_alt_ref_frame = TRUE;
+  cpi->refresh_golden_frame = TRUE;
+  cpi->refresh_alt_ref_frame = TRUE;
 
   vp9_init_mode_contexts(&cpi->common);
   vpx_memcpy(&cpi->common.lfc, &cpi->common.fc, sizeof(cpi->common.fc));
@@ -285,7 +285,7 @@ void vp9_setup_key_frame(VP9_COMP *cpi) {
 }
 
 void vp9_setup_inter_frame(VP9_COMP *cpi) {
-  if (cpi->common.refresh_alt_ref_frame) {
+  if (cpi->refresh_alt_ref_frame) {
     vpx_memcpy(&cpi->common.fc,
                &cpi->common.lfc_a,
                sizeof(cpi->common.fc));
@@ -358,7 +358,7 @@ static void calc_pframe_target_size(VP9_COMP *cpi) {
 
 
   // Special alt reference frame case
-  if (cpi->common.refresh_alt_ref_frame) {
+  if (cpi->refresh_alt_ref_frame) {
     // Per frame bit target for the alt ref frame
     cpi->per_frame_bandwidth = cpi->twopass.gf_bits;
     cpi->this_frame_target = cpi->per_frame_bandwidth;
@@ -377,7 +377,7 @@ static void calc_pframe_target_size(VP9_COMP *cpi) {
   if (cpi->this_frame_target < min_frame_target)
     cpi->this_frame_target = min_frame_target;
 
-  if (!cpi->common.refresh_alt_ref_frame)
+  if (!cpi->refresh_alt_ref_frame)
     // Note the baseline target data rate for this inter frame.
     cpi->inter_frame_target = cpi->this_frame_target;
 
@@ -386,7 +386,7 @@ static void calc_pframe_target_size(VP9_COMP *cpi) {
     // int Boost = 0;
     int Q = (cpi->oxcf.fixed_q < 0) ? cpi->last_q[INTER_FRAME] : cpi->oxcf.fixed_q;
 
-    cpi->common.refresh_golden_frame = TRUE;
+    cpi->refresh_golden_frame = TRUE;
 
     calc_gf_params(cpi);
 
@@ -431,7 +431,7 @@ void vp9_update_rate_correction_factors(VP9_COMP *cpi, int damp_var) {
   if (cpi->common.frame_type == KEY_FRAME) {
     rate_correction_factor = cpi->key_frame_rate_correction_factor;
   } else {
-    if (cpi->common.refresh_alt_ref_frame || cpi->common.refresh_golden_frame)
+    if (cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame)
       rate_correction_factor = cpi->gf_rate_correction_factor;
     else
       rate_correction_factor = cpi->rate_correction_factor;
@@ -505,7 +505,7 @@ void vp9_update_rate_correction_factors(VP9_COMP *cpi, int damp_var) {
   if (cpi->common.frame_type == KEY_FRAME)
     cpi->key_frame_rate_correction_factor = rate_correction_factor;
   else {
-    if (cpi->common.refresh_alt_ref_frame || cpi->common.refresh_golden_frame)
+    if (cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame)
       cpi->gf_rate_correction_factor = rate_correction_factor;
     else
       cpi->rate_correction_factor = rate_correction_factor;
@@ -529,7 +529,7 @@ int vp9_regulate_q(VP9_COMP *cpi, int target_bits_per_frame) {
   if (cpi->common.frame_type == KEY_FRAME)
     correction_factor = cpi->key_frame_rate_correction_factor;
   else {
-    if (cpi->common.refresh_alt_ref_frame || cpi->common.refresh_golden_frame)
+    if (cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame)
       correction_factor = cpi->gf_rate_correction_factor;
     else
       correction_factor = cpi->rate_correction_factor;
@@ -570,7 +570,8 @@ int vp9_regulate_q(VP9_COMP *cpi, int target_bits_per_frame) {
 
     if (cpi->common.frame_type == KEY_FRAME)
       zbin_oqmax = 0; // ZBIN_OQ_MAX/16
-    else if (cpi->common.refresh_alt_ref_frame || (cpi->common.refresh_golden_frame && !cpi->source_alt_ref_active))
+    else if (cpi->refresh_alt_ref_frame
+             || (cpi->refresh_golden_frame && !cpi->source_alt_ref_active))
       zbin_oqmax = 16;
     else
       zbin_oqmax = ZBIN_OQ_MAX;
@@ -671,7 +672,7 @@ void vp9_compute_frame_size_bounds(VP9_COMP *cpi, int *frame_under_shoot_limit,
       *frame_over_shoot_limit  = cpi->this_frame_target * 9 / 8;
       *frame_under_shoot_limit = cpi->this_frame_target * 7 / 8;
     } else {
-      if (cpi->common.refresh_alt_ref_frame || cpi->common.refresh_golden_frame) {
+      if (cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame) {
         *frame_over_shoot_limit  = cpi->this_frame_target * 9 / 8;
         *frame_under_shoot_limit = cpi->this_frame_target * 7 / 8;
       } else {
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 5e76d9372..5252bfd41 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -3592,22 +3592,22 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   }
 
   if (cpi->ref_frame_flags & VP9_LAST_FLAG) {
-    setup_buffer_inter(cpi, x, cpi->common.lst_fb_idx, LAST_FRAME,
-                       BLOCK_16X16, recon_yoffset, recon_uvoffset,
+    setup_buffer_inter(cpi, x, cpi->common.active_ref_idx[cpi->lst_fb_idx],
+                       LAST_FRAME, BLOCK_16X16, recon_yoffset, recon_uvoffset,
                        frame_mv[NEARESTMV], frame_mv[NEARMV],
                        frame_mdcounts, y_buffer, u_buffer, v_buffer);
   }
 
   if (cpi->ref_frame_flags & VP9_GOLD_FLAG) {
-    setup_buffer_inter(cpi, x, cpi->common.gld_fb_idx, GOLDEN_FRAME,
-                       BLOCK_16X16, recon_yoffset, recon_uvoffset,
+    setup_buffer_inter(cpi, x, cpi->common.active_ref_idx[cpi->gld_fb_idx],
+                       GOLDEN_FRAME, BLOCK_16X16, recon_yoffset, recon_uvoffset,
                        frame_mv[NEARESTMV], frame_mv[NEARMV],
                        frame_mdcounts, y_buffer, u_buffer, v_buffer);
   }
 
   if (cpi->ref_frame_flags & VP9_ALT_FLAG) {
-    setup_buffer_inter(cpi, x, cpi->common.alt_fb_idx, ALTREF_FRAME,
-                       BLOCK_16X16, recon_yoffset, recon_uvoffset,
+    setup_buffer_inter(cpi, x, cpi->common.active_ref_idx[cpi->alt_fb_idx],
+                       ALTREF_FRAME, BLOCK_16X16, recon_yoffset, recon_uvoffset,
                        frame_mv[NEARESTMV], frame_mv[NEARMV],
                        frame_mdcounts, y_buffer, u_buffer, v_buffer);
   }
@@ -4476,8 +4476,10 @@ static int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
   uint8_t *v_buffer[4];
   static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
                                     VP9_ALT_FLAG };
-  int idx_list[4] = { 0, cpi->common.lst_fb_idx, cpi->common.gld_fb_idx,
-                      cpi->common.alt_fb_idx };
+  int idx_list[4] = {0,
+                     cpi->common.active_ref_idx[cpi->lst_fb_idx],
+                     cpi->common.active_ref_idx[cpi->gld_fb_idx],
+                     cpi->common.active_ref_idx[cpi->alt_fb_idx]};
   int mdcounts[4];
   int near_sadidx[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
   int saddone = 0;
diff --git a/vp9/encoder/vp9_segmentation.c b/vp9/encoder/vp9_segmentation.c
index 49195e80c..05c335d4a 100644
--- a/vp9/encoder/vp9_segmentation.c
+++ b/vp9/encoder/vp9_segmentation.c
@@ -21,7 +21,7 @@ void vp9_update_gf_useage_maps(VP9_COMP *cpi, VP9_COMMON *cm, MACROBLOCK *x) {
 
   x->gf_active_ptr = (signed char *)cpi->gf_active_flags;
 
-  if ((cm->frame_type == KEY_FRAME) || (cm->refresh_golden_frame)) {
+  if ((cm->frame_type == KEY_FRAME) || (cpi->refresh_golden_frame)) {
     // Reset Gf useage monitors
     vpx_memset(cpi->gf_active_flags, 1, (cm->mb_rows * cm->mb_cols));
     cpi->gf_active_count = cm->mb_rows * cm->mb_cols;
diff --git a/vp9/vp9_dx_iface.c b/vp9/vp9_dx_iface.c
index 2d7e41369..05d4edbe8 100644
--- a/vp9/vp9_dx_iface.c
+++ b/vp9/vp9_dx_iface.c
@@ -645,9 +645,7 @@ static vpx_codec_err_t vp8_get_last_ref_updates(vpx_codec_alg_priv_t *ctx,
   VP9D_COMP *pbi = (VP9D_COMP *)ctx->pbi;
 
   if (update_info) {
-    *update_info = pbi->common.refresh_alt_ref_frame * (int) VP8_ALTR_FRAME
-                   + pbi->common.refresh_golden_frame * (int) VP8_GOLD_FRAME
-                   + pbi->common.refresh_last_frame * (int) VP8_LAST_FRAME;
+    *update_info = pbi->refresh_frame_flags;
 
     return VPX_CODEC_OK;
   } else