diff --git a/vp9/common/vp9_onyxc_int.h b/vp9/common/vp9_onyxc_int.h
index 751accf02..6d14128a8 100644
--- a/vp9/common/vp9_onyxc_int.h
+++ b/vp9/common/vp9_onyxc_int.h
@@ -238,8 +238,10 @@ static int get_free_fb(VP9_COMMON *cm) {
 }
 
 static void ref_cnt_fb(int *buf, int *idx, int new_idx) {
-  if (buf[*idx] > 0)
-    buf[*idx]--;
+  const int ref_index = *idx;
+
+  if (ref_index > 0 && buf[ref_index] > 0)
+    buf[ref_index]--;
 
   *idx = new_idx;
 
diff --git a/vp9/decoder/vp9_decodeframe.c b/vp9/decoder/vp9_decodeframe.c
index 82bace00d..516aa88cb 100644
--- a/vp9/decoder/vp9_decodeframe.c
+++ b/vp9/decoder/vp9_decodeframe.c
@@ -687,12 +687,6 @@ static void apply_frame_size(VP9D_COMP *pbi, int width, int height) {
 
   if (cm->width != width || cm->height != height) {
     // Change in frame size.
-    if (cm->width == 0 || cm->height == 0) {
-      // Assign new frame buffer on first call.
-      cm->new_fb_idx = NUM_YV12_BUFFERS - 1;
-      cm->fb_idx_ref_cnt[cm->new_fb_idx] = 1;
-    }
-
     // TODO(agrange) Don't test width/height, check overall size.
     if (width > cm->width || height > cm->height) {
       // Rescale frame buffers only if they're not big enough already.
diff --git a/vp9/decoder/vp9_onyxd_if.c b/vp9/decoder/vp9_onyxd_if.c
index 25fb3d6d2..4c0cd45a9 100644
--- a/vp9/decoder/vp9_onyxd_if.c
+++ b/vp9/decoder/vp9_onyxd_if.c
@@ -125,6 +125,9 @@ VP9D_PTR vp9_create_decompressor(VP9D_CONFIG *oxcf) {
 
   vp9_zero(*pbi);
 
+  // Initialize the references to not point to any frame buffers.
+  memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map));
+
   if (setjmp(cm->error.jmp)) {
     cm->error.setjmp = 0;
     vp9_remove_decompressor(pbi);