diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
index ccbf3f64a..f847e1276 100644
--- a/vp9/common/vp9_alloccommon.c
+++ b/vp9/common/vp9_alloccommon.c
@@ -97,14 +97,15 @@ static void free_mi(VP9_COMMON *cm) {
 
 void vp9_free_frame_buffers(VP9_COMMON *cm) {
   int i;
+  BufferPool *const pool = cm->buffer_pool;
 
   for (i = 0; i < FRAME_BUFFERS; ++i) {
-    vp9_free_frame_buffer(&cm->frame_bufs[i].buf);
+    vp9_free_frame_buffer(&pool->frame_bufs[i].buf);
 
-    if (cm->frame_bufs[i].ref_count > 0 &&
-        cm->frame_bufs[i].raw_frame_buffer.data != NULL) {
-      cm->release_fb_cb(cm->cb_priv, &cm->frame_bufs[i].raw_frame_buffer);
-      cm->frame_bufs[i].ref_count = 0;
+    if (pool->frame_bufs[i].ref_count > 0 &&
+        pool->frame_bufs[i].raw_frame_buffer.data != NULL) {
+      pool->release_fb_cb(pool->cb_priv, &pool->frame_bufs[i].raw_frame_buffer);
+      pool->frame_bufs[i].ref_count = 0;
     }
   }
 
@@ -176,13 +177,14 @@ int vp9_resize_frame_buffers(VP9_COMMON *cm, int width, int height) {
 
 static void init_frame_bufs(VP9_COMMON *cm) {
   int i;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   cm->new_fb_idx = FRAME_BUFFERS - 1;
-  cm->frame_bufs[cm->new_fb_idx].ref_count = 1;
+  frame_bufs[cm->new_fb_idx].ref_count = 1;
 
   for (i = 0; i < REF_FRAMES; ++i) {
     cm->ref_frame_map[i] = i;
-    cm->frame_bufs[i].ref_count = 1;
+    frame_bufs[i].ref_count = 1;
   }
 }
 
@@ -190,12 +192,13 @@ int vp9_alloc_frame_buffers(VP9_COMMON *cm, int width, int height) {
   int i;
   const int ss_x = cm->subsampling_x;
   const int ss_y = cm->subsampling_y;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   vp9_free_frame_buffers(cm);
 
   for (i = 0; i < FRAME_BUFFERS; ++i) {
-    cm->frame_bufs[i].ref_count = 0;
-    if (vp9_alloc_frame_buffer(&cm->frame_bufs[i].buf, width, height,
+    frame_bufs[i].ref_count = 0;
+    if (vp9_alloc_frame_buffer(&frame_bufs[i].buf, width, height,
                                ss_x, ss_y, VP9_ENC_BORDER_IN_PIXELS) < 0)
       goto fail;
   }
@@ -256,7 +259,7 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
 void vp9_remove_common(VP9_COMMON *cm) {
   vp9_free_frame_buffers(cm);
   vp9_free_context_buffers(cm);
-  vp9_free_internal_frame_buffers(&cm->int_frame_buffers);
+  vp9_free_internal_frame_buffers(&cm->buffer_pool->int_frame_buffers);
 }
 
 void vp9_update_frame_size(VP9_COMMON *cm) {
diff --git a/vp9/common/vp9_onyxc_int.h b/vp9/common/vp9_onyxc_int.h
index e1753a11b..640e953e7 100644
--- a/vp9/common/vp9_onyxc_int.h
+++ b/vp9/common/vp9_onyxc_int.h
@@ -11,6 +11,7 @@
 #ifndef VP9_COMMON_VP9_ONYXC_INT_H_
 #define VP9_COMMON_VP9_ONYXC_INT_H_
 
+#include <pthread.h>
 #include "./vpx_config.h"
 #include "vpx/internal/vpx_codec_internal.h"
 #include "./vp9_rtcd.h"
@@ -63,6 +64,26 @@ typedef struct {
   YV12_BUFFER_CONFIG buf;
 } RefCntBuffer;
 
+typedef struct {
+  // Protect BufferPool from being accessed by several FrameWorkers at
+  // the same time during frame parallel decode.
+  // TODO(hkuang): Try to use atomic variable instead of locking the whole pool.
+#if CONFIG_MULTITHREAD
+  pthread_mutex_t pool_mutex;
+#endif
+
+  // Private data associated with the frame buffer callbacks.
+  void *cb_priv;
+
+  vpx_get_frame_buffer_cb_fn_t get_fb_cb;
+  vpx_release_frame_buffer_cb_fn_t release_fb_cb;
+
+  RefCntBuffer frame_bufs[FRAME_BUFFERS];
+
+  // Frame buffers allocated internally by the codec.
+  InternalFrameBufferList int_frame_buffers;
+} BufferPool;
+
 typedef struct VP9Common {
   struct vpx_internal_error_info  error;
 
@@ -89,8 +110,6 @@ typedef struct VP9Common {
 
   YV12_BUFFER_CONFIG *frame_to_show;
 
-  RefCntBuffer frame_bufs[FRAME_BUFFERS];
-
   int ref_frame_map[REF_FRAMES]; /* maps fb_idx to reference slot */
 
   // TODO(jkoleszar): could expand active_ref_idx to 4, with 0 as intra, and
@@ -202,30 +221,26 @@ typedef struct VP9Common {
 
   int log2_tile_cols, log2_tile_rows;
 
-  // Private data associated with the frame buffer callbacks.
-  void *cb_priv;
-  vpx_get_frame_buffer_cb_fn_t get_fb_cb;
-  vpx_release_frame_buffer_cb_fn_t release_fb_cb;
-
-  // Handles memory for the codec.
-  InternalFrameBufferList int_frame_buffers;
+  // External BufferPool passed from outside.
+  BufferPool *buffer_pool;
 
   PARTITION_CONTEXT *above_seg_context;
   ENTROPY_CONTEXT *above_context;
 } VP9_COMMON;
 
 static INLINE YV12_BUFFER_CONFIG *get_frame_new_buffer(VP9_COMMON *cm) {
-  return &cm->frame_bufs[cm->new_fb_idx].buf;
+  return &cm->buffer_pool->frame_bufs[cm->new_fb_idx].buf;
 }
 
 static INLINE int get_free_fb(VP9_COMMON *cm) {
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
   int i;
-  for (i = 0; i < FRAME_BUFFERS; i++)
-    if (cm->frame_bufs[i].ref_count == 0)
+  for (i = 0; i < FRAME_BUFFERS; ++i)
+    if (frame_bufs[i].ref_count == 0)
       break;
 
   assert(i < FRAME_BUFFERS);
-  cm->frame_bufs[i].ref_count = 1;
+  frame_bufs[i].ref_count = 1;
   return i;
 }
 
@@ -310,7 +325,6 @@ static INLINE void update_partition_context(MACROBLOCKD *xd,
   PARTITION_CONTEXT *const above_ctx = xd->above_seg_context + mi_col;
   PARTITION_CONTEXT *const left_ctx = xd->left_seg_context + (mi_row & MI_MASK);
 
-  // num_4x4_blocks_wide_lookup[bsize] / 2
   const int bs = num_8x8_blocks_wide_lookup[bsize];
 
   // update the partition context at the end notes. set partition bits
diff --git a/vp9/decoder/vp9_decodeframe.c b/vp9/decoder/vp9_decodeframe.c
index 8b96abb9d..5b892bace 100644
--- a/vp9/decoder/vp9_decodeframe.c
+++ b/vp9/decoder/vp9_decodeframe.c
@@ -621,6 +621,7 @@ static void setup_display_size(VP9_COMMON *cm, struct vp9_read_bit_buffer *rb) {
 }
 
 static void apply_frame_size(VP9_COMMON *cm, int width, int height) {
+  BufferPool *const pool = cm->buffer_pool;
   if (cm->width != width || cm->height != height) {
     // Change in frame size.
     // TODO(agrange) Don't test width/height, check overall size.
@@ -640,8 +641,8 @@ static void apply_frame_size(VP9_COMMON *cm, int width, int height) {
   if (vp9_realloc_frame_buffer(
           get_frame_new_buffer(cm), cm->width, cm->height,
           cm->subsampling_x, cm->subsampling_y, VP9_DEC_BORDER_IN_PIXELS,
-          &cm->frame_bufs[cm->new_fb_idx].raw_frame_buffer, cm->get_fb_cb,
-          cm->cb_priv)) {
+          &pool->frame_bufs[cm->new_fb_idx].raw_frame_buffer, pool->get_fb_cb,
+          pool->cb_priv)) {
     vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                        "Failed to allocate frame buffer");
   }
@@ -1076,6 +1077,7 @@ static BITSTREAM_PROFILE read_profile(struct vp9_read_bit_buffer *rb) {
 static size_t read_uncompressed_header(VP9Decoder *pbi,
                                        struct vp9_read_bit_buffer *rb) {
   VP9_COMMON *const cm = &pbi->common;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
   size_t sz;
   int i;
 
@@ -1095,12 +1097,12 @@ static size_t read_uncompressed_header(VP9Decoder *pbi,
     // Show an existing frame directly.
     const int frame_to_show = cm->ref_frame_map[vp9_rb_read_literal(rb, 3)];
 
-    if (frame_to_show < 0 || cm->frame_bufs[frame_to_show].ref_count < 1)
+    if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1)
       vpx_internal_error(&cm->error, VPX_CODEC_UNSUP_BITSTREAM,
                          "Buffer %d does not contain a decoded frame",
                          frame_to_show);
 
-    ref_cnt_fb(cm->frame_bufs, &cm->new_fb_idx, frame_to_show);
+    ref_cnt_fb(frame_bufs, &cm->new_fb_idx, frame_to_show);
     pbi->refresh_frame_flags = 0;
     cm->lf.filter_level = 0;
     cm->show_frame = 1;
@@ -1161,7 +1163,7 @@ static size_t read_uncompressed_header(VP9Decoder *pbi,
         const int idx = cm->ref_frame_map[ref];
         RefBuffer *const ref_frame = &cm->frame_refs[i];
         ref_frame->idx = idx;
-        ref_frame->buf = &cm->frame_bufs[idx].buf;
+        ref_frame->buf = &frame_bufs[idx].buf;
         cm->ref_frame_sign_bias[LAST_FRAME + i] = vp9_rb_read_bit(rb);
       }
 
diff --git a/vp9/decoder/vp9_decoder.c b/vp9/decoder/vp9_decoder.c
index 13f9d71d7..75283ab93 100644
--- a/vp9/decoder/vp9_decoder.c
+++ b/vp9/decoder/vp9_decoder.c
@@ -41,7 +41,7 @@ static void initialize_dec() {
   }
 }
 
-VP9Decoder *vp9_decoder_create() {
+VP9Decoder *vp9_decoder_create(BufferPool *const pool) {
   VP9Decoder *const pbi = vpx_memalign(32, sizeof(*pbi));
   VP9_COMMON *const cm = pbi ? &pbi->common : NULL;
 
@@ -66,6 +66,7 @@ VP9Decoder *vp9_decoder_create() {
 
   cm->current_video_frame = 0;
   pbi->ready_for_new_data = 1;
+  pbi->common.buffer_pool = pool;
 
   // vp9_init_dequantizer() is first called here. Add check in
   // frame_init_dequantizer() to avoid unnecessary calling of
@@ -124,7 +125,7 @@ vpx_codec_err_t vp9_copy_reference_dec(VP9Decoder *pbi,
    */
   if (ref_frame_flag == VP9_LAST_FLAG) {
     const YV12_BUFFER_CONFIG *const cfg =
-        &cm->frame_bufs[cm->ref_frame_map[0]].buf;
+        &cm->buffer_pool->frame_bufs[cm->ref_frame_map[0]].buf;
     if (!equal_dimensions(cfg, sd))
       vpx_internal_error(&cm->error, VPX_CODEC_ERROR,
                          "Incorrect buffer dimensions");
@@ -143,6 +144,7 @@ vpx_codec_err_t vp9_set_reference_dec(VP9_COMMON *cm,
                                       VP9_REFFRAME ref_frame_flag,
                                       YV12_BUFFER_CONFIG *sd) {
   RefBuffer *ref_buf = NULL;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   // TODO(jkoleszar): The decoder doesn't have any real knowledge of what the
   // encoder is using the frame buffers for. This is just a stub to keep the
@@ -170,11 +172,11 @@ vpx_codec_err_t vp9_set_reference_dec(VP9_COMMON *cm,
     const int free_fb = get_free_fb(cm);
     // Decrease ref_count since it will be increased again in
     // ref_cnt_fb() below.
-    cm->frame_bufs[free_fb].ref_count--;
+    --frame_bufs[free_fb].ref_count;
 
     // Manage the reference counters and copy image.
-    ref_cnt_fb(cm->frame_bufs, ref_fb_ptr, free_fb);
-    ref_buf->buf = &cm->frame_bufs[*ref_fb_ptr].buf;
+    ref_cnt_fb(frame_bufs, ref_fb_ptr, free_fb);
+    ref_buf->buf = &frame_bufs[*ref_fb_ptr].buf;
     vp8_yv12_copy_frame(sd, ref_buf->buf);
   }
 
@@ -184,27 +186,30 @@ vpx_codec_err_t vp9_set_reference_dec(VP9_COMMON *cm,
 
 int vp9_get_reference_dec(VP9Decoder *pbi, int index, YV12_BUFFER_CONFIG **fb) {
   VP9_COMMON *cm = &pbi->common;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   if (index < 0 || index >= REF_FRAMES)
     return -1;
 
-  *fb = &cm->frame_bufs[cm->ref_frame_map[index]].buf;
+  *fb = &frame_bufs[cm->ref_frame_map[index]].buf;
   return 0;
 }
 
 /* If any buffer updating is signaled it should be done here. */
 static void swap_frame_buffers(VP9Decoder *pbi) {
   int ref_index = 0, mask;
-  VP9_COMMON *const cm = &pbi->common;
+  VP9_COMMON * const cm = &pbi->common;
+  BufferPool * const pool = cm->buffer_pool;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) {
     if (mask & 1) {
       const int old_idx = cm->ref_frame_map[ref_index];
-      ref_cnt_fb(cm->frame_bufs, &cm->ref_frame_map[ref_index],
+      ref_cnt_fb(frame_bufs, &cm->ref_frame_map[ref_index],
                  cm->new_fb_idx);
-      if (old_idx >= 0 && cm->frame_bufs[old_idx].ref_count == 0)
-        cm->release_fb_cb(cm->cb_priv,
-                          &cm->frame_bufs[old_idx].raw_frame_buffer);
+      if (old_idx >= 0 && frame_bufs[old_idx].ref_count == 0)
+        pool->release_fb_cb(pool->cb_priv,
+                            &frame_bufs[old_idx].raw_frame_buffer);
     }
     ++ref_index;
   }
@@ -212,7 +217,7 @@ static void swap_frame_buffers(VP9Decoder *pbi) {
   cm->frame_to_show = get_frame_new_buffer(cm);
 
   if (!pbi->frame_parallel_decode || !cm->show_frame) {
-    --cm->frame_bufs[cm->new_fb_idx].ref_count;
+    --frame_bufs[cm->new_fb_idx].ref_count;
   }
 
   // Invalidate these references until the next frame starts.
@@ -223,6 +228,8 @@ static void swap_frame_buffers(VP9Decoder *pbi) {
 int vp9_receive_compressed_data(VP9Decoder *pbi,
                                 size_t size, const uint8_t **psource) {
   VP9_COMMON *const cm = &pbi->common;
+  BufferPool *const pool = cm->buffer_pool;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
   const uint8_t *source = *psource;
   int retcode = 0;
 
@@ -244,9 +251,9 @@ int vp9_receive_compressed_data(VP9Decoder *pbi,
   // Check if the previous frame was a frame without any references to it.
   // Release frame buffer if not decoding in frame parallel mode.
   if (!pbi->frame_parallel_decode && cm->new_fb_idx >= 0
-      && cm->frame_bufs[cm->new_fb_idx].ref_count == 0)
-    cm->release_fb_cb(cm->cb_priv,
-                      &cm->frame_bufs[cm->new_fb_idx].raw_frame_buffer);
+      && frame_bufs[cm->new_fb_idx].ref_count == 0)
+    pool->release_fb_cb(pool->cb_priv,
+                        &frame_bufs[cm->new_fb_idx].raw_frame_buffer);
   cm->new_fb_idx = get_free_fb(cm);
 
   if (setjmp(cm->error.jmp)) {
@@ -262,8 +269,8 @@ int vp9_receive_compressed_data(VP9Decoder *pbi,
     if (cm->frame_refs[0].idx != INT_MAX && cm->frame_refs[0].buf != NULL)
       cm->frame_refs[0].buf->corrupted = 1;
 
-    if (cm->new_fb_idx > 0 && cm->frame_bufs[cm->new_fb_idx].ref_count > 0)
-      cm->frame_bufs[cm->new_fb_idx].ref_count--;
+    if (frame_bufs[cm->new_fb_idx].ref_count > 0)
+      --frame_bufs[cm->new_fb_idx].ref_count;
 
     return -1;
   }
diff --git a/vp9/decoder/vp9_decoder.h b/vp9/decoder/vp9_decoder.h
index ab4f9a2c3..29b18ca8f 100644
--- a/vp9/decoder/vp9_decoder.h
+++ b/vp9/decoder/vp9_decoder.h
@@ -78,7 +78,7 @@ vpx_codec_err_t vp9_set_reference_dec(VP9_COMMON *cm,
 int vp9_get_reference_dec(struct VP9Decoder *pbi,
                           int index, YV12_BUFFER_CONFIG **fb);
 
-struct VP9Decoder *vp9_decoder_create();
+struct VP9Decoder *vp9_decoder_create(BufferPool *const pool);
 
 void vp9_decoder_remove(struct VP9Decoder *pbi);
 
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index 4f17a15a9..a1c6ca07d 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -713,7 +713,8 @@ static void cal_nmvsadcosts_hp(int *mvsadcost[2]) {
 }
 
 
-VP9_COMP *vp9_create_compressor(VP9EncoderConfig *oxcf) {
+VP9_COMP *vp9_create_compressor(VP9EncoderConfig *oxcf,
+                                BufferPool *const pool) {
   unsigned int i, j;
   VP9_COMP *const cpi = vpx_memalign(32, sizeof(VP9_COMP));
   VP9_COMMON *const cm = cpi != NULL ? &cpi->common : NULL;
@@ -734,6 +735,7 @@ VP9_COMP *vp9_create_compressor(VP9EncoderConfig *oxcf) {
   vp9_rtcd();
 
   cpi->use_svc = 0;
+  cpi->common.buffer_pool = pool;
 
   init_config(cpi, oxcf);
   vp9_rc_init(&cpi->oxcf, cpi->pass, &cpi->rc);
@@ -1273,7 +1275,7 @@ int vp9_get_reference_enc(VP9_COMP *cpi, int index, YV12_BUFFER_CONFIG **fb) {
   if (index < 0 || index >= REF_FRAMES)
     return -1;
 
-  *fb = &cm->frame_bufs[cm->ref_frame_map[index]].buf;
+  *fb = &cm->buffer_pool->frame_bufs[cm->ref_frame_map[index]].buf;
   return 0;
 }
 
@@ -1542,14 +1544,13 @@ static int recode_loop_test(const VP9_COMP *cpi,
 
 void vp9_update_reference_frames(VP9_COMP *cpi) {
   VP9_COMMON * const cm = &cpi->common;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   // At this point the new frame has been encoded.
   // If any buffer copy / swapping is signaled it should be done here.
   if (cm->frame_type == KEY_FRAME) {
-    ref_cnt_fb(cm->frame_bufs,
-               &cm->ref_frame_map[cpi->gld_fb_idx], cm->new_fb_idx);
-    ref_cnt_fb(cm->frame_bufs,
-               &cm->ref_frame_map[cpi->alt_fb_idx], cm->new_fb_idx);
+    ref_cnt_fb(frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], cm->new_fb_idx);
+    ref_cnt_fb(frame_bufs, &cm->ref_frame_map[cpi->alt_fb_idx], cm->new_fb_idx);
   } else if (!cpi->multi_arf_allowed && cpi->refresh_golden_frame &&
              cpi->rc.is_src_frame_alt_ref && !cpi->use_svc) {
     /* Preserve the previously existing golden frame and update the frame in
@@ -1563,8 +1564,7 @@ void vp9_update_reference_frames(VP9_COMP *cpi) {
      */
     int tmp;
 
-    ref_cnt_fb(cm->frame_bufs,
-               &cm->ref_frame_map[cpi->alt_fb_idx], cm->new_fb_idx);
+    ref_cnt_fb(frame_bufs, &cm->ref_frame_map[cpi->alt_fb_idx], cm->new_fb_idx);
 
     tmp = cpi->alt_fb_idx;
     cpi->alt_fb_idx = cpi->gld_fb_idx;
@@ -1577,19 +1577,17 @@ void vp9_update_reference_frames(VP9_COMP *cpi) {
         arf_idx = gf_group->arf_update_idx[gf_group->index];
       }
 
-      ref_cnt_fb(cm->frame_bufs,
-                 &cm->ref_frame_map[arf_idx], cm->new_fb_idx);
+      ref_cnt_fb(frame_bufs, &cm->ref_frame_map[arf_idx], cm->new_fb_idx);
     }
 
     if (cpi->refresh_golden_frame) {
-      ref_cnt_fb(cm->frame_bufs,
+      ref_cnt_fb(frame_bufs,
                  &cm->ref_frame_map[cpi->gld_fb_idx], cm->new_fb_idx);
     }
   }
 
   if (cpi->refresh_last_frame) {
-    ref_cnt_fb(cm->frame_bufs,
-               &cm->ref_frame_map[cpi->lst_fb_idx], cm->new_fb_idx);
+    ref_cnt_fb(frame_bufs, &cm->ref_frame_map[cpi->lst_fb_idx], cm->new_fb_idx);
   }
 #if CONFIG_DENOISING
   vp9_denoiser_update_frame_info(&cpi->denoiser,
@@ -1630,34 +1628,36 @@ void vp9_scale_references(VP9_COMP *cpi) {
   VP9_COMMON *cm = &cpi->common;
   MV_REFERENCE_FRAME ref_frame;
   const VP9_REFFRAME ref_mask[3] = {VP9_LAST_FLAG, VP9_GOLD_FLAG, VP9_ALT_FLAG};
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     const int idx = cm->ref_frame_map[get_ref_frame_idx(cpi, ref_frame)];
-    const YV12_BUFFER_CONFIG *const ref = &cm->frame_bufs[idx].buf;
+    const YV12_BUFFER_CONFIG *const ref = &frame_bufs[idx].buf;
 
     // Need to convert from VP9_REFFRAME to index into ref_mask (subtract 1).
     if ((cpi->ref_frame_flags & ref_mask[ref_frame - 1]) &&
         (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height)) {
       const int new_fb = get_free_fb(cm);
-      vp9_realloc_frame_buffer(&cm->frame_bufs[new_fb].buf,
+      vp9_realloc_frame_buffer(&frame_bufs[new_fb].buf,
                                cm->width, cm->height,
                                cm->subsampling_x, cm->subsampling_y,
                                VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL);
-      scale_and_extend_frame(ref, &cm->frame_bufs[new_fb].buf);
+      scale_and_extend_frame(ref, &frame_bufs[new_fb].buf);
       cpi->scaled_ref_idx[ref_frame - 1] = new_fb;
     } else {
       cpi->scaled_ref_idx[ref_frame - 1] = idx;
-      cm->frame_bufs[idx].ref_count++;
+      ++frame_bufs[idx].ref_count;
     }
   }
 }
 
 static void release_scaled_references(VP9_COMP *cpi) {
   VP9_COMMON *cm = &cpi->common;
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
   int i;
 
-  for (i = 0; i < 3; i++)
-    cm->frame_bufs[cpi->scaled_ref_idx[i]].ref_count--;
+  for (i = 0; i < 3; ++i)
+    --frame_bufs[cpi->scaled_ref_idx[i]].ref_count;
 }
 
 static void full_to_model_count(unsigned int *model_count,
@@ -2520,6 +2520,7 @@ int vp9_get_compressed_data(VP9_COMP *cpi, unsigned int *frame_flags,
   const int is_spatial_svc = cpi->use_svc &&
                              (cpi->svc.number_temporal_layers == 1) &&
                              (cpi->svc.number_spatial_layers > 1);
+  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   if (!cpi)
     return -1;
@@ -2602,7 +2603,8 @@ int vp9_get_compressed_data(VP9_COMP *cpi, unsigned int *frame_flags,
       cm->show_frame = 1;
       cm->intra_only = 0;
 
-      // Check to see if the frame should be encoded as an arf overlay.
+      // Check to see if the frame to be encoded is an overlay for a previous
+      // arf frame and if so configure it as such.
       check_src_altref(cpi);
     }
   }
@@ -2656,7 +2658,7 @@ int vp9_get_compressed_data(VP9_COMP *cpi, unsigned int *frame_flags,
   /* find a free buffer for the new frame, releasing the reference previously
    * held.
    */
-  cm->frame_bufs[cm->new_fb_idx].ref_count--;
+  --frame_bufs[cm->new_fb_idx].ref_count;
   cm->new_fb_idx = get_free_fb(cm);
 
   if (!cpi->use_svc && cpi->multi_arf_allowed) {
@@ -2690,7 +2692,7 @@ int vp9_get_compressed_data(VP9_COMP *cpi, unsigned int *frame_flags,
 
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     const int idx = cm->ref_frame_map[get_ref_frame_idx(cpi, ref_frame)];
-    YV12_BUFFER_CONFIG *const buf = &cm->frame_bufs[idx].buf;
+    YV12_BUFFER_CONFIG *const buf = &frame_bufs[idx].buf;
     RefBuffer *const ref_buf = &cm->frame_refs[ref_frame - 1];
     ref_buf->buf = buf;
     ref_buf->idx = idx;
diff --git a/vp9/encoder/vp9_encoder.h b/vp9/encoder/vp9_encoder.h
index b38f9c246..072c1322a 100644
--- a/vp9/encoder/vp9_encoder.h
+++ b/vp9/encoder/vp9_encoder.h
@@ -436,7 +436,8 @@ typedef struct VP9_COMP {
 
 void vp9_initialize_enc();
 
-struct VP9_COMP *vp9_create_compressor(VP9EncoderConfig *oxcf);
+struct VP9_COMP *vp9_create_compressor(VP9EncoderConfig *oxcf,
+                                       BufferPool *const pool);
 void vp9_remove_compressor(VP9_COMP *cpi);
 
 void vp9_change_config(VP9_COMP *cpi, const VP9EncoderConfig *oxcf);
@@ -494,8 +495,9 @@ static INLINE int get_ref_frame_idx(const VP9_COMP *cpi,
 
 static INLINE YV12_BUFFER_CONFIG *get_ref_frame_buffer(
     VP9_COMP *cpi, MV_REFERENCE_FRAME ref_frame) {
-  VP9_COMMON * const cm = &cpi->common;
-  return &cm->frame_bufs[cm->ref_frame_map[get_ref_frame_idx(cpi, ref_frame)]]
+  VP9_COMMON *const cm = &cpi->common;
+  BufferPool *const pool = cm->buffer_pool;
+  return &pool->frame_bufs[cm->ref_frame_map[get_ref_frame_idx(cpi, ref_frame)]]
       .buf;
 }
 
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index e1a03a62f..e577017e6 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2271,9 +2271,10 @@ void vp9_setup_buffer_inter(VP9_COMP *cpi, MACROBLOCK *x,
 const YV12_BUFFER_CONFIG *vp9_get_scaled_ref_frame(const VP9_COMP *cpi,
                                                    int ref_frame) {
   const VP9_COMMON *const cm = &cpi->common;
+  const RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
   const int ref_idx = cm->ref_frame_map[get_ref_frame_idx(cpi, ref_frame)];
   const int scaled_idx = cpi->scaled_ref_idx[ref_frame - 1];
-  return (scaled_idx != ref_idx) ? &cm->frame_bufs[scaled_idx].buf : NULL;
+  return (scaled_idx != ref_idx) ? &frame_bufs[scaled_idx].buf : NULL;
 }
 
 int vp9_get_switchable_rate(const VP9_COMP *cpi) {
diff --git a/vp9/vp9_cx_iface.c b/vp9/vp9_cx_iface.c
index b1501619e..09949f24b 100644
--- a/vp9/vp9_cx_iface.c
+++ b/vp9/vp9_cx_iface.c
@@ -90,6 +90,8 @@ struct vpx_codec_alg_priv {
   vp8_postproc_cfg_t      preview_ppcfg;
   vpx_codec_pkt_list_decl(128) pkt_list;
   unsigned int                 fixed_kf_cntr;
+  // BufferPool that holds all reference frames.
+  BufferPool              *buffer_pool;
 };
 
 static VP9_REFFRAME ref_frame_to_vp9_reframe(vpx_ref_frame_type_t frame) {
@@ -630,6 +632,10 @@ static vpx_codec_err_t encoder_init(vpx_codec_ctx_t *ctx,
     ctx->priv->alg_priv = priv;
     ctx->priv->init_flags = ctx->init_flags;
     ctx->priv->enc.total_encoders = 1;
+    ctx->priv->alg_priv->buffer_pool =
+        (BufferPool *)vpx_calloc(1, sizeof(BufferPool));
+    if (ctx->priv->alg_priv->buffer_pool == NULL)
+      return VPX_CODEC_MEM_ERROR;
 
     if (ctx->config.enc) {
       // Update the reference to the config structure to an internal copy.
@@ -667,7 +673,8 @@ static vpx_codec_err_t encoder_init(vpx_codec_ctx_t *ctx,
       set_encoder_config(&ctx->priv->alg_priv->oxcf,
                          &ctx->priv->alg_priv->cfg,
                          &ctx->priv->alg_priv->extra_cfg);
-      cpi = vp9_create_compressor(&ctx->priv->alg_priv->oxcf);
+      cpi = vp9_create_compressor(&ctx->priv->alg_priv->oxcf,
+                                  ctx->priv->alg_priv->buffer_pool);
       if (cpi == NULL)
         res = VPX_CODEC_MEM_ERROR;
       else
@@ -681,6 +688,7 @@ static vpx_codec_err_t encoder_init(vpx_codec_ctx_t *ctx,
 static vpx_codec_err_t encoder_destroy(vpx_codec_alg_priv_t *ctx) {
   free(ctx->cx_data);
   vp9_remove_compressor(ctx->cpi);
+  vpx_free(ctx->buffer_pool);
   free(ctx);
   return VPX_CODEC_OK;
 }
diff --git a/vp9/vp9_dx_iface.c b/vp9/vp9_dx_iface.c
index 95897ae63..3a73eeaa1 100644
--- a/vp9/vp9_dx_iface.c
+++ b/vp9/vp9_dx_iface.c
@@ -46,6 +46,9 @@ struct vpx_codec_alg_priv {
   int                     next_submit_thread_id;
   int                     next_output_thread_id;
 
+  // BufferPool that holds all reference frames. Shared by all the FrameWorkers.
+  BufferPool              *buffer_pool;
+
   // External frame buffer info to save for VP9 common.
   void *ext_priv;  // Private data associated with the external frame buffers.
   vpx_get_frame_buffer_cb_fn_t get_ext_fb_cb;
@@ -100,6 +103,7 @@ static vpx_codec_err_t decoder_destroy(vpx_codec_alg_priv_t *ctx) {
   }
 
   vpx_free(ctx->frame_workers);
+  vpx_free(ctx->buffer_pool);
   vpx_free(ctx);
 
   return VPX_CODEC_OK;
@@ -218,21 +222,22 @@ static void init_buffer_callbacks(vpx_codec_alg_priv_t *ctx) {
     VP9Worker *const worker = &ctx->frame_workers[i];
     FrameWorkerData *const worker_data = (FrameWorkerData *)worker->data1;
     VP9_COMMON *const cm = &worker_data->pbi->common;
+    BufferPool *const pool = cm->buffer_pool;
 
     cm->new_fb_idx = -1;
     if (ctx->get_ext_fb_cb != NULL && ctx->release_ext_fb_cb != NULL) {
-      cm->get_fb_cb = ctx->get_ext_fb_cb;
-      cm->release_fb_cb = ctx->release_ext_fb_cb;
-      cm->cb_priv = ctx->ext_priv;
+      pool->get_fb_cb = ctx->get_ext_fb_cb;
+      pool->release_fb_cb = ctx->release_ext_fb_cb;
+      pool->cb_priv = ctx->ext_priv;
     } else {
-      cm->get_fb_cb = vp9_get_frame_buffer;
-      cm->release_fb_cb = vp9_release_frame_buffer;
+      pool->get_fb_cb = vp9_get_frame_buffer;
+      pool->release_fb_cb = vp9_release_frame_buffer;
 
-      if (vp9_alloc_internal_frame_buffers(&cm->int_frame_buffers))
+      if (vp9_alloc_internal_frame_buffers(&pool->int_frame_buffers))
         vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                            "Failed to initialize internal frame buffers");
 
-      cm->cb_priv = &cm->int_frame_buffers;
+      pool->cb_priv = &pool->int_frame_buffers;
     }
   }
 }
@@ -272,6 +277,9 @@ static vpx_codec_err_t init_decoder(vpx_codec_alg_priv_t *ctx) {
   ctx->next_output_thread_id = 0;
   ctx->num_frame_workers =
       (ctx->frame_parallel_decode == 1) ? ctx->cfg.threads: 1;
+  ctx->buffer_pool = (BufferPool *)vpx_calloc(1, sizeof(BufferPool));
+  if (ctx->buffer_pool == NULL)
+    return VPX_CODEC_MEM_ERROR;
 
   ctx->frame_workers = (VP9Worker *)
       vpx_malloc(ctx->num_frame_workers * sizeof(*ctx->frame_workers));
@@ -290,7 +298,7 @@ static vpx_codec_err_t init_decoder(vpx_codec_alg_priv_t *ctx) {
       return VPX_CODEC_MEM_ERROR;
     }
     worker_data = (FrameWorkerData *)worker->data1;
-    worker_data->pbi = vp9_decoder_create();
+    worker_data->pbi = vp9_decoder_create(ctx->buffer_pool);
     if (worker_data->pbi == NULL) {
       set_error_detail(ctx, "Failed to allocate worker_data");
       return VPX_CODEC_MEM_ERROR;
@@ -563,16 +571,18 @@ static vpx_image_t *decoder_get_frame(vpx_codec_alg_priv_t *ctx,
     FrameWorkerData *const worker_data = (FrameWorkerData *)worker->data1;
     if (vp9_get_raw_frame(worker_data->pbi, &sd, &flags) == 0) {
       VP9_COMMON *const cm = &worker_data->pbi->common;
+      BufferPool *const pool = cm->buffer_pool;
+      RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
       yuvconfig2image(&ctx->img, &sd, worker_data->user_priv);
-      ctx->img.fb_priv = cm->frame_bufs[cm->new_fb_idx].raw_frame_buffer.priv;
+      ctx->img.fb_priv = frame_bufs[cm->new_fb_idx].raw_frame_buffer.priv;
       img = &ctx->img;
       *iter = img;
       // Decrease reference count of last output frame in frame parallel mode.
       if (ctx->frame_parallel_decode && ctx->last_show_frame >= 0) {
-        --cm->frame_bufs[ctx->last_show_frame].ref_count;
-        if (cm->frame_bufs[ctx->last_show_frame].ref_count == 0) {
-          cm->release_fb_cb(cm->cb_priv,
-              &cm->frame_bufs[ctx->last_show_frame].raw_frame_buffer);
+        --frame_bufs[ctx->last_show_frame].ref_count;
+        if (frame_bufs[ctx->last_show_frame].ref_count == 0) {
+          pool->release_fb_cb(pool->cb_priv,
+              &frame_bufs[ctx->last_show_frame].raw_frame_buffer);
         }
       }
       ctx->last_show_frame = worker_data->pbi->common.new_fb_idx;