diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
index b19256eae..c0121626e 100644
--- a/vp9/common/vp9_alloccommon.c
+++ b/vp9/common/vp9_alloccommon.c
@@ -58,6 +58,7 @@ void vp9_free_frame_buffers(VP9_COMMON *oci) {
   vpx_free(oci->mip);
   vpx_free(oci->prev_mip);
   vpx_free(oci->above_seg_context);
+  vpx_free(oci->last_frame_seg_map);
 
   vpx_free(oci->above_context[0]);
   for (i = 0; i < MAX_MB_PLANE; i++)
@@ -65,6 +66,7 @@ void vp9_free_frame_buffers(VP9_COMMON *oci) {
   oci->mip = NULL;
   oci->prev_mip = NULL;
   oci->above_seg_context = NULL;
+  oci->last_frame_seg_map = NULL;
 }
 
 static void set_mb_mi(VP9_COMMON *cm, int aligned_width, int aligned_height) {
@@ -154,6 +156,11 @@ int vp9_alloc_frame_buffers(VP9_COMMON *oci, int width, int height) {
   if (!oci->above_seg_context)
     goto fail;
 
+  // Create the segmentation map structure and set to 0.
+  oci->last_frame_seg_map = vpx_calloc(oci->mi_rows * oci->mi_cols, 1);
+  if (!oci->last_frame_seg_map)
+    goto fail;
+
   return 0;
 
  fail:
@@ -195,4 +202,8 @@ void vp9_update_frame_size(VP9_COMMON *cm) {
   for (i = 1; i < MAX_MB_PLANE; i++)
     cm->above_context[i] =
         cm->above_context[0] + i * sizeof(ENTROPY_CONTEXT) * 2 * mi_cols;
+
+  // Initialize the previous frame segment map to 0.
+  if (cm->last_frame_seg_map)
+    vpx_memset(cm->last_frame_seg_map, 0, cm->mi_rows * cm->mi_cols);
 }
diff --git a/vp9/decoder/vp9_decodframe.c b/vp9/decoder/vp9_decodframe.c
index 3edb98187..0a38d6fa1 100644
--- a/vp9/decoder/vp9_decodframe.c
+++ b/vp9/decoder/vp9_decodframe.c
@@ -980,11 +980,6 @@ int vp9_decode_frame(VP9D_COMP *pbi, const uint8_t **p_data_end) {
 
   new_fb->corrupted |= read_compressed_header(pbi, data, first_partition_size);
 
-  // Create the segmentation map structure and set to 0
-  if (!pc->last_frame_seg_map)
-    CHECK_MEM_ERROR(pc, pc->last_frame_seg_map,
-                    vpx_calloc((pc->mi_rows * pc->mi_cols), 1));
-
   setup_block_dptrs(xd, pc->subsampling_x, pc->subsampling_y);
 
   // clear out the coeff buffer
diff --git a/vp9/decoder/vp9_onyxd_if.c b/vp9/decoder/vp9_onyxd_if.c
index e20994add..b609f9f31 100644
--- a/vp9/decoder/vp9_onyxd_if.c
+++ b/vp9/decoder/vp9_onyxd_if.c
@@ -161,9 +161,6 @@ void vp9_remove_decompressor(VP9D_PTR ptr) {
   if (!pbi)
     return;
 
-  if (pbi->common.last_frame_seg_map)
-    vpx_free(pbi->common.last_frame_seg_map);
-
   vp9_remove_common(&pbi->common);
   vp9_worker_end(&pbi->lf_worker);
   vpx_free(pbi->lf_worker.data1);