diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
index 554a31730dadb2814a32dfdc81426f6f811a137f..b99fdfa44ee751e94f38f05a83ba883b57b85ca1 100644
--- a/vp9/common/vp9_alloccommon.c
+++ b/vp9/common/vp9_alloccommon.c
@@ -58,6 +58,7 @@ void vp9_free_frame_buffers(VP9_COMMON *oci) {
   vpx_free(oci->mip);
   vpx_free(oci->prev_mip);
   vpx_free(oci->above_seg_context);
+  vpx_free(oci->last_frame_seg_map);
 
   vpx_free(oci->above_context[0]);
   for (i = 0; i < MAX_MB_PLANE; i++)
@@ -65,6 +66,7 @@ void vp9_free_frame_buffers(VP9_COMMON *oci) {
   oci->mip = NULL;
   oci->prev_mip = NULL;
   oci->above_seg_context = NULL;
+  oci->last_frame_seg_map = NULL;
 }
 
 static void set_mb_mi(VP9_COMMON *cm, int aligned_width, int aligned_height) {
@@ -157,6 +159,11 @@ int vp9_alloc_frame_buffers(VP9_COMMON *oci, int width, int height) {
   if (!oci->above_seg_context)
     goto fail;
 
+  // Create the segmentation map structure and set to 0.
+  oci->last_frame_seg_map = vpx_calloc(oci->mi_rows * oci->mi_cols, 1);
+  if (!oci->last_frame_seg_map)
+    goto fail;
+
   return 0;
 
  fail:
@@ -198,4 +205,8 @@ void vp9_update_frame_size(VP9_COMMON *cm) {
   for (i = 1; i < MAX_MB_PLANE; i++)
     cm->above_context[i] =
         cm->above_context[0] + i * sizeof(ENTROPY_CONTEXT) * 2 * mi_cols;
+
+  // Initialize the previous frame segment map to 0.
+  if (cm->last_frame_seg_map)
+    vpx_memset(cm->last_frame_seg_map, 0, cm->mi_rows * cm->mi_cols);
 }
diff --git a/vp9/decoder/vp9_decodframe.c b/vp9/decoder/vp9_decodframe.c
index 58fe52c33dc141f714034a352df1254dd0e65a08..cda3d7c3ed8ac5b70c1514d170a5c3267534fd73 100644
--- a/vp9/decoder/vp9_decodframe.c
+++ b/vp9/decoder/vp9_decodframe.c
@@ -989,11 +989,6 @@ int vp9_decode_frame(VP9D_COMP *pbi, const uint8_t **p_data_end) {
 
   new_fb->corrupted |= read_compressed_header(pbi, data, first_partition_size);
 
-  // Create the segmentation map structure and set to 0
-  if (!pc->last_frame_seg_map)
-    CHECK_MEM_ERROR(pc, pc->last_frame_seg_map,
-                    vpx_calloc((pc->mi_rows * pc->mi_cols), 1));
-
   setup_block_dptrs(xd, pc->subsampling_x, pc->subsampling_y);
 
   // clear out the coeff buffer
diff --git a/vp9/decoder/vp9_onyxd_if.c b/vp9/decoder/vp9_onyxd_if.c
index 91e26ec3e9c97633ee4c75026348e4a2c357ba75..2727c7511128b3fd4e7d7eb26e13ea324511033e 100644
--- a/vp9/decoder/vp9_onyxd_if.c
+++ b/vp9/decoder/vp9_onyxd_if.c
@@ -160,9 +160,6 @@ void vp9_remove_decompressor(VP9D_PTR ptr) {
   if (!pbi)
     return;
 
-  if (pbi->common.last_frame_seg_map)
-    vpx_free(pbi->common.last_frame_seg_map);
-
   vp9_remove_common(&pbi->common);
   vp9_worker_end(&pbi->lf_worker);
   vpx_free(pbi->lf_worker.data1);