diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
index ccbf3f6..b379656 100644
--- a/vp9/common/vp9_alloccommon.c
+++ b/vp9/common/vp9_alloccommon.c
@@ -28,7 +28,10 @@
     vpx_memset(&mi[i * cm->mi_stride], 0, sizeof(*mi));
 }
 
-static void set_mb_mi(VP9_COMMON *cm, int aligned_width, int aligned_height) {
+void vp9_set_mb_mi(VP9_COMMON *cm, int width, int height) {
+  const int aligned_width = ALIGN_POWER_OF_TWO(width, MI_SIZE_LOG2);
+  const int aligned_height = ALIGN_POWER_OF_TWO(height, MI_SIZE_LOG2);
+
   cm->mi_cols = aligned_width >> MI_SIZE_LOG2;
   cm->mi_rows = aligned_height >> MI_SIZE_LOG2;
   cm->mi_stride = cm->mi_cols + MI_BLOCK_SIZE;
@@ -95,7 +98,7 @@
   cm->prev_mi_grid_base = NULL;
 }
 
-void vp9_free_frame_buffers(VP9_COMMON *cm) {
+void vp9_free_ref_frame_buffers(VP9_COMMON *cm) {
   int i;
 
   for (i = 0; i < FRAME_BUFFERS; ++i) {
@@ -124,52 +127,27 @@
   cm->above_seg_context = NULL;
 }
 
-int vp9_resize_frame_buffers(VP9_COMMON *cm, int width, int height) {
-  const int aligned_width = ALIGN_POWER_OF_TWO(width, MI_SIZE_LOG2);
-  const int aligned_height = ALIGN_POWER_OF_TWO(height, MI_SIZE_LOG2);
-#if CONFIG_INTERNAL_STATS || CONFIG_VP9_POSTPROC
-  const int ss_x = cm->subsampling_x;
-  const int ss_y = cm->subsampling_y;
+int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
+  vp9_free_context_buffers(cm);
 
-  // TODO(agrange): this should be conditionally allocated.
-  if (vp9_realloc_frame_buffer(&cm->post_proc_buffer, width, height, ss_x, ss_y,
-                               VP9_DEC_BORDER_IN_PIXELS, NULL, NULL, NULL) < 0)
-    goto fail;
-#endif
+  vp9_set_mb_mi(cm, width, height);
+  if (alloc_mi(cm, cm->mi_stride * (cm->mi_rows + MI_BLOCK_SIZE))) goto fail;
 
-  set_mb_mi(cm, aligned_width, aligned_height);
-
-  free_mi(cm);
-  if (alloc_mi(cm, cm->mi_stride * (cm->mi_rows + MI_BLOCK_SIZE)))
-    goto fail;
-
-  setup_mi(cm);
-
-  // Create the segmentation map structure and set to 0.
-  vpx_free(cm->last_frame_seg_map);
   cm->last_frame_seg_map = (uint8_t *)vpx_calloc(cm->mi_rows * cm->mi_cols, 1);
-  if (!cm->last_frame_seg_map)
-    goto fail;
+  if (!cm->last_frame_seg_map) goto fail;
 
-  vpx_free(cm->above_context);
-  cm->above_context =
-      (ENTROPY_CONTEXT *)vpx_calloc(2 * mi_cols_aligned_to_sb(cm->mi_cols) *
-                                        MAX_MB_PLANE,
-                                    sizeof(*cm->above_context));
-  if (!cm->above_context)
-    goto fail;
+  cm->above_context = (ENTROPY_CONTEXT *)vpx_calloc(
+      2 * mi_cols_aligned_to_sb(cm->mi_cols) * MAX_MB_PLANE,
+      sizeof(*cm->above_context));
+  if (!cm->above_context) goto fail;
 
-  vpx_free(cm->above_seg_context);
-  cm->above_seg_context =
-     (PARTITION_CONTEXT *)vpx_calloc(mi_cols_aligned_to_sb(cm->mi_cols),
-                                     sizeof(*cm->above_seg_context));
-  if (!cm->above_seg_context)
-    goto fail;
+  cm->above_seg_context = (PARTITION_CONTEXT *)vpx_calloc(
+      mi_cols_aligned_to_sb(cm->mi_cols), sizeof(*cm->above_seg_context));
+  if (!cm->above_seg_context) goto fail;
 
   return 0;
 
  fail:
-  vp9_free_frame_buffers(cm);
   vp9_free_context_buffers(cm);
   return 1;
 }
@@ -186,12 +164,12 @@
   }
 }
 
-int vp9_alloc_frame_buffers(VP9_COMMON *cm, int width, int height) {
+int vp9_alloc_ref_frame_buffers(VP9_COMMON *cm, int width, int height) {
   int i;
   const int ss_x = cm->subsampling_x;
   const int ss_y = cm->subsampling_y;
 
-  vp9_free_frame_buffers(cm);
+  vp9_free_ref_frame_buffers(cm);
 
   for (i = 0; i < FRAME_BUFFERS; ++i) {
     cm->frame_bufs[i].ref_count = 0;
@@ -211,62 +189,18 @@
   return 0;
 
  fail:
-  vp9_free_frame_buffers(cm);
-  return 1;
-}
-
-int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
-  const int aligned_width = ALIGN_POWER_OF_TWO(width, MI_SIZE_LOG2);
-  const int aligned_height = ALIGN_POWER_OF_TWO(height, MI_SIZE_LOG2);
-
-  vp9_free_context_buffers(cm);
-
-  set_mb_mi(cm, aligned_width, aligned_height);
-
-  if (alloc_mi(cm, cm->mi_stride * (cm->mi_rows + MI_BLOCK_SIZE)))
-    goto fail;
-
-  setup_mi(cm);
-
-  // Create the segmentation map structure and set to 0.
-  cm->last_frame_seg_map = (uint8_t *)vpx_calloc(cm->mi_rows * cm->mi_cols, 1);
-  if (!cm->last_frame_seg_map)
-    goto fail;
-
-  cm->above_context =
-      (ENTROPY_CONTEXT *)vpx_calloc(2 * mi_cols_aligned_to_sb(cm->mi_cols) *
-                                        MAX_MB_PLANE,
-                                    sizeof(*cm->above_context));
-  if (!cm->above_context)
-    goto fail;
-
-  cm->above_seg_context =
-      (PARTITION_CONTEXT *)vpx_calloc(mi_cols_aligned_to_sb(cm->mi_cols),
-                                      sizeof(*cm->above_seg_context));
-  if (!cm->above_seg_context)
-    goto fail;
-
-  return 0;
-
- fail:
-  vp9_free_context_buffers(cm);
+  vp9_free_ref_frame_buffers(cm);
   return 1;
 }
 
 void vp9_remove_common(VP9_COMMON *cm) {
-  vp9_free_frame_buffers(cm);
+  vp9_free_ref_frame_buffers(cm);
   vp9_free_context_buffers(cm);
   vp9_free_internal_frame_buffers(&cm->int_frame_buffers);
 }
 
-void vp9_update_frame_size(VP9_COMMON *cm) {
-  const int aligned_width = ALIGN_POWER_OF_TWO(cm->width, MI_SIZE_LOG2);
-  const int aligned_height = ALIGN_POWER_OF_TWO(cm->height, MI_SIZE_LOG2);
-
-  set_mb_mi(cm, aligned_width, aligned_height);
+void vp9_init_context_buffers(VP9_COMMON *cm) {
   setup_mi(cm);
-
-  // Initialize the previous frame segment map to 0.
   if (cm->last_frame_seg_map)
     vpx_memset(cm->last_frame_seg_map, 0, cm->mi_rows * cm->mi_cols);
 }
diff --git a/vp9/common/vp9_alloccommon.h b/vp9/common/vp9_alloccommon.h
index c4b1b8d..c5b893f 100644
--- a/vp9/common/vp9_alloccommon.h
+++ b/vp9/common/vp9_alloccommon.h
@@ -20,18 +20,17 @@
 
 void vp9_remove_common(struct VP9Common *cm);
 
-int vp9_resize_frame_buffers(struct VP9Common *cm, int width, int height);
-
-int vp9_alloc_frame_buffers(struct VP9Common *cm, int width, int height);
-int vp9_alloc_state_buffers(struct VP9Common *cm, int width, int height);
 int vp9_alloc_context_buffers(struct VP9Common *cm, int width, int height);
-
-void vp9_free_frame_buffers(struct VP9Common *cm);
-void vp9_free_state_buffers(struct VP9Common *cm);
+void vp9_init_context_buffers(struct VP9Common *cm);
 void vp9_free_context_buffers(struct VP9Common *cm);
 
-void vp9_update_frame_size(struct VP9Common *cm);
+int vp9_alloc_ref_frame_buffers(struct VP9Common *cm, int width, int height);
+void vp9_free_ref_frame_buffers(struct VP9Common *cm);
 
+int vp9_alloc_state_buffers(struct VP9Common *cm, int width, int height);
+void vp9_free_state_buffers(struct VP9Common *cm);
+
+void vp9_set_mb_mi(struct VP9Common *cm, int width, int height);
 void vp9_swap_mi_and_prev_mi(struct VP9Common *cm);
 
 #ifdef __cplusplus
diff --git a/vp9/common/vp9_postproc.c b/vp9/common/vp9_postproc.c
index 9f32104..7d96e6e 100644
--- a/vp9/common/vp9_postproc.c
+++ b/vp9/common/vp9_postproc.c
@@ -411,6 +411,14 @@
 
   vp9_clear_system_state();
 
+#if CONFIG_VP9_POSTPROC || CONFIG_INTERNAL_STATS
+  if (vp9_realloc_frame_buffer(&cm->post_proc_buffer, cm->width, cm->height,
+                               cm->subsampling_x, cm->subsampling_y,
+                               VP9_DEC_BORDER_IN_PIXELS, NULL, NULL, NULL) < 0)
+    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+                       "Failed to allocate post-processing buffer");
+#endif
+
   if (flags & VP9D_DEMACROBLOCK) {
     deblock_and_de_macro_block(cm->frame_to_show, ppbuf,
                                q + (ppflags->deblocking_level - 5) * 10, 1, 0);
diff --git a/vp9/decoder/vp9_decodeframe.c b/vp9/decoder/vp9_decodeframe.c
index 615fa4c..5c4d74f 100644
--- a/vp9/decoder/vp9_decodeframe.c
+++ b/vp9/decoder/vp9_decodeframe.c
@@ -620,28 +620,28 @@
     vp9_read_frame_size(rb, &cm->display_width, &cm->display_height);
 }
 
-static void apply_frame_size(VP9_COMMON *cm, int width, int height) {
-#if CONFIG_SIZE_LIMIT
-  if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT)
-    vpx_internal_error(&cm->error, VPX_CODEC_CORRUPT_FRAME,
-                       "Width and height beyond allowed size.");
-#endif
-
+static void resize_context_buffers(VP9_COMMON *cm, int width, int height) {
   if (cm->width != width || cm->height != height) {
-    // Change in frame size.
-    // TODO(agrange) Don't test width/height, check overall size.
-    if (width > cm->width || height > cm->height) {
-      // Rescale frame buffers only if they're not big enough already.
-      if (vp9_resize_frame_buffers(cm, width, height))
+    // Change in frame size (assumption: color format does not change).
+    if (cm->width == 0 || cm->height == 0 ||
+        width * height > cm->width * cm->height) {
+      if (vp9_alloc_context_buffers(cm, width, height))
         vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                            "Failed to allocate frame buffers");
+    } else {
+      vp9_set_mb_mi(cm, width, height);
     }
-
+    vp9_init_context_buffers(cm);
     cm->width = width;
     cm->height = height;
-
-    vp9_update_frame_size(cm);
   }
+}
+
+static void setup_frame_size(VP9_COMMON *cm, struct vp9_read_bit_buffer *rb) {
+  int width, height;
+  vp9_read_frame_size(rb, &width, &height);
+  resize_context_buffers(cm, width, height);
+  setup_display_size(cm, rb);
 
   if (vp9_realloc_frame_buffer(
           get_frame_new_buffer(cm), cm->width, cm->height,
@@ -653,13 +653,6 @@
   }
 }
 
-static void setup_frame_size(VP9_COMMON *cm, struct vp9_read_bit_buffer *rb) {
-  int width, height;
-  vp9_read_frame_size(rb, &width, &height);
-  apply_frame_size(cm, width, height);
-  setup_display_size(cm, rb);
-}
-
 static void setup_frame_size_with_refs(VP9_COMMON *cm,
                                        struct vp9_read_bit_buffer *rb) {
   int width, height;
@@ -681,16 +674,23 @@
   // dimensions.
   for (i = 0; i < REFS_PER_FRAME; ++i) {
     RefBuffer *const ref_frame = &cm->frame_refs[i];
-    const int ref_width = ref_frame->buf->y_width;
-    const int ref_height = ref_frame->buf->y_height;
-
-    if (!valid_ref_frame_size(ref_width, ref_height, width, height))
+    if (!valid_ref_frame_size(ref_frame->buf->y_width, ref_frame->buf->y_height,
+                              width, height))
       vpx_internal_error(&cm->error, VPX_CODEC_CORRUPT_FRAME,
                          "Referenced frame has invalid size");
   }
 
-  apply_frame_size(cm, width, height);
+  resize_context_buffers(cm, width, height);
   setup_display_size(cm, rb);
+
+  if (vp9_realloc_frame_buffer(
+          get_frame_new_buffer(cm), cm->width, cm->height,
+          cm->subsampling_x, cm->subsampling_y, VP9_DEC_BORDER_IN_PIXELS,
+          &cm->frame_bufs[cm->new_fb_idx].raw_frame_buffer, cm->get_fb_cb,
+          cm->cb_priv)) {
+    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+                       "Failed to allocate frame buffer");
+  }
 }
 
 static void setup_tile_info(VP9_COMMON *cm, struct vp9_read_bit_buffer *rb) {
@@ -1144,8 +1144,8 @@
     pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1;
 
     for (i = 0; i < REFS_PER_FRAME; ++i) {
-      cm->frame_refs[i].idx = cm->new_fb_idx;
-      cm->frame_refs[i].buf = get_frame_new_buffer(cm);
+      cm->frame_refs[i].idx = -1;
+      cm->frame_refs[i].buf = NULL;
     }
 
     setup_frame_size(cm, rb);
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index d3e0e17..a65c9a0 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -172,7 +172,7 @@
   vp9_cyclic_refresh_free(cpi->cyclic_refresh);
   cpi->cyclic_refresh = NULL;
 
-  vp9_free_frame_buffers(cm);
+  vp9_free_ref_frame_buffers(cm);
   vp9_free_context_buffers(cm);
 
   vp9_free_frame_buffer(&cpi->last_frame_uf);
@@ -424,7 +424,7 @@
 
 static void alloc_ref_frame_buffers(VP9_COMP *cpi) {
   VP9_COMMON *const cm = &cpi->common;
-  if (vp9_alloc_frame_buffers(cm, cm->width, cm->height))
+  if (vp9_alloc_ref_frame_buffers(cm, cm->width, cm->height))
     vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                        "Failed to allocate frame buffers");
 }
@@ -471,7 +471,9 @@
 static void update_frame_size(VP9_COMP *cpi) {
   VP9_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->mb.e_mbd;
-  vp9_update_frame_size(cm);
+
+  vp9_set_mb_mi(cm, cm->width, cm->height);
+  vp9_init_context_buffers(cm);
   init_macroblockd(cm, xd);
 
   if (cpi->use_svc && cpi->svc.number_temporal_layers == 1) {
@@ -2842,6 +2844,8 @@
           PSNR_STATS psnr2;
           double frame_ssim2 = 0, weight = 0;
 #if CONFIG_VP9_POSTPROC
+          // TODO(agrange) Add resizing of post-proc buffer in here when the
+          // encoder is changed to use on-demand buffer allocation.
           vp9_deblock(cm->frame_to_show, &cm->post_proc_buffer,
                       cm->lf.filter_level * 10 / 6);
 #endif
@@ -2959,10 +2963,11 @@
   // always go to the next whole number
   cm->width = (hs - 1 + cpi->oxcf.width * hr) / hs;
   cm->height = (vs - 1 + cpi->oxcf.height * vr) / vs;
-
   assert(cm->width <= cpi->initial_width);
   assert(cm->height <= cpi->initial_height);
+
   update_frame_size(cpi);
+
   return 0;
 }
 
@@ -2995,10 +3000,11 @@
       printf("Warning: Desired height too large, changed to %d\n", cm->height);
     }
   }
-
   assert(cm->width <= cpi->initial_width);
   assert(cm->height <= cpi->initial_height);
+
   update_frame_size(cpi);
+
   return 0;
 }