diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c
index e9b5aa25884db085ef9bc4899c5c7fcde7e2928c..71c98ecb53653c30f6255065ef04d36cda91e2df 100644
--- a/vp9/encoder/vp9_firstpass.c
+++ b/vp9/encoder/vp9_firstpass.c
@@ -2155,6 +2155,10 @@ void configure_buffer_updates(VP9_COMP *cpi) {
       break;
   }
   if (is_two_pass_svc(cpi)) {
+    if (cpi->svc.temporal_layer_id > 0) {
+      cpi->refresh_last_frame = 0;
+      cpi->refresh_golden_frame = 0;
+    }
     if (cpi->svc.layer_context[cpi->svc.spatial_layer_id].gold_ref_idx < 0)
       cpi->refresh_golden_frame = 0;
     if (cpi->alt_ref_source == NULL)
diff --git a/vp9/encoder/vp9_svc_layercontext.c b/vp9/encoder/vp9_svc_layercontext.c
index 7545d87b30caf95238b2abbe23ff67455833d2ec..8c735abee3e4218fe36ee1a593d2e5bf711c5654 100644
--- a/vp9/encoder/vp9_svc_layercontext.c
+++ b/vp9/encoder/vp9_svc_layercontext.c
@@ -19,7 +19,7 @@ void vp9_init_layer_context(VP9_COMP *const cpi) {
   const VP9EncoderConfig *const oxcf = &cpi->oxcf;
   int layer;
   int layer_end;
-  int alt_ref_idx = svc->number_spatial_layers * svc->number_temporal_layers;
+  int alt_ref_idx = svc->number_spatial_layers;
 
   svc->spatial_layer_id = 0;
   svc->temporal_layer_id = 0;
@@ -286,30 +286,17 @@ static int copy_svc_params(VP9_COMP *const cpi, struct lookahead_entry *buf) {
     count >>= 1;
   }
 
-  cpi->lst_fb_idx =
-      cpi->svc.spatial_layer_id * cpi->svc.number_temporal_layers +
-      cpi->svc.temporal_layer_id;
-  if (lc->frames_from_key_frame < cpi->svc.number_temporal_layers)
-    cpi->ref_frame_flags &= ~VP9_LAST_FLAG;
-
-  if (cpi->svc.spatial_layer_id == 0) {
-    if (cpi->svc.temporal_layer_id == 0)
-      cpi->gld_fb_idx = lc->gold_ref_idx >= 0 ?
-                        lc->gold_ref_idx : cpi->lst_fb_idx;
-    else
-      cpi->gld_fb_idx = cpi->lst_fb_idx - 1;
-  } else {
-    if (cpi->svc.temporal_layer_id == 0)
-      cpi->gld_fb_idx = cpi->svc.spatial_layer_id -
-                        cpi->svc.number_temporal_layers;
-    else
-      cpi->gld_fb_idx = cpi->lst_fb_idx - 1;
-  }
+  cpi->lst_fb_idx = cpi->svc.spatial_layer_id;
+
+  if (cpi->svc.spatial_layer_id == 0)
+    cpi->gld_fb_idx = (lc->gold_ref_idx >= 0) ?
+                      lc->gold_ref_idx : cpi->lst_fb_idx;
+  else
+    cpi->gld_fb_idx = cpi->svc.spatial_layer_id - 1;
 
   if (lc->current_video_frame_in_layer == 0) {
     if (cpi->svc.spatial_layer_id >= 2) {
-      cpi->alt_fb_idx =
-          cpi->svc.spatial_layer_id - 2 * cpi->svc.number_temporal_layers;
+      cpi->alt_fb_idx = cpi->svc.spatial_layer_id - 2;
     } else {
       cpi->alt_fb_idx = cpi->lst_fb_idx;
       cpi->ref_frame_flags &= (~VP9_LAST_FLAG & ~VP9_ALT_FLAG);
@@ -331,8 +318,7 @@ static int copy_svc_params(VP9_COMP *const cpi, struct lookahead_entry *buf) {
             lc_lower->alt_ref_source != NULL)
           cpi->alt_fb_idx = lc_lower->alt_ref_idx;
         else if (cpi->svc.spatial_layer_id >= 2)
-          cpi->alt_fb_idx =
-              cpi->svc.spatial_layer_id - 2 * cpi->svc.number_temporal_layers;
+          cpi->alt_fb_idx = cpi->svc.spatial_layer_id - 2;
         else
           cpi->alt_fb_idx = cpi->lst_fb_idx;
       }
diff --git a/vp9/vp9_cx_iface.c b/vp9/vp9_cx_iface.c
index 4b24960ef306737674f0dd05221afaa279195482..0d6ad26a64312623c6604e78b1f73f2d6fa7f809 100644
--- a/vp9/vp9_cx_iface.c
+++ b/vp9/vp9_cx_iface.c
@@ -178,8 +178,6 @@ static vpx_codec_err_t validate_config(vpx_codec_alg_priv_t *ctx,
   }
 
 #if CONFIG_SPATIAL_SVC
-  if (cfg->ss_number_layers * cfg->ts_number_layers > REF_FRAMES)
-    ERROR("Too many layers. Maximum 8 layers could be set");
 
   if ((cfg->ss_number_layers > 1 || cfg->ts_number_layers > 1) &&
       cfg->g_pass == VPX_RC_LAST_PASS) {
@@ -188,8 +186,7 @@ static vpx_codec_err_t validate_config(vpx_codec_alg_priv_t *ctx,
       if (cfg->ss_enable_auto_alt_ref[i])
         ++alt_ref_sum;
     }
-    if (alt_ref_sum >
-        REF_FRAMES - cfg->ss_number_layers * cfg->ts_number_layers)
+    if (alt_ref_sum > REF_FRAMES - cfg->ss_number_layers)
       ERROR("Not enough ref buffers for svc alt ref frames");
     if ((cfg->ss_number_layers > 3 ||
          cfg->ss_number_layers * cfg->ts_number_layers > 4) &&