diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index 841b46521..2434aaff9 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -889,7 +889,12 @@ static void write_tile_info(VP9_COMMON *cm, struct vp9_write_bit_buffer *wb) {
 
 static int get_refresh_mask(VP9_COMP *cpi) {
   if (!cpi->multi_arf_allowed && cpi->refresh_golden_frame &&
-      cpi->rc.is_src_frame_alt_ref && !cpi->use_svc) {
+      cpi->rc.is_src_frame_alt_ref &&
+      (!cpi->use_svc ||      // Add spatial svc base layer case here
+       (cpi->svc.number_temporal_layers == 1 &&
+        cpi->svc.spatial_layer_id == 0 &&
+        cpi->svc.layer_context[0].gold_ref_idx >=0 &&
+        cpi->oxcf.ss_play_alternate[0]))) {
     // Preserve the previously existing golden frame and update the frame in
     // the alt ref slot instead. This is highly specific to the use of
     // alt-ref as a forward reference, and this needs to be generalized as
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index f8d26110d..b1c5326f4 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -1565,7 +1565,12 @@ void vp9_update_reference_frames(VP9_COMP *cpi) {
     ref_cnt_fb(cm->frame_bufs,
                &cm->ref_frame_map[cpi->alt_fb_idx], cm->new_fb_idx);
   } else if (!cpi->multi_arf_allowed && cpi->refresh_golden_frame &&
-             cpi->rc.is_src_frame_alt_ref && !cpi->use_svc) {
+             cpi->rc.is_src_frame_alt_ref &&
+             (!cpi->use_svc ||      // Add spatial svc base layer case here
+              (cpi->svc.number_temporal_layers == 1 &&
+               cpi->svc.spatial_layer_id == 0 &&
+               cpi->svc.layer_context[0].gold_ref_idx >=0 &&
+               cpi->oxcf.ss_play_alternate[0]))) {
     /* Preserve the previously existing golden frame and update the frame in
      * the alt ref slot instead. This is highly specific to the current use of
      * alt-ref as a forward reference, and this needs to be generalized as
@@ -1583,6 +1588,11 @@ void vp9_update_reference_frames(VP9_COMP *cpi) {
     tmp = cpi->alt_fb_idx;
     cpi->alt_fb_idx = cpi->gld_fb_idx;
     cpi->gld_fb_idx = tmp;
+
+    if (cpi->use_svc && cpi->svc.number_temporal_layers == 1) {
+      cpi->svc.layer_context[0].gold_ref_idx = cpi->gld_fb_idx;
+      cpi->svc.layer_context[0].alt_ref_idx = cpi->alt_fb_idx;
+    }
   } else { /* For non key/golden frames */
     if (cpi->refresh_alt_ref_frame) {
       int arf_idx = cpi->alt_fb_idx;
diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c
index 1b574758b..1578bf413 100644
--- a/vp9/encoder/vp9_firstpass.c
+++ b/vp9/encoder/vp9_firstpass.c
@@ -2119,7 +2119,8 @@ void configure_buffer_updates(VP9_COMP *cpi) {
       assert(0);
   }
   if (cpi->use_svc && cpi->svc.number_temporal_layers == 1) {
-    cpi->refresh_golden_frame = 0;
+    if (cpi->svc.layer_context[cpi->svc.spatial_layer_id].gold_ref_idx < 0)
+      cpi->refresh_golden_frame = 0;
     if (cpi->alt_ref_source == NULL)
       cpi->refresh_alt_ref_frame = 0;
   }
@@ -2219,6 +2220,9 @@ void vp9_rc_get_second_pass_params(VP9_COMP *cpi) {
   if (is_spatial_svc) {
     if (cpi->svc.spatial_layer_id == 0) {
       lc->is_key_frame = (cm->frame_type == KEY_FRAME);
+      if (lc->is_key_frame)
+        cpi->ref_frame_flags &=
+            (~VP9_LAST_FLAG & ~VP9_GOLD_FLAG & ~VP9_ALT_FLAG);
     } else {
       cm->frame_type = INTER_FRAME;
       lc->is_key_frame = cpi->svc.layer_context[0].is_key_frame;
diff --git a/vp9/encoder/vp9_ratectrl.c b/vp9/encoder/vp9_ratectrl.c
index 1adbad9cf..1a479f112 100644
--- a/vp9/encoder/vp9_ratectrl.c
+++ b/vp9/encoder/vp9_ratectrl.c
@@ -1238,7 +1238,8 @@ void vp9_rc_get_svc_params(VP9_COMP *cpi) {
 
     if (cpi->use_svc && cpi->svc.number_temporal_layers == 1) {
       cpi->svc.layer_context[cpi->svc.spatial_layer_id].is_key_frame = 1;
-      cpi->ref_frame_flags &= (~VP9_ALT_FLAG);
+      cpi->ref_frame_flags &=
+          (~VP9_LAST_FLAG & ~VP9_GOLD_FLAG & ~VP9_ALT_FLAG);
     }
 
     if (cpi->pass == 0 && cpi->oxcf.rc_mode == VPX_CBR) {
diff --git a/vp9/encoder/vp9_svc_layercontext.c b/vp9/encoder/vp9_svc_layercontext.c
index f8d1a83c3..0e921be8c 100644
--- a/vp9/encoder/vp9_svc_layercontext.c
+++ b/vp9/encoder/vp9_svc_layercontext.c
@@ -35,6 +35,7 @@ void vp9_init_layer_context(VP9_COMP *const cpi) {
     RATE_CONTROL *const lrc = &lc->rc;
     int i;
     lc->current_video_frame_in_layer = 0;
+    lc->layer_size = 0;
     lrc->ni_av_qi = oxcf->worst_allowed_q;
     lrc->total_actual_bits = 0;
     lrc->total_target_vs_actual = 0;
@@ -48,7 +49,6 @@ void vp9_init_layer_context(VP9_COMP *const cpi) {
     for (i = 0; i < RATE_FACTOR_LEVELS; ++i) {
       lrc->rate_correction_factors[i] = 1.0;
     }
-    lc->layer_size = 0;
 
     if (svc->number_temporal_layers > 1) {
       lc->target_bandwidth = oxcf->ts_target_bitrate[layer];
@@ -66,12 +66,17 @@ void vp9_init_layer_context(VP9_COMP *const cpi) {
         lc->alt_ref_idx = alt_ref_idx++;
       else
         lc->alt_ref_idx = -1;
+      lc->gold_ref_idx = -1;
     }
 
     lrc->buffer_level = vp9_rescale((int)(oxcf->starting_buffer_level_ms),
                                     lc->target_bandwidth, 1000);
     lrc->bits_off_target = lrc->buffer_level;
   }
+
+  // Still have extra buffer for base layer golden frame
+  if (svc->number_spatial_layers > 1 && alt_ref_idx < REF_FRAMES)
+    svc->layer_context[0].gold_ref_idx = alt_ref_idx;
 }
 
 // Update the layer context from a change_config() call.
@@ -266,21 +271,25 @@ static int copy_svc_params(VP9_COMP *const cpi, struct lookahead_entry *buf) {
   layer_param = &buf->svc_params[layer_id];
   cpi->svc.spatial_layer_id = layer_param->spatial_layer;
   cpi->svc.temporal_layer_id = layer_param->temporal_layer;
+  cpi->ref_frame_flags = VP9_ALT_FLAG | VP9_GOLD_FLAG | VP9_LAST_FLAG;
+
+  lc = &cpi->svc.layer_context[cpi->svc.spatial_layer_id];
 
   cpi->lst_fb_idx = cpi->svc.spatial_layer_id;
 
   if (cpi->svc.spatial_layer_id < 1)
-    cpi->gld_fb_idx = cpi->lst_fb_idx;
+      cpi->gld_fb_idx = lc->gold_ref_idx >= 0 ?
+                        lc->gold_ref_idx : cpi->lst_fb_idx;
   else
     cpi->gld_fb_idx = cpi->svc.spatial_layer_id - 1;
 
-  lc = &cpi->svc.layer_context[cpi->svc.spatial_layer_id];
-
   if (lc->current_video_frame_in_layer == 0) {
-    if (cpi->svc.spatial_layer_id >= 2)
+    if (cpi->svc.spatial_layer_id >= 2) {
       cpi->alt_fb_idx = cpi->svc.spatial_layer_id - 2;
-    else
+    } else {
       cpi->alt_fb_idx = cpi->lst_fb_idx;
+      cpi->ref_frame_flags &= (~VP9_LAST_FLAG & ~VP9_ALT_FLAG);
+    }
   } else {
     if (cpi->oxcf.ss_play_alternate[cpi->svc.spatial_layer_id]) {
       cpi->alt_fb_idx = lc->alt_ref_idx;
diff --git a/vp9/encoder/vp9_svc_layercontext.h b/vp9/encoder/vp9_svc_layercontext.h
index 7b533e467..801449b6f 100644
--- a/vp9/encoder/vp9_svc_layercontext.h
+++ b/vp9/encoder/vp9_svc_layercontext.h
@@ -31,6 +31,7 @@ typedef struct {
   vpx_svc_parameters_t svc_params_received;
   struct lookahead_entry  *alt_ref_source;
   int alt_ref_idx;
+  int gold_ref_idx;
   int has_alt_frame;
   size_t layer_size;
 } LAYER_CONTEXT;