diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index c23e150..3c4bdc9 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -1241,12 +1241,21 @@
       vpx_wb_write_literal(wb, vp9_get_refresh_mask(cpi), REF_FRAMES);
       write_frame_size(cm, wb);
     } else {
+      static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
+                                        VP9_ALT_FLAG };
+      const MV_REFERENCE_FRAME first_ref = get_first_ref_frame(cpi);
+      const int first_ref_map_idx = get_ref_frame_map_idx(cpi, first_ref);
       MV_REFERENCE_FRAME ref_frame;
       vpx_wb_write_literal(wb, vp9_get_refresh_mask(cpi), REF_FRAMES);
-      for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-        assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
-        vpx_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame),
-                             REF_FRAMES_LOG2);
+
+      // If a reference frame is not referenced, then set the index for that
+      // reference to the first one used/referenced.
+      for (ref_frame = LAST_FRAME; ref_frame < MAX_REF_FRAMES; ++ref_frame) {
+        const int referenced = cpi->ref_frame_flags & flag_list[ref_frame];
+        const int map_idx = referenced ? get_ref_frame_map_idx(cpi, ref_frame)
+                                       : first_ref_map_idx;
+        assert(map_idx != INVALID_IDX);
+        vpx_wb_write_literal(wb, map_idx, REF_FRAMES_LOG2);
         vpx_wb_write_bit(wb, cm->ref_frame_sign_bias[ref_frame]);
       }
 
diff --git a/vp9/encoder/vp9_encoder.h b/vp9/encoder/vp9_encoder.h
index 9774a64..1bca7de 100644
--- a/vp9/encoder/vp9_encoder.h
+++ b/vp9/encoder/vp9_encoder.h
@@ -1196,14 +1196,24 @@
          (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref);
 }
 
+static INLINE MV_REFERENCE_FRAME get_first_ref_frame(VP9_COMP *const cpi) {
+  static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
+                                    VP9_ALT_FLAG };
+  MV_REFERENCE_FRAME ref_frame = LAST_FRAME;
+  while (ref_frame < MAX_REF_FRAMES) {
+    if (cpi->ref_frame_flags & flag_list[ref_frame]) break;
+    ref_frame++;
+  }
+  return ref_frame;
+}
+
 static INLINE int get_ref_frame_map_idx(const VP9_COMP *cpi,
                                         MV_REFERENCE_FRAME ref_frame) {
-  if (ref_frame == LAST_FRAME) {
-    return cpi->lst_fb_idx;
-  } else if (ref_frame == GOLDEN_FRAME) {
-    return cpi->gld_fb_idx;
-  } else {
-    return cpi->alt_fb_idx;
+  switch (ref_frame) {
+    case LAST_FRAME: return cpi->lst_fb_idx;
+    case GOLDEN_FRAME: return cpi->gld_fb_idx;
+    case ALTREF_FRAME: return cpi->alt_fb_idx;
+    default: return INVALID_IDX;
   }
 }
 
diff --git a/vp9/encoder/vp9_svc_layercontext.c b/vp9/encoder/vp9_svc_layercontext.c
index ad3a8f7..f01cb17 100644
--- a/vp9/encoder/vp9_svc_layercontext.c
+++ b/vp9/encoder/vp9_svc_layercontext.c
@@ -73,7 +73,7 @@
     svc->downsample_filter_type[sl] = BILINEAR;
     svc->downsample_filter_phase[sl] = 8;  // Set to 8 for averaging filter.
     svc->framedrop_thresh[sl] = oxcf->drop_frames_water_mark;
-    svc->fb_idx_upd_tl0[sl] = -1;
+    svc->fb_idx_upd_tl0[sl] = INVALID_IDX;
     svc->drop_count[sl] = 0;
     svc->spatial_layer_sync[sl] = 0;
     svc->force_drop_constrained_from_above[sl] = 0;
@@ -462,32 +462,21 @@
   // fb_idx for that reference to the first one used/referenced.
   // This is to avoid setting fb_idx for a reference to a slot that is not
   // used/needed (i.e., since that reference is not referenced or refreshed).
-  static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
-                                    VP9_ALT_FLAG };
-  MV_REFERENCE_FRAME ref_frame;
-  MV_REFERENCE_FRAME first_ref = 0;
-  int first_fb_idx = 0;
-  int fb_idx[3] = { cpi->lst_fb_idx, cpi->gld_fb_idx, cpi->alt_fb_idx };
-  for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
-    if (cpi->ref_frame_flags & flag_list[ref_frame]) {
-      first_ref = ref_frame;
-      first_fb_idx = fb_idx[ref_frame - 1];
-      break;
+  const MV_REFERENCE_FRAME first_ref = get_first_ref_frame(cpi);
+  const int map_idx = get_ref_frame_map_idx(cpi, first_ref);
+  if (map_idx != INVALID_IDX) {
+    if (!(cpi->ref_frame_flags & VP9_LAST_FLAG ||
+          cpi->ext_refresh_last_frame)) {
+      cpi->lst_fb_idx = map_idx;
     }
-  }
-  if (first_ref > 0) {
-    if (first_ref != LAST_FRAME &&
-        !(cpi->ref_frame_flags & flag_list[LAST_FRAME]) &&
-        !cpi->ext_refresh_last_frame)
-      cpi->lst_fb_idx = first_fb_idx;
-    else if (first_ref != GOLDEN_FRAME &&
-             !(cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) &&
-             !cpi->ext_refresh_golden_frame)
-      cpi->gld_fb_idx = first_fb_idx;
-    else if (first_ref != ALTREF_FRAME &&
-             !(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]) &&
-             !cpi->ext_refresh_alt_ref_frame)
-      cpi->alt_fb_idx = first_fb_idx;
+    if (!(cpi->ref_frame_flags & VP9_GOLD_FLAG ||
+          cpi->ext_refresh_golden_frame)) {
+      cpi->gld_fb_idx = map_idx;
+    }
+    if (!(cpi->ref_frame_flags & VP9_ALT_FLAG ||
+          cpi->ext_refresh_alt_ref_frame)) {
+      cpi->alt_fb_idx = map_idx;
+    }
   }
 }
 
@@ -716,9 +705,9 @@
   int sl = svc->spatial_layer_id = svc->spatial_layer_to_encode;
   cpi->svc.temporal_layer_id = cpi->svc.temporal_layer_id_per_spatial[sl];
   cpi->ext_refresh_frame_flags_pending = 1;
-  cpi->lst_fb_idx = svc->lst_fb_idx[sl];
-  cpi->gld_fb_idx = svc->gld_fb_idx[sl];
-  cpi->alt_fb_idx = svc->alt_fb_idx[sl];
+  if (svc->reference_last[sl]) cpi->lst_fb_idx = svc->lst_fb_idx[sl];
+  if (svc->reference_golden[sl]) cpi->gld_fb_idx = svc->gld_fb_idx[sl];
+  if (svc->reference_altref[sl]) cpi->alt_fb_idx = svc->alt_fb_idx[sl];
   cpi->ext_refresh_last_frame = 0;
   cpi->ext_refresh_golden_frame = 0;
   cpi->ext_refresh_alt_ref_frame = 0;
@@ -875,9 +864,9 @@
     // flags are passed via the encode call (bypass mode). Issue is that we're
     // resetting ext_refresh_frame_flags_pending to 0 on frame drops.
     if (svc->temporal_layering_mode != VP9E_TEMPORAL_LAYERING_MODE_BYPASS) {
-      memset(&svc->lst_fb_idx, -1, sizeof(svc->lst_fb_idx));
-      memset(&svc->gld_fb_idx, -1, sizeof(svc->lst_fb_idx));
-      memset(&svc->alt_fb_idx, -1, sizeof(svc->lst_fb_idx));
+      memset(&svc->lst_fb_idx, INVALID_IDX, sizeof(svc->lst_fb_idx));
+      memset(&svc->gld_fb_idx, INVALID_IDX, sizeof(svc->lst_fb_idx));
+      memset(&svc->alt_fb_idx, INVALID_IDX, sizeof(svc->lst_fb_idx));
       // These are set by API before the superframe is encoded and they are
       // passed to encoder layer by layer. Don't reset them on layer 0 in bypass
       // mode.
@@ -970,7 +959,7 @@
 
   if (svc->temporal_layering_mode != VP9E_TEMPORAL_LAYERING_MODE_BYPASS &&
       svc->last_layer_dropped[svc->spatial_layer_id] &&
-      svc->fb_idx_upd_tl0[svc->spatial_layer_id] != -1 &&
+      svc->fb_idx_upd_tl0[svc->spatial_layer_id] != INVALID_IDX &&
       !svc->layer_context[svc->temporal_layer_id].is_key_frame) {
     // For fixed/non-flexible mode, if the previous frame (same spatial layer
     // from previous superframe) was dropped, make sure the lst_fb_idx
diff --git a/vpx/vp8cx.h b/vpx/vp8cx.h
index 47c38d3..b3c50a9 100644
--- a/vpx/vp8cx.h
+++ b/vpx/vp8cx.h
@@ -897,13 +897,16 @@
   int alt_fb_idx[VPX_SS_MAX_LAYERS];         /**< Altref buffer index. */
   int update_buffer_slot[VPX_SS_MAX_LAYERS]; /**< Update reference frames. */
   // TODO(jianj): Remove update_last/golden/alt_ref, these are deprecated.
-  int update_last[VPX_SS_MAX_LAYERS];       /**< Update last. */
-  int update_golden[VPX_SS_MAX_LAYERS];     /**< Update golden. */
-  int update_alt_ref[VPX_SS_MAX_LAYERS];    /**< Update altref. */
-  int reference_last[VPX_SS_MAX_LAYERS];    /**< Last as reference. */
-  int reference_golden[VPX_SS_MAX_LAYERS];  /**< Golden as reference. */
-  int reference_alt_ref[VPX_SS_MAX_LAYERS]; /**< Altref as reference. */
-  int64_t duration[VPX_SS_MAX_LAYERS];      /**< Duration per spatial layer. */
+  int update_last[VPX_SS_MAX_LAYERS];    /**< Update last. */
+  int update_golden[VPX_SS_MAX_LAYERS];  /**< Update golden. */
+  int update_alt_ref[VPX_SS_MAX_LAYERS]; /**< Update altref. */
+  int reference_last[VPX_SS_MAX_LAYERS];
+  /**< Last as reference. Use first referenced index if FALSE. */
+  int reference_golden[VPX_SS_MAX_LAYERS];
+  /**< Golden as reference. Use first referenced index if FALSE. */
+  int reference_alt_ref[VPX_SS_MAX_LAYERS];
+  /**< Altref as reference. Use first referenced index if FALSE. */
+  int64_t duration[VPX_SS_MAX_LAYERS]; /**< Duration per spatial layer. */
 } vpx_svc_ref_frame_config_t;
 
 /*!\brief VP9 svc frame dropping mode.