diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 517674e5f..9cb697e26 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2632,6 +2632,12 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     int64_t total_sse = INT64_MAX;
     int early_term = 0;
 
+    this_mode = vp9_mode_order[mode_index].mode;
+    ref_frame = vp9_mode_order[mode_index].ref_frame[0];
+    if (ref_frame != INTRA_FRAME && !(inter_mode_mask & (1 << this_mode)))
+      continue;
+    second_ref_frame = vp9_mode_order[mode_index].ref_frame[1];
+
     // Look at the reference frame of the best mode so far and set the
     // skip mask to look at a subset of the remaining modes.
     if (mode_index == mode_skip_start && best_mode_index >= 0) {
@@ -2653,6 +2659,13 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
           break;
       }
     }
+
+    if (cpi->sf.alt_ref_search_fp && cpi->rc.is_src_frame_alt_ref) {
+      mode_skip_mask = 0;
+      if (!(ref_frame == ALTREF_FRAME && second_ref_frame == NONE))
+        continue;
+    }
+
     if (mode_skip_mask & (1 << mode_index))
       continue;
 
@@ -2661,12 +2674,6 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
                             rd_thresh_freq_fact[mode_index]))
       continue;
 
-    this_mode = vp9_mode_order[mode_index].mode;
-    ref_frame = vp9_mode_order[mode_index].ref_frame[0];
-    if (ref_frame != INTRA_FRAME && !(inter_mode_mask & (1 << this_mode)))
-      continue;
-    second_ref_frame = vp9_mode_order[mode_index].ref_frame[1];
-
     if (cpi->sf.motion_field_mode_search) {
       const int mi_width  = MIN(num_8x8_blocks_wide_lookup[bsize],
                                 tile->mi_col_end - mi_col);
diff --git a/vp9/encoder/vp9_speed_features.c b/vp9/encoder/vp9_speed_features.c
index 67b6e269e..a2f4583a5 100644
--- a/vp9/encoder/vp9_speed_features.c
+++ b/vp9/encoder/vp9_speed_features.c
@@ -119,6 +119,7 @@ static void set_good_speed_feature(VP9_COMP *cpi, VP9_COMMON *cm,
     sf->adaptive_pred_interp_filter = 0;
     sf->cb_partition_search = frame_is_boosted(cpi) ? 0 : 1;
     sf->cb_pred_filter_search = 1;
+    sf->alt_ref_search_fp = 1;
     sf->motion_field_mode_search = frame_is_boosted(cpi) ? 0 : 1;
     sf->lf_motion_threshold = LOW_MOTION_THRESHOLD;
     sf->last_partitioning_redo_frequency = 3;
@@ -347,6 +348,7 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
   sf->cb_pred_filter_search = 0;
   sf->cb_partition_search = 0;
   sf->motion_field_mode_search = 0;
+  sf->alt_ref_search_fp = 0;
   sf->use_quant_fp = 0;
   sf->reference_masking = 0;
   sf->partition_search_type = SEARCH_PARTITION;
diff --git a/vp9/encoder/vp9_speed_features.h b/vp9/encoder/vp9_speed_features.h
index 8edcb1d72..3e8215f0c 100644
--- a/vp9/encoder/vp9_speed_features.h
+++ b/vp9/encoder/vp9_speed_features.h
@@ -291,6 +291,8 @@ typedef struct SPEED_FEATURES {
 
   int motion_field_mode_search;
 
+  int alt_ref_search_fp;
+
   // Fast quantization process path
   int use_quant_fp;