diff --git a/vp9/decoder/decodemv.c b/vp9/decoder/decodemv.c
index e9f37958c..e7560765a 100644
--- a/vp9/decoder/decodemv.c
+++ b/vp9/decoder/decodemv.c
@@ -717,7 +717,11 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
   }
 
   // Read the reference frame
-  mbmi->ref_frame = read_ref_frame(pbi, bc, mbmi->segment_id);
+  if (vp9_segfeature_active(xd, mbmi->segment_id, SEG_LVL_MODE)
+      && vp9_get_segdata(xd, mbmi->segment_id, SEG_LVL_MODE) < NEARESTMV)
+    mbmi->ref_frame = INTRA_FRAME;
+  else
+    mbmi->ref_frame = read_ref_frame(pbi, bc, mbmi->segment_id);
 
   // If reference frame is an Inter frame
   if (mbmi->ref_frame) {
diff --git a/vp9/encoder/bitstream.c b/vp9/encoder/bitstream.c
index d88cfd055..65614a176 100644
--- a/vp9/encoder/bitstream.c
+++ b/vp9/encoder/bitstream.c
@@ -863,7 +863,12 @@ static void pack_inter_mode_mvs(VP9_COMP *const cpi, vp9_writer *const bc) {
         }
 
         // Encode the reference frame.
-        encode_ref_frame(bc, pc, xd, segment_id, rf);
+        if (!vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE)
+            || vp9_get_segdata(xd, segment_id, SEG_LVL_MODE) >= NEARESTMV) {
+          encode_ref_frame(bc, pc, xd, segment_id, rf);
+        } else {
+          assert(rf == INTRA_FRAME);
+        }
 
         if (rf == INTRA_FRAME) {
 #ifdef ENTROPY_STATS