diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 2a522fd3ed..3e29e881c5 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1010,8 +1010,6 @@ int ff_set_ref_count(H264Context *h, H264SliceContext *sl)
         unsigned max[2];
         max[0] = max[1] = h->picture_structure == PICT_FRAME ? 15 : 31;
 
-        if (sl->slice_type_nos == AV_PICTURE_TYPE_B)
-            sl->direct_spatial_mv_pred = get_bits1(&sl->gb);
         num_ref_idx_active_override_flag = get_bits1(&sl->gb);
 
         if (num_ref_idx_active_override_flag) {
diff --git a/libavcodec/h264_parser.c b/libavcodec/h264_parser.c
index 6cf4a236a0..418fe0a488 100644
--- a/libavcodec/h264_parser.c
+++ b/libavcodec/h264_parser.c
@@ -150,6 +150,9 @@ static int scan_mmco_reset(AVCodecParserContext *s)
     if (h->pps.redundant_pic_cnt_present)
         get_ue_golomb(&sl->gb); // redundant_pic_count
 
+    if (sl->slice_type_nos == AV_PICTURE_TYPE_B)
+        get_bits1(&sl->gb); // direct_spatial_mv_pred
+
     if (ff_set_ref_count(h, sl) < 0)
         return AVERROR_INVALIDDATA;
 
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 4e744ea003..0fcb5631b2 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -1689,6 +1689,9 @@ int ff_h264_decode_slice_header(H264Context *h, H264SliceContext *sl)
     if (h->pps.redundant_pic_cnt_present)
         sl->redundant_pic_count = get_ue_golomb(&sl->gb);
 
+    if (sl->slice_type_nos == AV_PICTURE_TYPE_B)
+        sl->direct_spatial_mv_pred = get_bits1(&sl->gb);
+
     ret = ff_set_ref_count(h, sl);
     if (ret < 0)
         return ret;