diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 942ba7f20a..c46c372466 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1561,6 +1561,7 @@ again:
                 if(!idr_cleared)
                     idr(h); // FIXME ensure we don't lose some frames if there is reordering
                 idr_cleared = 1;
+                h->has_recovery_point = 1;
             case NAL_SLICE:
                 init_get_bits(&hx->gb, ptr, bit_length);
                 hx->intra_gb_ptr      =
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index a25b5785a9..c057c9709d 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -727,6 +727,8 @@ typedef struct H264Context {
 
     int frame_recovered;    ///< Initial frame has been completely recovered
 
+    int has_recovery_point;
+
     int luma_weight_flag[2];    ///< 7.4.3.2 luma_weight_lX_flag
     int chroma_weight_flag[2];  ///< 7.4.3.2 chroma_weight_lX_flag
 
diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 9bb065d24d..de1f60a41f 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -775,7 +775,7 @@ int ff_h264_execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count)
     if (   err >= 0
         && h->long_ref_count==0
         && (h->short_ref_count<=2 || h->pps.ref_count[0] <= 1 && h->pps.ref_count[1] <= 1 && pps_count == 1)
-        && h->pps.ref_count[0]<=2 + (h->picture_structure != PICT_FRAME)
+        && h->pps.ref_count[0]<=2 + (h->picture_structure != PICT_FRAME) + (2*!h->has_recovery_point)
         && h->cur_pic_ptr->f.pict_type == AV_PICTURE_TYPE_I){
         h->cur_pic_ptr->recovered |= 1;
         if(!h->avctx->has_b_frames)
diff --git a/libavcodec/h264_sei.c b/libavcodec/h264_sei.c
index 4e2f981181..88bf0f4977 100644
--- a/libavcodec/h264_sei.c
+++ b/libavcodec/h264_sei.c
@@ -183,6 +183,8 @@ static int decode_recovery_point(H264Context *h)
     if (h->avctx->debug & FF_DEBUG_PICT_INFO)
         av_log(h->avctx, AV_LOG_DEBUG, "sei_recovery_frame_cnt: %d\n", h->sei_recovery_frame_cnt);
 
+    h->has_recovery_point = 1;
+
     return 0;
 }