diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 7e62881740..3c07477d39 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1776,8 +1776,11 @@ static int decode_update_thread_context(AVCodecContext *dst,
 
     h->cur_pic_ptr = REBASE_PICTURE(h1->cur_pic_ptr, h, h1);
     unref_picture(h, &h->cur_pic);
-    if ((ret = ref_picture(h, &h->cur_pic, &h1->cur_pic)) < 0)
-        return ret;
+    if (h1->cur_pic.f.buf[0]) {
+        ret = ref_picture(h, &h->cur_pic, &h1->cur_pic);
+        if (ret < 0)
+            return ret;
+    }
 
     h->workaround_bugs = h1->workaround_bugs;
     h->low_delay       = h1->low_delay;