diff --git a/libavcodec/dxva2_h264.c b/libavcodec/dxva2_h264.c
index 22c76df601..b0b126a70a 100644
--- a/libavcodec/dxva2_h264.c
+++ b/libavcodec/dxva2_h264.c
@@ -46,7 +46,7 @@ static void fill_picture_parameters(struct dxva_context *ctx, const H264Context
 {
     const MpegEncContext *s = &h->s;
     const Picture *current_picture = s->current_picture_ptr;
-    int i;
+    int i, j;
 
     memset(pp, 0, sizeof(*pp));
     /* Configure current picture */
@@ -56,16 +56,16 @@ static void fill_picture_parameters(struct dxva_context *ctx, const H264Context
     /* Configure the set of references */
     pp->UsedForReferenceFlags  = 0;
     pp->NonExistingFrameFlags  = 0;
-    for (i = 0; i < FF_ARRAY_ELEMS(pp->RefFrameList); i++) {
-        if (i < h->short_ref_count + h->long_ref_count) {
+    for (i = 0, j = 0; i < FF_ARRAY_ELEMS(pp->RefFrameList); i++) {
             const Picture *r;
-            if (i < h->short_ref_count) {
-                r = h->short_ref[i];
-                assert(!r->long_ref);
+            if (j < h->short_ref_count) {
+                r = h->short_ref[j++];
             } else {
-                r = h->long_ref[i - h->short_ref_count];
-                assert(r->long_ref);
+                r = NULL;
+                while (!r && j < h->short_ref_count + 16)
+                    r = h->long_ref[j++ - h->short_ref_count];
             }
+        if (r) {
             fill_picture_entry(&pp->RefFrameList[i],
                                ff_dxva2_get_surface_index(ctx, r),
                                r->long_ref != 0);