diff --git a/libavformat/mov.c b/libavformat/mov.c
index 785d284b86..0e9566fe3a 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -1830,7 +1830,7 @@ static void mov_build_index(MOVContext *mov, AVStream *st)
 
         current_dts -= sc->dts_shift;
 
-        if (!sc->sample_count)
+        if (!sc->sample_count || st->nb_index_entries)
             return;
         if (sc->sample_count >= UINT_MAX / sizeof(*st->index_entries))
             return;