diff --git a/tools/ismindex.c b/tools/ismindex.c
index 9b4d5717e7..b1877904b3 100644
--- a/tools/ismindex.c
+++ b/tools/ismindex.c
@@ -298,8 +298,6 @@ static int handle_file(struct Tracks *tracks, const char *file, int split)
         fprintf(stderr, "No streams found in %s\n", file);
         goto fail;
     }
-    if (!tracks->duration)
-        tracks->duration = ctx->duration;
 
     for (i = 0; i < ctx->nb_streams; i++) {
         struct Track **temp;
@@ -338,6 +336,10 @@ static int handle_file(struct Tracks *tracks, const char *file, int split)
             continue;
         }
 
+        tracks->duration = FFMAX(tracks->duration,
+                                 av_rescale_rnd(track->duration, AV_TIME_BASE,
+                                                track->timescale, AV_ROUND_UP));
+
         if (track->is_audio) {
             if (tracks->audio_track < 0)
                 tracks->audio_track = tracks->nb_tracks;