diff --git a/libavcodec/xsubdec.c b/libavcodec/xsubdec.c
index 2e2cef550d..3838722c1f 100644
--- a/libavcodec/xsubdec.c
+++ b/libavcodec/xsubdec.c
@@ -30,7 +30,7 @@ static av_cold int decode_init(AVCodecContext *avctx) {
 static const uint8_t tc_offsets[9] = { 0, 1, 3, 4, 6, 7, 9, 10, 11 };
 static const uint8_t tc_muls[9] = { 10, 6, 10, 6, 10, 10, 10, 10, 1 };
 
-static uint64_t parse_timecode(const uint8_t *buf) {
+static int64_t parse_timecode(const uint8_t *buf, int64_t packet_time) {
     int i;
     int64_t ms = 0;
     if (buf[2] != ':' || buf[5] != ':' || buf[8] != '.')
@@ -40,7 +40,7 @@ static uint64_t parse_timecode(const uint8_t *buf) {
         if (c > 9) return AV_NOPTS_VALUE;
         ms = (ms + c) * tc_muls[i];
     }
-    return ms;
+    return ms - packet_time;
 }
 
 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
@@ -51,6 +51,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     const uint8_t *buf_end = buf + buf_size;
     uint8_t *bitmap;
     int w, h, x, y, rlelen, i;
+    int64_t packet_time = 0;
     GetBitContext gb;
 
     memset(sub, 0, sizeof(*sub));
@@ -66,8 +67,10 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         av_log(avctx, AV_LOG_ERROR, "invalid time code\n");
         return -1;
     }
-    sub->start_display_time = parse_timecode(buf +  1);
-    sub->end_display_time   = parse_timecode(buf + 14);
+    if (avpkt->pts != AV_NOPTS_VALUE)
+        packet_time = av_rescale_q(avpkt->pts, AV_TIME_BASE_Q, (AVRational){1, 1000});
+    sub->start_display_time = parse_timecode(buf +  1, packet_time);
+    sub->end_display_time   = parse_timecode(buf + 14, packet_time);
     buf += 27;
 
     // read header