diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index 7c10f33e6e..3a6bfa9ff2 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -45,6 +45,7 @@
 #include "libavutil/intfloat.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/lzo.h"
+#include "libavutil/mathematics.h"
 
 #include "libavcodec/bytestream.h"
 #include "libavcodec/mpeg4audio.h"
@@ -170,12 +171,12 @@ typedef struct {
     uint64_t default_duration;
     uint64_t flag_default;
     uint64_t flag_forced;
-    uint64_t codec_delay;
     uint64_t seek_preroll;
     MatroskaTrackVideo video;
     MatroskaTrackAudio audio;
     MatroskaTrackOperation operation;
     EbmlList encodings;
+    uint64_t codec_delay;
 
     AVStream *stream;
     int64_t end_timecode;
@@ -410,6 +411,7 @@ static EbmlSyntax matroska_track[] = {
     { MATROSKA_ID_TRACKTYPE,             EBML_UINT,  0, offsetof(MatroskaTrack, type) },
     { MATROSKA_ID_CODECID,               EBML_STR,   0, offsetof(MatroskaTrack, codec_id) },
     { MATROSKA_ID_CODECPRIVATE,          EBML_BIN,   0, offsetof(MatroskaTrack, codec_priv) },
+    { MATROSKA_ID_CODECDELAY,            EBML_UINT,  0, offsetof(MatroskaTrack, codec_delay) },
     { MATROSKA_ID_TRACKLANGUAGE,         EBML_UTF8,  0, offsetof(MatroskaTrack, language),     { .s = "eng" } },
     { MATROSKA_ID_TRACKDEFAULTDURATION,  EBML_UINT,  0, offsetof(MatroskaTrack, default_duration) },
     { MATROSKA_ID_TRACKTIMECODESCALE,    EBML_FLOAT, 0, offsetof(MatroskaTrack, time_scale),   { .f = 1.0 } },
@@ -420,7 +422,6 @@ static EbmlSyntax matroska_track[] = {
     { MATROSKA_ID_TRACKOPERATION,        EBML_NEST,  0, offsetof(MatroskaTrack, operation),    { .n = matroska_track_operation } },
     { MATROSKA_ID_TRACKCONTENTENCODINGS, EBML_NEST,  0, 0,                                     { .n = matroska_track_encodings } },
     { MATROSKA_ID_TRACKMAXBLKADDID,      EBML_UINT,  0, offsetof(MatroskaTrack, max_block_additional_id) },
-    { MATROSKA_ID_CODECDELAY,            EBML_UINT,  0, offsetof(MatroskaTrack, codec_delay) },
     { MATROSKA_ID_SEEKPREROLL,           EBML_UINT,  0, offsetof(MatroskaTrack, seek_preroll) },
     { MATROSKA_ID_TRACKFLAGENABLED,      EBML_NONE },
     { MATROSKA_ID_TRACKFLAGLACING,       EBML_NONE },
@@ -1931,6 +1932,11 @@ static int matroska_read_header(AVFormatContext *s)
         avpriv_set_pts_info(st, 64, matroska->time_scale * track->time_scale,
                             1000 * 1000 * 1000);    /* 64 bit pts in ns */
 
+        /* convert the delay from ns to the track timebase */
+        track->codec_delay = av_rescale_q(track->codec_delay,
+                                          (AVRational){ 1, 1000000000 },
+                                          st->time_base);
+
         st->codec->codec_id = codec_id;
 
         if (strcmp(track->language, "und"))
@@ -2013,7 +2019,7 @@ static int matroska_read_header(AVFormatContext *s)
                 st->need_parsing = AVSTREAM_PARSE_HEADERS;
             if (track->codec_delay > 0) {
                 st->codec->delay = av_rescale_q(track->codec_delay,
-                                                (AVRational){1, 1000000000},
+                                                st->time_base,
                                                 (AVRational){1, st->codec->sample_rate});
             }
             if (track->seek_preroll > 0) {
@@ -2702,7 +2708,7 @@ static int matroska_parse_block(MatroskaDemuxContext *matroska, uint8_t *data,
 
     if (cluster_time != (uint64_t) -1 &&
         (block_time >= 0 || cluster_time >= -block_time)) {
-        timecode = cluster_time + block_time;
+        timecode = cluster_time + block_time - track->codec_delay;
         if (track->type == MATROSKA_TRACK_TYPE_SUBTITLE &&
             timecode < track->end_timecode)
             is_keyframe = 0;  /* overlapping subtitles are not key frame */