diff --git a/libavcodec/audio_frame_queue.c b/libavcodec/audio_frame_queue.c
index a5c22ce643..7db0091c0c 100644
--- a/libavcodec/audio_frame_queue.c
+++ b/libavcodec/audio_frame_queue.c
@@ -102,6 +102,7 @@ void ff_af_queue_remove(AudioFrameQueue *afq, int nb_samples, int64_t *pts,
 
     if(nb_samples){
         av_assert0(!afq->frame_count);
+        av_assert0(afq->remaining_samples == afq->remaining_delay);
         if(afq->frames && afq->frames[0].pts != AV_NOPTS_VALUE)
             afq->frames[0].pts += nb_samples;
         av_log(afq->avctx, AV_LOG_DEBUG, "Trying to remove %d more samples than are in the que\n", nb_samples);