diff --git a/libavcodec/aacdec.c b/libavcodec/aacdec.c
index d1774a1f98..6000fe190b 100644
--- a/libavcodec/aacdec.c
+++ b/libavcodec/aacdec.c
@@ -2509,8 +2509,9 @@ static int latm_decode_frame(AVCodecContext *avctx, void *out, int *out_size,
             *out_size = 0;
             return avpkt->size;
         } else {
-            aac_decode_close(avctx);
-            if ((err = aac_decode_init(avctx)) < 0)
+            if ((err = decode_audio_specific_config(
+                    &latmctx->aac_ctx, avctx, &latmctx->aac_ctx.m4ac,
+                    avctx->extradata, avctx->extradata_size)) < 0)
                 return err;
             latmctx->initialized = 1;
         }