diff --git a/vp9/encoder/vp9_denoiser.c b/vp9/encoder/vp9_denoiser.c
index a4da44408..bed64397d 100644
--- a/vp9/encoder/vp9_denoiser.c
+++ b/vp9/encoder/vp9_denoiser.c
@@ -444,7 +444,7 @@ void vp9_denoiser_update_frame_info(
       svc_base_is_key) {
     int i;
     // Start at 1 so as not to overwrite the INTRA_FRAME
-    for (i = 1; i < DENOISER_REF_FRAMES; ++i)
+    for (i = 1; i < denoiser->num_ref_frames; ++i)
       copy_frame(&denoiser->running_avg_y[i], &src);
     denoiser->reset = 0;
     return;
@@ -512,8 +512,8 @@ void vp9_denoiser_update_frame_stats(MODE_INFO *mi, unsigned int sse,
   }
 }
 
-int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height, int ssx,
-                       int ssy,
+int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
+                       int width, int height, int ssx, int ssy,
 #if CONFIG_VP9_HIGHBITDEPTH
                        int use_highbitdepth,
 #endif
@@ -522,7 +522,11 @@ int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height, int ssx,
   const int legacy_byte_alignment = 0;
   assert(denoiser != NULL);
 
-  for (i = 0; i < DENOISER_REF_FRAMES; ++i) {
+  denoiser->num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
+  CHECK_MEM_ERROR(
+      cm, denoiser->running_avg_y,
+      vpx_calloc(denoiser->num_ref_frames, sizeof(denoiser->running_avg_y[0])));
+  for (i = 0; i < denoiser->num_ref_frames; ++i) {
     fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
                                   ssx, ssy,
 #if CONFIG_VP9_HIGHBITDEPTH
@@ -574,9 +578,11 @@ void vp9_denoiser_free(VP9_DENOISER *denoiser) {
     return;
   }
   denoiser->frame_buffer_initialized = 0;
-  for (i = 0; i < DENOISER_REF_FRAMES; ++i) {
+  for (i = 0; i < denoiser->num_ref_frames; ++i) {
     vpx_free_frame_buffer(&denoiser->running_avg_y[i]);
   }
+  vpx_free(denoiser->running_avg_y);
+  denoiser->running_avg_y = NULL;
   vpx_free_frame_buffer(&denoiser->mc_running_avg_y);
   vpx_free_frame_buffer(&denoiser->last_source);
 }
diff --git a/vp9/encoder/vp9_denoiser.h b/vp9/encoder/vp9_denoiser.h
index 7ed20856e..f23e6faea 100644
--- a/vp9/encoder/vp9_denoiser.h
+++ b/vp9/encoder/vp9_denoiser.h
@@ -21,9 +21,9 @@ extern "C" {
 
 #define MOTION_MAGNITUDE_THRESHOLD (8 * 3)
 
-// Denoiser is used in real-time mode which does not use alt-ref, so no need to
-// allocate for it, and hence we need MAX_REF_FRAME - 1
-#define DENOISER_REF_FRAMES MAX_REF_FRAMES - 1
+// Denoiser is used in non svc real-time mode which does not use alt-ref, so no
+// need to allocate for it, and hence we need MAX_REF_FRAME - 1
+#define NONSVC_REF_FRAMES MAX_REF_FRAMES - 1
 
 typedef enum vp9_denoiser_decision {
   COPY_BLOCK,
@@ -39,11 +39,12 @@ typedef enum vp9_denoiser_level {
 } VP9_DENOISER_LEVEL;
 
 typedef struct vp9_denoiser {
-  YV12_BUFFER_CONFIG running_avg_y[DENOISER_REF_FRAMES];
+  YV12_BUFFER_CONFIG *running_avg_y;
   YV12_BUFFER_CONFIG mc_running_avg_y;
   YV12_BUFFER_CONFIG last_source;
   int frame_buffer_initialized;
   int reset;
+  int num_ref_frames;
   VP9_DENOISER_LEVEL denoising_level;
   VP9_DENOISER_LEVEL prev_denoising_level;
 } VP9_DENOISER;
@@ -80,8 +81,8 @@ void vp9_denoiser_update_frame_stats(MODE_INFO *mi, unsigned int sse,
                                      PREDICTION_MODE mode,
                                      PICK_MODE_CONTEXT *ctx);
 
-int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height, int ssx,
-                       int ssy,
+int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
+                       int width, int height, int ssx, int ssy,
 #if CONFIG_VP9_HIGHBITDEPTH
                        int use_highbitdepth,
 #endif
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index 17e0de734..db651290d 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -3234,8 +3234,8 @@ static void setup_denoiser_buffer(VP9_COMP *cpi) {
   VP9_COMMON *const cm = &cpi->common;
   if (cpi->oxcf.noise_sensitivity > 0 &&
       !cpi->denoiser.frame_buffer_initialized) {
-    if (vp9_denoiser_alloc(&cpi->denoiser, cm->width, cm->height,
-                           cm->subsampling_x, cm->subsampling_y,
+    if (vp9_denoiser_alloc(cm, cpi->use_svc, &cpi->denoiser, cm->width,
+                           cm->height, cm->subsampling_x, cm->subsampling_y,
 #if CONFIG_VP9_HIGHBITDEPTH
                            cm->use_highbitdepth,
 #endif