diff --git a/codec/encoder/core/src/ref_list_mgr_svc.cpp b/codec/encoder/core/src/ref_list_mgr_svc.cpp
index 372b0cac..73101ed9 100644
--- a/codec/encoder/core/src/ref_list_mgr_svc.cpp
+++ b/codec/encoder/core/src/ref_list_mgr_svc.cpp
@@ -315,6 +315,10 @@ static inline void LTRMarkProcess (sWelsEncCtx* pCtx) {
     }
     pLongRefList[0]	 = pShortRefList[i];
     pRefList->uiLongRefCount++;
+    if (pRefList->uiLongRefCount > pCtx->pSvcParam->iLTRRefNum) {
+      SetUnref (pRefList->pLongRefList[pRefList->uiLongRefCount - 1]);
+      DeleteLTRFromLongList (pCtx, pRefList->uiLongRefCount - 1);
+    }
     DeleteSTRFromShortList (pCtx, i);
   }
 }