From 32ce337eb09a64bc4d97803c08cc0aefd757f840 Mon Sep 17 00:00:00 2001 From: ruil2 Date: Tue, 1 Jul 2014 14:50:50 +0800 Subject: [PATCH] check each WriteBlockResidualCavlc to avoid overflow --- .../encoder/core/src/svc_set_mb_syn_cavlc.cpp | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/codec/encoder/core/src/svc_set_mb_syn_cavlc.cpp b/codec/encoder/core/src/svc_set_mb_syn_cavlc.cpp index 87b05e30..96e9de91 100644 --- a/codec/encoder/core/src/svc_set_mb_syn_cavlc.cpp +++ b/codec/encoder/core/src/svc_set_mb_syn_cavlc.cpp @@ -282,7 +282,8 @@ int32_t WelsWriteMbResidual (SMbCache* sMbCacheInfo, SMB* pCurMb, SBitStringAux* iA = pNonZeroCoeffCount[iIdx - 1]; iB = pNonZeroCoeffCount[iIdx - 8]; WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB); - WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, LUMA_AC, iC, pBs); + if (WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, LUMA_AC, iC, pBs)) + return ENC_RETURN_VLCOVERFLOWFOUND; pBlock += 16; } } @@ -301,22 +302,26 @@ int32_t WelsWriteMbResidual (SMbCache* sMbCacheInfo, SMB* pCurMb, SBitStringAux* iA = pNonZeroCoeffCount[iIdx - 1]; iB = pNonZeroCoeffCount[iIdx - 8]; WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB); - WriteBlockResidualCavlc (pBlock, 15, kiA > 0, LUMA_4x4, iC, pBs); + if (WriteBlockResidualCavlc (pBlock, 15, kiA > 0, LUMA_4x4, iC, pBs)) + return ENC_RETURN_VLCOVERFLOWFOUND; iA = kiA; iB = pNonZeroCoeffCount[iIdx - 7]; WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB); - WriteBlockResidualCavlc (pBlock + 16, 15, kiB > 0, LUMA_4x4, iC, pBs); + if (WriteBlockResidualCavlc (pBlock + 16, 15, kiB > 0, LUMA_4x4, iC, pBs)) + return ENC_RETURN_VLCOVERFLOWFOUND; iA = pNonZeroCoeffCount[iIdx + 7]; iB = kiA; WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB); - WriteBlockResidualCavlc (pBlock + 32, 15, kiC > 0, LUMA_4x4, iC, pBs); + if (WriteBlockResidualCavlc (pBlock + 32, 15, kiC > 0, LUMA_4x4, iC, pBs)) + return ENC_RETURN_VLCOVERFLOWFOUND; iA = kiC; iB = kiB; WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB); - WriteBlockResidualCavlc (pBlock + 48, 15, kiD > 0, LUMA_4x4, iC, pBs); + if (WriteBlockResidualCavlc (pBlock + 48, 15, kiD > 0, LUMA_4x4, iC, pBs)) + return ENC_RETURN_VLCOVERFLOWFOUND; } pBlock += 64; } @@ -343,7 +348,8 @@ int32_t WelsWriteMbResidual (SMbCache* sMbCacheInfo, SMB* pCurMb, SBitStringAux* iA = pNonZeroCoeffCount[iIdx - 1]; iB = pNonZeroCoeffCount[iIdx - 8]; WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB); - WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs); + if (WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs)) + return ENC_RETURN_VLCOVERFLOWFOUND; pBlock += 16; } @@ -354,7 +360,8 @@ int32_t WelsWriteMbResidual (SMbCache* sMbCacheInfo, SMB* pCurMb, SBitStringAux* iA = pNonZeroCoeffCount[iIdx - 1]; iB = pNonZeroCoeffCount[iIdx - 8]; WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB); - WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs); + if (WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs)) + return ENC_RETURN_VLCOVERFLOWFOUND; pBlock += 16; } }