add protection for decoder and data format

codec/decoder/core/src/decoder.cpp

@@ -273,9 +273,10 @@ void WelsFreeMem (PWelsDecoderContext pCtx) {
 /*!
  * \brief	Open decoder
  */
-void WelsOpenDecoder (PWelsDecoderContext pCtx) {
+int32_t WelsOpenDecoder (PWelsDecoderContext pCtx) {
   // function pointers
   //initial MC function pointer--
+  int iRet = ERR_NONE;
   InitMcFunc (& (pCtx->sMcFunc), pCtx->uiCpuFlag);
   InitErrorCon (pCtx);
 
@@ -286,8 +287,9 @@ void WelsOpenDecoder (PWelsDecoderContext pCtx) {
   InitVlcTable (&pCtx->sVlcTable);
 
   // startup memory
-  if (ERR_NONE != WelsInitMemory (pCtx))
+  iRet = WelsInitMemory (pCtx);
-    return;
+  if (ERR_NONE != iRet)
+    return iRet;
 
 #ifdef LONG_TERM_REF
   pCtx->bParamSetsLostFlag = true;
@@ -298,6 +300,7 @@ void WelsOpenDecoder (PWelsDecoderContext pCtx) {
   pCtx->bDecErrorConedFlag = false; //default: decoder normal status
   pCtx->bPrintFrameErrorTraceFlag = true;
   pCtx->iIgnoredErrorInfoPacketCount = 0;
+  return iRet;
 }
 
 /*!
@@ -333,6 +336,9 @@ int32_t DecoderConfigParam (PWelsDecoderContext pCtx, const SDecodingParam* kpPa
 
   memcpy (pCtx->pParam, kpParam, sizeof (SDecodingParam));
   pCtx->eOutputColorFormat	= pCtx->pParam->eOutputColorFormat;
+  int32_t iRet = DecoderSetCsp (pCtx, pCtx->pParam->eOutputColorFormat);
+  if (iRet)
+    return iRet;
   pCtx->eErrorConMethod = pCtx->pParam->eEcActiveIdc;
 
   if (VIDEO_BITSTREAM_SVC == pCtx->pParam->sVideoProperty.eVideoBsType ||
@@ -368,10 +374,7 @@ int32_t WelsInitDecoder (PWelsDecoderContext pCtx, SLogContext* pLogCtx) {
   WelsDecoderDefaults (pCtx, pLogCtx);
 
   // open decoder
-  WelsOpenDecoder (pCtx);
+  return WelsOpenDecoder (pCtx);
-
-
-  return ERR_NONE;
 }
 
 /*!
@@ -596,9 +599,12 @@ int32_t DecoderSetCsp (PWelsDecoderContext pCtx, const int32_t kiColorFormat) {
   }
 
   //For now, support only videoFormatI420!
-  if (kiColorFormat != (int32_t) videoFormatI420) {
+  if (kiColorFormat == (int32_t) videoFormatInternal) {
+    pCtx->pParam->eOutputColorFormat = pCtx->eOutputColorFormat = videoFormatI420;
+  } else if (kiColorFormat != (int32_t) videoFormatI420) {
     WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING, "Support I420 output only for now! Change to I420...");
     pCtx->pParam->eOutputColorFormat = pCtx->eOutputColorFormat = videoFormatI420;
+    return cmUnsupportedData;
   }
 
   return 0;

codec/decoder/plus/inc/welsDecoderExt.h

@@ -101,7 +101,7 @@ virtual long EXTAPI GetOption (DECODER_OPTION eOptID, void* pOption);
 PWelsDecoderContext 				m_pDecContext;
 welsCodecTrace*			m_pWelsTrace;
 
-void InitDecoder (void);
+int32_t InitDecoder (void);
 void UninitDecoder (void);
 
 #ifdef OUTPUT_BIT_STREAM

codec/decoder/plus/src/welsDecoderExt.cpp

@@ -185,6 +185,7 @@ CWelsDecoder::~CWelsDecoder() {
 }
 
 long CWelsDecoder::Initialize (const SDecodingParam* pParam) {
+  int iRet = ERR_NONE;
   if (m_pWelsTrace == NULL) {
     return cmMallocMemeError;
   }
@@ -195,9 +196,13 @@ long CWelsDecoder::Initialize (const SDecodingParam* pParam) {
   }
 
   // H.264 decoder initialization,including memory allocation,then open it ready to decode
-  InitDecoder();
+  iRet = InitDecoder();
+  if (iRet)
+    return iRet;
 
-  DecoderConfigParam (m_pDecContext, pParam);
+  iRet = DecoderConfigParam (m_pDecContext, pParam);
+  if (iRet)
+    return iRet;
 
   return cmResultSuccess;
 }
@@ -226,15 +231,16 @@ void CWelsDecoder::UninitDecoder (void) {
 }
 
 // the return value of this function is not suitable, it need report failure info to upper layer.
-void CWelsDecoder::InitDecoder (void) {
+int32_t CWelsDecoder::InitDecoder (void) {
 
   WelsLog (&m_pWelsTrace->m_sLogCtx, WELS_LOG_INFO, "CWelsDecoder::init_decoder(), openh264 codec version = %s",
            VERSION_NUMBER);
 
   m_pDecContext	= (PWelsDecoderContext)WelsMalloc (sizeof (SWelsDecoderContext), "m_pDecContext");
+  if (NULL == m_pDecContext)
+    return cmMallocMemeError;
 
-  WelsInitDecoder (m_pDecContext, &m_pWelsTrace->m_sLogCtx);
+  return WelsInitDecoder (m_pDecContext, &m_pWelsTrace->m_sLogCtx);
-
 }
 
 /*

test/decoder/DecUT_DecExt.cpp

@@ -86,7 +86,11 @@ void DecoderInterfaceTest::Init() {
   m_szBuffer[3] = 1;
   m_iBufLength = 4;
   CM_RETURN eRet = (CM_RETURN) m_pDec->Initialize (&m_sDecParam);
-  ASSERT_EQ (eRet, cmResultSuccess);
+  if ((m_sDecParam.eOutputColorFormat != videoFormatI420) ||
+      (m_sDecParam.eOutputColorFormat != videoFormatInternal))
+    ASSERT_EQ (eRet, cmUnsupportedData);
+  else
+    ASSERT_EQ (eRet, cmResultSuccess);
 }
 
 void DecoderInterfaceTest::Uninit() {
@@ -154,7 +158,7 @@ void DecoderInterfaceTest::TestInitUninit() {
   m_pDec->Initialize (&m_sDecParam);
   eRet = (CM_RETURN) m_pDec->GetOption (DECODER_OPTION_DATAFORMAT, &iOutput);
   EXPECT_EQ (eRet, cmResultSuccess);
-  EXPECT_EQ (20, iOutput);
+  EXPECT_EQ ((int32_t) videoFormatI420, iOutput);
 
   //Uninitialize, no GetOption can be done
   m_pDec->Uninitialize();
@@ -180,7 +184,10 @@ void DecoderInterfaceTest::TestDataFormat() {
 
   //valid input
   eRet = (CM_RETURN) m_pDec->SetOption (DECODER_OPTION_DATAFORMAT, &iTmp);
-  EXPECT_EQ (eRet, cmResultSuccess);
+  if ((iTmp != (int32_t) videoFormatI420) && (iTmp != (int32_t) videoFormatInternal))
+    EXPECT_EQ (eRet, cmUnsupportedData);
+  else
+    EXPECT_EQ (eRet, cmResultSuccess);
   eRet = (CM_RETURN) m_pDec->GetOption (DECODER_OPTION_DATAFORMAT, &iOut);
   EXPECT_EQ (eRet, cmResultSuccess);