From 53c057a8a8c57df06864cd428e81b37fd7cca86b Mon Sep 17 00:00:00 2001 From: Sijia Chen Date: Mon, 21 Sep 2015 15:42:17 -0700 Subject: [PATCH] fix for simulcast case when the frame rate of lower resolution is higher, and add test case --- codec/encoder/core/inc/encoder.h | 1 + codec/encoder/core/inc/encoder_context.h | 2 +- codec/encoder/core/src/encoder.cpp | 50 ++++++-- codec/encoder/core/src/encoder_ext.cpp | 17 +-- test/api/encode_decode_api_test.cpp | 138 +++++++++++++++++++++++ 5 files changed, 183 insertions(+), 25 deletions(-) diff --git a/codec/encoder/core/inc/encoder.h b/codec/encoder/core/inc/encoder.h index 2afe57f7..406d6e1c 100644 --- a/codec/encoder/core/inc/encoder.h +++ b/codec/encoder/core/inc/encoder.h @@ -83,6 +83,7 @@ int32_t InitFunctionPointers (sWelsEncCtx* pEncCtx, SWelsSvcCodingParam* _param, * \brief initialize frame coding */ void InitFrameCoding (sWelsEncCtx* pEncCtx, const EVideoFrameType keFrameType); +void LoadBackFrameNum(sWelsEncCtx* pEncCtx); EVideoFrameType DecideFrameType (sWelsEncCtx* pEncCtx, const int8_t kiSpatialNum); diff --git a/codec/encoder/core/inc/encoder_context.h b/codec/encoder/core/inc/encoder_context.h index 1e7a0f7c..98d8b966 100644 --- a/codec/encoder/core/inc/encoder_context.h +++ b/codec/encoder/core/inc/encoder_context.h @@ -156,7 +156,7 @@ typedef struct TagWelsEncCtx { EWelsSliceType eSliceType; // currently coding slice type EWelsNalUnitType eNalType; // NAL type EWelsNalRefIdc eNalPriority; // NAL_Reference_Idc currently - EWelsNalRefIdc eLastNalPriority; // NAL_Reference_Idc in last frame + EWelsNalRefIdc eLastNalPriority[MAX_DEPENDENCY_LAYER]; // NAL_Reference_Idc in last frame uint8_t iNumRef0; uint8_t uiDependencyId; // Idc of dependecy layer to be coded diff --git a/codec/encoder/core/src/encoder.cpp b/codec/encoder/core/src/encoder.cpp index 15441c2e..01a8b01f 100644 --- a/codec/encoder/core/src/encoder.cpp +++ b/codec/encoder/core/src/encoder.cpp @@ -226,6 +226,41 @@ int32_t InitFunctionPointers (sWelsEncCtx* pEncCtx, SWelsSvcCodingParam* pParam, return iReturn; } +void UpdateFrameNum(sWelsEncCtx* pEncCtx) { + bool bNeedFrameNumIncreasing = false; + for (int32_t i=0; ieLastNalPriority[i]) { + bNeedFrameNumIncreasing = true; + } + } + if (bNeedFrameNumIncreasing) { + if (pEncCtx->iFrameNum < (1 << pEncCtx->pSps->uiLog2MaxFrameNum) - 1) + ++ pEncCtx->iFrameNum; + else + pEncCtx->iFrameNum = 0; // if iFrameNum overflow + } + for (int32_t i=0; ieLastNalPriority[i] = NRI_PRI_LOWEST; + } +} + + +void LoadBackFrameNum(sWelsEncCtx* pEncCtx) { + bool bNeedFrameNumIncreasing = false; + for (int32_t i=0; ieLastNalPriority[i]) { + bNeedFrameNumIncreasing = true; + } + } + if (bNeedFrameNumIncreasing) { + if (pEncCtx->iFrameNum != 0) { + pEncCtx->iFrameNum --; + } else { + pEncCtx->iFrameNum = (1 << pEncCtx->pSps->uiLog2MaxFrameNum) - 1; + } + } +} + /*! * \brief initialize frame coding */ @@ -245,12 +280,8 @@ void InitFrameCoding (sWelsEncCtx* pEncCtx, const EVideoFrameType keFrameType) { else pEncCtx->iPOC = 0; - if (pEncCtx->eLastNalPriority != 0) { - if (pEncCtx->iFrameNum < (1 << pEncCtx->pSps->uiLog2MaxFrameNum) - 1) - ++ pEncCtx->iFrameNum; - else - pEncCtx->iFrameNum = 0; // if iFrameNum overflow - } + UpdateFrameNum(pEncCtx); + pEncCtx->eNalType = NAL_UNIT_CODED_SLICE; pEncCtx->eSliceType = P_SLICE; pEncCtx->eNalPriority = NRI_PRI_HIGH; @@ -275,12 +306,7 @@ void InitFrameCoding (sWelsEncCtx* pEncCtx, const EVideoFrameType keFrameType) { else pEncCtx->iPOC = 0; - if (pEncCtx->eLastNalPriority != 0) { - if (pEncCtx->iFrameNum < (1 << pEncCtx->pSps->uiLog2MaxFrameNum) - 1) - ++ pEncCtx->iFrameNum; - else - pEncCtx->iFrameNum = 0; // if iFrameNum overflow - } + UpdateFrameNum(pEncCtx); pEncCtx->eNalType = NAL_UNIT_CODED_SLICE; pEncCtx->eSliceType = I_SLICE; diff --git a/codec/encoder/core/src/encoder_ext.cpp b/codec/encoder/core/src/encoder_ext.cpp index 2dbc7535..a617e4e0 100644 --- a/codec/encoder/core/src/encoder_ext.cpp +++ b/codec/encoder/core/src/encoder_ext.cpp @@ -3481,7 +3481,7 @@ int32_t WelsEncoderEncodeParameterSets (sWelsEncCtx* pCtx, void* pDst) { pLayerBsInfo->uiLayerType = NON_VIDEO_CODING_LAYER; pLayerBsInfo->iNalCount = iCountNal; - pCtx->eLastNalPriority = NRI_PRI_HIGHEST; + //pCtx->eLastNalPriority = NRI_PRI_HIGHEST; pFbi->iLayerNum = 1; pFbi->eFrameType = videoFrameTypeInvalid; @@ -3712,17 +3712,10 @@ void StackBackEncoderStatus (sWelsEncCtx* pEncCtx, pEncCtx->iPOC = (1 << pEncCtx->pSps->iLog2MaxPocLsb) - 2; } - if (pEncCtx->eLastNalPriority != 0) { - if (pEncCtx->iFrameNum != 0) { - pEncCtx->iFrameNum --; - } else { - pEncCtx->iFrameNum = (1 << pEncCtx->pSps->uiLog2MaxFrameNum) - 1; - } - } - + LoadBackFrameNum(pEncCtx); pEncCtx->eNalType = NAL_UNIT_CODED_SLICE; pEncCtx->eSliceType = P_SLICE; - pEncCtx->eNalPriority = pEncCtx->eLastNalPriority; + //pEncCtx->eNalPriority = pEncCtx->eLastNalPriority; //not need this since eNalPriority will be updated at the beginning of coding a frame } else if (keFrameType == videoFrameTypeIDR) { pEncCtx->uiIdrPicId --; @@ -3865,7 +3858,7 @@ int32_t WelsEncoderEncodeExt (sWelsEncCtx* pCtx, SFrameBSInfo* pFbi, const SSour while (iSpatialIdx < iSpatialNum) { const int32_t iDidIdx = (pSpatialIndexMap + iSpatialIdx)->iDid; // get iDid SSpatialLayerConfig* pParam = &pSvcParam->sSpatialLayers[iDidIdx]; - int32_t iDecompositionStages = pSvcParam->sDependencyLayers[iSpatialIdx].iDecompositionStages; + int32_t iDecompositionStages = pSvcParam->sDependencyLayers[iDidIdx].iDecompositionStages; pCtx->uiDependencyId = iCurDid = (int8_t)iDidIdx; pCtx->pVpp->AnalyzeSpatialPic (pCtx, iDidIdx); @@ -4387,6 +4380,7 @@ int32_t WelsEncoderEncodeExt (sWelsEncCtx* pCtx, SFrameBSInfo* pFbi, const SSour #endif//#if defined(MT_DEBUG) } + pCtx->eLastNalPriority[iDidIdx] = eNalRefIdc; ++ iSpatialIdx; if (iCurDid + 1 < pSvcParam->iSpatialLayerNum) { @@ -4447,7 +4441,6 @@ int32_t WelsEncoderEncodeExt (sWelsEncCtx* pCtx, SFrameBSInfo* pFbi, const SSour } ++ pCtx->iCodingIndex; - pCtx->eLastNalPriority = eNalRefIdc; pFbi->iLayerNum = iLayerNum; pFbi->iSubSeqId = GetSubSequenceId (pCtx, eFrameType); diff --git a/test/api/encode_decode_api_test.cpp b/test/api/encode_decode_api_test.cpp index 31be0de9..6cadc1dd 100644 --- a/test/api/encode_decode_api_test.cpp +++ b/test/api/encode_decode_api_test.cpp @@ -3363,6 +3363,12 @@ TEST_F (EncodeDecodeTestAPI, SimulcastAVC) { WelsDestroyDecoder (decoder[iIdx]); } } +#ifdef DEBUG_FILE_SAVE_SimulcastAVCDiffFps + for (int i = 0; i <)MAX_SPATIAL_LAYER_NUM; + i++) { + fclose (fEnc[i]); + } +#endif } TEST_F (EncodeDecodeTestAPI, SimulcastAVC_SPS_PPS_LISTING) { @@ -3587,3 +3593,135 @@ TEST_P (EncodeTestAPI, SetEncOptionSize) { } } + + +TEST_F (EncodeDecodeTestAPI, SimulcastAVCDiffFps) { +//#define DEBUG_FILE_SAVE3 + int iSpatialLayerNum = WelsClip3 ((rand() % MAX_SPATIAL_LAYER_NUM), 2, MAX_SPATIAL_LAYER_NUM); + int iWidth = WelsClip3 ((((rand() % MAX_WIDTH) >> 1) + 1) << 1, 1 << iSpatialLayerNum, MAX_WIDTH); + int iHeight = WelsClip3 ((((rand() % MAX_HEIGHT) >> 1) + 1) << 1, 1 << iSpatialLayerNum, MAX_HEIGHT); + float fFrameRate = 30; + int iEncFrameNum = WelsClip3 ((rand() % ENCODE_FRAME_NUM) + 1, 1, ENCODE_FRAME_NUM); + int iSliceNum = 1; + encoder_->GetDefaultParams (¶m_); + prepareParam (iSpatialLayerNum, iSliceNum, iWidth, iHeight, fFrameRate, ¶m_); + + //set flag of bSimulcastAVC + param_.bSimulcastAVC = true; + param_.iTemporalLayerNum = 3; + + int rv = encoder_->InitializeExt (¶m_); + ASSERT_TRUE (rv == cmResultSuccess); + + unsigned char* pBsBuf[MAX_SPATIAL_LAYER_NUM]; + int aLen[MAX_SPATIAL_LAYER_NUM] = {0}; + ISVCDecoder* decoder[MAX_SPATIAL_LAYER_NUM]; + +#ifdef DEBUG_FILE_SAVE_SimulcastAVCDiffFps + FILE* fEnc[MAX_SPATIAL_LAYER_NUM]; + fEnc[0] = fopen ("enc0.264", "wb"); + fEnc[1] = fopen ("enc1.264", "wb"); + fEnc[2] = fopen ("enc2.264", "wb"); + fEnc[3] = fopen ("enc3.264", "wb"); +#endif + + int iIdx = 0; + + //create decoder + for (iIdx = 0; iIdx < iSpatialLayerNum; iIdx++) { + pBsBuf[iIdx] = static_cast (malloc (iWidth * iHeight * 3 * sizeof (unsigned char) / 2)); + EXPECT_TRUE (pBsBuf[iIdx] != NULL); + aLen[iIdx] = 0; + + long rv = WelsCreateDecoder (&decoder[iIdx]); + ASSERT_EQ (0, rv); + EXPECT_TRUE (decoder[iIdx] != NULL); + + SDecodingParam decParam; + memset (&decParam, 0, sizeof (SDecodingParam)); + decParam.eOutputColorFormat = videoFormatI420; + decParam.uiTargetDqLayer = UCHAR_MAX; + decParam.eEcActiveIdc = ERROR_CON_SLICE_COPY; + decParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_DEFAULT; + + rv = decoder[iIdx]->Initialize (&decParam); + ASSERT_EQ (0, rv); + } + +#define PATTERN_NUM (16) + const int32_t iTemporalPattern[PATTERN_NUM][MAX_SPATIAL_LAYER_NUM] = { {2, 1, 1, 1}, {2, 2, 2, 1}, {4, 1, 1, 1}, {4, 2, 1, 1}, + {1, 2, 1, 1}, {1, 1, 2, 1}, {1, 4, 1, 1}, {2, 4, 2, 1}, {1, 4, 2, 1}, + {1, 2, 2, 1}, {1, 2, 4, 1}, + {1, 1, 1, 2}, {1, 2, 2, 2}, {1, 2, 2, 4}, {1, 2, 4, 2}, {1, 4, 4, 4}, + }; + for (int iPatternIdx = 0; iPatternIdx < PATTERN_NUM; iPatternIdx++) { + for (int i = 0; i < iSpatialLayerNum; i++) { + param_.sSpatialLayers[i].fFrameRate = (fFrameRate / iTemporalPattern[iPatternIdx][i]); + } + + iEncFrameNum = 10; + for (int iFrame = 0; iFrame < iEncFrameNum; iFrame++) { + int iResult; + int iLayerLen = 0; + unsigned char* pData[3] = { NULL }; + + InitialEncDec (param_.iPicWidth, param_.iPicHeight); + EncodeOneFrame (0); + + // init + for (iIdx = 0; iIdx < iSpatialLayerNum; iIdx++) { + aLen[iIdx] = 0; + } + for (int iLayer = 0; iLayer < info.iLayerNum; ++iLayer) { + iLayerLen = 0; + const SLayerBSInfo& layerInfo = info.sLayerInfo[iLayer]; + const int kiFirstNalType = ((* (layerInfo.pBsBuf + 4)) & 0x1f); + ASSERT_TRUE ((kiFirstNalType == NAL_SPS) || (kiFirstNalType == NAL_PPS) || (kiFirstNalType == NAL_SLICE) + || (kiFirstNalType == NAL_SLICE_IDR) || (kiFirstNalType == NAL_SEI)); + for (int iNal = 0; iNal < layerInfo.iNalCount; ++iNal) { + iLayerLen += layerInfo.pNalLengthInByte[iNal]; + } + + iIdx = layerInfo.uiSpatialId; + EXPECT_TRUE (iIdx < iSpatialLayerNum); + memcpy ((pBsBuf[iIdx] + aLen[iIdx]), layerInfo.pBsBuf, iLayerLen * sizeof (unsigned char)); + aLen[iIdx] += iLayerLen; + } + + for (iIdx = 0; iIdx < iSpatialLayerNum; iIdx++) { + pData[0] = pData[1] = pData[2] = 0; + memset (&dstBufInfo_, 0, sizeof (SBufferInfo)); + + if (aLen[iIdx] > 0) { +#ifdef DEBUG_FILE_SAVE_SimulcastAVCDiffFps + fwrite (pBsBuf[iIdx], aLen[iIdx], 1, fEnc[iIdx]); +#endif + iResult = decoder[iIdx]->DecodeFrame2 (pBsBuf[iIdx], aLen[iIdx], pData, &dstBufInfo_); + EXPECT_TRUE (iResult == cmResultSuccess) << "iResult=" << iResult << "LayerIdx=" << iIdx; + + iResult = decoder[iIdx]->DecodeFrame2 (NULL, 0, pData, &dstBufInfo_); + EXPECT_TRUE (iResult == cmResultSuccess) << "iResult=" << iResult << "LayerIdx=" << iIdx; + EXPECT_EQ (dstBufInfo_.iBufferStatus, 1) << "LayerIdx=" << iIdx; + } + } + } + } + + for (iIdx = 0; iIdx < iSpatialLayerNum; iIdx++) { + free (pBsBuf[iIdx]); + + if (decoder[iIdx] != NULL) { + decoder[iIdx]->Uninitialize(); + WelsDestroyDecoder (decoder[iIdx]); + } + } +#ifdef DEBUG_FILE_SAVE_SimulcastAVCDiffFps + for (int i = 0; i <)MAX_SPATIAL_LAYER_NUM; + i++) { + fclose (fEnc[i]); + } +#endif +} + + +