Merge pull request #1286 from ethanhugg/FF33-patch

avoid encoder crash with random pixel input

Makefile

@@ -17,7 +17,7 @@ SHARED=-shared
 OBJ=o
 PROJECT_NAME=openh264
 MODULE_NAME=gmpopenh264
-GMP_API_BRANCH=master
+GMP_API_BRANCH=Firefox33
 CCASFLAGS=$(CFLAGS)
 
 ifeq (,$(wildcard ./gmp-api))

codec/encoder/core/src/encoder_ext.cpp

@@ -2976,6 +2976,9 @@ bool CheckFrameSkipBasedMaxbr (sWelsEncCtx* pCtx, int32_t iSpatialNum) {
  * \return	EFrameType (videoFrameTypeIDR/videoFrameTypeI/videoFrameTypeP)
  */
 int32_t WelsEncoderEncodeExt (sWelsEncCtx* pCtx, SFrameBSInfo* pFbi, const SSourcePicture* pSrcPic) {
+  if( pCtx == NULL ) {
+    return ENC_RETURN_MEMALLOCERR;
+  }
   SLayerBSInfo* pLayerBsInfo					= &pFbi->sLayerInfo[0];
   SWelsSvcCodingParam* pSvcParam	= pCtx->pSvcParam;
   SSpatialPicIndex* pSpatialIndexMap = &pCtx->sSpatialIndexMap[0];

codec/encoder/core/src/ref_list_mgr_svc.cpp

@@ -127,6 +127,7 @@ static void DeleteNonSceneLTR (sWelsEncCtx* pCtx) {
         (pCtx->uiTemporalId < pRef->uiTemporalId || pCtx->bCurFrameMarkedAsSceneLtr)) {
       SetUnref (pRef);
       DeleteLTRFromLongList (pCtx, i);
+      i--;
     }
   }
 }

gmpopenh264.info

@@ -1,4 +1,4 @@
 Name: gmpopenh264
 Description: GMP Plugin for OpenH264.
 Version: 1.0
-APIs: encode-video[h264:vp8], decode-video[h264:vp8]
+APIs: encode-video[h264], decode-video[h264]

module/gmp-openh264.cpp

@@ -608,13 +608,15 @@ class OpenH264VideoDecoder : public GMPVideoDecoder {
       assert (false);
       break;
     }
-
+    DECODING_STATE dState = dsErrorFree;
     worker_thread_->Post (WrapTask (
                             this, &OpenH264VideoDecoder::Decode_w,
                             inputFrame,
                             missingFrames,
+                            dState,
                             renderTimeMs));
-
+    if (dState)
+      return GMPGenericErr;
     return GMPNoErr;
   }
 
@@ -633,6 +635,7 @@ class OpenH264VideoDecoder : public GMPVideoDecoder {
  private:
   void Decode_w (GMPVideoEncodedFrame* inputFrame,
                  bool missingFrames,
+                 DECODING_STATE& dState,
                  int64_t renderTimeMs = -1) {
     GMPLOG (GL_DEBUG, "Frame decode on worker thread length = "
             << inputFrame->Size());
@@ -642,13 +645,13 @@ class OpenH264VideoDecoder : public GMPVideoDecoder {
     memset (&decoded, 0, sizeof (decoded));
     unsigned char* data[3] = {nullptr, nullptr, nullptr};
 
-    int rv = decoder_->DecodeFrame2 (inputFrame->Buffer(),
+    dState = decoder_->DecodeFrame2 (inputFrame->Buffer(),
                                      inputFrame->Size(),
                                      data,
                                      &decoded);
 
-    if (rv) {
-      GMPLOG (GL_ERROR, "Decoding error rv=" << rv);
+    if (dState) {
+      GMPLOG (GL_ERROR, "Decoding error dState=" << dState);
     } else {
       valid = true;
     }