Improve Android camera error handling.

- Set Camera.ErrorCallback callback when opening camera to
receive camera server error notifications.
- Allow user to provide interface for handling camera errors
happening on camera thread.
- Run camera observer on camera thread and monitor camera fps
and amount of callback buffers, print statistics and report error
if camera stops generating frames.
- Query camera formats starting from front camera instead of back
camera to detect camera failures as fast as possible.
- Change all DCHECK to CHECK in androidvideocapturer.cc to detect
camera error on release builds.
- Plus adding some extra logging.

R=hbos@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/52519004

Cr-Commit-Position: refs/heads/master@{#9221}

talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java

@@ -129,7 +129,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
 
   void starCapturerAndRender(String deviceName) throws InterruptedException {
     PeerConnectionFactory factory = new PeerConnectionFactory();
-    VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName);
+    VideoCapturerAndroid capturer =
+        VideoCapturerAndroid.create(deviceName, null);
     VideoSource source =
         factory.createVideoSource(capturer, new MediaConstraints());
     VideoTrack track = factory.createVideoTrack("dummy", source);
@@ -150,7 +151,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
 
   @SmallTest
   public void testCreateAndRelease() throws Exception {
-    VideoCapturerAndroid capturer = VideoCapturerAndroid.create("");
+    VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null);
     assertNotNull(capturer);
     capturer.dispose();
   }
@@ -158,7 +159,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
   @SmallTest
   public void testCreateNoneExistingCamera() throws Exception {
     VideoCapturerAndroid capturer = VideoCapturerAndroid.create(
-        "none existing camera");
+        "none existing camera", null);
     assertNull(capturer);
   }
 
@@ -195,7 +196,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
   // It tests both the Java and the C++ layer.
   public void testSwitchVideoCapturer() throws Exception {
     PeerConnectionFactory factory = new PeerConnectionFactory();
-    VideoCapturerAndroid capturer = VideoCapturerAndroid.create("");
+    VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null);
     VideoSource source =
         factory.createVideoSource(capturer, new MediaConstraints());
     VideoTrack track = factory.createVideoTrack("dummy", source);
@@ -222,7 +223,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
   // be stopped and restarted. It tests both the Java and the C++ layer.
   public void testStopRestartVideoSource() throws Exception {
     PeerConnectionFactory factory = new PeerConnectionFactory();
-    VideoCapturerAndroid capturer = VideoCapturerAndroid.create("");
+    VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null);
     VideoSource source =
         factory.createVideoSource(capturer, new MediaConstraints());
     VideoTrack track = factory.createVideoTrack("dummy", source);
@@ -251,7 +252,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
     String deviceName = VideoCapturerAndroid.getDeviceName(0);
     ArrayList<CaptureFormat> formats =
         VideoCapturerAndroid.getSupportedFormats(0);
-    VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName);
+    VideoCapturerAndroid capturer =
+        VideoCapturerAndroid.create(deviceName, null);
 
     for(int i = 0; i < 3 ; ++i) {
       VideoCapturerAndroid.CaptureFormat format = formats.get(i);
@@ -275,7 +277,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
     String deviceName = VideoCapturerAndroid.getDeviceName(0);
     ArrayList<CaptureFormat> formats =
         VideoCapturerAndroid.getSupportedFormats(0);
-    VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName);
+    VideoCapturerAndroid capturer =
+        VideoCapturerAndroid.create(deviceName, null);
 
     VideoCapturerAndroid.CaptureFormat format = formats.get(0);
     capturer.startCapture(format.width, format.height, format.maxFramerate,

talk/app/webrtc/androidvideocapturer.cc

@@ -89,10 +89,10 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory {
     // |captured_frame_.data| is only guaranteed to be valid during the scope
     // of |AndroidVideoCapturer::OnIncomingFrame_w|.
     // Check that captured_frame is actually our frame.
-    DCHECK(captured_frame == &captured_frame_);
+    CHECK(captured_frame == &captured_frame_);
 
     if (!apply_rotation_ || captured_frame->rotation == kVideoRotation_0) {
-      DCHECK(captured_frame->fourcc == cricket::FOURCC_YV12);
+      CHECK(captured_frame->fourcc == cricket::FOURCC_YV12);
       const uint8_t* y_plane = static_cast<uint8_t*>(captured_frame_.data);
 
       // Android guarantees that the stride is a multiple of 16.
@@ -160,7 +160,7 @@ AndroidVideoCapturer::AndroidVideoCapturer(
   std::vector<cricket::VideoFormat> formats;
   for (Json::ArrayIndex i = 0; i < json_values.size(); ++i) {
       const Json::Value& json_value = json_values[i];
-      DCHECK(!json_value["width"].isNull() && !json_value["height"].isNull() &&
+      CHECK(!json_value["width"].isNull() && !json_value["height"].isNull() &&
              !json_value["framerate"].isNull());
       cricket::VideoFormat format(
           json_value["width"].asInt(),
@@ -173,16 +173,16 @@ AndroidVideoCapturer::AndroidVideoCapturer(
 }
 
 AndroidVideoCapturer::~AndroidVideoCapturer() {
-  DCHECK(!running_);
+  CHECK(!running_);
 }
 
 cricket::CaptureState AndroidVideoCapturer::Start(
     const cricket::VideoFormat& capture_format) {
-  DCHECK(thread_checker_.CalledOnValidThread());
-  DCHECK(!running_);
-
   LOG(LS_INFO) << " AndroidVideoCapturer::Start w = " << capture_format.width
                << " h = " << capture_format.height;
+  CHECK(thread_checker_.CalledOnValidThread());
+  CHECK(!running_);
+
   frame_factory_ = new AndroidVideoCapturer::FrameFactory(
       capture_format.width, capture_format.height, delegate_.get());
   set_frame_factory(frame_factory_);
@@ -197,9 +197,9 @@ cricket::CaptureState AndroidVideoCapturer::Start(
 }
 
 void AndroidVideoCapturer::Stop() {
-  DCHECK(thread_checker_.CalledOnValidThread());
   LOG(LS_INFO) << " AndroidVideoCapturer::Stop ";
-  DCHECK(running_);
+  CHECK(thread_checker_.CalledOnValidThread());
+  CHECK(running_);
   running_ = false;
   SetCaptureFormat(NULL);
 
@@ -209,18 +209,18 @@ void AndroidVideoCapturer::Stop() {
 }
 
 bool AndroidVideoCapturer::IsRunning() {
-  DCHECK(thread_checker_.CalledOnValidThread());
+  CHECK(thread_checker_.CalledOnValidThread());
   return running_;
 }
 
 bool AndroidVideoCapturer::GetPreferredFourccs(std::vector<uint32>* fourccs) {
-  DCHECK(thread_checker_.CalledOnValidThread());
+  CHECK(thread_checker_.CalledOnValidThread());
   fourccs->push_back(cricket::FOURCC_YV12);
   return true;
 }
 
 void AndroidVideoCapturer::OnCapturerStarted(bool success) {
-  DCHECK(thread_checker_.CalledOnValidThread());
+  CHECK(thread_checker_.CalledOnValidThread());
   cricket::CaptureState new_state =
       success ? cricket::CS_RUNNING : cricket::CS_FAILED;
   if (new_state == current_state_)
@@ -237,7 +237,7 @@ void AndroidVideoCapturer::OnIncomingFrame(void* frame_data,
                                            int length,
                                            int rotation,
                                            int64 time_stamp) {
-  DCHECK(thread_checker_.CalledOnValidThread());
+  CHECK(thread_checker_.CalledOnValidThread());
   frame_factory_->UpdateCapturedFrame(frame_data, length, rotation, time_stamp);
   SignalFrameCaptured(this, frame_factory_->GetCapturedFrame());
 }

talk/app/webrtc/java/jni/androidvideocapturer_jni.cc

@@ -54,8 +54,10 @@ AndroidVideoCapturerJni::Create(JNIEnv* jni,
   rtc::scoped_refptr<AndroidVideoCapturerJni> capturer(
       new rtc::RefCountedObject<AndroidVideoCapturerJni>(jni, j_video_capture));
 
-  if (capturer->Init(device_name))
+  if (capturer->Init(device_name)) {
     return capturer;
+  }
+  LOG(LS_ERROR) << "AndroidVideoCapturerJni init fails";
   return nullptr;
 }
 

talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java

@@ -51,6 +51,7 @@ import org.json.JSONObject;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Exchanger;
 import java.util.concurrent.TimeUnit;
@@ -73,6 +74,7 @@ import java.util.concurrent.TimeUnit;
 @SuppressWarnings("deprecation")
 public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallback {
   private final static String TAG = "VideoCapturerAndroid";
+  private final static int CAMERA_OBSERVER_PERIOD_MS = 5000;
 
   private Camera camera;  // Only non-null while capturing.
   private CameraThread cameraThread;
@@ -86,12 +88,69 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
   private int width;
   private int height;
   private int framerate;
+  private int cameraFramesCount;
+  private int captureBuffersCount;
   private volatile boolean pendingCameraSwitch;
   private CapturerObserver frameObserver = null;
+  private CameraErrorHandler errorHandler = null;
   // List of formats supported by all cameras. This list is filled once in order
   // to be able to switch cameras.
   private static List<List<CaptureFormat>> supportedFormats;
 
+  // Camera error callback.
+  private final Camera.ErrorCallback cameraErrorCallback =
+      new Camera.ErrorCallback() {
+    @Override
+    public void onError(int error, Camera camera) {
+      String errorMessage;
+      if (error == android.hardware.Camera.CAMERA_ERROR_SERVER_DIED) {
+        errorMessage = "Camera server died!";
+      } else {
+        errorMessage = "Camera error: " + error;
+      }
+      Log.e(TAG, errorMessage);
+      if (errorHandler != null) {
+        errorHandler.onCameraError(errorMessage);
+      }
+    }
+  };
+
+  // Camera observer - monitors camera framerate and amount of available
+  // camera buffers. Observer is excecuted on camera thread.
+  private final Runnable cameraObserver = new Runnable() {
+    @Override
+    public void run() {
+      int cameraFps = (cameraFramesCount * 1000 + CAMERA_OBSERVER_PERIOD_MS / 2)
+          / CAMERA_OBSERVER_PERIOD_MS;
+      double averageCaptureBuffersCount = 0;
+      if (cameraFramesCount > 0) {
+        averageCaptureBuffersCount =
+            (double)captureBuffersCount / cameraFramesCount;
+      }
+      Log.d(TAG, "Camera fps: " + cameraFps + ". CaptureBuffers: " +
+          String.format("%.1f", averageCaptureBuffersCount) +
+          ". Pending buffers: [" + videoBuffers.pendingFramesTimeStamps() + "]");
+      if (cameraFramesCount == 0) {
+        Log.e(TAG, "Camera freezed.");
+        if (errorHandler != null) {
+          errorHandler.onCameraError("Camera failure.");
+        }
+      } else {
+        cameraFramesCount = 0;
+        captureBuffersCount = 0;
+        if (cameraThreadHandler != null) {
+          cameraThreadHandler.postDelayed(this, CAMERA_OBSERVER_PERIOD_MS);
+        }
+      }
+    }
+  };
+
+  // Camera error handler - invoked when camera stops receiving frames
+  // or any camera exception happens on camera thread.
+  public static interface CameraErrorHandler {
+    public void onCameraError(String errorDescription);
+  }
+
   // Returns device names that can be used to create a new VideoCapturerAndroid.
   public static String[] getDeviceNames() {
     String[] names = new String[Camera.getNumberOfCameras()];
@@ -155,10 +214,14 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     return null;
   }
 
-  public static VideoCapturerAndroid create(String name) {
+  public static VideoCapturerAndroid create(String name,
+      CameraErrorHandler errorHandler) {
     VideoCapturer capturer = VideoCapturer.create(name);
-    if (capturer != null)
+    if (capturer != null) {
-      return (VideoCapturerAndroid) capturer;
+      VideoCapturerAndroid capturerAndroid = (VideoCapturerAndroid) capturer;
+      capturerAndroid.errorHandler = errorHandler;
+      return capturerAndroid;
+    }
     return null;
   }
 
@@ -218,7 +281,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
   // If deviceName is empty, the first available device is used in order to be
   // compatible with the generic VideoCapturer class.
   synchronized boolean init(String deviceName) {
-    Log.d(TAG, "init " + deviceName);
+    Log.d(TAG, "init: " + deviceName);
     if (deviceName == null || !initStatics())
       return false;
 
@@ -245,9 +308,20 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       Log.d(TAG, "Get supported formats.");
       supportedFormats =
           new ArrayList<List<CaptureFormat>>(Camera.getNumberOfCameras());
-      for (int i = 0; i < Camera.getNumberOfCameras(); ++i) {
+      // Start requesting supported formats from camera with the highest index
-        supportedFormats.add(getSupportedFormats(i));
+      // (back camera) first. If it fails then likely camera is in bad state.
+      for (int i = Camera.getNumberOfCameras() - 1; i >= 0; i--) {
+        ArrayList<CaptureFormat> supportedFormat = getSupportedFormats(i);
+        if (supportedFormat.size() == 0) {
+          Log.e(TAG, "Fail to get supported formats for camera " + i);
+          supportedFormats = null;
+          return false;
+        }
+        supportedFormats.add(supportedFormat);
       }
+      // Reverse the list since it is filled in reverse order.
+      Collections.reverse(supportedFormats);
+      Log.d(TAG, "Get supported formats done.");
       return true;
     } catch (Exception e) {
       supportedFormats = null;
@@ -328,6 +402,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
 
     Camera camera;
     try {
+      Log.d(TAG, "Opening camera " + id);
       camera = Camera.open(id);
     } catch (Exception e) {
       Log.e(TAG, "Open camera failed on id " + id, e);
@@ -353,6 +428,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       Log.e(TAG, "getSupportedFormats failed on id " + id, e);
     }
     camera.release();
+    camera = null;
     return formatList;
   }
 
@@ -411,6 +487,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     this.applicationContext = applicationContext;
     this.frameObserver = frameObserver;
     try {
+      Log.d(TAG, "Opening camera " + id);
       camera = Camera.open(id);
       info = new Camera.CameraInfo();
       Camera.getCameraInfo(id, info);
@@ -454,6 +531,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
         parameters.setVideoStabilization(true);
       }
 
+      camera.setErrorCallback(cameraErrorCallback);
+
       int androidFramerate = framerate * 1000;
       int[] range = getFramerateRange(parameters, androidFramerate);
       if (range != null) {
@@ -480,6 +559,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       camera.setPreviewCallbackWithBuffer(this);
       camera.startPreview();
       frameObserver.OnCapturerStarted(true);
+
+      // Start camera observer.
+      cameraFramesCount = 0;
+      captureBuffersCount = 0;
+      cameraThreadHandler.postDelayed(cameraObserver, CAMERA_OBSERVER_PERIOD_MS);
       return;
     } catch (RuntimeException e) {
       error = e;
@@ -488,6 +572,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     stopCaptureOnCameraThread();
     cameraThreadHandler = null;
     frameObserver.OnCapturerStarted(false);
+    if (errorHandler != null) {
+      errorHandler.onCameraError("Camera can not be started.");
+    }
     return;
   }
 
@@ -509,17 +596,18 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
   }
 
   private void stopCaptureOnCameraThread() {
-    Log.d(TAG, "stopCaptureOnCameraThread");
+    doStopCaptureOnCameraThread();
-    doStopCaptureOnCamerathread();
     Looper.myLooper().quit();
     return;
   }
 
-  private void doStopCaptureOnCamerathread() {
+  private void doStopCaptureOnCameraThread() {
+    Log.d(TAG, "stopCaptureOnCameraThread");
     if (camera == null) {
       return;
     }
     try {
+      cameraThreadHandler.removeCallbacks(cameraObserver);
       Log.d(TAG, "Stop preview.");
       camera.stopPreview();
       camera.setPreviewCallbackWithBuffer(null);
@@ -543,7 +631,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
   private void switchCameraOnCameraThread(Runnable switchDoneEvent) {
     Log.d(TAG, "switchCameraOnCameraThread");
 
-    doStopCaptureOnCamerathread();
+    doStopCaptureOnCameraThread();
     startCaptureOnCameraThread(width, height, framerate, frameObserver,
         applicationContext);
     pendingCameraSwitch = false;
@@ -638,6 +726,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     long captureTimeNs =
         TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());
 
+    cameraFramesCount++;
+    captureBuffersCount += videoBuffers.numCaptureBuffersAvailable;
     int rotation = getDeviceOrientation();
     if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) {
       rotation = 360 - rotation;
@@ -687,6 +777,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     private static int numCaptureBuffers = 3;
     private final List<Frame> cameraFrames = new ArrayList<Frame>();
     public int frameSize = 0;
+    public int numCaptureBuffersAvailable = 0;
     private Camera camera;
 
     private static class Frame {
@@ -713,7 +804,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       this.camera = camera;
       int newFrameSize = CaptureFormat.frameSize(width, height, format);
 
-      int numberOfEnquedCameraBuffers = 0;
+      numCaptureBuffersAvailable = 0;
       if (newFrameSize != frameSize) {
         // Create new frames and add to the camera.
         // The old frames will be released when frames are returned.
@@ -722,33 +813,39 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
           cameraFrames.add(frame);
           this.camera.addCallbackBuffer(frame.data());
         }
-        numberOfEnquedCameraBuffers = numCaptureBuffers;
+        numCaptureBuffersAvailable = numCaptureBuffers;
       } else {
         // Add all frames that have been returned.
         for (Frame frame : cameraFrames) {
           if (frame.timeStamp < 0) {
             camera.addCallbackBuffer(frame.data());
-            ++numberOfEnquedCameraBuffers;
+            numCaptureBuffersAvailable++;
           }
         }
       }
       frameSize = newFrameSize;
-      Log.d(TAG, "queueCameraBuffers enqued " + numberOfEnquedCameraBuffers
+      Log.d(TAG, "queueCameraBuffers enqued " + numCaptureBuffersAvailable
           + " buffers of size " + frameSize + ".");
     }
 
+    String pendingFramesTimeStamps() {
+      String pendingTimeStamps = new String();
+      for (Frame frame : cameraFrames) {
+        if (frame.timeStamp > -1) {
+          pendingTimeStamps += " " +
+              TimeUnit.NANOSECONDS.toMillis(frame.timeStamp);
+        }
+      }
+      return pendingTimeStamps;
+    }
+
     void stopReturnBuffersToCamera() {
       this.camera = null;
-      String pendingTimeStamps = new String();
+      String pendingTimeStamps = pendingFramesTimeStamps();
-      for (Frame frame : cameraFrames) {
-        if (frame.timeStamp > -1) {
-          pendingTimeStamps+= " " + frame.timeStamp;
-        }
-      }
       Log.d(TAG, "stopReturnBuffersToCamera called."
             + (pendingTimeStamps.isEmpty() ?
                    " All buffers have been returned."
-                   : " Pending buffers " + pendingTimeStamps + "."));
+                   : " Pending buffers: [" + pendingTimeStamps + "]."));
 
     }
 
@@ -759,6 +856,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
             throw new RuntimeException("Frame already in use !");
           }
           frame.timeStamp = timeStamp;
+          numCaptureBuffersAvailable--;
+          if (numCaptureBuffersAvailable == 0) {
+            Log.v(TAG, "Camera is running out of capture buffers."
+                + " Pending buffers: [" + pendingFramesTimeStamps() + "]");
+          }
           return;
         }
       }
@@ -782,11 +884,17 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
 
       if (camera != null && returnedFrame.frameSize == frameSize) {
         camera.addCallbackBuffer(returnedFrame.data());
+        if (numCaptureBuffersAvailable == 0) {
+          Log.v(TAG, "Frame returned when camera is running out of capture"
+              + " buffers for TS " + TimeUnit.NANOSECONDS.toMillis(timeStamp));
+        }
+        numCaptureBuffersAvailable++;
         return;
       }
 
       if (returnedFrame.frameSize != frameSize) {
-        Log.d(TAG, "returnBuffer with time stamp "+ timeStamp
+        Log.d(TAG, "returnBuffer with time stamp "
+            + TimeUnit.NANOSECONDS.toMillis(timeStamp)
             + " called with old frame size, " + returnedFrame.frameSize + ".");
         // Since this frame has the wrong size, remove it from the list. Frames
         // with the correct size is created in queueCameraBuffers so this must
@@ -795,7 +903,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
         return;
       }
 
-      Log.d(TAG, "returnBuffer with time stamp "+ timeStamp
+      Log.d(TAG, "returnBuffer with time stamp "
+          + TimeUnit.NANOSECONDS.toMillis(timeStamp)
           + " called after camera has been stopped.");
     }
   }

talk/examples/android/src/org/appspot/apprtc/PeerConnectionClient.java

@@ -439,7 +439,7 @@ public class PeerConnectionClient {
         cameraDeviceName = frontCameraDeviceName;
       }
       Log.d(TAG, "Opening camera: " + cameraDeviceName);
-      videoCapturer = VideoCapturerAndroid.create(cameraDeviceName);
+      videoCapturer = VideoCapturerAndroid.create(cameraDeviceName, null);
       if (videoCapturer == null) {
         reportError("Failed to open camera");
         return;