diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java index 32544a6f1..e4141f02a 100644 --- a/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java +++ b/talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java @@ -129,7 +129,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { void starCapturerAndRender(String deviceName) throws InterruptedException { PeerConnectionFactory factory = new PeerConnectionFactory(); - VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName); + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create(deviceName, null); VideoSource source = factory.createVideoSource(capturer, new MediaConstraints()); VideoTrack track = factory.createVideoTrack("dummy", source); @@ -150,7 +151,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { @SmallTest public void testCreateAndRelease() throws Exception { - VideoCapturerAndroid capturer = VideoCapturerAndroid.create(""); + VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null); assertNotNull(capturer); capturer.dispose(); } @@ -158,7 +159,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { @SmallTest public void testCreateNoneExistingCamera() throws Exception { VideoCapturerAndroid capturer = VideoCapturerAndroid.create( - "none existing camera"); + "none existing camera", null); assertNull(capturer); } @@ -195,7 +196,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { // It tests both the Java and the C++ layer. public void testSwitchVideoCapturer() throws Exception { PeerConnectionFactory factory = new PeerConnectionFactory(); - VideoCapturerAndroid capturer = VideoCapturerAndroid.create(""); + VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null); VideoSource source = factory.createVideoSource(capturer, new MediaConstraints()); VideoTrack track = factory.createVideoTrack("dummy", source); @@ -222,7 +223,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { // be stopped and restarted. It tests both the Java and the C++ layer. public void testStopRestartVideoSource() throws Exception { PeerConnectionFactory factory = new PeerConnectionFactory(); - VideoCapturerAndroid capturer = VideoCapturerAndroid.create(""); + VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null); VideoSource source = factory.createVideoSource(capturer, new MediaConstraints()); VideoTrack track = factory.createVideoTrack("dummy", source); @@ -251,7 +252,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { String deviceName = VideoCapturerAndroid.getDeviceName(0); ArrayList formats = VideoCapturerAndroid.getSupportedFormats(0); - VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName); + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create(deviceName, null); for(int i = 0; i < 3 ; ++i) { VideoCapturerAndroid.CaptureFormat format = formats.get(i); @@ -275,7 +277,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase { String deviceName = VideoCapturerAndroid.getDeviceName(0); ArrayList formats = VideoCapturerAndroid.getSupportedFormats(0); - VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName); + VideoCapturerAndroid capturer = + VideoCapturerAndroid.create(deviceName, null); VideoCapturerAndroid.CaptureFormat format = formats.get(0); capturer.startCapture(format.width, format.height, format.maxFramerate, diff --git a/talk/app/webrtc/androidvideocapturer.cc b/talk/app/webrtc/androidvideocapturer.cc index 89ab48696..2c9c03102 100644 --- a/talk/app/webrtc/androidvideocapturer.cc +++ b/talk/app/webrtc/androidvideocapturer.cc @@ -89,10 +89,10 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory { // |captured_frame_.data| is only guaranteed to be valid during the scope // of |AndroidVideoCapturer::OnIncomingFrame_w|. // Check that captured_frame is actually our frame. - DCHECK(captured_frame == &captured_frame_); + CHECK(captured_frame == &captured_frame_); if (!apply_rotation_ || captured_frame->rotation == kVideoRotation_0) { - DCHECK(captured_frame->fourcc == cricket::FOURCC_YV12); + CHECK(captured_frame->fourcc == cricket::FOURCC_YV12); const uint8_t* y_plane = static_cast(captured_frame_.data); // Android guarantees that the stride is a multiple of 16. @@ -160,7 +160,7 @@ AndroidVideoCapturer::AndroidVideoCapturer( std::vector formats; for (Json::ArrayIndex i = 0; i < json_values.size(); ++i) { const Json::Value& json_value = json_values[i]; - DCHECK(!json_value["width"].isNull() && !json_value["height"].isNull() && + CHECK(!json_value["width"].isNull() && !json_value["height"].isNull() && !json_value["framerate"].isNull()); cricket::VideoFormat format( json_value["width"].asInt(), @@ -173,16 +173,16 @@ AndroidVideoCapturer::AndroidVideoCapturer( } AndroidVideoCapturer::~AndroidVideoCapturer() { - DCHECK(!running_); + CHECK(!running_); } cricket::CaptureState AndroidVideoCapturer::Start( const cricket::VideoFormat& capture_format) { - DCHECK(thread_checker_.CalledOnValidThread()); - DCHECK(!running_); - LOG(LS_INFO) << " AndroidVideoCapturer::Start w = " << capture_format.width << " h = " << capture_format.height; + CHECK(thread_checker_.CalledOnValidThread()); + CHECK(!running_); + frame_factory_ = new AndroidVideoCapturer::FrameFactory( capture_format.width, capture_format.height, delegate_.get()); set_frame_factory(frame_factory_); @@ -197,9 +197,9 @@ cricket::CaptureState AndroidVideoCapturer::Start( } void AndroidVideoCapturer::Stop() { - DCHECK(thread_checker_.CalledOnValidThread()); LOG(LS_INFO) << " AndroidVideoCapturer::Stop "; - DCHECK(running_); + CHECK(thread_checker_.CalledOnValidThread()); + CHECK(running_); running_ = false; SetCaptureFormat(NULL); @@ -209,18 +209,18 @@ void AndroidVideoCapturer::Stop() { } bool AndroidVideoCapturer::IsRunning() { - DCHECK(thread_checker_.CalledOnValidThread()); + CHECK(thread_checker_.CalledOnValidThread()); return running_; } bool AndroidVideoCapturer::GetPreferredFourccs(std::vector* fourccs) { - DCHECK(thread_checker_.CalledOnValidThread()); + CHECK(thread_checker_.CalledOnValidThread()); fourccs->push_back(cricket::FOURCC_YV12); return true; } void AndroidVideoCapturer::OnCapturerStarted(bool success) { - DCHECK(thread_checker_.CalledOnValidThread()); + CHECK(thread_checker_.CalledOnValidThread()); cricket::CaptureState new_state = success ? cricket::CS_RUNNING : cricket::CS_FAILED; if (new_state == current_state_) @@ -237,7 +237,7 @@ void AndroidVideoCapturer::OnIncomingFrame(void* frame_data, int length, int rotation, int64 time_stamp) { - DCHECK(thread_checker_.CalledOnValidThread()); + CHECK(thread_checker_.CalledOnValidThread()); frame_factory_->UpdateCapturedFrame(frame_data, length, rotation, time_stamp); SignalFrameCaptured(this, frame_factory_->GetCapturedFrame()); } diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc index a51c6efa7..1fcf635ac 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc @@ -54,8 +54,10 @@ AndroidVideoCapturerJni::Create(JNIEnv* jni, rtc::scoped_refptr capturer( new rtc::RefCountedObject(jni, j_video_capture)); - if (capturer->Init(device_name)) + if (capturer->Init(device_name)) { return capturer; + } + LOG(LS_ERROR) << "AndroidVideoCapturerJni init fails"; return nullptr; } diff --git a/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java b/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java index ea4a512d5..e40cd1ac2 100644 --- a/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java +++ b/talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java @@ -51,6 +51,7 @@ import org.json.JSONObject; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.Exchanger; import java.util.concurrent.TimeUnit; @@ -73,6 +74,7 @@ import java.util.concurrent.TimeUnit; @SuppressWarnings("deprecation") public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallback { private final static String TAG = "VideoCapturerAndroid"; + private final static int CAMERA_OBSERVER_PERIOD_MS = 5000; private Camera camera; // Only non-null while capturing. private CameraThread cameraThread; @@ -86,12 +88,69 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private int width; private int height; private int framerate; + private int cameraFramesCount; + private int captureBuffersCount; private volatile boolean pendingCameraSwitch; private CapturerObserver frameObserver = null; + private CameraErrorHandler errorHandler = null; // List of formats supported by all cameras. This list is filled once in order // to be able to switch cameras. private static List> supportedFormats; + // Camera error callback. + private final Camera.ErrorCallback cameraErrorCallback = + new Camera.ErrorCallback() { + @Override + public void onError(int error, Camera camera) { + String errorMessage; + if (error == android.hardware.Camera.CAMERA_ERROR_SERVER_DIED) { + errorMessage = "Camera server died!"; + } else { + errorMessage = "Camera error: " + error; + } + Log.e(TAG, errorMessage); + if (errorHandler != null) { + errorHandler.onCameraError(errorMessage); + } + } + }; + + // Camera observer - monitors camera framerate and amount of available + // camera buffers. Observer is excecuted on camera thread. + private final Runnable cameraObserver = new Runnable() { + @Override + public void run() { + int cameraFps = (cameraFramesCount * 1000 + CAMERA_OBSERVER_PERIOD_MS / 2) + / CAMERA_OBSERVER_PERIOD_MS; + double averageCaptureBuffersCount = 0; + if (cameraFramesCount > 0) { + averageCaptureBuffersCount = + (double)captureBuffersCount / cameraFramesCount; + } + Log.d(TAG, "Camera fps: " + cameraFps + ". CaptureBuffers: " + + String.format("%.1f", averageCaptureBuffersCount) + + ". Pending buffers: [" + videoBuffers.pendingFramesTimeStamps() + "]"); + if (cameraFramesCount == 0) { + Log.e(TAG, "Camera freezed."); + if (errorHandler != null) { + errorHandler.onCameraError("Camera failure."); + } + } else { + cameraFramesCount = 0; + captureBuffersCount = 0; + if (cameraThreadHandler != null) { + cameraThreadHandler.postDelayed(this, CAMERA_OBSERVER_PERIOD_MS); + } + } + } + }; + + // Camera error handler - invoked when camera stops receiving frames + // or any camera exception happens on camera thread. + public static interface CameraErrorHandler { + public void onCameraError(String errorDescription); + } + // Returns device names that can be used to create a new VideoCapturerAndroid. public static String[] getDeviceNames() { String[] names = new String[Camera.getNumberOfCameras()]; @@ -155,10 +214,14 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba return null; } - public static VideoCapturerAndroid create(String name) { + public static VideoCapturerAndroid create(String name, + CameraErrorHandler errorHandler) { VideoCapturer capturer = VideoCapturer.create(name); - if (capturer != null) - return (VideoCapturerAndroid) capturer; + if (capturer != null) { + VideoCapturerAndroid capturerAndroid = (VideoCapturerAndroid) capturer; + capturerAndroid.errorHandler = errorHandler; + return capturerAndroid; + } return null; } @@ -218,7 +281,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // If deviceName is empty, the first available device is used in order to be // compatible with the generic VideoCapturer class. synchronized boolean init(String deviceName) { - Log.d(TAG, "init " + deviceName); + Log.d(TAG, "init: " + deviceName); if (deviceName == null || !initStatics()) return false; @@ -245,9 +308,20 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba Log.d(TAG, "Get supported formats."); supportedFormats = new ArrayList>(Camera.getNumberOfCameras()); - for (int i = 0; i < Camera.getNumberOfCameras(); ++i) { - supportedFormats.add(getSupportedFormats(i)); + // Start requesting supported formats from camera with the highest index + // (back camera) first. If it fails then likely camera is in bad state. + for (int i = Camera.getNumberOfCameras() - 1; i >= 0; i--) { + ArrayList supportedFormat = getSupportedFormats(i); + if (supportedFormat.size() == 0) { + Log.e(TAG, "Fail to get supported formats for camera " + i); + supportedFormats = null; + return false; + } + supportedFormats.add(supportedFormat); } + // Reverse the list since it is filled in reverse order. + Collections.reverse(supportedFormats); + Log.d(TAG, "Get supported formats done."); return true; } catch (Exception e) { supportedFormats = null; @@ -328,6 +402,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba Camera camera; try { + Log.d(TAG, "Opening camera " + id); camera = Camera.open(id); } catch (Exception e) { Log.e(TAG, "Open camera failed on id " + id, e); @@ -353,6 +428,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba Log.e(TAG, "getSupportedFormats failed on id " + id, e); } camera.release(); + camera = null; return formatList; } @@ -411,6 +487,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba this.applicationContext = applicationContext; this.frameObserver = frameObserver; try { + Log.d(TAG, "Opening camera " + id); camera = Camera.open(id); info = new Camera.CameraInfo(); Camera.getCameraInfo(id, info); @@ -454,6 +531,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba parameters.setVideoStabilization(true); } + camera.setErrorCallback(cameraErrorCallback); + int androidFramerate = framerate * 1000; int[] range = getFramerateRange(parameters, androidFramerate); if (range != null) { @@ -480,6 +559,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba camera.setPreviewCallbackWithBuffer(this); camera.startPreview(); frameObserver.OnCapturerStarted(true); + + // Start camera observer. + cameraFramesCount = 0; + captureBuffersCount = 0; + cameraThreadHandler.postDelayed(cameraObserver, CAMERA_OBSERVER_PERIOD_MS); return; } catch (RuntimeException e) { error = e; @@ -488,6 +572,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba stopCaptureOnCameraThread(); cameraThreadHandler = null; frameObserver.OnCapturerStarted(false); + if (errorHandler != null) { + errorHandler.onCameraError("Camera can not be started."); + } return; } @@ -509,17 +596,18 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } private void stopCaptureOnCameraThread() { - Log.d(TAG, "stopCaptureOnCameraThread"); - doStopCaptureOnCamerathread(); + doStopCaptureOnCameraThread(); Looper.myLooper().quit(); return; } - private void doStopCaptureOnCamerathread() { + private void doStopCaptureOnCameraThread() { + Log.d(TAG, "stopCaptureOnCameraThread"); if (camera == null) { return; } try { + cameraThreadHandler.removeCallbacks(cameraObserver); Log.d(TAG, "Stop preview."); camera.stopPreview(); camera.setPreviewCallbackWithBuffer(null); @@ -543,7 +631,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private void switchCameraOnCameraThread(Runnable switchDoneEvent) { Log.d(TAG, "switchCameraOnCameraThread"); - doStopCaptureOnCamerathread(); + doStopCaptureOnCameraThread(); startCaptureOnCameraThread(width, height, framerate, frameObserver, applicationContext); pendingCameraSwitch = false; @@ -638,6 +726,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba long captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); + cameraFramesCount++; + captureBuffersCount += videoBuffers.numCaptureBuffersAvailable; int rotation = getDeviceOrientation(); if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { rotation = 360 - rotation; @@ -687,6 +777,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private static int numCaptureBuffers = 3; private final List cameraFrames = new ArrayList(); public int frameSize = 0; + public int numCaptureBuffersAvailable = 0; private Camera camera; private static class Frame { @@ -713,7 +804,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba this.camera = camera; int newFrameSize = CaptureFormat.frameSize(width, height, format); - int numberOfEnquedCameraBuffers = 0; + numCaptureBuffersAvailable = 0; if (newFrameSize != frameSize) { // Create new frames and add to the camera. // The old frames will be released when frames are returned. @@ -722,33 +813,39 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba cameraFrames.add(frame); this.camera.addCallbackBuffer(frame.data()); } - numberOfEnquedCameraBuffers = numCaptureBuffers; + numCaptureBuffersAvailable = numCaptureBuffers; } else { // Add all frames that have been returned. for (Frame frame : cameraFrames) { if (frame.timeStamp < 0) { camera.addCallbackBuffer(frame.data()); - ++numberOfEnquedCameraBuffers; + numCaptureBuffersAvailable++; } } } frameSize = newFrameSize; - Log.d(TAG, "queueCameraBuffers enqued " + numberOfEnquedCameraBuffers + Log.d(TAG, "queueCameraBuffers enqued " + numCaptureBuffersAvailable + " buffers of size " + frameSize + "."); } + String pendingFramesTimeStamps() { + String pendingTimeStamps = new String(); + for (Frame frame : cameraFrames) { + if (frame.timeStamp > -1) { + pendingTimeStamps += " " + + TimeUnit.NANOSECONDS.toMillis(frame.timeStamp); + } + } + return pendingTimeStamps; + } + void stopReturnBuffersToCamera() { this.camera = null; - String pendingTimeStamps = new String(); - for (Frame frame : cameraFrames) { - if (frame.timeStamp > -1) { - pendingTimeStamps+= " " + frame.timeStamp; - } - } + String pendingTimeStamps = pendingFramesTimeStamps(); Log.d(TAG, "stopReturnBuffersToCamera called." + (pendingTimeStamps.isEmpty() ? " All buffers have been returned." - : " Pending buffers " + pendingTimeStamps + ".")); + : " Pending buffers: [" + pendingTimeStamps + "].")); } @@ -759,6 +856,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba throw new RuntimeException("Frame already in use !"); } frame.timeStamp = timeStamp; + numCaptureBuffersAvailable--; + if (numCaptureBuffersAvailable == 0) { + Log.v(TAG, "Camera is running out of capture buffers." + + " Pending buffers: [" + pendingFramesTimeStamps() + "]"); + } return; } } @@ -782,11 +884,17 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba if (camera != null && returnedFrame.frameSize == frameSize) { camera.addCallbackBuffer(returnedFrame.data()); + if (numCaptureBuffersAvailable == 0) { + Log.v(TAG, "Frame returned when camera is running out of capture" + + " buffers for TS " + TimeUnit.NANOSECONDS.toMillis(timeStamp)); + } + numCaptureBuffersAvailable++; return; } if (returnedFrame.frameSize != frameSize) { - Log.d(TAG, "returnBuffer with time stamp "+ timeStamp + Log.d(TAG, "returnBuffer with time stamp " + + TimeUnit.NANOSECONDS.toMillis(timeStamp) + " called with old frame size, " + returnedFrame.frameSize + "."); // Since this frame has the wrong size, remove it from the list. Frames // with the correct size is created in queueCameraBuffers so this must @@ -795,7 +903,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba return; } - Log.d(TAG, "returnBuffer with time stamp "+ timeStamp + Log.d(TAG, "returnBuffer with time stamp " + + TimeUnit.NANOSECONDS.toMillis(timeStamp) + " called after camera has been stopped."); } } diff --git a/talk/examples/android/src/org/appspot/apprtc/PeerConnectionClient.java b/talk/examples/android/src/org/appspot/apprtc/PeerConnectionClient.java index 967a2ea50..739c49eb7 100644 --- a/talk/examples/android/src/org/appspot/apprtc/PeerConnectionClient.java +++ b/talk/examples/android/src/org/appspot/apprtc/PeerConnectionClient.java @@ -439,7 +439,7 @@ public class PeerConnectionClient { cameraDeviceName = frontCameraDeviceName; } Log.d(TAG, "Opening camera: " + cameraDeviceName); - videoCapturer = VideoCapturerAndroid.create(cameraDeviceName); + videoCapturer = VideoCapturerAndroid.create(cameraDeviceName, null); if (videoCapturer == null) { reportError("Failed to open camera"); return;