generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve Android camera error handling.

Browse Source 
- Set Camera.ErrorCallback callback when opening camera to
receive camera server error notifications.
- Allow user to provide interface for handling camera errors
happening on camera thread.
- Run camera observer on camera thread and monitor camera fps
and amount of callback buffers, print statistics and report error
if camera stops generating frames.
- Query camera formats starting from front camera instead of back
camera to detect camera failures as fast as possible.
- Change all DCHECK to CHECK in androidvideocapturer.cc to detect
camera error on release builds.
- Plus adding some extra logging.

R=hbos@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/52519004

Cr-Commit-Position: refs/heads/master@{#9221}
This commit is contained in:
Alex Glaznev
2015-05-19 10:56:32 -07:00
parent 68898a2652
commit 2f5be9ad63
5 changed files with 159 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java
View File

@@ -129,7 +129,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
void starCapturerAndRender(String deviceName) throws InterruptedException {
PeerConnectionFactory factory = new PeerConnectionFactory();
VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName);
VideoCapturerAndroid capturer =
VideoCapturerAndroid.create(deviceName, null);
VideoSource source =
factory.createVideoSource(capturer, new MediaConstraints());
VideoTrack track = factory.createVideoTrack("dummy", source);
@@ -150,7 +151,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
@SmallTest
public void testCreateAndRelease() throws Exception {
VideoCapturerAndroid capturer = VideoCapturerAndroid.create("");
VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null);
assertNotNull(capturer);
capturer.dispose();
}
@@ -158,7 +159,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
@SmallTest
public void testCreateNoneExistingCamera() throws Exception {
VideoCapturerAndroid capturer = VideoCapturerAndroid.create(
"none existing camera");
"none existing camera", null);
assertNull(capturer);
}
@@ -195,7 +196,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
// It tests both the Java and the C++ layer.
public void testSwitchVideoCapturer() throws Exception {
PeerConnectionFactory factory = new PeerConnectionFactory();
VideoCapturerAndroid capturer = VideoCapturerAndroid.create("");
VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null);
VideoSource source =
factory.createVideoSource(capturer, new MediaConstraints());
VideoTrack track = factory.createVideoTrack("dummy", source);
@@ -222,7 +223,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
// be stopped and restarted. It tests both the Java and the C++ layer.
public void testStopRestartVideoSource() throws Exception {
PeerConnectionFactory factory = new PeerConnectionFactory();
VideoCapturerAndroid capturer = VideoCapturerAndroid.create("");
VideoCapturerAndroid capturer = VideoCapturerAndroid.create("", null);
VideoSource source =
factory.createVideoSource(capturer, new MediaConstraints());
VideoTrack track = factory.createVideoTrack("dummy", source);
@@ -251,7 +252,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
String deviceName = VideoCapturerAndroid.getDeviceName(0);
ArrayList<CaptureFormat> formats =
VideoCapturerAndroid.getSupportedFormats(0);
VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName);
VideoCapturerAndroid capturer =
VideoCapturerAndroid.create(deviceName, null);
for(int i = 0; i < 3 ; ++i) {
VideoCapturerAndroid.CaptureFormat format = formats.get(i);
@@ -275,7 +277,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
String deviceName = VideoCapturerAndroid.getDeviceName(0);
ArrayList<CaptureFormat> formats =
VideoCapturerAndroid.getSupportedFormats(0);
VideoCapturerAndroid capturer = VideoCapturerAndroid.create(deviceName);
VideoCapturerAndroid capturer =
VideoCapturerAndroid.create(deviceName, null);
VideoCapturerAndroid.CaptureFormat format = formats.get(0);
capturer.startCapture(format.width, format.height, format.maxFramerate,

26
talk/app/webrtc/androidvideocapturer.cc
View File

@@ -89,10 +89,10 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory {
// |captured_frame_.data| is only guaranteed to be valid during the scope
// of |AndroidVideoCapturer::OnIncomingFrame_w|.
// Check that captured_frame is actually our frame.
DCHECK(captured_frame == &captured_frame_);
CHECK(captured_frame == &captured_frame_);
if (!apply_rotation_ || captured_frame->rotation == kVideoRotation_0) {
DCHECK(captured_frame->fourcc == cricket::FOURCC_YV12);
CHECK(captured_frame->fourcc == cricket::FOURCC_YV12);
const uint8_t* y_plane = static_cast<uint8_t*>(captured_frame_.data);
// Android guarantees that the stride is a multiple of 16.
@@ -160,7 +160,7 @@ AndroidVideoCapturer::AndroidVideoCapturer(
std::vector<cricket::VideoFormat> formats;
for (Json::ArrayIndex i = 0; i < json_values.size(); ++i) {
const Json::Value& json_value = json_values[i];
DCHECK(!json_value["width"].isNull() && !json_value["height"].isNull() &&
CHECK(!json_value["width"].isNull() && !json_value["height"].isNull() &&
!json_value["framerate"].isNull());
cricket::VideoFormat format(
json_value["width"].asInt(),
@@ -173,16 +173,16 @@ AndroidVideoCapturer::AndroidVideoCapturer(
}
AndroidVideoCapturer::~AndroidVideoCapturer() {
DCHECK(!running_);
CHECK(!running_);
}
cricket::CaptureState AndroidVideoCapturer::Start(
const cricket::VideoFormat& capture_format) {
DCHECK(thread_checker_.CalledOnValidThread());
DCHECK(!running_);
LOG(LS_INFO) << " AndroidVideoCapturer::Start w = " << capture_format.width
<< " h = " << capture_format.height;
CHECK(thread_checker_.CalledOnValidThread());
CHECK(!running_);
frame_factory_ = new AndroidVideoCapturer::FrameFactory(
capture_format.width, capture_format.height, delegate_.get());
set_frame_factory(frame_factory_);
@@ -197,9 +197,9 @@ cricket::CaptureState AndroidVideoCapturer::Start(
}
void AndroidVideoCapturer::Stop() {
DCHECK(thread_checker_.CalledOnValidThread());
LOG(LS_INFO) << " AndroidVideoCapturer::Stop ";
DCHECK(running_);
CHECK(thread_checker_.CalledOnValidThread());
CHECK(running_);
running_ = false;
SetCaptureFormat(NULL);
@@ -209,18 +209,18 @@ void AndroidVideoCapturer::Stop() {
}
bool AndroidVideoCapturer::IsRunning() {
DCHECK(thread_checker_.CalledOnValidThread());
CHECK(thread_checker_.CalledOnValidThread());
return running_;
}
bool AndroidVideoCapturer::GetPreferredFourccs(std::vector<uint32>* fourccs) {
DCHECK(thread_checker_.CalledOnValidThread());
CHECK(thread_checker_.CalledOnValidThread());
fourccs->push_back(cricket::FOURCC_YV12);
return true;
}
void AndroidVideoCapturer::OnCapturerStarted(bool success) {
DCHECK(thread_checker_.CalledOnValidThread());
CHECK(thread_checker_.CalledOnValidThread());
cricket::CaptureState new_state =
success ? cricket::CS_RUNNING : cricket::CS_FAILED;
if (new_state == current_state_)
@@ -237,7 +237,7 @@ void AndroidVideoCapturer::OnIncomingFrame(void* frame_data,
int length,
int rotation,
int64 time_stamp) {
DCHECK(thread_checker_.CalledOnValidThread());
CHECK(thread_checker_.CalledOnValidThread());
frame_factory_->UpdateCapturedFrame(frame_data, length, rotation, time_stamp);
SignalFrameCaptured(this, frame_factory_->GetCapturedFrame());
}

4
talk/app/webrtc/java/jni/androidvideocapturer_jni.cc
View File

@@ -54,8 +54,10 @@ AndroidVideoCapturerJni::Create(JNIEnv* jni,
rtc::scoped_refptr<AndroidVideoCapturerJni> capturer(
new rtc::RefCountedObject<AndroidVideoCapturerJni>(jni, j_video_capture));
if (capturer->Init(device_name))
if (capturer->Init(device_name)) {
return capturer;
}
LOG(LS_ERROR) << "AndroidVideoCapturerJni init fails";
return nullptr;
}

155
talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java
View File

@@ -51,6 +51,7 @@ import org.json.JSONObject;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Exchanger;
import java.util.concurrent.TimeUnit;
@@ -73,6 +74,7 @@ import java.util.concurrent.TimeUnit;
@SuppressWarnings("deprecation")
public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallback {
private final static String TAG = "VideoCapturerAndroid";
private final static int CAMERA_OBSERVER_PERIOD_MS = 5000;
private Camera camera; // Only non-null while capturing.
private CameraThread cameraThread;
@@ -86,12 +88,69 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
private int width;
private int height;
private int framerate;
private int cameraFramesCount;
private int captureBuffersCount;
private volatile boolean pendingCameraSwitch;
private CapturerObserver frameObserver = null;
private CameraErrorHandler errorHandler = null;
// List of formats supported by all cameras. This list is filled once in order
// to be able to switch cameras.
private static List<List<CaptureFormat>> supportedFormats;
// Camera error callback.
private final Camera.ErrorCallback cameraErrorCallback =
new Camera.ErrorCallback() {
@Override
public void onError(int error, Camera camera) {
String errorMessage;
if (error == android.hardware.Camera.CAMERA_ERROR_SERVER_DIED) {
errorMessage = "Camera server died!";
} else {
errorMessage = "Camera error: " + error;
}
Log.e(TAG, errorMessage);
if (errorHandler != null) {
errorHandler.onCameraError(errorMessage);
}
}
};
// Camera observer - monitors camera framerate and amount of available
// camera buffers. Observer is excecuted on camera thread.
private final Runnable cameraObserver = new Runnable() {
@Override
public void run() {
int cameraFps = (cameraFramesCount * 1000 + CAMERA_OBSERVER_PERIOD_MS / 2)
/ CAMERA_OBSERVER_PERIOD_MS;
double averageCaptureBuffersCount = 0;
if (cameraFramesCount > 0) {
averageCaptureBuffersCount =
(double)captureBuffersCount / cameraFramesCount;
}
Log.d(TAG, "Camera fps: " + cameraFps + ". CaptureBuffers: " +
String.format("%.1f", averageCaptureBuffersCount) +
". Pending buffers: [" + videoBuffers.pendingFramesTimeStamps() + "]");
if (cameraFramesCount == 0) {
Log.e(TAG, "Camera freezed.");
if (errorHandler != null) {
errorHandler.onCameraError("Camera failure.");
}
} else {
cameraFramesCount = 0;
captureBuffersCount = 0;
if (cameraThreadHandler != null) {
cameraThreadHandler.postDelayed(this, CAMERA_OBSERVER_PERIOD_MS);
}
}
}
};
// Camera error handler - invoked when camera stops receiving frames
// or any camera exception happens on camera thread.
public static interface CameraErrorHandler {
public void onCameraError(String errorDescription);
}
// Returns device names that can be used to create a new VideoCapturerAndroid.
public static String[] getDeviceNames() {
String[] names = new String[Camera.getNumberOfCameras()];
@@ -155,10 +214,14 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
return null;
}
public static VideoCapturerAndroid create(String name) {
public static VideoCapturerAndroid create(String name,
CameraErrorHandler errorHandler) {
VideoCapturer capturer = VideoCapturer.create(name);
if (capturer != null)
return (VideoCapturerAndroid) capturer;
if (capturer != null) {
VideoCapturerAndroid capturerAndroid = (VideoCapturerAndroid) capturer;
capturerAndroid.errorHandler = errorHandler;
return capturerAndroid;
}
return null;
}
@@ -218,7 +281,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
// If deviceName is empty, the first available device is used in order to be
// compatible with the generic VideoCapturer class.
synchronized boolean init(String deviceName) {
Log.d(TAG, "init " + deviceName);
Log.d(TAG, "init: " + deviceName);
if (deviceName == null || !initStatics())
return false;
@@ -245,9 +308,20 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
Log.d(TAG, "Get supported formats.");
supportedFormats =
new ArrayList<List<CaptureFormat>>(Camera.getNumberOfCameras());
for (int i = 0; i < Camera.getNumberOfCameras(); ++i) {
supportedFormats.add(getSupportedFormats(i));
// Start requesting supported formats from camera with the highest index
// (back camera) first. If it fails then likely camera is in bad state.
for (int i = Camera.getNumberOfCameras() - 1; i >= 0; i--) {
ArrayList<CaptureFormat> supportedFormat = getSupportedFormats(i);
if (supportedFormat.size() == 0) {
Log.e(TAG, "Fail to get supported formats for camera " + i);
supportedFormats = null;
return false;
}
supportedFormats.add(supportedFormat);
}
// Reverse the list since it is filled in reverse order.
Collections.reverse(supportedFormats);
Log.d(TAG, "Get supported formats done.");
return true;
} catch (Exception e) {
supportedFormats = null;
@@ -328,6 +402,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
Camera camera;
try {
Log.d(TAG, "Opening camera " + id);
camera = Camera.open(id);
} catch (Exception e) {
Log.e(TAG, "Open camera failed on id " + id, e);
@@ -353,6 +428,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
Log.e(TAG, "getSupportedFormats failed on id " + id, e);
}
camera.release();
camera = null;
return formatList;
}
@@ -411,6 +487,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
this.applicationContext = applicationContext;
this.frameObserver = frameObserver;
try {
Log.d(TAG, "Opening camera " + id);
camera = Camera.open(id);
info = new Camera.CameraInfo();
Camera.getCameraInfo(id, info);
@@ -454,6 +531,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
parameters.setVideoStabilization(true);
}
camera.setErrorCallback(cameraErrorCallback);
int androidFramerate = framerate * 1000;
int[] range = getFramerateRange(parameters, androidFramerate);
if (range != null) {
@@ -480,6 +559,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
camera.setPreviewCallbackWithBuffer(this);
camera.startPreview();
frameObserver.OnCapturerStarted(true);
// Start camera observer.
cameraFramesCount = 0;
captureBuffersCount = 0;
cameraThreadHandler.postDelayed(cameraObserver, CAMERA_OBSERVER_PERIOD_MS);
return;
} catch (RuntimeException e) {
error = e;
@@ -488,6 +572,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
stopCaptureOnCameraThread();
cameraThreadHandler = null;
frameObserver.OnCapturerStarted(false);
if (errorHandler != null) {
errorHandler.onCameraError("Camera can not be started.");
}
return;
}
@@ -509,17 +596,18 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
}
private void stopCaptureOnCameraThread() {
Log.d(TAG, "stopCaptureOnCameraThread");
doStopCaptureOnCamerathread();
doStopCaptureOnCameraThread();
Looper.myLooper().quit();
return;
}
private void doStopCaptureOnCamerathread() {
private void doStopCaptureOnCameraThread() {
Log.d(TAG, "stopCaptureOnCameraThread");
if (camera == null) {
return;
}
try {
cameraThreadHandler.removeCallbacks(cameraObserver);
Log.d(TAG, "Stop preview.");
camera.stopPreview();
camera.setPreviewCallbackWithBuffer(null);
@@ -543,7 +631,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
private void switchCameraOnCameraThread(Runnable switchDoneEvent) {
Log.d(TAG, "switchCameraOnCameraThread");
doStopCaptureOnCamerathread();
doStopCaptureOnCameraThread();
startCaptureOnCameraThread(width, height, framerate, frameObserver,
applicationContext);
pendingCameraSwitch = false;
@@ -638,6 +726,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
long captureTimeNs =
TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());
cameraFramesCount++;
captureBuffersCount += videoBuffers.numCaptureBuffersAvailable;
int rotation = getDeviceOrientation();
if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) {
rotation = 360 - rotation;
@@ -687,6 +777,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
private static int numCaptureBuffers = 3;
private final List<Frame> cameraFrames = new ArrayList<Frame>();
public int frameSize = 0;
public int numCaptureBuffersAvailable = 0;
private Camera camera;
private static class Frame {
@@ -713,7 +804,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
this.camera = camera;
int newFrameSize = CaptureFormat.frameSize(width, height, format);
int numberOfEnquedCameraBuffers = 0;
numCaptureBuffersAvailable = 0;
if (newFrameSize != frameSize) {
// Create new frames and add to the camera.
// The old frames will be released when frames are returned.
@@ -722,33 +813,39 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
cameraFrames.add(frame);
this.camera.addCallbackBuffer(frame.data());
}
numberOfEnquedCameraBuffers = numCaptureBuffers;
numCaptureBuffersAvailable = numCaptureBuffers;
} else {
// Add all frames that have been returned.
for (Frame frame : cameraFrames) {
if (frame.timeStamp < 0) {
camera.addCallbackBuffer(frame.data());
++numberOfEnquedCameraBuffers;
numCaptureBuffersAvailable++;
}
}
}
frameSize = newFrameSize;
Log.d(TAG, "queueCameraBuffers enqued " + numberOfEnquedCameraBuffers
Log.d(TAG, "queueCameraBuffers enqued " + numCaptureBuffersAvailable
+ " buffers of size " + frameSize + ".");
}
String pendingFramesTimeStamps() {
String pendingTimeStamps = new String();
for (Frame frame : cameraFrames) {
if (frame.timeStamp > -1) {
pendingTimeStamps += " " +
TimeUnit.NANOSECONDS.toMillis(frame.timeStamp);
}
}
return pendingTimeStamps;
}
void stopReturnBuffersToCamera() {
this.camera = null;
String pendingTimeStamps = new String();
for (Frame frame : cameraFrames) {
if (frame.timeStamp > -1) {
pendingTimeStamps+= " " + frame.timeStamp;
}
}
String pendingTimeStamps = pendingFramesTimeStamps();
Log.d(TAG, "stopReturnBuffersToCamera called."
+ (pendingTimeStamps.isEmpty() ?
" All buffers have been returned."
: " Pending buffers " + pendingTimeStamps + "."));
: " Pending buffers: [" + pendingTimeStamps + "]."));
}
@@ -759,6 +856,11 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
throw new RuntimeException("Frame already in use !");
}
frame.timeStamp = timeStamp;
numCaptureBuffersAvailable--;
if (numCaptureBuffersAvailable == 0) {
Log.v(TAG, "Camera is running out of capture buffers."
+ " Pending buffers: [" + pendingFramesTimeStamps() + "]");
}
return;
}
}
@@ -782,11 +884,17 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
if (camera != null && returnedFrame.frameSize == frameSize) {
camera.addCallbackBuffer(returnedFrame.data());
if (numCaptureBuffersAvailable == 0) {
Log.v(TAG, "Frame returned when camera is running out of capture"
+ " buffers for TS " + TimeUnit.NANOSECONDS.toMillis(timeStamp));
}
numCaptureBuffersAvailable++;
return;
}
if (returnedFrame.frameSize != frameSize) {
Log.d(TAG, "returnBuffer with time stamp "+ timeStamp
Log.d(TAG, "returnBuffer with time stamp "
+ TimeUnit.NANOSECONDS.toMillis(timeStamp)
+ " called with old frame size, " + returnedFrame.frameSize + ".");
// Since this frame has the wrong size, remove it from the list. Frames
// with the correct size is created in queueCameraBuffers so this must
@@ -795,7 +903,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
return;
}
Log.d(TAG, "returnBuffer with time stamp "+ timeStamp
Log.d(TAG, "returnBuffer with time stamp "
+ TimeUnit.NANOSECONDS.toMillis(timeStamp)
+ " called after camera has been stopped.");
}
}

2
talk/examples/android/src/org/appspot/apprtc/PeerConnectionClient.java
View File

@@ -439,7 +439,7 @@ public class PeerConnectionClient {
cameraDeviceName = frontCameraDeviceName;
}
Log.d(TAG, "Opening camera: " + cameraDeviceName);
videoCapturer = VideoCapturerAndroid.create(cameraDeviceName);
videoCapturer = VideoCapturerAndroid.create(cameraDeviceName, null);
if (videoCapturer == null) {
reportError("Failed to open camera");
return;