From 1f72815637b255b6d9e9a88522720c43e0d0cf47 Mon Sep 17 00:00:00 2001 From: Nguyen Nguyen Date: Mon, 4 May 2015 12:38:55 -0400 Subject: [PATCH 1/6] Exporting videoFileString to outside world --- modules/highgui/include/opencv2/highgui/cap_ios.h | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/highgui/include/opencv2/highgui/cap_ios.h b/modules/highgui/include/opencv2/highgui/cap_ios.h index db3928f13..82230550f 100644 --- a/modules/highgui/include/opencv2/highgui/cap_ios.h +++ b/modules/highgui/include/opencv2/highgui/cap_ios.h @@ -142,6 +142,7 @@ - (void)layoutPreviewLayer; - (void)saveVideo; - (NSURL *)videoFileURL; +- (NSString *)videoFileString; @end From b69ffe6376d7ba8bf2eaf55f8a37cf62d4bc34e6 Mon Sep 17 00:00:00 2001 From: Nguyen Nguyen Date: Mon, 4 May 2015 16:21:25 -0400 Subject: [PATCH 2/6] Fixed memory leak when recording video in iOS --- modules/highgui/src/cap_ios_video_camera.mm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/highgui/src/cap_ios_video_camera.mm b/modules/highgui/src/cap_ios_video_camera.mm index 45db2b60a..42fd75555 100644 --- a/modules/highgui/src/cap_ios_video_camera.mm +++ b/modules/highgui/src/cap_ios_video_camera.mm @@ -557,6 +557,8 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; withPresentationTime:lastSampleTime] ) { NSLog(@"Video Writing Error"); } + if (pixelBuffer != nullptr) + CVPixelBufferRelease(pixelBuffer); } } From 9682430a698f5c287885463f4300338f0e4d85d7 Mon Sep 17 00:00:00 2001 From: Nguyen Nguyen Date: Tue, 5 May 2015 13:14:26 -0400 Subject: [PATCH 3/6] Adding code to deal with audio output --- .../highgui/include/opencv2/highgui/cap_ios.h | 3 +- .../highgui/src/cap_ios_abstract_camera.mm | 8 ++ modules/highgui/src/cap_ios_video_camera.mm | 76 ++++++++++++++++++- 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/cap_ios.h b/modules/highgui/include/opencv2/highgui/cap_ios.h index 82230550f..bf20174db 100644 --- a/modules/highgui/include/opencv2/highgui/cap_ios.h +++ b/modules/highgui/include/opencv2/highgui/cap_ios.h @@ -68,6 +68,7 @@ @property (nonatomic, readonly) BOOL captureSessionLoaded; @property (nonatomic, assign) int defaultFPS; +@property (nonatomic, readonly) AVCaptureVideoPreviewLayer *captureVideoPreviewLayer; @property (nonatomic, assign) AVCaptureDevicePosition defaultAVCaptureDevicePosition; @property (nonatomic, assign) AVCaptureVideoOrientation defaultAVCaptureVideoOrientation; @property (nonatomic, assign) BOOL useAVCaptureVideoPreviewLayer; @@ -110,7 +111,7 @@ @end -@interface CvVideoCamera : CvAbstractCamera +@interface CvVideoCamera : CvAbstractCamera { AVCaptureVideoDataOutput *videoDataOutput; diff --git a/modules/highgui/src/cap_ios_abstract_camera.mm b/modules/highgui/src/cap_ios_abstract_camera.mm index bc44c83ff..6073a31bf 100644 --- a/modules/highgui/src/cap_ios_abstract_camera.mm +++ b/modules/highgui/src/cap_ios_abstract_camera.mm @@ -272,6 +272,14 @@ } else { NSLog(@"[Camera] Error: could not set session preset"); } + + AVCaptureDevice *audioCaptureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + NSError *error = nil; + AVCaptureDeviceInput *audioInput = [AVCaptureDeviceInput deviceInputWithDevice:audioCaptureDevice error:&error]; + if (audioInput) { + NSLog(@"Adding audio capture devices "); + [self.captureSession addInput:audioInput]; + } } - (void)createCaptureDevice; diff --git a/modules/highgui/src/cap_ios_video_camera.mm b/modules/highgui/src/cap_ios_video_camera.mm index 42fd75555..50c515b19 100644 --- a/modules/highgui/src/cap_ios_video_camera.mm +++ b/modules/highgui/src/cap_ios_video_camera.mm @@ -41,14 +41,19 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; -@interface CvVideoCamera () +@interface CvVideoCamera () { + NSString* mediaPath; +} + - (void)createVideoDataOutput; - (void)createVideoFileOutput; - +- (void)createMovieFileOutput; +- (NSString*) mediaFileString; @property (nonatomic, retain) CALayer *customPreviewLayer; @property (nonatomic, retain) AVCaptureVideoDataOutput *videoDataOutput; +@property (nonatomic, retain) AVCaptureMovieFileOutput *movieFileOutput; @end @@ -68,6 +73,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; @synthesize customPreviewLayer; @synthesize videoDataOutput; +@synthesize movieFileOutput; @synthesize recordVideo; @synthesize rotateVideo; @@ -78,6 +84,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; + #pragma mark - Constructors - (id)initWithParentView:(UIView*)parent; @@ -105,9 +112,18 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; if ([[NSFileManager defaultManager] fileExistsAtPath:[self videoFileString]]) { [[NSFileManager defaultManager] removeItemAtPath:[self videoFileString] error:&error]; } + if (error == nil) { NSLog(@"[Camera] Delete file %@", [self videoFileString]); } + +#if 0 + if ([[NSFileManager defaultManager] fileExistsAtPath:[self mediaFileString]]) { + [[NSFileManager defaultManager] removeItemAtPath:[self mediaFileString] error:&error]; + } + + [self.movieFileOutput startRecordingToOutputFileURL:[self mediaFileURL] recordingDelegate:self]; +#endif } } @@ -333,6 +349,25 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } +- (void)createMovieFileOutput; +{ + self.movieFileOutput = [[AVCaptureMovieFileOutput alloc] init]; + CMTime maxDuration = CMTimeMake(30*60, 1); + movieFileOutput.maxRecordedDuration = maxDuration; + movieFileOutput.minFreeDiskSpaceLimit = (1024L)*(1024L*1024L); + movieFileOutput.maxRecordedFileSize = (400L)*(1024L*1024L); + + NSLog(@"createVideoFileOutput..."); + + if ([self.captureSession canAddOutput:movieFileOutput]) { + [captureSession addOutput:movieFileOutput]; + NSLog(@"Successfully added movie output "); + } + else { + NSLog(@"Couldn't add movie output "); + } +} + - (void)createVideoFileOutput; { @@ -378,6 +413,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; [self createVideoDataOutput]; if (self.recordVideo == YES) { [self createVideoFileOutput]; + //[self createMovieFileOutput]; } } @@ -419,6 +455,24 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; return pxbuffer; } + +- (void)captureOutput:(AVCaptureFileOutput *)captureOutput + didFinishRecordingToOutputFileAtURL:(NSURL *)outputFileURL + fromConnections:(NSArray *)connections + error:(NSError *)error { + +#if 0 + BOOL recordedSuccessfully = YES; + if ([error code] != noErr) { + // A problem occurred: Find out if the recording was successful. + id value = [[error userInfo] objectForKey:AVErrorRecordingSuccessfullyFinishedKey]; + if (value) { + recordedSuccessfully = [value boolValue]; + } + } +#endif + NSLog(@"Capture File output done "); +} #pragma mark - Protocol AVCaptureVideoDataOutputSampleBufferDelegate @@ -601,7 +655,8 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; - (NSURL *)videoFileURL; { - NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"output.mov"]; + //NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"output.mov"]; + NSString *outputPath = self.videoFileString; NSURL *outputURL = [NSURL fileURLWithPath:outputPath]; NSFileManager *fileManager = [NSFileManager defaultManager]; if ([fileManager fileExistsAtPath:outputPath]) { @@ -611,6 +666,16 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } +- (NSURL *)mediaFileURL; +{ + NSString *outputPath = self.mediaFileString; + NSURL *outputURL = [NSURL fileURLWithPath:outputPath]; + NSFileManager *fileManager = [NSFileManager defaultManager]; + if ([fileManager fileExistsAtPath:outputPath]) { + NSLog(@"file exists"); + } + return outputURL; +} - (NSString *)videoFileString; { @@ -618,4 +683,9 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; return outputPath; } + +- (NSString*) mediaFileString { + NSString *outputPath = [[NSString alloc] initWithFormat:@"%@-%@", NSTemporaryDirectory(), @"media.mov"]; + return outputPath; +} @end From a5814871d9996a938a51216842fca76cae9bf9b8 Mon Sep 17 00:00:00 2001 From: Nguyen Nguyen Date: Wed, 6 May 2015 17:01:38 -0400 Subject: [PATCH 4/6] Adding audio to opencv --- .../highgui/include/opencv2/highgui/cap_ios.h | 2 + .../highgui/src/cap_ios_abstract_camera.mm | 155 +++++++++++++++++- modules/highgui/src/cap_ios_video_camera.mm | 22 ++- 3 files changed, 165 insertions(+), 14 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/cap_ios.h b/modules/highgui/include/opencv2/highgui/cap_ios.h index bf20174db..e031764db 100644 --- a/modules/highgui/include/opencv2/highgui/cap_ios.h +++ b/modules/highgui/include/opencv2/highgui/cap_ios.h @@ -40,6 +40,7 @@ { AVCaptureSession* captureSession; AVCaptureConnection* videoCaptureConnection; + AVCaptureConnection* audioCaptureConnection; AVCaptureVideoPreviewLayer *captureVideoPreviewLayer; UIDeviceOrientation currentDeviceOrientation; @@ -63,6 +64,7 @@ @property (nonatomic, retain) AVCaptureSession* captureSession; @property (nonatomic, retain) AVCaptureConnection* videoCaptureConnection; +@property (nonatomic, retain) AVCaptureConnection* audioCaptureConnection; @property (nonatomic, readonly) BOOL running; @property (nonatomic, readonly) BOOL captureSessionLoaded; diff --git a/modules/highgui/src/cap_ios_abstract_camera.mm b/modules/highgui/src/cap_ios_abstract_camera.mm index 6073a31bf..f4f706afe 100644 --- a/modules/highgui/src/cap_ios_abstract_camera.mm +++ b/modules/highgui/src/cap_ios_abstract_camera.mm @@ -35,7 +35,9 @@ #pragma mark - Private Interface -@interface CvAbstractCamera () +@interface CvAbstractCamera () { + AVCaptureAudioDataOutput *audioOut; +} @property (nonatomic, retain) AVCaptureVideoPreviewLayer* captureVideoPreviewLayer; @@ -72,6 +74,7 @@ @synthesize captureSession; @synthesize captureVideoPreviewLayer; @synthesize videoCaptureConnection; +@synthesize audioCaptureConnection; @synthesize running; @synthesize captureSessionLoaded; @synthesize useAVCaptureVideoPreviewLayer; @@ -206,6 +209,7 @@ self.captureSession = nil; self.captureVideoPreviewLayer = nil; self.videoCaptureConnection = nil; + self.audioCaptureConnection = nil; captureSessionLoaded = NO; } @@ -273,15 +277,122 @@ NSLog(@"[Camera] Error: could not set session preset"); } - AVCaptureDevice *audioCaptureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; - NSError *error = nil; - AVCaptureDeviceInput *audioInput = [AVCaptureDeviceInput deviceInputWithDevice:audioCaptureDevice error:&error]; - if (audioInput) { - NSLog(@"Adding audio capture devices "); - [self.captureSession addInput:audioInput]; - } } +#if 0 +- (void)sampleCaptureSessionSetup { + + if ( _captureSession ) { + return; + } + + _captureSession = [[AVCaptureSession alloc] init]; + + [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(captureSessionNotification:) name:nil object:_captureSession]; + _applicationWillEnterForegroundNotificationObserver = [[NSNotificationCenter defaultCenter] addObserverForName:UIApplicationWillEnterForegroundNotification object:[UIApplication sharedApplication] queue:nil usingBlock:^(NSNotification *note) { + // Retain self while the capture session is alive by referencing it in this observer block which is tied to the session lifetime + // Client must stop us running before we can be deallocated + [self applicationWillEnterForeground]; + }]; + +#if RECORD_AUDIO + /* Audio */ + AVCaptureDevice *audioDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + AVCaptureDeviceInput *audioIn = [[AVCaptureDeviceInput alloc] initWithDevice:audioDevice error:nil]; + if ( [_captureSession canAddInput:audioIn] ) { + [_captureSession addInput:audioIn]; + } + [audioIn release]; + + AVCaptureAudioDataOutput *audioOut = [[AVCaptureAudioDataOutput alloc] init]; + // Put audio on its own queue to ensure that our video processing doesn't cause us to drop audio + dispatch_queue_t audioCaptureQueue = dispatch_queue_create( "com.apple.sample.capturepipeline.audio", DISPATCH_QUEUE_SERIAL ); + [audioOut setSampleBufferDelegate:self queue:audioCaptureQueue]; + [audioCaptureQueue release]; + + if ( [_captureSession canAddOutput:audioOut] ) { + [_captureSession addOutput:audioOut]; + } + self.audioConnection = [audioOut connectionWithMediaType:AVMediaTypeAudio]; + [audioOut release]; +#endif // RECORD_AUDIO + + /* Video */ + AVCaptureDevice *videoDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; + _videoDevice = videoDevice; + AVCaptureDeviceInput *videoIn = [[AVCaptureDeviceInput alloc] initWithDevice:videoDevice error:nil]; + if ( [_captureSession canAddInput:videoIn] ) { + [_captureSession addInput:videoIn]; + } + [videoIn release]; + + AVCaptureVideoDataOutput *videoOut = [[AVCaptureVideoDataOutput alloc] init]; + videoOut.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey : @(_renderer.inputPixelFormat) }; + [videoOut setSampleBufferDelegate:self queue:_videoDataOutputQueue]; + + // RosyWriter records videos and we prefer not to have any dropped frames in the video recording. + // By setting alwaysDiscardsLateVideoFrames to NO we ensure that minor fluctuations in system load or in our processing time for a given frame won't cause framedrops. + // We do however need to ensure that on average we can process frames in realtime. + // If we were doing preview only we would probably want to set alwaysDiscardsLateVideoFrames to YES. + videoOut.alwaysDiscardsLateVideoFrames = NO; + + if ( [_captureSession canAddOutput:videoOut] ) { + [_captureSession addOutput:videoOut]; + } + + _videoConnection = [videoOut connectionWithMediaType:AVMediaTypeVideo]; + + int frameRate; + NSString *sessionPreset = AVCaptureSessionPresetHigh; + CMTime frameDuration = kCMTimeInvalid; + // For single core systems like iPhone 4 and iPod Touch 4th Generation we use a lower resolution and framerate to maintain real-time performance. + if ( [NSProcessInfo processInfo].processorCount == 1 ) + { + if ( [_captureSession canSetSessionPreset:AVCaptureSessionPreset640x480] ) { + sessionPreset = AVCaptureSessionPreset640x480; + } + frameRate = 15; + } + else + { +#if ! USE_OPENGL_RENDERER + // When using the CPU renderers or the CoreImage renderer we lower the resolution to 720p so that all devices can maintain real-time performance (this is primarily for A5 based devices like iPhone 4s and iPod Touch 5th Generation). + if ( [_captureSession canSetSessionPreset:AVCaptureSessionPreset1280x720] ) { + sessionPreset = AVCaptureSessionPreset1280x720; + } +#endif // ! USE_OPENGL_RENDERER + + frameRate = 30; + } + + _captureSession.sessionPreset = sessionPreset; + + frameDuration = CMTimeMake( 1, frameRate ); + + NSError *error = nil; + if ( [videoDevice lockForConfiguration:&error] ) { + videoDevice.activeVideoMaxFrameDuration = frameDuration; + videoDevice.activeVideoMinFrameDuration = frameDuration; + [videoDevice unlockForConfiguration]; + } + else { + NSLog( @"videoDevice lockForConfiguration returned error %@", error ); + } + + // Get the recommended compression settings after configuring the session/device. +#if RECORD_AUDIO + _audioCompressionSettings = [[audioOut recommendedAudioSettingsForAssetWriterWithOutputFileType:AVFileTypeQuickTimeMovie] copy]; +#endif + _videoCompressionSettings = [[videoOut recommendedVideoSettingsForAssetWriterWithOutputFileType:AVFileTypeQuickTimeMovie] copy]; + + self.videoOrientation = _videoConnection.videoOrientation; + + [videoOut release]; + + return; +} +#endif + - (void)createCaptureDevice; { // setup the device @@ -289,6 +400,34 @@ [self setDesiredCameraPosition:self.defaultAVCaptureDevicePosition]; NSLog(@"[Camera] device connected? %@", device.connected ? @"YES" : @"NO"); NSLog(@"[Camera] device position %@", (device.position == AVCaptureDevicePositionBack) ? @"back" : @"front"); + + AVCaptureDevice *audioCaptureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + NSError *error = nil; + //AVCaptureDeviceInput *audioInput = [AVCaptureDeviceInput deviceInputWithDevice:audioCaptureDevice error:&error]; + AVCaptureDeviceInput *audioInput = [[AVCaptureDeviceInput alloc] initWithDevice:audioCaptureDevice error:nil]; + if (audioInput) { + NSLog(@"Adding audio capture devices "); + [self.captureSession addInput:audioInput]; + [audioInput release]; + } + + + // Put audio on its own queue to ensure that our video processing doesn't cause us to drop audio + audioOut = [[AVCaptureAudioDataOutput alloc] init]; + dispatch_queue_t audioCaptureQueue = dispatch_queue_create("opencv.ios.audio", DISPATCH_QUEUE_SERIAL ); + [audioOut setSampleBufferDelegate:self queue:audioCaptureQueue]; + + if ( [self.captureSession canAddOutput:audioOut] ) { + [self.captureSession addOutput:audioOut]; + NSLog(@"audioOut added "); + } + + [audioCaptureQueue release]; + + self.audioCaptureConnection = [audioOut connectionWithMediaType:AVMediaTypeAudio]; + + NSLog(@"Audio has been setup with callback "); + } diff --git a/modules/highgui/src/cap_ios_video_camera.mm b/modules/highgui/src/cap_ios_video_camera.mm index 50c515b19..343ffde51 100644 --- a/modules/highgui/src/cap_ios_video_camera.mm +++ b/modules/highgui/src/cap_ios_video_camera.mm @@ -117,13 +117,9 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; NSLog(@"[Camera] Delete file %@", [self videoFileString]); } -#if 0 if ([[NSFileManager defaultManager] fileExistsAtPath:[self mediaFileString]]) { [[NSFileManager defaultManager] removeItemAtPath:[self mediaFileString] error:&error]; } - - [self.movieFileOutput startRecordingToOutputFileURL:[self mediaFileURL] recordingDelegate:self]; -#endif } } @@ -309,6 +305,9 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } [[self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo] setEnabled:YES]; + //self.videoCaptureConnection = [self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo]; + //[self.videoCaptureConnection setEnabled:YES]; + // set default FPS if ([self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo].supportsVideoMinFrameDuration) { @@ -351,13 +350,13 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; - (void)createMovieFileOutput; { + NSLog(@"createVideoFileOutput..."); self.movieFileOutput = [[AVCaptureMovieFileOutput alloc] init]; CMTime maxDuration = CMTimeMake(30*60, 1); movieFileOutput.maxRecordedDuration = maxDuration; movieFileOutput.minFreeDiskSpaceLimit = (1024L)*(1024L*1024L); movieFileOutput.maxRecordedFileSize = (400L)*(1024L*1024L); - NSLog(@"createVideoFileOutput..."); if ([self.captureSession canAddOutput:movieFileOutput]) { [captureSession addOutput:movieFileOutput]; @@ -366,6 +365,9 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; else { NSLog(@"Couldn't add movie output "); } + + if (self.recordVideo == YES) + [self.movieFileOutput startRecordingToOutputFileURL:[self mediaFileURL] recordingDelegate:self]; } @@ -480,6 +482,13 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; { (void)captureOutput; (void)connection; + + if (connection == self.audioCaptureConnection) { + NSLog(@"Audio Sample came in "); + return; + } + + //NSLog(@"Video sample came in "); if (self.delegate) { // convert from Core Media to Core Video @@ -674,6 +683,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; if ([fileManager fileExistsAtPath:outputPath]) { NSLog(@"file exists"); } + NSLog(@"media URL %@", outputURL); return outputURL; } @@ -685,7 +695,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; - (NSString*) mediaFileString { - NSString *outputPath = [[NSString alloc] initWithFormat:@"%@-%@", NSTemporaryDirectory(), @"media.mov"]; + NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"media.mov"]; return outputPath; } @end From 53ff537f1c12c4fad0ccb6ef425c0902956b5daa Mon Sep 17 00:00:00 2001 From: Nguyen Nguyen Date: Fri, 8 May 2015 11:33:11 -0400 Subject: [PATCH 5/6] temporary fixes --- .../highgui/include/opencv2/highgui/cap_ios.h | 4 +- .../highgui/src/cap_ios_abstract_camera.mm | 2 + modules/highgui/src/cap_ios_video_camera.mm | 59 +++++++++++++++---- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/cap_ios.h b/modules/highgui/include/opencv2/highgui/cap_ios.h index e031764db..aeb6c42be 100644 --- a/modules/highgui/include/opencv2/highgui/cap_ios.h +++ b/modules/highgui/include/opencv2/highgui/cap_ios.h @@ -109,6 +109,7 @@ #ifdef __cplusplus // delegate method for processing image frames - (void)processImage:(cv::Mat&)image; + #endif @end @@ -128,8 +129,6 @@ AVAssetWriterInputPixelBufferAdaptor* recordPixelBufferAdaptor; AVAssetWriter* recordAssetWriter; - CMTime lastSampleTime; - } @property (nonatomic, assign) id delegate; @@ -140,6 +139,7 @@ @property (nonatomic, retain) AVAssetWriterInput* recordAssetWriterInput; @property (nonatomic, retain) AVAssetWriterInputPixelBufferAdaptor* recordPixelBufferAdaptor; @property (nonatomic, retain) AVAssetWriter* recordAssetWriter; +@property (nonatomic, readonly) int64_t timestampMs; - (void)adjustLayoutToInterfaceOrientation:(UIInterfaceOrientation)interfaceOrientation; - (void)layoutPreviewLayer; diff --git a/modules/highgui/src/cap_ios_abstract_camera.mm b/modules/highgui/src/cap_ios_abstract_camera.mm index f4f706afe..61ea00ada 100644 --- a/modules/highgui/src/cap_ios_abstract_camera.mm +++ b/modules/highgui/src/cap_ios_abstract_camera.mm @@ -401,6 +401,7 @@ NSLog(@"[Camera] device connected? %@", device.connected ? @"YES" : @"NO"); NSLog(@"[Camera] device position %@", (device.position == AVCaptureDevicePositionBack) ? @"back" : @"front"); +#if 0 AVCaptureDevice *audioCaptureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; NSError *error = nil; //AVCaptureDeviceInput *audioInput = [AVCaptureDeviceInput deviceInputWithDevice:audioCaptureDevice error:&error]; @@ -427,6 +428,7 @@ self.audioCaptureConnection = [audioOut connectionWithMediaType:AVMediaTypeAudio]; NSLog(@"Audio has been setup with callback "); +#endif } diff --git a/modules/highgui/src/cap_ios_video_camera.mm b/modules/highgui/src/cap_ios_video_camera.mm index 343ffde51..fc3cfd4f4 100644 --- a/modules/highgui/src/cap_ios_video_camera.mm +++ b/modules/highgui/src/cap_ios_video_camera.mm @@ -43,6 +43,11 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; @interface CvVideoCamera () { NSString* mediaPath; + int recordCountDown; + CMTime _lastSampleTime; + int64_t _timestampMs; + dispatch_queue_t movieWriterQueue; + } @@ -54,6 +59,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; @property (nonatomic, retain) CALayer *customPreviewLayer; @property (nonatomic, retain) AVCaptureVideoDataOutput *videoDataOutput; @property (nonatomic, retain) AVCaptureMovieFileOutput *movieFileOutput; +@property (nonatomic, retain) dispatch_queue_t movieWriterQueue; @end @@ -82,6 +88,9 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; @synthesize recordPixelBufferAdaptor; @synthesize recordAssetWriter; +@synthesize timestampMs = _timestampMs; + + @@ -89,12 +98,14 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; - (id)initWithParentView:(UIView*)parent; { + recordCountDown = 1000000000; self = [super initWithParentView:parent]; if (self) { self.useAVCaptureVideoPreviewLayer = NO; self.recordVideo = NO; self.rotateVideo = NO; } + movieWriterQueue = nil; return self; } @@ -105,6 +116,8 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; - (void)start; { + recordCountDown = 5; + movieWriterQueue = nil; [super start]; if (self.recordVideo == YES) { @@ -116,11 +129,8 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; if (error == nil) { NSLog(@"[Camera] Delete file %@", [self videoFileString]); } - - if ([[NSFileManager defaultManager] fileExistsAtPath:[self mediaFileString]]) { - [[NSFileManager defaultManager] removeItemAtPath:[self mediaFileString] error:&error]; - } } + } @@ -146,6 +156,9 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; self.recordAssetWriter = nil; self.recordAssetWriterInput = nil; self.recordPixelBufferAdaptor = nil; + if (movieWriterQueue) + dispatch_release(movieWriterQueue); + self.movieWriterQueue = nil; } [self.customPreviewLayer removeFromSuperlayer]; @@ -344,6 +357,9 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; [self.videoDataOutput setSampleBufferDelegate:self queue:videoDataOutputQueue]; + if (self.recordVideo == YES && movieWriterQueue == nil) { + movieWriterQueue = dispatch_queue_create("opencv_movieWriter", DISPATCH_QUEUE_SERIAL); + } NSLog(@"[Camera] created AVCaptureVideoDataOutput at %d FPS", self.defaultFPS); } @@ -477,6 +493,15 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } #pragma mark - Protocol AVCaptureVideoDataOutputSampleBufferDelegate +- (void)captureOutput:(AVCaptureOutput *)captureOutput didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection +{ + if (connection == self.audioCaptureConnection) { + NSLog(@"Audio sample did drop "); + return; + } + NSLog(@"Video Frame did drop "); +} + - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { @@ -484,7 +509,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; (void)connection; if (connection == self.audioCaptureConnection) { - NSLog(@"Audio Sample came in "); + //NSLog(@"Audio Sample came in "); return; } @@ -526,15 +551,22 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } + + CMTime lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + int64_t msec = lastSampleTime.value / (lastSampleTime.timescale / 1000); + _timestampMs = msec; + //NSLog(@"Timestamp %u / %u, msec = %lu ", lastSampleTime.value, lastSampleTime.timescale, msec); + + // delegate image processing to the delegate cv::Mat image((int)height, (int)width, format_opencv, bufferAddress, bytesPerRow); - CGImage* dstImage; - if ([self.delegate respondsToSelector:@selector(processImage:)]) { [self.delegate processImage:image]; } + CGImage* dstImage; + // check if matrix data pointer or dimensions were changed by the delegate bool iOSimage = false; if (height == (size_t)image.rows && width == (size_t)image.cols && format_opencv == image.type() && bufferAddress == image.data && bytesPerRow == image.step) { @@ -595,17 +627,20 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; // render buffer + //dispatch_sync(dispatch_get_main_queue(), ^{ dispatch_sync(dispatch_get_main_queue(), ^{ self.customPreviewLayer.contents = (__bridge id)dstImage; }); + if (recordCountDown > 0) + recordCountDown--; + + if (self.recordVideo == YES && recordCountDown <= 0) { + //CMTimeShow(lastSampleTime); - if (self.recordVideo == YES) { - lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); -// CMTimeShow(lastSampleTime); if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) { [self.recordAssetWriter startWriting]; - [self.recordAssetWriter startSessionAtSourceTime:lastSampleTime]; + [self.recordAssetWriter startSessionAtSourceTime:_lastSampleTime]; if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) { NSLog(@"[Camera] Recording Error: asset writer status is not writing: %@", self.recordAssetWriter.error); return; @@ -623,10 +658,8 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; if (pixelBuffer != nullptr) CVPixelBufferRelease(pixelBuffer); } - } - // cleanup CGImageRelease(dstImage); From 58dd70e8dcfc618a5d6c706604909e53d8e21d32 Mon Sep 17 00:00:00 2001 From: Nguyen Nguyen Date: Fri, 8 May 2015 12:06:03 -0400 Subject: [PATCH 6/6] Revert support for audio --- .../highgui/include/opencv2/highgui/cap_ios.h | 8 +- .../highgui/src/cap_ios_abstract_camera.mm | 151 +----------------- modules/highgui/src/cap_ios_video_camera.mm | 135 ++-------------- 3 files changed, 17 insertions(+), 277 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/cap_ios.h b/modules/highgui/include/opencv2/highgui/cap_ios.h index aeb6c42be..4c931d400 100644 --- a/modules/highgui/include/opencv2/highgui/cap_ios.h +++ b/modules/highgui/include/opencv2/highgui/cap_ios.h @@ -40,7 +40,6 @@ { AVCaptureSession* captureSession; AVCaptureConnection* videoCaptureConnection; - AVCaptureConnection* audioCaptureConnection; AVCaptureVideoPreviewLayer *captureVideoPreviewLayer; UIDeviceOrientation currentDeviceOrientation; @@ -64,7 +63,6 @@ @property (nonatomic, retain) AVCaptureSession* captureSession; @property (nonatomic, retain) AVCaptureConnection* videoCaptureConnection; -@property (nonatomic, retain) AVCaptureConnection* audioCaptureConnection; @property (nonatomic, readonly) BOOL running; @property (nonatomic, readonly) BOOL captureSessionLoaded; @@ -109,12 +107,11 @@ #ifdef __cplusplus // delegate method for processing image frames - (void)processImage:(cv::Mat&)image; - #endif @end -@interface CvVideoCamera : CvAbstractCamera +@interface CvVideoCamera : CvAbstractCamera { AVCaptureVideoDataOutput *videoDataOutput; @@ -129,6 +126,8 @@ AVAssetWriterInputPixelBufferAdaptor* recordPixelBufferAdaptor; AVAssetWriter* recordAssetWriter; + CMTime lastSampleTime; + } @property (nonatomic, assign) id delegate; @@ -139,7 +138,6 @@ @property (nonatomic, retain) AVAssetWriterInput* recordAssetWriterInput; @property (nonatomic, retain) AVAssetWriterInputPixelBufferAdaptor* recordPixelBufferAdaptor; @property (nonatomic, retain) AVAssetWriter* recordAssetWriter; -@property (nonatomic, readonly) int64_t timestampMs; - (void)adjustLayoutToInterfaceOrientation:(UIInterfaceOrientation)interfaceOrientation; - (void)layoutPreviewLayer; diff --git a/modules/highgui/src/cap_ios_abstract_camera.mm b/modules/highgui/src/cap_ios_abstract_camera.mm index 61ea00ada..bc44c83ff 100644 --- a/modules/highgui/src/cap_ios_abstract_camera.mm +++ b/modules/highgui/src/cap_ios_abstract_camera.mm @@ -35,9 +35,7 @@ #pragma mark - Private Interface -@interface CvAbstractCamera () { - AVCaptureAudioDataOutput *audioOut; -} +@interface CvAbstractCamera () @property (nonatomic, retain) AVCaptureVideoPreviewLayer* captureVideoPreviewLayer; @@ -74,7 +72,6 @@ @synthesize captureSession; @synthesize captureVideoPreviewLayer; @synthesize videoCaptureConnection; -@synthesize audioCaptureConnection; @synthesize running; @synthesize captureSessionLoaded; @synthesize useAVCaptureVideoPreviewLayer; @@ -209,7 +206,6 @@ self.captureSession = nil; self.captureVideoPreviewLayer = nil; self.videoCaptureConnection = nil; - self.audioCaptureConnection = nil; captureSessionLoaded = NO; } @@ -276,123 +272,8 @@ } else { NSLog(@"[Camera] Error: could not set session preset"); } - } -#if 0 -- (void)sampleCaptureSessionSetup { - - if ( _captureSession ) { - return; - } - - _captureSession = [[AVCaptureSession alloc] init]; - - [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(captureSessionNotification:) name:nil object:_captureSession]; - _applicationWillEnterForegroundNotificationObserver = [[NSNotificationCenter defaultCenter] addObserverForName:UIApplicationWillEnterForegroundNotification object:[UIApplication sharedApplication] queue:nil usingBlock:^(NSNotification *note) { - // Retain self while the capture session is alive by referencing it in this observer block which is tied to the session lifetime - // Client must stop us running before we can be deallocated - [self applicationWillEnterForeground]; - }]; - -#if RECORD_AUDIO - /* Audio */ - AVCaptureDevice *audioDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; - AVCaptureDeviceInput *audioIn = [[AVCaptureDeviceInput alloc] initWithDevice:audioDevice error:nil]; - if ( [_captureSession canAddInput:audioIn] ) { - [_captureSession addInput:audioIn]; - } - [audioIn release]; - - AVCaptureAudioDataOutput *audioOut = [[AVCaptureAudioDataOutput alloc] init]; - // Put audio on its own queue to ensure that our video processing doesn't cause us to drop audio - dispatch_queue_t audioCaptureQueue = dispatch_queue_create( "com.apple.sample.capturepipeline.audio", DISPATCH_QUEUE_SERIAL ); - [audioOut setSampleBufferDelegate:self queue:audioCaptureQueue]; - [audioCaptureQueue release]; - - if ( [_captureSession canAddOutput:audioOut] ) { - [_captureSession addOutput:audioOut]; - } - self.audioConnection = [audioOut connectionWithMediaType:AVMediaTypeAudio]; - [audioOut release]; -#endif // RECORD_AUDIO - - /* Video */ - AVCaptureDevice *videoDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; - _videoDevice = videoDevice; - AVCaptureDeviceInput *videoIn = [[AVCaptureDeviceInput alloc] initWithDevice:videoDevice error:nil]; - if ( [_captureSession canAddInput:videoIn] ) { - [_captureSession addInput:videoIn]; - } - [videoIn release]; - - AVCaptureVideoDataOutput *videoOut = [[AVCaptureVideoDataOutput alloc] init]; - videoOut.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey : @(_renderer.inputPixelFormat) }; - [videoOut setSampleBufferDelegate:self queue:_videoDataOutputQueue]; - - // RosyWriter records videos and we prefer not to have any dropped frames in the video recording. - // By setting alwaysDiscardsLateVideoFrames to NO we ensure that minor fluctuations in system load or in our processing time for a given frame won't cause framedrops. - // We do however need to ensure that on average we can process frames in realtime. - // If we were doing preview only we would probably want to set alwaysDiscardsLateVideoFrames to YES. - videoOut.alwaysDiscardsLateVideoFrames = NO; - - if ( [_captureSession canAddOutput:videoOut] ) { - [_captureSession addOutput:videoOut]; - } - - _videoConnection = [videoOut connectionWithMediaType:AVMediaTypeVideo]; - - int frameRate; - NSString *sessionPreset = AVCaptureSessionPresetHigh; - CMTime frameDuration = kCMTimeInvalid; - // For single core systems like iPhone 4 and iPod Touch 4th Generation we use a lower resolution and framerate to maintain real-time performance. - if ( [NSProcessInfo processInfo].processorCount == 1 ) - { - if ( [_captureSession canSetSessionPreset:AVCaptureSessionPreset640x480] ) { - sessionPreset = AVCaptureSessionPreset640x480; - } - frameRate = 15; - } - else - { -#if ! USE_OPENGL_RENDERER - // When using the CPU renderers or the CoreImage renderer we lower the resolution to 720p so that all devices can maintain real-time performance (this is primarily for A5 based devices like iPhone 4s and iPod Touch 5th Generation). - if ( [_captureSession canSetSessionPreset:AVCaptureSessionPreset1280x720] ) { - sessionPreset = AVCaptureSessionPreset1280x720; - } -#endif // ! USE_OPENGL_RENDERER - - frameRate = 30; - } - - _captureSession.sessionPreset = sessionPreset; - - frameDuration = CMTimeMake( 1, frameRate ); - - NSError *error = nil; - if ( [videoDevice lockForConfiguration:&error] ) { - videoDevice.activeVideoMaxFrameDuration = frameDuration; - videoDevice.activeVideoMinFrameDuration = frameDuration; - [videoDevice unlockForConfiguration]; - } - else { - NSLog( @"videoDevice lockForConfiguration returned error %@", error ); - } - - // Get the recommended compression settings after configuring the session/device. -#if RECORD_AUDIO - _audioCompressionSettings = [[audioOut recommendedAudioSettingsForAssetWriterWithOutputFileType:AVFileTypeQuickTimeMovie] copy]; -#endif - _videoCompressionSettings = [[videoOut recommendedVideoSettingsForAssetWriterWithOutputFileType:AVFileTypeQuickTimeMovie] copy]; - - self.videoOrientation = _videoConnection.videoOrientation; - - [videoOut release]; - - return; -} -#endif - - (void)createCaptureDevice; { // setup the device @@ -400,36 +281,6 @@ [self setDesiredCameraPosition:self.defaultAVCaptureDevicePosition]; NSLog(@"[Camera] device connected? %@", device.connected ? @"YES" : @"NO"); NSLog(@"[Camera] device position %@", (device.position == AVCaptureDevicePositionBack) ? @"back" : @"front"); - -#if 0 - AVCaptureDevice *audioCaptureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; - NSError *error = nil; - //AVCaptureDeviceInput *audioInput = [AVCaptureDeviceInput deviceInputWithDevice:audioCaptureDevice error:&error]; - AVCaptureDeviceInput *audioInput = [[AVCaptureDeviceInput alloc] initWithDevice:audioCaptureDevice error:nil]; - if (audioInput) { - NSLog(@"Adding audio capture devices "); - [self.captureSession addInput:audioInput]; - [audioInput release]; - } - - - // Put audio on its own queue to ensure that our video processing doesn't cause us to drop audio - audioOut = [[AVCaptureAudioDataOutput alloc] init]; - dispatch_queue_t audioCaptureQueue = dispatch_queue_create("opencv.ios.audio", DISPATCH_QUEUE_SERIAL ); - [audioOut setSampleBufferDelegate:self queue:audioCaptureQueue]; - - if ( [self.captureSession canAddOutput:audioOut] ) { - [self.captureSession addOutput:audioOut]; - NSLog(@"audioOut added "); - } - - [audioCaptureQueue release]; - - self.audioCaptureConnection = [audioOut connectionWithMediaType:AVMediaTypeAudio]; - - NSLog(@"Audio has been setup with callback "); -#endif - } diff --git a/modules/highgui/src/cap_ios_video_camera.mm b/modules/highgui/src/cap_ios_video_camera.mm index fc3cfd4f4..25b652b40 100644 --- a/modules/highgui/src/cap_ios_video_camera.mm +++ b/modules/highgui/src/cap_ios_video_camera.mm @@ -42,24 +42,15 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; @interface CvVideoCamera () { - NSString* mediaPath; - int recordCountDown; - CMTime _lastSampleTime; - int64_t _timestampMs; - dispatch_queue_t movieWriterQueue; - + int recordingCountDown; } - - (void)createVideoDataOutput; - (void)createVideoFileOutput; -- (void)createMovieFileOutput; -- (NSString*) mediaFileString; + @property (nonatomic, retain) CALayer *customPreviewLayer; @property (nonatomic, retain) AVCaptureVideoDataOutput *videoDataOutput; -@property (nonatomic, retain) AVCaptureMovieFileOutput *movieFileOutput; -@property (nonatomic, retain) dispatch_queue_t movieWriterQueue; @end @@ -79,7 +70,6 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; @synthesize customPreviewLayer; @synthesize videoDataOutput; -@synthesize movieFileOutput; @synthesize recordVideo; @synthesize rotateVideo; @@ -88,24 +78,18 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; @synthesize recordPixelBufferAdaptor; @synthesize recordAssetWriter; -@synthesize timestampMs = _timestampMs; - - - #pragma mark - Constructors - (id)initWithParentView:(UIView*)parent; { - recordCountDown = 1000000000; self = [super initWithParentView:parent]; if (self) { self.useAVCaptureVideoPreviewLayer = NO; self.recordVideo = NO; self.rotateVideo = NO; } - movieWriterQueue = nil; return self; } @@ -116,8 +100,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; - (void)start; { - recordCountDown = 5; - movieWriterQueue = nil; + recordingCountDown = 10; [super start]; if (self.recordVideo == YES) { @@ -125,12 +108,10 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; if ([[NSFileManager defaultManager] fileExistsAtPath:[self videoFileString]]) { [[NSFileManager defaultManager] removeItemAtPath:[self videoFileString] error:&error]; } - if (error == nil) { NSLog(@"[Camera] Delete file %@", [self videoFileString]); } } - } @@ -156,9 +137,6 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; self.recordAssetWriter = nil; self.recordAssetWriterInput = nil; self.recordPixelBufferAdaptor = nil; - if (movieWriterQueue) - dispatch_release(movieWriterQueue); - self.movieWriterQueue = nil; } [self.customPreviewLayer removeFromSuperlayer]; @@ -318,9 +296,6 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } [[self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo] setEnabled:YES]; - //self.videoCaptureConnection = [self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo]; - //[self.videoCaptureConnection setEnabled:YES]; - // set default FPS if ([self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo].supportsVideoMinFrameDuration) { @@ -357,35 +332,10 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; [self.videoDataOutput setSampleBufferDelegate:self queue:videoDataOutputQueue]; - if (self.recordVideo == YES && movieWriterQueue == nil) { - movieWriterQueue = dispatch_queue_create("opencv_movieWriter", DISPATCH_QUEUE_SERIAL); - } NSLog(@"[Camera] created AVCaptureVideoDataOutput at %d FPS", self.defaultFPS); } -- (void)createMovieFileOutput; -{ - NSLog(@"createVideoFileOutput..."); - self.movieFileOutput = [[AVCaptureMovieFileOutput alloc] init]; - CMTime maxDuration = CMTimeMake(30*60, 1); - movieFileOutput.maxRecordedDuration = maxDuration; - movieFileOutput.minFreeDiskSpaceLimit = (1024L)*(1024L*1024L); - movieFileOutput.maxRecordedFileSize = (400L)*(1024L*1024L); - - - if ([self.captureSession canAddOutput:movieFileOutput]) { - [captureSession addOutput:movieFileOutput]; - NSLog(@"Successfully added movie output "); - } - else { - NSLog(@"Couldn't add movie output "); - } - - if (self.recordVideo == YES) - [self.movieFileOutput startRecordingToOutputFileURL:[self mediaFileURL] recordingDelegate:self]; -} - - (void)createVideoFileOutput; { @@ -431,7 +381,6 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; [self createVideoDataOutput]; if (self.recordVideo == YES) { [self createVideoFileOutput]; - //[self createMovieFileOutput]; } } @@ -473,47 +422,13 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; return pxbuffer; } - -- (void)captureOutput:(AVCaptureFileOutput *)captureOutput - didFinishRecordingToOutputFileAtURL:(NSURL *)outputFileURL - fromConnections:(NSArray *)connections - error:(NSError *)error { - -#if 0 - BOOL recordedSuccessfully = YES; - if ([error code] != noErr) { - // A problem occurred: Find out if the recording was successful. - id value = [[error userInfo] objectForKey:AVErrorRecordingSuccessfullyFinishedKey]; - if (value) { - recordedSuccessfully = [value boolValue]; - } - } -#endif - NSLog(@"Capture File output done "); -} #pragma mark - Protocol AVCaptureVideoDataOutputSampleBufferDelegate -- (void)captureOutput:(AVCaptureOutput *)captureOutput didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection -{ - if (connection == self.audioCaptureConnection) { - NSLog(@"Audio sample did drop "); - return; - } - NSLog(@"Video Frame did drop "); -} - - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { (void)captureOutput; (void)connection; - - if (connection == self.audioCaptureConnection) { - //NSLog(@"Audio Sample came in "); - return; - } - - //NSLog(@"Video sample came in "); if (self.delegate) { // convert from Core Media to Core Video @@ -551,22 +466,15 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } - - CMTime lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); - int64_t msec = lastSampleTime.value / (lastSampleTime.timescale / 1000); - _timestampMs = msec; - //NSLog(@"Timestamp %u / %u, msec = %lu ", lastSampleTime.value, lastSampleTime.timescale, msec); - - // delegate image processing to the delegate cv::Mat image((int)height, (int)width, format_opencv, bufferAddress, bytesPerRow); + CGImage* dstImage; + if ([self.delegate respondsToSelector:@selector(processImage:)]) { [self.delegate processImage:image]; } - CGImage* dstImage; - // check if matrix data pointer or dimensions were changed by the delegate bool iOSimage = false; if (height == (size_t)image.rows && width == (size_t)image.cols && format_opencv == image.type() && bufferAddress == image.data && bytesPerRow == image.step) { @@ -627,20 +535,18 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; // render buffer - //dispatch_sync(dispatch_get_main_queue(), ^{ dispatch_sync(dispatch_get_main_queue(), ^{ self.customPreviewLayer.contents = (__bridge id)dstImage; }); - if (recordCountDown > 0) - recordCountDown--; - - if (self.recordVideo == YES && recordCountDown <= 0) { - //CMTimeShow(lastSampleTime); + recordingCountDown--; + if (self.recordVideo == YES && recordingCountDown < 0) { + lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); +// CMTimeShow(lastSampleTime); if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) { [self.recordAssetWriter startWriting]; - [self.recordAssetWriter startSessionAtSourceTime:_lastSampleTime]; + [self.recordAssetWriter startSessionAtSourceTime:lastSampleTime]; if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) { NSLog(@"[Camera] Recording Error: asset writer status is not writing: %@", self.recordAssetWriter.error); return; @@ -658,8 +564,10 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; if (pixelBuffer != nullptr) CVPixelBufferRelease(pixelBuffer); } + } + // cleanup CGImageRelease(dstImage); @@ -697,8 +605,7 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; - (NSURL *)videoFileURL; { - //NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"output.mov"]; - NSString *outputPath = self.videoFileString; + NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"output.mov"]; NSURL *outputURL = [NSURL fileURLWithPath:outputPath]; NSFileManager *fileManager = [NSFileManager defaultManager]; if ([fileManager fileExistsAtPath:outputPath]) { @@ -708,17 +615,6 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; } -- (NSURL *)mediaFileURL; -{ - NSString *outputPath = self.mediaFileString; - NSURL *outputURL = [NSURL fileURLWithPath:outputPath]; - NSFileManager *fileManager = [NSFileManager defaultManager]; - if ([fileManager fileExistsAtPath:outputPath]) { - NSLog(@"file exists"); - } - NSLog(@"media URL %@", outputURL); - return outputURL; -} - (NSString *)videoFileString; { @@ -726,9 +622,4 @@ static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}; return outputPath; } - -- (NSString*) mediaFileString { - NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"media.mov"]; - return outputPath; -} @end