From d08cb6b357f779e2099d00ea9d3b6530271ff45a Mon Sep 17 00:00:00 2001 From: Maxim Kostin Date: Fri, 15 May 2015 16:28:47 +0300 Subject: [PATCH] Added WinRT support for videoio. Signed-off-by: Maxim Kostin --- modules/videoio/include/opencv2/videoio.hpp | 6 +- .../include/opencv2/videoio/cap_winrt.hpp | 130 ++++++ modules/videoio/src/cap.cpp | 42 +- .../src/cap_winrt/CaptureFrameGrabber.cpp | 173 ++++++++ .../src/cap_winrt/CaptureFrameGrabber.hpp | 85 ++++ modules/videoio/src/cap_winrt/MFIncludes.hpp | 172 ++++++++ modules/videoio/src/cap_winrt/MediaSink.hpp | 396 ++++++++++++++++++ .../videoio/src/cap_winrt/MediaStreamSink.cpp | 386 +++++++++++++++++ .../videoio/src/cap_winrt/MediaStreamSink.hpp | 114 +++++ modules/videoio/src/cap_winrt_bridge.cpp | 89 ++++ modules/videoio/src/cap_winrt_bridge.hpp | 96 +++++ modules/videoio/src/cap_winrt_capture.cpp | 279 ++++++++++++ modules/videoio/src/cap_winrt_capture.hpp | 82 ++++ modules/videoio/src/cap_winrt_video.cpp | 322 ++++++++++++++ modules/videoio/src/cap_winrt_video.hpp | 73 ++++ 15 files changed, 2442 insertions(+), 3 deletions(-) create mode 100644 modules/videoio/include/opencv2/videoio/cap_winrt.hpp create mode 100644 modules/videoio/src/cap_winrt/CaptureFrameGrabber.cpp create mode 100644 modules/videoio/src/cap_winrt/CaptureFrameGrabber.hpp create mode 100644 modules/videoio/src/cap_winrt/MFIncludes.hpp create mode 100644 modules/videoio/src/cap_winrt/MediaSink.hpp create mode 100644 modules/videoio/src/cap_winrt/MediaStreamSink.cpp create mode 100644 modules/videoio/src/cap_winrt/MediaStreamSink.hpp create mode 100644 modules/videoio/src/cap_winrt_bridge.cpp create mode 100644 modules/videoio/src/cap_winrt_bridge.hpp create mode 100644 modules/videoio/src/cap_winrt_capture.cpp create mode 100644 modules/videoio/src/cap_winrt_capture.hpp create mode 100644 modules/videoio/src/cap_winrt_video.cpp create mode 100644 modules/videoio/src/cap_winrt_video.hpp diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index b0245b1df..a9a42dfb5 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -50,6 +50,7 @@ @{ @defgroup videoio_c C API @defgroup videoio_ios iOS glue + @defgroup videoio_winrt WinRT glue @} */ @@ -85,8 +86,9 @@ enum { CAP_ANY = 0, // autodetect CAP_AVFOUNDATION = 1200, // AVFoundation framework for iOS (OS X Lion will have the same API) CAP_GIGANETIX = 1300, // Smartek Giganetix GigEVisionSDK CAP_MSMF = 1400, // Microsoft Media Foundation (via videoInput) - CAP_INTELPERC = 1500, // Intel Perceptual Computing SDK - CAP_OPENNI2 = 1600, // OpenNI2 (for Kinect) + CAP_WINRT = 1410, // Microsoft Windows Runtime using Media Foundation + CAP_INTELPERC = 1500, // Intel Perceptual Computing SDK + CAP_OPENNI2 = 1600, // OpenNI2 (for Kinect) CAP_OPENNI2_ASUS = 1610 // OpenNI2 (for Asus Xtion and Occipital Structure sensors) }; diff --git a/modules/videoio/include/opencv2/videoio/cap_winrt.hpp b/modules/videoio/include/opencv2/videoio/cap_winrt.hpp new file mode 100644 index 000000000..39f98d3a8 --- /dev/null +++ b/modules/videoio/include/opencv2/videoio/cap_winrt.hpp @@ -0,0 +1,130 @@ +// Video support for Windows Runtime + +// Copyright (c) Microsoft Open Technologies, Inc. +// All rights reserved. +// +// (3 - clause BSD License) +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that +// the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the +// following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +// promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include + +using namespace Windows::UI::Xaml::Controls; + +namespace cv +{ + +//! @addtogroup videoio_winrt +//! @{ + +enum { + OPEN_CAMERA = 300, + CLOSE_CAMERA, + UPDATE_IMAGE_ELEMENT, + SHOW_TRACKBAR +}; + +/********************************** WinRT API ************************************************/ + +template +CV_EXPORTS void winrt_startMessageLoop(std::function&& callback, Args... args); + +template +CV_EXPORTS void winrt_startMessageLoop(void callback(Args...), Args... args); + +/** @brief +@note + Sets the reporter method for the HighguiAssist singleton. Starts the main OpenCV as + an async thread in WinRT. See VideoCapture for the example of callback implementation. + Here is how the class can be used: +@code + void cvMain() + { + Mat frame; + VideoCapture cam; + cam.open(0); + + while (1) + { + cam >> frame; + + // don't reprocess the same frame again + if (!cam.grab()) continue; + + // your processing logic goes here + + // obligatory step to get XAML image component updated + winrt_imshow(); + } + } + + MainPage::MainPage() + { + InitializeComponent(); + + cv::winrt_setFrameContainer(cvImage); + cv::winrt_startMessageLoop(cvMain); + } +@endcode +*/ +template +CV_EXPORTS void winrt_startMessageLoop(void callback(void)); + +/** @brief +@note + Must be called from WinRT specific callback to handle image grabber state. + Here is how the class can be used: +@code + MainPage::MainPage() + { + // ... + Window::Current->VisibilityChanged += ref new Windows::UI::Xaml::WindowVisibilityChangedEventHandler(this, &Application::MainPage::OnVisibilityChanged); + // ... + } + + void Application::MainPage::OnVisibilityChanged(Platform::Object ^sender, + Windows::UI::Core::VisibilityChangedEventArgs ^e) + { + cv::winrt_onVisibilityChanged(e->Visible); + } +@endcode +*/ +CV_EXPORTS void winrt_onVisibilityChanged(bool visible); + +/** @brief +@note + Must be called to assign WinRT control holding image you're working with. + Code sample is available for winrt_startMessageLoop(). +*/ +CV_EXPORTS void winrt_setFrameContainer(::Windows::UI::Xaml::Controls::Image^ image); + +/** @brief +@note + Must be called to update attached image source. + Code sample is available for winrt_startMessageLoop(). +*/ +CV_EXPORTS void winrt_imshow(); + +//! @} videoio_winrt + +} // cv \ No newline at end of file diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index 1fedb0b66..67eb28395 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -43,6 +43,13 @@ #include "cap_intelperc.hpp" #include "cap_dshow.hpp" +// All WinRT versions older than 8.0 should provide classes used for video support +#if defined(WINRT) && !defined(WINRT_8_0) +# include "cap_winrt_capture.hpp" +# include "cap_winrt_bridge.hpp" +# define WINRT_VIDEO +#endif + #if defined _M_X64 && defined _MSC_VER && !defined CV_ICC #pragma optimize("",off) #pragma warning(disable: 4748) @@ -508,6 +515,9 @@ static Ptr IVideoCapture_create(int index) #endif #ifdef HAVE_INTELPERC CV_CAP_INTELPERC, +#endif +#ifdef WINRT_VIDEO + CAP_WINRT, #endif -1, -1 }; @@ -526,6 +536,7 @@ static Ptr IVideoCapture_create(int index) { #if defined(HAVE_DSHOW) || \ defined(HAVE_INTELPERC) || \ + defined(WINRT_VIDEO) || \ (0) Ptr capture; @@ -540,6 +551,13 @@ static Ptr IVideoCapture_create(int index) case CV_CAP_INTELPERC: capture = makePtr(); break; // CV_CAP_INTEL_PERC +#endif +#ifdef WINRT_VIDEO + case CAP_WINRT: + capture = Ptr(new cv::VideoCapture_WinRT(index)); + if (capture) + return capture; + break; // CAP_WINRT #endif } if (capture && capture->isOpened()) @@ -664,7 +682,29 @@ bool VideoCapture::read(OutputArray image) VideoCapture& VideoCapture::operator >> (Mat& image) { +#ifdef WINRT_VIDEO + if (grab()) + { + if (retrieve(image)) + { + std::lock_guard lock(VideoioBridge::getInstance().inputBufferMutex); + VideoioBridge& bridge = VideoioBridge::getInstance(); + + // double buffering + bridge.swapInputBuffers(); + auto p = bridge.frontInputPtr; + + bridge.bIsFrameNew = false; + + // needed here because setting Mat 'image' is not allowed by OutputArray in read() + Mat m(bridge.height, bridge.width, CV_8UC3, p); + image = m; + } + } +#else read(image); +#endif + return *this; } @@ -760,4 +800,4 @@ int VideoWriter::fourcc(char c1, char c2, char c3, char c4) return (c1 & 255) + ((c2 & 255) << 8) + ((c3 & 255) << 16) + ((c4 & 255) << 24); } -} +} \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt/CaptureFrameGrabber.cpp b/modules/videoio/src/cap_winrt/CaptureFrameGrabber.cpp new file mode 100644 index 000000000..236e22766 --- /dev/null +++ b/modules/videoio/src/cap_winrt/CaptureFrameGrabber.cpp @@ -0,0 +1,173 @@ +// Copyright (c) Microsoft. All rights reserved. +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "MediaStreamSink.hpp" +#include "MediaSink.hpp" +#include "CaptureFrameGrabber.hpp" + +using namespace Media; +using namespace Platform; +using namespace Windows::Foundation; +using namespace Windows::Media; +using namespace Windows::Media::Capture; +using namespace Windows::Media::MediaProperties; +using namespace concurrency; +using namespace Microsoft::WRL::Details; +using namespace Microsoft::WRL; + +task Media::CaptureFrameGrabber::CreateAsync(_In_ MediaCapture^ capture, _In_ VideoEncodingProperties^ props, CaptureStreamType streamType) +{ + auto reader = ref new Media::CaptureFrameGrabber(capture, props, streamType); + + auto profile = ref new MediaEncodingProfile(); + profile->Video = props; + + task task; + if (reader->_streamType == CaptureStreamType::Preview) + { + task = create_task(capture->StartPreviewToCustomSinkAsync(profile, reader->_mediaExtension)); + } + else + { + task = create_task(capture->StartRecordToCustomSinkAsync(profile, reader->_mediaExtension)); + } + + return task.then([reader]() + { + reader->_state = State::Started; + return reader; + }); +} + +Media::CaptureFrameGrabber::CaptureFrameGrabber(_In_ MediaCapture^ capture, _In_ VideoEncodingProperties^ props, CaptureStreamType streamType) +: _state(State::Created) +, _streamType(streamType) +, _capture(capture) +{ + auto videoSampleHandler = ref new MediaSampleHandler(this, &Media::CaptureFrameGrabber::ProcessSample); + + _mediaSink = Make(nullptr, props, nullptr, videoSampleHandler); + _mediaExtension = reinterpret_cast(static_cast(_mediaSink.Get())); +} + +Media::CaptureFrameGrabber::~CaptureFrameGrabber() +{ + if (_state == State::Started) + { + if (_streamType == CaptureStreamType::Preview) + { + (void)_capture->StopPreviewAsync(); + } + else + { + (void)_capture->StopRecordAsync(); + } + } + + if (_mediaSink != nullptr) + { + (void)_mediaSink->Shutdown(); + _mediaSink = nullptr; + } + _mediaExtension = nullptr; + _capture = nullptr; +} + +void Media::CaptureFrameGrabber::ShowCameraSettings() +{ +#if WINAPI_FAMILY!=WINAPI_FAMILY_PHONE_APP + if (_state == State::Started) + { + CameraOptionsUI::Show(_capture.Get()); + } +#endif +} + +task Media::CaptureFrameGrabber::FinishAsync() +{ + auto lock = _lock.LockExclusive(); + + if (_state != State::Started) + { + throw ref new COMException(E_UNEXPECTED, L"State"); + } + _state = State::Closing; + + if (_mediaSink != nullptr) + { + (void)_mediaSink->Shutdown(); + _mediaSink = nullptr; + } + _mediaExtension = nullptr; + + task task; + if (_streamType == CaptureStreamType::Preview) + { + task = create_task(_capture->StopPreviewAsync()); + } + else + { + task = create_task(_capture->StopRecordAsync()); + } + + return task.then([this]() + { + auto lock = _lock.LockExclusive(); + _state = State::Closed; + _capture = nullptr; + }); +} + +task> Media::CaptureFrameGrabber::GetFrameAsync() +{ + auto lock = _lock.LockExclusive(); + + if (_state != State::Started) + { + throw ref new COMException(E_UNEXPECTED, L"State"); + } + + _mediaSink->RequestVideoSample(); + + task_completion_event> taskEvent; + _videoSampleRequestQueue.push(taskEvent); + + return create_task(taskEvent); +} + +void Media::CaptureFrameGrabber::ProcessSample(_In_ MediaSample^ sample) +{ + task_completion_event> t; + + { + auto lock = _lock.LockExclusive(); + + t = _videoSampleRequestQueue.front(); + _videoSampleRequestQueue.pop(); + } + + ComPtr buffer; + CHK(sample->Sample->ConvertToContiguousBuffer(&buffer)); + + // Dispatch without the lock taken to avoid deadlocks + t.set(As(buffer)); +} \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt/CaptureFrameGrabber.hpp b/modules/videoio/src/cap_winrt/CaptureFrameGrabber.hpp new file mode 100644 index 000000000..d3bcec20d --- /dev/null +++ b/modules/videoio/src/cap_winrt/CaptureFrameGrabber.hpp @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft. All rights reserved. +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include "MFIncludes.hpp" + + +namespace Media { + +class MediaSink; + +enum class CaptureStreamType +{ + Preview = 0, + Record +}; + +ref class CaptureFrameGrabber sealed +{ +public: + + // IClosable + virtual ~CaptureFrameGrabber(); + + virtual void ShowCameraSettings(); + +internal: + + static concurrency::task CreateAsync(_In_ WMC::MediaCapture^ capture, _In_ WMMp::VideoEncodingProperties^ props) + { + return CreateAsync(capture, props, CaptureStreamType::Preview); + } + + static concurrency::task CreateAsync(_In_ WMC::MediaCapture^ capture, _In_ WMMp::VideoEncodingProperties^ props, CaptureStreamType streamType); + + concurrency::task> GetFrameAsync(); + concurrency::task FinishAsync(); + +private: + + CaptureFrameGrabber(_In_ WMC::MediaCapture^ capture, _In_ WMMp::VideoEncodingProperties^ props, CaptureStreamType streamType); + + void ProcessSample(_In_ MediaSample^ sample); + + Platform::Agile _capture; + ::Windows::Media::IMediaExtension^ _mediaExtension; + + MW::ComPtr _mediaSink; + + CaptureStreamType _streamType; + + enum class State + { + Created, + Started, + Closing, + Closed + } _state; + + std::queue>> _videoSampleRequestQueue; + AutoMF _mf; + MWW::SRWLock _lock; +}; + +} \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt/MFIncludes.hpp b/modules/videoio/src/cap_winrt/MFIncludes.hpp new file mode 100644 index 000000000..de831eeb1 --- /dev/null +++ b/modules/videoio/src/cap_winrt/MFIncludes.hpp @@ -0,0 +1,172 @@ +// Header for standard system include files. + +// Copyright (c) Microsoft. All rights reserved. +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include + +#include +#include + +namespace AWM = ::ABI::Windows::Media; +namespace AWMMp = ::ABI::Windows::Media::MediaProperties; +namespace AWFC = ::ABI::Windows::Foundation::Collections; +namespace MW = ::Microsoft::WRL; +namespace MWD = ::Microsoft::WRL::Details; +namespace MWW = ::Microsoft::WRL::Wrappers; +namespace WMC = ::Windows::Media::Capture; +namespace WF = ::Windows::Foundation; +namespace WMMp = ::Windows::Media::MediaProperties; +namespace WSS = ::Windows::Storage::Streams; + +// Exception-based error handling +#define CHK(statement) {HRESULT _hr = (statement); if (FAILED(_hr)) { throw ref new Platform::COMException(_hr); };} +#define CHKNULL(p) {if ((p) == nullptr) { throw ref new Platform::NullReferenceException(L#p); };} + +// Exception-free error handling +#define CHK_RETURN(statement) {hr = (statement); if (FAILED(hr)) { return hr; };} + +// Cast a C++/CX msartpointer to an ABI smartpointer +template +MW::ComPtr As(U^ in) +{ + MW::ComPtr out; + CHK(reinterpret_cast(in)->QueryInterface(IID_PPV_ARGS(&out))); + return out; +} + +// Cast an ABI smartpointer +template +Microsoft::WRL::ComPtr As(const Microsoft::WRL::ComPtr& in) +{ + Microsoft::WRL::ComPtr out; + CHK(in.As(&out)); + return out; +} + +// Cast an ABI smartpointer +template +Microsoft::WRL::ComPtr As(U* in) +{ + Microsoft::WRL::ComPtr out; + CHK(in->QueryInterface(IID_PPV_ARGS(&out))); + return out; +} + +// Get access to bytes in IBuffer +inline unsigned char* GetData(_In_ WSS::IBuffer^ buffer) +{ + unsigned char* bytes = nullptr; + CHK(As(buffer)->Buffer(&bytes)); + return bytes; +} + +// Class to start and shutdown Media Foundation +class AutoMF +{ +public: + AutoMF() + : _bInitialized(false) + { + CHK(MFStartup(MF_VERSION)); + } + + ~AutoMF() + { + if (_bInitialized) + { + (void)MFShutdown(); + } + } + +private: + bool _bInitialized; +}; + +// Class to track error origin +template +HRESULT OriginateError(__in HRESULT hr, __in wchar_t const (&str)[N]) +{ + if (FAILED(hr)) + { + ::RoOriginateErrorW(hr, N - 1, str); + } + return hr; +} + +// Class to track error origin +inline HRESULT OriginateError(__in HRESULT hr) +{ + if (FAILED(hr)) + { + ::RoOriginateErrorW(hr, 0, nullptr); + } + return hr; +} + +// Converts exceptions into HRESULTs +template +HRESULT ExceptionBoundary(Lambda&& lambda) +{ + try + { + lambda(); + return S_OK; + } + catch (Platform::Exception^ e) + { + return e->HResult; + } + catch (const std::bad_alloc&) + { + return E_OUTOFMEMORY; + } + catch (const std::exception&) + { + return E_FAIL; + } +} + +// Wraps an IMFSample in a C++/CX class to be able to define a callback delegate +ref class MediaSample sealed +{ +internal: + MW::ComPtr Sample; +}; + +delegate void MediaSampleHandler(MediaSample^ sample); \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt/MediaSink.hpp b/modules/videoio/src/cap_winrt/MediaSink.hpp new file mode 100644 index 000000000..40fe0e40b --- /dev/null +++ b/modules/videoio/src/cap_winrt/MediaSink.hpp @@ -0,0 +1,396 @@ +// Copyright (c) Microsoft. All rights reserved. +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include "MediaStreamSink.hpp" +#include "MFIncludes.hpp" + +namespace Media { + +const unsigned int c_audioStreamSinkId = 0; +const unsigned int c_videoStreamSinkId = 1; + +class MediaSink WrlSealed + : public MW::RuntimeClass< + MW::RuntimeClassFlags< + MW::RuntimeClassType::WinRtClassicComMix> + , AWM::IMediaExtension + , IMFMediaSink + , IMFClockStateSink + , MW::FtmBase + > +{ + InspectableClass(L"MediaSink", BaseTrust) + +public: + + MediaSink( + _In_opt_ WMMp::AudioEncodingProperties^ audioProps, + _In_opt_ WMMp::VideoEncodingProperties^ videoProps, + _In_opt_ MediaSampleHandler^ audioSampleHandler, + _In_opt_ MediaSampleHandler^ videoSampleHandler + ) + : _shutdown(false) + { + MW::ComPtr audioMT; + if (audioProps != nullptr) + { + CHK(MFCreateMediaTypeFromProperties(As(audioProps).Get(), &audioMT)); + _audioStreamSink = MW::Make( + this, + c_audioStreamSinkId, + audioMT, + audioSampleHandler + ); + } + + MW::ComPtr videoMT; + if (videoProps != nullptr) + { + CHK(MFCreateMediaTypeFromProperties(As(videoProps).Get(), &videoMT)); + _videoStreamSink = MW::Make( + this, + c_videoStreamSinkId, + videoMT, + videoSampleHandler + ); + } + } + + void RequestAudioSample() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + _audioStreamSink->RequestSample(); + } + + void RequestVideoSample() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + _videoStreamSink->RequestSample(); + } + + void SetCurrentAudioMediaType(_In_ IMFMediaType* mt) + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + _audioStreamSink->InternalSetCurrentMediaType(mt); + } + + void SetCurrentVideoMediaType(_In_ IMFMediaType* mt) + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + _videoStreamSink->InternalSetCurrentMediaType(mt); + } + + // + // IMediaExtension + // + + IFACEMETHODIMP SetProperties(_In_ AWFC::IPropertySet * /*configuration*/) + { + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + }); + } + + // + // IMFMediaSink + // + + IFACEMETHODIMP GetCharacteristics(_Out_ DWORD *characteristics) + { + return ExceptionBoundary([this, characteristics]() + { + _VerifyNotShutdown(); + + CHKNULL(characteristics); + *characteristics = MEDIASINK_RATELESS | MEDIASINK_FIXED_STREAMS; + }); + } + + IFACEMETHODIMP AddStreamSink( + DWORD /*streamSinkIdentifier*/, + _In_ IMFMediaType * /*mediaType*/, + _COM_Outptr_ IMFStreamSink **streamSink + ) + { + return ExceptionBoundary([this, streamSink]() + { + _VerifyNotShutdown(); + + CHKNULL(streamSink); + *streamSink = nullptr; + + CHK(MF_E_STREAMSINKS_FIXED); + }); + } + + IFACEMETHODIMP RemoveStreamSink(DWORD /*streamSinkIdentifier*/) + { + return ExceptionBoundary([this]() + { + _VerifyNotShutdown(); + + CHK(MF_E_STREAMSINKS_FIXED); + }); + } + + IFACEMETHODIMP GetStreamSinkCount(_Out_ DWORD *streamSinkCount) + { + return ExceptionBoundary([this, streamSinkCount]() + { + CHKNULL(streamSinkCount); + + _VerifyNotShutdown(); + + *streamSinkCount = (_audioStreamSink != nullptr) + (_videoStreamSink != nullptr); + }); + } + + IFACEMETHODIMP GetStreamSinkByIndex(DWORD index, _COM_Outptr_ IMFStreamSink **streamSink) + { + return ExceptionBoundary([this, index, streamSink]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(streamSink); + *streamSink = nullptr; + + _VerifyNotShutdown(); + + switch (index) + { + case 0: + if (_audioStreamSink != nullptr) + { + CHK(_audioStreamSink.CopyTo(streamSink)); + } + else + { + CHK(_videoStreamSink.CopyTo(streamSink)); + } + break; + + case 1: + if ((_audioStreamSink != nullptr) && (_videoStreamSink != nullptr)) + { + CHK(_videoStreamSink.CopyTo(streamSink)); + } + else + { + CHK(E_INVALIDARG); + } + break; + + default: + CHK(E_INVALIDARG); + } + }); + } + + IFACEMETHODIMP GetStreamSinkById(DWORD identifier, _COM_Outptr_ IMFStreamSink **streamSink) + { + return ExceptionBoundary([this, identifier, streamSink]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(streamSink); + *streamSink = nullptr; + + _VerifyNotShutdown(); + + if ((identifier == 0) && (_audioStreamSink != nullptr)) + { + CHK(_audioStreamSink.CopyTo(streamSink)); + } + else if ((identifier == 1) && (_videoStreamSink != nullptr)) + { + CHK(_videoStreamSink.CopyTo(streamSink)); + } + else + { + CHK(E_INVALIDARG); + } + }); + } + + IFACEMETHODIMP SetPresentationClock(_In_ IMFPresentationClock *clock) + { + return ExceptionBoundary([this, clock]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + if (_clock != nullptr) + { + CHK(_clock->RemoveClockStateSink(this)); + _clock = nullptr; + } + + if (clock != nullptr) + { + CHK(clock->AddClockStateSink(this)); + _clock = clock; + } + }); + } + + IFACEMETHODIMP GetPresentationClock(_COM_Outptr_ IMFPresentationClock **clock) + { + return ExceptionBoundary([this, clock]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(clock); + *clock = nullptr; + + _VerifyNotShutdown(); + + if (_clock != nullptr) + { + CHK(_clock.CopyTo(clock)) + } + }); + } + + IFACEMETHODIMP Shutdown() + { + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + if (_shutdown) + { + return; + } + _shutdown = true; + + if (_audioStreamSink != nullptr) + { + _audioStreamSink->Shutdown(); + _audioStreamSink = nullptr; + } + + if (_videoStreamSink != nullptr) + { + _videoStreamSink->Shutdown(); + _videoStreamSink = nullptr; + } + + if (_clock != nullptr) + { + (void)_clock->RemoveClockStateSink(this); + _clock = nullptr; + } + }); + } + + // + // IMFClockStateSink methods + // + + IFACEMETHODIMP OnClockStart(MFTIME /*hnsSystemTime*/, LONGLONG /*llClockStartOffset*/) + { + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + }); + } + + IFACEMETHODIMP OnClockStop(MFTIME /*hnsSystemTime*/) + { + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + }); + } + + IFACEMETHODIMP OnClockPause(MFTIME /*hnsSystemTime*/) + { + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + }); + } + + IFACEMETHODIMP OnClockRestart(MFTIME /*hnsSystemTime*/) + { + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + }); + } + + IFACEMETHODIMP OnClockSetRate(MFTIME /*hnsSystemTime*/, float /*flRate*/) + { + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + }); + } + +private: + + bool _shutdown; + + void _VerifyNotShutdown() + { + if (_shutdown) + { + CHK(MF_E_SHUTDOWN); + } + } + + MW::ComPtr _audioStreamSink; + MW::ComPtr _videoStreamSink; + MW::ComPtr _clock; + + MWW::SRWLock _lock; +}; + +} \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt/MediaStreamSink.cpp b/modules/videoio/src/cap_winrt/MediaStreamSink.cpp new file mode 100644 index 000000000..1df3b4cfb --- /dev/null +++ b/modules/videoio/src/cap_winrt/MediaStreamSink.cpp @@ -0,0 +1,386 @@ +// Copyright (c) Microsoft. All rights reserved. +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "MediaStreamSink.hpp" +#include "MFIncludes.hpp" + +using namespace Media; +using namespace Microsoft::WRL; +using namespace Platform; +using namespace Windows::Foundation; + +MediaStreamSink::MediaStreamSink( + __in const MW::ComPtr& sink, + __in DWORD id, + __in const MW::ComPtr& mt, + __in MediaSampleHandler^ sampleHandler + ) + : _shutdown(false) + , _id(-1) + , _width(0) + , _height(0) +{ + CHK(MFCreateEventQueue(&_eventQueue)); + CHK(MFCreateMediaType(&_curMT)); + + _UpdateMediaType(mt); + + _sink = sink; + _id = id; + _sampleHandler = sampleHandler; +} + +HRESULT MediaStreamSink::GetMediaSink(__deref_out IMFMediaSink **sink) +{ + return ExceptionBoundary([this, sink]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(sink); + *sink = nullptr; + + _VerifyNotShutdown(); + + CHK(_sink.CopyTo(sink)); + }); +} + +HRESULT MediaStreamSink::GetIdentifier(__out DWORD *identifier) +{ + return ExceptionBoundary([this, identifier]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(identifier); + + _VerifyNotShutdown(); + + *identifier = _id; + }); +} + +HRESULT MediaStreamSink::GetMediaTypeHandler(__deref_out IMFMediaTypeHandler **handler) +{ + return ExceptionBoundary([this, handler]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(handler); + *handler = nullptr; + + _VerifyNotShutdown(); + + *handler = this; + this->AddRef(); + + }); +} + +void MediaStreamSink::RequestSample() +{ + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + CHK(_eventQueue->QueueEventParamVar(MEStreamSinkRequestSample, GUID_NULL, S_OK, nullptr)); +} + +HRESULT MediaStreamSink::ProcessSample(__in_opt IMFSample *sample) +{ + return ExceptionBoundary([this, sample]() + { + MediaSampleHandler^ sampleHandler; + auto mediaSample = ref new MediaSample(); + + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + if (sample == nullptr) + { + return; + } + + mediaSample->Sample = sample; + sampleHandler = _sampleHandler; + } + + // Call back without the lock taken to avoid deadlocks + sampleHandler(mediaSample); + }); +} + +HRESULT MediaStreamSink::PlaceMarker(__in MFSTREAMSINK_MARKER_TYPE /*markerType*/, __in const PROPVARIANT * /*markerValue*/, __in const PROPVARIANT * contextValue) +{ + return ExceptionBoundary([this, contextValue]() + { + auto lock = _lock.LockExclusive(); + CHKNULL(contextValue); + + _VerifyNotShutdown(); + + CHK(_eventQueue->QueueEventParamVar(MEStreamSinkMarker, GUID_NULL, S_OK, contextValue)); + }); +} + +HRESULT MediaStreamSink::Flush() +{ + return ExceptionBoundary([this]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + }); +} + +HRESULT MediaStreamSink::GetEvent(__in DWORD flags, __deref_out IMFMediaEvent **event) +{ + return ExceptionBoundary([this, flags, event]() + { + CHKNULL(event); + *event = nullptr; + + ComPtr eventQueue; + + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + eventQueue = _eventQueue; + } + + // May block for a while + CHK(eventQueue->GetEvent(flags, event)); + }); +} + +HRESULT MediaStreamSink::BeginGetEvent(__in IMFAsyncCallback *callback, __in_opt IUnknown *state) +{ + return ExceptionBoundary([this, callback, state]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + CHK(_eventQueue->BeginGetEvent(callback, state)); + }); +} + + +HRESULT MediaStreamSink::EndGetEvent(__in IMFAsyncResult *result, __deref_out IMFMediaEvent **event) +{ + return ExceptionBoundary([this, result, event]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(event); + *event = nullptr; + + _VerifyNotShutdown(); + + CHK(_eventQueue->EndGetEvent(result, event)); + }); +} + +HRESULT MediaStreamSink::QueueEvent( + __in MediaEventType met, + __in REFGUID extendedType, + __in HRESULT status, + __in_opt const PROPVARIANT *value + ) +{ + return ExceptionBoundary([this, met, extendedType, status, value]() + { + auto lock = _lock.LockExclusive(); + + _VerifyNotShutdown(); + + CHK(_eventQueue->QueueEventParamVar(met, extendedType, status, value)); + }); +} + +HRESULT MediaStreamSink::IsMediaTypeSupported(__in IMFMediaType *mediaType, __deref_out_opt IMFMediaType **closestMediaType) +{ + bool supported = false; + + HRESULT hr = ExceptionBoundary([this, mediaType, closestMediaType, &supported]() + { + auto lock = _lock.LockExclusive(); + HRESULT hr = S_OK; + + if (closestMediaType != nullptr) + { + *closestMediaType = nullptr; + } + + CHKNULL(mediaType); + + _VerifyNotShutdown(); + + supported = _IsMediaTypeSupported(mediaType); + }); + + // Avoid throwing an exception to return MF_E_INVALIDMEDIATYPE as this is not a exceptional case + return FAILED(hr) ? hr : supported ? S_OK : MF_E_INVALIDMEDIATYPE; +} + +HRESULT MediaStreamSink::GetMediaTypeCount(__out DWORD *typeCount) +{ + return ExceptionBoundary([this, typeCount]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(typeCount); + + _VerifyNotShutdown(); + + // No media type provided by default (app needs to specify it) + *typeCount = 0; + }); +} + +HRESULT MediaStreamSink::GetMediaTypeByIndex(__in DWORD /*index*/, __deref_out IMFMediaType **mediaType) +{ + HRESULT hr = ExceptionBoundary([this, mediaType]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(mediaType); + *mediaType = nullptr; + + _VerifyNotShutdown(); + }); + + // Avoid throwing an exception to return MF_E_NO_MORE_TYPES as this is not a exceptional case + return FAILED(hr) ? hr : MF_E_NO_MORE_TYPES; +} + +HRESULT MediaStreamSink::SetCurrentMediaType(__in IMFMediaType *mediaType) +{ + return ExceptionBoundary([this, mediaType]() + { + auto lock = _lock.LockExclusive(); + HRESULT hr = S_OK; + + CHKNULL(mediaType); + + _VerifyNotShutdown(); + + if (!_IsMediaTypeSupported(mediaType)) + { + CHK(MF_E_INVALIDMEDIATYPE); + } + + _UpdateMediaType(mediaType); + }); +} + +HRESULT MediaStreamSink::GetCurrentMediaType(__deref_out_opt IMFMediaType **mediaType) +{ + return ExceptionBoundary([this, mediaType]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(mediaType); + *mediaType = nullptr; + + _VerifyNotShutdown(); + + ComPtr mt; + CHK(MFCreateMediaType(&mt)); + CHK(_curMT->CopyAllItems(mt.Get())); + *mediaType = mt.Detach(); + }); +} + +HRESULT MediaStreamSink::GetMajorType(__out GUID *majorType) +{ + return ExceptionBoundary([this, majorType]() + { + auto lock = _lock.LockExclusive(); + + CHKNULL(majorType); + + _VerifyNotShutdown(); + + *majorType = _majorType; + }); +} + +void MediaStreamSink::InternalSetCurrentMediaType(__in const ComPtr& mediaType) +{ + auto lock = _lock.LockExclusive(); + + CHKNULL(mediaType); + + _VerifyNotShutdown(); + + _UpdateMediaType(mediaType); +} + +void MediaStreamSink::Shutdown() +{ + auto lock = _lock.LockExclusive(); + + if (_shutdown) + { + return; + } + _shutdown = true; + + (void)_eventQueue->Shutdown(); + _eventQueue = nullptr; + + _curMT = nullptr; + _sink = nullptr; + _sampleHandler = nullptr; +} + +bool MediaStreamSink::_IsMediaTypeSupported(__in const ComPtr& mt) const +{ + GUID majorType; + GUID subType; + if (SUCCEEDED(mt->GetGUID(MF_MT_MAJOR_TYPE, &majorType)) && + SUCCEEDED(mt->GetGUID(MF_MT_SUBTYPE, &subType)) && + (majorType == _majorType) && + (subType == _subType)) + { + return true; + } + + return false; +} + +void MediaStreamSink::_UpdateMediaType(__in const ComPtr& mt) +{ + CHK(mt->GetGUID(MF_MT_MAJOR_TYPE, &_majorType)); + CHK(mt->GetGUID(MF_MT_SUBTYPE, &_subType)); + + if (_majorType == MFMediaType_Video) + { + CHK(MFGetAttributeSize(mt.Get(), MF_MT_FRAME_SIZE, &_width, &_height)); + } + + CHK(mt->CopyAllItems(_curMT.Get())); +} \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt/MediaStreamSink.hpp b/modules/videoio/src/cap_winrt/MediaStreamSink.hpp new file mode 100644 index 000000000..3434adff0 --- /dev/null +++ b/modules/videoio/src/cap_winrt/MediaStreamSink.hpp @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft. All rights reserved. +// +// The MIT License (MIT) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include "MFIncludes.hpp" + +namespace Media { + +class MediaStreamSink WrlSealed : + public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags, + IMFStreamSink, + IMFMediaEventGenerator, + IMFMediaTypeHandler + > +{ +public: + + MediaStreamSink( + __in const MW::ComPtr& sink, + __in DWORD id, + __in const MW::ComPtr& mt, + __in MediaSampleHandler^ sampleHandler + ); + + // + // IMFStreamSink + // + + IFACEMETHODIMP GetMediaSink(__deref_out IMFMediaSink **sink); + IFACEMETHODIMP GetIdentifier(__out DWORD *identifier); + IFACEMETHODIMP GetMediaTypeHandler(__deref_out IMFMediaTypeHandler **handler); + IFACEMETHODIMP ProcessSample(__in_opt IMFSample *sample); + IFACEMETHODIMP PlaceMarker(__in MFSTREAMSINK_MARKER_TYPE markerType, __in const PROPVARIANT * markerValue, __in const PROPVARIANT * contextValue); + IFACEMETHODIMP Flush(); + + // + // IMFMediaEventGenerator + // + + IFACEMETHODIMP GetEvent(__in DWORD flags, __deref_out IMFMediaEvent **event); + IFACEMETHODIMP BeginGetEvent(__in IMFAsyncCallback *callback, __in_opt IUnknown *state); + IFACEMETHODIMP EndGetEvent(__in IMFAsyncResult *result, __deref_out IMFMediaEvent **event); + IFACEMETHODIMP QueueEvent(__in MediaEventType met, __in REFGUID extendedType, __in HRESULT status, __in_opt const PROPVARIANT *value); + + // + // IMFMediaTypeHandler + // + + IFACEMETHODIMP IsMediaTypeSupported(__in IMFMediaType *mediaType, __deref_out_opt IMFMediaType **closestMediaType); + IFACEMETHODIMP GetMediaTypeCount(__out DWORD *typeCount); + IFACEMETHODIMP GetMediaTypeByIndex(__in DWORD index, __deref_out IMFMediaType **mediaType); + IFACEMETHODIMP SetCurrentMediaType(__in IMFMediaType *mediaType); + IFACEMETHODIMP GetCurrentMediaType(__deref_out_opt IMFMediaType **mediaType); + IFACEMETHODIMP GetMajorType(__out GUID *majorType); + + // + // Misc + // + + void InternalSetCurrentMediaType(__in const MW::ComPtr& mediaType); + void RequestSample(); + void Shutdown(); + +private: + + bool _IsMediaTypeSupported(__in const MW::ComPtr& mt) const; + void _UpdateMediaType(__in const MW::ComPtr& mt); + + void _VerifyNotShutdown() + { + if (_shutdown) + { + CHK(MF_E_SHUTDOWN); + } + } + + MW::ComPtr _sink; + MW::ComPtr _eventQueue; + MW::ComPtr _curMT; + + MediaSampleHandler^ _sampleHandler; + + GUID _majorType; + GUID _subType; + unsigned int _width; + unsigned int _height; + DWORD _id; + bool _shutdown; + + MWW::SRWLock _lock; +}; + +} \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt_bridge.cpp b/modules/videoio/src/cap_winrt_bridge.cpp new file mode 100644 index 000000000..11bd0b715 --- /dev/null +++ b/modules/videoio/src/cap_winrt_bridge.cpp @@ -0,0 +1,89 @@ +// videoio to XAML bridge for OpenCV + +// Copyright (c) Microsoft Open Technologies, Inc. +// All rights reserved. +// +// (3 - clause BSD License) +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that +// the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the +// following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +// promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "opencv2\videoio\cap_winrt.hpp" +#include "cap_winrt_capture.hpp" +#include "cap_winrt_bridge.hpp" +#include "cap_winrt_video.hpp" + +using namespace Windows::Foundation; +using namespace Windows::Media::Capture; +using namespace Windows::Media::MediaProperties; +using namespace Windows::Devices::Enumeration; + +using namespace Windows::UI::Xaml::Media::Imaging; +using namespace Microsoft::WRL; + +using namespace Platform; +using namespace ::Concurrency; + +using namespace ::std; + +/***************************** VideoioBridge class ******************************/ + +// non-blocking +void VideoioBridge::requestForUIthreadAsync(int action, int widthp, int heightp) +{ + reporter.report(action); +} + +VideoioBridge& VideoioBridge::getInstance() +{ + static VideoioBridge instance; + return instance; +} + +void VideoioBridge::swapInputBuffers() +{ + // TODO: already locked, check validity + // lock_guard lock(inputBufferMutex); + swap(backInputPtr, frontInputPtr); + //if (currentFrame != frameCounter) + //{ + // currentFrame = frameCounter; + // swap(backInputPtr, frontInputPtr); + //} +} + +void VideoioBridge::swapOutputBuffers() +{ + lock_guard lock(outputBufferMutex); + swap(frontOutputBuffer, backOutputBuffer); +} + +void VideoioBridge::allocateOutputBuffers() +{ + frontOutputBuffer = ref new WriteableBitmap(width, height); + backOutputBuffer = ref new WriteableBitmap(width, height); +} + +void VideoioBridge::imshow() +{ + VideoioBridge::getInstance().swapOutputBuffers(); + VideoioBridge::getInstance().requestForUIthreadAsync(cv::UPDATE_IMAGE_ELEMENT); +} + +// end \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt_bridge.hpp b/modules/videoio/src/cap_winrt_bridge.hpp new file mode 100644 index 000000000..02f57b034 --- /dev/null +++ b/modules/videoio/src/cap_winrt_bridge.hpp @@ -0,0 +1,96 @@ +// videoio to XAML bridge for OpenCV + +// Copyright (c) Microsoft Open Technologies, Inc. +// All rights reserved. +// +// (3 - clause BSD License) +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that +// the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the +// following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +// promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// this header is included in the XAML App, so it cannot include any +// OpenCV headers, or a static assert will be raised + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +// Class VideoioBridge (singleton) is needed because the interface for +// VideoCapture_WinRT in cap_winrt_capture.hpp is fixed by OpenCV. +class VideoioBridge +{ +public: + + static VideoioBridge& getInstance(); + + // call after initialization + void setReporter(Concurrency::progress_reporter pr) { reporter = pr; } + + // to be called from cvMain via cap_winrt on bg thread - non-blocking (async) + void requestForUIthreadAsync( int action, int width=0, int height=0 ); + + // TODO: modify in window.cpp: void cv::imshow( const String& winname, InputArray _img ) + void imshow(/*cv::InputArray matToShow*/); // shows Mat in the cvImage element + void swapInputBuffers(); + void allocateOutputBuffers(); + void swapOutputBuffers(); + + + int deviceIndex, width, height; + std::atomic bIsFrameNew; + std::mutex inputBufferMutex; // input is double buffered + unsigned char * frontInputPtr; // OpenCV reads this + unsigned char * backInputPtr; // Video grabber writes this + std::atomic frameCounter; + unsigned long currentFrame; + + std::mutex outputBufferMutex; // output is double buffered + Windows::UI::Xaml::Media::Imaging::WriteableBitmap^ frontOutputBuffer; // OpenCV write this + Windows::UI::Xaml::Media::Imaging::WriteableBitmap^ backOutputBuffer; // XAML reads this + Windows::UI::Xaml::Controls::Image ^cvImage; + +private: + + VideoioBridge() { + deviceIndex = 0; + width = 640; + height = 480; + deviceReady = false; + bIsFrameNew = false; + currentFrame = 0; + frameCounter = 0; + }; + + // singleton + VideoioBridge(VideoioBridge const &); + void operator=(const VideoioBridge &); + + std::atomic deviceReady; + Concurrency::progress_reporter reporter; +}; \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt_capture.cpp b/modules/videoio/src/cap_winrt_capture.cpp new file mode 100644 index 000000000..882172279 --- /dev/null +++ b/modules/videoio/src/cap_winrt_capture.cpp @@ -0,0 +1,279 @@ +// Capture support for WinRT + +// Copyright (c) Microsoft Open Technologies, Inc. +// All rights reserved. +// +// (3 - clause BSD License) +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that +// the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the +// following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +// promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + + +#include "precomp.hpp" +#include "cap_winrt_capture.hpp" +#include "cap_winrt_bridge.hpp" +#include "cap_winrt_video.hpp" +#include + +using namespace Windows::Foundation; +using namespace Windows::Media::Capture; +using namespace Windows::Media::MediaProperties; +using namespace Windows::Devices::Enumeration; + +using namespace Platform; + +using namespace Windows::UI::Xaml::Media::Imaging; +using namespace Microsoft::WRL; + +using namespace ::std; + + +// nb. VideoCapture_WinRT is not a singleton, so the Mats are made file statics +// we do not support more than one capture device simultaneously with the +// design at this time + +// nb. inputBufferMutex was not able to guarantee that OpenCV Mats were +// ready to accept data in the UI thread (memory access exceptions were thrown +// even though buffer address was good). +// Therefore allocation of Mats is also done on the UI thread before the video +// device is initialized. + +static cv::Mat frontInputMat; +static cv::Mat backInputMat; + +namespace cv { + + /***************************** exported control functions ******************************/ + + template + void winrt_startMessageLoop(std::function&& callback, Args... args) + { + auto asyncTask = ::concurrency::create_async([=](::concurrency::progress_reporter reporter) + { + VideoioBridge::getInstance().setReporter(reporter); + + // frame reading loop + callback(args...); + }); + + asyncTask->Progress = ref new AsyncActionProgressHandler([=](IAsyncActionWithProgress^ act, int progress) + { + int action = progress; + + // these actions will be processed on the UI thread asynchronously + switch (action) + { + case OPEN_CAMERA: + winrt_openCamera(); + break; + case CLOSE_CAMERA: + winrt_closeGrabber(); + break; + case UPDATE_IMAGE_ELEMENT: + winrt_updateFrameContainer(); + break; + } + }); + } + + template + void winrt_startMessageLoop(void callback(Args...), Args... args) + { + winrt_startMessageLoop(std::function(callback), args...); + } + + void winrt_onVisibilityChanged(bool visible) { + if (visible) + { + VideoioBridge& bridge = VideoioBridge::getInstance(); + + // only start the grabber if the camera was opened in OpenCV + if (bridge.backInputPtr != nullptr) + { + if (Video::getInstance().isStarted()) return; + + int device = bridge.deviceIndex; + int width = bridge.width; + int height = bridge.height; + + winrt_initGrabber(device, width, height); + } + } else + { + //grabberStarted = false; + winrt_closeGrabber(); + } + } + + void winrt_imshow() { + VideoioBridge::getInstance().imshow(); + } + + void winrt_setFrameContainer(::Windows::UI::Xaml::Controls::Image^ image) { + VideoioBridge::getInstance().cvImage = image; + } + + + /********************************* Internal helpers ************************************/ + + void winrt_updateFrameContainer() + { + // copy output Mat to WBM + winrt_copyOutput(); + + // set XAML image element with image WBM + VideoioBridge::getInstance().cvImage->Source = VideoioBridge::getInstance().backOutputBuffer; + } + + // performed on UI thread + bool winrt_openCamera() + { + VideoioBridge& bridge = VideoioBridge::getInstance(); + + int device = bridge.deviceIndex; + int width = bridge.width; + int height = bridge.height; + + // buffers must alloc'd on UI thread + winrt_allocateBuffers(width, height); + + // nb. video capture device init must be done on UI thread; + if (!Video::getInstance().isStarted()) + { + winrt_initGrabber(device, width, height); + return true; + } + + return false; + } + + // performed on UI thread + void winrt_allocateBuffers(int width, int height) + { + VideoioBridge& bridge = VideoioBridge::getInstance(); + + // allocate input Mats (bgra8 = CV_8UC4, RGB24 = CV_8UC3) + frontInputMat.create(height, width, CV_8UC3); + backInputMat.create(height, width, CV_8UC3); + + bridge.frontInputPtr = frontInputMat.ptr(0); + bridge.backInputPtr = backInputMat.ptr(0); + + bridge.allocateOutputBuffers(); + } + + // non-blocking + bool winrt_initGrabber(int device, int w, int h) { + // nb. Video class is not exported outside of this DLL + // due to complexities in the CaptureFrameGrabber ref class + // as written in the header not mixing well with pure C++ classes + return Video::getInstance().initGrabber(device, w, h); + } + + void winrt_closeGrabber() { + Video::getInstance().closeGrabber(); + } + + // nb on UI thread + void winrt_copyOutput() { + Video::getInstance().CopyOutput(); + } + + + /********************************* VideoCapture_WinRT class ****************************/ + + VideoCapture_WinRT::VideoCapture_WinRT(int device) : started(false) + { + VideoioBridge::getInstance().deviceIndex = device; + } + + bool VideoCapture_WinRT::isOpened() const + { + return true; // started; + } + + // grab a frame: + // this will NOT block per spec + // should be called on the image processing thread, not the UI thread + bool VideoCapture_WinRT::grabFrame() + { + // if device is not started we must return true so retrieveFrame() is called to start device + // nb. we cannot start the device here because we do not know the size of the input Mat + if (!started) return true; + + if (VideoioBridge::getInstance().bIsFrameNew) + { + return true; + } + + // nb. if blocking is to be added: + // unique_lock lock(VideoioBridge::getInstance().frameReadyMutex); + // VideoioBridge::getInstance().frameReadyEvent.wait(lock); + return false; + } + + // should be called on the image processing thread after grabFrame + // see VideoCapture::read + bool VideoCapture_WinRT::retrieveFrame(int channel, cv::OutputArray outArray) + { + if (!started) { + + int width, height; + width = outArray.size().width; + height = outArray.size().height; + if (width == 0) width = 640; + if (height == 0) height = 480; + + VideoioBridge::getInstance().width = width; + VideoioBridge::getInstance().height = height; + + // nb. Mats will be alloc'd on UI thread + + // request device init on UI thread - this does not block, and is async + VideoioBridge::getInstance().requestForUIthreadAsync(OPEN_CAMERA, + outArray.size().width, outArray.size().height); + + started = true; + return false; + } + + if (!started) return false; + + return VideoioBridge::getInstance().bIsFrameNew; + } + + + bool VideoCapture_WinRT::setProperty(int property_id, double value) + { + switch (property_id) + { + case CAP_PROP_FRAME_WIDTH: + size.width = (int)value; + break; + case CAP_PROP_FRAME_HEIGHT: + size.height = (int)value; + break; + default: + return false; + } + return true; + } +} + +// end \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt_capture.hpp b/modules/videoio/src/cap_winrt_capture.hpp new file mode 100644 index 000000000..918a4d6fc --- /dev/null +++ b/modules/videoio/src/cap_winrt_capture.hpp @@ -0,0 +1,82 @@ +// Capture support for WinRT + +// Copyright (c) Microsoft Open Technologies, Inc. +// All rights reserved. +// +// (3 - clause BSD License) +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that +// the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the +// following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +// promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "precomp.hpp" + +#include +#include +#include +#include + +#include + + +// nb. implemented the newer IVideoCapture C++ interface so that we can work +// directly with Mat, not the older C cv interface +// (which may have added overhead for IPL file conversion) + +namespace cv { + + /******************* Internal helpers **************************************/ + + void winrt_updateFrameContainer(); + bool winrt_openCamera(); + bool winrt_initGrabber(int device, int w, int h); + void winrt_closeGrabber(); + void winrt_copyOutput(); + void winrt_allocateBuffers(int width, int height); + + /******************* VideoCapture_WinRT class ******************************/ + + class VideoCapture_WinRT : public IVideoCapture + { + public: + VideoCapture_WinRT() : started(false) {} + VideoCapture_WinRT(int device); + virtual ~VideoCapture_WinRT() {} + + // from base class IVideoCapture + virtual double getProperty(int) { return 0; } + virtual bool setProperty(int, double); + virtual bool grabFrame(); + virtual bool retrieveFrame(int channel, cv::OutputArray outArray); + + // Return the type of the capture object + virtual int getCaptureDomain() { return CAP_WINRT; } + + virtual bool isOpened() const; + + protected: + + bool started; + CvSize size; + int bytesPerPixel; + unsigned long frameCurrent; + std::atomic isFrameNew; + }; +} \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt_video.cpp b/modules/videoio/src/cap_winrt_video.cpp new file mode 100644 index 000000000..a41e1f9e3 --- /dev/null +++ b/modules/videoio/src/cap_winrt_video.cpp @@ -0,0 +1,322 @@ +// Video support with XAML + +// Copyright (c) Microsoft Open Technologies, Inc. +// All rights reserved. +// +// (3 - clause BSD License) +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that +// the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the +// following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +// promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "cap_winrt_video.hpp" + +#include +#include +#include +#include + +#include +#include +#include + + +using namespace ::concurrency; +using namespace ::Windows::Foundation; +using namespace ::std; + +using namespace Microsoft::WRL; +using namespace Windows::Media::Devices; +using namespace Windows::Media::MediaProperties; +using namespace Windows::Media::Capture; +using namespace Windows::UI::Xaml::Media::Imaging; +using namespace Windows::Devices::Enumeration; + +#include "cap_winrt/CaptureFrameGrabber.hpp" + +// pull in Media Foundation libs +#pragma comment(lib, "mfplat") +#pragma comment(lib, "mf") +#pragma comment(lib, "mfuuid") + +#if (WINAPI_FAMILY!=WINAPI_FAMILY_PHONE_APP) && !defined(_M_ARM) +#pragma comment(lib, "Shlwapi") +#endif + +#include "cap_winrt_bridge.hpp" + +Video::Video() {} + +Video &Video::getInstance() { + static Video v; + return v; +} + +bool Video::isStarted() { + return bGrabberInited.load(); +} + +void Video::closeGrabber() { + // assigning nullptr causes deref of grabber and thus closes the device + m_frameGrabber = nullptr; + bGrabberInited = false; + bGrabberInitInProgress = false; +} + + +bool Video::initGrabber(int device, int w, int h) { + // already started? + if (bGrabberInited || bGrabberInitInProgress) return false; + + width = w; + height = h; + + bGrabberInited = false; + bGrabberInitInProgress = true; + + m_deviceID = device; + + create_task(DeviceInformation::FindAllAsync(DeviceClass::VideoCapture)) + .then([this](task findTask) + { + m_devices = findTask.get(); + + // got selected device? + if ((unsigned)m_deviceID >= m_devices.Get()->Size) + { + OutputDebugStringA("Video::initGrabber - no video device found\n"); + return false; + } + + auto devInfo = m_devices.Get()->GetAt(m_deviceID); + + auto settings = ref new MediaCaptureInitializationSettings(); + settings->StreamingCaptureMode = StreamingCaptureMode::Video; // Video-only capture + settings->VideoDeviceId = devInfo->Id; + + auto location = devInfo->EnclosureLocation; + bFlipImageX = true; + if (location != nullptr && location->Panel == Windows::Devices::Enumeration::Panel::Back) + { + bFlipImageX = false; + } + + m_capture = ref new MediaCapture(); + create_task(m_capture->InitializeAsync(settings)).then([this](){ + + auto props = safe_cast(m_capture->VideoDeviceController->GetMediaStreamProperties(MediaStreamType::VideoPreview)); + + // for 24 bpp + props->Subtype = MediaEncodingSubtypes::Rgb24; bytesPerPixel = 3; + + // format used by XAML & WBM (for testing) + // props->Subtype = MediaEncodingSubtypes::Bgra8; bytesPerPixel = 4; + + props->Width = width; + props->Height = height; + + return ::Media::CaptureFrameGrabber::CreateAsync(m_capture.Get(), props); + + }).then([this](::Media::CaptureFrameGrabber^ frameGrabber) + { + m_frameGrabber = frameGrabber; + bGrabberInited = true; + bGrabberInitInProgress = false; + //ready = true; + _GrabFrameAsync(frameGrabber); + }); + + return true; + }); + + // nb. cannot block here - this will lock the UI thread: + + return true; +} + + +void Video::_GrabFrameAsync(::Media::CaptureFrameGrabber^ frameGrabber) { + // use rgb24 layout + create_task(frameGrabber->GetFrameAsync()).then([this, frameGrabber](const ComPtr& buffer) + { + // do the RGB swizzle while copying the pixels from the IMF2DBuffer2 + BYTE *pbScanline; + LONG plPitch; + unsigned int colBytes = width * bytesPerPixel; + CHK(buffer->Lock2D(&pbScanline, &plPitch)); + + // flip + if (bFlipImageX) + { + std::lock_guard lock(VideoioBridge::getInstance().inputBufferMutex); + + // ptr to input Mat data array + auto buf = VideoioBridge::getInstance().backInputPtr; + + for (unsigned int row = 0; row < height; row++) + { + unsigned int i = 0; + unsigned int j = colBytes - 1; + + while (i < colBytes) + { + // reverse the scan line + // as a side effect this also swizzles R and B channels + buf[j--] = pbScanline[i++]; + buf[j--] = pbScanline[i++]; + buf[j--] = pbScanline[i++]; + } + pbScanline += plPitch; + buf += colBytes; + } + VideoioBridge::getInstance().bIsFrameNew = true; + } else + { + std::lock_guard lock(VideoioBridge::getInstance().inputBufferMutex); + + // ptr to input Mat data array + auto buf = VideoioBridge::getInstance().backInputPtr; + + for (unsigned int row = 0; row < height; row++) + { + // used for Bgr8: + //for (unsigned int i = 0; i < colBytes; i++ ) + // buf[i] = pbScanline[i]; + + // used for RGB24: + for (unsigned int i = 0; i < colBytes; i += bytesPerPixel) + { + // swizzle the R and B values (BGR to RGB) + buf[i] = pbScanline[i + 2]; + buf[i + 1] = pbScanline[i + 1]; + buf[i + 2] = pbScanline[i]; + + // no swizzle + //buf[i] = pbScanline[i]; + //buf[i + 1] = pbScanline[i + 1]; + //buf[i + 2] = pbScanline[i + 2]; + } + + pbScanline += plPitch; + buf += colBytes; + } + VideoioBridge::getInstance().bIsFrameNew = true; + } + CHK(buffer->Unlock2D()); + + VideoioBridge::getInstance().frameCounter++; + + if (bGrabberInited) + { + _GrabFrameAsync(frameGrabber); + } + }, task_continuation_context::use_current()); +} + + +// copy from input Mat to output WBM +// must be on UI thread +void Video::CopyOutput() { + { + std::lock_guard lock(VideoioBridge::getInstance().outputBufferMutex); + + auto inAr = VideoioBridge::getInstance().frontInputPtr; + auto outAr = GetData(VideoioBridge::getInstance().frontOutputBuffer->PixelBuffer); + + const unsigned int bytesPerPixel = 3; + auto pbScanline = inAr; + auto plPitch = width * bytesPerPixel; + + auto buf = outAr; + unsigned int colBytes = width * 4; + + // copy RGB24 to bgra8 + for (unsigned int row = 0; row < height; row++) + { + // used for Bgr8: + // nb. no alpha + // for (unsigned int i = 0; i < colBytes; i++ ) buf[i] = pbScanline[i]; + + // used for RGB24: + // nb. alpha is set to full opaque + for (unsigned int i = 0, j = 0; i < plPitch; i += bytesPerPixel, j += 4) + { + // swizzle the R and B values (RGB24 to Bgr8) + buf[j] = pbScanline[i + 2]; + buf[j + 1] = pbScanline[i + 1]; + buf[j + 2] = pbScanline[i]; + buf[j + 3] = 0xff; + + // if no swizzle is desired: + //buf[i] = pbScanline[i]; + //buf[i + 1] = pbScanline[i + 1]; + //buf[i + 2] = pbScanline[i + 2]; + //buf[i + 3] = 0xff; + } + + pbScanline += plPitch; + buf += colBytes; + } + VideoioBridge::getInstance().frontOutputBuffer->PixelBuffer->Length = width * height * 4; + } +} + + +bool Video::listDevicesTask() { + std::atomic ready(false); + + auto settings = ref new MediaCaptureInitializationSettings(); + + //vector devices; + + create_task(DeviceInformation::FindAllAsync(DeviceClass::VideoCapture)) + .then([this, &ready](task findTask) + { + m_devices = findTask.get(); + + for (size_t i = 0; i < m_devices->Size; i++) + { + // ofVideoDevice deviceInfo; + auto d = m_devices->GetAt(i); + //deviceInfo.bAvailable = true; + //deviceInfo.deviceName = PlatformStringToString(d->Name); + //deviceInfo.hardwareName = deviceInfo.deviceName; + // devices.push_back(deviceInfo); + } + + ready = true; + }); + + // wait for async task to complete + int count = 0; + while (!ready) + { + count++; + } + + return true; +} + + +bool Video::listDevices() { + // synchronous version of listing video devices on WinRT + std::future result = std::async(std::launch::async, &Video::listDevicesTask, this); + return result.get(); +} + +// end \ No newline at end of file diff --git a/modules/videoio/src/cap_winrt_video.hpp b/modules/videoio/src/cap_winrt_video.hpp new file mode 100644 index 000000000..9ece69e1f --- /dev/null +++ b/modules/videoio/src/cap_winrt_video.hpp @@ -0,0 +1,73 @@ +// Video support with XAML + +// Copyright (c) Microsoft Open Technologies, Inc. +// All rights reserved. +// +// (3 - clause BSD License) +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that +// the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the +// following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +// promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cap_winrt/CaptureFrameGrabber.hpp" + +#include +#include + +class Video { +public: + + bool initGrabber(int device, int w, int h); + void closeGrabber(); + bool isStarted(); + + // singleton + static Video &getInstance(); + + void CopyOutput(); + +private: + // singleton + Video(); + + void _GrabFrameAsync(::Media::CaptureFrameGrabber^ frameGrabber); + + bool listDevices(); + + Platform::Agile m_capture; + Platform::Agile m_devices; + + ::Media::CaptureFrameGrabber^ m_frameGrabber; + + bool listDevicesTask(); + + bool bChooseDevice; + bool bVerbose; + bool bFlipImageX; + //std::atomic bGrabberInited; + int m_deviceID; + int attemptFramerate; + std::atomic bIsFrameNew; + std::atomic bGrabberInited; + std::atomic bGrabberInitInProgress; + unsigned int width, height; + int bytesPerPixel; + +}; \ No newline at end of file