Add MouseCursorCapturer interface with implementation for X11.

The new interface will be used to capture cursor shape and position and
blend it into the image captured with desktop capturers.

BUG=crbug.com/173265
R=wez@chromium.org
TBR=andrew@webrtc.org (modules.gyp)

Review URL: https://webrtc-codereview.appspot.com/2386005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4967 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
sergeyu@chromium.org 2013-10-16 02:42:38 +00:00
parent 3555303cb0
commit 2767b53f66
14 changed files with 622 additions and 2 deletions

View File

@ -15,6 +15,7 @@
'<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers',
],
'sources': [
"desktop_capture_types.h",
"desktop_capturer.h",
"desktop_frame.cc",
"desktop_frame.h",
@ -35,6 +36,12 @@
"mac/desktop_configuration.mm",
"mac/scoped_pixel_buffer_object.cc",
"mac/scoped_pixel_buffer_object.h",
"mouse_cursor.cc",
"mouse_cursor.h",
"mouse_cursor_monitor.h",
"mouse_cursor_monitor_mac.mm",
"mouse_cursor_monitor_win.cc",
"mouse_cursor_monitor_x11.cc",
"mouse_cursor_shape.h",
"screen_capture_frame_queue.cc",
"screen_capture_frame_queue.h",
@ -88,6 +95,7 @@
}],
['OS!="win" and OS!="mac" and use_x11==0', {
'sources': [
"mouse_cursor_monitor_null.cc",
"screen_capturer_null.cc",
"window_capturer_null.cc",
],

View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_TYPES_H_
#define WEBRTC_MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_TYPES_H_
#include <stdint.h>
#include "webrtc/modules/desktop_capture/desktop_geometry.h"
#include "webrtc/typedefs.h"
namespace webrtc {
// Type used to identify windows on the desktop. Values are platform-specific:
// - On Windows: HWND cast to intptr_t.
// - On Linux (with X11): X11 Window (unsigned long) type cast to intptr_t.
// - On OSX: integer window number.
typedef intptr_t WindowId;
const WindowId kNullWindowId = 0;
} // namespace webrtc
#endif // WEBRTC_MODULES_DESKTOP_CAPTURE_DESKTOP_CAPTURE_TYPES_H_

View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/desktop_capture/mouse_cursor.h"
#include "webrtc/modules/desktop_capture/desktop_frame.h"
namespace webrtc {
MouseCursor::MouseCursor(DesktopFrame* image, const DesktopVector& hotspot)
: image_(image),
hotspot_(hotspot) {
assert(0 <= hotspot_.x() && hotspot_.x() <= image_->size().width());
assert(0 <= hotspot_.y() && hotspot_.y() <= image_->size().height());
}
MouseCursor::~MouseCursor() {}
} // namespace webrtc

View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_MOUSE_CURSOR_H_
#define WEBRTC_MODULES_DESKTOP_CAPTURE_MOUSE_CURSOR_H_
#include "webrtc/modules/desktop_capture/desktop_geometry.h"
#include "webrtc/system_wrappers/interface/constructor_magic.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
namespace webrtc {
class DesktopFrame;
class MouseCursor {
public:
// Takes ownership of |image|. |hotspot| must be within |image| boundaries.
MouseCursor(DesktopFrame* image, const DesktopVector& hotspot);
~MouseCursor();
const DesktopFrame& image() { return *image_; }
const DesktopVector& hotspot() { return hotspot_; }
private:
scoped_ptr<DesktopFrame> image_;
DesktopVector hotspot_;
DISALLOW_COPY_AND_ASSIGN(MouseCursor);
};
} // namespace webrtc
#endif // WEBRTC_MODULES_DESKTOP_CAPTURE_MOUSE_CURSOR_H_

View File

@ -0,0 +1,89 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_MOUSE_CURSOR_MONITOR_H_
#define WEBRTC_MODULES_DESKTOP_CAPTURE_MOUSE_CURSOR_MONITOR_H_
#include "webrtc/modules/desktop_capture/desktop_capture_types.h"
#include "webrtc/modules/desktop_capture/desktop_geometry.h"
#include "webrtc/typedefs.h"
namespace webrtc {
class DesktopCaptureOptions;
class DesktopFrame;
class MouseCursor;
// Captures mouse shape and position.
class MouseCursorMonitor {
public:
enum CursorState {
// Cursor on top of the window including window decorations.
INSIDE,
// Cursor is outside of the window.
OUTSIDE,
};
enum Mode {
// Capture only shape of the mouse cursor, but not position.
SHAPE_ONLY,
// Capture both, mouse cursor shape and position.
SHAPE_AND_POSITION,
};
// Callback interface used to pass current mouse cursor position and shape.
class Callback {
public:
// Called in response to Capture() when the cursor shape has changed. Must
// take ownership of |cursor|.
virtual void OnMouseCursor(MouseCursor* cursor) = 0;
// Called in response to Capture(). |position| indicates cursor position
// relative to the |window| specified in the constructor.
virtual void OnMouseCursorPosition(CursorState state,
const DesktopVector& position) = 0;
protected:
virtual ~Callback() {}
};
virtual ~MouseCursorMonitor() {}
// Creates a capturer that notifies of mouse cursor events while the cursor is
// over the specified window.
static MouseCursorMonitor* CreateForWindow(
const DesktopCaptureOptions& options,
WindowId window);
// Creates a capturer that monitors the mouse cursor shape and position across
// the entire desktop.
//
// TODO(sergeyu): Provide a way to select a specific screen.
static MouseCursorMonitor* CreateForScreen(
const DesktopCaptureOptions& options);
// Initializes the monitor with the |callback|, which must remain valid until
// capturer is destroyed.
virtual void Init(Callback* callback, Mode mode) = 0;
// Captures current cursor shape and position (depending on the |mode| passed
// to Init()). Calls Callback::OnMouseCursor() if cursor shape has
// changed since the last call (or when Capture() is called for the first
// time) and then Callback::OnMouseCursorPosition() if mode is set to
// SHAPE_AND_POSITION.
virtual void Capture() = 0;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_DESKTOP_CAPTURE_MOUSE_CURSOR_MONITOR_H_

View File

@ -0,0 +1,28 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
#include <cstddef>
namespace webrtc {
// TODO(sergeyu): Implement MouseCursorMonitor for Mac.
MouseCursorMonitor* MouseCursorMonitor::CreateForWindow(
const DesktopCaptureOptions& options, WindowId window) {
return NULL;
}
MouseCursorMonitor* MouseCursorMonitor::CreateForScreen(
const DesktopCaptureOptions& options) {
return NULL;
}
} // namespace webrtc

View File

@ -0,0 +1,22 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
#include <cstddef>
namespace webrtc {
MouseCursorMonitor* MouseCursorMonitor::CreateForScreen(
const DesktopCaptureOptions& options) {
return NULL;
}
} // namespace webrtc

View File

@ -0,0 +1,118 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
#include "gtest/gtest.h"
#include "webrtc/modules/desktop_capture/desktop_capture_options.h"
#include "webrtc/modules/desktop_capture/desktop_frame.h"
#include "webrtc/modules/desktop_capture/mouse_cursor.h"
#include "webrtc/modules/desktop_capture/window_capturer.h"
#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
namespace webrtc {
class MouseCursorMonitorTest : public testing::Test,
public MouseCursorMonitor::Callback {
public:
MouseCursorMonitorTest()
: position_received_(false) {
}
// MouseCursorMonitor::Callback interface
virtual void OnMouseCursor(MouseCursor* cursor_image) OVERRIDE {
cursor_image_.reset(cursor_image);
}
virtual void OnMouseCursorPosition(MouseCursorMonitor::CursorState state,
const DesktopVector& position) OVERRIDE {
state_ = state;
position_ = position;
position_received_ = true;
}
protected:
scoped_ptr<MouseCursor> cursor_image_;
MouseCursorMonitor::CursorState state_;
DesktopVector position_;
bool position_received_;
};
// TODO(sergeyu): Enable tests on all platforms.
#if defined(USE_X11)
#define MAYBE(x) x
#else
#define MAYBE(x) DISABLED_##x
#endif
TEST_F(MouseCursorMonitorTest, MAYBE(FromScreen)) {
scoped_ptr<MouseCursorMonitor> capturer(MouseCursorMonitor::CreateForScreen(
DesktopCaptureOptions::CreateDefault()));
assert(capturer.get());
capturer->Init(this, MouseCursorMonitor::SHAPE_AND_POSITION);
capturer->Capture();
EXPECT_TRUE(cursor_image_.get());
EXPECT_GE(cursor_image_->hotspot().x(), 0);
EXPECT_LE(cursor_image_->hotspot().x(),
cursor_image_->image().size().width());
EXPECT_GE(cursor_image_->hotspot().y(), 0);
EXPECT_LE(cursor_image_->hotspot().y(),
cursor_image_->image().size().height());
EXPECT_TRUE(position_received_);
EXPECT_EQ(MouseCursorMonitor::INSIDE, state_);
}
TEST_F(MouseCursorMonitorTest, MAYBE(FromWindow)) {
DesktopCaptureOptions options = DesktopCaptureOptions::CreateDefault();
// First get list of windows.
scoped_ptr<WindowCapturer> window_capturer(WindowCapturer::Create(options));
// If window capturing is not supported then skip this test.
if (!window_capturer.get())
return;
WindowCapturer::WindowList windows;
EXPECT_TRUE(window_capturer->GetWindowList(&windows));
// Iterate over all windows and try capturing mouse cursor for each of them.
for (size_t i = 0; i < windows.size(); ++i) {
cursor_image_.reset();
position_received_ = false;
scoped_ptr<MouseCursorMonitor> capturer(
MouseCursorMonitor::CreateForWindow(
DesktopCaptureOptions::CreateDefault(), windows[i].id));
assert(capturer.get());
capturer->Init(this, MouseCursorMonitor::SHAPE_AND_POSITION);
capturer->Capture();
EXPECT_TRUE(cursor_image_.get());
EXPECT_TRUE(position_received_);
}
}
// Make sure that OnMouseCursorPosition() is not called in the SHAPE_ONLY mode.
TEST_F(MouseCursorMonitorTest, MAYBE(ShapeOnly)) {
scoped_ptr<MouseCursorMonitor> capturer(MouseCursorMonitor::CreateForScreen(
DesktopCaptureOptions::CreateDefault()));
assert(capturer.get());
capturer->Init(this, MouseCursorMonitor::SHAPE_ONLY);
capturer->Capture();
EXPECT_TRUE(cursor_image_.get());
EXPECT_FALSE(position_received_);
}
} // namespace webrtc

View File

@ -0,0 +1,28 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
#include <cstddef>
namespace webrtc {
// TODO(sergeyu): Implement MouseCursorMonitor for Windows.
MouseCursorMonitor* MouseCursorMonitor::CreateForWindow(
const DesktopCaptureOptions& options, WindowId window) {
return NULL;
}
MouseCursorMonitor* MouseCursorMonitor::CreateForScreen(
const DesktopCaptureOptions& options) {
return NULL;
}
} // namespace webrtc

View File

@ -0,0 +1,224 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
#include <X11/extensions/Xfixes.h>
#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include "webrtc/modules/desktop_capture/desktop_capture_options.h"
#include "webrtc/modules/desktop_capture/desktop_frame.h"
#include "webrtc/modules/desktop_capture/mouse_cursor.h"
#include "webrtc/system_wrappers/interface/logging.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
namespace {
// WindowCapturer returns window IDs of X11 windows with WM_STATE attribute.
// These windows may not be immediate children of the root window, because
// window managers may re-parent them to add decorations. However,
// XQueryPointer() expects to be passed children of the root. This function
// searches up the list of the windows to find the root child that corresponds
// to |window|.
Window GetTopLevelWindow(Display* display, Window window) {
while (true) {
// If the window is in WithdrawnState then look at all of its children.
::Window root, parent;
::Window *children;
unsigned int num_children;
if (!XQueryTree(display, window, &root, &parent, &children,
&num_children)) {
LOG(LS_ERROR) << "Failed to query for child windows although window"
<< "does not have a valid WM_STATE.";
return None;
}
if (children)
XFree(children);
if (parent == root)
break;
window = parent;
}
return window;
}
} // namespace
namespace webrtc {
class MouseCursorMonitorX11 : public MouseCursorMonitor,
public SharedXDisplay::XEventHandler {
public:
MouseCursorMonitorX11(const DesktopCaptureOptions& options, Window window);
virtual ~MouseCursorMonitorX11();
virtual void Init(Callback* callback, Mode mode) OVERRIDE;
virtual void Capture() OVERRIDE;
private:
// SharedXDisplay::XEventHandler interface.
virtual bool HandleXEvent(const XEvent& event) OVERRIDE;
Display* display() { return x_display_->display(); }
// Captures current cursor shape and stores it in |cursor_shape_|.
void CaptureCursor();
scoped_refptr<SharedXDisplay> x_display_;
Callback* callback_;
Mode mode_;
Window window_;
bool have_xfixes_;
int xfixes_event_base_;
int xfixes_error_base_;
scoped_ptr<MouseCursor> cursor_shape_;
};
MouseCursorMonitorX11::MouseCursorMonitorX11(
const DesktopCaptureOptions& options,
Window window)
: x_display_(options.x_display()),
callback_(NULL),
mode_(SHAPE_AND_POSITION),
window_(window),
have_xfixes_(false),
xfixes_event_base_(-1),
xfixes_error_base_(-1) {}
MouseCursorMonitorX11::~MouseCursorMonitorX11() {
if (have_xfixes_) {
x_display_->RemoveEventHandler(xfixes_event_base_ + XFixesCursorNotify,
this);
}
}
void MouseCursorMonitorX11::Init(Callback* callback, Mode mode) {
// Init can be called only once per instance of MouseCursorMonitor.
assert(!callback_);
assert(callback);
callback_ = callback;
mode_ = mode;
have_xfixes_ =
XFixesQueryExtension(display(), &xfixes_event_base_, &xfixes_error_base_);
if (have_xfixes_) {
// Register for changes to the cursor shape.
XFixesSelectCursorInput(display(), window_, XFixesDisplayCursorNotifyMask);
x_display_->AddEventHandler(xfixes_event_base_ + XFixesCursorNotify, this);
CaptureCursor();
} else {
LOG(LS_INFO) << "X server does not support XFixes.";
}
}
void MouseCursorMonitorX11::Capture() {
assert(callback_);
// Process X11 events in case XFixes has sent cursor notification.
x_display_->ProcessPendingXEvents();
// cursor_shape_| is set only if we were notified of a cursor shape change.
if (cursor_shape_.get())
callback_->OnMouseCursor(cursor_shape_.release());
// Get cursor position if necessary.
if (mode_ == SHAPE_AND_POSITION) {
int root_x;
int root_y;
int win_x;
int win_y;
Window root_window;
Window child_window;
unsigned int mask;
Bool result = XQueryPointer(display(), window_, &root_window, &child_window,
&root_x, &root_y, &win_x, &win_y, &mask);
CursorState state;
if (!result) {
state = OUTSIDE;
} else {
// In screen mode (window_ == root_window) the mouse is always inside.
// XQueryPointer() sets |child_window| to None if the cursor is outside
// |window_|.
state =
(window_ == root_window || child_window != None) ? INSIDE : OUTSIDE;
}
callback_->OnMouseCursorPosition(state,
webrtc::DesktopVector(win_x, win_y));
}
}
bool MouseCursorMonitorX11::HandleXEvent(const XEvent& event) {
if (have_xfixes_ && event.type == xfixes_event_base_ + XFixesCursorNotify) {
const XFixesCursorNotifyEvent* cursor_event =
reinterpret_cast<const XFixesCursorNotifyEvent*>(&event);
if (cursor_event->subtype == XFixesDisplayCursorNotify) {
CaptureCursor();
}
// Return false, even if the event has been handled, because there might be
// other listeners for cursor notifications.
}
return false;
}
void MouseCursorMonitorX11::CaptureCursor() {
assert(have_xfixes_);
XFixesCursorImage* img = XFixesGetCursorImage(display());
if (!img)
return;
scoped_ptr<DesktopFrame> image(
new BasicDesktopFrame(DesktopSize(img->width, img->height)));
// Xlib stores 32-bit data in longs, even if longs are 64-bits long.
unsigned long* src = img->pixels;
uint32_t* dst = reinterpret_cast<uint32_t*>(image->data());
uint32_t* dst_end = dst + (img->width * img->height);
while (dst < dst_end) {
*dst++ = static_cast<uint32_t>(*src++);
}
DesktopVector hotspot(std::min(img->width, img->xhot),
std::min(img->height, img->yhot));
XFree(img);
cursor_shape_.reset(new MouseCursor(image.release(), hotspot));
}
// static
MouseCursorMonitor* MouseCursorMonitor::CreateForWindow(
const DesktopCaptureOptions& options, WindowId window) {
if (!options.x_display())
return NULL;
window = GetTopLevelWindow(options.x_display()->display(), window);
if (window == None)
return NULL;
return new MouseCursorMonitorX11(options, window);
}
MouseCursorMonitor* MouseCursorMonitor::CreateForScreen(
const DesktopCaptureOptions& options) {
if (!options.x_display())
return NULL;
return new MouseCursorMonitorX11(
options, DefaultRootWindow(options.x_display()->display()));
}
} // namespace webrtc

View File

@ -18,6 +18,8 @@
namespace webrtc {
// Type used to return mouse cursor shape from video capturers.
//
// TODO(sergeyu): Remove this type and use MouseCursor instead.
struct MouseCursorShape {
// Size of the cursor in screen pixels.
DesktopSize size;

View File

@ -896,7 +896,7 @@ void ScreenCapturerMac::DisplaysReconfiguredCallback(
} // namespace
// static
ScreenCapturer* ScreenCapturer::Create(const DesktopCaptureOptions& context) {
ScreenCapturer* ScreenCapturer::Create(const DesktopCaptureOptions& options) {
scoped_ptr<ScreenCapturerMac> capturer(new ScreenCapturerMac());
if (!capturer->Init())
capturer.reset();

View File

@ -14,6 +14,7 @@
#include <vector>
#include <string>
#include "webrtc/modules/desktop_capture/desktop_capture_types.h"
#include "webrtc/modules/desktop_capture/desktop_capturer.h"
#include "webrtc/system_wrappers/interface/constructor_magic.h"
#include "webrtc/typedefs.h"
@ -24,7 +25,7 @@ class DesktopCaptureOptions;
class WindowCapturer : public DesktopCapturer {
public:
typedef intptr_t WindowId;
typedef webrtc::WindowId WindowId;
struct Window {
WindowId id;

View File

@ -156,6 +156,7 @@
'audio_processing/utility/delay_estimator_unittest.cc',
'audio_processing/utility/ring_buffer_unittest.cc',
'bitrate_controller/bitrate_controller_unittest.cc',
'desktop_capture/mouse_cursor_monitor_unittest.cc',
'desktop_capture/desktop_region_unittest.cc',
'desktop_capture/differ_block_unittest.cc',
'desktop_capture/differ_unittest.cc',
@ -237,6 +238,7 @@
# supported.
['desktop_capture_supported==0', {
'sources!': [
'desktop_capture/mouse_cursor_monitor_unittest.cc',
'desktop_capture/screen_capturer_helper_unittest.cc',
'desktop_capture/screen_capturer_mac_unittest.cc',
'desktop_capture/screen_capturer_mock_objects.h',