Update to the peerconnection sample app.

* Fixes bug where remote video wasn't renderered.


* Update the Conductor class in accordance to the latest changes in the API.
  We now process the stream add/remove callbacks asynchronously.

* When a remote peer connects to us, we now call AddStream for our local streams
  to share with the peer if we haven't already done so.  To do that, we maintain
  a set of streams we have already shared.

BUG=11
Review URL: http://webrtc-codereview.appspot.com/131011

git-svn-id: http://webrtc.googlecode.com/svn/trunk@506 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
tommi@webrtc.org 2011-09-01 08:37:05 +00:00
parent 84519ec0a2
commit c6e54a97a7
5 changed files with 165 additions and 130 deletions

View File

@ -15,10 +15,18 @@
#include "talk/p2p/client/basicportallocator.h"
#include "talk/session/phone/videorendererfactory.h"
namespace {
// Used when passing stream information from callback threads to the UI thread.
struct StreamInfo {
StreamInfo(const std::string& id, bool video) : id_(id), video_(video) {}
std::string id_;
bool video_;
};
} // end anonymous.
Conductor::Conductor(PeerConnectionClient* client, MainWindow* main_wnd)
: waiting_for_audio_(false),
waiting_for_video_(false),
peer_id_(-1),
: peer_id_(-1),
client_(client),
main_wnd_(main_wnd) {
client_->RegisterObserver(this);
@ -89,11 +97,8 @@ bool Conductor::InitializePeerConnection() {
void Conductor::DeletePeerConnection() {
peer_connection_.reset();
worker_thread_.reset();
video_channel_.clear();
audio_channel_.clear();
active_streams_.clear();
peer_connection_factory_.reset();
waiting_for_audio_ = false;
waiting_for_video_ = false;
peer_id_ = -1;
}
@ -105,8 +110,6 @@ void Conductor::StartCaptureDevice() {
if (peer_connection_->SetVideoCapture("")) {
peer_connection_->SetLocalVideoRenderer(main_wnd_->local_renderer());
} else {
ASSERT(false);
}
}
}
@ -131,41 +134,15 @@ void Conductor::OnSignalingMessage(const std::string& msg) {
void Conductor::OnAddStream(const std::string& stream_id, bool video) {
LOG(INFO) << __FUNCTION__ << " " << stream_id;
if (video) {
// ASSERT(video_channel_.empty());
video_channel_ = stream_id;
waiting_for_video_ = false;
LOG(INFO) << "Setting video renderer for stream: " << stream_id;
bool ok = peer_connection_->SetVideoRenderer(stream_id,
main_wnd_->remote_renderer());
if (!ok)
LOG(LS_ERROR) << "SetVideoRenderer failed for : " << stream_id;
} else {
// ASSERT(audio_channel_.empty());
audio_channel_ = stream_id;
waiting_for_audio_ = false;
}
if (!waiting_for_video_)
main_wnd_->QueueUIThreadCallback(MEDIA_CHANNELS_INITIALIZED, NULL);
main_wnd_->QueueUIThreadCallback(NEW_STREAM_ADDED,
new StreamInfo(stream_id, video));
}
void Conductor::OnRemoveStream(const std::string& stream_id, bool video) {
LOG(INFO) << __FUNCTION__ << (video ? " video: " : " audio: ") << stream_id;
if (video) {
video_channel_.clear();
} else {
audio_channel_.clear();
}
if (video_channel_.empty() && audio_channel_.empty()) {
LOG(INFO) << "All streams have been closed.";
main_wnd_->QueueUIThreadCallback(PEER_CONNECTION_CLOSED, NULL);
} else {
LOG(INFO) << "Remaining streams: '" << video_channel_ << "', '"
<< audio_channel_ << "'";
}
main_wnd_->QueueUIThreadCallback(STREAM_REMOVED,
new StreamInfo(stream_id, video));
}
//
@ -221,8 +198,6 @@ void Conductor::OnMessageFromPeer(int peer_id, const std::string& message) {
LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
client_->SignOut();
return;
} else {
StartCaptureDevice();
}
} else if (peer_id != peer_id_) {
ASSERT(peer_id_ != -1);
@ -274,24 +249,38 @@ void Conductor::ConnectToPeer(int peer_id) {
if (InitializePeerConnection()) {
peer_id_ = peer_id;
main_wnd_->SwitchToStreamingUI();
StartCaptureDevice();
AddStreams();
} else {
main_wnd_->MessageBox("Error", "Failed to initialize PeerConnection", true);
}
}
bool Conductor::AddStream(const std::string& id, bool video) {
// NOTE: Must be called from the UI thread.
if (active_streams_.find(id) != active_streams_.end())
return false; // Already added.
active_streams_.insert(id);
bool ret = peer_connection_->AddStream(id, video);
if (!ret) {
active_streams_.erase(id);
} else if (video) {
LOG(INFO) << "Setting video renderer for stream: " << id;
bool ok = peer_connection_->SetVideoRenderer(id,
main_wnd_->remote_renderer());
ASSERT(ok);
}
return ret;
}
void Conductor::AddStreams() {
ASSERT(!waiting_for_video_);
ASSERT(!waiting_for_audio_);
int streams = 0;
if (AddStream(kVideoLabel, true))
++streams;
waiting_for_video_ = true;
waiting_for_audio_ = true;
if (!peer_connection_->AddStream(kVideoLabel, true))
waiting_for_video_ = false;
if (!peer_connection_->AddStream(kAudioLabel, false))
waiting_for_audio_ = false;
if (AddStream(kAudioLabel, false))
++streams;
// At the initiator of the call, after adding streams we need
// kick start the ICE candidates discovery process, which
@ -299,7 +288,7 @@ void Conductor::AddStreams() {
// getting the OnLocalStreamInitialized callback which is removed
// now. Connect will trigger OnSignalingMessage callback when
// ICE candidates are available.
if (waiting_for_audio_ || waiting_for_video_)
if (streams)
peer_connection_->Connect();
}
@ -315,54 +304,95 @@ void Conductor::DisconnectFromCurrentPeer() {
}
void Conductor::UIThreadCallback(int msg_id, void* data) {
if (msg_id == MEDIA_CHANNELS_INITIALIZED) {
StartCaptureDevice();
// When we get an OnSignalingMessage notification, we'll send our
// json encoded signaling message to the peer, which is the first step
// of establishing a connection.
} else if (msg_id == PEER_CONNECTION_CLOSED) {
LOG(INFO) << "PEER_CONNECTION_CLOSED";
DeletePeerConnection();
switch (msg_id) {
case PEER_CONNECTION_CLOSED:
LOG(INFO) << "PEER_CONNECTION_CLOSED";
DeletePeerConnection();
waiting_for_audio_ = false;
waiting_for_video_ = false;
ASSERT(video_channel_.empty());
ASSERT(audio_channel_.empty());
if (main_wnd_->IsWindow()) {
if (client_->is_connected()) {
main_wnd_->SwitchToPeerList(client_->peers());
} else {
main_wnd_->SwitchToConnectUI();
}
} else {
DisconnectFromServer();
}
} else if (msg_id == SEND_MESSAGE_TO_PEER) {
LOG(INFO) << "SEND_MESSAGE_TO_PEER";
std::string* msg = reinterpret_cast<std::string*>(data);
if (client_->IsSendingMessage()) {
ASSERT(msg != NULL);
pending_messages_.push_back(msg);
} else {
if (!msg && !pending_messages_.empty()) {
msg = pending_messages_.front();
pending_messages_.pop_front();
}
if (msg) {
bool ok = client_->SendToPeer(peer_id_, *msg);
if (!ok && peer_id_ != -1) {
LOG(LS_ERROR) << "SendToPeer failed";
DisconnectFromServer();
ASSERT(active_streams_.empty());
if (main_wnd_->IsWindow()) {
if (client_->is_connected()) {
main_wnd_->SwitchToPeerList(client_->peers());
} else {
main_wnd_->SwitchToConnectUI();
}
delete msg;
} else {
DisconnectFromServer();
}
break;
case SEND_MESSAGE_TO_PEER: {
LOG(INFO) << "SEND_MESSAGE_TO_PEER";
std::string* msg = reinterpret_cast<std::string*>(data);
if (client_->IsSendingMessage()) {
ASSERT(msg != NULL);
pending_messages_.push_back(msg);
} else {
if (!msg && !pending_messages_.empty()) {
msg = pending_messages_.front();
pending_messages_.pop_front();
}
if (msg) {
bool ok = client_->SendToPeer(peer_id_, *msg);
if (!ok && peer_id_ != -1) {
LOG(LS_ERROR) << "SendToPeer failed";
DisconnectFromServer();
}
delete msg;
}
if (!peer_connection_.get())
peer_id_ = -1;
}
break;
}
case PEER_CONNECTION_ADDSTREAMS:
AddStreams();
break;
case PEER_CONNECTION_ERROR:
main_wnd_->MessageBox("Error", "an unknown error occurred", true);
break;
case NEW_STREAM_ADDED: {
talk_base::scoped_ptr<StreamInfo> info(
reinterpret_cast<StreamInfo*>(data));
if (info->video_) {
LOG(INFO) << "Setting video renderer for stream: " << info->id_;
bool ok = peer_connection_->SetVideoRenderer(info->id_,
main_wnd_->remote_renderer());
ASSERT(ok);
if (!ok)
LOG(LS_ERROR) << "SetVideoRenderer failed for : " << info->id_;
// TODO(tommi): For the initiator, we shouldn't have to make this call
// here (which is actually the second time this is called for the
// initiator). Look into why this is needed.
StartCaptureDevice();
}
if (!peer_connection_.get())
peer_id_ = -1;
// If we haven't shared any streams with this peer (we're the receiver)
// then do so now.
if (active_streams_.empty())
AddStreams();
break;
}
} else if (msg_id == PEER_CONNECTION_ADDSTREAMS) {
AddStreams();
} else if (msg_id == PEER_CONNECTION_ERROR) {
main_wnd_->MessageBox("Error", "an unknown error occurred", true);
case STREAM_REMOVED: {
talk_base::scoped_ptr<StreamInfo> info(
reinterpret_cast<StreamInfo*>(data));
active_streams_.erase(info->id_);
if (active_streams_.empty()) {
LOG(INFO) << "All streams have been closed.";
main_wnd_->QueueUIThreadCallback(PEER_CONNECTION_CLOSED, NULL);
}
break;
}
default:
ASSERT(false);
break;
}
}

View File

@ -13,6 +13,7 @@
#pragma once
#include <deque>
#include <set>
#include <string>
#include "peerconnection/samples/client/main_wnd.h"
@ -36,6 +37,8 @@ class Conductor
SEND_MESSAGE_TO_PEER,
PEER_CONNECTION_ADDSTREAMS,
PEER_CONNECTION_ERROR,
NEW_STREAM_ADDED,
STREAM_REMOVED,
};
Conductor(PeerConnectionClient* client, MainWindow* main_wnd);
@ -49,6 +52,7 @@ class Conductor
bool InitializePeerConnection();
void DeletePeerConnection();
void StartCaptureDevice();
bool AddStream(const std::string& id, bool video);
void AddStreams();
//
@ -94,17 +98,14 @@ class Conductor
virtual void UIThreadCallback(int msg_id, void* data);
protected:
bool waiting_for_audio_;
bool waiting_for_video_;
int peer_id_;
talk_base::scoped_ptr<webrtc::PeerConnection> peer_connection_;
talk_base::scoped_ptr<webrtc::PeerConnectionFactory> peer_connection_factory_;
talk_base::scoped_ptr<talk_base::Thread> worker_thread_;
PeerConnectionClient* client_;
MainWindow* main_wnd_;
std::string video_channel_;
std::string audio_channel_;
std::deque<std::string*> pending_messages_;
std::set<std::string> active_streams_;
};
#endif // PEERCONNECTION_SAMPLES_CLIENT_CONDUCTOR_H_

View File

@ -91,7 +91,6 @@ gboolean Redraw(gpointer data) {
wnd->OnRedraw();
return false;
}
} // end anonymous
//
@ -126,8 +125,8 @@ void GtkMainWnd::MessageBox(const char* caption, const char* text,
is_error ? GTK_MESSAGE_ERROR : GTK_MESSAGE_INFO,
GTK_BUTTONS_CLOSE, "%s", text);
gtk_window_set_title(GTK_WINDOW(dialog), caption);
gtk_dialog_run(GTK_DIALOG (dialog));
gtk_widget_destroy (dialog);
gtk_dialog_run(GTK_DIALOG(dialog));
gtk_widget_destroy(dialog);
}
MainWindow::UI GtkMainWnd::current_ui() {
@ -141,12 +140,14 @@ MainWindow::UI GtkMainWnd::current_ui() {
}
cricket::VideoRenderer* GtkMainWnd::local_renderer() {
ASSERT(local_renderer_.get() != NULL);
if (!local_renderer_.get())
local_renderer_.reset(new VideoRenderer(this));
return local_renderer_.get();
}
cricket::VideoRenderer* GtkMainWnd::remote_renderer() {
ASSERT(remote_renderer_.get() != NULL);
if (!remote_renderer_.get())
remote_renderer_.reset(new VideoRenderer(this));
return remote_renderer_.get();
}
@ -272,12 +273,6 @@ void GtkMainWnd::SwitchToStreamingUI() {
ASSERT(draw_area_ == NULL);
// Prepare new buffers for the new conversation. We don't
// reuse buffers across sessions to avoid possibly rendering
// a frame from the previous conversation.
remote_renderer_.reset(new VideoRenderer(this));
local_renderer_.reset(new VideoRenderer(this));
gtk_container_set_border_width(GTK_CONTAINER(window_), 0);
if (peer_list_) {
gtk_widget_destroy(peer_list_);
@ -348,7 +343,7 @@ void GtkMainWnd::OnRowActivated(GtkTreeView* tree_view, GtkTreePath* path,
if (id != -1)
callback_->ConnectToPeer(id);
g_free(text);
}
}
}
void GtkMainWnd::OnRedraw() {
@ -381,7 +376,7 @@ void GtkMainWnd::OnRedraw() {
image += width;
scaled += width * 2;
}
image = reinterpret_cast<const uint32*>(local_renderer_->image());
scaled = reinterpret_cast<uint32*>(draw_buffer_.get());
// Position the local preview on the right side.
@ -389,7 +384,7 @@ void GtkMainWnd::OnRedraw() {
// right margin...
scaled -= 10;
// ... towards the bottom.
scaled += (height * width * 4) -
scaled += (height * width * 4) -
((local_renderer_->height() / 2) *
(local_renderer_->width() / 2) * 4);
// bottom margin...
@ -415,7 +410,7 @@ void GtkMainWnd::OnRedraw() {
gdk_threads_leave();
}
GtkMainWnd::VideoRenderer::VideoRenderer(GtkMainWnd* main_wnd)
GtkMainWnd::VideoRenderer::VideoRenderer(GtkMainWnd* main_wnd)
: width_(0), height_(0), main_wnd_(main_wnd) {
}

View File

@ -146,6 +146,9 @@ void MainWnd::SwitchToConnectUI() {
}
void MainWnd::SwitchToPeerList(const Peers& peers) {
remote_video_.reset();
local_video_.reset();
LayoutConnectUI(false);
::SendMessage(listbox_, LB_RESETCONTENT, 0, 0);
@ -161,9 +164,6 @@ void MainWnd::SwitchToPeerList(const Peers& peers) {
}
void MainWnd::SwitchToStreamingUI() {
remote_video_.reset(new VideoRenderer(handle(), 1, 1));
local_video_.reset(new VideoRenderer(handle(), 1, 1));
LayoutConnectUI(false);
LayoutPeerListUI(false);
ui_ = STREAMING;
@ -177,6 +177,18 @@ void MainWnd::MessageBox(const char* caption, const char* text, bool is_error) {
::MessageBoxA(handle(), text, caption, flags);
}
cricket::VideoRenderer* MainWnd::local_renderer() {
if (!local_video_.get())
local_video_.reset(new VideoRenderer(handle(), 1, 1));
return local_video_.get();
}
cricket::VideoRenderer* MainWnd::remote_renderer() {
if (!remote_video_.get())
remote_video_.reset(new VideoRenderer(handle(), 1, 1));
return remote_video_.get();
}
void MainWnd::QueueUIThreadCallback(int msg_id, void* data) {
::PostThreadMessage(ui_thread_id_, UI_THREAD_CALLBACK,
static_cast<WPARAM>(msg_id), reinterpret_cast<LPARAM>(data));
@ -194,8 +206,8 @@ void MainWnd::OnPaint() {
AutoLock<VideoRenderer> remote_lock(remote_video_.get());
const BITMAPINFO& bmi = remote_video_->bmi();
long height = abs(bmi.bmiHeader.biHeight);
long width = bmi.bmiHeader.biWidth;
int height = abs(bmi.bmiHeader.biHeight);
int width = bmi.bmiHeader.biWidth;
const uint8* image = remote_video_->image();
if (image != NULL) {
@ -231,8 +243,8 @@ void MainWnd::OnPaint() {
if ((rc.right - rc.left) > 200 && (rc.bottom - rc.top) > 200) {
const BITMAPINFO& bmi = local_video_->bmi();
image = local_video_->image();
long thumb_width = bmi.bmiHeader.biWidth / 4;
long thumb_height = abs(bmi.bmiHeader.biHeight) / 4;
int thumb_width = bmi.bmiHeader.biWidth / 4;
int thumb_height = abs(bmi.bmiHeader.biHeight) / 4;
StretchDIBits(dc_mem,
logical_area.x - thumb_width - 10,
logical_area.y - thumb_height - 10,

View File

@ -35,6 +35,8 @@ class MainWndCallback {
// Pure virtual interface for the main window.
class MainWindow {
public:
virtual ~MainWindow() {}
enum UI {
CONNECT_TO_SERVER,
LIST_PEERS,
@ -85,13 +87,8 @@ class MainWnd : public MainWindow {
bool is_error);
virtual UI current_ui() { return ui_; }
virtual cricket::VideoRenderer* local_renderer() {
return local_video_.get();
}
virtual cricket::VideoRenderer* remote_renderer() {
return remote_video_.get();
}
virtual cricket::VideoRenderer* local_renderer();
virtual cricket::VideoRenderer* remote_renderer();
virtual void QueueUIThreadCallback(int msg_id, void* data);
@ -135,7 +132,7 @@ class MainWnd : public MainWindow {
template <typename T>
class AutoLock {
public:
AutoLock(T* obj) : obj_(obj) { obj_->Lock(); }
explicit AutoLock(T* obj) : obj_(obj) { obj_->Lock(); }
~AutoLock() { obj_->Unlock(); }
protected:
T* obj_;