1 /*
2  *  Copyright 2012 The WebRTC Project Authors. All rights reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "examples/peerconnection/client/conductor.h"
12 
13 #include <stddef.h>
14 #include <stdint.h>
15 
16 #include <memory>
17 #include <utility>
18 #include <vector>
19 
20 #include "absl/memory/memory.h"
21 #include "absl/types/optional.h"
22 #include "api/audio/audio_mixer.h"
23 #include "api/audio_codecs/audio_decoder_factory.h"
24 #include "api/audio_codecs/audio_encoder_factory.h"
25 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
26 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
27 #include "api/audio_options.h"
28 #include "api/create_peerconnection_factory.h"
29 #include "api/rtp_sender_interface.h"
30 #include "api/video_codecs/builtin_video_decoder_factory.h"
31 #include "api/video_codecs/builtin_video_encoder_factory.h"
32 #include "api/video_codecs/video_decoder_factory.h"
33 #include "api/video_codecs/video_encoder_factory.h"
34 #include "examples/peerconnection/client/defaults.h"
35 #include "modules/audio_device/include/audio_device.h"
36 #include "modules/audio_processing/include/audio_processing.h"
37 #include "modules/video_capture/video_capture.h"
38 #include "modules/video_capture/video_capture_factory.h"
39 #include "p2p/base/port_allocator.h"
40 #include "pc/video_track_source.h"
41 #include "rtc_base/checks.h"
42 #include "rtc_base/logging.h"
43 #include "rtc_base/ref_counted_object.h"
44 #include "rtc_base/rtc_certificate_generator.h"
45 #include "rtc_base/strings/json.h"
46 #include "test/vcm_capturer.h"
47 
48 namespace {
49 // Names used for a IceCandidate JSON object.
50 const char kCandidateSdpMidName[] = "sdpMid";
51 const char kCandidateSdpMlineIndexName[] = "sdpMLineIndex";
52 const char kCandidateSdpName[] = "candidate";
53 
54 // Names used for a SessionDescription JSON object.
55 const char kSessionDescriptionTypeName[] = "type";
56 const char kSessionDescriptionSdpName[] = "sdp";
57 
58 class DummySetSessionDescriptionObserver
59     : public webrtc::SetSessionDescriptionObserver {
60  public:
Create()61   static DummySetSessionDescriptionObserver* Create() {
62     return new rtc::RefCountedObject<DummySetSessionDescriptionObserver>();
63   }
OnSuccess()64   virtual void OnSuccess() { RTC_LOG(INFO) << __FUNCTION__; }
OnFailure(webrtc::RTCError error)65   virtual void OnFailure(webrtc::RTCError error) {
66     RTC_LOG(INFO) << __FUNCTION__ << " " << ToString(error.type()) << ": "
67                   << error.message();
68   }
69 };
70 
71 class CapturerTrackSource : public webrtc::VideoTrackSource {
72  public:
Create()73   static rtc::scoped_refptr<CapturerTrackSource> Create() {
74     const size_t kWidth = 640;
75     const size_t kHeight = 480;
76     const size_t kFps = 30;
77     std::unique_ptr<webrtc::test::VcmCapturer> capturer;
78     std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
79         webrtc::VideoCaptureFactory::CreateDeviceInfo());
80     if (!info) {
81       return nullptr;
82     }
83     int num_devices = info->NumberOfDevices();
84     for (int i = 0; i < num_devices; ++i) {
85       capturer = absl::WrapUnique(
86           webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, i));
87       if (capturer) {
88         return new rtc::RefCountedObject<CapturerTrackSource>(
89             std::move(capturer));
90       }
91     }
92 
93     return nullptr;
94   }
95 
96  protected:
CapturerTrackSource(std::unique_ptr<webrtc::test::VcmCapturer> capturer)97   explicit CapturerTrackSource(
98       std::unique_ptr<webrtc::test::VcmCapturer> capturer)
99       : VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}
100 
101  private:
source()102   rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
103     return capturer_.get();
104   }
105   std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
106 };
107 
108 }  // namespace
109 
Conductor(PeerConnectionClient * client,MainWindow * main_wnd)110 Conductor::Conductor(PeerConnectionClient* client, MainWindow* main_wnd)
111     : peer_id_(-1), loopback_(false), client_(client), main_wnd_(main_wnd) {
112   client_->RegisterObserver(this);
113   main_wnd->RegisterObserver(this);
114 }
115 
~Conductor()116 Conductor::~Conductor() {
117   RTC_DCHECK(!peer_connection_);
118 }
119 
connection_active() const120 bool Conductor::connection_active() const {
121   return peer_connection_ != nullptr;
122 }
123 
Close()124 void Conductor::Close() {
125   client_->SignOut();
126   DeletePeerConnection();
127 }
128 
InitializePeerConnection()129 bool Conductor::InitializePeerConnection() {
130   RTC_DCHECK(!peer_connection_factory_);
131   RTC_DCHECK(!peer_connection_);
132 
133   peer_connection_factory_ = webrtc::CreatePeerConnectionFactory(
134       nullptr /* network_thread */, nullptr /* worker_thread */,
135       nullptr /* signaling_thread */, nullptr /* default_adm */,
136       webrtc::CreateBuiltinAudioEncoderFactory(),
137       webrtc::CreateBuiltinAudioDecoderFactory(),
138       webrtc::CreateBuiltinVideoEncoderFactory(),
139       webrtc::CreateBuiltinVideoDecoderFactory(), nullptr /* audio_mixer */,
140       nullptr /* audio_processing */);
141 
142   if (!peer_connection_factory_) {
143     main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory",
144                           true);
145     DeletePeerConnection();
146     return false;
147   }
148 
149   if (!CreatePeerConnection(/*dtls=*/true)) {
150     main_wnd_->MessageBox("Error", "CreatePeerConnection failed", true);
151     DeletePeerConnection();
152   }
153 
154   AddTracks();
155 
156   return peer_connection_ != nullptr;
157 }
158 
ReinitializePeerConnectionForLoopback()159 bool Conductor::ReinitializePeerConnectionForLoopback() {
160   loopback_ = true;
161   std::vector<rtc::scoped_refptr<webrtc::RtpSenderInterface>> senders =
162       peer_connection_->GetSenders();
163   peer_connection_ = nullptr;
164   if (CreatePeerConnection(/*dtls=*/false)) {
165     for (const auto& sender : senders) {
166       peer_connection_->AddTrack(sender->track(), sender->stream_ids());
167     }
168     peer_connection_->CreateOffer(
169         this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
170   }
171   return peer_connection_ != nullptr;
172 }
173 
CreatePeerConnection(bool dtls)174 bool Conductor::CreatePeerConnection(bool dtls) {
175   RTC_DCHECK(peer_connection_factory_);
176   RTC_DCHECK(!peer_connection_);
177 
178   webrtc::PeerConnectionInterface::RTCConfiguration config;
179   config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
180   config.enable_dtls_srtp = dtls;
181   webrtc::PeerConnectionInterface::IceServer server;
182   server.uri = GetPeerConnectionString();
183   config.servers.push_back(server);
184 
185   peer_connection_ = peer_connection_factory_->CreatePeerConnection(
186       config, nullptr, nullptr, this);
187   return peer_connection_ != nullptr;
188 }
189 
DeletePeerConnection()190 void Conductor::DeletePeerConnection() {
191   main_wnd_->StopLocalRenderer();
192   main_wnd_->StopRemoteRenderer();
193   peer_connection_ = nullptr;
194   peer_connection_factory_ = nullptr;
195   peer_id_ = -1;
196   loopback_ = false;
197 }
198 
EnsureStreamingUI()199 void Conductor::EnsureStreamingUI() {
200   RTC_DCHECK(peer_connection_);
201   if (main_wnd_->IsWindow()) {
202     if (main_wnd_->current_ui() != MainWindow::STREAMING)
203       main_wnd_->SwitchToStreamingUI();
204   }
205 }
206 
207 //
208 // PeerConnectionObserver implementation.
209 //
210 
OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> & streams)211 void Conductor::OnAddTrack(
212     rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
213     const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>>&
214         streams) {
215   RTC_LOG(INFO) << __FUNCTION__ << " " << receiver->id();
216   main_wnd_->QueueUIThreadCallback(NEW_TRACK_ADDED,
217                                    receiver->track().release());
218 }
219 
OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver)220 void Conductor::OnRemoveTrack(
221     rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) {
222   RTC_LOG(INFO) << __FUNCTION__ << " " << receiver->id();
223   main_wnd_->QueueUIThreadCallback(TRACK_REMOVED, receiver->track().release());
224 }
225 
OnIceCandidate(const webrtc::IceCandidateInterface * candidate)226 void Conductor::OnIceCandidate(const webrtc::IceCandidateInterface* candidate) {
227   RTC_LOG(INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
228   // For loopback test. To save some connecting delay.
229   if (loopback_) {
230     if (!peer_connection_->AddIceCandidate(candidate)) {
231       RTC_LOG(WARNING) << "Failed to apply the received candidate";
232     }
233     return;
234   }
235 
236   Json::StyledWriter writer;
237   Json::Value jmessage;
238 
239   jmessage[kCandidateSdpMidName] = candidate->sdp_mid();
240   jmessage[kCandidateSdpMlineIndexName] = candidate->sdp_mline_index();
241   std::string sdp;
242   if (!candidate->ToString(&sdp)) {
243     RTC_LOG(LS_ERROR) << "Failed to serialize candidate";
244     return;
245   }
246   jmessage[kCandidateSdpName] = sdp;
247   SendMessage(writer.write(jmessage));
248 }
249 
250 //
251 // PeerConnectionClientObserver implementation.
252 //
253 
OnSignedIn()254 void Conductor::OnSignedIn() {
255   RTC_LOG(INFO) << __FUNCTION__;
256   main_wnd_->SwitchToPeerList(client_->peers());
257 }
258 
OnDisconnected()259 void Conductor::OnDisconnected() {
260   RTC_LOG(INFO) << __FUNCTION__;
261 
262   DeletePeerConnection();
263 
264   if (main_wnd_->IsWindow())
265     main_wnd_->SwitchToConnectUI();
266 }
267 
OnPeerConnected(int id,const std::string & name)268 void Conductor::OnPeerConnected(int id, const std::string& name) {
269   RTC_LOG(INFO) << __FUNCTION__;
270   // Refresh the list if we're showing it.
271   if (main_wnd_->current_ui() == MainWindow::LIST_PEERS)
272     main_wnd_->SwitchToPeerList(client_->peers());
273 }
274 
OnPeerDisconnected(int id)275 void Conductor::OnPeerDisconnected(int id) {
276   RTC_LOG(INFO) << __FUNCTION__;
277   if (id == peer_id_) {
278     RTC_LOG(INFO) << "Our peer disconnected";
279     main_wnd_->QueueUIThreadCallback(PEER_CONNECTION_CLOSED, NULL);
280   } else {
281     // Refresh the list if we're showing it.
282     if (main_wnd_->current_ui() == MainWindow::LIST_PEERS)
283       main_wnd_->SwitchToPeerList(client_->peers());
284   }
285 }
286 
OnMessageFromPeer(int peer_id,const std::string & message)287 void Conductor::OnMessageFromPeer(int peer_id, const std::string& message) {
288   RTC_DCHECK(peer_id_ == peer_id || peer_id_ == -1);
289   RTC_DCHECK(!message.empty());
290 
291   if (!peer_connection_.get()) {
292     RTC_DCHECK(peer_id_ == -1);
293     peer_id_ = peer_id;
294 
295     if (!InitializePeerConnection()) {
296       RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
297       client_->SignOut();
298       return;
299     }
300   } else if (peer_id != peer_id_) {
301     RTC_DCHECK(peer_id_ != -1);
302     RTC_LOG(WARNING)
303         << "Received a message from unknown peer while already in a "
304            "conversation with a different peer.";
305     return;
306   }
307 
308   Json::Reader reader;
309   Json::Value jmessage;
310   if (!reader.parse(message, jmessage)) {
311     RTC_LOG(WARNING) << "Received unknown message. " << message;
312     return;
313   }
314   std::string type_str;
315   std::string json_object;
316 
317   rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName,
318                                &type_str);
319   if (!type_str.empty()) {
320     if (type_str == "offer-loopback") {
321       // This is a loopback call.
322       // Recreate the peerconnection with DTLS disabled.
323       if (!ReinitializePeerConnectionForLoopback()) {
324         RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
325         DeletePeerConnection();
326         client_->SignOut();
327       }
328       return;
329     }
330     absl::optional<webrtc::SdpType> type_maybe =
331         webrtc::SdpTypeFromString(type_str);
332     if (!type_maybe) {
333       RTC_LOG(LS_ERROR) << "Unknown SDP type: " << type_str;
334       return;
335     }
336     webrtc::SdpType type = *type_maybe;
337     std::string sdp;
338     if (!rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionSdpName,
339                                       &sdp)) {
340       RTC_LOG(WARNING) << "Can't parse received session description message.";
341       return;
342     }
343     webrtc::SdpParseError error;
344     std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
345         webrtc::CreateSessionDescription(type, sdp, &error);
346     if (!session_description) {
347       RTC_LOG(WARNING) << "Can't parse received session description message. "
348                           "SdpParseError was: "
349                        << error.description;
350       return;
351     }
352     RTC_LOG(INFO) << " Received session description :" << message;
353     peer_connection_->SetRemoteDescription(
354         DummySetSessionDescriptionObserver::Create(),
355         session_description.release());
356     if (type == webrtc::SdpType::kOffer) {
357       peer_connection_->CreateAnswer(
358           this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
359     }
360   } else {
361     std::string sdp_mid;
362     int sdp_mlineindex = 0;
363     std::string sdp;
364     if (!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpMidName,
365                                       &sdp_mid) ||
366         !rtc::GetIntFromJsonObject(jmessage, kCandidateSdpMlineIndexName,
367                                    &sdp_mlineindex) ||
368         !rtc::GetStringFromJsonObject(jmessage, kCandidateSdpName, &sdp)) {
369       RTC_LOG(WARNING) << "Can't parse received message.";
370       return;
371     }
372     webrtc::SdpParseError error;
373     std::unique_ptr<webrtc::IceCandidateInterface> candidate(
374         webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error));
375     if (!candidate.get()) {
376       RTC_LOG(WARNING) << "Can't parse received candidate message. "
377                           "SdpParseError was: "
378                        << error.description;
379       return;
380     }
381     if (!peer_connection_->AddIceCandidate(candidate.get())) {
382       RTC_LOG(WARNING) << "Failed to apply the received candidate";
383       return;
384     }
385     RTC_LOG(INFO) << " Received candidate :" << message;
386   }
387 }
388 
OnMessageSent(int err)389 void Conductor::OnMessageSent(int err) {
390   // Process the next pending message if any.
391   main_wnd_->QueueUIThreadCallback(SEND_MESSAGE_TO_PEER, NULL);
392 }
393 
OnServerConnectionFailure()394 void Conductor::OnServerConnectionFailure() {
395   main_wnd_->MessageBox("Error", ("Failed to connect to " + server_).c_str(),
396                         true);
397 }
398 
399 //
400 // MainWndCallback implementation.
401 //
402 
StartLogin(const std::string & server,int port)403 void Conductor::StartLogin(const std::string& server, int port) {
404   if (client_->is_connected())
405     return;
406   server_ = server;
407   client_->Connect(server, port, GetPeerName());
408 }
409 
DisconnectFromServer()410 void Conductor::DisconnectFromServer() {
411   if (client_->is_connected())
412     client_->SignOut();
413 }
414 
ConnectToPeer(int peer_id)415 void Conductor::ConnectToPeer(int peer_id) {
416   RTC_DCHECK(peer_id_ == -1);
417   RTC_DCHECK(peer_id != -1);
418 
419   if (peer_connection_.get()) {
420     main_wnd_->MessageBox(
421         "Error", "We only support connecting to one peer at a time", true);
422     return;
423   }
424 
425   if (InitializePeerConnection()) {
426     peer_id_ = peer_id;
427     peer_connection_->CreateOffer(
428         this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
429   } else {
430     main_wnd_->MessageBox("Error", "Failed to initialize PeerConnection", true);
431   }
432 }
433 
AddTracks()434 void Conductor::AddTracks() {
435   if (!peer_connection_->GetSenders().empty()) {
436     return;  // Already added tracks.
437   }
438 
439   rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track(
440       peer_connection_factory_->CreateAudioTrack(
441           kAudioLabel, peer_connection_factory_->CreateAudioSource(
442                            cricket::AudioOptions())));
443   auto result_or_error = peer_connection_->AddTrack(audio_track, {kStreamId});
444   if (!result_or_error.ok()) {
445     RTC_LOG(LS_ERROR) << "Failed to add audio track to PeerConnection: "
446                       << result_or_error.error().message();
447   }
448 
449   rtc::scoped_refptr<CapturerTrackSource> video_device =
450       CapturerTrackSource::Create();
451   if (video_device) {
452     rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_(
453         peer_connection_factory_->CreateVideoTrack(kVideoLabel, video_device));
454     main_wnd_->StartLocalRenderer(video_track_);
455 
456     result_or_error = peer_connection_->AddTrack(video_track_, {kStreamId});
457     if (!result_or_error.ok()) {
458       RTC_LOG(LS_ERROR) << "Failed to add video track to PeerConnection: "
459                         << result_or_error.error().message();
460     }
461   } else {
462     RTC_LOG(LS_ERROR) << "OpenVideoCaptureDevice failed";
463   }
464 
465   main_wnd_->SwitchToStreamingUI();
466 }
467 
DisconnectFromCurrentPeer()468 void Conductor::DisconnectFromCurrentPeer() {
469   RTC_LOG(INFO) << __FUNCTION__;
470   if (peer_connection_.get()) {
471     client_->SendHangUp(peer_id_);
472     DeletePeerConnection();
473   }
474 
475   if (main_wnd_->IsWindow())
476     main_wnd_->SwitchToPeerList(client_->peers());
477 }
478 
UIThreadCallback(int msg_id,void * data)479 void Conductor::UIThreadCallback(int msg_id, void* data) {
480   switch (msg_id) {
481     case PEER_CONNECTION_CLOSED:
482       RTC_LOG(INFO) << "PEER_CONNECTION_CLOSED";
483       DeletePeerConnection();
484 
485       if (main_wnd_->IsWindow()) {
486         if (client_->is_connected()) {
487           main_wnd_->SwitchToPeerList(client_->peers());
488         } else {
489           main_wnd_->SwitchToConnectUI();
490         }
491       } else {
492         DisconnectFromServer();
493       }
494       break;
495 
496     case SEND_MESSAGE_TO_PEER: {
497       RTC_LOG(INFO) << "SEND_MESSAGE_TO_PEER";
498       std::string* msg = reinterpret_cast<std::string*>(data);
499       if (msg) {
500         // For convenience, we always run the message through the queue.
501         // This way we can be sure that messages are sent to the server
502         // in the same order they were signaled without much hassle.
503         pending_messages_.push_back(msg);
504       }
505 
506       if (!pending_messages_.empty() && !client_->IsSendingMessage()) {
507         msg = pending_messages_.front();
508         pending_messages_.pop_front();
509 
510         if (!client_->SendToPeer(peer_id_, *msg) && peer_id_ != -1) {
511           RTC_LOG(LS_ERROR) << "SendToPeer failed";
512           DisconnectFromServer();
513         }
514         delete msg;
515       }
516 
517       if (!peer_connection_.get())
518         peer_id_ = -1;
519 
520       break;
521     }
522 
523     case NEW_TRACK_ADDED: {
524       auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
525       if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
526         auto* video_track = static_cast<webrtc::VideoTrackInterface*>(track);
527         main_wnd_->StartRemoteRenderer(video_track);
528       }
529       track->Release();
530       break;
531     }
532 
533     case TRACK_REMOVED: {
534       // Remote peer stopped sending a track.
535       auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
536       track->Release();
537       break;
538     }
539 
540     default:
541       RTC_NOTREACHED();
542       break;
543   }
544 }
545 
OnSuccess(webrtc::SessionDescriptionInterface * desc)546 void Conductor::OnSuccess(webrtc::SessionDescriptionInterface* desc) {
547   peer_connection_->SetLocalDescription(
548       DummySetSessionDescriptionObserver::Create(), desc);
549 
550   std::string sdp;
551   desc->ToString(&sdp);
552 
553   // For loopback test. To save some connecting delay.
554   if (loopback_) {
555     // Replace message type from "offer" to "answer"
556     std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
557         webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp);
558     peer_connection_->SetRemoteDescription(
559         DummySetSessionDescriptionObserver::Create(),
560         session_description.release());
561     return;
562   }
563 
564   Json::StyledWriter writer;
565   Json::Value jmessage;
566   jmessage[kSessionDescriptionTypeName] =
567       webrtc::SdpTypeToString(desc->GetType());
568   jmessage[kSessionDescriptionSdpName] = sdp;
569   SendMessage(writer.write(jmessage));
570 }
571 
OnFailure(webrtc::RTCError error)572 void Conductor::OnFailure(webrtc::RTCError error) {
573   RTC_LOG(LERROR) << ToString(error.type()) << ": " << error.message();
574 }
575 
SendMessage(const std::string & json_object)576 void Conductor::SendMessage(const std::string& json_object) {
577   std::string* msg = new std::string(json_object);
578   main_wnd_->QueueUIThreadCallback(SEND_MESSAGE_TO_PEER, msg);
579 }
580