1 /* 2 * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ 12 #define MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "api/audio_codecs/audio_encoder_factory.h" 20 #include "api/scoped_refptr.h" 21 #include "api/task_queue/task_queue_factory.h" 22 #include "api/transport/rtp/rtp_source.h" 23 #include "call/audio_state.h" 24 #include "call/call.h" 25 #include "media/base/media_engine.h" 26 #include "media/base/rtp_utils.h" 27 #include "rtc_base/buffer.h" 28 #include "rtc_base/constructor_magic.h" 29 #include "rtc_base/network_route.h" 30 #include "rtc_base/task_queue.h" 31 #include "rtc_base/thread_checker.h" 32 33 namespace cricket { 34 35 class AudioDeviceModule; 36 class AudioMixer; 37 class AudioSource; 38 class WebRtcVoiceMediaChannel; 39 40 // WebRtcVoiceEngine is a class to be used with CompositeMediaEngine. 41 // It uses the WebRtc VoiceEngine library for audio handling. 42 class WebRtcVoiceEngine final : public VoiceEngineInterface { 43 friend class WebRtcVoiceMediaChannel; 44 45 public: 46 WebRtcVoiceEngine( 47 webrtc::TaskQueueFactory* task_queue_factory, 48 webrtc::AudioDeviceModule* adm, 49 const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory, 50 const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory, 51 rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer, 52 rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing); 53 ~WebRtcVoiceEngine() override; 54 55 // Does initialization that needs to occur on the worker thread. 56 void Init() override; 57 58 rtc::scoped_refptr<webrtc::AudioState> GetAudioState() const override; 59 VoiceMediaChannel* CreateMediaChannel( 60 webrtc::Call* call, 61 const MediaConfig& config, 62 const AudioOptions& options, 63 const webrtc::CryptoOptions& crypto_options) override; 64 65 const std::vector<AudioCodec>& send_codecs() const override; 66 const std::vector<AudioCodec>& recv_codecs() const override; 67 std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() 68 const override; 69 70 // For tracking WebRtc channels. Needed because we have to pause them 71 // all when switching devices. 72 // May only be called by WebRtcVoiceMediaChannel. 73 void RegisterChannel(WebRtcVoiceMediaChannel* channel); 74 void UnregisterChannel(WebRtcVoiceMediaChannel* channel); 75 76 // Starts AEC dump using an existing file. A maximum file size in bytes can be 77 // specified. When the maximum file size is reached, logging is stopped and 78 // the file is closed. If max_size_bytes is set to <= 0, no limit will be 79 // used. 80 bool StartAecDump(webrtc::FileWrapper file, int64_t max_size_bytes) override; 81 82 // Stops AEC dump. 83 void StopAecDump() override; 84 85 private: 86 // Every option that is "set" will be applied. Every option not "set" will be 87 // ignored. This allows us to selectively turn on and off different options 88 // easily at any time. 89 bool ApplyOptions(const AudioOptions& options); 90 91 int CreateVoEChannel(); 92 93 webrtc::TaskQueueFactory* const task_queue_factory_; 94 std::unique_ptr<rtc::TaskQueue> low_priority_worker_queue_; 95 96 webrtc::AudioDeviceModule* adm(); 97 webrtc::AudioProcessing* apm() const; 98 webrtc::AudioState* audio_state(); 99 100 std::vector<AudioCodec> CollectCodecs( 101 const std::vector<webrtc::AudioCodecSpec>& specs) const; 102 103 rtc::ThreadChecker signal_thread_checker_; 104 rtc::ThreadChecker worker_thread_checker_; 105 106 // The audio device module. 107 rtc::scoped_refptr<webrtc::AudioDeviceModule> adm_; 108 rtc::scoped_refptr<webrtc::AudioEncoderFactory> encoder_factory_; 109 rtc::scoped_refptr<webrtc::AudioDecoderFactory> decoder_factory_; 110 rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer_; 111 // The audio processing module. 112 rtc::scoped_refptr<webrtc::AudioProcessing> apm_; 113 // The primary instance of WebRtc VoiceEngine. 114 rtc::scoped_refptr<webrtc::AudioState> audio_state_; 115 std::vector<AudioCodec> send_codecs_; 116 std::vector<AudioCodec> recv_codecs_; 117 std::vector<WebRtcVoiceMediaChannel*> channels_; 118 bool is_dumping_aec_ = false; 119 bool initialized_ = false; 120 121 // Cache experimental_ns and apply in case they are missing in the audio 122 // options. We need to do this because SetExtraOptions() will revert to 123 // defaults for options which are not provided. 124 absl::optional<bool> experimental_ns_; 125 // Jitter buffer settings for new streams. 126 size_t audio_jitter_buffer_max_packets_ = 200; 127 bool audio_jitter_buffer_fast_accelerate_ = false; 128 int audio_jitter_buffer_min_delay_ms_ = 0; 129 bool audio_jitter_buffer_enable_rtx_handling_ = false; 130 131 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcVoiceEngine); 132 }; 133 134 // WebRtcVoiceMediaChannel is an implementation of VoiceMediaChannel that uses 135 // WebRtc Voice Engine. 136 class WebRtcVoiceMediaChannel final : public VoiceMediaChannel, 137 public webrtc::Transport { 138 public: 139 WebRtcVoiceMediaChannel(WebRtcVoiceEngine* engine, 140 const MediaConfig& config, 141 const AudioOptions& options, 142 const webrtc::CryptoOptions& crypto_options, 143 webrtc::Call* call); 144 ~WebRtcVoiceMediaChannel() override; 145 options()146 const AudioOptions& options() const { return options_; } 147 148 bool SetSendParameters(const AudioSendParameters& params) override; 149 bool SetRecvParameters(const AudioRecvParameters& params) override; 150 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; 151 webrtc::RTCError SetRtpSendParameters( 152 uint32_t ssrc, 153 const webrtc::RtpParameters& parameters) override; 154 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; 155 webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; 156 157 void SetPlayout(bool playout) override; 158 void SetSend(bool send) override; 159 bool SetAudioSend(uint32_t ssrc, 160 bool enable, 161 const AudioOptions* options, 162 AudioSource* source) override; 163 bool AddSendStream(const StreamParams& sp) override; 164 bool RemoveSendStream(uint32_t ssrc) override; 165 bool AddRecvStream(const StreamParams& sp) override; 166 bool RemoveRecvStream(uint32_t ssrc) override; 167 void ResetUnsignaledRecvStream() override; 168 169 // E2EE Frame API 170 // Set a frame decryptor to a particular ssrc that will intercept all 171 // incoming audio payloads and attempt to decrypt them before forwarding the 172 // result. 173 void SetFrameDecryptor(uint32_t ssrc, 174 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> 175 frame_decryptor) override; 176 // Set a frame encryptor to a particular ssrc that will intercept all 177 // outgoing audio payloads frames and attempt to encrypt them and forward the 178 // result to the packetizer. 179 void SetFrameEncryptor(uint32_t ssrc, 180 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> 181 frame_encryptor) override; 182 183 bool SetOutputVolume(uint32_t ssrc, double volume) override; 184 // Applies the new volume to current and future unsignaled streams. 185 bool SetDefaultOutputVolume(double volume) override; 186 187 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; 188 absl::optional<int> GetBaseMinimumPlayoutDelayMs( 189 uint32_t ssrc) const override; 190 191 bool CanInsertDtmf() override; 192 bool InsertDtmf(uint32_t ssrc, int event, int duration) override; 193 194 void OnPacketReceived(rtc::CopyOnWriteBuffer packet, 195 int64_t packet_time_us) override; 196 void OnNetworkRouteChanged(const std::string& transport_name, 197 const rtc::NetworkRoute& network_route) override; 198 void OnReadyToSend(bool ready) override; 199 bool GetStats(VoiceMediaInfo* info) override; 200 201 // Set the audio sink for an existing stream. 202 void SetRawAudioSink( 203 uint32_t ssrc, 204 std::unique_ptr<webrtc::AudioSinkInterface> sink) override; 205 // Will set the audio sink on the latest unsignaled stream, future or 206 // current. Only one stream at a time will use the sink. 207 void SetDefaultRawAudioSink( 208 std::unique_ptr<webrtc::AudioSinkInterface> sink) override; 209 210 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; 211 212 // Sets a frame transformer between encoder and packetizer, to transform 213 // encoded frames before sending them out the network. 214 void SetEncoderToPacketizerFrameTransformer( 215 uint32_t ssrc, 216 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 217 override; 218 void SetDepacketizerToDecoderFrameTransformer( 219 uint32_t ssrc, 220 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) 221 override; 222 223 // implements Transport interface SendRtp(const uint8_t * data,size_t len,const webrtc::PacketOptions & options)224 bool SendRtp(const uint8_t* data, 225 size_t len, 226 const webrtc::PacketOptions& options) override { 227 rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); 228 rtc::PacketOptions rtc_options; 229 rtc_options.packet_id = options.packet_id; 230 if (DscpEnabled()) { 231 rtc_options.dscp = PreferredDscp(); 232 } 233 rtc_options.info_signaled_after_sent.included_in_feedback = 234 options.included_in_feedback; 235 rtc_options.info_signaled_after_sent.included_in_allocation = 236 options.included_in_allocation; 237 return VoiceMediaChannel::SendPacket(&packet, rtc_options); 238 } 239 SendRtcp(const uint8_t * data,size_t len)240 bool SendRtcp(const uint8_t* data, size_t len) override { 241 rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen); 242 rtc::PacketOptions rtc_options; 243 if (DscpEnabled()) { 244 rtc_options.dscp = PreferredDscp(); 245 } 246 247 return VoiceMediaChannel::SendRtcp(&packet, rtc_options); 248 } 249 250 private: 251 bool SetOptions(const AudioOptions& options); 252 bool SetRecvCodecs(const std::vector<AudioCodec>& codecs); 253 bool SetSendCodecs(const std::vector<AudioCodec>& codecs); 254 bool SetLocalSource(uint32_t ssrc, AudioSource* source); 255 bool MuteStream(uint32_t ssrc, bool mute); 256 engine()257 WebRtcVoiceEngine* engine() { return engine_; } 258 void ChangePlayout(bool playout); 259 int CreateVoEChannel(); 260 bool DeleteVoEChannel(int channel); 261 bool SetMaxSendBitrate(int bps); 262 void SetupRecording(); 263 // Check if 'ssrc' is an unsignaled stream, and if so mark it as not being 264 // unsignaled anymore (i.e. it is now removed, or signaled), and return true. 265 bool MaybeDeregisterUnsignaledRecvStream(uint32_t ssrc); 266 267 rtc::ThreadChecker worker_thread_checker_; 268 269 WebRtcVoiceEngine* const engine_ = nullptr; 270 std::vector<AudioCodec> send_codecs_; 271 272 // TODO(kwiberg): decoder_map_ and recv_codecs_ store the exact same 273 // information, in slightly different formats. Eliminate recv_codecs_. 274 std::map<int, webrtc::SdpAudioFormat> decoder_map_; 275 std::vector<AudioCodec> recv_codecs_; 276 277 int max_send_bitrate_bps_ = 0; 278 AudioOptions options_; 279 absl::optional<int> dtmf_payload_type_; 280 int dtmf_payload_freq_ = -1; 281 bool recv_transport_cc_enabled_ = false; 282 bool recv_nack_enabled_ = false; 283 bool desired_playout_ = false; 284 bool playout_ = false; 285 bool send_ = false; 286 webrtc::Call* const call_ = nullptr; 287 288 const MediaConfig::Audio audio_config_; 289 290 // Queue of unsignaled SSRCs; oldest at the beginning. 291 std::vector<uint32_t> unsignaled_recv_ssrcs_; 292 293 // This is a stream param that comes from the remote description, but wasn't 294 // signaled with any a=ssrc lines. It holds the information that was signaled 295 // before the unsignaled receive stream is created when the first packet is 296 // received. 297 StreamParams unsignaled_stream_params_; 298 299 // Volume for unsignaled streams, which may be set before the stream exists. 300 double default_recv_volume_ = 1.0; 301 302 // Delay for unsignaled streams, which may be set before the stream exists. 303 int default_recv_base_minimum_delay_ms_ = 0; 304 305 // Sink for latest unsignaled stream - may be set before the stream exists. 306 std::unique_ptr<webrtc::AudioSinkInterface> default_sink_; 307 // Default SSRC to use for RTCP receiver reports in case of no signaled 308 // send streams. See: https://code.google.com/p/webrtc/issues/detail?id=4740 309 // and https://code.google.com/p/chromium/issues/detail?id=547661 310 uint32_t receiver_reports_ssrc_ = 0xFA17FA17u; 311 312 class WebRtcAudioSendStream; 313 std::map<uint32_t, WebRtcAudioSendStream*> send_streams_; 314 std::vector<webrtc::RtpExtension> send_rtp_extensions_; 315 std::string mid_; 316 317 class WebRtcAudioReceiveStream; 318 std::map<uint32_t, WebRtcAudioReceiveStream*> recv_streams_; 319 std::vector<webrtc::RtpExtension> recv_rtp_extensions_; 320 321 absl::optional<webrtc::AudioSendStream::Config::SendCodecSpec> 322 send_codec_spec_; 323 324 // TODO(kwiberg): Per-SSRC codec pair IDs? 325 const webrtc::AudioCodecPairId codec_pair_id_ = 326 webrtc::AudioCodecPairId::Create(); 327 328 // Per peer connection crypto options that last for the lifetime of the peer 329 // connection. 330 const webrtc::CryptoOptions crypto_options_; 331 // Unsignaled streams have an option to have a frame decryptor set on them. 332 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> 333 unsignaled_frame_decryptor_; 334 335 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcVoiceMediaChannel); 336 }; 337 } // namespace cricket 338 339 #endif // MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ 340