1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/renderer/media/webrtc_audio_renderer.h"
6
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/strings/string_util.h"
10 #include "base/strings/stringprintf.h"
11 #include "content/renderer/media/audio_device_factory.h"
12 #include "content/renderer/media/media_stream_dispatcher.h"
13 #include "content/renderer/media/webrtc_audio_device_impl.h"
14 #include "content/renderer/media/webrtc_logging.h"
15 #include "content/renderer/render_frame_impl.h"
16 #include "media/audio/audio_output_device.h"
17 #include "media/audio/audio_parameters.h"
18 #include "media/audio/sample_rates.h"
19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
21
22
23 #if defined(OS_WIN)
24 #include "base/win/windows_version.h"
25 #include "media/audio/win/core_audio_util_win.h"
26 #endif
27
28 namespace content {
29
30 namespace {
31
32 // We add a UMA histogram measuring the execution time of the Render() method
33 // every |kNumCallbacksBetweenRenderTimeHistograms| callback. Assuming 10ms
34 // between each callback leads to one UMA update each 100ms.
35 const int kNumCallbacksBetweenRenderTimeHistograms = 10;
36
37 // This is a simple wrapper class that's handed out to users of a shared
38 // WebRtcAudioRenderer instance. This class maintains the per-user 'playing'
39 // and 'started' states to avoid problems related to incorrect usage which
40 // might violate the implementation assumptions inside WebRtcAudioRenderer
41 // (see the play reference count).
42 class SharedAudioRenderer : public MediaStreamAudioRenderer {
43 public:
44 // Callback definition for a callback that is called when when Play(), Pause()
45 // or SetVolume are called (whenever the internal |playing_state_| changes).
46 typedef base::Callback<
47 void(const scoped_refptr<webrtc::MediaStreamInterface>&,
48 WebRtcAudioRenderer::PlayingState*)> OnPlayStateChanged;
49
SharedAudioRenderer(const scoped_refptr<MediaStreamAudioRenderer> & delegate,const scoped_refptr<webrtc::MediaStreamInterface> & media_stream,const OnPlayStateChanged & on_play_state_changed)50 SharedAudioRenderer(
51 const scoped_refptr<MediaStreamAudioRenderer>& delegate,
52 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
53 const OnPlayStateChanged& on_play_state_changed)
54 : delegate_(delegate), media_stream_(media_stream), started_(false),
55 on_play_state_changed_(on_play_state_changed) {
56 DCHECK(!on_play_state_changed_.is_null());
57 DCHECK(media_stream_.get());
58 }
59
60 protected:
~SharedAudioRenderer()61 virtual ~SharedAudioRenderer() {
62 DCHECK(thread_checker_.CalledOnValidThread());
63 DVLOG(1) << __FUNCTION__;
64 Stop();
65 }
66
Start()67 virtual void Start() OVERRIDE {
68 DCHECK(thread_checker_.CalledOnValidThread());
69 if (started_)
70 return;
71 started_ = true;
72 delegate_->Start();
73 }
74
Play()75 virtual void Play() OVERRIDE {
76 DCHECK(thread_checker_.CalledOnValidThread());
77 DCHECK(started_);
78 if (playing_state_.playing())
79 return;
80 playing_state_.set_playing(true);
81 on_play_state_changed_.Run(media_stream_, &playing_state_);
82 }
83
Pause()84 virtual void Pause() OVERRIDE {
85 DCHECK(thread_checker_.CalledOnValidThread());
86 DCHECK(started_);
87 if (!playing_state_.playing())
88 return;
89 playing_state_.set_playing(false);
90 on_play_state_changed_.Run(media_stream_, &playing_state_);
91 }
92
Stop()93 virtual void Stop() OVERRIDE {
94 DCHECK(thread_checker_.CalledOnValidThread());
95 if (!started_)
96 return;
97 Pause();
98 started_ = false;
99 delegate_->Stop();
100 }
101
SetVolume(float volume)102 virtual void SetVolume(float volume) OVERRIDE {
103 DCHECK(thread_checker_.CalledOnValidThread());
104 DCHECK(volume >= 0.0f && volume <= 1.0f);
105 playing_state_.set_volume(volume);
106 on_play_state_changed_.Run(media_stream_, &playing_state_);
107 }
108
GetCurrentRenderTime() const109 virtual base::TimeDelta GetCurrentRenderTime() const OVERRIDE {
110 DCHECK(thread_checker_.CalledOnValidThread());
111 return delegate_->GetCurrentRenderTime();
112 }
113
IsLocalRenderer() const114 virtual bool IsLocalRenderer() const OVERRIDE {
115 DCHECK(thread_checker_.CalledOnValidThread());
116 return delegate_->IsLocalRenderer();
117 }
118
119 private:
120 base::ThreadChecker thread_checker_;
121 const scoped_refptr<MediaStreamAudioRenderer> delegate_;
122 const scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
123 bool started_;
124 WebRtcAudioRenderer::PlayingState playing_state_;
125 OnPlayStateChanged on_play_state_changed_;
126 };
127
128 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
129 // depending on whether or not an input element is currently open with
130 // ducking enabled.
GetCurrentDuckingFlag(int render_frame_id)131 int GetCurrentDuckingFlag(int render_frame_id) {
132 RenderFrameImpl* const frame =
133 RenderFrameImpl::FromRoutingID(render_frame_id);
134 MediaStreamDispatcher* const dispatcher = frame ?
135 frame->GetMediaStreamDispatcher() : NULL;
136 if (dispatcher && dispatcher->IsAudioDuckingActive()) {
137 return media::AudioParameters::DUCKING;
138 }
139
140 return media::AudioParameters::NO_EFFECTS;
141 }
142
143 // Helper method to get platform specific optimal buffer size.
GetOptimalBufferSize(int sample_rate,int hardware_buffer_size)144 int GetOptimalBufferSize(int sample_rate, int hardware_buffer_size) {
145 // Use native hardware buffer size as default. On Windows, we strive to open
146 // up using this native hardware buffer size to achieve best
147 // possible performance and to ensure that no FIFO is needed on the browser
148 // side to match the client request. That is why there is no #if case for
149 // Windows below.
150 int frames_per_buffer = hardware_buffer_size;
151
152 #if defined(OS_LINUX) || defined(OS_MACOSX)
153 // On Linux and MacOS, the low level IO implementations on the browser side
154 // supports all buffer size the clients want. We use the native peer
155 // connection buffer size (10ms) to achieve best possible performance.
156 frames_per_buffer = sample_rate / 100;
157 #elif defined(OS_ANDROID)
158 // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
159 // cases. Might not be possible to come up with the perfect solution using
160 // the render side only.
161 int frames_per_10ms = sample_rate / 100;
162 if (frames_per_buffer < 2 * frames_per_10ms) {
163 // Examples of low-latency frame sizes and the resulting |buffer_size|:
164 // Nexus 7 : 240 audio frames => 2*480 = 960
165 // Nexus 10 : 256 => 2*441 = 882
166 // Galaxy Nexus: 144 => 2*441 = 882
167 frames_per_buffer = 2 * frames_per_10ms;
168 DVLOG(1) << "Low-latency output detected on Android";
169 }
170 #endif
171
172 DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer;
173 return frames_per_buffer;
174 }
175
176 } // namespace
177
WebRtcAudioRenderer(const scoped_refptr<webrtc::MediaStreamInterface> & media_stream,int source_render_view_id,int source_render_frame_id,int session_id,int sample_rate,int frames_per_buffer)178 WebRtcAudioRenderer::WebRtcAudioRenderer(
179 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
180 int source_render_view_id,
181 int source_render_frame_id,
182 int session_id,
183 int sample_rate,
184 int frames_per_buffer)
185 : state_(UNINITIALIZED),
186 source_render_view_id_(source_render_view_id),
187 source_render_frame_id_(source_render_frame_id),
188 session_id_(session_id),
189 media_stream_(media_stream),
190 source_(NULL),
191 play_ref_count_(0),
192 start_ref_count_(0),
193 audio_delay_milliseconds_(0),
194 fifo_delay_milliseconds_(0),
195 sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
196 media::CHANNEL_LAYOUT_STEREO, sample_rate, 16,
197 frames_per_buffer,
198 GetCurrentDuckingFlag(source_render_frame_id)),
199 render_callback_count_(0) {
200 WebRtcLogMessage(base::StringPrintf(
201 "WAR::WAR. source_render_view_id=%d"
202 ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
203 source_render_view_id,
204 session_id,
205 sample_rate,
206 frames_per_buffer,
207 sink_params_.effects()));
208 }
209
~WebRtcAudioRenderer()210 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
211 DCHECK(thread_checker_.CalledOnValidThread());
212 DCHECK_EQ(state_, UNINITIALIZED);
213 }
214
Initialize(WebRtcAudioRendererSource * source)215 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
216 DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
217 DCHECK(thread_checker_.CalledOnValidThread());
218 base::AutoLock auto_lock(lock_);
219 DCHECK_EQ(state_, UNINITIALIZED);
220 DCHECK(source);
221 DCHECK(!sink_.get());
222 DCHECK(!source_);
223
224 // WebRTC does not yet support higher rates than 96000 on the client side
225 // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
226 // we change the rate to 48000 instead. The consequence is that the native
227 // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
228 // which will then be resampled by the audio converted on the browser side
229 // to match the native audio layer.
230 int sample_rate = sink_params_.sample_rate();
231 DVLOG(1) << "Audio output hardware sample rate: " << sample_rate;
232 if (sample_rate == 192000) {
233 DVLOG(1) << "Resampling from 48000 to 192000 is required";
234 sample_rate = 48000;
235 }
236 media::AudioSampleRate asr;
237 if (media::ToAudioSampleRate(sample_rate, &asr)) {
238 UMA_HISTOGRAM_ENUMERATION(
239 "WebRTC.AudioOutputSampleRate", asr, media::kAudioSampleRateMax + 1);
240 } else {
241 UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
242 sample_rate);
243 }
244
245 // Set up audio parameters for the source, i.e., the WebRTC client.
246
247 // The WebRTC client only supports multiples of 10ms as buffer size where
248 // 10ms is preferred for lowest possible delay.
249 media::AudioParameters source_params;
250 const int frames_per_10ms = (sample_rate / 100);
251 DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms;
252
253 source_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
254 sink_params_.channel_layout(), sink_params_.channels(),
255 sample_rate, 16, frames_per_10ms);
256
257 const int frames_per_buffer =
258 GetOptimalBufferSize(sample_rate, sink_params_.frames_per_buffer());
259
260 sink_params_.Reset(sink_params_.format(), sink_params_.channel_layout(),
261 sink_params_.channels(), sample_rate, 16,
262 frames_per_buffer);
263
264 // Create a FIFO if re-buffering is required to match the source input with
265 // the sink request. The source acts as provider here and the sink as
266 // consumer.
267 fifo_delay_milliseconds_ = 0;
268 if (source_params.frames_per_buffer() != sink_params_.frames_per_buffer()) {
269 DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
270 << " to " << sink_params_.frames_per_buffer();
271 audio_fifo_.reset(new media::AudioPullFifo(
272 source_params.channels(),
273 source_params.frames_per_buffer(),
274 base::Bind(
275 &WebRtcAudioRenderer::SourceCallback,
276 base::Unretained(this))));
277
278 if (sink_params_.frames_per_buffer() > source_params.frames_per_buffer()) {
279 int frame_duration_milliseconds = base::Time::kMillisecondsPerSecond /
280 static_cast<double>(source_params.sample_rate());
281 fifo_delay_milliseconds_ = (sink_params_.frames_per_buffer() -
282 source_params.frames_per_buffer()) * frame_duration_milliseconds;
283 }
284 }
285
286 source_ = source;
287
288 // Configure the audio rendering client and start rendering.
289 sink_ = AudioDeviceFactory::NewOutputDevice(
290 source_render_view_id_, source_render_frame_id_);
291
292 DCHECK_GE(session_id_, 0);
293 sink_->InitializeWithSessionId(sink_params_, this, session_id_);
294
295 sink_->Start();
296
297 // User must call Play() before any audio can be heard.
298 state_ = PAUSED;
299
300 return true;
301 }
302
303 scoped_refptr<MediaStreamAudioRenderer>
CreateSharedAudioRendererProxy(const scoped_refptr<webrtc::MediaStreamInterface> & media_stream)304 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
305 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream) {
306 content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed =
307 base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged, this);
308 return new SharedAudioRenderer(this, media_stream, on_play_state_changed);
309 }
310
IsStarted() const311 bool WebRtcAudioRenderer::IsStarted() const {
312 DCHECK(thread_checker_.CalledOnValidThread());
313 return start_ref_count_ != 0;
314 }
315
Start()316 void WebRtcAudioRenderer::Start() {
317 DVLOG(1) << "WebRtcAudioRenderer::Start()";
318 DCHECK(thread_checker_.CalledOnValidThread());
319 ++start_ref_count_;
320 }
321
Play()322 void WebRtcAudioRenderer::Play() {
323 DVLOG(1) << "WebRtcAudioRenderer::Play()";
324 DCHECK(thread_checker_.CalledOnValidThread());
325
326 if (playing_state_.playing())
327 return;
328
329 playing_state_.set_playing(true);
330 render_callback_count_ = 0;
331
332 OnPlayStateChanged(media_stream_, &playing_state_);
333 }
334
EnterPlayState()335 void WebRtcAudioRenderer::EnterPlayState() {
336 DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
337 DCHECK(thread_checker_.CalledOnValidThread());
338 DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
339 base::AutoLock auto_lock(lock_);
340 if (state_ == UNINITIALIZED)
341 return;
342
343 DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
344 ++play_ref_count_;
345
346 if (state_ != PLAYING) {
347 state_ = PLAYING;
348
349 if (audio_fifo_) {
350 audio_delay_milliseconds_ = 0;
351 audio_fifo_->Clear();
352 }
353 }
354 }
355
Pause()356 void WebRtcAudioRenderer::Pause() {
357 DVLOG(1) << "WebRtcAudioRenderer::Pause()";
358 DCHECK(thread_checker_.CalledOnValidThread());
359 if (!playing_state_.playing())
360 return;
361
362 playing_state_.set_playing(false);
363
364 OnPlayStateChanged(media_stream_, &playing_state_);
365 }
366
EnterPauseState()367 void WebRtcAudioRenderer::EnterPauseState() {
368 DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
369 DCHECK(thread_checker_.CalledOnValidThread());
370 DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
371 base::AutoLock auto_lock(lock_);
372 if (state_ == UNINITIALIZED)
373 return;
374
375 DCHECK_EQ(state_, PLAYING);
376 DCHECK_GT(play_ref_count_, 0);
377 if (!--play_ref_count_)
378 state_ = PAUSED;
379 }
380
Stop()381 void WebRtcAudioRenderer::Stop() {
382 DVLOG(1) << "WebRtcAudioRenderer::Stop()";
383 DCHECK(thread_checker_.CalledOnValidThread());
384 {
385 base::AutoLock auto_lock(lock_);
386 if (state_ == UNINITIALIZED)
387 return;
388
389 if (--start_ref_count_)
390 return;
391
392 DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
393
394 source_->RemoveAudioRenderer(this);
395 source_ = NULL;
396 state_ = UNINITIALIZED;
397 }
398
399 // Make sure to stop the sink while _not_ holding the lock since the Render()
400 // callback may currently be executing and try to grab the lock while we're
401 // stopping the thread on which it runs.
402 sink_->Stop();
403 }
404
SetVolume(float volume)405 void WebRtcAudioRenderer::SetVolume(float volume) {
406 DCHECK(thread_checker_.CalledOnValidThread());
407 DCHECK(volume >= 0.0f && volume <= 1.0f);
408
409 playing_state_.set_volume(volume);
410 OnPlayStateChanged(media_stream_, &playing_state_);
411 }
412
GetCurrentRenderTime() const413 base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() const {
414 DCHECK(thread_checker_.CalledOnValidThread());
415 base::AutoLock auto_lock(lock_);
416 return current_time_;
417 }
418
IsLocalRenderer() const419 bool WebRtcAudioRenderer::IsLocalRenderer() const {
420 return false;
421 }
422
Render(media::AudioBus * audio_bus,int audio_delay_milliseconds)423 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
424 int audio_delay_milliseconds) {
425 base::AutoLock auto_lock(lock_);
426 if (!source_)
427 return 0;
428
429 DVLOG(2) << "WebRtcAudioRenderer::Render()";
430 DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
431
432 audio_delay_milliseconds_ = audio_delay_milliseconds;
433
434 if (audio_fifo_)
435 audio_fifo_->Consume(audio_bus, audio_bus->frames());
436 else
437 SourceCallback(0, audio_bus);
438
439 return (state_ == PLAYING) ? audio_bus->frames() : 0;
440 }
441
OnRenderError()442 void WebRtcAudioRenderer::OnRenderError() {
443 NOTIMPLEMENTED();
444 LOG(ERROR) << "OnRenderError()";
445 }
446
447 // Called by AudioPullFifo when more data is necessary.
SourceCallback(int fifo_frame_delay,media::AudioBus * audio_bus)448 void WebRtcAudioRenderer::SourceCallback(
449 int fifo_frame_delay, media::AudioBus* audio_bus) {
450 base::TimeTicks start_time = base::TimeTicks::Now() ;
451 DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
452 << fifo_frame_delay << ", "
453 << audio_bus->frames() << ")";
454
455 int output_delay_milliseconds = audio_delay_milliseconds_;
456 output_delay_milliseconds += fifo_delay_milliseconds_;
457 DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
458
459 // We need to keep render data for the |source_| regardless of |state_|,
460 // otherwise the data will be buffered up inside |source_|.
461 source_->RenderData(audio_bus, sink_params_.sample_rate(),
462 output_delay_milliseconds,
463 ¤t_time_);
464
465 // Avoid filling up the audio bus if we are not playing; instead
466 // return here and ensure that the returned value in Render() is 0.
467 if (state_ != PLAYING)
468 audio_bus->Zero();
469
470 if (++render_callback_count_ == kNumCallbacksBetweenRenderTimeHistograms) {
471 base::TimeDelta elapsed = base::TimeTicks::Now() - start_time;
472 render_callback_count_ = 0;
473 UMA_HISTOGRAM_TIMES("WebRTC.AudioRenderTimes", elapsed);
474 }
475 }
476
UpdateSourceVolume(webrtc::AudioSourceInterface * source)477 void WebRtcAudioRenderer::UpdateSourceVolume(
478 webrtc::AudioSourceInterface* source) {
479 DCHECK(thread_checker_.CalledOnValidThread());
480
481 // Note: If there are no playing audio renderers, then the volume will be
482 // set to 0.0.
483 float volume = 0.0f;
484
485 SourcePlayingStates::iterator entry = source_playing_states_.find(source);
486 if (entry != source_playing_states_.end()) {
487 PlayingStates& states = entry->second;
488 for (PlayingStates::const_iterator it = states.begin();
489 it != states.end(); ++it) {
490 if ((*it)->playing())
491 volume += (*it)->volume();
492 }
493 }
494
495 // The valid range for volume scaling of a remote webrtc source is
496 // 0.0-10.0 where 1.0 is no attenuation/boost.
497 DCHECK(volume >= 0.0f);
498 if (volume > 10.0f)
499 volume = 10.0f;
500
501 DVLOG(1) << "Setting remote source volume: " << volume;
502 source->SetVolume(volume);
503 }
504
AddPlayingState(webrtc::AudioSourceInterface * source,PlayingState * state)505 bool WebRtcAudioRenderer::AddPlayingState(
506 webrtc::AudioSourceInterface* source,
507 PlayingState* state) {
508 DCHECK(thread_checker_.CalledOnValidThread());
509 DCHECK(state->playing());
510 // Look up or add the |source| to the map.
511 PlayingStates& array = source_playing_states_[source];
512 if (std::find(array.begin(), array.end(), state) != array.end())
513 return false;
514
515 array.push_back(state);
516
517 return true;
518 }
519
RemovePlayingState(webrtc::AudioSourceInterface * source,PlayingState * state)520 bool WebRtcAudioRenderer::RemovePlayingState(
521 webrtc::AudioSourceInterface* source,
522 PlayingState* state) {
523 DCHECK(thread_checker_.CalledOnValidThread());
524 DCHECK(!state->playing());
525 SourcePlayingStates::iterator found = source_playing_states_.find(source);
526 if (found == source_playing_states_.end())
527 return false;
528
529 PlayingStates& array = found->second;
530 PlayingStates::iterator state_it =
531 std::find(array.begin(), array.end(), state);
532 if (state_it == array.end())
533 return false;
534
535 array.erase(state_it);
536
537 if (array.empty())
538 source_playing_states_.erase(found);
539
540 return true;
541 }
542
OnPlayStateChanged(const scoped_refptr<webrtc::MediaStreamInterface> & media_stream,PlayingState * state)543 void WebRtcAudioRenderer::OnPlayStateChanged(
544 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
545 PlayingState* state) {
546 webrtc::AudioTrackVector tracks(media_stream->GetAudioTracks());
547 for (webrtc::AudioTrackVector::iterator it = tracks.begin();
548 it != tracks.end(); ++it) {
549 webrtc::AudioSourceInterface* source = (*it)->GetSource();
550 DCHECK(source);
551 if (!state->playing()) {
552 if (RemovePlayingState(source, state))
553 EnterPauseState();
554 } else if (AddPlayingState(source, state)) {
555 EnterPlayState();
556 }
557 UpdateSourceVolume(source);
558 }
559 }
560
561 } // namespace content
562