1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ 12 #define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ 13 14 #include <memory> 15 16 #include "audio_session_observer.h" 17 #include "modules/audio_device/audio_device_generic.h" 18 #include "rtc_base/buffer.h" 19 #include "rtc_base/thread.h" 20 #include "rtc_base/thread_annotations.h" 21 #include "rtc_base/thread_checker.h" 22 #include "sdk/objc/base/RTCMacros.h" 23 #include "voice_processing_audio_unit.h" 24 25 RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter); 26 27 namespace webrtc { 28 29 class FineAudioBuffer; 30 31 namespace ios_adm { 32 33 // Implements full duplex 16-bit mono PCM audio support for iOS using a 34 // Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit 35 // supports audio echo cancellation. It also adds automatic gain control, 36 // adjustment of voice-processing quality and muting. 37 // 38 // An instance must be created and destroyed on one and the same thread. 39 // All supported public methods must also be called on the same thread. 40 // A thread checker will RTC_DCHECK if any supported method is called on an 41 // invalid thread. 42 // 43 // Recorded audio will be delivered on a real-time internal I/O thread in the 44 // audio unit. The audio unit will also ask for audio data to play out on this 45 // same thread. 46 class AudioDeviceIOS : public AudioDeviceGeneric, 47 public AudioSessionObserver, 48 public VoiceProcessingAudioUnitObserver, 49 public rtc::MessageHandler { 50 public: 51 AudioDeviceIOS(); 52 ~AudioDeviceIOS() override; 53 54 void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; 55 56 InitStatus Init() override; 57 int32_t Terminate() override; 58 bool Initialized() const override; 59 60 int32_t InitPlayout() override; 61 bool PlayoutIsInitialized() const override; 62 63 int32_t InitRecording() override; 64 bool RecordingIsInitialized() const override; 65 66 int32_t StartPlayout() override; 67 int32_t StopPlayout() override; 68 bool Playing() const override; 69 70 int32_t StartRecording() override; 71 int32_t StopRecording() override; 72 bool Recording() const override; 73 74 // These methods returns hard-coded delay values and not dynamic delay 75 // estimates. The reason is that iOS supports a built-in AEC and the WebRTC 76 // AEC will always be disabled in the Libjingle layer to avoid running two 77 // AEC implementations at the same time. And, it saves resources to avoid 78 // updating these delay values continuously. 79 // TODO(henrika): it would be possible to mark these two methods as not 80 // implemented since they are only called for A/V-sync purposes today and 81 // A/V-sync is not supported on iOS. However, we avoid adding error messages 82 // the log by using these dummy implementations instead. 83 int32_t PlayoutDelay(uint16_t& delayMS) const override; 84 85 // No implementation for playout underrun on iOS. We override it to avoid a 86 // periodic log that it isn't available from the base class. GetPlayoutUnderrunCount()87 int32_t GetPlayoutUnderrunCount() const override { return -1; } 88 89 // Native audio parameters stored during construction. 90 // These methods are unique for the iOS implementation. 91 int GetPlayoutAudioParameters(AudioParameters* params) const override; 92 int GetRecordAudioParameters(AudioParameters* params) const override; 93 94 // These methods are currently not fully implemented on iOS: 95 96 // See audio_device_not_implemented.cc for trivial implementations. 97 int32_t ActiveAudioLayer( 98 AudioDeviceModule::AudioLayer& audioLayer) const override; 99 int32_t PlayoutIsAvailable(bool& available) override; 100 int32_t RecordingIsAvailable(bool& available) override; 101 int16_t PlayoutDevices() override; 102 int16_t RecordingDevices() override; 103 int32_t PlayoutDeviceName(uint16_t index, 104 char name[kAdmMaxDeviceNameSize], 105 char guid[kAdmMaxGuidSize]) override; 106 int32_t RecordingDeviceName(uint16_t index, 107 char name[kAdmMaxDeviceNameSize], 108 char guid[kAdmMaxGuidSize]) override; 109 int32_t SetPlayoutDevice(uint16_t index) override; 110 int32_t SetPlayoutDevice( 111 AudioDeviceModule::WindowsDeviceType device) override; 112 int32_t SetRecordingDevice(uint16_t index) override; 113 int32_t SetRecordingDevice( 114 AudioDeviceModule::WindowsDeviceType device) override; 115 int32_t InitSpeaker() override; 116 bool SpeakerIsInitialized() const override; 117 int32_t InitMicrophone() override; 118 bool MicrophoneIsInitialized() const override; 119 int32_t SpeakerVolumeIsAvailable(bool& available) override; 120 int32_t SetSpeakerVolume(uint32_t volume) override; 121 int32_t SpeakerVolume(uint32_t& volume) const override; 122 int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; 123 int32_t MinSpeakerVolume(uint32_t& minVolume) const override; 124 int32_t MicrophoneVolumeIsAvailable(bool& available) override; 125 int32_t SetMicrophoneVolume(uint32_t volume) override; 126 int32_t MicrophoneVolume(uint32_t& volume) const override; 127 int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; 128 int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; 129 int32_t MicrophoneMuteIsAvailable(bool& available) override; 130 int32_t SetMicrophoneMute(bool enable) override; 131 int32_t MicrophoneMute(bool& enabled) const override; 132 int32_t SpeakerMuteIsAvailable(bool& available) override; 133 int32_t SetSpeakerMute(bool enable) override; 134 int32_t SpeakerMute(bool& enabled) const override; 135 int32_t StereoPlayoutIsAvailable(bool& available) override; 136 int32_t SetStereoPlayout(bool enable) override; 137 int32_t StereoPlayout(bool& enabled) const override; 138 int32_t StereoRecordingIsAvailable(bool& available) override; 139 int32_t SetStereoRecording(bool enable) override; 140 int32_t StereoRecording(bool& enabled) const override; 141 142 // AudioSessionObserver methods. May be called from any thread. 143 void OnInterruptionBegin() override; 144 void OnInterruptionEnd() override; 145 void OnValidRouteChange() override; 146 void OnCanPlayOrRecordChange(bool can_play_or_record) override; 147 void OnChangedOutputVolume() override; 148 149 // VoiceProcessingAudioUnitObserver methods. 150 OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, 151 const AudioTimeStamp* time_stamp, 152 UInt32 bus_number, 153 UInt32 num_frames, 154 AudioBufferList* io_data) override; 155 OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags, 156 const AudioTimeStamp* time_stamp, 157 UInt32 bus_number, 158 UInt32 num_frames, 159 AudioBufferList* io_data) override; 160 161 // Handles messages from posts. 162 void OnMessage(rtc::Message* msg) override; 163 164 bool IsInterrupted(); 165 166 private: 167 // Called by the relevant AudioSessionObserver methods on |thread_|. 168 void HandleInterruptionBegin(); 169 void HandleInterruptionEnd(); 170 void HandleValidRouteChange(); 171 void HandleCanPlayOrRecordChange(bool can_play_or_record); 172 void HandleSampleRateChange(float sample_rate); 173 void HandlePlayoutGlitchDetected(); 174 void HandleOutputVolumeChange(); 175 176 // Uses current |playout_parameters_| and |record_parameters_| to inform the 177 // audio device buffer (ADB) about our internal audio parameters. 178 void UpdateAudioDeviceBuffer(); 179 180 // Since the preferred audio parameters are only hints to the OS, the actual 181 // values may be different once the AVAudioSession has been activated. 182 // This method asks for the current hardware parameters and takes actions 183 // if they should differ from what we have asked for initially. It also 184 // defines |playout_parameters_| and |record_parameters_|. 185 void SetupAudioBuffersForActiveAudioSession(); 186 187 // Creates the audio unit. 188 bool CreateAudioUnit(); 189 190 // Updates the audio unit state based on current state. 191 void UpdateAudioUnit(bool can_play_or_record); 192 193 // Configures the audio session for WebRTC. 194 bool ConfigureAudioSession(); 195 // Unconfigures the audio session. 196 void UnconfigureAudioSession(); 197 198 // Activates our audio session, creates and initializes the voice-processing 199 // audio unit and verifies that we got the preferred native audio parameters. 200 bool InitPlayOrRecord(); 201 202 // Closes and deletes the voice-processing I/O unit. 203 void ShutdownPlayOrRecord(); 204 205 // Resets thread-checkers before a call is restarted. 206 void PrepareForNewStart(); 207 208 // Ensures that methods are called from the same thread as this object is 209 // created on. 210 rtc::ThreadChecker thread_checker_; 211 212 // Native I/O audio thread checker. 213 rtc::ThreadChecker io_thread_checker_; 214 215 // Thread that this object is created on. 216 rtc::Thread* thread_; 217 218 // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the 219 // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). 220 // The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance 221 // and therefore outlives this object. 222 AudioDeviceBuffer* audio_device_buffer_; 223 224 // Contains audio parameters (sample rate, #channels, buffer size etc.) for 225 // the playout and recording sides. These structure is set in two steps: 226 // first, native sample rate and #channels are defined in Init(). Next, the 227 // audio session is activated and we verify that the preferred parameters 228 // were granted by the OS. At this stage it is also possible to add a third 229 // component to the parameters; the native I/O buffer duration. 230 // A RTC_CHECK will be hit if we for some reason fail to open an audio session 231 // using the specified parameters. 232 AudioParameters playout_parameters_; 233 AudioParameters record_parameters_; 234 235 // The AudioUnit used to play and record audio. 236 std::unique_ptr<VoiceProcessingAudioUnit> audio_unit_; 237 238 // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data 239 // in chunks of 10ms. It then allows for this data to be pulled in 240 // a finer or coarser granularity. I.e. interacting with this class instead 241 // of directly with the AudioDeviceBuffer one can ask for any number of 242 // audio data samples. Is also supports a similar scheme for the recording 243 // side. 244 // Example: native buffer size can be 128 audio frames at 16kHz sample rate. 245 // WebRTC will provide 480 audio frames per 10ms but iOS asks for 128 246 // in each callback (one every 8ms). This class can then ask for 128 and the 247 // FineAudioBuffer will ask WebRTC for new data only when needed and also 248 // cache non-utilized audio between callbacks. On the recording side, iOS 249 // can provide audio data frames of size 128 and these are accumulated until 250 // enough data to supply one 10ms call exists. This 10ms chunk is then sent 251 // to WebRTC and the remaining part is stored. 252 std::unique_ptr<FineAudioBuffer> fine_audio_buffer_; 253 254 // Temporary storage for recorded data. AudioUnitRender() renders into this 255 // array as soon as a frame of the desired buffer size has been recorded. 256 // On real iOS devices, the size will be fixed and set once. For iOS 257 // simulators, the size can vary from callback to callback and the size 258 // will be changed dynamically to account for this behavior. 259 rtc::BufferT<int16_t> record_audio_buffer_; 260 261 // Set to 1 when recording is active and 0 otherwise. 262 volatile int recording_; 263 264 // Set to 1 when playout is active and 0 otherwise. 265 volatile int playing_; 266 267 // Set to true after successful call to Init(), false otherwise. 268 bool initialized_ RTC_GUARDED_BY(thread_checker_); 269 270 // Set to true after successful call to InitRecording() or InitPlayout(), 271 // false otherwise. 272 bool audio_is_initialized_; 273 274 // Set to true if audio session is interrupted, false otherwise. 275 bool is_interrupted_; 276 277 // Audio interruption observer instance. 278 RTCNativeAudioSessionDelegateAdapter* audio_session_observer_ 279 RTC_GUARDED_BY(thread_checker_); 280 281 // Set to true if we've activated the audio session. 282 bool has_configured_session_ RTC_GUARDED_BY(thread_checker_); 283 284 // Counts number of detected audio glitches on the playout side. 285 int64_t num_detected_playout_glitches_ RTC_GUARDED_BY(thread_checker_); 286 int64_t last_playout_time_ RTC_GUARDED_BY(io_thread_checker_); 287 288 // Counts number of playout callbacks per call. 289 // The value isupdated on the native I/O thread and later read on the 290 // creating thread (see thread_checker_) but at this stage no audio is 291 // active. Hence, it is a "thread safe" design and no lock is needed. 292 int64_t num_playout_callbacks_; 293 294 // Contains the time for when the last output volume change was detected. 295 int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_checker_); 296 }; 297 } // namespace ios_adm 298 } // namespace webrtc 299 300 #endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ 301