1 /*
2 * Copyright 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <cassert>
18
19 #include <SLES/OpenSLES.h>
20 #include <SLES/OpenSLES_Android.h>
21
22 #include "oboe/AudioStreamBuilder.h"
23 #include "AudioInputStreamOpenSLES.h"
24 #include "AudioStreamOpenSLES.h"
25 #include "OpenSLESUtilities.h"
26
27 using namespace oboe;
28
OpenSLES_convertInputPreset(InputPreset oboePreset)29 static SLuint32 OpenSLES_convertInputPreset(InputPreset oboePreset) {
30 SLuint32 openslPreset = SL_ANDROID_RECORDING_PRESET_NONE;
31 switch(oboePreset) {
32 case InputPreset::Generic:
33 openslPreset = SL_ANDROID_RECORDING_PRESET_GENERIC;
34 break;
35 case InputPreset::Camcorder:
36 openslPreset = SL_ANDROID_RECORDING_PRESET_CAMCORDER;
37 break;
38 case InputPreset::VoiceRecognition:
39 case InputPreset::VoicePerformance:
40 openslPreset = SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION;
41 break;
42 case InputPreset::VoiceCommunication:
43 openslPreset = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;
44 break;
45 case InputPreset::Unprocessed:
46 openslPreset = SL_ANDROID_RECORDING_PRESET_UNPROCESSED;
47 break;
48 default:
49 break;
50 }
51 return openslPreset;
52 }
53
AudioInputStreamOpenSLES(const AudioStreamBuilder & builder)54 AudioInputStreamOpenSLES::AudioInputStreamOpenSLES(const AudioStreamBuilder &builder)
55 : AudioStreamOpenSLES(builder) {
56 }
57
~AudioInputStreamOpenSLES()58 AudioInputStreamOpenSLES::~AudioInputStreamOpenSLES() {
59 }
60
61 // Calculate masks specific to INPUT streams.
channelCountToChannelMask(int channelCount) const62 SLuint32 AudioInputStreamOpenSLES::channelCountToChannelMask(int channelCount) const {
63 // Derived from internal sles_channel_in_mask_from_count(chanCount);
64 // in "frameworks/wilhelm/src/android/channels.cpp".
65 // Yes, it seems strange to use SPEAKER constants to describe inputs.
66 // But that is how OpenSL ES does it internally.
67 switch (channelCount) {
68 case 1:
69 return SL_SPEAKER_FRONT_LEFT;
70 case 2:
71 return SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
72 default:
73 return channelCountToChannelMaskDefault(channelCount);
74 }
75 }
76
open()77 Result AudioInputStreamOpenSLES::open() {
78 logUnsupportedAttributes();
79
80 SLAndroidConfigurationItf configItf = nullptr;
81
82 if (getSdkVersion() < __ANDROID_API_M__ && mFormat == AudioFormat::Float){
83 // TODO: Allow floating point format on API <23 using float->int16 converter
84 return Result::ErrorInvalidFormat;
85 }
86
87 // If audio format is unspecified then choose a suitable default.
88 // API 23+: FLOAT
89 // API <23: INT16
90 if (mFormat == AudioFormat::Unspecified){
91 mFormat = (getSdkVersion() < __ANDROID_API_M__) ?
92 AudioFormat::I16 : AudioFormat::Float;
93 }
94
95 Result oboeResult = AudioStreamOpenSLES::open();
96 if (Result::OK != oboeResult) return oboeResult;
97
98 SLuint32 bitsPerSample = static_cast<SLuint32>(getBytesPerSample() * kBitsPerByte);
99
100 // configure audio sink
101 SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {
102 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, // locatorType
103 static_cast<SLuint32>(kBufferQueueLength)}; // numBuffers
104
105 // Define the audio data format.
106 SLDataFormat_PCM format_pcm = {
107 SL_DATAFORMAT_PCM, // formatType
108 static_cast<SLuint32>(mChannelCount), // numChannels
109 static_cast<SLuint32>(mSampleRate * kMillisPerSecond), // milliSamplesPerSec
110 bitsPerSample, // bitsPerSample
111 bitsPerSample, // containerSize;
112 channelCountToChannelMask(mChannelCount), // channelMask
113 getDefaultByteOrder(),
114 };
115
116 SLDataSink audioSink = {&loc_bufq, &format_pcm};
117
118 /**
119 * API 23 (Marshmallow) introduced support for floating-point data representation and an
120 * extended data format type: SLAndroidDataFormat_PCM_EX for recording streams (playback streams
121 * got this in API 21). If running on API 23+ use this newer format type, creating it from our
122 * original format.
123 */
124 SLAndroidDataFormat_PCM_EX format_pcm_ex;
125 if (getSdkVersion() >= __ANDROID_API_M__) {
126 SLuint32 representation = OpenSLES_ConvertFormatToRepresentation(getFormat());
127 // Fill in the format structure.
128 format_pcm_ex = OpenSLES_createExtendedFormat(format_pcm, representation);
129 // Use in place of the previous format.
130 audioSink.pFormat = &format_pcm_ex;
131 }
132
133
134 // configure audio source
135 SLDataLocator_IODevice loc_dev = {SL_DATALOCATOR_IODEVICE,
136 SL_IODEVICE_AUDIOINPUT,
137 SL_DEFAULTDEVICEID_AUDIOINPUT,
138 NULL};
139 SLDataSource audioSrc = {&loc_dev, NULL};
140
141 SLresult result = EngineOpenSLES::getInstance().createAudioRecorder(&mObjectInterface,
142 &audioSrc,
143 &audioSink);
144
145 if (SL_RESULT_SUCCESS != result) {
146 LOGE("createAudioRecorder() result:%s", getSLErrStr(result));
147 goto error;
148 }
149
150 // Configure the stream.
151 result = (*mObjectInterface)->GetInterface(mObjectInterface,
152 SL_IID_ANDROIDCONFIGURATION,
153 &configItf);
154
155 if (SL_RESULT_SUCCESS != result) {
156 LOGW("%s() GetInterface(SL_IID_ANDROIDCONFIGURATION) failed with %s",
157 __func__, getSLErrStr(result));
158 } else {
159 if (getInputPreset() == InputPreset::VoicePerformance) {
160 LOGD("OpenSL ES does not support InputPreset::VoicePerformance. Use VoiceRecognition.");
161 mInputPreset = InputPreset::VoiceRecognition;
162 }
163 SLuint32 presetValue = OpenSLES_convertInputPreset(getInputPreset());
164 result = (*configItf)->SetConfiguration(configItf,
165 SL_ANDROID_KEY_RECORDING_PRESET,
166 &presetValue,
167 sizeof(SLuint32));
168 if (SL_RESULT_SUCCESS != result
169 && presetValue != SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION) {
170 presetValue = SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION;
171 LOGD("Setting InputPreset %d failed. Using VoiceRecognition instead.", getInputPreset());
172 mInputPreset = InputPreset::VoiceRecognition;
173 (*configItf)->SetConfiguration(configItf,
174 SL_ANDROID_KEY_RECORDING_PRESET,
175 &presetValue,
176 sizeof(SLuint32));
177 }
178
179 result = configurePerformanceMode(configItf);
180 if (SL_RESULT_SUCCESS != result) {
181 goto error;
182 }
183 }
184
185 result = (*mObjectInterface)->Realize(mObjectInterface, SL_BOOLEAN_FALSE);
186 if (SL_RESULT_SUCCESS != result) {
187 LOGE("Realize recorder object result:%s", getSLErrStr(result));
188 goto error;
189 }
190
191 result = (*mObjectInterface)->GetInterface(mObjectInterface, SL_IID_RECORD, &mRecordInterface);
192 if (SL_RESULT_SUCCESS != result) {
193 LOGE("GetInterface RECORD result:%s", getSLErrStr(result));
194 goto error;
195 }
196
197 result = AudioStreamOpenSLES::registerBufferQueueCallback();
198 if (SL_RESULT_SUCCESS != result) {
199 goto error;
200 }
201
202 result = updateStreamParameters(configItf);
203 if (SL_RESULT_SUCCESS != result) {
204 goto error;
205 }
206
207 oboeResult = configureBufferSizes(mSampleRate);
208 if (Result::OK != oboeResult) {
209 goto error;
210 }
211
212 allocateFifo();
213
214 setState(StreamState::Open);
215 return Result::OK;
216
217 error:
218 return Result::ErrorInternal; // TODO convert error from SLES to OBOE
219 }
220
close()221 Result AudioInputStreamOpenSLES::close() {
222 LOGD("AudioInputStreamOpenSLES::%s()", __func__);
223 std::lock_guard<std::mutex> lock(mLock);
224 Result result = Result::OK;
225 if (getState() == StreamState::Closed){
226 result = Result::ErrorClosed;
227 } else {
228 requestStop_l();
229 // invalidate any interfaces
230 mRecordInterface = nullptr;
231 result = AudioStreamOpenSLES::close_l();
232 }
233 return result;
234 }
235
setRecordState_l(SLuint32 newState)236 Result AudioInputStreamOpenSLES::setRecordState_l(SLuint32 newState) {
237 LOGD("AudioInputStreamOpenSLES::%s(%u)", __func__, newState);
238 Result result = Result::OK;
239
240 if (mRecordInterface == nullptr) {
241 LOGE("AudioInputStreamOpenSLES::%s() mRecordInterface is null", __func__);
242 return Result::ErrorInvalidState;
243 }
244 SLresult slResult = (*mRecordInterface)->SetRecordState(mRecordInterface, newState);
245 //LOGD("AudioInputStreamOpenSLES::%s(%u) returned %u", __func__, newState, slResult);
246 if (SL_RESULT_SUCCESS != slResult) {
247 LOGE("AudioInputStreamOpenSLES::%s(%u) returned error %s",
248 __func__, newState, getSLErrStr(slResult));
249 result = Result::ErrorInternal; // TODO review
250 }
251 return result;
252 }
253
requestStart()254 Result AudioInputStreamOpenSLES::requestStart() {
255 LOGD("AudioInputStreamOpenSLES(): %s() called", __func__);
256 std::lock_guard<std::mutex> lock(mLock);
257 StreamState initialState = getState();
258 switch (initialState) {
259 case StreamState::Starting:
260 case StreamState::Started:
261 return Result::OK;
262 case StreamState::Closed:
263 return Result::ErrorClosed;
264 default:
265 break;
266 }
267
268 // We use a callback if the user requests one
269 // OR if we have an internal callback to fill the blocking IO buffer.
270 setDataCallbackEnabled(true);
271
272 setState(StreamState::Starting);
273 Result result = setRecordState_l(SL_RECORDSTATE_RECORDING);
274 if (result == Result::OK) {
275 setState(StreamState::Started);
276 // Enqueue the first buffer to start the streaming.
277 // This does not call the callback function.
278 enqueueCallbackBuffer(mSimpleBufferQueueInterface);
279 } else {
280 setState(initialState);
281 }
282 return result;
283 }
284
285
requestPause()286 Result AudioInputStreamOpenSLES::requestPause() {
287 LOGW("AudioInputStreamOpenSLES::%s() is intentionally not implemented for input "
288 "streams", __func__);
289 return Result::ErrorUnimplemented; // Matches AAudio behavior.
290 }
291
requestFlush()292 Result AudioInputStreamOpenSLES::requestFlush() {
293 LOGW("AudioInputStreamOpenSLES::%s() is intentionally not implemented for input "
294 "streams", __func__);
295 return Result::ErrorUnimplemented; // Matches AAudio behavior.
296 }
297
requestStop()298 Result AudioInputStreamOpenSLES::requestStop() {
299 LOGD("AudioInputStreamOpenSLES(): %s() called", __func__);
300 std::lock_guard<std::mutex> lock(mLock);
301 return requestStop_l();
302 }
303
304 // Call under mLock
requestStop_l()305 Result AudioInputStreamOpenSLES::requestStop_l() {
306 StreamState initialState = getState();
307 switch (initialState) {
308 case StreamState::Stopping:
309 case StreamState::Stopped:
310 return Result::OK;
311 case StreamState::Closed:
312 return Result::ErrorClosed;
313 default:
314 break;
315 }
316
317 setState(StreamState::Stopping);
318
319 Result result = setRecordState_l(SL_RECORDSTATE_STOPPED);
320 if (result == Result::OK) {
321 mPositionMillis.reset32(); // OpenSL ES resets its millisecond position when stopped.
322 setState(StreamState::Stopped);
323 } else {
324 setState(initialState);
325 }
326 return result;
327 }
328
updateFramesWritten()329 void AudioInputStreamOpenSLES::updateFramesWritten() {
330 if (usingFIFO()) {
331 AudioStreamBuffered::updateFramesWritten();
332 } else {
333 mFramesWritten = getFramesProcessedByServer();
334 }
335 }
336
updateServiceFrameCounter()337 Result AudioInputStreamOpenSLES::updateServiceFrameCounter() {
338 Result result = Result::OK;
339 // Avoid deadlock if another thread is trying to stop or close this stream
340 // and this is being called from a callback.
341 if (mLock.try_lock()) {
342
343 if (mRecordInterface == nullptr) {
344 mLock.unlock();
345 return Result::ErrorNull;
346 }
347 SLmillisecond msec = 0;
348 SLresult slResult = (*mRecordInterface)->GetPosition(mRecordInterface, &msec);
349 if (SL_RESULT_SUCCESS != slResult) {
350 LOGW("%s(): GetPosition() returned %s", __func__, getSLErrStr(slResult));
351 // set result based on SLresult
352 result = Result::ErrorInternal;
353 } else {
354 mPositionMillis.update32(msec);
355 }
356 mLock.unlock();
357 }
358 return result;
359 }
360