1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.soundtrigger; 18 19 import static com.android.server.soundtrigger.DeviceStateHandler.SoundTriggerDeviceState; 20 import static com.android.server.soundtrigger.SoundTriggerEvent.SessionEvent.Type; 21 import static com.android.server.utils.EventLogger.Event.ALOGW; 22 import android.annotation.NonNull; 23 import android.annotation.Nullable; 24 import android.content.Context; 25 import android.hardware.soundtrigger.IRecognitionStatusCallback; 26 import android.hardware.soundtrigger.ModelParams; 27 import android.hardware.soundtrigger.SoundTrigger; 28 import android.hardware.soundtrigger.SoundTrigger.GenericRecognitionEvent; 29 import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; 30 import android.hardware.soundtrigger.SoundTrigger.Keyphrase; 31 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 32 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 33 import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 34 import android.hardware.soundtrigger.SoundTrigger.ModelParamRange; 35 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 36 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 37 import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 38 import android.hardware.soundtrigger.SoundTrigger.SoundModel; 39 import android.hardware.soundtrigger.SoundTriggerModule; 40 import android.os.Binder; 41 import android.os.DeadObjectException; 42 import android.os.Handler; 43 import android.os.IBinder; 44 import android.os.Looper; 45 import android.os.Message; 46 import android.os.RemoteException; 47 import android.util.Slog; 48 49 import com.android.internal.annotations.GuardedBy; 50 import com.android.internal.logging.MetricsLogger; 51 import com.android.server.soundtrigger.SoundTriggerEvent.SessionEvent; 52 import com.android.server.utils.EventLogger.Event; 53 import com.android.server.utils.EventLogger; 54 55 import java.io.FileDescriptor; 56 import java.io.PrintWriter; 57 import java.util.ArrayList; 58 import java.util.HashMap; 59 import java.util.Iterator; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.Objects; 63 import java.util.UUID; 64 import java.util.function.Function; 65 import java.util.function.Supplier; 66 67 /** 68 * Helper for {@link SoundTrigger} APIs. Supports two types of models: 69 * (i) A voice model which is exported via the {@link VoiceInteractionService}. There can only be 70 * a single voice model running on the DSP at any given time. 71 * 72 * (ii) Generic sound-trigger models: Supports multiple of these. 73 * 74 * Currently this just acts as an abstraction over all SoundTrigger API calls. 75 * @hide 76 */ 77 public class SoundTriggerHelper implements SoundTrigger.StatusListener { 78 static final String TAG = "SoundTriggerHelper"; 79 80 // Module ID if there is no available module to connect to. 81 public static final int INVALID_MODULE_ID = -1; 82 83 /** 84 * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, 85 * IRecognitionStatusCallback, RecognitionConfig)}, 86 * {@link #stopRecognition(int, IRecognitionStatusCallback)} 87 */ 88 public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 89 public static final int STATUS_OK = SoundTrigger.STATUS_OK; 90 91 private static final int INVALID_VALUE = Integer.MIN_VALUE; 92 93 private SoundTriggerModule mModule; 94 private final Object mLock = new Object(); 95 private final Context mContext; 96 97 // The SoundTriggerManager layer handles multiple recognition models of type generic and 98 // keyphrase. We store the ModelData here in a hashmap. 99 private final HashMap<UUID, ModelData> mModelDataMap = new HashMap<>(); 100 101 // An index of keyphrase sound models so that we can reach them easily. We support indexing 102 // keyphrase sound models with a keyphrase ID. Sound model with the same keyphrase ID will 103 // replace an existing model, thus there is a 1:1 mapping from keyphrase ID to a voice 104 // sound model. 105 private final HashMap<Integer, UUID> mKeyphraseUuidMap = new HashMap<>(); 106 107 // Whether ANY recognition (keyphrase or generic) has been requested. 108 private boolean mRecognitionRequested = false; 109 110 // TODO(b/269366605) Temporary solution to query correct moduleProperties 111 private final int mModuleId; 112 private final Function<SoundTrigger.StatusListener, SoundTriggerModule> mModuleProvider; 113 private final Supplier<List<ModuleProperties>> mModulePropertiesProvider; 114 private final EventLogger mEventLogger; 115 116 @GuardedBy("mLock") 117 private boolean mIsDetached = false; 118 119 @GuardedBy("mLock") 120 private SoundTriggerDeviceState mDeviceState = SoundTriggerDeviceState.DISABLE; 121 122 @GuardedBy("mLock") 123 private boolean mIsAppOpPermitted = true; 124 SoundTriggerHelper(Context context, EventLogger eventLogger, @NonNull Function<SoundTrigger.StatusListener, SoundTriggerModule> moduleProvider, int moduleId, @NonNull Supplier<List<ModuleProperties>> modulePropertiesProvider)125 SoundTriggerHelper(Context context, EventLogger eventLogger, 126 @NonNull Function<SoundTrigger.StatusListener, SoundTriggerModule> moduleProvider, 127 int moduleId, 128 @NonNull Supplier<List<ModuleProperties>> modulePropertiesProvider) { 129 mModuleId = moduleId; 130 mContext = context; 131 mModuleProvider = moduleProvider; 132 mEventLogger = eventLogger; 133 mModulePropertiesProvider = modulePropertiesProvider; 134 if (moduleId == INVALID_MODULE_ID) { 135 mModule = null; 136 } else { 137 mModule = mModuleProvider.apply(this); 138 } 139 } 140 141 /** 142 * Starts recognition for the given generic sound model ID. This is a wrapper around {@link 143 * startRecognition()}. 144 * 145 * @param modelId UUID of the sound model. 146 * @param soundModel The generic sound model to use for recognition. 147 * @param callback Callack for the recognition events related to the given keyphrase. 148 * @param recognitionConfig Instance of RecognitionConfig containing the parameters for the 149 * recognition. 150 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 151 */ startGenericRecognition(UUID modelId, GenericSoundModel soundModel, IRecognitionStatusCallback callback, RecognitionConfig recognitionConfig, boolean runInBatterySaverMode)152 public int startGenericRecognition(UUID modelId, GenericSoundModel soundModel, 153 IRecognitionStatusCallback callback, RecognitionConfig recognitionConfig, 154 boolean runInBatterySaverMode) { 155 MetricsLogger.count(mContext, "sth_start_recognition", 1); 156 if (modelId == null || soundModel == null || callback == null || 157 recognitionConfig == null) { 158 Slog.w(TAG, "Passed in bad data to startGenericRecognition()."); 159 return STATUS_ERROR; 160 } 161 162 synchronized (mLock) { 163 if (mIsDetached) { 164 throw new IllegalStateException("SoundTriggerHelper has been detached"); 165 } 166 ModelData modelData = getOrCreateGenericModelDataLocked(modelId); 167 if (modelData == null) { 168 Slog.w(TAG, "Irrecoverable error occurred, check UUID / sound model data."); 169 return STATUS_ERROR; 170 } 171 return startRecognition(soundModel, modelData, callback, recognitionConfig, 172 INVALID_VALUE /* keyphraseId */, runInBatterySaverMode); 173 } 174 } 175 176 /** 177 * Starts recognition for the given keyphraseId. 178 * 179 * @param keyphraseId The identifier of the keyphrase for which 180 * the recognition is to be started. 181 * @param soundModel The sound model to use for recognition. 182 * @param callback The callback for the recognition events related to the given keyphrase. 183 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 184 */ startKeyphraseRecognition(int keyphraseId, KeyphraseSoundModel soundModel, IRecognitionStatusCallback callback, RecognitionConfig recognitionConfig, boolean runInBatterySaverMode)185 public int startKeyphraseRecognition(int keyphraseId, KeyphraseSoundModel soundModel, 186 IRecognitionStatusCallback callback, RecognitionConfig recognitionConfig, 187 boolean runInBatterySaverMode) { 188 synchronized (mLock) { 189 MetricsLogger.count(mContext, "sth_start_recognition", 1); 190 if (soundModel == null || callback == null || recognitionConfig == null) { 191 return STATUS_ERROR; 192 } 193 194 if (mIsDetached) { 195 throw new IllegalStateException("SoundTriggerHelper has been detached"); 196 } 197 198 ModelData model = getKeyphraseModelDataLocked(keyphraseId); 199 if (model != null && !model.isKeyphraseModel()) { 200 Slog.e(TAG, "Generic model with same UUID exists."); 201 return STATUS_ERROR; 202 } 203 204 // Process existing model first. 205 if (model != null && !model.getModelId().equals(soundModel.getUuid())) { 206 // The existing model has a different UUID, should be replaced. 207 int status = cleanUpExistingKeyphraseModelLocked(model); 208 if (status != STATUS_OK) { 209 return status; 210 } 211 removeKeyphraseModelLocked(keyphraseId); 212 model = null; 213 } 214 215 // We need to create a new one: either no previous models existed for given keyphrase id 216 // or the existing model had a different UUID and was cleaned up. 217 if (model == null) { 218 model = createKeyphraseModelDataLocked(soundModel.getUuid(), keyphraseId); 219 } 220 221 return startRecognition(soundModel, model, callback, recognitionConfig, 222 keyphraseId, runInBatterySaverMode); 223 } 224 } 225 cleanUpExistingKeyphraseModelLocked(ModelData modelData)226 private int cleanUpExistingKeyphraseModelLocked(ModelData modelData) { 227 // Stop and clean up a previous ModelData if one exists. This usually is used when the 228 // previous model has a different UUID for the same keyphrase ID. 229 int status = tryStopAndUnloadLocked(modelData, true /* stop */, true /* unload */); 230 if (status != STATUS_OK) { 231 Slog.w(TAG, "Unable to stop or unload previous model: " + 232 modelData.toString()); 233 } 234 return status; 235 } 236 prepareForRecognition(ModelData modelData)237 private int prepareForRecognition(ModelData modelData) { 238 if (mModule == null) { 239 Slog.w(TAG, "prepareForRecognition: cannot attach to sound trigger module"); 240 return STATUS_ERROR; 241 } 242 // Load the model if it is not loaded. 243 if (!modelData.isModelLoaded()) { 244 // Before we try and load this model, we should first make sure that any other 245 // models that don't have an active recognition/dead callback are unloaded. Since 246 // there is a finite limit on the number of models that the hardware may be able to 247 // have loaded, we want to make sure there's room for our model. 248 stopAndUnloadDeadModelsLocked(); 249 int[] handle = new int[] { 0 }; 250 int status = mModule.loadSoundModel(modelData.getSoundModel(), handle); 251 if (status != SoundTrigger.STATUS_OK) { 252 Slog.w(TAG, "prepareForRecognition: loadSoundModel failed with status: " + status); 253 return status; 254 } 255 modelData.setHandle(handle[0]); 256 modelData.setLoaded(); 257 } 258 return STATUS_OK; 259 } 260 261 262 /** 263 * Starts recognition for the given sound model. A single routine for both keyphrase and 264 * generic sound models. 265 * 266 * @param soundModel The sound model to use for recognition. 267 * @param modelData Instance of {@link #ModelData} for the given model. 268 * @param callback Callback for the recognition events related to the given keyphrase. 269 * @param recognitionConfig Instance of {@link RecognitionConfig} containing the parameters 270 * @param keyphraseId Keyphrase ID for keyphrase models only. Pass in INVALID_VALUE for other 271 * models. 272 * for the recognition. 273 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 274 */ startRecognition(SoundModel soundModel, ModelData modelData, IRecognitionStatusCallback callback, RecognitionConfig recognitionConfig, int keyphraseId, boolean runInBatterySaverMode)275 private int startRecognition(SoundModel soundModel, ModelData modelData, 276 IRecognitionStatusCallback callback, RecognitionConfig recognitionConfig, 277 int keyphraseId, boolean runInBatterySaverMode) { 278 synchronized (mLock) { 279 // TODO Remove previous callback handling 280 IRecognitionStatusCallback oldCallback = modelData.getCallback(); 281 if (oldCallback != null && oldCallback.asBinder() != callback.asBinder()) { 282 Slog.w(TAG, "Canceling previous recognition for model id: " 283 + modelData.getModelId()); 284 try { 285 oldCallback.onPreempted(); 286 } catch (RemoteException e) { 287 Slog.w(TAG, "RemoteException in onDetectionStopped", e); 288 } 289 modelData.clearCallback(); 290 } 291 292 // If the existing SoundModel is different (for the same UUID for Generic and same 293 // keyphrase ID for voice), ensure that it is unloaded and stopped before proceeding. 294 // This works for both keyphrase and generic models. This logic also ensures that a 295 // previously loaded (or started) model is appropriately stopped. Since this is a 296 // generalization of the previous logic with a single keyphrase model, we should have 297 // no regression with the previous version of this code as was given in the 298 // startKeyphrase() routine. 299 if (modelData.getSoundModel() != null) { 300 boolean stopModel = false; // Stop the model after checking that it is started. 301 boolean unloadModel = false; 302 if (modelData.getSoundModel().equals(soundModel) && modelData.isModelStarted()) { 303 // The model has not changed, but the previous model is "started". 304 // Stop the previously running model. 305 stopModel = true; 306 unloadModel = false; // No need to unload if the model hasn't changed. 307 } else if (!modelData.getSoundModel().equals(soundModel)) { 308 // We have a different model for this UUID. Stop and unload if needed. This 309 // helps maintain the singleton restriction for keyphrase sound models. 310 stopModel = modelData.isModelStarted(); 311 unloadModel = modelData.isModelLoaded(); 312 } 313 if (stopModel || unloadModel) { 314 int status = tryStopAndUnloadLocked(modelData, stopModel, unloadModel); 315 if (status != STATUS_OK) { 316 Slog.w(TAG, "Unable to stop or unload previous model: " + 317 modelData.toString()); 318 return status; 319 } 320 } 321 } 322 323 modelData.setCallback(callback); 324 modelData.setRequested(true); 325 modelData.setRecognitionConfig(recognitionConfig); 326 modelData.setRunInBatterySaverMode(runInBatterySaverMode); 327 modelData.setSoundModel(soundModel); 328 329 if (isRecognitionAllowed(modelData)) { 330 int startRecoResult = updateRecognitionLocked(modelData, 331 false /* Don't notify for synchronous calls */); 332 if (startRecoResult == SoundTrigger.STATUS_OK) { 333 return startRecoResult; 334 } else if (startRecoResult != SoundTrigger.STATUS_BUSY) { 335 // If we are returning an unexpected error, don't mark the model as requested 336 modelData.setRequested(false); 337 return startRecoResult; 338 } 339 } 340 // Either recognition isn't allowed by device state, or the module is busy. 341 // Dispatch a pause. 342 try { 343 if (callback != null) { 344 mEventLogger.enqueue(new SessionEvent(Type.PAUSE, modelData.getModelId())); 345 callback.onRecognitionPaused(); 346 } 347 } catch (RemoteException e) { 348 mEventLogger.enqueue(new SessionEvent( 349 Type.PAUSE, modelData.getModelId(), "RemoteException") 350 .printLog(ALOGW, TAG)); 351 forceStopAndUnloadModelLocked(modelData, e); 352 } 353 return STATUS_OK; 354 } 355 } 356 357 /** 358 * Stops recognition for the given generic sound model. This is a wrapper for {@link 359 * #stopRecognition}. 360 * 361 * @param modelId The identifier of the generic sound model for which 362 * the recognition is to be stopped. 363 * @param callback The callback for the recognition events related to the given sound model. 364 * 365 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 366 */ stopGenericRecognition(UUID modelId, IRecognitionStatusCallback callback)367 public int stopGenericRecognition(UUID modelId, IRecognitionStatusCallback callback) { 368 synchronized (mLock) { 369 MetricsLogger.count(mContext, "sth_stop_recognition", 1); 370 if (callback == null || modelId == null) { 371 Slog.e(TAG, "Null callbackreceived for stopGenericRecognition() for modelid:" + 372 modelId); 373 return STATUS_ERROR; 374 } 375 if (mIsDetached) { 376 throw new IllegalStateException("SoundTriggerHelper has been detached"); 377 } 378 ModelData modelData = mModelDataMap.get(modelId); 379 if (modelData == null || !modelData.isGenericModel()) { 380 Slog.w(TAG, "Attempting stopRecognition on invalid model with id:" + modelId); 381 return STATUS_ERROR; 382 } 383 384 int status = stopRecognition(modelData, callback); 385 if (status != SoundTrigger.STATUS_OK) { 386 Slog.w(TAG, "stopGenericRecognition failed: " + status); 387 } 388 return status; 389 } 390 } 391 392 /** 393 * Stops recognition for the given {@link Keyphrase} if a recognition is 394 * currently active. This is a wrapper for {@link #stopRecognition()}. 395 * 396 * @param keyphraseId The identifier of the keyphrase for which 397 * the recognition is to be stopped. 398 * @param callback The callback for the recognition events related to the given keyphrase. 399 * 400 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 401 */ stopKeyphraseRecognition(int keyphraseId, IRecognitionStatusCallback callback)402 public int stopKeyphraseRecognition(int keyphraseId, IRecognitionStatusCallback callback) { 403 synchronized (mLock) { 404 MetricsLogger.count(mContext, "sth_stop_recognition", 1); 405 if (callback == null) { 406 Slog.e(TAG, "Null callback received for stopKeyphraseRecognition() for keyphraseId:" + 407 keyphraseId); 408 return STATUS_ERROR; 409 } 410 if (mIsDetached) { 411 throw new IllegalStateException("SoundTriggerHelper has been detached"); 412 } 413 ModelData modelData = getKeyphraseModelDataLocked(keyphraseId); 414 if (modelData == null || !modelData.isKeyphraseModel()) { 415 Slog.w(TAG, "No model exists for given keyphrase Id " + keyphraseId); 416 return STATUS_ERROR; 417 } 418 419 int status = stopRecognition(modelData, callback); 420 if (status != SoundTrigger.STATUS_OK) { 421 return status; 422 } 423 424 return status; 425 } 426 } 427 428 /** 429 * Stops recognition for the given ModelData instance. 430 * 431 * @param modelData Instance of {@link #ModelData} sound model. 432 * @param callback The callback for the recognition events related to the given keyphrase. 433 * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. 434 */ stopRecognition(ModelData modelData, IRecognitionStatusCallback callback)435 private int stopRecognition(ModelData modelData, IRecognitionStatusCallback callback) { 436 synchronized (mLock) { 437 if (callback == null) { 438 return STATUS_ERROR; 439 } 440 if (mModule == null) { 441 Slog.w(TAG, "Attempting stopRecognition after detach"); 442 return STATUS_ERROR; 443 } 444 445 IRecognitionStatusCallback currentCallback = modelData.getCallback(); 446 if (modelData == null || currentCallback == null || 447 (!modelData.isRequested() && !modelData.isModelStarted())) { 448 // startGenericRecognition hasn't been called or it failed. 449 Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); 450 return STATUS_ERROR; 451 } 452 453 if (currentCallback.asBinder() != callback.asBinder()) { 454 // We don't allow a different listener to stop the recognition than the one 455 // that started it. 456 Slog.w(TAG, "Attempting stopRecognition for another recognition"); 457 return STATUS_ERROR; 458 } 459 460 // Request stop recognition via the update() method. 461 modelData.setRequested(false); 462 int status = updateRecognitionLocked(modelData, false); 463 if (status != SoundTrigger.STATUS_OK) { 464 return status; 465 } 466 467 // We leave the sound model loaded but not started, this helps us when we start back. 468 // Also clear the internal state once the recognition has been stopped. 469 modelData.setLoaded(); 470 modelData.clearCallback(); 471 modelData.setRecognitionConfig(null); 472 return status; 473 } 474 } 475 476 // Stop a previously started model if it was started. Optionally, unload if the previous model 477 // is stale and is about to be replaced. 478 // Needs to be called with the mLock held. tryStopAndUnloadLocked(ModelData modelData, boolean stopModel, boolean unloadModel)479 private int tryStopAndUnloadLocked(ModelData modelData, boolean stopModel, 480 boolean unloadModel) { 481 int status = STATUS_OK; 482 if (modelData.isModelNotLoaded()) { 483 return status; 484 } 485 if (stopModel && modelData.isModelStarted()) { 486 status = stopRecognitionLocked(modelData, 487 false /* don't notify for synchronous calls */); 488 if (status != SoundTrigger.STATUS_OK) { 489 Slog.w(TAG, "stopRecognition failed: " + status); 490 return status; 491 } 492 } 493 494 if (unloadModel && modelData.isModelLoaded()) { 495 Slog.d(TAG, "Unloading previously loaded stale model."); 496 if (mModule == null) { 497 return STATUS_ERROR; 498 } 499 status = mModule.unloadSoundModel(modelData.getHandle()); 500 MetricsLogger.count(mContext, "sth_unloading_stale_model", 1); 501 if (status != SoundTrigger.STATUS_OK) { 502 Slog.w(TAG, "unloadSoundModel call failed with " + status); 503 } else { 504 // Clear the ModelData state if successful. 505 modelData.clearState(); 506 } 507 } 508 return status; 509 } 510 getModuleProperties()511 public ModuleProperties getModuleProperties() { 512 synchronized (mLock) { 513 if (mIsDetached) { 514 throw new IllegalStateException("SoundTriggerHelper has been detached"); 515 } 516 } 517 for (ModuleProperties moduleProperties : mModulePropertiesProvider.get()) { 518 if (moduleProperties.getId() == mModuleId) { 519 return moduleProperties; 520 } 521 } 522 Slog.e(TAG, "Module properties not found for existing moduleId " + mModuleId); 523 return null; 524 } 525 unloadKeyphraseSoundModel(int keyphraseId)526 public int unloadKeyphraseSoundModel(int keyphraseId) { 527 synchronized (mLock) { 528 MetricsLogger.count(mContext, "sth_unload_keyphrase_sound_model", 1); 529 ModelData modelData = getKeyphraseModelDataLocked(keyphraseId); 530 if (mModule == null || modelData == null || !modelData.isModelLoaded() 531 || !modelData.isKeyphraseModel()) { 532 return STATUS_ERROR; 533 } 534 if (mIsDetached) { 535 throw new IllegalStateException("SoundTriggerHelper has been detached"); 536 } 537 // Stop recognition if it's the current one. 538 modelData.setRequested(false); 539 int status = updateRecognitionLocked(modelData, false); 540 if (status != SoundTrigger.STATUS_OK) { 541 Slog.w(TAG, "Stop recognition failed for keyphrase ID:" + status); 542 } 543 544 status = mModule.unloadSoundModel(modelData.getHandle()); 545 if (status != SoundTrigger.STATUS_OK) { 546 Slog.w(TAG, "unloadKeyphraseSoundModel call failed with " + status); 547 } 548 549 // Remove it from existence. 550 removeKeyphraseModelLocked(keyphraseId); 551 return status; 552 } 553 } 554 unloadGenericSoundModel(UUID modelId)555 public int unloadGenericSoundModel(UUID modelId) { 556 synchronized (mLock) { 557 MetricsLogger.count(mContext, "sth_unload_generic_sound_model", 1); 558 if (modelId == null || mModule == null) { 559 return STATUS_ERROR; 560 } 561 if (mIsDetached) { 562 throw new IllegalStateException("SoundTriggerHelper has been detached"); 563 } 564 ModelData modelData = mModelDataMap.get(modelId); 565 if (modelData == null || !modelData.isGenericModel()) { 566 Slog.w(TAG, "Unload error: Attempting unload invalid generic model with id:" + 567 modelId); 568 return STATUS_ERROR; 569 } 570 if (!modelData.isModelLoaded()) { 571 // Nothing to do here. 572 Slog.i(TAG, "Unload: Given generic model is not loaded:" + modelId); 573 return STATUS_OK; 574 } 575 if (modelData.isModelStarted()) { 576 int status = stopRecognitionLocked(modelData, 577 false /* don't notify for synchronous calls */); 578 if (status != SoundTrigger.STATUS_OK) { 579 Slog.w(TAG, "stopGenericRecognition failed: " + status); 580 } 581 } 582 583 if (mModule == null) { 584 return STATUS_ERROR; 585 } 586 int status = mModule.unloadSoundModel(modelData.getHandle()); 587 if (status != SoundTrigger.STATUS_OK) { 588 Slog.w(TAG, "unloadGenericSoundModel() call failed with " + status); 589 Slog.w(TAG, "unloadGenericSoundModel() force-marking model as unloaded."); 590 } 591 592 // Remove it from existence. 593 mModelDataMap.remove(modelId); 594 return status; 595 } 596 } 597 isRecognitionRequested(UUID modelId)598 public boolean isRecognitionRequested(UUID modelId) { 599 synchronized (mLock) { 600 if (mIsDetached) { 601 throw new IllegalStateException("SoundTriggerHelper has been detached"); 602 } 603 ModelData modelData = mModelDataMap.get(modelId); 604 return modelData != null && modelData.isRequested(); 605 } 606 } 607 onDeviceStateChanged(SoundTriggerDeviceState state)608 public void onDeviceStateChanged(SoundTriggerDeviceState state) { 609 synchronized (mLock) { 610 if (mIsDetached || mDeviceState == state) { 611 // Nothing to update 612 return; 613 } 614 mDeviceState = state; 615 updateAllRecognitionsLocked(); 616 } 617 } 618 onAppOpStateChanged(boolean isPermitted)619 public void onAppOpStateChanged(boolean isPermitted) { 620 synchronized (mLock) { 621 if (mIsAppOpPermitted == isPermitted) { 622 return; 623 } 624 mIsAppOpPermitted = isPermitted; 625 updateAllRecognitionsLocked(); 626 } 627 } 628 getGenericModelState(UUID modelId)629 public int getGenericModelState(UUID modelId) { 630 synchronized (mLock) { 631 MetricsLogger.count(mContext, "sth_get_generic_model_state", 1); 632 if (modelId == null || mModule == null) { 633 return STATUS_ERROR; 634 } 635 if (mIsDetached) { 636 throw new IllegalStateException("SoundTriggerHelper has been detached"); 637 } 638 ModelData modelData = mModelDataMap.get(modelId); 639 if (modelData == null || !modelData.isGenericModel()) { 640 Slog.w(TAG, "GetGenericModelState error: Invalid generic model id:" + 641 modelId); 642 return STATUS_ERROR; 643 } 644 if (!modelData.isModelLoaded()) { 645 Slog.i(TAG, "GetGenericModelState: Given generic model is not loaded:" + modelId); 646 return STATUS_ERROR; 647 } 648 if (!modelData.isModelStarted()) { 649 Slog.i(TAG, "GetGenericModelState: Given generic model is not started:" + modelId); 650 return STATUS_ERROR; 651 } 652 653 return mModule.getModelState(modelData.getHandle()); 654 } 655 } 656 setParameter(UUID modelId, @ModelParams int modelParam, int value)657 public int setParameter(UUID modelId, @ModelParams int modelParam, int value) { 658 synchronized (mLock) { 659 if (mIsDetached) { 660 throw new IllegalStateException("SoundTriggerHelper has been detached"); 661 } 662 return setParameterLocked(mModelDataMap.get(modelId), modelParam, value); 663 } 664 } 665 setKeyphraseParameter(int keyphraseId, @ModelParams int modelParam, int value)666 public int setKeyphraseParameter(int keyphraseId, @ModelParams int modelParam, int value) { 667 synchronized (mLock) { 668 if (mIsDetached) { 669 throw new IllegalStateException("SoundTriggerHelper has been detached"); 670 } 671 return setParameterLocked(getKeyphraseModelDataLocked(keyphraseId), modelParam, value); 672 } 673 } 674 setParameterLocked(@ullable ModelData modelData, @ModelParams int modelParam, int value)675 private int setParameterLocked(@Nullable ModelData modelData, @ModelParams int modelParam, 676 int value) { 677 MetricsLogger.count(mContext, "sth_set_parameter", 1); 678 if (mModule == null) { 679 return SoundTrigger.STATUS_NO_INIT; 680 } 681 if (modelData == null || !modelData.isModelLoaded()) { 682 Slog.i(TAG, "SetParameter: Given model is not loaded:" + modelData); 683 return SoundTrigger.STATUS_BAD_VALUE; 684 } 685 686 return mModule.setParameter(modelData.getHandle(), modelParam, value); 687 } 688 getParameter(@onNull UUID modelId, @ModelParams int modelParam)689 public int getParameter(@NonNull UUID modelId, @ModelParams int modelParam) { 690 synchronized (mLock) { 691 if (mIsDetached) { 692 throw new IllegalStateException("SoundTriggerHelper has been detached"); 693 } 694 return getParameterLocked(mModelDataMap.get(modelId), modelParam); 695 } 696 } 697 getKeyphraseParameter(int keyphraseId, @ModelParams int modelParam)698 public int getKeyphraseParameter(int keyphraseId, @ModelParams int modelParam) { 699 synchronized (mLock) { 700 if (mIsDetached) { 701 throw new IllegalStateException("SoundTriggerHelper has been detached"); 702 } 703 return getParameterLocked(getKeyphraseModelDataLocked(keyphraseId), modelParam); 704 } 705 } 706 getParameterLocked(@ullable ModelData modelData, @ModelParams int modelParam)707 private int getParameterLocked(@Nullable ModelData modelData, @ModelParams int modelParam) { 708 MetricsLogger.count(mContext, "sth_get_parameter", 1); 709 if (mModule == null) { 710 throw new UnsupportedOperationException("SoundTriggerModule not initialized"); 711 } 712 713 if (modelData == null) { 714 throw new IllegalArgumentException("Invalid model id"); 715 } 716 if (!modelData.isModelLoaded()) { 717 throw new UnsupportedOperationException("Given model is not loaded:" + modelData); 718 } 719 720 return mModule.getParameter(modelData.getHandle(), modelParam); 721 } 722 723 @Nullable queryParameter(@onNull UUID modelId, @ModelParams int modelParam)724 public ModelParamRange queryParameter(@NonNull UUID modelId, @ModelParams int modelParam) { 725 synchronized (mLock) { 726 if (mIsDetached) { 727 throw new IllegalStateException("SoundTriggerHelper has been detached"); 728 } 729 return queryParameterLocked(mModelDataMap.get(modelId), modelParam); 730 } 731 } 732 733 @Nullable queryKeyphraseParameter(int keyphraseId, @ModelParams int modelParam)734 public ModelParamRange queryKeyphraseParameter(int keyphraseId, @ModelParams int modelParam) { 735 synchronized (mLock) { 736 if (mIsDetached) { 737 throw new IllegalStateException("SoundTriggerHelper has been detached"); 738 } 739 return queryParameterLocked(getKeyphraseModelDataLocked(keyphraseId), modelParam); 740 } 741 } 742 743 @Nullable queryParameterLocked(@ullable ModelData modelData, @ModelParams int modelParam)744 private ModelParamRange queryParameterLocked(@Nullable ModelData modelData, 745 @ModelParams int modelParam) { 746 MetricsLogger.count(mContext, "sth_query_parameter", 1); 747 if (mModule == null) { 748 return null; 749 } 750 if (modelData == null) { 751 Slog.w(TAG, "queryParameter: Invalid model id"); 752 return null; 753 } 754 if (!modelData.isModelLoaded()) { 755 Slog.i(TAG, "queryParameter: Given model is not loaded:" + modelData); 756 return null; 757 } 758 759 return mModule.queryParameter(modelData.getHandle(), modelParam); 760 } 761 762 //---- SoundTrigger.StatusListener methods 763 @Override onRecognition(RecognitionEvent event)764 public void onRecognition(RecognitionEvent event) { 765 if (event == null) { 766 Slog.w(TAG, "Null recognition event!"); 767 return; 768 } 769 770 if (!(event instanceof KeyphraseRecognitionEvent) && 771 !(event instanceof GenericRecognitionEvent)) { 772 Slog.w(TAG, "Invalid recognition event type (not one of generic or keyphrase)!"); 773 return; 774 } 775 776 synchronized (mLock) { 777 switch (event.status) { 778 case SoundTrigger.RECOGNITION_STATUS_ABORT: 779 onRecognitionAbortLocked(event); 780 break; 781 case SoundTrigger.RECOGNITION_STATUS_FAILURE: 782 case SoundTrigger.RECOGNITION_STATUS_SUCCESS: 783 case SoundTrigger.RECOGNITION_STATUS_GET_STATE_RESPONSE: 784 if (isKeyphraseRecognitionEvent(event)) { 785 onKeyphraseRecognitionLocked((KeyphraseRecognitionEvent) event); 786 } else { 787 onGenericRecognitionLocked((GenericRecognitionEvent) event); 788 } 789 break; 790 } 791 } 792 } 793 isKeyphraseRecognitionEvent(RecognitionEvent event)794 private boolean isKeyphraseRecognitionEvent(RecognitionEvent event) { 795 return event instanceof KeyphraseRecognitionEvent; 796 } 797 798 @GuardedBy("mLock") onGenericRecognitionLocked(GenericRecognitionEvent event)799 private void onGenericRecognitionLocked(GenericRecognitionEvent event) { 800 MetricsLogger.count(mContext, "sth_generic_recognition_event", 1); 801 if (event.status != SoundTrigger.RECOGNITION_STATUS_SUCCESS 802 && event.status != SoundTrigger.RECOGNITION_STATUS_GET_STATE_RESPONSE) { 803 return; 804 } 805 ModelData model = getModelDataForLocked(event.soundModelHandle); 806 if (model == null || !model.isGenericModel()) { 807 Slog.w(TAG, "Generic recognition event: Model does not exist for handle: " 808 + event.soundModelHandle); 809 return; 810 } 811 if (!Objects.equals(event.getToken(), model.getToken())) { 812 // Stale event, do nothing 813 return; 814 } 815 816 IRecognitionStatusCallback callback = model.getCallback(); 817 if (callback == null) { 818 Slog.w(TAG, "Generic recognition event: Null callback for model handle: " 819 + event.soundModelHandle); 820 return; 821 } 822 823 if (!event.recognitionStillActive) { 824 model.setStopped(); 825 } 826 827 try { 828 mEventLogger.enqueue(new SessionEvent(Type.RECOGNITION, model.getModelId())); 829 callback.onGenericSoundTriggerDetected((GenericRecognitionEvent) event); 830 } catch (RemoteException e) { 831 mEventLogger.enqueue(new SessionEvent( 832 Type.RECOGNITION, model.getModelId(), "RemoteException") 833 .printLog(ALOGW, TAG)); 834 forceStopAndUnloadModelLocked(model, e); 835 return; 836 } 837 838 RecognitionConfig config = model.getRecognitionConfig(); 839 if (config == null) { 840 Slog.w(TAG, "Generic recognition event: Null RecognitionConfig for model handle: " 841 + event.soundModelHandle); 842 return; 843 } 844 845 model.setRequested(config.allowMultipleTriggers); 846 // TODO: Remove this block if the lower layer supports multiple triggers. 847 if (model.isRequested()) { 848 updateRecognitionLocked(model, true); 849 } 850 } 851 852 @Override onModelUnloaded(int modelHandle)853 public void onModelUnloaded(int modelHandle) { 854 synchronized (mLock) { 855 MetricsLogger.count(mContext, "sth_sound_model_updated", 1); 856 onModelUnloadedLocked(modelHandle); 857 } 858 } 859 860 @Override onResourcesAvailable()861 public void onResourcesAvailable() { 862 synchronized (mLock) { 863 onResourcesAvailableLocked(); 864 } 865 } 866 867 @Override onServiceDied()868 public void onServiceDied() { 869 Slog.e(TAG, "onServiceDied!!"); 870 MetricsLogger.count(mContext, "sth_service_died", 1); 871 synchronized (mLock) { 872 onServiceDiedLocked(); 873 } 874 } 875 onModelUnloadedLocked(int modelHandle)876 private void onModelUnloadedLocked(int modelHandle) { 877 ModelData modelData = getModelDataForLocked(modelHandle); 878 if (modelData != null) { 879 modelData.setNotLoaded(); 880 } 881 } 882 883 @GuardedBy("mLock") onResourcesAvailableLocked()884 private void onResourcesAvailableLocked() { 885 mEventLogger.enqueue(new SessionEvent(Type.RESOURCES_AVAILABLE, null)); 886 updateAllRecognitionsLocked(); 887 } 888 onRecognitionAbortLocked(RecognitionEvent event)889 private void onRecognitionAbortLocked(RecognitionEvent event) { 890 Slog.w(TAG, "Recognition aborted"); 891 MetricsLogger.count(mContext, "sth_recognition_aborted", 1); 892 ModelData modelData = getModelDataForLocked(event.soundModelHandle); 893 if (modelData != null && modelData.isModelStarted()) { 894 if (!Objects.equals(event.getToken(), modelData.getToken())) { 895 // Stale event, do nothing 896 return; 897 } 898 modelData.setStopped(); 899 try { 900 IRecognitionStatusCallback callback = modelData.getCallback(); 901 if (callback != null) { 902 mEventLogger.enqueue(new SessionEvent(Type.PAUSE, modelData.getModelId())); 903 callback.onRecognitionPaused(); 904 } 905 } catch (RemoteException e) { 906 mEventLogger.enqueue(new SessionEvent( 907 Type.PAUSE, modelData.getModelId(), "RemoteException") 908 .printLog(ALOGW, TAG)); 909 forceStopAndUnloadModelLocked(modelData, e); 910 } 911 } 912 } 913 getKeyphraseIdFromEvent(KeyphraseRecognitionEvent event)914 private int getKeyphraseIdFromEvent(KeyphraseRecognitionEvent event) { 915 if (event == null) { 916 Slog.w(TAG, "Null RecognitionEvent received."); 917 return INVALID_VALUE; 918 } 919 KeyphraseRecognitionExtra[] keyphraseExtras = 920 ((KeyphraseRecognitionEvent) event).keyphraseExtras; 921 if (keyphraseExtras == null || keyphraseExtras.length == 0) { 922 Slog.w(TAG, "Invalid keyphrase recognition event!"); 923 return INVALID_VALUE; 924 } 925 // TODO: Handle more than one keyphrase extras. 926 return keyphraseExtras[0].id; 927 } 928 929 @GuardedBy("mLock") onKeyphraseRecognitionLocked(KeyphraseRecognitionEvent event)930 private void onKeyphraseRecognitionLocked(KeyphraseRecognitionEvent event) { 931 Slog.i(TAG, "Recognition success"); 932 MetricsLogger.count(mContext, "sth_keyphrase_recognition_event", 1); 933 int keyphraseId = getKeyphraseIdFromEvent(event); 934 ModelData modelData = getKeyphraseModelDataLocked(keyphraseId); 935 936 if (modelData == null || !modelData.isKeyphraseModel()) { 937 Slog.e(TAG, "Keyphase model data does not exist for ID:" + keyphraseId); 938 return; 939 } 940 if (!Objects.equals(event.getToken(), modelData.getToken())) { 941 // Stale event, do nothing 942 return; 943 } 944 if (modelData.getCallback() == null) { 945 Slog.w(TAG, "Received onRecognition event without callback for keyphrase model."); 946 return; 947 } 948 949 if (!event.recognitionStillActive) { 950 modelData.setStopped(); 951 } 952 953 try { 954 mEventLogger.enqueue(new SessionEvent(Type.RECOGNITION, modelData.getModelId())); 955 modelData.getCallback().onKeyphraseDetected((KeyphraseRecognitionEvent) event); 956 } catch (RemoteException e) { 957 mEventLogger.enqueue(new SessionEvent( 958 Type.RECOGNITION, modelData.getModelId(), "RemoteException") 959 .printLog(ALOGW, TAG)); 960 forceStopAndUnloadModelLocked(modelData, e); 961 return; 962 } 963 964 RecognitionConfig config = modelData.getRecognitionConfig(); 965 if (config != null) { 966 // Whether we should continue by starting this again. 967 modelData.setRequested(config.allowMultipleTriggers); 968 } 969 // TODO: Remove this block if the lower layer supports multiple triggers. 970 if (modelData.isRequested()) { 971 updateRecognitionLocked(modelData, true); 972 } 973 } 974 975 @GuardedBy("mLock") updateAllRecognitionsLocked()976 private void updateAllRecognitionsLocked() { 977 // updateRecognitionLocked can possibly update the list of models 978 ArrayList<ModelData> modelDatas = new ArrayList<ModelData>(mModelDataMap.values()); 979 for (ModelData modelData : modelDatas) { 980 updateRecognitionLocked(modelData, true); 981 } 982 } 983 984 @GuardedBy("mLock") updateRecognitionLocked(ModelData model, boolean notifyClientOnError)985 private int updateRecognitionLocked(ModelData model, boolean notifyClientOnError) { 986 boolean shouldStartModel = model.isRequested() && isRecognitionAllowed(model); 987 if (shouldStartModel == model.isModelStarted()) { 988 // No-op. 989 return STATUS_OK; 990 } 991 if (shouldStartModel) { 992 int status = prepareForRecognition(model); 993 if (status != STATUS_OK) { 994 Slog.w(TAG, "startRecognition failed to prepare model for recognition"); 995 return status; 996 } 997 status = startRecognitionLocked(model, notifyClientOnError); 998 return status; 999 } else { 1000 return stopRecognitionLocked(model, notifyClientOnError); 1001 } 1002 } 1003 onServiceDiedLocked()1004 private void onServiceDiedLocked() { 1005 try { 1006 MetricsLogger.count(mContext, "sth_service_died", 1); 1007 for (ModelData modelData : mModelDataMap.values()) { 1008 IRecognitionStatusCallback callback = modelData.getCallback(); 1009 if (callback != null) { 1010 try { 1011 mEventLogger.enqueue(new SessionEvent(Type.MODULE_DIED, 1012 modelData.getModelId()).printLog(ALOGW, TAG)); 1013 callback.onModuleDied(); 1014 } catch (RemoteException e) { 1015 mEventLogger.enqueue(new SessionEvent(Type.MODULE_DIED, 1016 modelData.getModelId(), "RemoteException") 1017 .printLog(ALOGW, TAG)); 1018 } 1019 } 1020 } 1021 } finally { 1022 internalClearModelStateLocked(); 1023 if (mModule != null) { 1024 mModule.detach(); 1025 try { 1026 // This is best effort 1027 // TODO (b/279507851) 1028 mModule = mModuleProvider.apply(this); 1029 } catch (Exception e) { 1030 mModule = null; 1031 } 1032 } 1033 } 1034 } 1035 1036 // Clears state for all models (generic and keyphrase). internalClearModelStateLocked()1037 private void internalClearModelStateLocked() { 1038 for (ModelData modelData : mModelDataMap.values()) { 1039 modelData.clearState(); 1040 } 1041 } 1042 1043 /** 1044 * Stops and unloads all models. This is intended as a clean-up call with the expectation that 1045 * this instance is not used after. 1046 * @hide 1047 */ detach()1048 public void detach() { 1049 synchronized (mLock) { 1050 if (mIsDetached) return; 1051 mIsDetached = true; 1052 for (ModelData model : mModelDataMap.values()) { 1053 forceStopAndUnloadModelLocked(model, null); 1054 } 1055 mModelDataMap.clear(); 1056 if (mModule != null) { 1057 mModule.detach(); 1058 mModule = null; 1059 } 1060 } 1061 } 1062 1063 /** 1064 * Stops and unloads a sound model, and removes any reference to the model if successful. 1065 * 1066 * @param modelData The model data to remove. 1067 * @param exception Optional exception to print in logcat. May be null. 1068 */ forceStopAndUnloadModelLocked(ModelData modelData, Exception exception)1069 private void forceStopAndUnloadModelLocked(ModelData modelData, Exception exception) { 1070 forceStopAndUnloadModelLocked(modelData, exception, null /* modelDataIterator */); 1071 } 1072 1073 /** 1074 * Stops and unloads a sound model, and removes any reference to the model if successful. 1075 * 1076 * @param modelData The model data to remove. 1077 * @param exception Optional exception to print in logcat. May be null. 1078 * @param modelDataIterator If this function is to be used while iterating over the 1079 * mModelDataMap, you can provide the iterator for the current model data to be used to 1080 * remove the modelData from the map. This avoids generating a 1081 * ConcurrentModificationException, since this function will try and remove the model 1082 * data from the mModelDataMap when it can successfully unload the model. 1083 */ forceStopAndUnloadModelLocked(ModelData modelData, Exception exception, Iterator modelDataIterator)1084 private void forceStopAndUnloadModelLocked(ModelData modelData, Exception exception, 1085 Iterator modelDataIterator) { 1086 if (exception != null) { 1087 Slog.e(TAG, "forceStopAndUnloadModel", exception); 1088 } 1089 if (mModule == null) { 1090 return; 1091 } 1092 if (modelData.isModelStarted()) { 1093 Slog.d(TAG, "Stopping previously started dangling model " + modelData.getHandle()); 1094 if (mModule.stopRecognition(modelData.getHandle()) == STATUS_OK) { 1095 modelData.setStopped(); 1096 modelData.setRequested(false); 1097 } else { 1098 Slog.e(TAG, "Failed to stop model " + modelData.getHandle()); 1099 } 1100 } 1101 if (modelData.isModelLoaded()) { 1102 Slog.d(TAG, "Unloading previously loaded dangling model " + modelData.getHandle()); 1103 if (mModule.unloadSoundModel(modelData.getHandle()) == STATUS_OK) { 1104 // Remove the model data from existence. 1105 if (modelDataIterator != null) { 1106 modelDataIterator.remove(); 1107 } else { 1108 mModelDataMap.remove(modelData.getModelId()); 1109 } 1110 Iterator it = mKeyphraseUuidMap.entrySet().iterator(); 1111 while (it.hasNext()) { 1112 Map.Entry pair = (Map.Entry) it.next(); 1113 if (pair.getValue().equals(modelData.getModelId())) { 1114 it.remove(); 1115 } 1116 } 1117 modelData.clearState(); 1118 } else { 1119 Slog.e(TAG, "Failed to unload model " + modelData.getHandle()); 1120 } 1121 } 1122 } 1123 stopAndUnloadDeadModelsLocked()1124 private void stopAndUnloadDeadModelsLocked() { 1125 Iterator it = mModelDataMap.entrySet().iterator(); 1126 while (it.hasNext()) { 1127 ModelData modelData = (ModelData) ((Map.Entry) it.next()).getValue(); 1128 if (!modelData.isModelLoaded()) { 1129 continue; 1130 } 1131 if (modelData.getCallback() == null 1132 || (modelData.getCallback().asBinder() != null 1133 && !modelData.getCallback().asBinder().pingBinder())) { 1134 // No one is listening on this model, so we might as well evict it. 1135 Slog.w(TAG, "Removing model " + modelData.getHandle() + " that has no clients"); 1136 forceStopAndUnloadModelLocked(modelData, null /* exception */, it); 1137 } 1138 } 1139 } 1140 getOrCreateGenericModelDataLocked(UUID modelId)1141 private ModelData getOrCreateGenericModelDataLocked(UUID modelId) { 1142 ModelData modelData = mModelDataMap.get(modelId); 1143 if (modelData == null) { 1144 modelData = ModelData.createGenericModelData(modelId); 1145 mModelDataMap.put(modelId, modelData); 1146 } else if (!modelData.isGenericModel()) { 1147 Slog.e(TAG, "UUID already used for non-generic model."); 1148 return null; 1149 } 1150 return modelData; 1151 } 1152 removeKeyphraseModelLocked(int keyphraseId)1153 private void removeKeyphraseModelLocked(int keyphraseId) { 1154 UUID uuid = mKeyphraseUuidMap.get(keyphraseId); 1155 if (uuid == null) { 1156 return; 1157 } 1158 mModelDataMap.remove(uuid); 1159 mKeyphraseUuidMap.remove(keyphraseId); 1160 } 1161 getKeyphraseModelDataLocked(int keyphraseId)1162 private ModelData getKeyphraseModelDataLocked(int keyphraseId) { 1163 UUID uuid = mKeyphraseUuidMap.get(keyphraseId); 1164 if (uuid == null) { 1165 return null; 1166 } 1167 return mModelDataMap.get(uuid); 1168 } 1169 1170 // Use this to create a new ModelData entry for a keyphrase Id. It will overwrite existing 1171 // mapping if one exists. createKeyphraseModelDataLocked(UUID modelId, int keyphraseId)1172 private ModelData createKeyphraseModelDataLocked(UUID modelId, int keyphraseId) { 1173 mKeyphraseUuidMap.remove(keyphraseId); 1174 mModelDataMap.remove(modelId); 1175 mKeyphraseUuidMap.put(keyphraseId, modelId); 1176 ModelData modelData = ModelData.createKeyphraseModelData(modelId); 1177 mModelDataMap.put(modelId, modelData); 1178 return modelData; 1179 } 1180 1181 // Instead of maintaining a second hashmap of modelHandle -> ModelData, we just 1182 // iterate through to find the right object (since we don't expect 100s of models 1183 // to be stored). getModelDataForLocked(int modelHandle)1184 private ModelData getModelDataForLocked(int modelHandle) { 1185 // Fetch ModelData object corresponding to the model handle. 1186 for (ModelData model : mModelDataMap.values()) { 1187 if (model.getHandle() == modelHandle) { 1188 return model; 1189 } 1190 } 1191 return null; 1192 } 1193 1194 /** 1195 * Determines if recognition is allowed at all based on device state 1196 * 1197 * <p>Depending on the state of the SoundTrigger service, whether a call is active, or if 1198 * battery saver mode is enabled, a specific model may or may not be able to run. The result 1199 * of this check is not permanent, and the state of the device can change at any time. 1200 * 1201 * @param modelData Model data to be used for recognition 1202 * @return True if recognition is allowed to run at this time. False if not. 1203 */ 1204 @GuardedBy("mLock") isRecognitionAllowed(ModelData modelData)1205 private boolean isRecognitionAllowed(ModelData modelData) { 1206 if (!mIsAppOpPermitted) { 1207 return false; 1208 } 1209 return switch (mDeviceState) { 1210 case DISABLE -> false; 1211 case CRITICAL -> modelData.shouldRunInBatterySaverMode(); 1212 case ENABLE -> true; 1213 default -> throw new AssertionError("Enum changed between compile and runtime"); 1214 }; 1215 } 1216 1217 // A single routine that implements the start recognition logic for both generic and keyphrase 1218 // models. 1219 @GuardedBy("mLock") startRecognitionLocked(ModelData modelData, boolean notifyClientOnError)1220 private int startRecognitionLocked(ModelData modelData, boolean notifyClientOnError) { 1221 IRecognitionStatusCallback callback = modelData.getCallback(); 1222 RecognitionConfig config = modelData.getRecognitionConfig(); 1223 if (callback == null || !modelData.isModelLoaded() || config == null) { 1224 // Nothing to do here. 1225 Slog.w(TAG, "startRecognition: Bad data passed in."); 1226 MetricsLogger.count(mContext, "sth_start_recognition_error", 1); 1227 return STATUS_ERROR; 1228 } 1229 1230 if (!isRecognitionAllowed(modelData)) { 1231 // Nothing to do here. 1232 Slog.w(TAG, "startRecognition requested but not allowed."); 1233 MetricsLogger.count(mContext, "sth_start_recognition_not_allowed", 1); 1234 return STATUS_OK; 1235 } 1236 1237 if (mModule == null) { 1238 return STATUS_ERROR; 1239 } 1240 int status = STATUS_OK; 1241 try { 1242 modelData.setToken(mModule.startRecognitionWithToken(modelData.getHandle(), config)); 1243 } catch (Exception e) { 1244 status = SoundTrigger.handleException(e); 1245 } 1246 if (status != SoundTrigger.STATUS_OK) { 1247 Slog.w(TAG, "startRecognition failed with " + status); 1248 MetricsLogger.count(mContext, "sth_start_recognition_error", 1); 1249 // Notify of error if needed. 1250 if (notifyClientOnError) { 1251 try { 1252 mEventLogger.enqueue(new SessionEvent(Type.RESUME_FAILED, 1253 modelData.getModelId(), String.valueOf(status)) 1254 .printLog(ALOGW, TAG)); 1255 modelData.setRequested(false); 1256 callback.onResumeFailed(status); 1257 } catch (RemoteException e) { 1258 mEventLogger.enqueue(new SessionEvent(Type.RESUME_FAILED, 1259 modelData.getModelId(), 1260 String.valueOf(status) + " - RemoteException") 1261 .printLog(ALOGW, TAG)); 1262 forceStopAndUnloadModelLocked(modelData, e); 1263 } 1264 } 1265 } else { 1266 Slog.i(TAG, "startRecognition successful."); 1267 MetricsLogger.count(mContext, "sth_start_recognition_success", 1); 1268 modelData.setStarted(); 1269 // Notify of resume if needed. 1270 if (notifyClientOnError) { 1271 try { 1272 mEventLogger.enqueue(new SessionEvent(Type.RESUME, 1273 modelData.getModelId())); 1274 callback.onRecognitionResumed(); 1275 } catch (RemoteException e) { 1276 mEventLogger.enqueue(new SessionEvent(Type.RESUME, 1277 modelData.getModelId(), "RemoteException").printLog(ALOGW, TAG)); 1278 forceStopAndUnloadModelLocked(modelData, e); 1279 } 1280 } 1281 } 1282 return status; 1283 } 1284 stopRecognitionLocked(ModelData modelData, boolean notify)1285 private int stopRecognitionLocked(ModelData modelData, boolean notify) { 1286 if (mModule == null) { 1287 return STATUS_ERROR; 1288 } 1289 1290 IRecognitionStatusCallback callback = modelData.getCallback(); 1291 // Stop recognition. 1292 int status = STATUS_OK; 1293 1294 status = mModule.stopRecognition(modelData.getHandle()); 1295 1296 if (status != SoundTrigger.STATUS_OK) { 1297 Slog.e(TAG, "stopRecognition call failed with " + status); 1298 MetricsLogger.count(mContext, "sth_stop_recognition_error", 1); 1299 if (notify) { 1300 try { 1301 mEventLogger.enqueue(new SessionEvent(Type.PAUSE_FAILED, 1302 modelData.getModelId(), String.valueOf(status)) 1303 .printLog(ALOGW, TAG)); 1304 modelData.setRequested(false); 1305 callback.onPauseFailed(status); 1306 } catch (RemoteException e) { 1307 mEventLogger.enqueue(new SessionEvent(Type.PAUSE_FAILED, 1308 modelData.getModelId(), 1309 String.valueOf(status) + " - RemoteException") 1310 .printLog(ALOGW, TAG)); 1311 forceStopAndUnloadModelLocked(modelData, e); 1312 } 1313 } 1314 } else { 1315 modelData.setStopped(); 1316 MetricsLogger.count(mContext, "sth_stop_recognition_success", 1); 1317 // Notify of pause if needed. 1318 if (notify) { 1319 try { 1320 mEventLogger.enqueue(new SessionEvent(Type.PAUSE, 1321 modelData.getModelId())); 1322 callback.onRecognitionPaused(); 1323 } catch (RemoteException e) { 1324 mEventLogger.enqueue(new SessionEvent(Type.PAUSE, 1325 modelData.getModelId(), "RemoteException").printLog(ALOGW, TAG)); 1326 forceStopAndUnloadModelLocked(modelData, e); 1327 } 1328 } 1329 } 1330 return status; 1331 } 1332 1333 // Computes whether we have any recognition running at all (voice or generic). Sets 1334 // the mRecognitionRequested variable with the result. computeRecognitionRequestedLocked()1335 private boolean computeRecognitionRequestedLocked() { 1336 if (mModule == null) { 1337 mRecognitionRequested = false; 1338 return mRecognitionRequested; 1339 } 1340 for (ModelData modelData : mModelDataMap.values()) { 1341 if (modelData.isRequested()) { 1342 mRecognitionRequested = true; 1343 return mRecognitionRequested; 1344 } 1345 } 1346 mRecognitionRequested = false; 1347 return mRecognitionRequested; 1348 } 1349 1350 // This class encapsulates the callbacks, state, handles and any other information that 1351 // represents a model. 1352 private static class ModelData { 1353 // Model not loaded (and hence not started). 1354 static final int MODEL_NOTLOADED = 0; 1355 1356 // Loaded implies model was successfully loaded. Model not started yet. 1357 static final int MODEL_LOADED = 1; 1358 1359 // Started implies model was successfully loaded and start was called. 1360 static final int MODEL_STARTED = 2; 1361 1362 // One of MODEL_NOTLOADED, MODEL_LOADED, MODEL_STARTED (which implies loaded). 1363 private int mModelState; 1364 private UUID mModelId; 1365 1366 // mRequested captures the explicit intent that a start was requested for this model. We 1367 // continue to capture and retain this state even after the model gets started, so that we 1368 // know when a model gets stopped due to "other" reasons, that we should start it again. 1369 // This was the intended behavior of the "mRequested" variable in the previous version of 1370 // this code that we are replicating here. 1371 // 1372 // The "other" reasons include power save, abort being called from the lower layer (due 1373 // to concurrent capture not being supported) and phone call state. Once we recover from 1374 // these transient disruptions, we would start such models again where mRequested == true. 1375 // Thus, mRequested gets reset only when there is an explicit intent to stop the model 1376 // coming from the SoundTriggerService layer that uses this class (and thus eventually 1377 // from the app that manages this model). 1378 private boolean mRequested = false; 1379 1380 // One of SoundModel.TYPE_GENERIC or SoundModel.TYPE_KEYPHRASE. Initially set 1381 // to SoundModel.TYPE_UNKNOWN; 1382 private int mModelType = SoundModel.TYPE_UNKNOWN; 1383 1384 private IRecognitionStatusCallback mCallback = null; 1385 private RecognitionConfig mRecognitionConfig = null; 1386 1387 // Model handle is an integer used by the HAL as an identifier for sound 1388 // models. 1389 private int mModelHandle; 1390 1391 /** 1392 * True if the service should continue listening when battery saver mode is enabled. 1393 * Having this flag set requires the client calling 1394 * {@link SoundTriggerModule#startRecognition(int, RecognitionConfig)} to be granted 1395 * {@link android.Manifest.permission#SOUND_TRIGGER_RUN_IN_BATTERY_SAVER}. 1396 */ 1397 public boolean mRunInBatterySaverMode = false; 1398 1399 // The SoundModel instance, one of KeyphraseSoundModel or GenericSoundModel. 1400 private SoundModel mSoundModel = null; 1401 1402 // Token used to disambiguate recognition sessions. 1403 private IBinder mRecognitionToken = null; 1404 ModelData(UUID modelId, int modelType)1405 private ModelData(UUID modelId, int modelType) { 1406 mModelId = modelId; 1407 // Private constructor, since we require modelType to be one of TYPE_GENERIC, 1408 // TYPE_KEYPHRASE or TYPE_UNKNOWN. 1409 mModelType = modelType; 1410 } 1411 createKeyphraseModelData(UUID modelId)1412 static ModelData createKeyphraseModelData(UUID modelId) { 1413 return new ModelData(modelId, SoundModel.TYPE_KEYPHRASE); 1414 } 1415 createGenericModelData(UUID modelId)1416 static ModelData createGenericModelData(UUID modelId) { 1417 return new ModelData(modelId, SoundModel.TYPE_GENERIC_SOUND); 1418 } 1419 1420 // Note that most of the functionality in this Java class will not work for 1421 // SoundModel.TYPE_UNKNOWN nevertheless we have it since lower layers support it. createModelDataOfUnknownType(UUID modelId)1422 static ModelData createModelDataOfUnknownType(UUID modelId) { 1423 return new ModelData(modelId, SoundModel.TYPE_UNKNOWN); 1424 } 1425 setCallback(IRecognitionStatusCallback callback)1426 synchronized void setCallback(IRecognitionStatusCallback callback) { 1427 mCallback = callback; 1428 } 1429 getCallback()1430 synchronized IRecognitionStatusCallback getCallback() { 1431 return mCallback; 1432 } 1433 isModelLoaded()1434 synchronized boolean isModelLoaded() { 1435 return (mModelState == MODEL_LOADED || mModelState == MODEL_STARTED); 1436 } 1437 isModelNotLoaded()1438 synchronized boolean isModelNotLoaded() { 1439 return mModelState == MODEL_NOTLOADED; 1440 } 1441 setStarted()1442 synchronized void setStarted() { 1443 mModelState = MODEL_STARTED; 1444 } 1445 setStopped()1446 synchronized void setStopped() { 1447 // If we are moving to the stopped state, we should clear out our 1448 // startRecognition token 1449 mRecognitionToken = null; 1450 mModelState = MODEL_LOADED; 1451 } 1452 setLoaded()1453 synchronized void setLoaded() { 1454 mModelState = MODEL_LOADED; 1455 } 1456 setNotLoaded()1457 synchronized void setNotLoaded() { 1458 mRecognitionToken = null; 1459 mModelState = MODEL_NOTLOADED; 1460 } 1461 isModelStarted()1462 synchronized boolean isModelStarted() { 1463 return mModelState == MODEL_STARTED; 1464 } 1465 clearState()1466 synchronized void clearState() { 1467 mModelState = MODEL_NOTLOADED; 1468 mRecognitionToken = null; 1469 mRecognitionConfig = null; 1470 mRequested = false; 1471 mCallback = null; 1472 } 1473 clearCallback()1474 synchronized void clearCallback() { 1475 mCallback = null; 1476 } 1477 setHandle(int handle)1478 synchronized void setHandle(int handle) { 1479 mModelHandle = handle; 1480 } 1481 setRecognitionConfig(RecognitionConfig config)1482 synchronized void setRecognitionConfig(RecognitionConfig config) { 1483 mRecognitionConfig = config; 1484 } 1485 setRunInBatterySaverMode(boolean runInBatterySaverMode)1486 synchronized void setRunInBatterySaverMode(boolean runInBatterySaverMode) { 1487 mRunInBatterySaverMode = runInBatterySaverMode; 1488 } 1489 shouldRunInBatterySaverMode()1490 synchronized boolean shouldRunInBatterySaverMode() { 1491 return mRunInBatterySaverMode; 1492 } 1493 getHandle()1494 synchronized int getHandle() { 1495 return mModelHandle; 1496 } 1497 getModelId()1498 synchronized UUID getModelId() { 1499 return mModelId; 1500 } 1501 getRecognitionConfig()1502 synchronized RecognitionConfig getRecognitionConfig() { 1503 return mRecognitionConfig; 1504 } 1505 1506 // Whether a start recognition was requested. isRequested()1507 synchronized boolean isRequested() { 1508 return mRequested; 1509 } 1510 setRequested(boolean requested)1511 synchronized void setRequested(boolean requested) { 1512 mRequested = requested; 1513 } 1514 setSoundModel(SoundModel soundModel)1515 synchronized void setSoundModel(SoundModel soundModel) { 1516 mSoundModel = soundModel; 1517 } 1518 getSoundModel()1519 synchronized SoundModel getSoundModel() { 1520 return mSoundModel; 1521 } 1522 getToken()1523 synchronized IBinder getToken() { 1524 return mRecognitionToken; 1525 } 1526 setToken(IBinder token)1527 synchronized void setToken(IBinder token) { 1528 mRecognitionToken = token; 1529 } 1530 getModelType()1531 synchronized int getModelType() { 1532 return mModelType; 1533 } 1534 isKeyphraseModel()1535 synchronized boolean isKeyphraseModel() { 1536 return mModelType == SoundModel.TYPE_KEYPHRASE; 1537 } 1538 isGenericModel()1539 synchronized boolean isGenericModel() { 1540 return mModelType == SoundModel.TYPE_GENERIC_SOUND; 1541 } 1542 stateToString()1543 synchronized String stateToString() { 1544 switch(mModelState) { 1545 case MODEL_NOTLOADED: return "NOT_LOADED"; 1546 case MODEL_LOADED: return "LOADED"; 1547 case MODEL_STARTED: return "STARTED"; 1548 } 1549 return "Unknown state"; 1550 } 1551 requestedToString()1552 synchronized String requestedToString() { 1553 return "Requested: " + (mRequested ? "Yes" : "No"); 1554 } 1555 callbackToString()1556 synchronized String callbackToString() { 1557 return "Callback: " + (mCallback != null ? mCallback.asBinder() : "null"); 1558 } 1559 uuidToString()1560 synchronized String uuidToString() { 1561 return "UUID: " + mModelId; 1562 } 1563 toString()1564 synchronized public String toString() { 1565 return "Handle: " + mModelHandle + "\n" + 1566 "ModelState: " + stateToString() + "\n" + 1567 requestedToString() + "\n" + 1568 callbackToString() + "\n" + 1569 uuidToString() + "\n" + 1570 modelTypeToString() + 1571 "RunInBatterySaverMode=" + mRunInBatterySaverMode; 1572 } 1573 modelTypeToString()1574 synchronized String modelTypeToString() { 1575 String type = null; 1576 switch (mModelType) { 1577 case SoundModel.TYPE_GENERIC_SOUND: type = "Generic"; break; 1578 case SoundModel.TYPE_UNKNOWN: type = "Unknown"; break; 1579 case SoundModel.TYPE_KEYPHRASE: type = "Keyphrase"; break; 1580 } 1581 return "Model type: " + type + "\n"; 1582 } 1583 } 1584 } 1585