1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.app.Activity; 23 import android.content.Intent; 24 import android.hardware.soundtrigger.IRecognitionStatusCallback; 25 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; 26 import android.hardware.soundtrigger.KeyphraseMetadata; 27 import android.hardware.soundtrigger.SoundTrigger; 28 import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; 29 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 30 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 31 import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 32 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 33 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 34 import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 35 import android.media.AudioFormat; 36 import android.os.AsyncTask; 37 import android.os.Handler; 38 import android.os.Message; 39 import android.os.RemoteException; 40 import android.util.Slog; 41 42 import com.android.internal.app.IVoiceInteractionManagerService; 43 44 import java.io.PrintWriter; 45 import java.lang.annotation.Retention; 46 import java.lang.annotation.RetentionPolicy; 47 import java.util.Locale; 48 49 /** 50 * A class that lets a VoiceInteractionService implementation interact with 51 * always-on keyphrase detection APIs. 52 */ 53 public class AlwaysOnHotwordDetector { 54 //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----// 55 /** 56 * Indicates that this hotword detector is no longer valid for any recognition 57 * and should not be used anymore. 58 */ 59 private static final int STATE_INVALID = -3; 60 61 /** 62 * Indicates that recognition for the given keyphrase is not available on the system 63 * because of the hardware configuration. 64 * No further interaction should be performed with the detector that returns this availability. 65 */ 66 public static final int STATE_HARDWARE_UNAVAILABLE = -2; 67 /** 68 * Indicates that recognition for the given keyphrase is not supported. 69 * No further interaction should be performed with the detector that returns this availability. 70 */ 71 public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; 72 /** 73 * Indicates that the given keyphrase is not enrolled. 74 * The caller may choose to begin an enrollment flow for the keyphrase. 75 */ 76 public static final int STATE_KEYPHRASE_UNENROLLED = 1; 77 /** 78 * Indicates that the given keyphrase is currently enrolled and it's possible to start 79 * recognition for it. 80 */ 81 public static final int STATE_KEYPHRASE_ENROLLED = 2; 82 83 /** 84 * Indicates that the detector isn't ready currently. 85 */ 86 private static final int STATE_NOT_READY = 0; 87 88 // Keyphrase management actions. Used in getManageIntent() ----// 89 @Retention(RetentionPolicy.SOURCE) 90 @IntDef(value = { 91 MANAGE_ACTION_ENROLL, 92 MANAGE_ACTION_RE_ENROLL, 93 MANAGE_ACTION_UN_ENROLL 94 }) 95 private @interface ManageActions {} 96 97 /** 98 * Indicates that we need to enroll. 99 * 100 * @hide 101 */ 102 public static final int MANAGE_ACTION_ENROLL = 0; 103 /** 104 * Indicates that we need to re-enroll. 105 * 106 * @hide 107 */ 108 public static final int MANAGE_ACTION_RE_ENROLL = 1; 109 /** 110 * Indicates that we need to un-enroll. 111 * 112 * @hide 113 */ 114 public static final int MANAGE_ACTION_UN_ENROLL = 2; 115 116 //-- Flags for startRecognition ----// 117 /** @hide */ 118 @Retention(RetentionPolicy.SOURCE) 119 @IntDef(flag = true, 120 value = { 121 RECOGNITION_FLAG_NONE, 122 RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO, 123 RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 124 }) 125 public @interface RecognitionFlags {} 126 127 /** 128 * Empty flag for {@link #startRecognition(int)}. 129 * 130 * @hide 131 */ 132 public static final int RECOGNITION_FLAG_NONE = 0; 133 /** 134 * Recognition flag for {@link #startRecognition(int)} that indicates 135 * whether the trigger audio for hotword needs to be captured. 136 */ 137 public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; 138 /** 139 * Recognition flag for {@link #startRecognition(int)} that indicates 140 * whether the recognition should keep going on even after the keyphrase triggers. 141 * If this flag is specified, it's possible to get multiple triggers after a 142 * call to {@link #startRecognition(int)} if the user speaks the keyphrase multiple times. 143 * When this isn't specified, the default behavior is to stop recognition once the 144 * keyphrase is spoken, till the caller starts recognition again. 145 */ 146 public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2; 147 148 //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----// 149 // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. 150 151 /** @hide */ 152 @Retention(RetentionPolicy.SOURCE) 153 @IntDef(flag = true, 154 value = { 155 RECOGNITION_MODE_VOICE_TRIGGER, 156 RECOGNITION_MODE_USER_IDENTIFICATION, 157 }) 158 public @interface RecognitionModes {} 159 160 /** 161 * Simple recognition of the key phrase. 162 * Returned by {@link #getSupportedRecognitionModes()} 163 */ 164 public static final int RECOGNITION_MODE_VOICE_TRIGGER 165 = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; 166 /** 167 * User identification performed with the keyphrase recognition. 168 * Returned by {@link #getSupportedRecognitionModes()} 169 */ 170 public static final int RECOGNITION_MODE_USER_IDENTIFICATION 171 = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; 172 173 static final String TAG = "AlwaysOnHotwordDetector"; 174 static final boolean DBG = false; 175 176 private static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 177 private static final int STATUS_OK = SoundTrigger.STATUS_OK; 178 179 private static final int MSG_AVAILABILITY_CHANGED = 1; 180 private static final int MSG_HOTWORD_DETECTED = 2; 181 private static final int MSG_DETECTION_ERROR = 3; 182 private static final int MSG_DETECTION_PAUSE = 4; 183 private static final int MSG_DETECTION_RESUME = 5; 184 185 private final String mText; 186 private final Locale mLocale; 187 /** 188 * The metadata of the Keyphrase, derived from the enrollment application. 189 * This may be null if this keyphrase isn't supported by the enrollment application. 190 */ 191 private final KeyphraseMetadata mKeyphraseMetadata; 192 private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; 193 private final IVoiceInteractionService mVoiceInteractionService; 194 private final IVoiceInteractionManagerService mModelManagementService; 195 private final SoundTriggerListener mInternalCallback; 196 private final Callback mExternalCallback; 197 private final Object mLock = new Object(); 198 private final Handler mHandler; 199 200 private int mAvailability = STATE_NOT_READY; 201 202 /** 203 * Additional payload for {@link Callback#onDetected}. 204 */ 205 public static class EventPayload { 206 private final boolean mTriggerAvailable; 207 // Indicates if {@code captureSession} can be used to continue capturing more audio 208 // from the DSP hardware. 209 private final boolean mCaptureAvailable; 210 // The session to use when attempting to capture more audio from the DSP hardware. 211 private final int mCaptureSession; 212 private final AudioFormat mAudioFormat; 213 // Raw data associated with the event. 214 // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true. 215 private final byte[] mData; 216 EventPayload(boolean triggerAvailable, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data)217 private EventPayload(boolean triggerAvailable, boolean captureAvailable, 218 AudioFormat audioFormat, int captureSession, byte[] data) { 219 mTriggerAvailable = triggerAvailable; 220 mCaptureAvailable = captureAvailable; 221 mCaptureSession = captureSession; 222 mAudioFormat = audioFormat; 223 mData = data; 224 } 225 226 /** 227 * Gets the format of the audio obtained using {@link #getTriggerAudio()}. 228 * May be null if there's no audio present. 229 */ 230 @Nullable getCaptureAudioFormat()231 public AudioFormat getCaptureAudioFormat() { 232 return mAudioFormat; 233 } 234 235 /** 236 * Gets the raw audio that triggered the keyphrase. 237 * This may be null if the trigger audio isn't available. 238 * If non-null, the format of the audio can be obtained by calling 239 * {@link #getCaptureAudioFormat()}. 240 * 241 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 242 */ 243 @Nullable getTriggerAudio()244 public byte[] getTriggerAudio() { 245 if (mTriggerAvailable) { 246 return mData; 247 } else { 248 return null; 249 } 250 } 251 252 /** 253 * Gets the session ID to start a capture from the DSP. 254 * This may be null if streaming capture isn't possible. 255 * If non-null, the format of the audio that can be captured can be 256 * obtained using {@link #getCaptureAudioFormat()}. 257 * 258 * TODO: Candidate for Public API when the API to start capture with a session ID 259 * is made public. 260 * 261 * TODO: Add this to {@link #getCaptureAudioFormat()}: 262 * "Gets the format of the audio obtained using {@link #getTriggerAudio()} 263 * or {@link #getCaptureSession()}. May be null if no audio can be obtained 264 * for either the trigger or a streaming session." 265 * 266 * TODO: Should this return a known invalid value instead? 267 * 268 * @hide 269 */ 270 @Nullable getCaptureSession()271 public Integer getCaptureSession() { 272 if (mCaptureAvailable) { 273 return mCaptureSession; 274 } else { 275 return null; 276 } 277 } 278 } 279 280 /** 281 * Callbacks for always-on hotword detection. 282 */ 283 public static abstract class Callback { 284 /** 285 * Called when the hotword availability changes. 286 * This indicates a change in the availability of recognition for the given keyphrase. 287 * It's called at least once with the initial availability.<p/> 288 * 289 * Availability implies whether the hardware on this system is capable of listening for 290 * the given keyphrase or not. <p/> 291 * 292 * @see AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE 293 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNSUPPORTED 294 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNENROLLED 295 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_ENROLLED 296 */ onAvailabilityChanged(int status)297 public abstract void onAvailabilityChanged(int status); 298 /** 299 * Called when the keyphrase is spoken. 300 * This implicitly stops listening for the keyphrase once it's detected. 301 * Clients should start a recognition again once they are done handling this 302 * detection. 303 * 304 * @param eventPayload Payload data for the detection event. 305 * This may contain the trigger audio, if requested when calling 306 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 307 */ onDetected(@onNull EventPayload eventPayload)308 public abstract void onDetected(@NonNull EventPayload eventPayload); 309 /** 310 * Called when the detection fails due to an error. 311 */ onError()312 public abstract void onError(); 313 /** 314 * Called when the recognition is paused temporarily for some reason. 315 * This is an informational callback, and the clients shouldn't be doing anything here 316 * except showing an indication on their UI if they have to. 317 */ onRecognitionPaused()318 public abstract void onRecognitionPaused(); 319 /** 320 * Called when the recognition is resumed after it was temporarily paused. 321 * This is an informational callback, and the clients shouldn't be doing anything here 322 * except showing an indication on their UI if they have to. 323 */ onRecognitionResumed()324 public abstract void onRecognitionResumed(); 325 } 326 327 /** 328 * @param text The keyphrase text to get the detector for. 329 * @param locale The java locale for the detector. 330 * @param callback A non-null Callback for receiving the recognition events. 331 * @param voiceInteractionService The current voice interaction service. 332 * @param modelManagementService A service that allows management of sound models. 333 * 334 * @hide 335 */ AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionService voiceInteractionService, IVoiceInteractionManagerService modelManagementService)336 public AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, 337 KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, 338 IVoiceInteractionService voiceInteractionService, 339 IVoiceInteractionManagerService modelManagementService) { 340 mText = text; 341 mLocale = locale; 342 mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; 343 mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale); 344 mExternalCallback = callback; 345 mHandler = new MyHandler(); 346 mInternalCallback = new SoundTriggerListener(mHandler); 347 mVoiceInteractionService = voiceInteractionService; 348 mModelManagementService = modelManagementService; 349 new RefreshAvailabiltyTask().execute(); 350 } 351 352 /** 353 * Gets the recognition modes supported by the associated keyphrase. 354 * 355 * @see #RECOGNITION_MODE_USER_IDENTIFICATION 356 * @see #RECOGNITION_MODE_VOICE_TRIGGER 357 * 358 * @throws UnsupportedOperationException if the keyphrase itself isn't supported. 359 * Callers should only call this method after a supported state callback on 360 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 361 * @throws IllegalStateException if the detector is in an invalid state. 362 * This may happen if another detector has been instantiated or the 363 * {@link VoiceInteractionService} hosting this detector has been shut down. 364 */ getSupportedRecognitionModes()365 public @RecognitionModes int getSupportedRecognitionModes() { 366 if (DBG) Slog.d(TAG, "getSupportedRecognitionModes()"); 367 synchronized (mLock) { 368 return getSupportedRecognitionModesLocked(); 369 } 370 } 371 getSupportedRecognitionModesLocked()372 private int getSupportedRecognitionModesLocked() { 373 if (mAvailability == STATE_INVALID) { 374 throw new IllegalStateException( 375 "getSupportedRecognitionModes called on an invalid detector"); 376 } 377 378 // This method only makes sense if we can actually support a recognition. 379 if (mAvailability != STATE_KEYPHRASE_ENROLLED 380 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 381 throw new UnsupportedOperationException( 382 "Getting supported recognition modes for the keyphrase is not supported"); 383 } 384 385 return mKeyphraseMetadata.recognitionModeFlags; 386 } 387 388 /** 389 * Starts recognition for the associated keyphrase. 390 * 391 * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 392 * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 393 * 394 * @param recognitionFlags The flags to control the recognition properties. 395 * @return Indicates whether the call succeeded or not. 396 * @throws UnsupportedOperationException if the recognition isn't supported. 397 * Callers should only call this method after a supported state callback on 398 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 399 * @throws IllegalStateException if the detector is in an invalid state. 400 * This may happen if another detector has been instantiated or the 401 * {@link VoiceInteractionService} hosting this detector has been shut down. 402 */ startRecognition(@ecognitionFlags int recognitionFlags)403 public boolean startRecognition(@RecognitionFlags int recognitionFlags) { 404 if (DBG) Slog.d(TAG, "startRecognition(" + recognitionFlags + ")"); 405 synchronized (mLock) { 406 if (mAvailability == STATE_INVALID) { 407 throw new IllegalStateException("startRecognition called on an invalid detector"); 408 } 409 410 // Check if we can start/stop a recognition. 411 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 412 throw new UnsupportedOperationException( 413 "Recognition for the given keyphrase is not supported"); 414 } 415 416 return startRecognitionLocked(recognitionFlags) == STATUS_OK; 417 } 418 } 419 420 /** 421 * Stops recognition for the associated keyphrase. 422 * 423 * @return Indicates whether the call succeeded or not. 424 * @throws UnsupportedOperationException if the recognition isn't supported. 425 * Callers should only call this method after a supported state callback on 426 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 427 * @throws IllegalStateException if the detector is in an invalid state. 428 * This may happen if another detector has been instantiated or the 429 * {@link VoiceInteractionService} hosting this detector has been shut down. 430 */ stopRecognition()431 public boolean stopRecognition() { 432 if (DBG) Slog.d(TAG, "stopRecognition()"); 433 synchronized (mLock) { 434 if (mAvailability == STATE_INVALID) { 435 throw new IllegalStateException("stopRecognition called on an invalid detector"); 436 } 437 438 // Check if we can start/stop a recognition. 439 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 440 throw new UnsupportedOperationException( 441 "Recognition for the given keyphrase is not supported"); 442 } 443 444 return stopRecognitionLocked() == STATUS_OK; 445 } 446 } 447 448 /** 449 * Creates an intent to start the enrollment for the associated keyphrase. 450 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 451 * Starting re-enrollment is only valid if the keyphrase is un-enrolled, 452 * i.e. {@link #STATE_KEYPHRASE_UNENROLLED}, 453 * otherwise {@link #createReEnrollIntent()} should be preferred. 454 * 455 * @return An {@link Intent} to start enrollment for the given keyphrase. 456 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 457 * Callers should only call this method after a supported state callback on 458 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 459 * @throws IllegalStateException if the detector is in an invalid state. 460 * This may happen if another detector has been instantiated or the 461 * {@link VoiceInteractionService} hosting this detector has been shut down. 462 */ createEnrollIntent()463 public Intent createEnrollIntent() { 464 if (DBG) Slog.d(TAG, "createEnrollIntent"); 465 synchronized (mLock) { 466 return getManageIntentLocked(MANAGE_ACTION_ENROLL); 467 } 468 } 469 470 /** 471 * Creates an intent to start the un-enrollment for the associated keyphrase. 472 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 473 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 474 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 475 * 476 * @return An {@link Intent} to start un-enrollment for the given keyphrase. 477 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 478 * Callers should only call this method after a supported state callback on 479 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 480 * @throws IllegalStateException if the detector is in an invalid state. 481 * This may happen if another detector has been instantiated or the 482 * {@link VoiceInteractionService} hosting this detector has been shut down. 483 */ createUnEnrollIntent()484 public Intent createUnEnrollIntent() { 485 if (DBG) Slog.d(TAG, "createUnEnrollIntent"); 486 synchronized (mLock) { 487 return getManageIntentLocked(MANAGE_ACTION_UN_ENROLL); 488 } 489 } 490 491 /** 492 * Creates an intent to start the re-enrollment for the associated keyphrase. 493 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 494 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 495 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 496 * 497 * @return An {@link Intent} to start re-enrollment for the given keyphrase. 498 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 499 * Callers should only call this method after a supported state callback on 500 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 501 * @throws IllegalStateException if the detector is in an invalid state. 502 * This may happen if another detector has been instantiated or the 503 * {@link VoiceInteractionService} hosting this detector has been shut down. 504 */ createReEnrollIntent()505 public Intent createReEnrollIntent() { 506 if (DBG) Slog.d(TAG, "createReEnrollIntent"); 507 synchronized (mLock) { 508 return getManageIntentLocked(MANAGE_ACTION_RE_ENROLL); 509 } 510 } 511 getManageIntentLocked(int action)512 private Intent getManageIntentLocked(int action) { 513 if (mAvailability == STATE_INVALID) { 514 throw new IllegalStateException("getManageIntent called on an invalid detector"); 515 } 516 517 // This method only makes sense if we can actually support a recognition. 518 if (mAvailability != STATE_KEYPHRASE_ENROLLED 519 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 520 throw new UnsupportedOperationException( 521 "Managing the given keyphrase is not supported"); 522 } 523 524 return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); 525 } 526 527 /** 528 * Invalidates this hotword detector so that any future calls to this result 529 * in an IllegalStateException. 530 * 531 * @hide 532 */ invalidate()533 void invalidate() { 534 synchronized (mLock) { 535 mAvailability = STATE_INVALID; 536 notifyStateChangedLocked(); 537 } 538 } 539 540 /** 541 * Reloads the sound models from the service. 542 * 543 * @hide 544 */ onSoundModelsChanged()545 void onSoundModelsChanged() { 546 synchronized (mLock) { 547 if (mAvailability == STATE_INVALID 548 || mAvailability == STATE_HARDWARE_UNAVAILABLE 549 || mAvailability == STATE_KEYPHRASE_UNSUPPORTED) { 550 Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config"); 551 return; 552 } 553 554 // Stop the recognition before proceeding. 555 // This is done because we want to stop the recognition on an older model if it changed 556 // or was deleted. 557 // The availability change callback should ensure that the client starts recognition 558 // again if needed. 559 stopRecognitionLocked(); 560 561 // Execute a refresh availability task - which should then notify of a change. 562 new RefreshAvailabiltyTask().execute(); 563 } 564 } 565 startRecognitionLocked(int recognitionFlags)566 private int startRecognitionLocked(int recognitionFlags) { 567 KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; 568 // TODO: Do we need to do something about the confidence level here? 569 recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id, 570 mKeyphraseMetadata.recognitionModeFlags, 0, new ConfidenceLevel[0]); 571 boolean captureTriggerAudio = 572 (recognitionFlags&RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; 573 boolean allowMultipleTriggers = 574 (recognitionFlags&RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0; 575 int code = STATUS_ERROR; 576 try { 577 code = mModelManagementService.startRecognition(mVoiceInteractionService, 578 mKeyphraseMetadata.id, mLocale.toLanguageTag(), mInternalCallback, 579 new RecognitionConfig(captureTriggerAudio, allowMultipleTriggers, 580 recognitionExtra, null /* additional data */)); 581 } catch (RemoteException e) { 582 Slog.w(TAG, "RemoteException in startRecognition!", e); 583 } 584 if (code != STATUS_OK) { 585 Slog.w(TAG, "startRecognition() failed with error code " + code); 586 } 587 return code; 588 } 589 stopRecognitionLocked()590 private int stopRecognitionLocked() { 591 int code = STATUS_ERROR; 592 try { 593 code = mModelManagementService.stopRecognition( 594 mVoiceInteractionService, mKeyphraseMetadata.id, mInternalCallback); 595 } catch (RemoteException e) { 596 Slog.w(TAG, "RemoteException in stopRecognition!", e); 597 } 598 599 if (code != STATUS_OK) { 600 Slog.w(TAG, "stopRecognition() failed with error code " + code); 601 } 602 return code; 603 } 604 notifyStateChangedLocked()605 private void notifyStateChangedLocked() { 606 Message message = Message.obtain(mHandler, MSG_AVAILABILITY_CHANGED); 607 message.arg1 = mAvailability; 608 message.sendToTarget(); 609 } 610 611 /** @hide */ 612 static final class SoundTriggerListener extends IRecognitionStatusCallback.Stub { 613 private final Handler mHandler; 614 SoundTriggerListener(Handler handler)615 public SoundTriggerListener(Handler handler) { 616 mHandler = handler; 617 } 618 619 @Override onKeyphraseDetected(KeyphraseRecognitionEvent event)620 public void onKeyphraseDetected(KeyphraseRecognitionEvent event) { 621 if (DBG) { 622 Slog.d(TAG, "onDetected(" + event + ")"); 623 } else { 624 Slog.i(TAG, "onDetected"); 625 } 626 Message.obtain(mHandler, MSG_HOTWORD_DETECTED, 627 new EventPayload(event.triggerInData, event.captureAvailable, 628 event.captureFormat, event.captureSession, event.data)) 629 .sendToTarget(); 630 } 631 @Override onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event)632 public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { 633 Slog.w(TAG, "Generic sound trigger event detected at AOHD: " + event); 634 } 635 636 @Override onError(int status)637 public void onError(int status) { 638 Slog.i(TAG, "onError: " + status); 639 mHandler.sendEmptyMessage(MSG_DETECTION_ERROR); 640 } 641 642 @Override onRecognitionPaused()643 public void onRecognitionPaused() { 644 Slog.i(TAG, "onRecognitionPaused"); 645 mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE); 646 } 647 648 @Override onRecognitionResumed()649 public void onRecognitionResumed() { 650 Slog.i(TAG, "onRecognitionResumed"); 651 mHandler.sendEmptyMessage(MSG_DETECTION_RESUME); 652 } 653 } 654 655 class MyHandler extends Handler { 656 @Override handleMessage(Message msg)657 public void handleMessage(Message msg) { 658 synchronized (mLock) { 659 if (mAvailability == STATE_INVALID) { 660 Slog.w(TAG, "Received message: " + msg.what + " for an invalid detector"); 661 return; 662 } 663 } 664 665 switch (msg.what) { 666 case MSG_AVAILABILITY_CHANGED: 667 mExternalCallback.onAvailabilityChanged(msg.arg1); 668 break; 669 case MSG_HOTWORD_DETECTED: 670 mExternalCallback.onDetected((EventPayload) msg.obj); 671 break; 672 case MSG_DETECTION_ERROR: 673 mExternalCallback.onError(); 674 break; 675 case MSG_DETECTION_PAUSE: 676 mExternalCallback.onRecognitionPaused(); 677 break; 678 case MSG_DETECTION_RESUME: 679 mExternalCallback.onRecognitionResumed(); 680 break; 681 default: 682 super.handleMessage(msg); 683 } 684 } 685 } 686 687 class RefreshAvailabiltyTask extends AsyncTask<Void, Void, Void> { 688 689 @Override doInBackground(Void... params)690 public Void doInBackground(Void... params) { 691 int availability = internalGetInitialAvailability(); 692 boolean enrolled = false; 693 // Fetch the sound model if the availability is one of the supported ones. 694 if (availability == STATE_NOT_READY 695 || availability == STATE_KEYPHRASE_UNENROLLED 696 || availability == STATE_KEYPHRASE_ENROLLED) { 697 enrolled = internalGetIsEnrolled(mKeyphraseMetadata.id, mLocale); 698 if (!enrolled) { 699 availability = STATE_KEYPHRASE_UNENROLLED; 700 } else { 701 availability = STATE_KEYPHRASE_ENROLLED; 702 } 703 } 704 705 synchronized (mLock) { 706 if (DBG) { 707 Slog.d(TAG, "Hotword availability changed from " + mAvailability 708 + " -> " + availability); 709 } 710 mAvailability = availability; 711 notifyStateChangedLocked(); 712 } 713 return null; 714 } 715 716 /** 717 * @return The initial availability without checking the enrollment status. 718 */ internalGetInitialAvailability()719 private int internalGetInitialAvailability() { 720 synchronized (mLock) { 721 // This detector has already been invalidated. 722 if (mAvailability == STATE_INVALID) { 723 return STATE_INVALID; 724 } 725 } 726 727 ModuleProperties dspModuleProperties = null; 728 try { 729 dspModuleProperties = 730 mModelManagementService.getDspModuleProperties(mVoiceInteractionService); 731 } catch (RemoteException e) { 732 Slog.w(TAG, "RemoteException in getDspProperties!", e); 733 } 734 // No DSP available 735 if (dspModuleProperties == null) { 736 return STATE_HARDWARE_UNAVAILABLE; 737 } 738 // No enrollment application supports this keyphrase/locale 739 if (mKeyphraseMetadata == null) { 740 return STATE_KEYPHRASE_UNSUPPORTED; 741 } 742 return STATE_NOT_READY; 743 } 744 745 /** 746 * @return The corresponding {@link KeyphraseSoundModel} or null if none is found. 747 */ internalGetIsEnrolled(int keyphraseId, Locale locale)748 private boolean internalGetIsEnrolled(int keyphraseId, Locale locale) { 749 try { 750 return mModelManagementService.isEnrolledForKeyphrase( 751 mVoiceInteractionService, keyphraseId, locale.toLanguageTag()); 752 } catch (RemoteException e) { 753 Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!", e); 754 } 755 return false; 756 } 757 } 758 759 /** @hide */ dump(String prefix, PrintWriter pw)760 public void dump(String prefix, PrintWriter pw) { 761 synchronized (mLock) { 762 pw.print(prefix); pw.print("Text="); pw.println(mText); 763 pw.print(prefix); pw.print("Locale="); pw.println(mLocale); 764 pw.print(prefix); pw.print("Availability="); pw.println(mAvailability); 765 pw.print(prefix); pw.print("KeyphraseMetadata="); pw.println(mKeyphraseMetadata); 766 pw.print(prefix); pw.print("EnrollmentInfo="); pw.println(mKeyphraseEnrollmentInfo); 767 } 768 } 769 } 770