1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.app.Activity; 23 import android.content.Intent; 24 import android.hardware.soundtrigger.IRecognitionStatusCallback; 25 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; 26 import android.hardware.soundtrigger.KeyphraseMetadata; 27 import android.hardware.soundtrigger.SoundTrigger; 28 import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; 29 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 30 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 31 import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 32 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 33 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 34 import android.media.AudioFormat; 35 import android.os.AsyncTask; 36 import android.os.Handler; 37 import android.os.Message; 38 import android.os.RemoteException; 39 import android.util.Slog; 40 41 import com.android.internal.app.IVoiceInteractionManagerService; 42 43 import java.io.PrintWriter; 44 import java.lang.annotation.Retention; 45 import java.lang.annotation.RetentionPolicy; 46 import java.util.Locale; 47 48 /** 49 * A class that lets a VoiceInteractionService implementation interact with 50 * always-on keyphrase detection APIs. 51 */ 52 public class AlwaysOnHotwordDetector { 53 //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----// 54 /** 55 * Indicates that this hotword detector is no longer valid for any recognition 56 * and should not be used anymore. 57 */ 58 private static final int STATE_INVALID = -3; 59 60 /** 61 * Indicates that recognition for the given keyphrase is not available on the system 62 * because of the hardware configuration. 63 * No further interaction should be performed with the detector that returns this availability. 64 */ 65 public static final int STATE_HARDWARE_UNAVAILABLE = -2; 66 /** 67 * Indicates that recognition for the given keyphrase is not supported. 68 * No further interaction should be performed with the detector that returns this availability. 69 */ 70 public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; 71 /** 72 * Indicates that the given keyphrase is not enrolled. 73 * The caller may choose to begin an enrollment flow for the keyphrase. 74 */ 75 public static final int STATE_KEYPHRASE_UNENROLLED = 1; 76 /** 77 * Indicates that the given keyphrase is currently enrolled and it's possible to start 78 * recognition for it. 79 */ 80 public static final int STATE_KEYPHRASE_ENROLLED = 2; 81 82 /** 83 * Indicates that the detector isn't ready currently. 84 */ 85 private static final int STATE_NOT_READY = 0; 86 87 // Keyphrase management actions. Used in getManageIntent() ----// 88 @Retention(RetentionPolicy.SOURCE) 89 @IntDef(value = { 90 MANAGE_ACTION_ENROLL, 91 MANAGE_ACTION_RE_ENROLL, 92 MANAGE_ACTION_UN_ENROLL 93 }) 94 private @interface ManageActions {} 95 96 /** 97 * Indicates that we need to enroll. 98 * 99 * @hide 100 */ 101 public static final int MANAGE_ACTION_ENROLL = 0; 102 /** 103 * Indicates that we need to re-enroll. 104 * 105 * @hide 106 */ 107 public static final int MANAGE_ACTION_RE_ENROLL = 1; 108 /** 109 * Indicates that we need to un-enroll. 110 * 111 * @hide 112 */ 113 public static final int MANAGE_ACTION_UN_ENROLL = 2; 114 115 //-- Flags for startRecognition ----// 116 /** @hide */ 117 @Retention(RetentionPolicy.SOURCE) 118 @IntDef(flag = true, 119 value = { 120 RECOGNITION_FLAG_NONE, 121 RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO, 122 RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 123 }) 124 public @interface RecognitionFlags {} 125 126 /** 127 * Empty flag for {@link #startRecognition(int)}. 128 * 129 * @hide 130 */ 131 public static final int RECOGNITION_FLAG_NONE = 0; 132 /** 133 * Recognition flag for {@link #startRecognition(int)} that indicates 134 * whether the trigger audio for hotword needs to be captured. 135 */ 136 public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; 137 /** 138 * Recognition flag for {@link #startRecognition(int)} that indicates 139 * whether the recognition should keep going on even after the keyphrase triggers. 140 * If this flag is specified, it's possible to get multiple triggers after a 141 * call to {@link #startRecognition(int)} if the user speaks the keyphrase multiple times. 142 * When this isn't specified, the default behavior is to stop recognition once the 143 * keyphrase is spoken, till the caller starts recognition again. 144 */ 145 public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2; 146 147 //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----// 148 // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. 149 150 /** @hide */ 151 @Retention(RetentionPolicy.SOURCE) 152 @IntDef(flag = true, 153 value = { 154 RECOGNITION_MODE_VOICE_TRIGGER, 155 RECOGNITION_MODE_USER_IDENTIFICATION, 156 }) 157 public @interface RecognitionModes {} 158 159 /** 160 * Simple recognition of the key phrase. 161 * Returned by {@link #getSupportedRecognitionModes()} 162 */ 163 public static final int RECOGNITION_MODE_VOICE_TRIGGER 164 = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; 165 /** 166 * User identification performed with the keyphrase recognition. 167 * Returned by {@link #getSupportedRecognitionModes()} 168 */ 169 public static final int RECOGNITION_MODE_USER_IDENTIFICATION 170 = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; 171 172 static final String TAG = "AlwaysOnHotwordDetector"; 173 static final boolean DBG = false; 174 175 private static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 176 private static final int STATUS_OK = SoundTrigger.STATUS_OK; 177 178 private static final int MSG_AVAILABILITY_CHANGED = 1; 179 private static final int MSG_HOTWORD_DETECTED = 2; 180 private static final int MSG_DETECTION_ERROR = 3; 181 private static final int MSG_DETECTION_PAUSE = 4; 182 private static final int MSG_DETECTION_RESUME = 5; 183 184 private final String mText; 185 private final Locale mLocale; 186 /** 187 * The metadata of the Keyphrase, derived from the enrollment application. 188 * This may be null if this keyphrase isn't supported by the enrollment application. 189 */ 190 private final KeyphraseMetadata mKeyphraseMetadata; 191 private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; 192 private final IVoiceInteractionService mVoiceInteractionService; 193 private final IVoiceInteractionManagerService mModelManagementService; 194 private final SoundTriggerListener mInternalCallback; 195 private final Callback mExternalCallback; 196 private final Object mLock = new Object(); 197 private final Handler mHandler; 198 199 private int mAvailability = STATE_NOT_READY; 200 201 /** 202 * Additional payload for {@link Callback#onDetected}. 203 */ 204 public static class EventPayload { 205 private final boolean mTriggerAvailable; 206 // Indicates if {@code captureSession} can be used to continue capturing more audio 207 // from the DSP hardware. 208 private final boolean mCaptureAvailable; 209 // The session to use when attempting to capture more audio from the DSP hardware. 210 private final int mCaptureSession; 211 private final AudioFormat mAudioFormat; 212 // Raw data associated with the event. 213 // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true. 214 private final byte[] mData; 215 EventPayload(boolean triggerAvailable, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data)216 private EventPayload(boolean triggerAvailable, boolean captureAvailable, 217 AudioFormat audioFormat, int captureSession, byte[] data) { 218 mTriggerAvailable = triggerAvailable; 219 mCaptureAvailable = captureAvailable; 220 mCaptureSession = captureSession; 221 mAudioFormat = audioFormat; 222 mData = data; 223 } 224 225 /** 226 * Gets the format of the audio obtained using {@link #getTriggerAudio()}. 227 * May be null if there's no audio present. 228 */ 229 @Nullable getCaptureAudioFormat()230 public AudioFormat getCaptureAudioFormat() { 231 return mAudioFormat; 232 } 233 234 /** 235 * Gets the raw audio that triggered the keyphrase. 236 * This may be null if the trigger audio isn't available. 237 * If non-null, the format of the audio can be obtained by calling 238 * {@link #getCaptureAudioFormat()}. 239 * 240 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 241 */ 242 @Nullable getTriggerAudio()243 public byte[] getTriggerAudio() { 244 if (mTriggerAvailable) { 245 return mData; 246 } else { 247 return null; 248 } 249 } 250 251 /** 252 * Gets the session ID to start a capture from the DSP. 253 * This may be null if streaming capture isn't possible. 254 * If non-null, the format of the audio that can be captured can be 255 * obtained using {@link #getCaptureAudioFormat()}. 256 * 257 * TODO: Candidate for Public API when the API to start capture with a session ID 258 * is made public. 259 * 260 * TODO: Add this to {@link #getCaptureAudioFormat()}: 261 * "Gets the format of the audio obtained using {@link #getTriggerAudio()} 262 * or {@link #getCaptureSession()}. May be null if no audio can be obtained 263 * for either the trigger or a streaming session." 264 * 265 * TODO: Should this return a known invalid value instead? 266 * 267 * @hide 268 */ 269 @Nullable getCaptureSession()270 public Integer getCaptureSession() { 271 if (mCaptureAvailable) { 272 return mCaptureSession; 273 } else { 274 return null; 275 } 276 } 277 } 278 279 /** 280 * Callbacks for always-on hotword detection. 281 */ 282 public static abstract class Callback { 283 /** 284 * Called when the hotword availability changes. 285 * This indicates a change in the availability of recognition for the given keyphrase. 286 * It's called at least once with the initial availability.<p/> 287 * 288 * Availability implies whether the hardware on this system is capable of listening for 289 * the given keyphrase or not. <p/> 290 * 291 * @see AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE 292 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNSUPPORTED 293 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNENROLLED 294 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_ENROLLED 295 */ onAvailabilityChanged(int status)296 public abstract void onAvailabilityChanged(int status); 297 /** 298 * Called when the keyphrase is spoken. 299 * This implicitly stops listening for the keyphrase once it's detected. 300 * Clients should start a recognition again once they are done handling this 301 * detection. 302 * 303 * @param eventPayload Payload data for the detection event. 304 * This may contain the trigger audio, if requested when calling 305 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 306 */ onDetected(@onNull EventPayload eventPayload)307 public abstract void onDetected(@NonNull EventPayload eventPayload); 308 /** 309 * Called when the detection fails due to an error. 310 */ onError()311 public abstract void onError(); 312 /** 313 * Called when the recognition is paused temporarily for some reason. 314 * This is an informational callback, and the clients shouldn't be doing anything here 315 * except showing an indication on their UI if they have to. 316 */ onRecognitionPaused()317 public abstract void onRecognitionPaused(); 318 /** 319 * Called when the recognition is resumed after it was temporarily paused. 320 * This is an informational callback, and the clients shouldn't be doing anything here 321 * except showing an indication on their UI if they have to. 322 */ onRecognitionResumed()323 public abstract void onRecognitionResumed(); 324 } 325 326 /** 327 * @param text The keyphrase text to get the detector for. 328 * @param locale The java locale for the detector. 329 * @param callback A non-null Callback for receiving the recognition events. 330 * @param voiceInteractionService The current voice interaction service. 331 * @param modelManagementService A service that allows management of sound models. 332 * 333 * @hide 334 */ AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionService voiceInteractionService, IVoiceInteractionManagerService modelManagementService)335 public AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, 336 KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, 337 IVoiceInteractionService voiceInteractionService, 338 IVoiceInteractionManagerService modelManagementService) { 339 mText = text; 340 mLocale = locale; 341 mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; 342 mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale); 343 mExternalCallback = callback; 344 mHandler = new MyHandler(); 345 mInternalCallback = new SoundTriggerListener(mHandler); 346 mVoiceInteractionService = voiceInteractionService; 347 mModelManagementService = modelManagementService; 348 new RefreshAvailabiltyTask().execute(); 349 } 350 351 /** 352 * Gets the recognition modes supported by the associated keyphrase. 353 * 354 * @see #RECOGNITION_MODE_USER_IDENTIFICATION 355 * @see #RECOGNITION_MODE_VOICE_TRIGGER 356 * 357 * @throws UnsupportedOperationException if the keyphrase itself isn't supported. 358 * Callers should only call this method after a supported state callback on 359 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 360 * @throws IllegalStateException if the detector is in an invalid state. 361 * This may happen if another detector has been instantiated or the 362 * {@link VoiceInteractionService} hosting this detector has been shut down. 363 */ getSupportedRecognitionModes()364 public @RecognitionModes int getSupportedRecognitionModes() { 365 if (DBG) Slog.d(TAG, "getSupportedRecognitionModes()"); 366 synchronized (mLock) { 367 return getSupportedRecognitionModesLocked(); 368 } 369 } 370 getSupportedRecognitionModesLocked()371 private int getSupportedRecognitionModesLocked() { 372 if (mAvailability == STATE_INVALID) { 373 throw new IllegalStateException( 374 "getSupportedRecognitionModes called on an invalid detector"); 375 } 376 377 // This method only makes sense if we can actually support a recognition. 378 if (mAvailability != STATE_KEYPHRASE_ENROLLED 379 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 380 throw new UnsupportedOperationException( 381 "Getting supported recognition modes for the keyphrase is not supported"); 382 } 383 384 return mKeyphraseMetadata.recognitionModeFlags; 385 } 386 387 /** 388 * Starts recognition for the associated keyphrase. 389 * 390 * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 391 * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 392 * 393 * @param recognitionFlags The flags to control the recognition properties. 394 * @return Indicates whether the call succeeded or not. 395 * @throws UnsupportedOperationException if the recognition isn't supported. 396 * Callers should only call this method after a supported state callback on 397 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 398 * @throws IllegalStateException if the detector is in an invalid state. 399 * This may happen if another detector has been instantiated or the 400 * {@link VoiceInteractionService} hosting this detector has been shut down. 401 */ startRecognition(@ecognitionFlags int recognitionFlags)402 public boolean startRecognition(@RecognitionFlags int recognitionFlags) { 403 if (DBG) Slog.d(TAG, "startRecognition(" + recognitionFlags + ")"); 404 synchronized (mLock) { 405 if (mAvailability == STATE_INVALID) { 406 throw new IllegalStateException("startRecognition called on an invalid detector"); 407 } 408 409 // Check if we can start/stop a recognition. 410 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 411 throw new UnsupportedOperationException( 412 "Recognition for the given keyphrase is not supported"); 413 } 414 415 return startRecognitionLocked(recognitionFlags) == STATUS_OK; 416 } 417 } 418 419 /** 420 * Stops recognition for the associated keyphrase. 421 * 422 * @return Indicates whether the call succeeded or not. 423 * @throws UnsupportedOperationException if the recognition isn't supported. 424 * Callers should only call this method after a supported state callback on 425 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 426 * @throws IllegalStateException if the detector is in an invalid state. 427 * This may happen if another detector has been instantiated or the 428 * {@link VoiceInteractionService} hosting this detector has been shut down. 429 */ stopRecognition()430 public boolean stopRecognition() { 431 if (DBG) Slog.d(TAG, "stopRecognition()"); 432 synchronized (mLock) { 433 if (mAvailability == STATE_INVALID) { 434 throw new IllegalStateException("stopRecognition called on an invalid detector"); 435 } 436 437 // Check if we can start/stop a recognition. 438 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 439 throw new UnsupportedOperationException( 440 "Recognition for the given keyphrase is not supported"); 441 } 442 443 return stopRecognitionLocked() == STATUS_OK; 444 } 445 } 446 447 /** 448 * Creates an intent to start the enrollment for the associated keyphrase. 449 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 450 * Starting re-enrollment is only valid if the keyphrase is un-enrolled, 451 * i.e. {@link #STATE_KEYPHRASE_UNENROLLED}, 452 * otherwise {@link #createReEnrollIntent()} should be preferred. 453 * 454 * @return An {@link Intent} to start enrollment for the given keyphrase. 455 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 456 * Callers should only call this method after a supported state callback on 457 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 458 * @throws IllegalStateException if the detector is in an invalid state. 459 * This may happen if another detector has been instantiated or the 460 * {@link VoiceInteractionService} hosting this detector has been shut down. 461 */ createEnrollIntent()462 public Intent createEnrollIntent() { 463 if (DBG) Slog.d(TAG, "createEnrollIntent"); 464 synchronized (mLock) { 465 return getManageIntentLocked(MANAGE_ACTION_ENROLL); 466 } 467 } 468 469 /** 470 * Creates an intent to start the un-enrollment for the associated keyphrase. 471 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 472 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 473 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 474 * 475 * @return An {@link Intent} to start un-enrollment for the given keyphrase. 476 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 477 * Callers should only call this method after a supported state callback on 478 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 479 * @throws IllegalStateException if the detector is in an invalid state. 480 * This may happen if another detector has been instantiated or the 481 * {@link VoiceInteractionService} hosting this detector has been shut down. 482 */ createUnEnrollIntent()483 public Intent createUnEnrollIntent() { 484 if (DBG) Slog.d(TAG, "createUnEnrollIntent"); 485 synchronized (mLock) { 486 return getManageIntentLocked(MANAGE_ACTION_UN_ENROLL); 487 } 488 } 489 490 /** 491 * Creates an intent to start the re-enrollment for the associated keyphrase. 492 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 493 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 494 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 495 * 496 * @return An {@link Intent} to start re-enrollment for the given keyphrase. 497 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 498 * Callers should only call this method after a supported state callback on 499 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 500 * @throws IllegalStateException if the detector is in an invalid state. 501 * This may happen if another detector has been instantiated or the 502 * {@link VoiceInteractionService} hosting this detector has been shut down. 503 */ createReEnrollIntent()504 public Intent createReEnrollIntent() { 505 if (DBG) Slog.d(TAG, "createReEnrollIntent"); 506 synchronized (mLock) { 507 return getManageIntentLocked(MANAGE_ACTION_RE_ENROLL); 508 } 509 } 510 getManageIntentLocked(int action)511 private Intent getManageIntentLocked(int action) { 512 if (mAvailability == STATE_INVALID) { 513 throw new IllegalStateException("getManageIntent called on an invalid detector"); 514 } 515 516 // This method only makes sense if we can actually support a recognition. 517 if (mAvailability != STATE_KEYPHRASE_ENROLLED 518 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 519 throw new UnsupportedOperationException( 520 "Managing the given keyphrase is not supported"); 521 } 522 523 return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); 524 } 525 526 /** 527 * Invalidates this hotword detector so that any future calls to this result 528 * in an IllegalStateException. 529 * 530 * @hide 531 */ invalidate()532 void invalidate() { 533 synchronized (mLock) { 534 mAvailability = STATE_INVALID; 535 notifyStateChangedLocked(); 536 } 537 } 538 539 /** 540 * Reloads the sound models from the service. 541 * 542 * @hide 543 */ onSoundModelsChanged()544 void onSoundModelsChanged() { 545 synchronized (mLock) { 546 if (mAvailability == STATE_INVALID 547 || mAvailability == STATE_HARDWARE_UNAVAILABLE 548 || mAvailability == STATE_KEYPHRASE_UNSUPPORTED) { 549 Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config"); 550 return; 551 } 552 553 // Stop the recognition before proceeding. 554 // This is done because we want to stop the recognition on an older model if it changed 555 // or was deleted. 556 // The availability change callback should ensure that the client starts recognition 557 // again if needed. 558 stopRecognitionLocked(); 559 560 // Execute a refresh availability task - which should then notify of a change. 561 new RefreshAvailabiltyTask().execute(); 562 } 563 } 564 startRecognitionLocked(int recognitionFlags)565 private int startRecognitionLocked(int recognitionFlags) { 566 KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; 567 // TODO: Do we need to do something about the confidence level here? 568 recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id, 569 mKeyphraseMetadata.recognitionModeFlags, 0, new ConfidenceLevel[0]); 570 boolean captureTriggerAudio = 571 (recognitionFlags&RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; 572 boolean allowMultipleTriggers = 573 (recognitionFlags&RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0; 574 int code = STATUS_ERROR; 575 try { 576 code = mModelManagementService.startRecognition(mVoiceInteractionService, 577 mKeyphraseMetadata.id, mLocale.toLanguageTag(), mInternalCallback, 578 new RecognitionConfig(captureTriggerAudio, allowMultipleTriggers, 579 recognitionExtra, null /* additional data */)); 580 } catch (RemoteException e) { 581 Slog.w(TAG, "RemoteException in startRecognition!", e); 582 } 583 if (code != STATUS_OK) { 584 Slog.w(TAG, "startRecognition() failed with error code " + code); 585 } 586 return code; 587 } 588 stopRecognitionLocked()589 private int stopRecognitionLocked() { 590 int code = STATUS_ERROR; 591 try { 592 code = mModelManagementService.stopRecognition( 593 mVoiceInteractionService, mKeyphraseMetadata.id, mInternalCallback); 594 } catch (RemoteException e) { 595 Slog.w(TAG, "RemoteException in stopRecognition!", e); 596 } 597 598 if (code != STATUS_OK) { 599 Slog.w(TAG, "stopRecognition() failed with error code " + code); 600 } 601 return code; 602 } 603 notifyStateChangedLocked()604 private void notifyStateChangedLocked() { 605 Message message = Message.obtain(mHandler, MSG_AVAILABILITY_CHANGED); 606 message.arg1 = mAvailability; 607 message.sendToTarget(); 608 } 609 610 /** @hide */ 611 static final class SoundTriggerListener extends IRecognitionStatusCallback.Stub { 612 private final Handler mHandler; 613 SoundTriggerListener(Handler handler)614 public SoundTriggerListener(Handler handler) { 615 mHandler = handler; 616 } 617 618 @Override onDetected(KeyphraseRecognitionEvent event)619 public void onDetected(KeyphraseRecognitionEvent event) { 620 if (DBG) { 621 Slog.d(TAG, "onDetected(" + event + ")"); 622 } else { 623 Slog.i(TAG, "onDetected"); 624 } 625 Message.obtain(mHandler, MSG_HOTWORD_DETECTED, 626 new EventPayload(event.triggerInData, event.captureAvailable, 627 event.captureFormat, event.captureSession, event.data)) 628 .sendToTarget(); 629 } 630 631 @Override onError(int status)632 public void onError(int status) { 633 Slog.i(TAG, "onError: " + status); 634 mHandler.sendEmptyMessage(MSG_DETECTION_ERROR); 635 } 636 637 @Override onRecognitionPaused()638 public void onRecognitionPaused() { 639 Slog.i(TAG, "onRecognitionPaused"); 640 mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE); 641 } 642 643 @Override onRecognitionResumed()644 public void onRecognitionResumed() { 645 Slog.i(TAG, "onRecognitionResumed"); 646 mHandler.sendEmptyMessage(MSG_DETECTION_RESUME); 647 } 648 } 649 650 class MyHandler extends Handler { 651 @Override handleMessage(Message msg)652 public void handleMessage(Message msg) { 653 synchronized (mLock) { 654 if (mAvailability == STATE_INVALID) { 655 Slog.w(TAG, "Received message: " + msg.what + " for an invalid detector"); 656 return; 657 } 658 } 659 660 switch (msg.what) { 661 case MSG_AVAILABILITY_CHANGED: 662 mExternalCallback.onAvailabilityChanged(msg.arg1); 663 break; 664 case MSG_HOTWORD_DETECTED: 665 mExternalCallback.onDetected((EventPayload) msg.obj); 666 break; 667 case MSG_DETECTION_ERROR: 668 mExternalCallback.onError(); 669 break; 670 case MSG_DETECTION_PAUSE: 671 mExternalCallback.onRecognitionPaused(); 672 break; 673 case MSG_DETECTION_RESUME: 674 mExternalCallback.onRecognitionResumed(); 675 break; 676 default: 677 super.handleMessage(msg); 678 } 679 } 680 } 681 682 class RefreshAvailabiltyTask extends AsyncTask<Void, Void, Void> { 683 684 @Override doInBackground(Void... params)685 public Void doInBackground(Void... params) { 686 int availability = internalGetInitialAvailability(); 687 boolean enrolled = false; 688 // Fetch the sound model if the availability is one of the supported ones. 689 if (availability == STATE_NOT_READY 690 || availability == STATE_KEYPHRASE_UNENROLLED 691 || availability == STATE_KEYPHRASE_ENROLLED) { 692 enrolled = internalGetIsEnrolled(mKeyphraseMetadata.id, mLocale); 693 if (!enrolled) { 694 availability = STATE_KEYPHRASE_UNENROLLED; 695 } else { 696 availability = STATE_KEYPHRASE_ENROLLED; 697 } 698 } 699 700 synchronized (mLock) { 701 if (DBG) { 702 Slog.d(TAG, "Hotword availability changed from " + mAvailability 703 + " -> " + availability); 704 } 705 mAvailability = availability; 706 notifyStateChangedLocked(); 707 } 708 return null; 709 } 710 711 /** 712 * @return The initial availability without checking the enrollment status. 713 */ internalGetInitialAvailability()714 private int internalGetInitialAvailability() { 715 synchronized (mLock) { 716 // This detector has already been invalidated. 717 if (mAvailability == STATE_INVALID) { 718 return STATE_INVALID; 719 } 720 } 721 722 ModuleProperties dspModuleProperties = null; 723 try { 724 dspModuleProperties = 725 mModelManagementService.getDspModuleProperties(mVoiceInteractionService); 726 } catch (RemoteException e) { 727 Slog.w(TAG, "RemoteException in getDspProperties!", e); 728 } 729 // No DSP available 730 if (dspModuleProperties == null) { 731 return STATE_HARDWARE_UNAVAILABLE; 732 } 733 // No enrollment application supports this keyphrase/locale 734 if (mKeyphraseMetadata == null) { 735 return STATE_KEYPHRASE_UNSUPPORTED; 736 } 737 return STATE_NOT_READY; 738 } 739 740 /** 741 * @return The corresponding {@link KeyphraseSoundModel} or null if none is found. 742 */ internalGetIsEnrolled(int keyphraseId, Locale locale)743 private boolean internalGetIsEnrolled(int keyphraseId, Locale locale) { 744 try { 745 return mModelManagementService.isEnrolledForKeyphrase( 746 mVoiceInteractionService, keyphraseId, locale.toLanguageTag()); 747 } catch (RemoteException e) { 748 Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!", e); 749 } 750 return false; 751 } 752 } 753 754 /** @hide */ dump(String prefix, PrintWriter pw)755 public void dump(String prefix, PrintWriter pw) { 756 synchronized (mLock) { 757 pw.print(prefix); pw.print("Text="); pw.println(mText); 758 pw.print(prefix); pw.print("Locale="); pw.println(mLocale); 759 pw.print(prefix); pw.print("Availability="); pw.println(mAvailability); 760 pw.print(prefix); pw.print("KeyphraseMetadata="); pw.println(mKeyphraseMetadata); 761 pw.print(prefix); pw.print("EnrollmentInfo="); pw.println(mKeyphraseEnrollmentInfo); 762 } 763 } 764 } 765