1 package com.android.systemui.assist; 2 3 import static com.android.systemui.DejankUtils.whitelistIpcs; 4 import static com.android.systemui.shared.system.QuickStepContract.SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED; 5 6 import android.annotation.NonNull; 7 import android.annotation.Nullable; 8 import android.app.ActivityManager; 9 import android.app.ActivityOptions; 10 import android.app.SearchManager; 11 import android.app.StatusBarManager; 12 import android.content.ActivityNotFoundException; 13 import android.content.ComponentName; 14 import android.content.Context; 15 import android.content.Intent; 16 import android.metrics.LogMaker; 17 import android.os.AsyncTask; 18 import android.os.Bundle; 19 import android.os.Handler; 20 import android.os.RemoteException; 21 import android.os.SystemClock; 22 import android.os.UserHandle; 23 import android.provider.Settings; 24 import android.service.voice.VisualQueryAttentionResult; 25 import android.service.voice.VoiceInteractionSession; 26 import android.util.Log; 27 28 import com.android.internal.app.AssistUtils; 29 import com.android.internal.app.IVisualQueryDetectionAttentionListener; 30 import com.android.internal.app.IVisualQueryRecognitionStatusListener; 31 import com.android.internal.app.IVoiceInteractionSessionListener; 32 import com.android.internal.logging.MetricsLogger; 33 import com.android.internal.logging.nano.MetricsProto.MetricsEvent; 34 import com.android.systemui.assist.domain.interactor.AssistInteractor; 35 import com.android.systemui.assist.ui.DefaultUiController; 36 import com.android.systemui.dagger.SysUISingleton; 37 import com.android.systemui.dagger.qualifiers.Main; 38 import com.android.systemui.model.SysUiState; 39 import com.android.systemui.recents.OverviewProxyService; 40 import com.android.systemui.res.R; 41 import com.android.systemui.settings.DisplayTracker; 42 import com.android.systemui.settings.UserTracker; 43 import com.android.systemui.statusbar.CommandQueue; 44 import com.android.systemui.statusbar.policy.DeviceProvisionedController; 45 import com.android.systemui.user.domain.interactor.SelectedUserInteractor; 46 import com.android.systemui.util.settings.SecureSettings; 47 48 import dagger.Lazy; 49 50 import java.util.ArrayList; 51 import java.util.Arrays; 52 import java.util.List; 53 54 import javax.inject.Inject; 55 56 /** 57 * Class to manage everything related to assist in SystemUI. 58 */ 59 @SysUISingleton 60 public class AssistManager { 61 62 /** 63 * Controls the UI for showing Assistant invocation progress. 64 */ 65 public interface UiController { 66 /** 67 * Updates the invocation progress. 68 * 69 * @param type one of INVOCATION_TYPE_GESTURE, INVOCATION_TYPE_ACTIVE_EDGE, 70 * INVOCATION_TYPE_VOICE, INVOCATION_TYPE_QUICK_SEARCH_BAR, 71 * INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS 72 * @param progress a float between 0 and 1 inclusive. 0 represents the beginning of the 73 * gesture; 1 represents the end. 74 */ onInvocationProgress(int type, float progress)75 void onInvocationProgress(int type, float progress); 76 77 /** 78 * Called when an invocation gesture completes. 79 * 80 * @param velocity the speed of the invocation gesture, in pixels per millisecond. For 81 * drags, this is 0. 82 */ onGestureCompletion(float velocity)83 void onGestureCompletion(float velocity); 84 85 /** 86 * Hides any SysUI for the assistant, but _does not_ close the assistant itself. 87 */ hide()88 void hide(); 89 } 90 91 /** 92 * An interface for a listener that receives notification that visual query attention has 93 * either been gained or lost. 94 */ 95 public interface VisualQueryAttentionListener { 96 /** Called when visual query attention has been gained. */ onAttentionGained()97 void onAttentionGained(); 98 99 /** Called when visual query attention has been lost. */ onAttentionLost()100 void onAttentionLost(); 101 } 102 103 private static final String TAG = "AssistManager"; 104 105 // Note that VERBOSE logging may leak PII (e.g. transcription contents). 106 private static final boolean VERBOSE = false; 107 108 private static final String INVOCATION_TIME_MS_KEY = "invocation_time_ms"; 109 private static final String INVOCATION_PHONE_STATE_KEY = "invocation_phone_state"; 110 protected static final String ACTION_KEY = "action"; 111 protected static final String SET_ASSIST_GESTURE_CONSTRAINED_ACTION = 112 "set_assist_gesture_constrained"; 113 protected static final String CONSTRAINED_KEY = "should_constrain"; 114 115 public static final String INVOCATION_TYPE_KEY = "invocation_type"; 116 public static final int INVOCATION_TYPE_UNKNOWN = 117 AssistUtils.INVOCATION_TYPE_UNKNOWN; 118 public static final int INVOCATION_TYPE_GESTURE = 119 AssistUtils.INVOCATION_TYPE_GESTURE; 120 public static final int INVOCATION_TYPE_OTHER = 121 AssistUtils.INVOCATION_TYPE_PHYSICAL_GESTURE; 122 public static final int INVOCATION_TYPE_VOICE = 123 AssistUtils.INVOCATION_TYPE_VOICE; 124 public static final int INVOCATION_TYPE_QUICK_SEARCH_BAR = 125 AssistUtils.INVOCATION_TYPE_QUICK_SEARCH_BAR; 126 public static final int INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS = 127 AssistUtils.INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS; 128 public static final int INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS = 129 AssistUtils.INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS; 130 public static final int INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS = 131 AssistUtils.INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS; 132 133 public static final int DISMISS_REASON_INVOCATION_CANCELLED = 1; 134 public static final int DISMISS_REASON_TAP = 2; 135 public static final int DISMISS_REASON_BACK = 3; 136 public static final int DISMISS_REASON_TIMEOUT = 4; 137 138 private static final long TIMEOUT_SERVICE = 2500; 139 private static final long TIMEOUT_ACTIVITY = 1000; 140 141 protected final Context mContext; 142 private final AssistDisclosure mAssistDisclosure; 143 private final PhoneStateMonitor mPhoneStateMonitor; 144 private final OverviewProxyService mOverviewProxyService; 145 private final UiController mUiController; 146 protected final Lazy<SysUiState> mSysUiState; 147 protected final AssistLogger mAssistLogger; 148 private final UserTracker mUserTracker; 149 private final DisplayTracker mDisplayTracker; 150 private final SecureSettings mSecureSettings; 151 private final SelectedUserInteractor mSelectedUserInteractor; 152 private final ActivityManager mActivityManager; 153 private final AssistInteractor mInteractor; 154 155 private final DeviceProvisionedController mDeviceProvisionedController; 156 157 private final List<VisualQueryAttentionListener> mVisualQueryAttentionListeners = 158 new ArrayList<>(); 159 160 private final IVisualQueryDetectionAttentionListener mVisualQueryDetectionAttentionListener = 161 new IVisualQueryDetectionAttentionListener.Stub() { 162 @Override 163 public void onAttentionGained(VisualQueryAttentionResult attentionResult) { 164 // TODO (b/319132184): Implemented this with different types. 165 handleVisualAttentionChanged(true); 166 } 167 168 @Override 169 public void onAttentionLost(int interactionIntention) { 170 //TODO (b/319132184): Implemented this with different types. 171 handleVisualAttentionChanged(false); 172 } 173 }; 174 175 private final CommandQueue mCommandQueue; 176 protected final AssistUtils mAssistUtils; 177 178 // Invocation types that should be sent over OverviewProxy instead of handled here. 179 private int[] mAssistOverrideInvocationTypes; 180 181 @Inject AssistManager( DeviceProvisionedController controller, Context context, AssistUtils assistUtils, CommandQueue commandQueue, PhoneStateMonitor phoneStateMonitor, OverviewProxyService overviewProxyService, Lazy<SysUiState> sysUiState, DefaultUiController defaultUiController, AssistLogger assistLogger, @Main Handler uiHandler, UserTracker userTracker, DisplayTracker displayTracker, SecureSettings secureSettings, SelectedUserInteractor selectedUserInteractor, ActivityManager activityManager, AssistInteractor interactor)182 public AssistManager( 183 DeviceProvisionedController controller, 184 Context context, 185 AssistUtils assistUtils, 186 CommandQueue commandQueue, 187 PhoneStateMonitor phoneStateMonitor, 188 OverviewProxyService overviewProxyService, 189 Lazy<SysUiState> sysUiState, 190 DefaultUiController defaultUiController, 191 AssistLogger assistLogger, 192 @Main Handler uiHandler, 193 UserTracker userTracker, 194 DisplayTracker displayTracker, 195 SecureSettings secureSettings, 196 SelectedUserInteractor selectedUserInteractor, 197 ActivityManager activityManager, 198 AssistInteractor interactor) { 199 mContext = context; 200 mDeviceProvisionedController = controller; 201 mCommandQueue = commandQueue; 202 mAssistUtils = assistUtils; 203 mAssistDisclosure = new AssistDisclosure(context, uiHandler); 204 mOverviewProxyService = overviewProxyService; 205 mPhoneStateMonitor = phoneStateMonitor; 206 mAssistLogger = assistLogger; 207 mUserTracker = userTracker; 208 mDisplayTracker = displayTracker; 209 mSecureSettings = secureSettings; 210 mSelectedUserInteractor = selectedUserInteractor; 211 mActivityManager = activityManager; 212 mInteractor = interactor; 213 214 registerVoiceInteractionSessionListener(); 215 registerVisualQueryRecognitionStatusListener(); 216 217 mUiController = defaultUiController; 218 219 mSysUiState = sysUiState; 220 221 mOverviewProxyService.addCallback(new OverviewProxyService.OverviewProxyListener() { 222 @Override 223 public void onAssistantProgress(float progress) { 224 // Progress goes from 0 to 1 to indicate how close the assist gesture is to 225 // completion. 226 onInvocationProgress(INVOCATION_TYPE_GESTURE, progress); 227 } 228 229 @Override 230 public void onAssistantGestureCompletion(float velocity) { 231 onGestureCompletion(velocity); 232 } 233 }); 234 } 235 registerVoiceInteractionSessionListener()236 protected void registerVoiceInteractionSessionListener() { 237 mAssistUtils.registerVoiceInteractionSessionListener( 238 new IVoiceInteractionSessionListener.Stub() { 239 @Override 240 public void onVoiceSessionShown() throws RemoteException { 241 if (VERBOSE) { 242 Log.v(TAG, "Voice open"); 243 } 244 mAssistLogger.reportAssistantSessionEvent( 245 AssistantSessionEvent.ASSISTANT_SESSION_UPDATE); 246 } 247 248 @Override 249 public void onVoiceSessionHidden() throws RemoteException { 250 if (VERBOSE) { 251 Log.v(TAG, "Voice closed"); 252 } 253 mAssistLogger.reportAssistantSessionEvent( 254 AssistantSessionEvent.ASSISTANT_SESSION_CLOSE); 255 } 256 257 @Override 258 public void onVoiceSessionWindowVisibilityChanged(boolean visible) 259 throws RemoteException { 260 if (VERBOSE) { 261 Log.v(TAG, "Window visibility changed: " + visible); 262 } 263 } 264 265 @Override 266 public void onSetUiHints(Bundle hints) { 267 if (VERBOSE) { 268 Log.v(TAG, "UI hints received"); 269 } 270 271 String action = hints.getString(ACTION_KEY); 272 if (SET_ASSIST_GESTURE_CONSTRAINED_ACTION.equals(action)) { 273 mSysUiState.get() 274 .setFlag( 275 SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED, 276 hints.getBoolean(CONSTRAINED_KEY, false)) 277 .commitUpdate(mDisplayTracker.getDefaultDisplayId()); 278 } 279 } 280 }); 281 } 282 startAssist(Bundle args)283 public void startAssist(Bundle args) { 284 if (mActivityManager.getLockTaskModeState() == ActivityManager.LOCK_TASK_MODE_LOCKED) { 285 return; 286 } 287 if (shouldOverrideAssist(args)) { 288 try { 289 if (mOverviewProxyService.getProxy() == null) { 290 Log.w(TAG, "No OverviewProxyService to invoke assistant override"); 291 return; 292 } 293 mOverviewProxyService.getProxy().onAssistantOverrideInvoked( 294 args.getInt(INVOCATION_TYPE_KEY)); 295 } catch (RemoteException e) { 296 Log.w(TAG, "Unable to invoke assistant via OverviewProxyService override", e); 297 } 298 return; 299 } 300 301 final ComponentName assistComponent = getAssistInfo(); 302 if (assistComponent == null) { 303 return; 304 } 305 306 final boolean isService = assistComponent.equals(getVoiceInteractorComponentName()); 307 308 if (args == null) { 309 args = new Bundle(); 310 } 311 int legacyInvocationType = args.getInt(INVOCATION_TYPE_KEY, 0); 312 int legacyDeviceState = mPhoneStateMonitor.getPhoneState(); 313 args.putInt(INVOCATION_PHONE_STATE_KEY, legacyDeviceState); 314 args.putLong(INVOCATION_TIME_MS_KEY, SystemClock.elapsedRealtime()); 315 mAssistLogger.reportAssistantInvocationEventFromLegacy( 316 legacyInvocationType, 317 /* isInvocationComplete = */ true, 318 assistComponent, 319 legacyDeviceState); 320 logStartAssistLegacy(legacyInvocationType, legacyDeviceState); 321 mInteractor.onAssistantStarted(legacyInvocationType); 322 startAssistInternal(args, assistComponent, isService); 323 } 324 shouldOverrideAssist(Bundle args)325 private boolean shouldOverrideAssist(Bundle args) { 326 if (args == null || !args.containsKey(INVOCATION_TYPE_KEY)) { 327 return false; 328 } 329 330 int invocationType = args.getInt(INVOCATION_TYPE_KEY); 331 return shouldOverrideAssist(invocationType); 332 } 333 334 /** @return true if the invocation type should be handled by OverviewProxy instead of SysUI. */ shouldOverrideAssist(int invocationType)335 public boolean shouldOverrideAssist(int invocationType) { 336 return mAssistOverrideInvocationTypes != null 337 && Arrays.stream(mAssistOverrideInvocationTypes).anyMatch( 338 override -> override == invocationType); 339 } 340 341 /** 342 * @param invocationTypes The invocation types that will henceforth be handled via 343 * OverviewProxy (Launcher); other invocation types should be handled by 344 * this class. 345 */ setAssistantOverridesRequested(int[] invocationTypes)346 public void setAssistantOverridesRequested(int[] invocationTypes) { 347 mAssistOverrideInvocationTypes = invocationTypes; 348 } 349 350 /** Called when the user is performing an assistant invocation action (e.g. Active Edge) */ onInvocationProgress(int type, float progress)351 public void onInvocationProgress(int type, float progress) { 352 mUiController.onInvocationProgress(type, progress); 353 } 354 355 /** 356 * Called when the user has invoked the assistant with the incoming velocity, in pixels per 357 * millisecond. For invocations without a velocity (e.g. slow drag), the velocity is set to 358 * zero. 359 */ onGestureCompletion(float velocity)360 public void onGestureCompletion(float velocity) { 361 mUiController.onGestureCompletion(velocity); 362 } 363 hideAssist()364 public void hideAssist() { 365 mAssistUtils.hideCurrentSession(); 366 } 367 368 /** 369 * Add the given {@link VisualQueryAttentionListener} to the list of listeners awaiting 370 * notification of gaining/losing visual query attention. 371 */ addVisualQueryAttentionListener(VisualQueryAttentionListener listener)372 public void addVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 373 if (!mVisualQueryAttentionListeners.contains(listener)) { 374 mVisualQueryAttentionListeners.add(listener); 375 } 376 } 377 378 /** 379 * Remove the given {@link VisualQueryAttentionListener} from the list of listeners awaiting 380 * notification of gaining/losing visual query attention. 381 */ removeVisualQueryAttentionListener(VisualQueryAttentionListener listener)382 public void removeVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 383 mVisualQueryAttentionListeners.remove(listener); 384 } 385 startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, boolean isService)386 private void startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, 387 boolean isService) { 388 if (isService) { 389 startVoiceInteractor(args); 390 } else { 391 startAssistActivity(args, assistComponent); 392 } 393 } 394 startAssistActivity(Bundle args, @NonNull ComponentName assistComponent)395 private void startAssistActivity(Bundle args, @NonNull ComponentName assistComponent) { 396 if (!mDeviceProvisionedController.isDeviceProvisioned()) { 397 return; 398 } 399 400 // Close Recent Apps if needed 401 mCommandQueue.animateCollapsePanels( 402 CommandQueue.FLAG_EXCLUDE_SEARCH_PANEL | CommandQueue.FLAG_EXCLUDE_RECENTS_PANEL, 403 false /* force */); 404 405 boolean structureEnabled = mSecureSettings.getIntForUser( 406 Settings.Secure.ASSIST_STRUCTURE_ENABLED, 1, UserHandle.USER_CURRENT) != 0; 407 408 final SearchManager searchManager = 409 (SearchManager) mContext.getSystemService(Context.SEARCH_SERVICE); 410 if (searchManager == null) { 411 return; 412 } 413 final Intent intent = searchManager.getAssistIntent(structureEnabled); 414 if (intent == null) { 415 return; 416 } 417 intent.setComponent(assistComponent); 418 intent.putExtras(args); 419 420 if (structureEnabled && AssistUtils.isDisclosureEnabled(mContext)) { 421 showDisclosure(); 422 } 423 424 try { 425 final ActivityOptions opts = ActivityOptions.makeCustomAnimation(mContext, 426 R.anim.search_launch_enter, R.anim.search_launch_exit); 427 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); 428 AsyncTask.execute(new Runnable() { 429 @Override 430 public void run() { 431 mContext.startActivityAsUser(intent, opts.toBundle(), 432 mUserTracker.getUserHandle()); 433 } 434 }); 435 } catch (ActivityNotFoundException e) { 436 Log.w(TAG, "Activity not found for " + intent.getAction()); 437 } 438 } 439 startVoiceInteractor(Bundle args)440 private void startVoiceInteractor(Bundle args) { 441 mAssistUtils.showSessionForActiveService(args, 442 VoiceInteractionSession.SHOW_SOURCE_ASSIST_GESTURE, mContext.getAttributionTag(), 443 null, null); 444 } 445 registerVisualQueryRecognitionStatusListener()446 private void registerVisualQueryRecognitionStatusListener() { 447 if (!mContext.getResources() 448 .getBoolean(R.bool.config_enableVisualQueryAttentionDetection)) { 449 return; 450 } 451 452 mAssistUtils.subscribeVisualQueryRecognitionStatus( 453 new IVisualQueryRecognitionStatusListener.Stub() { 454 @Override 455 public void onStartPerceiving() { 456 mAssistUtils.enableVisualQueryDetection( 457 mVisualQueryDetectionAttentionListener); 458 final StatusBarManager statusBarManager = 459 mContext.getSystemService(StatusBarManager.class); 460 if (statusBarManager != null) { 461 statusBarManager.setIcon("assist_attention", 462 R.drawable.ic_assistant_attention_indicator, 463 0, "Attention Icon for Assistant"); 464 statusBarManager.setIconVisibility("assist_attention", false); 465 } 466 } 467 468 @Override 469 public void onStopPerceiving() { 470 // Treat this as a signal that attention has been lost (and inform listeners 471 // accordingly). 472 handleVisualAttentionChanged(false); 473 mAssistUtils.disableVisualQueryDetection(); 474 final StatusBarManager statusBarManager = 475 mContext.getSystemService(StatusBarManager.class); 476 if (statusBarManager != null) { 477 statusBarManager.removeIcon("assist_attention"); 478 } 479 } 480 }); 481 } 482 483 // TODO (b/319132184): Implemented this with different types. handleVisualAttentionChanged(boolean attentionGained)484 private void handleVisualAttentionChanged(boolean attentionGained) { 485 final StatusBarManager statusBarManager = mContext.getSystemService(StatusBarManager.class); 486 if (statusBarManager != null) { 487 statusBarManager.setIconVisibility("assist_attention", attentionGained); 488 } 489 mVisualQueryAttentionListeners.forEach( 490 attentionGained 491 ? VisualQueryAttentionListener::onAttentionGained 492 : VisualQueryAttentionListener::onAttentionLost); 493 } 494 launchVoiceAssistFromKeyguard()495 public void launchVoiceAssistFromKeyguard() { 496 mAssistUtils.launchVoiceAssistFromKeyguard(); 497 } 498 canVoiceAssistBeLaunchedFromKeyguard()499 public boolean canVoiceAssistBeLaunchedFromKeyguard() { 500 // TODO(b/140051519) 501 return whitelistIpcs(() -> mAssistUtils.activeServiceSupportsLaunchFromKeyguard()); 502 } 503 getVoiceInteractorComponentName()504 public ComponentName getVoiceInteractorComponentName() { 505 return mAssistUtils.getActiveServiceComponentName(); 506 } 507 isVoiceSessionRunning()508 private boolean isVoiceSessionRunning() { 509 return mAssistUtils.isSessionRunning(); 510 } 511 512 @Nullable getAssistInfoForUser(int userId)513 public ComponentName getAssistInfoForUser(int userId) { 514 return mAssistUtils.getAssistComponentForUser(userId); 515 } 516 517 @Nullable getAssistInfo()518 private ComponentName getAssistInfo() { 519 return getAssistInfoForUser(mSelectedUserInteractor.getSelectedUserId()); 520 } 521 showDisclosure()522 public void showDisclosure() { 523 mAssistDisclosure.postShow(); 524 } 525 onLockscreenShown()526 public void onLockscreenShown() { 527 AsyncTask.execute(new Runnable() { 528 @Override 529 public void run() { 530 mAssistUtils.onLockscreenShown(); 531 } 532 }); 533 } 534 535 /** Returns the logging flags for the given Assistant invocation type. */ toLoggingSubType(int invocationType)536 public int toLoggingSubType(int invocationType) { 537 return toLoggingSubType(invocationType, mPhoneStateMonitor.getPhoneState()); 538 } 539 logStartAssistLegacy(int invocationType, int phoneState)540 protected void logStartAssistLegacy(int invocationType, int phoneState) { 541 MetricsLogger.action( 542 new LogMaker(MetricsEvent.ASSISTANT) 543 .setType(MetricsEvent.TYPE_OPEN) 544 .setSubtype(toLoggingSubType(invocationType, phoneState))); 545 } 546 toLoggingSubType(int invocationType, int phoneState)547 protected final int toLoggingSubType(int invocationType, int phoneState) { 548 // Note that this logic will break if the number of Assistant invocation types exceeds 7. 549 // There are currently 5 invocation types, but we will be migrating to the new logging 550 // framework in the next update. 551 int subType = 0; 552 subType |= invocationType << 1; 553 subType |= phoneState << 4; 554 return subType; 555 } 556 } 557