1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.voiceinteraction; 18 19 import static android.app.AppOpsManager.OP_ASSIST_SCREENSHOT; 20 import static android.app.AppOpsManager.OP_ASSIST_STRUCTURE; 21 import static android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION; 22 import static android.content.Intent.FLAG_GRANT_WRITE_URI_PERMISSION; 23 import static android.service.voice.VoiceInteractionSession.KEY_FOREGROUND_ACTIVITIES; 24 import static android.view.Display.DEFAULT_DISPLAY; 25 import static android.view.WindowManager.LayoutParams.TYPE_VOICE_INTERACTION; 26 27 import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_ACTIVITY_ID; 28 import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_KEY_CONTENT; 29 import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_KEY_DATA; 30 import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_KEY_STRUCTURE; 31 import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_TASK_ID; 32 33 import android.annotation.NonNull; 34 import android.annotation.Nullable; 35 import android.app.ActivityManager; 36 import android.app.ActivityTaskManager; 37 import android.app.AppOpsManager; 38 import android.app.IActivityManager; 39 import android.app.IActivityTaskManager; 40 import android.app.UriGrantsManager; 41 import android.app.assist.AssistContent; 42 import android.app.assist.AssistStructure; 43 import android.content.ClipData; 44 import android.content.ComponentName; 45 import android.content.ContentProvider; 46 import android.content.Context; 47 import android.content.Intent; 48 import android.content.ServiceConnection; 49 import android.graphics.Bitmap; 50 import android.hardware.power.Boost; 51 import android.net.Uri; 52 import android.os.Binder; 53 import android.os.Bundle; 54 import android.os.Handler; 55 import android.os.IBinder; 56 import android.os.PowerManager; 57 import android.os.PowerManagerInternal; 58 import android.os.RemoteException; 59 import android.os.ServiceManager; 60 import android.os.UserHandle; 61 import android.provider.Settings; 62 import android.service.voice.IVoiceInteractionSession; 63 import android.service.voice.IVoiceInteractionSessionService; 64 import android.service.voice.VisibleActivityInfo; 65 import android.service.voice.VoiceInteractionService; 66 import android.service.voice.VoiceInteractionSession; 67 import android.util.ArrayMap; 68 import android.util.Slog; 69 import android.view.IWindowManager; 70 71 import com.android.internal.app.AssistUtils; 72 import com.android.internal.app.IVoiceInteractionSessionShowCallback; 73 import com.android.internal.app.IVoiceInteractor; 74 import com.android.server.FgThread; 75 import com.android.server.LocalServices; 76 import com.android.server.am.AssistDataRequester; 77 import com.android.server.am.AssistDataRequester.AssistDataRequesterCallbacks; 78 import com.android.server.power.LowPowerStandbyControllerInternal; 79 import com.android.server.statusbar.StatusBarManagerInternal; 80 import com.android.server.uri.UriGrantsManagerInternal; 81 import com.android.server.wm.ActivityAssistInfo; 82 import com.android.server.wm.ActivityTaskManagerInternal; 83 84 import java.io.PrintWriter; 85 import java.time.Instant; 86 import java.util.ArrayList; 87 import java.util.List; 88 import java.util.concurrent.Executors; 89 import java.util.concurrent.ScheduledExecutorService; 90 91 final class VoiceInteractionSessionConnection implements ServiceConnection, 92 AssistDataRequesterCallbacks { 93 94 static final String TAG = "VoiceInteractionServiceManager"; 95 static final boolean DEBUG = false; 96 static final int POWER_BOOST_TIMEOUT_MS = Integer.parseInt( 97 System.getProperty("vendor.powerhal.interaction.max", "200")); 98 static final int BOOST_TIMEOUT_MS = 300; 99 /** 100 * The maximum time an app can stay on the Low Power Standby allowlist when 101 * the session is shown. There to safeguard against apps that don't call hide. 102 */ 103 private static final int LOW_POWER_STANDBY_ALLOWLIST_TIMEOUT_MS = 120_000; 104 // TODO: To avoid ap doesn't call hide, only 10 secs for now, need a better way to manage it 105 // in the future. 106 static final int MAX_POWER_BOOST_TIMEOUT = 10_000; 107 108 final IBinder mToken = new Binder(); 109 final Object mLock; 110 final ComponentName mSessionComponentName; 111 final Intent mBindIntent; 112 final int mUser; 113 final Context mContext; 114 final Callback mCallback; 115 final int mCallingUid; 116 final Handler mHandler; 117 final IActivityTaskManager mActivityTaskManager; 118 final IActivityManager mAm; 119 final UriGrantsManagerInternal mUgmInternal; 120 final IWindowManager mIWindowManager; 121 final AppOpsManager mAppOps; 122 final IBinder mPermissionOwner; 123 boolean mShown; 124 Bundle mShowArgs; 125 int mShowFlags; 126 boolean mBound; 127 boolean mFullyBound; 128 boolean mCanceled; 129 IVoiceInteractionSessionService mService; 130 IVoiceInteractionSession mSession; 131 IVoiceInteractor mInteractor; 132 ArrayList<IVoiceInteractionSessionShowCallback> mPendingShowCallbacks = new ArrayList<>(); 133 private List<ActivityAssistInfo> mPendingHandleAssistWithoutData = new ArrayList<>(); 134 AssistDataRequester mAssistDataRequester; 135 private boolean mListeningVisibleActivity; 136 private final ScheduledExecutorService mScheduledExecutorService = 137 Executors.newSingleThreadScheduledExecutor(); 138 // Records the visible activity information the system has already called onVisible, without 139 // confirming the result of callback. When activity visible state is changed, we use this to 140 // determine to call onVisible or onInvisible to assistant application. 141 private final ArrayMap<IBinder, VisibleActivityInfo> mVisibleActivityInfoForToken = 142 new ArrayMap<>(); 143 private final PowerManagerInternal mPowerManagerInternal; 144 private final LowPowerStandbyControllerInternal mLowPowerStandbyControllerInternal; 145 private final Runnable mRemoveFromLowPowerStandbyAllowlistRunnable = 146 this::removeFromLowPowerStandbyAllowlist; 147 private boolean mLowPowerStandbyAllowlisted; 148 private PowerBoostSetter mSetPowerBoostRunnable; 149 private final Handler mFgHandler; 150 151 class PowerBoostSetter implements Runnable { 152 153 private boolean mCanceled; 154 private final Instant mExpiryTime; 155 PowerBoostSetter(Instant expiryTime)156 PowerBoostSetter(Instant expiryTime) { 157 mExpiryTime = expiryTime; 158 } 159 160 @Override run()161 public void run() { 162 synchronized (mLock) { 163 if (mCanceled) { 164 return; 165 } 166 // To avoid voice interaction service does not call hide to cancel setting 167 // power boost. We will cancel set boost when reaching the max timeout. 168 if (Instant.now().isBefore(mExpiryTime)) { 169 mPowerManagerInternal.setPowerBoost(Boost.INTERACTION, BOOST_TIMEOUT_MS); 170 if (mSetPowerBoostRunnable != null) { 171 mFgHandler.postDelayed(mSetPowerBoostRunnable, POWER_BOOST_TIMEOUT_MS); 172 } 173 } else { 174 Slog.w(TAG, "Reset power boost INTERACTION because reaching max timeout."); 175 mPowerManagerInternal.setPowerBoost(Boost.INTERACTION, /* durationMs */ -1); 176 } 177 } 178 } 179 cancel()180 void cancel() { 181 synchronized (mLock) { 182 mCanceled = true; 183 } 184 } 185 } 186 187 IVoiceInteractionSessionShowCallback mShowCallback = 188 new IVoiceInteractionSessionShowCallback.Stub() { 189 @Override 190 public void onFailed() throws RemoteException { 191 synchronized (mLock) { 192 notifyPendingShowCallbacksFailedLocked(); 193 } 194 } 195 196 @Override 197 public void onShown() throws RemoteException { 198 synchronized (mLock) { 199 // TODO: Figure out whether this is good enough or whether we need to hook into 200 // Window manager to actually wait for the window to be drawn. 201 notifyPendingShowCallbacksShownLocked(); 202 } 203 } 204 }; 205 206 public interface Callback { sessionConnectionGone(VoiceInteractionSessionConnection connection)207 public void sessionConnectionGone(VoiceInteractionSessionConnection connection); onSessionShown(VoiceInteractionSessionConnection connection)208 public void onSessionShown(VoiceInteractionSessionConnection connection); onSessionHidden(VoiceInteractionSessionConnection connection)209 public void onSessionHidden(VoiceInteractionSessionConnection connection); 210 } 211 212 final ServiceConnection mFullConnection = new ServiceConnection() { 213 @Override 214 public void onServiceConnected(ComponentName name, IBinder service) { 215 } 216 @Override 217 public void onServiceDisconnected(ComponentName name) { 218 } 219 }; 220 VoiceInteractionSessionConnection(Object lock, ComponentName component, int user, Context context, Callback callback, int callingUid, Handler handler)221 public VoiceInteractionSessionConnection(Object lock, ComponentName component, int user, 222 Context context, Callback callback, int callingUid, Handler handler) { 223 mLock = lock; 224 mSessionComponentName = component; 225 mUser = user; 226 mContext = context; 227 mCallback = callback; 228 mCallingUid = callingUid; 229 mHandler = handler; 230 mActivityTaskManager = ActivityTaskManager.getService(); 231 mAm = ActivityManager.getService(); 232 mUgmInternal = LocalServices.getService(UriGrantsManagerInternal.class); 233 mIWindowManager = IWindowManager.Stub.asInterface( 234 ServiceManager.getService(Context.WINDOW_SERVICE)); 235 mPowerManagerInternal = LocalServices.getService(PowerManagerInternal.class); 236 mLowPowerStandbyControllerInternal = LocalServices.getService( 237 LowPowerStandbyControllerInternal.class); 238 mAppOps = context.getSystemService(AppOpsManager.class); 239 mFgHandler = FgThread.getHandler(); 240 mAssistDataRequester = new AssistDataRequester(mContext, mIWindowManager, 241 (AppOpsManager) mContext.getSystemService(Context.APP_OPS_SERVICE), 242 this, mLock, OP_ASSIST_STRUCTURE, OP_ASSIST_SCREENSHOT); 243 final IBinder permOwner = mUgmInternal.newUriPermissionOwner("voicesession:" 244 + component.flattenToShortString()); 245 mPermissionOwner = permOwner; 246 mBindIntent = new Intent(VoiceInteractionService.SERVICE_INTERFACE); 247 mBindIntent.setComponent(mSessionComponentName); 248 mBound = mContext.bindServiceAsUser(mBindIntent, this, 249 Context.BIND_AUTO_CREATE | Context.BIND_WAIVE_PRIORITY 250 | Context.BIND_ALLOW_OOM_MANAGEMENT 251 | Context.BIND_ALLOW_BACKGROUND_ACTIVITY_STARTS, new UserHandle(mUser)); 252 if (mBound) { 253 try { 254 mIWindowManager.addWindowToken(mToken, TYPE_VOICE_INTERACTION, DEFAULT_DISPLAY, 255 null /* options */); 256 } catch (RemoteException e) { 257 Slog.w(TAG, "Failed adding window token", e); 258 } 259 } else { 260 Slog.w(TAG, "Failed binding to voice interaction session service " 261 + mSessionComponentName); 262 } 263 } 264 getUserDisabledShowContextLocked()265 public int getUserDisabledShowContextLocked() { 266 int flags = 0; 267 if (Settings.Secure.getIntForUser(mContext.getContentResolver(), 268 Settings.Secure.ASSIST_STRUCTURE_ENABLED, 1, mUser) == 0) { 269 flags |= VoiceInteractionSession.SHOW_WITH_ASSIST; 270 } 271 if (Settings.Secure.getIntForUser(mContext.getContentResolver(), 272 Settings.Secure.ASSIST_SCREENSHOT_ENABLED, 1, mUser) == 0) { 273 flags |= VoiceInteractionSession.SHOW_WITH_SCREENSHOT; 274 } 275 return flags; 276 } 277 showLocked(@onNull Bundle args, int flags, @Nullable String attributionTag, int disabledContext, @Nullable IVoiceInteractionSessionShowCallback showCallback, @NonNull List<ActivityAssistInfo> topActivities)278 public boolean showLocked(@NonNull Bundle args, int flags, @Nullable String attributionTag, 279 int disabledContext, @Nullable IVoiceInteractionSessionShowCallback showCallback, 280 @NonNull List<ActivityAssistInfo> topActivities) { 281 if (mBound) { 282 if (!mFullyBound) { 283 mFullyBound = mContext.bindServiceAsUser(mBindIntent, mFullConnection, 284 Context.BIND_AUTO_CREATE | Context.BIND_TREAT_LIKE_ACTIVITY 285 | Context.BIND_SCHEDULE_LIKE_TOP_APP 286 | Context.BIND_TREAT_LIKE_VISIBLE_FOREGROUND_SERVICE 287 | Context.BIND_ALLOW_BACKGROUND_ACTIVITY_STARTS, 288 new UserHandle(mUser)); 289 } 290 291 mShown = true; 292 mShowArgs = args; 293 mShowFlags = flags; 294 295 disabledContext |= getUserDisabledShowContextLocked(); 296 297 boolean fetchData = (flags & VoiceInteractionSession.SHOW_WITH_ASSIST) != 0; 298 boolean fetchScreenshot = (flags & VoiceInteractionSession.SHOW_WITH_SCREENSHOT) != 0; 299 boolean assistDataRequestNeeded = fetchData || fetchScreenshot; 300 301 if (assistDataRequestNeeded) { 302 int topActivitiesCount = topActivities.size(); 303 final ArrayList<IBinder> topActivitiesToken = new ArrayList<>(topActivitiesCount); 304 for (int i = 0; i < topActivitiesCount; i++) { 305 topActivitiesToken.add(topActivities.get(i).getActivityToken()); 306 } 307 boolean fetchDataAllowed = 308 (disabledContext & VoiceInteractionSession.SHOW_WITH_ASSIST) == 0; 309 310 // Ensure that the current activity supports assist data 311 boolean isAssistDataAllowed = false; 312 try { 313 isAssistDataAllowed = mActivityTaskManager.isAssistDataAllowed(); 314 } catch (RemoteException e) { 315 // Should never happen 316 } 317 318 // TODO: Refactor to have all assist data allowed checks in one place. 319 if (fetchDataAllowed && isAssistDataAllowed) { 320 ArrayList<ComponentName> topComponents = new ArrayList<>(topActivitiesCount); 321 for (int i = 0; i < topActivitiesCount; i++) { 322 topComponents.add(topActivities.get(i).getComponentName()); 323 } 324 mShowArgs.putParcelableArrayList(KEY_FOREGROUND_ACTIVITIES, topComponents); 325 } 326 327 mAssistDataRequester.requestAssistData(topActivitiesToken, 328 fetchData, 329 fetchScreenshot, 330 fetchDataAllowed, 331 (disabledContext & VoiceInteractionSession.SHOW_WITH_SCREENSHOT) == 0, 332 mCallingUid, 333 mSessionComponentName.getPackageName(), 334 attributionTag); 335 336 boolean needDisclosure = mAssistDataRequester.getPendingDataCount() > 0 337 || mAssistDataRequester.getPendingScreenshotCount() > 0; 338 if (needDisclosure && AssistUtils.shouldDisclose(mContext, mSessionComponentName)) { 339 mHandler.post(mShowAssistDisclosureRunnable); 340 } 341 } 342 if (mSession != null) { 343 try { 344 mSession.show(mShowArgs, mShowFlags, showCallback); 345 mShowArgs = null; 346 mShowFlags = 0; 347 } catch (RemoteException e) { 348 } 349 if (assistDataRequestNeeded) { 350 mAssistDataRequester.processPendingAssistData(); 351 } else { 352 doHandleAssistWithoutData(topActivities); 353 } 354 } else { 355 if (showCallback != null) { 356 mPendingShowCallbacks.add(showCallback); 357 } 358 if (!assistDataRequestNeeded) { 359 // If no data are required we are not passing trough mAssistDataRequester. As 360 // a consequence, when a new session is delivered it is needed to process those 361 // requests manually. 362 mPendingHandleAssistWithoutData = topActivities; 363 } 364 } 365 // remove if already existing one. 366 if (mSetPowerBoostRunnable != null) { 367 mSetPowerBoostRunnable.cancel(); 368 } 369 mSetPowerBoostRunnable = new PowerBoostSetter( 370 Instant.now().plusMillis(MAX_POWER_BOOST_TIMEOUT)); 371 mFgHandler.post(mSetPowerBoostRunnable); 372 373 if (mLowPowerStandbyControllerInternal != null) { 374 mLowPowerStandbyControllerInternal.addToAllowlist(mCallingUid, 375 PowerManager.LOW_POWER_STANDBY_ALLOWED_REASON_VOICE_INTERACTION); 376 mLowPowerStandbyAllowlisted = true; 377 mFgHandler.removeCallbacks(mRemoveFromLowPowerStandbyAllowlistRunnable); 378 mFgHandler.postDelayed(mRemoveFromLowPowerStandbyAllowlistRunnable, 379 LOW_POWER_STANDBY_ALLOWLIST_TIMEOUT_MS); 380 } 381 382 mCallback.onSessionShown(this); 383 return true; 384 } 385 if (showCallback != null) { 386 try { 387 showCallback.onFailed(); 388 } catch (RemoteException e) { 389 } 390 } 391 return false; 392 } 393 doHandleAssistWithoutData(List<ActivityAssistInfo> topActivities)394 private void doHandleAssistWithoutData(List<ActivityAssistInfo> topActivities) { 395 final int activityCount = topActivities.size(); 396 for (int i = 0; i < activityCount; i++) { 397 final ActivityAssistInfo topActivity = topActivities.get(i); 398 final IBinder assistToken = topActivity.getAssistToken(); 399 final int taskId = topActivity.getTaskId(); 400 final int activityIndex = i; 401 try { 402 mSession.handleAssist( 403 taskId, 404 assistToken, 405 /* assistData = */ null, 406 /* assistStructure = */ null, 407 /* assistContent = */ null, 408 activityIndex, 409 activityCount); 410 } catch (RemoteException e) { 411 // Ignore 412 } 413 } 414 } 415 416 @Override canHandleReceivedAssistDataLocked()417 public boolean canHandleReceivedAssistDataLocked() { 418 return mSession != null; 419 } 420 421 @Override onAssistDataReceivedLocked(Bundle data, int activityIndex, int activityCount)422 public void onAssistDataReceivedLocked(Bundle data, int activityIndex, int activityCount) { 423 // Return early if we have no session 424 if (mSession == null) { 425 return; 426 } 427 428 if (data == null) { 429 try { 430 mSession.handleAssist(-1, null, null, null, null, 0, 0); 431 } catch (RemoteException e) { 432 // Ignore 433 } 434 } else { 435 final int taskId = data.getInt(ASSIST_TASK_ID); 436 final IBinder activityId = data.getBinder(ASSIST_ACTIVITY_ID); 437 final Bundle assistData = data.getBundle(ASSIST_KEY_DATA); 438 final AssistStructure structure = data.getParcelable(ASSIST_KEY_STRUCTURE, android.app.assist.AssistStructure.class); 439 final AssistContent content = data.getParcelable(ASSIST_KEY_CONTENT, android.app.assist.AssistContent.class); 440 int uid = -1; 441 if (assistData != null) { 442 uid = assistData.getInt(Intent.EXTRA_ASSIST_UID, -1); 443 } 444 if (uid >= 0 && content != null) { 445 Intent intent = content.getIntent(); 446 if (intent != null) { 447 ClipData clipData = intent.getClipData(); 448 if (clipData != null && Intent.isAccessUriMode(intent.getFlags())) { 449 grantClipDataPermissions(clipData, intent.getFlags(), uid, 450 mCallingUid, mSessionComponentName.getPackageName()); 451 } 452 } 453 ClipData clipData = content.getClipData(); 454 if (clipData != null) { 455 grantClipDataPermissions(clipData, FLAG_GRANT_READ_URI_PERMISSION, 456 uid, mCallingUid, mSessionComponentName.getPackageName()); 457 } 458 } 459 try { 460 mSession.handleAssist(taskId, activityId, assistData, structure, 461 content, activityIndex, activityCount); 462 } catch (RemoteException e) { 463 // Ignore 464 } 465 } 466 } 467 468 @Override onAssistScreenshotReceivedLocked(Bitmap screenshot)469 public void onAssistScreenshotReceivedLocked(Bitmap screenshot) { 470 // Return early if we have no session 471 if (mSession == null) { 472 return; 473 } 474 475 try { 476 mSession.handleScreenshot(screenshot); 477 } catch (RemoteException e) { 478 // Ignore 479 } 480 } 481 grantUriPermission(Uri uri, int mode, int srcUid, int destUid, String destPkg)482 void grantUriPermission(Uri uri, int mode, int srcUid, int destUid, String destPkg) { 483 if (!"content".equals(uri.getScheme())) { 484 return; 485 } 486 final long ident = Binder.clearCallingIdentity(); 487 try { 488 // This will throw SecurityException for us. 489 mUgmInternal.checkGrantUriPermission(srcUid, null, 490 ContentProvider.getUriWithoutUserId(uri), mode, 491 ContentProvider.getUserIdFromUri(uri, UserHandle.getUserId(srcUid))); 492 // No security exception, do the grant. 493 int sourceUserId = ContentProvider.getUserIdFromUri(uri, mUser); 494 uri = ContentProvider.getUriWithoutUserId(uri); 495 UriGrantsManager.getService().grantUriPermissionFromOwner(mPermissionOwner, srcUid, 496 destPkg, uri, FLAG_GRANT_READ_URI_PERMISSION, sourceUserId, mUser); 497 } catch (RemoteException e) { 498 } catch (SecurityException e) { 499 Slog.w(TAG, "Can't propagate permission", e); 500 } finally { 501 Binder.restoreCallingIdentity(ident); 502 } 503 504 } 505 grantClipDataItemPermission(ClipData.Item item, int mode, int srcUid, int destUid, String destPkg)506 void grantClipDataItemPermission(ClipData.Item item, int mode, int srcUid, int destUid, 507 String destPkg) { 508 if (item.getUri() != null) { 509 grantUriPermission(item.getUri(), mode, srcUid, destUid, destPkg); 510 } 511 Intent intent = item.getIntent(); 512 if (intent != null && intent.getData() != null) { 513 grantUriPermission(intent.getData(), mode, srcUid, destUid, destPkg); 514 } 515 } 516 grantClipDataPermissions(ClipData data, int mode, int srcUid, int destUid, String destPkg)517 void grantClipDataPermissions(ClipData data, int mode, int srcUid, int destUid, 518 String destPkg) { 519 final int N = data.getItemCount(); 520 for (int i=0; i<N; i++) { 521 grantClipDataItemPermission(data.getItemAt(i), mode, srcUid, destUid, destPkg); 522 } 523 } 524 hideLocked()525 public boolean hideLocked() { 526 if (mBound) { 527 if (mShown) { 528 mShown = false; 529 mShowArgs = null; 530 mShowFlags = 0; 531 mAssistDataRequester.cancel(); 532 mPendingShowCallbacks.clear(); 533 if (mSession != null) { 534 try { 535 mSession.hide(); 536 } catch (RemoteException e) { 537 } 538 } 539 mUgmInternal.revokeUriPermissionFromOwner(mPermissionOwner, null, 540 FLAG_GRANT_READ_URI_PERMISSION | FLAG_GRANT_WRITE_URI_PERMISSION, mUser); 541 if (mSession != null) { 542 try { 543 ActivityTaskManager.getService().finishVoiceTask(mSession); 544 } catch (RemoteException e) { 545 } 546 } 547 if (mSetPowerBoostRunnable != null) { 548 mSetPowerBoostRunnable.cancel(); 549 mSetPowerBoostRunnable = null; 550 } 551 // A negative value indicates canceling previous boost. 552 mPowerManagerInternal.setPowerBoost(Boost.INTERACTION, /* durationMs */ -1); 553 if (mLowPowerStandbyControllerInternal != null) { 554 removeFromLowPowerStandbyAllowlist(); 555 } 556 mCallback.onSessionHidden(this); 557 } 558 if (mFullyBound) { 559 mContext.unbindService(mFullConnection); 560 mFullyBound = false; 561 } 562 return true; 563 } 564 return false; 565 } 566 cancelLocked(boolean finishTask)567 public void cancelLocked(boolean finishTask) { 568 mListeningVisibleActivity = false; 569 mVisibleActivityInfoForToken.clear(); 570 hideLocked(); 571 mCanceled = true; 572 if (mBound) { 573 if (mSession != null) { 574 try { 575 mSession.destroy(); 576 } catch (RemoteException e) { 577 Slog.w(TAG, "Voice interation session already dead"); 578 } 579 } 580 if (finishTask && mSession != null) { 581 try { 582 ActivityTaskManager.getService().finishVoiceTask(mSession); 583 } catch (RemoteException e) { 584 } 585 } 586 mContext.unbindService(this); 587 try { 588 mIWindowManager.removeWindowToken(mToken, DEFAULT_DISPLAY); 589 } catch (RemoteException e) { 590 Slog.w(TAG, "Failed removing window token", e); 591 } 592 mBound = false; 593 mService = null; 594 mSession = null; 595 mInteractor = null; 596 } 597 if (mFullyBound) { 598 mContext.unbindService(mFullConnection); 599 mFullyBound = false; 600 } 601 } 602 deliverNewSessionLocked(IVoiceInteractionSession session, IVoiceInteractor interactor)603 public boolean deliverNewSessionLocked(IVoiceInteractionSession session, 604 IVoiceInteractor interactor) { 605 mSession = session; 606 mInteractor = interactor; 607 if (mShown) { 608 try { 609 session.show(mShowArgs, mShowFlags, mShowCallback); 610 mShowArgs = null; 611 mShowFlags = 0; 612 } catch (RemoteException e) { 613 } 614 mAssistDataRequester.processPendingAssistData(); 615 if (!mPendingHandleAssistWithoutData.isEmpty()) { 616 doHandleAssistWithoutData(mPendingHandleAssistWithoutData); 617 mPendingHandleAssistWithoutData.clear(); 618 } 619 } 620 return true; 621 } 622 notifyPendingShowCallbacksShownLocked()623 private void notifyPendingShowCallbacksShownLocked() { 624 for (int i = 0; i < mPendingShowCallbacks.size(); i++) { 625 try { 626 mPendingShowCallbacks.get(i).onShown(); 627 } catch (RemoteException e) { 628 } 629 } 630 mPendingShowCallbacks.clear(); 631 } 632 notifyPendingShowCallbacksFailedLocked()633 private void notifyPendingShowCallbacksFailedLocked() { 634 for (int i = 0; i < mPendingShowCallbacks.size(); i++) { 635 try { 636 mPendingShowCallbacks.get(i).onFailed(); 637 } catch (RemoteException e) { 638 } 639 } 640 mPendingShowCallbacks.clear(); 641 } 642 startListeningVisibleActivityChangedLocked()643 void startListeningVisibleActivityChangedLocked() { 644 if (DEBUG) { 645 Slog.d(TAG, "startListeningVisibleActivityChangedLocked"); 646 } 647 648 if (!mShown || mCanceled || mSession == null) { 649 return; 650 } 651 652 mListeningVisibleActivity = true; 653 mVisibleActivityInfoForToken.clear(); 654 655 // It should only need to report which activities are visible 656 final ArrayMap<IBinder, VisibleActivityInfo> newVisibleActivityInfos = 657 getTopVisibleActivityInfosLocked(); 658 659 if (newVisibleActivityInfos == null || newVisibleActivityInfos.isEmpty()) { 660 return; 661 } 662 notifyVisibleActivitiesChangedLocked(newVisibleActivityInfos, 663 VisibleActivityInfo.TYPE_ACTIVITY_ADDED); 664 mVisibleActivityInfoForToken.putAll(newVisibleActivityInfos); 665 } 666 stopListeningVisibleActivityChangedLocked()667 void stopListeningVisibleActivityChangedLocked() { 668 if (DEBUG) { 669 Slog.d(TAG, "stopListeningVisibleActivityChangedLocked"); 670 } 671 mListeningVisibleActivity = false; 672 mVisibleActivityInfoForToken.clear(); 673 } 674 notifyActivityEventChangedLocked(@onNull IBinder activityToken, int type)675 void notifyActivityEventChangedLocked(@NonNull IBinder activityToken, int type) { 676 if (DEBUG) { 677 Slog.d(TAG, "notifyActivityEventChangedLocked activityToken=" + activityToken 678 + ", type=" + type); 679 } 680 if (!mListeningVisibleActivity) { 681 if (DEBUG) { 682 Slog.d(TAG, "not enable listening visible activity"); 683 } 684 return; 685 } 686 mScheduledExecutorService.execute(() -> { 687 synchronized (mLock) { 688 handleVisibleActivitiesLocked(activityToken, type); 689 } 690 }); 691 } 692 getTopVisibleActivityInfosLocked()693 private ArrayMap<IBinder, VisibleActivityInfo> getTopVisibleActivityInfosLocked() { 694 if (DEBUG) { 695 Slog.d(TAG, "getTopVisibleActivityInfosLocked"); 696 } 697 List<ActivityAssistInfo> allVisibleActivities = 698 LocalServices.getService(ActivityTaskManagerInternal.class) 699 .getTopVisibleActivities(); 700 if (DEBUG) { 701 Slog.d(TAG, "getTopVisibleActivityInfosLocked: allVisibleActivities=" 702 + allVisibleActivities); 703 } 704 if (allVisibleActivities.isEmpty()) { 705 Slog.w(TAG, "no visible activity"); 706 return null; 707 } 708 final int count = allVisibleActivities.size(); 709 final ArrayMap<IBinder, VisibleActivityInfo> visibleActivityInfoArrayMap = 710 new ArrayMap<>(count); 711 for (int i = 0; i < count; i++) { 712 ActivityAssistInfo info = allVisibleActivities.get(i); 713 if (DEBUG) { 714 Slog.d(TAG, "ActivityAssistInfo : activityToken=" + info.getActivityToken() 715 + ", assistToken=" + info.getAssistToken() 716 + ", taskId=" + info.getTaskId()); 717 } 718 visibleActivityInfoArrayMap.put(info.getActivityToken(), 719 new VisibleActivityInfo(info.getTaskId(), info.getAssistToken())); 720 } 721 return visibleActivityInfoArrayMap; 722 } 723 724 // TODO(b/242359988): Split this method up handleVisibleActivitiesLocked(@onNull IBinder activityToken, int type)725 private void handleVisibleActivitiesLocked(@NonNull IBinder activityToken, int type) { 726 if (DEBUG) { 727 Slog.d(TAG, "handleVisibleActivitiesLocked activityToken=" + activityToken 728 + ", type=" + type); 729 } 730 731 if (!mListeningVisibleActivity) { 732 if (DEBUG) { 733 Slog.d(TAG, "not enable listening visible activity"); 734 } 735 return; 736 } 737 if (!mShown || mCanceled || mSession == null) { 738 return; 739 } 740 741 // We use this local variable to determine to call onVisible or onInvisible. 742 boolean notifyOnVisible = false; 743 VisibleActivityInfo notifyVisibleActivityInfo = null; 744 745 if (type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_START 746 || type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_RESUME) { 747 // It seems that the onStart is unnecessary. But if we have it, the assistant 748 // application can request the directActions early. Even if we have the onStart, 749 // we still need the onResume because it is possible that the activity goes to 750 // onResume from onPause with invisible before the activity goes to onStop from 751 // onPause. 752 753 // Check if we have reported this activity as visible. If we have reported it as 754 // visible, do nothing. 755 if (mVisibleActivityInfoForToken.containsKey(activityToken)) { 756 return; 757 } 758 759 // Before reporting this activity as visible, we need to make sure the activity 760 // is really visible. 761 notifyVisibleActivityInfo = getVisibleActivityInfoFromTopVisibleActivity( 762 activityToken); 763 if (notifyVisibleActivityInfo == null) { 764 return; 765 } 766 notifyOnVisible = true; 767 } else if (type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_PAUSE) { 768 // For the onPause stage, the Activity is not necessarily invisible now, so we need 769 // to check its state. 770 // Note: After syncing with Activity owner, before the onPause is called, the 771 // visibility state has been updated. 772 notifyVisibleActivityInfo = getVisibleActivityInfoFromTopVisibleActivity( 773 activityToken); 774 if (notifyVisibleActivityInfo != null) { 775 return; 776 } 777 778 // Also make sure we previously reported this Activity as visible. 779 notifyVisibleActivityInfo = mVisibleActivityInfoForToken.get(activityToken); 780 if (notifyVisibleActivityInfo == null) { 781 return; 782 } 783 } else if (type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_STOP) { 784 // For the onStop stage, the activity is in invisible state. We only need to consider if 785 // we have reported this activity as visible. If we have reported it as visible, we 786 // need to report it as invisible. 787 // Why we still need onStop? Because it is possible that the activity is in a visible 788 // state during onPause stage, when the activity enters onStop from onPause, we may 789 // need to notify onInvisible. 790 // Note: After syncing with Activity owner, before the onStop is called, the 791 // visibility state has been updated. 792 notifyVisibleActivityInfo = mVisibleActivityInfoForToken.get(activityToken); 793 if (notifyVisibleActivityInfo == null) { 794 return; 795 } 796 } else { 797 Slog.w(TAG, "notifyActivityEventChangedLocked unexpected type=" + type); 798 return; 799 } 800 801 try { 802 mSession.notifyVisibleActivityInfoChanged(notifyVisibleActivityInfo, 803 notifyOnVisible ? VisibleActivityInfo.TYPE_ACTIVITY_ADDED 804 : VisibleActivityInfo.TYPE_ACTIVITY_REMOVED); 805 } catch (RemoteException e) { 806 if (DEBUG) { 807 Slog.w(TAG, "handleVisibleActivitiesLocked RemoteException : " + e); 808 } 809 } 810 811 if (notifyOnVisible) { 812 mVisibleActivityInfoForToken.put(activityToken, notifyVisibleActivityInfo); 813 } else { 814 mVisibleActivityInfoForToken.remove(activityToken); 815 } 816 } 817 notifyVisibleActivitiesChangedLocked( ArrayMap<IBinder, VisibleActivityInfo> visibleActivityInfos, int type)818 private void notifyVisibleActivitiesChangedLocked( 819 ArrayMap<IBinder, VisibleActivityInfo> visibleActivityInfos, int type) { 820 if (visibleActivityInfos == null || visibleActivityInfos.isEmpty()) { 821 return; 822 } 823 if (mSession == null) { 824 return; 825 } 826 try { 827 for (int i = 0; i < visibleActivityInfos.size(); i++) { 828 mSession.notifyVisibleActivityInfoChanged(visibleActivityInfos.valueAt(i), type); 829 } 830 } catch (RemoteException e) { 831 if (DEBUG) { 832 Slog.w(TAG, "notifyVisibleActivitiesChangedLocked RemoteException : " + e); 833 } 834 } 835 if (DEBUG) { 836 Slog.d(TAG, "notifyVisibleActivitiesChangedLocked type=" + type + ", count=" 837 + visibleActivityInfos.size()); 838 } 839 } 840 getVisibleActivityInfoFromTopVisibleActivity( @onNull IBinder activityToken)841 private VisibleActivityInfo getVisibleActivityInfoFromTopVisibleActivity( 842 @NonNull IBinder activityToken) { 843 final ArrayMap<IBinder, VisibleActivityInfo> visibleActivityInfos = 844 getTopVisibleActivityInfosLocked(); 845 if (visibleActivityInfos == null) { 846 return null; 847 } 848 return visibleActivityInfos.get(activityToken); 849 } 850 notifyActivityDestroyedLocked(@onNull IBinder activityToken)851 void notifyActivityDestroyedLocked(@NonNull IBinder activityToken) { 852 if (DEBUG) { 853 Slog.d(TAG, "notifyActivityDestroyedLocked activityToken=" + activityToken); 854 } 855 if (!mListeningVisibleActivity) { 856 if (DEBUG) { 857 Slog.d(TAG, "not enable listening visible activity"); 858 } 859 return; 860 } 861 mScheduledExecutorService.execute(() -> { 862 synchronized (mLock) { 863 if (!mListeningVisibleActivity) { 864 return; 865 } 866 if (!mShown || mCanceled || mSession == null) { 867 return; 868 } 869 870 VisibleActivityInfo visibleActivityInfo = mVisibleActivityInfoForToken.remove( 871 activityToken); 872 if (visibleActivityInfo != null) { 873 try { 874 mSession.notifyVisibleActivityInfoChanged(visibleActivityInfo, 875 VisibleActivityInfo.TYPE_ACTIVITY_REMOVED); 876 } catch (RemoteException e) { 877 if (DEBUG) { 878 Slog.w(TAG, "notifyVisibleActivityInfoChanged RemoteException : " + e); 879 } 880 } 881 } 882 } 883 }); 884 } 885 removeFromLowPowerStandbyAllowlist()886 private void removeFromLowPowerStandbyAllowlist() { 887 synchronized (mLock) { 888 if (mLowPowerStandbyAllowlisted) { 889 mFgHandler.removeCallbacks(mRemoveFromLowPowerStandbyAllowlistRunnable); 890 mLowPowerStandbyControllerInternal.removeFromAllowlist(mCallingUid, 891 PowerManager.LOW_POWER_STANDBY_ALLOWED_REASON_VOICE_INTERACTION); 892 mLowPowerStandbyAllowlisted = false; 893 } 894 } 895 } 896 897 @Override onServiceConnected(ComponentName name, IBinder service)898 public void onServiceConnected(ComponentName name, IBinder service) { 899 synchronized (mLock) { 900 mService = IVoiceInteractionSessionService.Stub.asInterface(service); 901 if (!mCanceled) { 902 try { 903 mService.newSession(mToken, mShowArgs, mShowFlags); 904 } catch (RemoteException e) { 905 Slog.w(TAG, "Failed adding window token", e); 906 } 907 } 908 } 909 } 910 911 @Override onServiceDisconnected(ComponentName name)912 public void onServiceDisconnected(ComponentName name) { 913 mCallback.sessionConnectionGone(this); 914 synchronized (mLock) { 915 mService = null; 916 } 917 } 918 dump(String prefix, PrintWriter pw)919 public void dump(String prefix, PrintWriter pw) { 920 pw.print(prefix); pw.print("mToken="); pw.println(mToken); 921 pw.print(prefix); pw.print("mShown="); pw.println(mShown); 922 pw.print(prefix); pw.print("mShowArgs="); pw.println(mShowArgs); 923 pw.print(prefix); pw.print("mShowFlags=0x"); pw.println(Integer.toHexString(mShowFlags)); 924 pw.print(prefix); pw.print("mBound="); pw.println(mBound); 925 if (mBound) { 926 pw.print(prefix); pw.print("mService="); pw.println(mService); 927 pw.print(prefix); pw.print("mSession="); pw.println(mSession); 928 pw.print(prefix); pw.print("mInteractor="); pw.println(mInteractor); 929 } 930 mAssistDataRequester.dump(prefix, pw); 931 } 932 933 private Runnable mShowAssistDisclosureRunnable = new Runnable() { 934 @Override 935 public void run() { 936 StatusBarManagerInternal statusBarInternal = LocalServices.getService( 937 StatusBarManagerInternal.class); 938 if (statusBarInternal != null) { 939 statusBarInternal.showAssistDisclosure(); 940 } 941 } 942 }; 943 } 944