1 /*
2  * Copyright (C) 2012 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.view;
18 
19 import static android.view.WindowManager.LayoutParams.TYPE_INPUT_METHOD;
20 import static android.view.accessibility.AccessibilityNodeInfo.ACTION_ARGUMENT_ACCESSIBLE_CLICKABLE_SPAN;
21 import static android.view.accessibility.AccessibilityNodeInfo.EXTRA_DATA_REQUESTED_KEY;
22 import static android.view.accessibility.AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_KEY;
23 
24 import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
25 
26 import android.accessibilityservice.AccessibilityService;
27 import android.annotation.NonNull;
28 import android.graphics.Matrix;
29 import android.graphics.Rect;
30 import android.graphics.RectF;
31 import android.graphics.Region;
32 import android.os.Bundle;
33 import android.os.Handler;
34 import android.os.Looper;
35 import android.os.Message;
36 import android.os.Parcelable;
37 import android.os.Process;
38 import android.os.RemoteException;
39 import android.os.SystemClock;
40 import android.text.style.AccessibilityClickableSpan;
41 import android.text.style.ClickableSpan;
42 import android.util.LongSparseArray;
43 import android.util.Slog;
44 import android.view.accessibility.AccessibilityInteractionClient;
45 import android.view.accessibility.AccessibilityManager;
46 import android.view.accessibility.AccessibilityNodeIdManager;
47 import android.view.accessibility.AccessibilityNodeInfo;
48 import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
49 import android.view.accessibility.AccessibilityNodeProvider;
50 import android.view.accessibility.AccessibilityRequestPreparer;
51 import android.view.accessibility.Flags;
52 import android.view.accessibility.IAccessibilityInteractionConnectionCallback;
53 import android.window.ScreenCapture;
54 
55 import com.android.internal.R;
56 import com.android.internal.annotations.GuardedBy;
57 import com.android.internal.annotations.VisibleForTesting;
58 import com.android.internal.os.SomeArgs;
59 import com.android.internal.util.function.pooled.PooledLambda;
60 
61 import java.util.ArrayDeque;
62 import java.util.ArrayList;
63 import java.util.HashSet;
64 import java.util.LinkedHashMap;
65 import java.util.LinkedList;
66 import java.util.List;
67 import java.util.Map;
68 import java.util.Queue;
69 import java.util.function.Predicate;
70 
71 /**
72  * Class for managing accessibility interactions initiated from the system
73  * and targeting the view hierarchy. A *ClientThread method is to be
74  * called from the interaction connection ViewAncestor gives the system to
75  * talk to it and a corresponding *UiThread method that is executed on the
76  * UI thread.
77  *
78  * @hide
79  */
80 @VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE)
81 public final class AccessibilityInteractionController {
82 
83     private static final String LOG_TAG = "AccessibilityInteractionController";
84 
85     // Debugging flag
86     private static final boolean ENFORCE_NODE_TREE_CONSISTENT = false;
87 
88     // Constants for readability
89     private static final boolean IGNORE_REQUEST_PREPARERS = true;
90     private static final boolean CONSIDER_REQUEST_PREPARERS = false;
91 
92     // If an app holds off accessibility for longer than this, the hold-off is canceled to prevent
93     // accessibility from hanging
94     private static final long REQUEST_PREPARER_TIMEOUT_MS = 500;
95 
96     // Callbacks should have the same configuration of the flags below to allow satisfying a pending
97     // node request on prefetch
98     private static final int FLAGS_AFFECTING_REPORTED_DATA = AccessibilityNodeInfo.FLAG_REPORT_MASK;
99 
100     private final ArrayList<AccessibilityNodeInfo> mTempAccessibilityNodeInfoList =
101         new ArrayList<AccessibilityNodeInfo>();
102 
103     private final Object mLock = new Object();
104 
105     private final PrivateHandler mHandler;
106 
107     private final ViewRootImpl mViewRootImpl;
108 
109     private final AccessibilityNodePrefetcher mPrefetcher;
110 
111     private final long mMyLooperThreadId;
112 
113     private final int mMyProcessId;
114 
115     private final AccessibilityManager mA11yManager;
116 
117     private final ArrayList<View> mTempArrayList = new ArrayList<View>();
118 
119     private final Rect mTempRect = new Rect();
120     private final RectF mTempRectF = new RectF();
121 
122     private AddNodeInfosForViewId mAddNodeInfosForViewId;
123 
124     @GuardedBy("mLock")
125     private ArrayList<Message> mPendingFindNodeByIdMessages;
126 
127     @GuardedBy("mLock")
128     private int mNumActiveRequestPreparers;
129     @GuardedBy("mLock")
130     private List<MessageHolder> mMessagesWaitingForRequestPreparer;
131     @GuardedBy("mLock")
132     private int mActiveRequestPreparerId;
133 
AccessibilityInteractionController(ViewRootImpl viewRootImpl)134     public AccessibilityInteractionController(ViewRootImpl viewRootImpl) {
135         Looper looper = viewRootImpl.mHandler.getLooper();
136         mMyLooperThreadId = looper.getThread().getId();
137         mMyProcessId = Process.myPid();
138         mHandler = new PrivateHandler(looper);
139         mViewRootImpl = viewRootImpl;
140         mPrefetcher = new AccessibilityNodePrefetcher();
141         mA11yManager = mViewRootImpl.mContext.getSystemService(AccessibilityManager.class);
142         mPendingFindNodeByIdMessages = new ArrayList<>();
143     }
144 
scheduleMessage(Message message, int interrogatingPid, long interrogatingTid, boolean ignoreRequestPreparers)145     private void scheduleMessage(Message message, int interrogatingPid, long interrogatingTid,
146             boolean ignoreRequestPreparers) {
147         if (ignoreRequestPreparers
148                 || !holdOffMessageIfNeeded(message, interrogatingPid, interrogatingTid)) {
149             // If the interrogation is performed by the same thread as the main UI
150             // thread in this process, set the message as a static reference so
151             // after this call completes the same thread but in the interrogating
152             // client can handle the message to generate the result.
153             if (interrogatingPid == mMyProcessId && interrogatingTid == mMyLooperThreadId
154                     && mHandler.hasAccessibilityCallback(message)) {
155                 AccessibilityInteractionClient.getInstanceForThread(
156                         interrogatingTid).setSameThreadMessage(message);
157             } else {
158                 // For messages without callback of interrogating client, just handle the
159                 // message immediately if this is UI thread.
160                 if (!mHandler.hasAccessibilityCallback(message)
161                         && Thread.currentThread().getId() == mMyLooperThreadId) {
162                     mHandler.handleMessage(message);
163                 } else {
164                     mHandler.sendMessage(message);
165                 }
166             }
167         }
168     }
169 
isShown(View view)170     private boolean isShown(View view) {
171         return (view != null) && (view.getWindowVisibility() == View.VISIBLE && view.isShown());
172     }
173 
isVisibleToAccessibilityService(View view)174     private boolean isVisibleToAccessibilityService(View view) {
175         return view != null && (mA11yManager.isRequestFromAccessibilityTool()
176                 || !view.isAccessibilityDataSensitive());
177     }
178 
findAccessibilityNodeInfoByAccessibilityIdClientThread( long accessibilityNodeId, Region interactiveRegion, int interactionId, IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid, long interrogatingTid, MagnificationSpec spec, float[] matrixValues, Bundle arguments)179     public void findAccessibilityNodeInfoByAccessibilityIdClientThread(
180             long accessibilityNodeId, Region interactiveRegion, int interactionId,
181             IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
182             long interrogatingTid, MagnificationSpec spec, float[] matrixValues,
183             Bundle arguments) {
184         final Message message = mHandler.obtainMessage();
185         message.what = PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID;
186         message.arg1 = flags;
187 
188         final SomeArgs args = SomeArgs.obtain();
189         args.argi1 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);
190         args.argi2 = AccessibilityNodeInfo.getVirtualDescendantId(accessibilityNodeId);
191         args.argi3 = interactionId;
192         args.arg1 = callback;
193         args.arg2 = spec;
194         args.arg3 = interactiveRegion;
195         args.arg4 = arguments;
196         args.arg5 = matrixValues;
197         message.obj = args;
198 
199         synchronized (mLock) {
200             mPendingFindNodeByIdMessages.add(message);
201             scheduleMessage(message, interrogatingPid, interrogatingTid,
202                     CONSIDER_REQUEST_PREPARERS);
203         }
204     }
205 
206     /**
207      * Check if this message needs to be held off while the app prepares to meet either this
208      * request, or a request ahead of it.
209      *
210      * @param originalMessage The message to be processed
211      * @param callingPid The calling process id
212      * @param callingTid The calling thread id
213      *
214      * @return {@code true} if the message is held off and will be processed later, {@code false} if
215      *         the message should be posted.
216      */
holdOffMessageIfNeeded( Message originalMessage, int callingPid, long callingTid)217     private boolean holdOffMessageIfNeeded(
218             Message originalMessage, int callingPid, long callingTid) {
219         synchronized (mLock) {
220             // If a request is already pending, queue this request for when it's finished
221             if (mNumActiveRequestPreparers != 0) {
222                 queueMessageToHandleOncePrepared(originalMessage, callingPid, callingTid);
223                 return true;
224             }
225 
226             // Currently the only message that can hold things off is findByA11yId with extra data.
227             if (originalMessage.what
228                     != PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID) {
229                 return false;
230             }
231             SomeArgs originalMessageArgs = (SomeArgs) originalMessage.obj;
232             Bundle requestArguments = (Bundle) originalMessageArgs.arg4;
233             if (requestArguments == null) {
234                 return false;
235             }
236 
237             // If nothing it registered for this view, nothing to do
238             int accessibilityViewId = originalMessageArgs.argi1;
239             final List<AccessibilityRequestPreparer> preparers =
240                     mA11yManager.getRequestPreparersForAccessibilityId(accessibilityViewId);
241             if (preparers == null) {
242                 return false;
243             }
244 
245             // If the bundle doesn't request the extra data, nothing to do
246             final String extraDataKey = requestArguments.getString(EXTRA_DATA_REQUESTED_KEY);
247             if (extraDataKey == null) {
248                 return false;
249             }
250 
251             // Send the request to the AccessibilityRequestPreparers on the UI thread
252             mNumActiveRequestPreparers = preparers.size();
253             for (int i = 0; i < preparers.size(); i++) {
254                 final Message requestPreparerMessage = mHandler.obtainMessage(
255                         PrivateHandler.MSG_PREPARE_FOR_EXTRA_DATA_REQUEST);
256                 final SomeArgs requestPreparerArgs = SomeArgs.obtain();
257                 // virtualDescendentId
258                 requestPreparerArgs.argi1 =
259                         (originalMessageArgs.argi2 == AccessibilityNodeInfo.UNDEFINED_ITEM_ID)
260                         ? AccessibilityNodeProvider.HOST_VIEW_ID : originalMessageArgs.argi2;
261                 requestPreparerArgs.arg1 = preparers.get(i);
262                 requestPreparerArgs.arg2 = extraDataKey;
263                 requestPreparerArgs.arg3 = requestArguments;
264                 Message preparationFinishedMessage = mHandler.obtainMessage(
265                         PrivateHandler.MSG_APP_PREPARATION_FINISHED);
266                 preparationFinishedMessage.arg1 = ++mActiveRequestPreparerId;
267                 requestPreparerArgs.arg4 = preparationFinishedMessage;
268 
269                 requestPreparerMessage.obj = requestPreparerArgs;
270                 scheduleMessage(requestPreparerMessage, callingPid, callingTid,
271                         IGNORE_REQUEST_PREPARERS);
272                 mHandler.obtainMessage(PrivateHandler.MSG_APP_PREPARATION_TIMEOUT);
273                 mHandler.sendEmptyMessageDelayed(PrivateHandler.MSG_APP_PREPARATION_TIMEOUT,
274                         REQUEST_PREPARER_TIMEOUT_MS);
275             }
276 
277             // Set the initial request aside
278             queueMessageToHandleOncePrepared(originalMessage, callingPid, callingTid);
279             return true;
280         }
281     }
282 
prepareForExtraDataRequestUiThread(Message message)283     private void prepareForExtraDataRequestUiThread(Message message) {
284         SomeArgs args = (SomeArgs) message.obj;
285         final int virtualDescendantId = args.argi1;
286         final AccessibilityRequestPreparer preparer = (AccessibilityRequestPreparer) args.arg1;
287         final String extraDataKey = (String) args.arg2;
288         final Bundle requestArguments = (Bundle) args.arg3;
289         final Message preparationFinishedMessage = (Message) args.arg4;
290 
291         preparer.onPrepareExtraData(virtualDescendantId, extraDataKey,
292                 requestArguments, preparationFinishedMessage);
293     }
294 
queueMessageToHandleOncePrepared(Message message, int interrogatingPid, long interrogatingTid)295     private void queueMessageToHandleOncePrepared(Message message, int interrogatingPid,
296             long interrogatingTid) {
297         if (mMessagesWaitingForRequestPreparer == null) {
298             mMessagesWaitingForRequestPreparer = new ArrayList<>(1);
299         }
300         MessageHolder messageHolder =
301                 new MessageHolder(message, interrogatingPid, interrogatingTid);
302         mMessagesWaitingForRequestPreparer.add(messageHolder);
303     }
304 
requestPreparerDoneUiThread(Message message)305     private void requestPreparerDoneUiThread(Message message) {
306         synchronized (mLock) {
307             if (message.arg1 != mActiveRequestPreparerId) {
308                 Slog.e(LOG_TAG, "Surprising AccessibilityRequestPreparer callback (likely late)");
309                 return;
310             }
311             mNumActiveRequestPreparers--;
312             if (mNumActiveRequestPreparers <= 0) {
313                 mHandler.removeMessages(PrivateHandler.MSG_APP_PREPARATION_TIMEOUT);
314                 scheduleAllMessagesWaitingForRequestPreparerLocked();
315             }
316         }
317     }
318 
requestPreparerTimeoutUiThread()319     private void requestPreparerTimeoutUiThread() {
320         synchronized (mLock) {
321             Slog.e(LOG_TAG, "AccessibilityRequestPreparer timed out");
322             scheduleAllMessagesWaitingForRequestPreparerLocked();
323         }
324     }
325 
326     @GuardedBy("mLock")
scheduleAllMessagesWaitingForRequestPreparerLocked()327     private void scheduleAllMessagesWaitingForRequestPreparerLocked() {
328         int numMessages = mMessagesWaitingForRequestPreparer.size();
329         for (int i = 0; i < numMessages; i++) {
330             MessageHolder request = mMessagesWaitingForRequestPreparer.get(i);
331             scheduleMessage(request.mMessage, request.mInterrogatingPid,
332                     request.mInterrogatingTid,
333                     (i == 0) /* the app is ready for the first request */);
334         }
335         mMessagesWaitingForRequestPreparer.clear();
336         mNumActiveRequestPreparers = 0; // Just to be safe - should be unnecessary
337         mActiveRequestPreparerId = -1;
338     }
339 
findAccessibilityNodeInfoByAccessibilityIdUiThread(Message message)340     private void findAccessibilityNodeInfoByAccessibilityIdUiThread(Message message) {
341         synchronized (mLock) {
342             mPendingFindNodeByIdMessages.remove(message);
343         }
344         final int flags = message.arg1;
345 
346         SomeArgs args = (SomeArgs) message.obj;
347         final int accessibilityViewId = args.argi1;
348         final int virtualDescendantId = args.argi2;
349         final int interactionId = args.argi3;
350         final IAccessibilityInteractionConnectionCallback callback =
351             (IAccessibilityInteractionConnectionCallback) args.arg1;
352         final MagnificationSpec spec = (MagnificationSpec) args.arg2;
353         final Region interactiveRegion = (Region) args.arg3;
354         final Bundle arguments = (Bundle) args.arg4;
355         final float[] matrixValues = (float[]) args.arg5;
356 
357         args.recycle();
358 
359         View requestedView = null;
360         AccessibilityNodeInfo requestedNode = null;
361         boolean interruptPrefetch =
362                 ((flags & AccessibilityNodeInfo.FLAG_PREFETCH_UNINTERRUPTIBLE) == 0);
363 
364         ArrayList<AccessibilityNodeInfo> infos = mTempAccessibilityNodeInfoList;
365         infos.clear();
366         try {
367             if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null) {
368                 return;
369             }
370             setAccessibilityFetchFlags(flags);
371             requestedView = findViewByAccessibilityId(accessibilityViewId);
372             if (requestedView != null && isShown(requestedView)) {
373                 requestedNode = populateAccessibilityNodeInfoForView(
374                         requestedView, arguments, virtualDescendantId);
375                 mPrefetcher.mInterruptPrefetch = interruptPrefetch;
376                 mPrefetcher.mFetchFlags = flags & AccessibilityNodeInfo.FLAG_PREFETCH_MASK;
377 
378                 if (!interruptPrefetch) {
379                     infos.add(requestedNode);
380                     mPrefetcher.prefetchAccessibilityNodeInfos(requestedView,
381                             requestedNode == null ? null : new AccessibilityNodeInfo(requestedNode),
382                             infos);
383                     resetAccessibilityFetchFlags();
384                 }
385             }
386         } finally {
387             if (!interruptPrefetch) {
388                 // Return found node and prefetched nodes in one IPC.
389                 updateInfosForViewportAndReturnFindNodeResult(infos, callback, interactionId, spec,
390                         matrixValues, interactiveRegion);
391 
392                 final SatisfiedFindAccessibilityNodeByAccessibilityIdRequest satisfiedRequest =
393                         getSatisfiedRequestInPrefetch(requestedNode == null ? null : requestedNode,
394                                infos, flags);
395                 if (satisfiedRequest != null) {
396                     returnFindNodeResult(satisfiedRequest);
397                 }
398                 return;
399             } else {
400                 // Return found node.
401                 updateInfoForViewportAndReturnFindNodeResult(
402                         requestedNode == null ? null : new AccessibilityNodeInfo(requestedNode),
403                         callback, interactionId, spec, matrixValues, interactiveRegion);
404             }
405         }
406         mPrefetcher.prefetchAccessibilityNodeInfos(requestedView,
407                 requestedNode == null ? null : new AccessibilityNodeInfo(requestedNode), infos);
408         resetAccessibilityFetchFlags();
409         updateInfosForViewPort(infos, spec, matrixValues, interactiveRegion);
410         final SatisfiedFindAccessibilityNodeByAccessibilityIdRequest satisfiedRequest =
411                 getSatisfiedRequestInPrefetch(requestedNode == null ? null : requestedNode, infos,
412                         flags);
413 
414         // Return prefetch result separately.
415         returnPrefetchResult(interactionId, infos, callback);
416 
417         if (satisfiedRequest != null) {
418             returnFindNodeResult(satisfiedRequest);
419         }
420     }
421 
populateAccessibilityNodeInfoForView( View view, Bundle arguments, int virtualViewId)422     private AccessibilityNodeInfo populateAccessibilityNodeInfoForView(
423             View view, Bundle arguments, int virtualViewId) {
424         AccessibilityNodeProvider provider = view.getAccessibilityNodeProvider();
425         // Determine if we'll be populating extra data
426         final String extraDataRequested = (arguments == null) ? null
427                 : arguments.getString(EXTRA_DATA_REQUESTED_KEY);
428         AccessibilityNodeInfo root = null;
429         if (provider == null) {
430             root = view.createAccessibilityNodeInfo();
431             if (root != null) {
432                 if (extraDataRequested != null) {
433                     view.addExtraDataToAccessibilityNodeInfo(root, extraDataRequested, arguments);
434                 }
435             }
436         } else {
437             root = provider.createAccessibilityNodeInfo(virtualViewId);
438             if (root != null) {
439                 if (extraDataRequested != null) {
440                     provider.addExtraDataToAccessibilityNodeInfo(
441                             virtualViewId, root, extraDataRequested, arguments);
442                 }
443             }
444         }
445         return root;
446     }
447 
findAccessibilityNodeInfosByViewIdClientThread(long accessibilityNodeId, String viewId, Region interactiveRegion, int interactionId, IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid, long interrogatingTid, MagnificationSpec spec, float[] matrixValues)448     public void findAccessibilityNodeInfosByViewIdClientThread(long accessibilityNodeId,
449             String viewId, Region interactiveRegion, int interactionId,
450             IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
451             long interrogatingTid, MagnificationSpec spec, float[] matrixValues) {
452         Message message = mHandler.obtainMessage();
453         message.what = PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID;
454         message.arg1 = flags;
455         message.arg2 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);
456 
457         SomeArgs args = SomeArgs.obtain();
458         args.argi1 = interactionId;
459         args.arg1 = callback;
460         args.arg2 = spec;
461         args.arg3 = viewId;
462         args.arg4 = interactiveRegion;
463         args.arg5 = matrixValues;
464         message.obj = args;
465 
466         scheduleMessage(message, interrogatingPid, interrogatingTid, CONSIDER_REQUEST_PREPARERS);
467     }
468 
findAccessibilityNodeInfosByViewIdUiThread(Message message)469     private void findAccessibilityNodeInfosByViewIdUiThread(Message message) {
470         final int flags = message.arg1;
471         final int accessibilityViewId = message.arg2;
472 
473         SomeArgs args = (SomeArgs) message.obj;
474         final int interactionId = args.argi1;
475         final IAccessibilityInteractionConnectionCallback callback =
476             (IAccessibilityInteractionConnectionCallback) args.arg1;
477         final MagnificationSpec spec = (MagnificationSpec) args.arg2;
478         final String viewId = (String) args.arg3;
479         final Region interactiveRegion = (Region) args.arg4;
480         final float[] matrixValues = (float[]) args.arg5;
481         args.recycle();
482 
483         final List<AccessibilityNodeInfo> infos = mTempAccessibilityNodeInfoList;
484         infos.clear();
485         try {
486             if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null
487                     || viewId == null) {
488                 return;
489             }
490             setAccessibilityFetchFlags(flags);
491             final View root = findViewByAccessibilityId(accessibilityViewId);
492             if (root != null) {
493                 final int resolvedViewId = root.getContext().getResources()
494                         .getIdentifier(viewId, null, null);
495                 if (resolvedViewId <= 0) {
496                     return;
497                 }
498                 if (mAddNodeInfosForViewId == null) {
499                     mAddNodeInfosForViewId = new AddNodeInfosForViewId();
500                 }
501                 mAddNodeInfosForViewId.init(resolvedViewId, infos);
502                 root.findViewByPredicate(mAddNodeInfosForViewId);
503                 mAddNodeInfosForViewId.reset();
504             }
505         } finally {
506             resetAccessibilityFetchFlags();
507             updateInfosForViewportAndReturnFindNodeResult(
508                     infos, callback, interactionId, spec, matrixValues, interactiveRegion);
509         }
510     }
511 
findAccessibilityNodeInfosByTextClientThread(long accessibilityNodeId, String text, Region interactiveRegion, int interactionId, IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid, long interrogatingTid, MagnificationSpec spec, float[] matrixValues)512     public void findAccessibilityNodeInfosByTextClientThread(long accessibilityNodeId,
513             String text, Region interactiveRegion, int interactionId,
514             IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
515             long interrogatingTid, MagnificationSpec spec, float[] matrixValues) {
516         Message message = mHandler.obtainMessage();
517         message.what = PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT;
518         message.arg1 = flags;
519 
520         SomeArgs args = SomeArgs.obtain();
521         args.arg1 = text;
522         args.arg2 = callback;
523         args.arg3 = spec;
524         args.argi1 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);
525         args.argi2 = AccessibilityNodeInfo.getVirtualDescendantId(accessibilityNodeId);
526         args.argi3 = interactionId;
527         args.arg4 = interactiveRegion;
528         args.arg5 = matrixValues;
529         message.obj = args;
530 
531         scheduleMessage(message, interrogatingPid, interrogatingTid, CONSIDER_REQUEST_PREPARERS);
532     }
533 
findAccessibilityNodeInfosByTextUiThread(Message message)534     private void findAccessibilityNodeInfosByTextUiThread(Message message) {
535         final int flags = message.arg1;
536 
537         SomeArgs args = (SomeArgs) message.obj;
538         final String text = (String) args.arg1;
539         final IAccessibilityInteractionConnectionCallback callback =
540             (IAccessibilityInteractionConnectionCallback) args.arg2;
541         final MagnificationSpec spec = (MagnificationSpec) args.arg3;
542         final int accessibilityViewId = args.argi1;
543         final int virtualDescendantId = args.argi2;
544         final int interactionId = args.argi3;
545         final Region interactiveRegion = (Region) args.arg4;
546         final float[] matrixValues = (float[]) args.arg5;
547         args.recycle();
548 
549         List<AccessibilityNodeInfo> infos = null;
550         try {
551             if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null) {
552                 return;
553             }
554             setAccessibilityFetchFlags(flags);
555             final View root = findViewByAccessibilityId(accessibilityViewId);
556             if (root != null && isShown(root)) {
557                 AccessibilityNodeProvider provider = root.getAccessibilityNodeProvider();
558                 if (provider != null) {
559                     infos = provider.findAccessibilityNodeInfosByText(text,
560                             virtualDescendantId);
561                 } else if (virtualDescendantId == AccessibilityNodeProvider.HOST_VIEW_ID) {
562                     ArrayList<View> foundViews = mTempArrayList;
563                     foundViews.clear();
564                     root.findViewsWithText(foundViews, text, View.FIND_VIEWS_WITH_TEXT
565                             | View.FIND_VIEWS_WITH_CONTENT_DESCRIPTION
566                             | View.FIND_VIEWS_WITH_ACCESSIBILITY_NODE_PROVIDERS);
567                     if (!foundViews.isEmpty()) {
568                         infos = mTempAccessibilityNodeInfoList;
569                         infos.clear();
570                         final int viewCount = foundViews.size();
571                         for (int i = 0; i < viewCount; i++) {
572                             View foundView = foundViews.get(i);
573                             if (isShown(foundView) && isVisibleToAccessibilityService(foundView)) {
574                                 provider = foundView.getAccessibilityNodeProvider();
575                                 if (provider != null) {
576                                     List<AccessibilityNodeInfo> infosFromProvider =
577                                         provider.findAccessibilityNodeInfosByText(text,
578                                                 AccessibilityNodeProvider.HOST_VIEW_ID);
579                                     if (infosFromProvider != null) {
580                                         infos.addAll(infosFromProvider);
581                                     }
582                                 } else  {
583                                     infos.add(foundView.createAccessibilityNodeInfo());
584                                 }
585                             }
586                         }
587                     }
588                 }
589             }
590         } finally {
591             resetAccessibilityFetchFlags();
592             updateInfosForViewportAndReturnFindNodeResult(
593                     infos, callback, interactionId, spec, matrixValues, interactiveRegion);
594         }
595     }
596 
597     /**
598      * Take a screenshot using {@link ScreenCapture} of this {@link ViewRootImpl}'s {@link
599      * SurfaceControl}.
600      */
takeScreenshotOfWindowClientThread(int interactionId, ScreenCapture.ScreenCaptureListener listener, IAccessibilityInteractionConnectionCallback callback)601     public void takeScreenshotOfWindowClientThread(int interactionId,
602             ScreenCapture.ScreenCaptureListener listener,
603             IAccessibilityInteractionConnectionCallback callback) {
604         Message message = PooledLambda.obtainMessage(
605                 AccessibilityInteractionController::takeScreenshotOfWindowUiThread,
606                 this, interactionId, listener, callback);
607 
608         // Screenshot results are returned to the service asynchronously, so the same-thread
609         // message wait logic from #scheduleMessage() is not needed.
610         mHandler.sendMessage(message);
611     }
612 
takeScreenshotOfWindowUiThread(int interactionId, ScreenCapture.ScreenCaptureListener listener, IAccessibilityInteractionConnectionCallback callback)613     private void takeScreenshotOfWindowUiThread(int interactionId,
614             ScreenCapture.ScreenCaptureListener listener,
615             IAccessibilityInteractionConnectionCallback callback) {
616         try {
617             if ((mViewRootImpl.getWindowFlags() & WindowManager.LayoutParams.FLAG_SECURE) != 0) {
618                 callback.sendTakeScreenshotOfWindowError(
619                         AccessibilityService.ERROR_TAKE_SCREENSHOT_SECURE_WINDOW, interactionId);
620                 return;
621             }
622             final ScreenCapture.LayerCaptureArgs captureArgs =
623                     new ScreenCapture.LayerCaptureArgs.Builder(mViewRootImpl.getSurfaceControl())
624                             .setChildrenOnly(false).setUid(Process.myUid()).build();
625             if (ScreenCapture.captureLayers(captureArgs, listener) != 0) {
626                 callback.sendTakeScreenshotOfWindowError(
627                         AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR, interactionId);
628             }
629         } catch (RemoteException re) {
630             /* ignore - the other side will time out */
631         }
632     }
633 
findFocusClientThread(long accessibilityNodeId, int focusType, Region interactiveRegion, int interactionId, IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid, long interrogatingTid, MagnificationSpec spec, float[] matrixValues)634     public void findFocusClientThread(long accessibilityNodeId, int focusType,
635             Region interactiveRegion, int interactionId,
636             IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
637             long interrogatingTid, MagnificationSpec spec, float[] matrixValues) {
638         Message message = mHandler.obtainMessage();
639         message.what = PrivateHandler.MSG_FIND_FOCUS;
640         message.arg1 = flags;
641         message.arg2 = focusType;
642 
643         SomeArgs args = SomeArgs.obtain();
644         args.argi1 = interactionId;
645         args.argi2 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);
646         args.argi3 = AccessibilityNodeInfo.getVirtualDescendantId(accessibilityNodeId);
647         args.arg1 = callback;
648         args.arg2 = spec;
649         args.arg3 = interactiveRegion;
650         args.arg4 = matrixValues;
651         message.obj = args;
652 
653         scheduleMessage(message, interrogatingPid, interrogatingTid, CONSIDER_REQUEST_PREPARERS);
654     }
655 
findFocusUiThread(Message message)656     private void findFocusUiThread(Message message) {
657         final int flags = message.arg1;
658         final int focusType = message.arg2;
659 
660         SomeArgs args = (SomeArgs) message.obj;
661         final int interactionId = args.argi1;
662         final int accessibilityViewId = args.argi2;
663         final int virtualDescendantId = args.argi3;
664         final IAccessibilityInteractionConnectionCallback callback =
665             (IAccessibilityInteractionConnectionCallback) args.arg1;
666         final MagnificationSpec spec = (MagnificationSpec) args.arg2;
667         final Region interactiveRegion = (Region) args.arg3;
668         final float[] matrixValues = (float[]) args.arg4;
669         args.recycle();
670 
671         AccessibilityNodeInfo focused = null;
672         try {
673             if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null) {
674                 return;
675             }
676             setAccessibilityFetchFlags(flags);
677             final View root = findViewByAccessibilityId(accessibilityViewId);
678             if (root != null && isShown(root)) {
679                 switch (focusType) {
680                     case AccessibilityNodeInfo.FOCUS_ACCESSIBILITY: {
681                         View host = mViewRootImpl.mAccessibilityFocusedHost;
682                         // If there is no accessibility focus host or it is not a descendant
683                         // of the root from which to start the search, then the search failed.
684                         if (host == null || !ViewRootImpl.isViewDescendantOf(host, root)) {
685                             break;
686                         }
687                         // The focused view not shown, we failed.
688                         if (!isShown(host)) {
689                             break;
690                         }
691                         if (!isVisibleToAccessibilityService(host)) {
692                             break;
693                         }
694                         // If the host has a provider ask this provider to search for the
695                         // focus instead fetching all provider nodes to do the search here.
696                         AccessibilityNodeProvider provider = host.getAccessibilityNodeProvider();
697                         if (provider != null) {
698                             final AccessibilityNodeInfo focusNode =
699                                     mViewRootImpl.mAccessibilityFocusedVirtualView;
700                             if (focusNode != null) {
701                                 final int virtualNodeId = AccessibilityNodeInfo
702                                         .getVirtualDescendantId(focusNode.getSourceNodeId());
703                                 focused = provider.createAccessibilityNodeInfo(virtualNodeId);
704                             }
705                         } else if (virtualDescendantId == AccessibilityNodeProvider.HOST_VIEW_ID) {
706                             focused = host.createAccessibilityNodeInfo();
707                         }
708                     } break;
709                     case AccessibilityNodeInfo.FOCUS_INPUT: {
710                         View target = root.findFocus();
711                         if (!isShown(target)) {
712                             break;
713                         }
714                         if (!isVisibleToAccessibilityService(target)) {
715                             break;
716                         }
717                         AccessibilityNodeProvider provider = target.getAccessibilityNodeProvider();
718                         if (provider != null) {
719                             focused = provider.findFocus(focusType);
720                         }
721                         if (focused == null) {
722                             focused = target.createAccessibilityNodeInfo();
723                         }
724                     } break;
725                     default:
726                         throw new IllegalArgumentException("Unknown focus type: " + focusType);
727                 }
728             }
729         } finally {
730             resetAccessibilityFetchFlags();
731             updateInfoForViewportAndReturnFindNodeResult(
732                     focused, callback, interactionId, spec, matrixValues, interactiveRegion);
733         }
734     }
735 
focusSearchClientThread(long accessibilityNodeId, int direction, Region interactiveRegion, int interactionId, IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid, long interrogatingTid, MagnificationSpec spec, float[] matrixValues)736     public void focusSearchClientThread(long accessibilityNodeId, int direction,
737             Region interactiveRegion, int interactionId,
738             IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
739             long interrogatingTid, MagnificationSpec spec, float[] matrixValues) {
740         Message message = mHandler.obtainMessage();
741         message.what = PrivateHandler.MSG_FOCUS_SEARCH;
742         message.arg1 = flags;
743         message.arg2 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);
744 
745         SomeArgs args = SomeArgs.obtain();
746         args.argi2 = direction;
747         args.argi3 = interactionId;
748         args.arg1 = callback;
749         args.arg2 = spec;
750         args.arg3 = interactiveRegion;
751         args.arg4 = matrixValues;
752 
753         message.obj = args;
754 
755         scheduleMessage(message, interrogatingPid, interrogatingTid, CONSIDER_REQUEST_PREPARERS);
756     }
757 
focusSearchUiThread(Message message)758     private void focusSearchUiThread(Message message) {
759         final int flags = message.arg1;
760         final int accessibilityViewId = message.arg2;
761 
762         SomeArgs args = (SomeArgs) message.obj;
763         final int direction = args.argi2;
764         final int interactionId = args.argi3;
765         final IAccessibilityInteractionConnectionCallback callback =
766             (IAccessibilityInteractionConnectionCallback) args.arg1;
767         final MagnificationSpec spec = (MagnificationSpec) args.arg2;
768         final Region interactiveRegion = (Region) args.arg3;
769         final float[] matrixValues = (float[]) args.arg4;
770         args.recycle();
771 
772         AccessibilityNodeInfo next = null;
773         try {
774             if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null) {
775                 return;
776             }
777             setAccessibilityFetchFlags(flags);
778             final View root = findViewByAccessibilityId(accessibilityViewId);
779             if (root != null && isShown(root)) {
780                 View nextView = root.focusSearch(direction);
781                 if (nextView != null) {
782                     next = nextView.createAccessibilityNodeInfo();
783                 }
784             }
785         } finally {
786             resetAccessibilityFetchFlags();
787             updateInfoForViewportAndReturnFindNodeResult(
788                     next, callback, interactionId, spec, matrixValues, interactiveRegion);
789         }
790     }
791 
performAccessibilityActionClientThread(long accessibilityNodeId, int action, Bundle arguments, int interactionId, IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid, long interrogatingTid)792     public void performAccessibilityActionClientThread(long accessibilityNodeId, int action,
793             Bundle arguments, int interactionId,
794             IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
795             long interrogatingTid) {
796         Message message = mHandler.obtainMessage();
797         message.what = PrivateHandler.MSG_PERFORM_ACCESSIBILITY_ACTION;
798         message.arg1 = flags;
799         message.arg2 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);
800 
801         SomeArgs args = SomeArgs.obtain();
802         args.argi1 = AccessibilityNodeInfo.getVirtualDescendantId(accessibilityNodeId);
803         args.argi2 = action;
804         args.argi3 = interactionId;
805         args.arg1 = callback;
806         args.arg2 = arguments;
807 
808         message.obj = args;
809 
810         scheduleMessage(message, interrogatingPid, interrogatingTid, CONSIDER_REQUEST_PREPARERS);
811     }
812 
performAccessibilityActionUiThread(Message message)813     private void performAccessibilityActionUiThread(Message message) {
814         final int flags = message.arg1;
815         final int accessibilityViewId = message.arg2;
816 
817         SomeArgs args = (SomeArgs) message.obj;
818         final int virtualDescendantId = args.argi1;
819         final int action = args.argi2;
820         final int interactionId = args.argi3;
821         final IAccessibilityInteractionConnectionCallback callback =
822             (IAccessibilityInteractionConnectionCallback) args.arg1;
823         Bundle arguments = (Bundle) args.arg2;
824 
825         args.recycle();
826 
827         boolean succeeded = false;
828         try {
829             if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null ||
830                     mViewRootImpl.mStopped || mViewRootImpl.mPausedForTransition) {
831                 return;
832             }
833             setAccessibilityFetchFlags(flags);
834             final View target = findViewByAccessibilityId(accessibilityViewId);
835             if (target != null && isShown(target) && isVisibleToAccessibilityService(target)) {
836                 mA11yManager.notifyPerformingAction(action);
837                 if (action == R.id.accessibilityActionClickOnClickableSpan) {
838                     // Handle this hidden action separately
839                     succeeded = handleClickableSpanActionUiThread(
840                             target, virtualDescendantId, arguments);
841                 } else {
842                     AccessibilityNodeProvider provider = target.getAccessibilityNodeProvider();
843                     if (provider != null) {
844                         succeeded = provider.performAction(virtualDescendantId, action,
845                                 arguments);
846                     } else if (virtualDescendantId == AccessibilityNodeProvider.HOST_VIEW_ID) {
847                         succeeded = target.performAccessibilityAction(action, arguments);
848                     }
849                 }
850                 mA11yManager.notifyPerformingAction(0);
851             }
852         } finally {
853             try {
854                 resetAccessibilityFetchFlags();
855                 callback.setPerformAccessibilityActionResult(succeeded, interactionId);
856             } catch (RemoteException re) {
857                 /* ignore - the other side will time out */
858             }
859         }
860     }
861 
862     /**
863      * Finds the accessibility focused node in the root, and clears the accessibility focus.
864      */
clearAccessibilityFocusClientThread()865     public void clearAccessibilityFocusClientThread() {
866         final Message message = mHandler.obtainMessage();
867         message.what = PrivateHandler.MSG_CLEAR_ACCESSIBILITY_FOCUS;
868 
869         // Don't care about pid and tid because there's no interrogating client for this message.
870         scheduleMessage(message, 0, 0, CONSIDER_REQUEST_PREPARERS);
871     }
872 
clearAccessibilityFocusUiThread()873     private void clearAccessibilityFocusUiThread() {
874         if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null) {
875             return;
876         }
877         try {
878             // Clearing focus does not expose sensitive data, so set fetch flags to ensure that the
879             // root view is always returned if present.
880             setAccessibilityFetchFlags(
881                     AccessibilityNodeInfo.FLAG_SERVICE_REQUESTS_INCLUDE_NOT_IMPORTANT_VIEWS
882                             | AccessibilityNodeInfo.FLAG_SERVICE_IS_ACCESSIBILITY_TOOL);
883             final View root = getRootView();
884             if (root != null && isShown(root)) {
885                 final View host = mViewRootImpl.mAccessibilityFocusedHost;
886                 // If there is no accessibility focus host or it is not a descendant
887                 // of the root from which to start the search, then the search failed.
888                 if (host == null || !ViewRootImpl.isViewDescendantOf(host, root)) {
889                     return;
890                 }
891                 final AccessibilityNodeProvider provider = host.getAccessibilityNodeProvider();
892                 final AccessibilityNodeInfo focusNode =
893                         mViewRootImpl.mAccessibilityFocusedVirtualView;
894                 if (provider != null && focusNode != null) {
895                     final int virtualNodeId = AccessibilityNodeInfo.getVirtualDescendantId(
896                             focusNode.getSourceNodeId());
897                     provider.performAction(virtualNodeId,
898                             AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS.getId(),
899                             null);
900                 } else {
901                     host.performAccessibilityAction(
902                             AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS.getId(),
903                             null);
904                 }
905             }
906         } finally {
907             resetAccessibilityFetchFlags();
908         }
909     }
910 
911     /**
912      * Notify outside touch event to the target window.
913      */
notifyOutsideTouchClientThread()914     public void notifyOutsideTouchClientThread() {
915         final Message message = mHandler.obtainMessage();
916         message.what = PrivateHandler.MSG_NOTIFY_OUTSIDE_TOUCH;
917 
918         // Don't care about pid and tid because there's no interrogating client for this message.
919         scheduleMessage(message, 0, 0, CONSIDER_REQUEST_PREPARERS);
920     }
921 
notifyOutsideTouchUiThread()922     private void notifyOutsideTouchUiThread() {
923         if (mViewRootImpl.mView == null || mViewRootImpl.mAttachInfo == null
924                 || mViewRootImpl.mStopped || mViewRootImpl.mPausedForTransition) {
925             return;
926         }
927         final View root = getRootView();
928         if (root != null && isShown(root)) {
929             // trigger ACTION_OUTSIDE to notify windows
930             final long now = SystemClock.uptimeMillis();
931             final MotionEvent event = MotionEvent.obtain(now, now, MotionEvent.ACTION_OUTSIDE,
932                     0, 0, 0);
933             event.setSource(InputDevice.SOURCE_TOUCHSCREEN);
934             mViewRootImpl.dispatchInputEvent(event);
935         }
936     }
937 
findViewByAccessibilityId(int accessibilityId)938     private View findViewByAccessibilityId(int accessibilityId) {
939         if (accessibilityId == AccessibilityNodeInfo.ROOT_ITEM_ID) {
940             return getRootView();
941         } else {
942             return AccessibilityNodeIdManager.getInstance().findView(accessibilityId);
943         }
944     }
945 
getRootView()946     private View getRootView() {
947         if (!isVisibleToAccessibilityService(mViewRootImpl.mView)) {
948             return null;
949         }
950         return mViewRootImpl.mView;
951     }
952 
setAccessibilityFetchFlags(int flags)953     private void setAccessibilityFetchFlags(int flags) {
954         mViewRootImpl.mAttachInfo.mAccessibilityFetchFlags = flags;
955         mA11yManager.setRequestFromAccessibilityTool(
956                 (flags & AccessibilityNodeInfo.FLAG_SERVICE_IS_ACCESSIBILITY_TOOL) != 0);
957     }
958 
resetAccessibilityFetchFlags()959     private void resetAccessibilityFetchFlags() {
960         mViewRootImpl.mAttachInfo.mAccessibilityFetchFlags = 0;
961         mA11yManager.setRequestFromAccessibilityTool(false);
962     }
963 
964     // The boundInScreen includes magnification effect, so we need to normalize it before
965     // determine the visibility.
adjustIsVisibleToUserIfNeeded(AccessibilityNodeInfo info, Region interactiveRegion, MagnificationSpec spec)966     private void adjustIsVisibleToUserIfNeeded(AccessibilityNodeInfo info,
967             Region interactiveRegion, MagnificationSpec spec) {
968         if (interactiveRegion == null || info == null) {
969             return;
970         }
971         Rect boundsInScreen = mTempRect;
972         info.getBoundsInScreen(boundsInScreen);
973         if (spec != null && !spec.isNop()) {
974             boundsInScreen.offset((int) -spec.offsetX, (int) -spec.offsetY);
975             boundsInScreen.scale(1 / spec.scale);
976         }
977 
978         if (interactiveRegion.quickReject(boundsInScreen) && !shouldBypassAdjustIsVisible()) {
979             info.setVisibleToUser(false);
980         }
981     }
982 
shouldBypassAdjustIsVisible()983     private boolean shouldBypassAdjustIsVisible() {
984         final int windowType = mViewRootImpl.mOrigWindowType;
985         if (windowType == TYPE_INPUT_METHOD) {
986             return true;
987         }
988         return false;
989     }
990 
991     /**
992      * Applies the host-window matrix to the embedded node. After this transform, The node bounds
993      *  will be transformed from embedded window coordinates to host-window coordinates.
994      *
995      */
applyHostWindowMatrixIfNeeded(AccessibilityNodeInfo info)996     private void applyHostWindowMatrixIfNeeded(AccessibilityNodeInfo info) {
997         if (info == null || shouldBypassApplyWindowMatrix()) {
998             return;
999         }
1000         final Rect boundsInScreen = mTempRect;
1001         final RectF transformedBounds = mTempRectF;
1002         final Matrix windowMatrix = mViewRootImpl.mAttachInfo.mWindowMatrixInEmbeddedHierarchy;
1003 
1004         info.getBoundsInScreen(boundsInScreen);
1005         transformedBounds.set(boundsInScreen);
1006         windowMatrix.mapRect(transformedBounds);
1007         boundsInScreen.set((int) transformedBounds.left, (int) transformedBounds.top,
1008                 (int) transformedBounds.right, (int) transformedBounds.bottom);
1009         info.setBoundsInScreen(boundsInScreen);
1010     }
1011 
shouldBypassApplyWindowMatrix()1012     private boolean shouldBypassApplyWindowMatrix() {
1013         final Matrix windowMatrix = mViewRootImpl.mAttachInfo.mWindowMatrixInEmbeddedHierarchy;
1014         return windowMatrix == null || windowMatrix.isIdentity();
1015     }
1016 
associateLeashedParentIfNeeded(AccessibilityNodeInfo info)1017     private void associateLeashedParentIfNeeded(AccessibilityNodeInfo info) {
1018         if (info == null || shouldBypassAssociateLeashedParent()) {
1019             return;
1020         }
1021         // The node id of root node in embedded maybe not be ROOT_NODE_ID so we compare the id
1022         // with root view.
1023         if (mViewRootImpl.mView.getAccessibilityViewId()
1024                 != AccessibilityNodeInfo.getAccessibilityViewId(info.getSourceNodeId())) {
1025             return;
1026         }
1027         info.setLeashedParent(mViewRootImpl.mAttachInfo.mLeashedParentToken,
1028                 mViewRootImpl.mAttachInfo.mLeashedParentAccessibilityViewId);
1029     }
1030 
shouldBypassAssociateLeashedParent()1031     private boolean shouldBypassAssociateLeashedParent() {
1032         return (mViewRootImpl.mAttachInfo.mLeashedParentToken == null
1033                 && mViewRootImpl.mAttachInfo.mLeashedParentAccessibilityViewId == View.NO_ID);
1034     }
1035 
shouldApplyAppScaleAndMagnificationSpec(float appScale, MagnificationSpec spec)1036     private boolean shouldApplyAppScaleAndMagnificationSpec(float appScale,
1037             MagnificationSpec spec) {
1038         return (appScale != 1.0f || (spec != null && !spec.isNop()));
1039     }
1040 
updateInfosForViewPort(List<AccessibilityNodeInfo> infos, MagnificationSpec spec, float[] matrixValues, Region interactiveRegion)1041     private void updateInfosForViewPort(List<AccessibilityNodeInfo> infos, MagnificationSpec spec,
1042             float[] matrixValues, Region interactiveRegion) {
1043         for (int i = 0; i < infos.size(); i++) {
1044             updateInfoForViewPort(infos.get(i), spec, matrixValues, interactiveRegion);
1045         }
1046     }
1047 
updateInfoForViewPort(AccessibilityNodeInfo info, MagnificationSpec spec, float[] matrixValues, Region interactiveRegion)1048     private void updateInfoForViewPort(AccessibilityNodeInfo info, MagnificationSpec spec,
1049             float[] matrixValues, Region interactiveRegion) {
1050         associateLeashedParentIfNeeded(info);
1051 
1052         applyHostWindowMatrixIfNeeded(info);
1053         // Transform view bounds from window coordinates to screen coordinates.
1054         transformBoundsWithScreenMatrix(info, matrixValues);
1055         adjustIsVisibleToUserIfNeeded(info, interactiveRegion, spec);
1056     }
1057 
1058 
1059     /**
1060      * Transforms the regions from local screen coordinate to global screen coordinate with the
1061      * given transform matrix used in on-screen coordinate.
1062      *
1063      * @param info the AccessibilityNodeInfo that has the region in application screen coordinate
1064      * @param matrixValues the matrix to be applied
1065      */
transformBoundsWithScreenMatrix(AccessibilityNodeInfo info, float[] matrixValues)1066     private void transformBoundsWithScreenMatrix(AccessibilityNodeInfo info,
1067             float[] matrixValues) {
1068         if (info == null || matrixValues == null) {
1069             return;
1070         }
1071         final Rect boundInScreen = mTempRect;
1072         final RectF transformedBounds = mTempRectF;
1073 
1074         info.getBoundsInScreen(boundInScreen);
1075         transformedBounds.set(boundInScreen);
1076 
1077         final Matrix transformMatrix = new Matrix();
1078         transformMatrix.setValues(matrixValues);
1079         final float applicationScale = mViewRootImpl.mAttachInfo.mApplicationScale;
1080         if (applicationScale != 1f) {
1081             transformMatrix.preScale(applicationScale, applicationScale);
1082         }
1083         // Transform the bounds from application screen coordinates to global window coordinates.
1084         // For the embedded node, the bounds we get is already in window coordinates, so we don't
1085         // need to do it.
1086         if (mViewRootImpl.mAttachInfo.mWindowMatrixInEmbeddedHierarchy == null) {
1087             transformMatrix.preTranslate(-mViewRootImpl.mAttachInfo.mWindowLeft,
1088                     -mViewRootImpl.mAttachInfo.mWindowTop);
1089         }
1090 
1091         if (transformMatrix.isIdentity()) {
1092             return;
1093         }
1094         transformMatrix.mapRect(transformedBounds);
1095         roundRectFToRect(transformedBounds, boundInScreen);
1096         info.setBoundsInScreen(boundInScreen);
1097         // Scale text locations if they are present
1098         if (info.hasExtras()) {
1099             final Bundle extras = info.getExtras();
1100             final RectF[] textLocations =
1101                     extras.getParcelableArray(EXTRA_DATA_TEXT_CHARACTER_LOCATION_KEY, RectF.class);
1102             if (textLocations != null) {
1103                 for (int i = 0; i < textLocations.length; i++) {
1104                     // Unchecked cast - an app that puts other objects in this bundle with this
1105                     // key will crash.
1106                     final RectF textLocation = textLocations[i];
1107                     if (textLocation != null) {
1108                         transformMatrix.mapRect(textLocation);
1109                     }
1110                 }
1111             }
1112         }
1113         applyTransformMatrixToBoundsInParentIfNeeded(info, transformMatrix);
1114     }
1115 
applyTransformMatrixToBoundsInParentIfNeeded(AccessibilityNodeInfo info, Matrix transformMatrix)1116     private void applyTransformMatrixToBoundsInParentIfNeeded(AccessibilityNodeInfo info,
1117             Matrix transformMatrix) {
1118         final float[] screenMatrixValues = new float[9];
1119         transformMatrix.getValues(screenMatrixValues);
1120         final Matrix scaleMatrix = new Matrix();
1121         scaleMatrix.setScale(screenMatrixValues[Matrix.MSCALE_X],
1122                 screenMatrixValues[Matrix.MSCALE_X]);
1123         if (scaleMatrix.isIdentity()) {
1124             return;
1125         }
1126         Rect boundsInParent = mTempRect;
1127         final RectF transformedBounds = mTempRectF;
1128         info.getBoundsInParent(boundsInParent);
1129         transformedBounds.set(boundsInParent);
1130         scaleMatrix.mapRect(transformedBounds);
1131         roundRectFToRect(transformedBounds, boundsInParent);
1132         info.setBoundsInParent(boundsInParent);
1133     }
1134 
updateInfosForViewportAndReturnFindNodeResult(List<AccessibilityNodeInfo> infos, IAccessibilityInteractionConnectionCallback callback, int interactionId, MagnificationSpec spec, float[] matrixValues, Region interactiveRegion)1135     private void updateInfosForViewportAndReturnFindNodeResult(List<AccessibilityNodeInfo> infos,
1136             IAccessibilityInteractionConnectionCallback callback, int interactionId,
1137             MagnificationSpec spec, float[] matrixValues, Region interactiveRegion) {
1138         if (infos != null) {
1139             updateInfosForViewPort(infos, spec, matrixValues, interactiveRegion);
1140         }
1141         returnFindNodesResult(infos, callback, interactionId);
1142     }
1143 
returnFindNodeResult(AccessibilityNodeInfo info, IAccessibilityInteractionConnectionCallback callback, int interactionId)1144     private void returnFindNodeResult(AccessibilityNodeInfo info,
1145                                       IAccessibilityInteractionConnectionCallback callback,
1146                                       int interactionId) {
1147         try {
1148             callback.setFindAccessibilityNodeInfoResult(info, interactionId);
1149         } catch (RemoteException re) {
1150             /* ignore - the other side will time out */
1151         }
1152     }
1153 
returnFindNodeResult(SatisfiedFindAccessibilityNodeByAccessibilityIdRequest satisfiedRequest)1154     private void returnFindNodeResult(SatisfiedFindAccessibilityNodeByAccessibilityIdRequest
1155             satisfiedRequest) {
1156         try {
1157             final AccessibilityNodeInfo info = satisfiedRequest.mSatisfiedRequestNode;
1158             final IAccessibilityInteractionConnectionCallback callback =
1159                     satisfiedRequest.mSatisfiedRequestCallback;
1160             final int interactionId = satisfiedRequest.mSatisfiedRequestInteractionId;
1161             callback.setFindAccessibilityNodeInfoResult(info, interactionId);
1162         } catch (RemoteException re) {
1163             /* ignore - the other side will time out */
1164         }
1165     }
1166 
returnFindNodesResult(List<AccessibilityNodeInfo> infos, IAccessibilityInteractionConnectionCallback callback, int interactionId)1167     private void returnFindNodesResult(List<AccessibilityNodeInfo> infos,
1168             IAccessibilityInteractionConnectionCallback callback, int interactionId) {
1169         try {
1170             callback.setFindAccessibilityNodeInfosResult(infos, interactionId);
1171             if (infos != null) {
1172                 infos.clear();
1173             }
1174         } catch (RemoteException re) {
1175             /* ignore - the other side will time out */
1176         }
1177     }
1178 
getSatisfiedRequestInPrefetch( AccessibilityNodeInfo requestedNode, List<AccessibilityNodeInfo> infos, int flags)1179     private SatisfiedFindAccessibilityNodeByAccessibilityIdRequest getSatisfiedRequestInPrefetch(
1180             AccessibilityNodeInfo requestedNode, List<AccessibilityNodeInfo> infos, int flags) {
1181         SatisfiedFindAccessibilityNodeByAccessibilityIdRequest satisfiedRequest = null;
1182         synchronized (mLock) {
1183             for (int i = 0; i < mPendingFindNodeByIdMessages.size(); i++) {
1184                 final Message pendingMessage = mPendingFindNodeByIdMessages.get(i);
1185                 final int pendingFlags = pendingMessage.arg1;
1186                 if ((pendingFlags & FLAGS_AFFECTING_REPORTED_DATA)
1187                         != (flags & FLAGS_AFFECTING_REPORTED_DATA)) {
1188                     continue;
1189                 }
1190                 SomeArgs args = (SomeArgs) pendingMessage.obj;
1191                 final int accessibilityViewId = args.argi1;
1192                 final int virtualDescendantId = args.argi2;
1193 
1194                 final AccessibilityNodeInfo satisfiedRequestNode = nodeWithIdFromList(requestedNode,
1195                         infos, AccessibilityNodeInfo.makeNodeId(
1196                                 accessibilityViewId, virtualDescendantId));
1197 
1198                 if (satisfiedRequestNode != null) {
1199                     mHandler.removeMessages(
1200                             PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID,
1201                             pendingMessage.obj);
1202                     final IAccessibilityInteractionConnectionCallback satisfiedRequestCallback =
1203                             (IAccessibilityInteractionConnectionCallback) args.arg1;
1204                     final int satisfiedRequestInteractionId = args.argi3;
1205                     satisfiedRequest = new SatisfiedFindAccessibilityNodeByAccessibilityIdRequest(
1206                                     satisfiedRequestNode, satisfiedRequestCallback,
1207                                     satisfiedRequestInteractionId);
1208                     args.recycle();
1209                     break;
1210                 }
1211             }
1212             mPendingFindNodeByIdMessages.clear();
1213             // Remove node from prefetched infos.
1214             if (satisfiedRequest != null && satisfiedRequest.mSatisfiedRequestNode
1215                     != requestedNode) {
1216                 infos.remove(satisfiedRequest.mSatisfiedRequestNode);
1217             }
1218             return satisfiedRequest;
1219         }
1220     }
1221 
nodeWithIdFromList(AccessibilityNodeInfo requestedNode, List<AccessibilityNodeInfo> infos, long nodeId)1222     private AccessibilityNodeInfo nodeWithIdFromList(AccessibilityNodeInfo requestedNode,
1223             List<AccessibilityNodeInfo> infos, long nodeId) {
1224         if (requestedNode != null && requestedNode.getSourceNodeId() == nodeId) {
1225             return requestedNode;
1226         }
1227         for (int j = 0; j < infos.size(); j++) {
1228             AccessibilityNodeInfo info = infos.get(j);
1229             if (info.getSourceNodeId() == nodeId) {
1230                 return info;
1231             }
1232         }
1233         return null;
1234     }
1235 
returnPrefetchResult(int interactionId, List<AccessibilityNodeInfo> infos, IAccessibilityInteractionConnectionCallback callback)1236     private void returnPrefetchResult(int interactionId, List<AccessibilityNodeInfo> infos,
1237                                       IAccessibilityInteractionConnectionCallback callback) {
1238         if (infos.size() > 0) {
1239             try {
1240                 callback.setPrefetchAccessibilityNodeInfoResult(infos, interactionId);
1241             } catch (RemoteException re) {
1242                 /* ignore - other side isn't too bothered if this doesn't arrive */
1243             }
1244         }
1245     }
1246 
updateInfoForViewportAndReturnFindNodeResult(AccessibilityNodeInfo info, IAccessibilityInteractionConnectionCallback callback, int interactionId, MagnificationSpec spec, float[] matrixValues, Region interactiveRegion)1247     private void updateInfoForViewportAndReturnFindNodeResult(AccessibilityNodeInfo info,
1248             IAccessibilityInteractionConnectionCallback callback, int interactionId,
1249             MagnificationSpec spec, float[] matrixValues, Region interactiveRegion) {
1250         updateInfoForViewPort(info, spec, matrixValues, interactiveRegion);
1251         returnFindNodeResult(info, callback, interactionId);
1252     }
1253 
handleClickableSpanActionUiThread( View view, int virtualDescendantId, Bundle arguments)1254     private boolean handleClickableSpanActionUiThread(
1255             View view, int virtualDescendantId, Bundle arguments) {
1256         Parcelable span = arguments.getParcelable(ACTION_ARGUMENT_ACCESSIBLE_CLICKABLE_SPAN);
1257         if (!(span instanceof AccessibilityClickableSpan)) {
1258             return false;
1259         }
1260 
1261         // Find the original ClickableSpan if it's still on the screen
1262         AccessibilityNodeInfo infoWithSpan = null;
1263         AccessibilityNodeProvider provider = view.getAccessibilityNodeProvider();
1264         if (provider != null) {
1265             infoWithSpan = provider.createAccessibilityNodeInfo(virtualDescendantId);
1266         } else if (virtualDescendantId == AccessibilityNodeProvider.HOST_VIEW_ID) {
1267             infoWithSpan = view.createAccessibilityNodeInfo();
1268         }
1269         if (infoWithSpan == null) {
1270             return false;
1271         }
1272 
1273         // Click on the corresponding span
1274         ClickableSpan clickableSpan = ((AccessibilityClickableSpan) span).findClickableSpan(
1275                 infoWithSpan.getOriginalText());
1276         if (clickableSpan != null) {
1277             clickableSpan.onClick(view);
1278             return true;
1279         }
1280         return false;
1281     }
1282 
roundRectFToRect(@onNull RectF sourceRectF, @NonNull Rect outRect)1283     private static void roundRectFToRect(@NonNull  RectF sourceRectF, @NonNull Rect outRect) {
1284         // Offset 0.5f to round after casting.
1285         outRect.set((int) (sourceRectF.left + 0.5), (int) (sourceRectF.top + 0.5),
1286                 (int) (sourceRectF.right + 0.5), (int) (sourceRectF.bottom + 0.5));
1287     }
1288 
1289     /**
1290      * Destroy {@link AccessibilityInteractionController} and clean up the pending actions.
1291      */
destroy()1292     public void destroy() {
1293         if (Flags.preventLeakingViewrootimpl()) {
1294             mHandler.removeCallbacksAndMessages(null);
1295         }
1296     }
1297 
1298     /**
1299      * This class encapsulates a prefetching strategy for the accessibility APIs for
1300      * querying window content. It is responsible to prefetch a batch of
1301      * AccessibilityNodeInfos in addition to the one for a requested node.
1302      */
1303     private class AccessibilityNodePrefetcher {
1304 
1305         private final ArrayList<View> mTempViewList = new ArrayList<View>();
1306         private boolean mInterruptPrefetch;
1307         private int mFetchFlags;
1308 
prefetchAccessibilityNodeInfos(View view, AccessibilityNodeInfo root, List<AccessibilityNodeInfo> outInfos)1309         public void prefetchAccessibilityNodeInfos(View view, AccessibilityNodeInfo root,
1310                 List<AccessibilityNodeInfo> outInfos) {
1311             if (root == null) {
1312                 return;
1313             }
1314             AccessibilityNodeProvider provider = view.getAccessibilityNodeProvider();
1315             final boolean prefetchPredecessors =
1316                     isFlagSet(AccessibilityNodeInfo.FLAG_PREFETCH_ANCESTORS);
1317             if (provider == null) {
1318                 if (prefetchPredecessors) {
1319                     prefetchPredecessorsOfRealNode(view, outInfos);
1320                 }
1321                 if (isFlagSet(AccessibilityNodeInfo.FLAG_PREFETCH_SIBLINGS)) {
1322                     prefetchSiblingsOfRealNode(view, outInfos, prefetchPredecessors);
1323                 }
1324                 if (isFlagSet(AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_HYBRID)) {
1325                     prefetchDescendantsOfRealNode(view, outInfos);
1326                 }
1327             } else {
1328                 if (prefetchPredecessors) {
1329                     prefetchPredecessorsOfVirtualNode(root, view, provider, outInfos);
1330                 }
1331                 if (isFlagSet(AccessibilityNodeInfo.FLAG_PREFETCH_SIBLINGS)) {
1332                     prefetchSiblingsOfVirtualNode(root, view, provider, outInfos,
1333                             prefetchPredecessors);
1334                 }
1335                 if (isFlagSet(AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_HYBRID)) {
1336                     prefetchDescendantsOfVirtualNode(root, provider, outInfos);
1337                 }
1338             }
1339             if (isFlagSet(AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_DEPTH_FIRST)
1340                     || isFlagSet(AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_BREADTH_FIRST)) {
1341                 if (shouldStopPrefetching(outInfos)) {
1342                     return;
1343                 }
1344                 PrefetchDeque<DequeNode> deque = new PrefetchDeque<>(
1345                         mFetchFlags & AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_MASK,
1346                         outInfos);
1347                 addChildrenOfRoot(view, root, provider, deque);
1348                 deque.performTraversalAndPrefetch();
1349             }
1350             if (ENFORCE_NODE_TREE_CONSISTENT) {
1351                 enforceNodeTreeConsistent(root, outInfos);
1352             }
1353         }
1354 
addChildrenOfRoot(View root, AccessibilityNodeInfo rootInfo, AccessibilityNodeProvider rootProvider, PrefetchDeque deque)1355         private void addChildrenOfRoot(View root, AccessibilityNodeInfo rootInfo,
1356                 AccessibilityNodeProvider rootProvider, PrefetchDeque deque) {
1357             DequeNode rootDequeNode;
1358             if (rootProvider == null) {
1359                 rootDequeNode = new ViewNode(root);
1360             } else {
1361                 rootDequeNode = new VirtualNode(
1362                         AccessibilityNodeProvider.HOST_VIEW_ID, rootProvider);
1363             }
1364             rootDequeNode.addChildren(rootInfo, deque);
1365         }
1366 
isFlagSet(@ccessibilityNodeInfo.PrefetchingStrategy int strategy)1367         private boolean isFlagSet(@AccessibilityNodeInfo.PrefetchingStrategy int strategy) {
1368             return (mFetchFlags & strategy) != 0;
1369         }
1370 
shouldStopPrefetching(List prefetchedInfos)1371         public boolean shouldStopPrefetching(List prefetchedInfos) {
1372             return ((mHandler.hasUserInteractiveMessagesWaiting() && mInterruptPrefetch)
1373                     || prefetchedInfos.size()
1374                     >= AccessibilityNodeInfo.MAX_NUMBER_OF_PREFETCHED_NODES);
1375         }
1376 
enforceNodeTreeConsistent( AccessibilityNodeInfo root, List<AccessibilityNodeInfo> nodes)1377         private void enforceNodeTreeConsistent(
1378                 AccessibilityNodeInfo root, List<AccessibilityNodeInfo> nodes) {
1379             LongSparseArray<AccessibilityNodeInfo> nodeMap =
1380                     new LongSparseArray<AccessibilityNodeInfo>();
1381             final int nodeCount = nodes.size();
1382             for (int i = 0; i < nodeCount; i++) {
1383                 AccessibilityNodeInfo node = nodes.get(i);
1384                 nodeMap.put(node.getSourceNodeId(), node);
1385             }
1386 
1387             // If the nodes are a tree it does not matter from
1388             // which node we start to search for the root.
1389             AccessibilityNodeInfo parent = root;
1390             while (parent != null) {
1391                 root = parent;
1392                 parent = nodeMap.get(parent.getParentNodeId());
1393             }
1394 
1395             // Traverse the tree and do some checks.
1396             AccessibilityNodeInfo accessFocus = null;
1397             AccessibilityNodeInfo inputFocus = null;
1398             HashSet<AccessibilityNodeInfo> seen = new HashSet<AccessibilityNodeInfo>();
1399             Queue<AccessibilityNodeInfo> fringe = new LinkedList<AccessibilityNodeInfo>();
1400             fringe.add(root);
1401 
1402             while (!fringe.isEmpty()) {
1403                 AccessibilityNodeInfo current = fringe.poll();
1404 
1405                 // Check for duplicates
1406                 if (!seen.add(current)) {
1407                     throw new IllegalStateException("Duplicate node: "
1408                             + current + " in window:"
1409                             + mViewRootImpl.mAttachInfo.mAccessibilityWindowId);
1410                 }
1411 
1412                 // Check for one accessibility focus.
1413                 if (current.isAccessibilityFocused()) {
1414                     if (accessFocus != null) {
1415                         throw new IllegalStateException("Duplicate accessibility focus:"
1416                                 + current
1417                                 + " in window:" + mViewRootImpl.mAttachInfo.mAccessibilityWindowId);
1418                     } else {
1419                         accessFocus = current;
1420                     }
1421                 }
1422 
1423                 // Check for one input focus.
1424                 if (current.isFocused()) {
1425                     if (inputFocus != null) {
1426                         throw new IllegalStateException("Duplicate input focus: "
1427                             + current + " in window:"
1428                             + mViewRootImpl.mAttachInfo.mAccessibilityWindowId);
1429                     } else {
1430                         inputFocus = current;
1431                     }
1432                 }
1433 
1434                 final int childCount = current.getChildCount();
1435                 for (int j = 0; j < childCount; j++) {
1436                     final long childId = current.getChildId(j);
1437                     final AccessibilityNodeInfo child = nodeMap.get(childId);
1438                     if (child != null) {
1439                         fringe.add(child);
1440                     }
1441                 }
1442             }
1443 
1444             // Check for disconnected nodes.
1445             for (int j = nodeMap.size() - 1; j >= 0; j--) {
1446                 AccessibilityNodeInfo info = nodeMap.valueAt(j);
1447                 if (!seen.contains(info)) {
1448                     throw new IllegalStateException("Disconnected node: " + info);
1449                 }
1450             }
1451         }
1452 
prefetchPredecessorsOfRealNode(View view, List<AccessibilityNodeInfo> outInfos)1453         private void prefetchPredecessorsOfRealNode(View view,
1454                 List<AccessibilityNodeInfo> outInfos) {
1455             if (shouldStopPrefetching(outInfos)) {
1456                 return;
1457             }
1458             ViewParent parent = view.getParentForAccessibility();
1459             while (parent instanceof View && !shouldStopPrefetching(outInfos)) {
1460                 View parentView = (View) parent;
1461                 AccessibilityNodeInfo info = parentView.createAccessibilityNodeInfo();
1462                 if (info != null) {
1463                     outInfos.add(info);
1464                 }
1465                 parent = parent.getParentForAccessibility();
1466             }
1467         }
1468 
prefetchSiblingsOfRealNode(View current, List<AccessibilityNodeInfo> outInfos, boolean predecessorsPrefetched)1469         private void prefetchSiblingsOfRealNode(View current,
1470                 List<AccessibilityNodeInfo> outInfos, boolean predecessorsPrefetched) {
1471             if (shouldStopPrefetching(outInfos)) {
1472                 return;
1473             }
1474             ViewParent parent = current.getParentForAccessibility();
1475             if (parent instanceof ViewGroup) {
1476                 ViewGroup parentGroup = (ViewGroup) parent;
1477                 ArrayList<View> children = mTempViewList;
1478                 children.clear();
1479                 try {
1480                     if (!predecessorsPrefetched) {
1481                         AccessibilityNodeInfo parentInfo =
1482                                 ((ViewGroup) parent).createAccessibilityNodeInfo();
1483                         if (parentInfo != null) {
1484                             outInfos.add(parentInfo);
1485                         }
1486                     }
1487                     parentGroup.addChildrenForAccessibility(children);
1488                     final int childCount = children.size();
1489                     for (int i = 0; i < childCount; i++) {
1490                         if (shouldStopPrefetching(outInfos)) {
1491                             return;
1492                         }
1493                         View child = children.get(i);
1494                         if (child.getAccessibilityViewId() != current.getAccessibilityViewId()
1495                                 && isShown(child)) {
1496                             AccessibilityNodeInfo info = null;
1497                             AccessibilityNodeProvider provider =
1498                                     child.getAccessibilityNodeProvider();
1499                             if (provider == null) {
1500                                 info = child.createAccessibilityNodeInfo();
1501                             } else {
1502                                 info = provider.createAccessibilityNodeInfo(
1503                                         AccessibilityNodeProvider.HOST_VIEW_ID);
1504                             }
1505                             if (info != null) {
1506                                 outInfos.add(info);
1507                             }
1508                         }
1509                     }
1510                 } finally {
1511                     children.clear();
1512                 }
1513             }
1514         }
1515 
prefetchDescendantsOfRealNode(View root, List<AccessibilityNodeInfo> outInfos)1516         private void prefetchDescendantsOfRealNode(View root,
1517                 List<AccessibilityNodeInfo> outInfos) {
1518             if (shouldStopPrefetching(outInfos) || !(root instanceof ViewGroup)) {
1519                 return;
1520             }
1521             LinkedHashMap<View, AccessibilityNodeInfo> addedChildren =
1522                     new LinkedHashMap<View, AccessibilityNodeInfo>();
1523             ArrayList<View> children = mTempViewList;
1524             children.clear();
1525             try {
1526                 root.addChildrenForAccessibility(children);
1527                 final int childCount = children.size();
1528                 for (int i = 0; i < childCount; i++) {
1529                     if (shouldStopPrefetching(outInfos)) {
1530                         return;
1531                     }
1532                     View child = children.get(i);
1533                     if (isShown(child)) {
1534                         AccessibilityNodeProvider provider = child.getAccessibilityNodeProvider();
1535                         if (provider == null) {
1536                             AccessibilityNodeInfo info = child.createAccessibilityNodeInfo();
1537                             if (info != null) {
1538                                 outInfos.add(info);
1539                                 addedChildren.put(child, null);
1540                             }
1541                         } else {
1542                             AccessibilityNodeInfo info = provider.createAccessibilityNodeInfo(
1543                                    AccessibilityNodeProvider.HOST_VIEW_ID);
1544                             if (info != null) {
1545                                 outInfos.add(info);
1546                                 addedChildren.put(child, info);
1547                             }
1548                         }
1549                     }
1550                 }
1551             } finally {
1552                 children.clear();
1553             }
1554             if (!shouldStopPrefetching(outInfos)) {
1555                 for (Map.Entry<View, AccessibilityNodeInfo> entry : addedChildren.entrySet()) {
1556                     View addedChild = entry.getKey();
1557                     AccessibilityNodeInfo virtualRoot = entry.getValue();
1558                     if (virtualRoot == null) {
1559                         prefetchDescendantsOfRealNode(addedChild, outInfos);
1560                     } else {
1561                         AccessibilityNodeProvider provider =
1562                             addedChild.getAccessibilityNodeProvider();
1563                         prefetchDescendantsOfVirtualNode(virtualRoot, provider, outInfos);
1564                     }
1565                 }
1566             }
1567         }
1568 
prefetchPredecessorsOfVirtualNode(AccessibilityNodeInfo root, View providerHost, AccessibilityNodeProvider provider, List<AccessibilityNodeInfo> outInfos)1569         private void prefetchPredecessorsOfVirtualNode(AccessibilityNodeInfo root,
1570                 View providerHost, AccessibilityNodeProvider provider,
1571                 List<AccessibilityNodeInfo> outInfos) {
1572             final int initialResultSize = outInfos.size();
1573             long parentNodeId = root.getParentNodeId();
1574             int accessibilityViewId = AccessibilityNodeInfo.getAccessibilityViewId(parentNodeId);
1575             while (accessibilityViewId != AccessibilityNodeInfo.UNDEFINED_ITEM_ID) {
1576                 if (shouldStopPrefetching(outInfos)) {
1577                     return;
1578                 }
1579                 final int virtualDescendantId =
1580                     AccessibilityNodeInfo.getVirtualDescendantId(parentNodeId);
1581                 if (virtualDescendantId != AccessibilityNodeProvider.HOST_VIEW_ID
1582                         || accessibilityViewId == providerHost.getAccessibilityViewId()) {
1583                     final AccessibilityNodeInfo parent;
1584                     parent = provider.createAccessibilityNodeInfo(virtualDescendantId);
1585                     if (parent == null) {
1586                         // Going up the parent relation we found a null predecessor,
1587                         // so remove these disconnected nodes from the result.
1588                         final int currentResultSize = outInfos.size();
1589                         for (int i = currentResultSize - 1; i >= initialResultSize; i--) {
1590                             outInfos.remove(i);
1591                         }
1592                         // Couldn't obtain the parent, which means we have a
1593                         // disconnected sub-tree. Abort prefetch immediately.
1594                         return;
1595                     }
1596                     outInfos.add(parent);
1597                     parentNodeId = parent.getParentNodeId();
1598                     accessibilityViewId = AccessibilityNodeInfo.getAccessibilityViewId(
1599                             parentNodeId);
1600                 } else {
1601                     prefetchPredecessorsOfRealNode(providerHost, outInfos);
1602                     return;
1603                 }
1604             }
1605         }
1606 
prefetchSiblingsOfVirtualNode(AccessibilityNodeInfo current, View providerHost, AccessibilityNodeProvider provider, List<AccessibilityNodeInfo> outInfos, boolean predecessorsPrefetched)1607         private void prefetchSiblingsOfVirtualNode(AccessibilityNodeInfo current, View providerHost,
1608                 AccessibilityNodeProvider provider, List<AccessibilityNodeInfo> outInfos,
1609                 boolean predecessorsPrefetched) {
1610             final long parentNodeId = current.getParentNodeId();
1611             final int parentAccessibilityViewId =
1612                     AccessibilityNodeInfo.getAccessibilityViewId(parentNodeId);
1613             final int parentVirtualDescendantId =
1614                     AccessibilityNodeInfo.getVirtualDescendantId(parentNodeId);
1615             if (parentVirtualDescendantId != AccessibilityNodeProvider.HOST_VIEW_ID
1616                     || parentAccessibilityViewId == providerHost.getAccessibilityViewId()) {
1617                 final AccessibilityNodeInfo parent =
1618                         provider.createAccessibilityNodeInfo(parentVirtualDescendantId);
1619                 if (parent != null) {
1620                     if (!predecessorsPrefetched) {
1621                         outInfos.add(parent);
1622                     }
1623                     final int childCount = parent.getChildCount();
1624                     for (int i = 0; i < childCount; i++) {
1625                         if (shouldStopPrefetching(outInfos)) {
1626                             return;
1627                         }
1628                         final long childNodeId = parent.getChildId(i);
1629                         if (childNodeId != current.getSourceNodeId()) {
1630                             final int childVirtualDescendantId =
1631                                 AccessibilityNodeInfo.getVirtualDescendantId(childNodeId);
1632                             AccessibilityNodeInfo child = provider.createAccessibilityNodeInfo(
1633                                     childVirtualDescendantId);
1634                             if (child != null) {
1635                                 outInfos.add(child);
1636                             }
1637                         }
1638                     }
1639                 }
1640             } else {
1641                 prefetchSiblingsOfRealNode(providerHost, outInfos, predecessorsPrefetched);
1642             }
1643         }
1644 
prefetchDescendantsOfVirtualNode(AccessibilityNodeInfo root, AccessibilityNodeProvider provider, List<AccessibilityNodeInfo> outInfos)1645         private void prefetchDescendantsOfVirtualNode(AccessibilityNodeInfo root,
1646                 AccessibilityNodeProvider provider, List<AccessibilityNodeInfo> outInfos) {
1647             final int initialOutInfosSize = outInfos.size();
1648             final int childCount = root.getChildCount();
1649             for (int i = 0; i < childCount; i++) {
1650                 if (shouldStopPrefetching(outInfos)) {
1651                     return;
1652                 }
1653                 final long childNodeId = root.getChildId(i);
1654                 AccessibilityNodeInfo child = provider.createAccessibilityNodeInfo(
1655                         AccessibilityNodeInfo.getVirtualDescendantId(childNodeId));
1656                 if (child != null) {
1657                     outInfos.add(child);
1658                 }
1659             }
1660             if (!shouldStopPrefetching(outInfos)) {
1661                 final int addedChildCount = outInfos.size() - initialOutInfosSize;
1662                 for (int i = 0; i < addedChildCount; i++) {
1663                     AccessibilityNodeInfo child = outInfos.get(initialOutInfosSize + i);
1664                     prefetchDescendantsOfVirtualNode(child, provider, outInfos);
1665                 }
1666             }
1667         }
1668     }
1669 
1670     private class PrivateHandler extends Handler {
1671         private static final int MSG_PERFORM_ACCESSIBILITY_ACTION = 1;
1672         private static final int MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID = 2;
1673         private static final int MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID = 3;
1674         private static final int MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT = 4;
1675         private static final int MSG_FIND_FOCUS = 5;
1676         private static final int MSG_FOCUS_SEARCH = 6;
1677         private static final int MSG_PREPARE_FOR_EXTRA_DATA_REQUEST = 7;
1678         private static final int MSG_APP_PREPARATION_FINISHED = 8;
1679         private static final int MSG_APP_PREPARATION_TIMEOUT = 9;
1680 
1681         // Uses FIRST_NO_ACCESSIBILITY_CALLBACK_MSG for messages that don't need to call back
1682         // results to interrogating client.
1683         private static final int FIRST_NO_ACCESSIBILITY_CALLBACK_MSG = 100;
1684         private static final int MSG_CLEAR_ACCESSIBILITY_FOCUS =
1685                 FIRST_NO_ACCESSIBILITY_CALLBACK_MSG + 1;
1686         private static final int MSG_NOTIFY_OUTSIDE_TOUCH =
1687                 FIRST_NO_ACCESSIBILITY_CALLBACK_MSG + 2;
1688 
PrivateHandler(Looper looper)1689         public PrivateHandler(Looper looper) {
1690             super(looper);
1691         }
1692 
1693         @Override
getMessageName(Message message)1694         public String getMessageName(Message message) {
1695             final int type = message.what;
1696             switch (type) {
1697                 case MSG_PERFORM_ACCESSIBILITY_ACTION:
1698                     return "MSG_PERFORM_ACCESSIBILITY_ACTION";
1699                 case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID:
1700                     return "MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID";
1701                 case MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID:
1702                     return "MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID";
1703                 case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT:
1704                     return "MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT";
1705                 case MSG_FIND_FOCUS:
1706                     return "MSG_FIND_FOCUS";
1707                 case MSG_FOCUS_SEARCH:
1708                     return "MSG_FOCUS_SEARCH";
1709                 case MSG_PREPARE_FOR_EXTRA_DATA_REQUEST:
1710                     return "MSG_PREPARE_FOR_EXTRA_DATA_REQUEST";
1711                 case MSG_APP_PREPARATION_FINISHED:
1712                     return "MSG_APP_PREPARATION_FINISHED";
1713                 case MSG_APP_PREPARATION_TIMEOUT:
1714                     return "MSG_APP_PREPARATION_TIMEOUT";
1715                 case MSG_CLEAR_ACCESSIBILITY_FOCUS:
1716                     return "MSG_CLEAR_ACCESSIBILITY_FOCUS";
1717                 case MSG_NOTIFY_OUTSIDE_TOUCH:
1718                     return "MSG_NOTIFY_OUTSIDE_TOUCH";
1719                 default:
1720                     throw new IllegalArgumentException("Unknown message type: " + type);
1721             }
1722         }
1723 
1724         @Override
handleMessage(Message message)1725         public void handleMessage(Message message) {
1726             final int type = message.what;
1727             switch (type) {
1728                 case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID: {
1729                     findAccessibilityNodeInfoByAccessibilityIdUiThread(message);
1730                 } break;
1731                 case MSG_PERFORM_ACCESSIBILITY_ACTION: {
1732                     performAccessibilityActionUiThread(message);
1733                 } break;
1734                 case MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID: {
1735                     findAccessibilityNodeInfosByViewIdUiThread(message);
1736                 } break;
1737                 case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT: {
1738                     findAccessibilityNodeInfosByTextUiThread(message);
1739                 } break;
1740                 case MSG_FIND_FOCUS: {
1741                     findFocusUiThread(message);
1742                 } break;
1743                 case MSG_FOCUS_SEARCH: {
1744                     focusSearchUiThread(message);
1745                 } break;
1746                 case MSG_PREPARE_FOR_EXTRA_DATA_REQUEST: {
1747                     prepareForExtraDataRequestUiThread(message);
1748                 } break;
1749                 case MSG_APP_PREPARATION_FINISHED: {
1750                     requestPreparerDoneUiThread(message);
1751                 } break;
1752                 case MSG_APP_PREPARATION_TIMEOUT: {
1753                     requestPreparerTimeoutUiThread();
1754                 } break;
1755                 case MSG_CLEAR_ACCESSIBILITY_FOCUS: {
1756                     clearAccessibilityFocusUiThread();
1757                 } break;
1758                 case MSG_NOTIFY_OUTSIDE_TOUCH: {
1759                     notifyOutsideTouchUiThread();
1760                 } break;
1761                 default:
1762                     throw new IllegalArgumentException("Unknown message type: " + type);
1763             }
1764         }
1765 
hasAccessibilityCallback(Message message)1766         boolean hasAccessibilityCallback(Message message) {
1767             return message.what < FIRST_NO_ACCESSIBILITY_CALLBACK_MSG ? true : false;
1768         }
1769 
hasUserInteractiveMessagesWaiting()1770         boolean hasUserInteractiveMessagesWaiting() {
1771             return hasMessagesOrCallbacks();
1772         }
1773     }
1774 
1775     private final class AddNodeInfosForViewId implements Predicate<View> {
1776         private int mViewId = View.NO_ID;
1777         private List<AccessibilityNodeInfo> mInfos;
1778 
init(int viewId, List<AccessibilityNodeInfo> infos)1779         public void init(int viewId, List<AccessibilityNodeInfo> infos) {
1780             mViewId = viewId;
1781             mInfos = infos;
1782         }
1783 
reset()1784         public void reset() {
1785             mViewId = View.NO_ID;
1786             mInfos = null;
1787         }
1788 
1789         @Override
test(View view)1790         public boolean test(View view) {
1791             if (view.getId() == mViewId && isShown(view) && isVisibleToAccessibilityService(view)) {
1792                 mInfos.add(view.createAccessibilityNodeInfo());
1793             }
1794             return false;
1795         }
1796     }
1797 
1798     private static final class MessageHolder {
1799         final Message mMessage;
1800         final int mInterrogatingPid;
1801         final long mInterrogatingTid;
1802 
MessageHolder(Message message, int interrogatingPid, long interrogatingTid)1803         MessageHolder(Message message, int interrogatingPid, long interrogatingTid) {
1804             mMessage = message;
1805             mInterrogatingPid = interrogatingPid;
1806             mInterrogatingTid = interrogatingTid;
1807         }
1808     }
1809 
1810     private static class SatisfiedFindAccessibilityNodeByAccessibilityIdRequest {
1811         final AccessibilityNodeInfo mSatisfiedRequestNode;
1812         final IAccessibilityInteractionConnectionCallback mSatisfiedRequestCallback;
1813         final int mSatisfiedRequestInteractionId;
1814 
SatisfiedFindAccessibilityNodeByAccessibilityIdRequest( AccessibilityNodeInfo satisfiedRequestNode, IAccessibilityInteractionConnectionCallback satisfiedRequestCallback, int satisfiedRequestInteractionId)1815         SatisfiedFindAccessibilityNodeByAccessibilityIdRequest(
1816                 AccessibilityNodeInfo satisfiedRequestNode,
1817                 IAccessibilityInteractionConnectionCallback satisfiedRequestCallback,
1818                 int satisfiedRequestInteractionId) {
1819             mSatisfiedRequestNode = satisfiedRequestNode;
1820             mSatisfiedRequestCallback = satisfiedRequestCallback;
1821             mSatisfiedRequestInteractionId = satisfiedRequestInteractionId;
1822         }
1823     }
1824 
1825     private class PrefetchDeque<E extends DequeNode>
1826             extends ArrayDeque<E> {
1827         int mStrategy;
1828         List<AccessibilityNodeInfo> mPrefetchOutput;
1829 
PrefetchDeque(int strategy, List<AccessibilityNodeInfo> output)1830         PrefetchDeque(int strategy, List<AccessibilityNodeInfo> output) {
1831             mStrategy = strategy;
1832             mPrefetchOutput = output;
1833         }
1834 
1835         /** Performs depth-first or breadth-first traversal.
1836          *
1837          * For depth-first search, we iterate through the children in backwards order and push them
1838          * to the stack before taking from the head. For breadth-first search, we iterate through
1839          * the children in order and push them to the stack before taking from the tail.
1840          *
1841          * Depth-first search:  0 has children 0, 1, 2, 4. 1 has children 5 and 6.
1842          * Head         Tail
1843          * 1  2  3  4 ->  pop: 1 -> 5  6  2  3  4
1844          *
1845          * Breadth-first search
1846          * Head         Tail
1847          * 4  3  2  1 -> remove last: 1 -> 6  5  3  2
1848          *
1849          **/
performTraversalAndPrefetch()1850         void performTraversalAndPrefetch() {
1851             try {
1852                 while (!isEmpty()) {
1853                     E child = getNext();
1854                     AccessibilityNodeInfo childInfo = child.getA11yNodeInfo();
1855                     if (childInfo != null) {
1856                         mPrefetchOutput.add(childInfo);
1857                     }
1858                     if (mPrefetcher.shouldStopPrefetching(mPrefetchOutput)) {
1859                         return;
1860                     }
1861                     // Add children to deque.
1862                     child.addChildren(childInfo, this);
1863                 }
1864             } finally {
1865                 clear();
1866             }
1867         }
1868 
getNext()1869         E getNext() {
1870             if (isStack()) {
1871                 return pop();
1872             }
1873             return removeLast();
1874         }
1875 
isStack()1876         boolean isStack() {
1877             return (mStrategy & AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_DEPTH_FIRST) != 0;
1878         }
1879     }
1880 
1881     interface DequeNode {
getA11yNodeInfo()1882         AccessibilityNodeInfo getA11yNodeInfo();
addChildren(AccessibilityNodeInfo virtualRoot, PrefetchDeque deque)1883         void addChildren(AccessibilityNodeInfo virtualRoot, PrefetchDeque deque);
1884     }
1885 
1886     private class ViewNode implements DequeNode {
1887         View mView;
1888         private final ArrayList<View> mTempViewList = new ArrayList<>();
1889 
ViewNode(View view)1890         ViewNode(View view) {
1891             mView = view;
1892         }
1893 
1894         @Override
getA11yNodeInfo()1895         public AccessibilityNodeInfo getA11yNodeInfo() {
1896             if (mView == null) {
1897                 return null;
1898             }
1899             return mView.createAccessibilityNodeInfo();
1900         }
1901 
1902         @Override
addChildren(AccessibilityNodeInfo virtualRoot, PrefetchDeque deque)1903         public void addChildren(AccessibilityNodeInfo virtualRoot, PrefetchDeque deque) {
1904             if (mView == null) {
1905                 return;
1906             }
1907             if (!(mView instanceof ViewGroup)) {
1908                 return;
1909             }
1910             ArrayList<View> children = mTempViewList;
1911             children.clear();
1912             try {
1913                 mView.addChildrenForAccessibility(children);
1914                 final int childCount = children.size();
1915 
1916                 if (deque.isStack()) {
1917                     for (int i = childCount - 1; i >= 0; i--) {
1918                         addChild(deque, children.get(i));
1919                     }
1920                 } else {
1921                     for (int i = 0; i < childCount; i++) {
1922                         addChild(deque, children.get(i));
1923                     }
1924                 }
1925             } finally {
1926                 children.clear();
1927             }
1928         }
1929 
addChild(ArrayDeque deque, View child)1930         private void addChild(ArrayDeque deque, View child) {
1931             if (isShown(child)) {
1932                 AccessibilityNodeProvider provider = child.getAccessibilityNodeProvider();
1933                 if (provider == null) {
1934                     deque.push(new ViewNode(child));
1935                 } else {
1936                     deque.push(new VirtualNode(AccessibilityNodeProvider.HOST_VIEW_ID,
1937                             provider));
1938                 }
1939             }
1940         }
1941     }
1942 
1943     private class VirtualNode implements DequeNode {
1944         long mInfoId;
1945         AccessibilityNodeProvider mProvider;
1946 
VirtualNode(long id, AccessibilityNodeProvider provider)1947         VirtualNode(long id, AccessibilityNodeProvider provider) {
1948             mInfoId = id;
1949             mProvider = provider;
1950         }
1951         @Override
getA11yNodeInfo()1952         public AccessibilityNodeInfo getA11yNodeInfo() {
1953             if (mProvider == null) {
1954                 return null;
1955             }
1956             return mProvider.createAccessibilityNodeInfo(
1957                     AccessibilityNodeInfo.getVirtualDescendantId(mInfoId));
1958         }
1959 
1960         @Override
addChildren(AccessibilityNodeInfo virtualRoot, PrefetchDeque deque)1961         public void addChildren(AccessibilityNodeInfo virtualRoot, PrefetchDeque deque) {
1962             if (virtualRoot == null) {
1963                 return;
1964             }
1965             final int childCount = virtualRoot.getChildCount();
1966             if (deque.isStack()) {
1967                 for (int i = childCount - 1; i >= 0; i--) {
1968                     final long childNodeId = virtualRoot.getChildId(i);
1969                     deque.push(new VirtualNode(childNodeId, mProvider));
1970                 }
1971             } else {
1972                 for (int i = 0; i < childCount; i++) {
1973                     final long childNodeId = virtualRoot.getChildId(i);
1974                     deque.push(new VirtualNode(childNodeId, mProvider));
1975                 }
1976             }
1977         }
1978     }
1979 
1980     /** Attaches an accessibility overlay to the specified window. */
attachAccessibilityOverlayToWindowClientThread( SurfaceControl sc, int interactionId, IAccessibilityInteractionConnectionCallback callback)1981     public void attachAccessibilityOverlayToWindowClientThread(
1982             SurfaceControl sc,
1983             int interactionId,
1984             IAccessibilityInteractionConnectionCallback callback) {
1985         mHandler.sendMessage(
1986                 obtainMessage(
1987                         AccessibilityInteractionController
1988                                 ::attachAccessibilityOverlayToWindowUiThread,
1989                         this,
1990                         sc,
1991                         interactionId,
1992                         callback));
1993     }
1994 
attachAccessibilityOverlayToWindowUiThread( SurfaceControl sc, int interactionId, IAccessibilityInteractionConnectionCallback callback)1995     private void attachAccessibilityOverlayToWindowUiThread(
1996             SurfaceControl sc,
1997             int interactionId,
1998             IAccessibilityInteractionConnectionCallback callback) {
1999         SurfaceControl parent = mViewRootImpl.getSurfaceControl();
2000         if (!parent.isValid()) {
2001             try {
2002                 callback.sendAttachOverlayResult(
2003                         AccessibilityService.OVERLAY_RESULT_INTERNAL_ERROR, interactionId);
2004                 return;
2005             } catch (RemoteException re) {
2006                 /* ignore - the other side will time out */
2007             }
2008         }
2009         SurfaceControl.Transaction t = new SurfaceControl.Transaction();
2010         t.reparent(sc, parent).apply();
2011         t.close();
2012         try {
2013             callback.sendAttachOverlayResult(
2014                     AccessibilityService.OVERLAY_RESULT_SUCCESS, interactionId);
2015         } catch (RemoteException re) {
2016             /* ignore - the other side will time out */
2017         }
2018     }
2019 }
2020