1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.app;
18 
19 import android.accessibilityservice.AccessibilityService.Callbacks;
20 import android.accessibilityservice.AccessibilityService.IAccessibilityServiceClientWrapper;
21 import android.accessibilityservice.AccessibilityServiceInfo;
22 import android.accessibilityservice.IAccessibilityServiceClient;
23 import android.accessibilityservice.IAccessibilityServiceConnection;
24 import android.annotation.NonNull;
25 import android.annotation.TestApi;
26 import android.graphics.Bitmap;
27 import android.graphics.Point;
28 import android.graphics.Rect;
29 import android.graphics.Region;
30 import android.hardware.display.DisplayManagerGlobal;
31 import android.os.Handler;
32 import android.os.HandlerThread;
33 import android.os.IBinder;
34 import android.os.Looper;
35 import android.os.ParcelFileDescriptor;
36 import android.os.RemoteException;
37 import android.os.SystemClock;
38 import android.os.UserHandle;
39 import android.util.Log;
40 import android.view.Display;
41 import android.view.InputEvent;
42 import android.view.KeyEvent;
43 import android.view.Surface;
44 import android.view.WindowAnimationFrameStats;
45 import android.view.WindowContentFrameStats;
46 import android.view.accessibility.AccessibilityEvent;
47 import android.view.accessibility.AccessibilityInteractionClient;
48 import android.view.accessibility.AccessibilityNodeInfo;
49 import android.view.accessibility.AccessibilityWindowInfo;
50 import android.view.accessibility.IAccessibilityInteractionConnection;
51 
52 import com.android.internal.util.function.pooled.PooledLambda;
53 import libcore.io.IoUtils;
54 
55 import java.io.IOException;
56 import java.util.ArrayList;
57 import java.util.List;
58 import java.util.concurrent.TimeoutException;
59 
60 /**
61  * Class for interacting with the device's UI by simulation user actions and
62  * introspection of the screen content. It relies on the platform accessibility
63  * APIs to introspect the screen and to perform some actions on the remote view
64  * tree. It also allows injecting of arbitrary raw input events simulating user
65  * interaction with keyboards and touch devices. One can think of a UiAutomation
66  * as a special type of {@link android.accessibilityservice.AccessibilityService}
67  * which does not provide hooks for the service life cycle and exposes other
68  * APIs that are useful for UI test automation.
69  * <p>
70  * The APIs exposed by this class are low-level to maximize flexibility when
71  * developing UI test automation tools and libraries. Generally, a UiAutomation
72  * client should be using a higher-level library or implement high-level functions.
73  * For example, performing a tap on the screen requires construction and injecting
74  * of a touch down and up events which have to be delivered to the system by a
75  * call to {@link #injectInputEvent(InputEvent, boolean)}.
76  * </p>
77  * <p>
78  * The APIs exposed by this class operate across applications enabling a client
79  * to write tests that cover use cases spanning over multiple applications. For
80  * example, going to the settings application to change a setting and then
81  * interacting with another application whose behavior depends on that setting.
82  * </p>
83  */
84 public final class UiAutomation {
85 
86     private static final String LOG_TAG = UiAutomation.class.getSimpleName();
87 
88     private static final boolean DEBUG = false;
89 
90     private static final int CONNECTION_ID_UNDEFINED = -1;
91 
92     private static final long CONNECT_TIMEOUT_MILLIS = 5000;
93 
94     /** Rotation constant: Unfreeze rotation (rotating the device changes its rotation state). */
95     public static final int ROTATION_UNFREEZE = -2;
96 
97     /** Rotation constant: Freeze rotation to its current state. */
98     public static final int ROTATION_FREEZE_CURRENT = -1;
99 
100     /** Rotation constant: Freeze rotation to 0 degrees (natural orientation) */
101     public static final int ROTATION_FREEZE_0 = Surface.ROTATION_0;
102 
103     /** Rotation constant: Freeze rotation to 90 degrees . */
104     public static final int ROTATION_FREEZE_90 = Surface.ROTATION_90;
105 
106     /** Rotation constant: Freeze rotation to 180 degrees . */
107     public static final int ROTATION_FREEZE_180 = Surface.ROTATION_180;
108 
109     /** Rotation constant: Freeze rotation to 270 degrees . */
110     public static final int ROTATION_FREEZE_270 = Surface.ROTATION_270;
111 
112     /**
113      * UiAutomation supresses accessibility services by default. This flag specifies that
114      * existing accessibility services should continue to run, and that new ones may start.
115      * This flag is set when obtaining the UiAutomation from
116      * {@link Instrumentation#getUiAutomation(int)}.
117      */
118     public static final int FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES = 0x00000001;
119 
120     private final Object mLock = new Object();
121 
122     private final ArrayList<AccessibilityEvent> mEventQueue = new ArrayList<AccessibilityEvent>();
123 
124     private final Handler mLocalCallbackHandler;
125 
126     private final IUiAutomationConnection mUiAutomationConnection;
127 
128     private HandlerThread mRemoteCallbackThread;
129 
130     private IAccessibilityServiceClient mClient;
131 
132     private int mConnectionId = CONNECTION_ID_UNDEFINED;
133 
134     private OnAccessibilityEventListener mOnAccessibilityEventListener;
135 
136     private boolean mWaitingForEventDelivery;
137 
138     private long mLastEventTimeMillis;
139 
140     private boolean mIsConnecting;
141 
142     private boolean mIsDestroyed;
143 
144     private int mFlags;
145 
146     /**
147      * Listener for observing the {@link AccessibilityEvent} stream.
148      */
149     public static interface OnAccessibilityEventListener {
150 
151         /**
152          * Callback for receiving an {@link AccessibilityEvent}.
153          * <p>
154          * <strong>Note:</strong> This method is <strong>NOT</strong> executed
155          * on the main test thread. The client is responsible for proper
156          * synchronization.
157          * </p>
158          * <p>
159          * <strong>Note:</strong> It is responsibility of the client
160          * to recycle the received events to minimize object creation.
161          * </p>
162          *
163          * @param event The received event.
164          */
onAccessibilityEvent(AccessibilityEvent event)165         public void onAccessibilityEvent(AccessibilityEvent event);
166     }
167 
168     /**
169      * Listener for filtering accessibility events.
170      */
171     public static interface AccessibilityEventFilter {
172 
173         /**
174          * Callback for determining whether an event is accepted or
175          * it is filtered out.
176          *
177          * @param event The event to process.
178          * @return True if the event is accepted, false to filter it out.
179          */
accept(AccessibilityEvent event)180         public boolean accept(AccessibilityEvent event);
181     }
182 
183     /**
184      * Creates a new instance that will handle callbacks from the accessibility
185      * layer on the thread of the provided looper and perform requests for privileged
186      * operations on the provided connection.
187      *
188      * @param looper The looper on which to execute accessibility callbacks.
189      * @param connection The connection for performing privileged operations.
190      *
191      * @hide
192      */
UiAutomation(Looper looper, IUiAutomationConnection connection)193     public UiAutomation(Looper looper, IUiAutomationConnection connection) {
194         if (looper == null) {
195             throw new IllegalArgumentException("Looper cannot be null!");
196         }
197         if (connection == null) {
198             throw new IllegalArgumentException("Connection cannot be null!");
199         }
200         mLocalCallbackHandler = new Handler(looper);
201         mUiAutomationConnection = connection;
202     }
203 
204     /**
205      * Connects this UiAutomation to the accessibility introspection APIs with default flags.
206      *
207      * @hide
208      */
connect()209     public void connect() {
210         connect(0);
211     }
212 
213     /**
214      * Connects this UiAutomation to the accessibility introspection APIs.
215      *
216      * @param flags Any flags to apply to the automation as it gets connected
217      *
218      * @hide
219      */
connect(int flags)220     public void connect(int flags) {
221         synchronized (mLock) {
222             throwIfConnectedLocked();
223             if (mIsConnecting) {
224                 return;
225             }
226             mIsConnecting = true;
227             mRemoteCallbackThread = new HandlerThread("UiAutomation");
228             mRemoteCallbackThread.start();
229             mClient = new IAccessibilityServiceClientImpl(mRemoteCallbackThread.getLooper());
230         }
231 
232         try {
233             // Calling out without a lock held.
234             mUiAutomationConnection.connect(mClient, flags);
235             mFlags = flags;
236         } catch (RemoteException re) {
237             throw new RuntimeException("Error while connecting UiAutomation", re);
238         }
239 
240         synchronized (mLock) {
241             final long startTimeMillis = SystemClock.uptimeMillis();
242             try {
243                 while (true) {
244                     if (isConnectedLocked()) {
245                         break;
246                     }
247                     final long elapsedTimeMillis = SystemClock.uptimeMillis() - startTimeMillis;
248                     final long remainingTimeMillis = CONNECT_TIMEOUT_MILLIS - elapsedTimeMillis;
249                     if (remainingTimeMillis <= 0) {
250                         throw new RuntimeException("Error while connecting UiAutomation");
251                     }
252                     try {
253                         mLock.wait(remainingTimeMillis);
254                     } catch (InterruptedException ie) {
255                         /* ignore */
256                     }
257                 }
258             } finally {
259                 mIsConnecting = false;
260             }
261         }
262     }
263 
264     /**
265      * Get the flags used to connect the service.
266      *
267      * @return The flags used to connect
268      *
269      * @hide
270      */
getFlags()271     public int getFlags() {
272         return mFlags;
273     }
274 
275     /**
276      * Disconnects this UiAutomation from the accessibility introspection APIs.
277      *
278      * @hide
279      */
disconnect()280     public void disconnect() {
281         synchronized (mLock) {
282             if (mIsConnecting) {
283                 throw new IllegalStateException(
284                         "Cannot call disconnect() while connecting!");
285             }
286             throwIfNotConnectedLocked();
287             mConnectionId = CONNECTION_ID_UNDEFINED;
288         }
289         try {
290             // Calling out without a lock held.
291             mUiAutomationConnection.disconnect();
292         } catch (RemoteException re) {
293             throw new RuntimeException("Error while disconnecting UiAutomation", re);
294         } finally {
295             mRemoteCallbackThread.quit();
296             mRemoteCallbackThread = null;
297         }
298     }
299 
300     /**
301      * The id of the {@link IAccessibilityInteractionConnection} for querying
302      * the screen content. This is here for legacy purposes since some tools use
303      * hidden APIs to introspect the screen.
304      *
305      * @hide
306      */
getConnectionId()307     public int getConnectionId() {
308         synchronized (mLock) {
309             throwIfNotConnectedLocked();
310             return mConnectionId;
311         }
312     }
313 
314     /**
315      * Reports if the object has been destroyed
316      *
317      * @return {code true} if the object has been destroyed.
318      *
319      * @hide
320      */
isDestroyed()321     public boolean isDestroyed() {
322         return mIsDestroyed;
323     }
324 
325     /**
326      * Sets a callback for observing the stream of {@link AccessibilityEvent}s.
327      * The callbacks are delivered on the main application thread.
328      *
329      * @param listener The callback.
330      */
setOnAccessibilityEventListener(OnAccessibilityEventListener listener)331     public void setOnAccessibilityEventListener(OnAccessibilityEventListener listener) {
332         synchronized (mLock) {
333             mOnAccessibilityEventListener = listener;
334         }
335     }
336 
337     /**
338      * Destroy this UiAutomation. After calling this method, attempting to use the object will
339      * result in errors.
340      *
341      * @hide
342      */
343     @TestApi
destroy()344     public void destroy() {
345         disconnect();
346         mIsDestroyed = true;
347     }
348 
349     /**
350      * Performs a global action. Such an action can be performed at any moment
351      * regardless of the current application or user location in that application.
352      * For example going back, going home, opening recents, etc.
353      *
354      * @param action The action to perform.
355      * @return Whether the action was successfully performed.
356      *
357      * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_BACK
358      * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_HOME
359      * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_NOTIFICATIONS
360      * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_RECENTS
361      */
performGlobalAction(int action)362     public final boolean performGlobalAction(int action) {
363         final IAccessibilityServiceConnection connection;
364         synchronized (mLock) {
365             throwIfNotConnectedLocked();
366             connection = AccessibilityInteractionClient.getInstance()
367                     .getConnection(mConnectionId);
368         }
369         // Calling out without a lock held.
370         if (connection != null) {
371             try {
372                 return connection.performGlobalAction(action);
373             } catch (RemoteException re) {
374                 Log.w(LOG_TAG, "Error while calling performGlobalAction", re);
375             }
376         }
377         return false;
378     }
379 
380     /**
381      * Find the view that has the specified focus type. The search is performed
382      * across all windows.
383      * <p>
384      * <strong>Note:</strong> In order to access the windows you have to opt-in
385      * to retrieve the interactive windows by setting the
386      * {@link AccessibilityServiceInfo#FLAG_RETRIEVE_INTERACTIVE_WINDOWS} flag.
387      * Otherwise, the search will be performed only in the active window.
388      * </p>
389      *
390      * @param focus The focus to find. One of {@link AccessibilityNodeInfo#FOCUS_INPUT} or
391      *         {@link AccessibilityNodeInfo#FOCUS_ACCESSIBILITY}.
392      * @return The node info of the focused view or null.
393      *
394      * @see AccessibilityNodeInfo#FOCUS_INPUT
395      * @see AccessibilityNodeInfo#FOCUS_ACCESSIBILITY
396      */
findFocus(int focus)397     public AccessibilityNodeInfo findFocus(int focus) {
398         return AccessibilityInteractionClient.getInstance().findFocus(mConnectionId,
399                 AccessibilityWindowInfo.ANY_WINDOW_ID, AccessibilityNodeInfo.ROOT_NODE_ID, focus);
400     }
401 
402     /**
403      * Gets the an {@link AccessibilityServiceInfo} describing this UiAutomation.
404      * This method is useful if one wants to change some of the dynamically
405      * configurable properties at runtime.
406      *
407      * @return The accessibility service info.
408      *
409      * @see AccessibilityServiceInfo
410      */
getServiceInfo()411     public final AccessibilityServiceInfo getServiceInfo() {
412         final IAccessibilityServiceConnection connection;
413         synchronized (mLock) {
414             throwIfNotConnectedLocked();
415             connection = AccessibilityInteractionClient.getInstance()
416                     .getConnection(mConnectionId);
417         }
418         // Calling out without a lock held.
419         if (connection != null) {
420             try {
421                 return connection.getServiceInfo();
422             } catch (RemoteException re) {
423                 Log.w(LOG_TAG, "Error while getting AccessibilityServiceInfo", re);
424             }
425         }
426         return null;
427     }
428 
429     /**
430      * Sets the {@link AccessibilityServiceInfo} that describes how this
431      * UiAutomation will be handled by the platform accessibility layer.
432      *
433      * @param info The info.
434      *
435      * @see AccessibilityServiceInfo
436      */
setServiceInfo(AccessibilityServiceInfo info)437     public final void setServiceInfo(AccessibilityServiceInfo info) {
438         final IAccessibilityServiceConnection connection;
439         synchronized (mLock) {
440             throwIfNotConnectedLocked();
441             AccessibilityInteractionClient.getInstance().clearCache();
442             connection = AccessibilityInteractionClient.getInstance()
443                     .getConnection(mConnectionId);
444         }
445         // Calling out without a lock held.
446         if (connection != null) {
447             try {
448                 connection.setServiceInfo(info);
449             } catch (RemoteException re) {
450                 Log.w(LOG_TAG, "Error while setting AccessibilityServiceInfo", re);
451             }
452         }
453     }
454 
455     /**
456      * Gets the windows on the screen. This method returns only the windows
457      * that a sighted user can interact with, as opposed to all windows.
458      * For example, if there is a modal dialog shown and the user cannot touch
459      * anything behind it, then only the modal window will be reported
460      * (assuming it is the top one). For convenience the returned windows
461      * are ordered in a descending layer order, which is the windows that
462      * are higher in the Z-order are reported first.
463      * <p>
464      * <strong>Note:</strong> In order to access the windows you have to opt-in
465      * to retrieve the interactive windows by setting the
466      * {@link AccessibilityServiceInfo#FLAG_RETRIEVE_INTERACTIVE_WINDOWS} flag.
467      * </p>
468      *
469      * @return The windows if there are windows such, otherwise an empty list.
470      */
getWindows()471     public List<AccessibilityWindowInfo> getWindows() {
472         final int connectionId;
473         synchronized (mLock) {
474             throwIfNotConnectedLocked();
475             connectionId = mConnectionId;
476         }
477         // Calling out without a lock held.
478         return AccessibilityInteractionClient.getInstance()
479                 .getWindows(connectionId);
480     }
481 
482     /**
483      * Gets the root {@link AccessibilityNodeInfo} in the active window.
484      *
485      * @return The root info.
486      */
getRootInActiveWindow()487     public AccessibilityNodeInfo getRootInActiveWindow() {
488         final int connectionId;
489         synchronized (mLock) {
490             throwIfNotConnectedLocked();
491             connectionId = mConnectionId;
492         }
493         // Calling out without a lock held.
494         return AccessibilityInteractionClient.getInstance()
495                 .getRootInActiveWindow(connectionId);
496     }
497 
498     /**
499      * A method for injecting an arbitrary input event.
500      * <p>
501      * <strong>Note:</strong> It is caller's responsibility to recycle the event.
502      * </p>
503      * @param event The event to inject.
504      * @param sync Whether to inject the event synchronously.
505      * @return Whether event injection succeeded.
506      */
injectInputEvent(InputEvent event, boolean sync)507     public boolean injectInputEvent(InputEvent event, boolean sync) {
508         synchronized (mLock) {
509             throwIfNotConnectedLocked();
510         }
511         try {
512             if (DEBUG) {
513                 Log.i(LOG_TAG, "Injecting: " + event + " sync: " + sync);
514             }
515             // Calling out without a lock held.
516             return mUiAutomationConnection.injectInputEvent(event, sync);
517         } catch (RemoteException re) {
518             Log.e(LOG_TAG, "Error while injecting input event!", re);
519         }
520         return false;
521     }
522 
523     /**
524      * Sets the device rotation. A client can freeze the rotation in
525      * desired state or freeze the rotation to its current state or
526      * unfreeze the rotation (rotating the device changes its rotation
527      * state).
528      *
529      * @param rotation The desired rotation.
530      * @return Whether the rotation was set successfully.
531      *
532      * @see #ROTATION_FREEZE_0
533      * @see #ROTATION_FREEZE_90
534      * @see #ROTATION_FREEZE_180
535      * @see #ROTATION_FREEZE_270
536      * @see #ROTATION_FREEZE_CURRENT
537      * @see #ROTATION_UNFREEZE
538      */
setRotation(int rotation)539     public boolean setRotation(int rotation) {
540         synchronized (mLock) {
541             throwIfNotConnectedLocked();
542         }
543         switch (rotation) {
544             case ROTATION_FREEZE_0:
545             case ROTATION_FREEZE_90:
546             case ROTATION_FREEZE_180:
547             case ROTATION_FREEZE_270:
548             case ROTATION_UNFREEZE:
549             case ROTATION_FREEZE_CURRENT: {
550                 try {
551                     // Calling out without a lock held.
552                     mUiAutomationConnection.setRotation(rotation);
553                     return true;
554                 } catch (RemoteException re) {
555                     Log.e(LOG_TAG, "Error while setting rotation!", re);
556                 }
557             } return false;
558             default: {
559                 throw new IllegalArgumentException("Invalid rotation.");
560             }
561         }
562     }
563 
564     /**
565      * Executes a command and waits for a specific accessibility event up to a
566      * given wait timeout. To detect a sequence of events one can implement a
567      * filter that keeps track of seen events of the expected sequence and
568      * returns true after the last event of that sequence is received.
569      * <p>
570      * <strong>Note:</strong> It is caller's responsibility to recycle the returned event.
571      * </p>
572      * @param command The command to execute.
573      * @param filter Filter that recognizes the expected event.
574      * @param timeoutMillis The wait timeout in milliseconds.
575      *
576      * @throws TimeoutException If the expected event is not received within the timeout.
577      */
executeAndWaitForEvent(Runnable command, AccessibilityEventFilter filter, long timeoutMillis)578     public AccessibilityEvent executeAndWaitForEvent(Runnable command,
579             AccessibilityEventFilter filter, long timeoutMillis) throws TimeoutException {
580         // Acquire the lock and prepare for receiving events.
581         synchronized (mLock) {
582             throwIfNotConnectedLocked();
583             mEventQueue.clear();
584             // Prepare to wait for an event.
585             mWaitingForEventDelivery = true;
586         }
587 
588         // Note: We have to release the lock since calling out with this lock held
589         // can bite. We will correctly filter out events from other interactions,
590         // so starting to collect events before running the action is just fine.
591 
592         // We will ignore events from previous interactions.
593         final long executionStartTimeMillis = SystemClock.uptimeMillis();
594         // Execute the command *without* the lock being held.
595         command.run();
596 
597         List<AccessibilityEvent> receivedEvents = new ArrayList<>();
598 
599         // Acquire the lock and wait for the event.
600         try {
601             // Wait for the event.
602             final long startTimeMillis = SystemClock.uptimeMillis();
603             while (true) {
604                 List<AccessibilityEvent> localEvents = new ArrayList<>();
605                 synchronized (mLock) {
606                     localEvents.addAll(mEventQueue);
607                     mEventQueue.clear();
608                 }
609                 // Drain the event queue
610                 while (!localEvents.isEmpty()) {
611                     AccessibilityEvent event = localEvents.remove(0);
612                     // Ignore events from previous interactions.
613                     if (event.getEventTime() < executionStartTimeMillis) {
614                         continue;
615                     }
616                     if (filter.accept(event)) {
617                         return event;
618                     }
619                     receivedEvents.add(event);
620                 }
621                 // Check if timed out and if not wait.
622                 final long elapsedTimeMillis = SystemClock.uptimeMillis() - startTimeMillis;
623                 final long remainingTimeMillis = timeoutMillis - elapsedTimeMillis;
624                 if (remainingTimeMillis <= 0) {
625                     throw new TimeoutException("Expected event not received within: "
626                             + timeoutMillis + " ms among: " + receivedEvents);
627                 }
628                 synchronized (mLock) {
629                     if (mEventQueue.isEmpty()) {
630                         try {
631                             mLock.wait(remainingTimeMillis);
632                         } catch (InterruptedException ie) {
633                             /* ignore */
634                         }
635                     }
636                 }
637             }
638         } finally {
639             int size = receivedEvents.size();
640             for (int i = 0; i < size; i++) {
641                 receivedEvents.get(i).recycle();
642             }
643 
644             synchronized (mLock) {
645                 mWaitingForEventDelivery = false;
646                 mEventQueue.clear();
647                 mLock.notifyAll();
648             }
649         }
650     }
651 
652     /**
653      * Waits for the accessibility event stream to become idle, which is not to
654      * have received an accessibility event within <code>idleTimeoutMillis</code>.
655      * The total time spent to wait for an idle accessibility event stream is bounded
656      * by the <code>globalTimeoutMillis</code>.
657      *
658      * @param idleTimeoutMillis The timeout in milliseconds between two events
659      *            to consider the device idle.
660      * @param globalTimeoutMillis The maximal global timeout in milliseconds in
661      *            which to wait for an idle state.
662      *
663      * @throws TimeoutException If no idle state was detected within
664      *            <code>globalTimeoutMillis.</code>
665      */
waitForIdle(long idleTimeoutMillis, long globalTimeoutMillis)666     public void waitForIdle(long idleTimeoutMillis, long globalTimeoutMillis)
667             throws TimeoutException {
668         synchronized (mLock) {
669             throwIfNotConnectedLocked();
670 
671             final long startTimeMillis = SystemClock.uptimeMillis();
672             if (mLastEventTimeMillis <= 0) {
673                 mLastEventTimeMillis = startTimeMillis;
674             }
675 
676             while (true) {
677                 final long currentTimeMillis = SystemClock.uptimeMillis();
678                 // Did we get idle state within the global timeout?
679                 final long elapsedGlobalTimeMillis = currentTimeMillis - startTimeMillis;
680                 final long remainingGlobalTimeMillis =
681                         globalTimeoutMillis - elapsedGlobalTimeMillis;
682                 if (remainingGlobalTimeMillis <= 0) {
683                     throw new TimeoutException("No idle state with idle timeout: "
684                             + idleTimeoutMillis + " within global timeout: "
685                             + globalTimeoutMillis);
686                 }
687                 // Did we get an idle state within the idle timeout?
688                 final long elapsedIdleTimeMillis = currentTimeMillis - mLastEventTimeMillis;
689                 final long remainingIdleTimeMillis = idleTimeoutMillis - elapsedIdleTimeMillis;
690                 if (remainingIdleTimeMillis <= 0) {
691                     return;
692                 }
693                 try {
694                      mLock.wait(remainingIdleTimeMillis);
695                 } catch (InterruptedException ie) {
696                      /* ignore */
697                 }
698             }
699         }
700     }
701 
702     /**
703      * Takes a screenshot.
704      *
705      * @return The screenshot bitmap on success, null otherwise.
706      */
takeScreenshot()707     public Bitmap takeScreenshot() {
708         synchronized (mLock) {
709             throwIfNotConnectedLocked();
710         }
711         Display display = DisplayManagerGlobal.getInstance()
712                 .getRealDisplay(Display.DEFAULT_DISPLAY);
713         Point displaySize = new Point();
714         display.getRealSize(displaySize);
715 
716         int rotation = display.getRotation();
717 
718         // Take the screenshot
719         Bitmap screenShot = null;
720         try {
721             // Calling out without a lock held.
722             screenShot = mUiAutomationConnection.takeScreenshot(
723                     new Rect(0, 0, displaySize.x, displaySize.y), rotation);
724             if (screenShot == null) {
725                 return null;
726             }
727         } catch (RemoteException re) {
728             Log.e(LOG_TAG, "Error while taking screnshot!", re);
729             return null;
730         }
731 
732         // Optimization
733         screenShot.setHasAlpha(false);
734 
735         return screenShot;
736     }
737 
738     /**
739      * Sets whether this UiAutomation to run in a "monkey" mode. Applications can query whether
740      * they are executed in a "monkey" mode, i.e. run by a test framework, and avoid doing
741      * potentially undesirable actions such as calling 911 or posting on public forums etc.
742      *
743      * @param enable whether to run in a "monkey" mode or not. Default is not.
744      * @see ActivityManager#isUserAMonkey()
745      */
setRunAsMonkey(boolean enable)746     public void setRunAsMonkey(boolean enable) {
747         synchronized (mLock) {
748             throwIfNotConnectedLocked();
749         }
750         try {
751             ActivityManager.getService().setUserIsMonkey(enable);
752         } catch (RemoteException re) {
753             Log.e(LOG_TAG, "Error while setting run as monkey!", re);
754         }
755     }
756 
757     /**
758      * Clears the frame statistics for the content of a given window. These
759      * statistics contain information about the most recently rendered content
760      * frames.
761      *
762      * @param windowId The window id.
763      * @return Whether the window is present and its frame statistics
764      *         were cleared.
765      *
766      * @see android.view.WindowContentFrameStats
767      * @see #getWindowContentFrameStats(int)
768      * @see #getWindows()
769      * @see AccessibilityWindowInfo#getId() AccessibilityWindowInfo.getId()
770      */
clearWindowContentFrameStats(int windowId)771     public boolean clearWindowContentFrameStats(int windowId) {
772         synchronized (mLock) {
773             throwIfNotConnectedLocked();
774         }
775         try {
776             if (DEBUG) {
777                 Log.i(LOG_TAG, "Clearing content frame stats for window: " + windowId);
778             }
779             // Calling out without a lock held.
780             return mUiAutomationConnection.clearWindowContentFrameStats(windowId);
781         } catch (RemoteException re) {
782             Log.e(LOG_TAG, "Error clearing window content frame stats!", re);
783         }
784         return false;
785     }
786 
787     /**
788      * Gets the frame statistics for a given window. These statistics contain
789      * information about the most recently rendered content frames.
790      * <p>
791      * A typical usage requires clearing the window frame statistics via {@link
792      * #clearWindowContentFrameStats(int)} followed by an interaction with the UI and
793      * finally getting the window frame statistics via calling this method.
794      * </p>
795      * <pre>
796      * // Assume we have at least one window.
797      * final int windowId = getWindows().get(0).getId();
798      *
799      * // Start with a clean slate.
800      * uiAutimation.clearWindowContentFrameStats(windowId);
801      *
802      * // Do stuff with the UI.
803      *
804      * // Get the frame statistics.
805      * WindowContentFrameStats stats = uiAutomation.getWindowContentFrameStats(windowId);
806      * </pre>
807      *
808      * @param windowId The window id.
809      * @return The window frame statistics, or null if the window is not present.
810      *
811      * @see android.view.WindowContentFrameStats
812      * @see #clearWindowContentFrameStats(int)
813      * @see #getWindows()
814      * @see AccessibilityWindowInfo#getId() AccessibilityWindowInfo.getId()
815      */
getWindowContentFrameStats(int windowId)816     public WindowContentFrameStats getWindowContentFrameStats(int windowId) {
817         synchronized (mLock) {
818             throwIfNotConnectedLocked();
819         }
820         try {
821             if (DEBUG) {
822                 Log.i(LOG_TAG, "Getting content frame stats for window: " + windowId);
823             }
824             // Calling out without a lock held.
825             return mUiAutomationConnection.getWindowContentFrameStats(windowId);
826         } catch (RemoteException re) {
827             Log.e(LOG_TAG, "Error getting window content frame stats!", re);
828         }
829         return null;
830     }
831 
832     /**
833      * Clears the window animation rendering statistics. These statistics contain
834      * information about the most recently rendered window animation frames, i.e.
835      * for window transition animations.
836      *
837      * @see android.view.WindowAnimationFrameStats
838      * @see #getWindowAnimationFrameStats()
839      * @see android.R.styleable#WindowAnimation
840      */
clearWindowAnimationFrameStats()841     public void clearWindowAnimationFrameStats() {
842         synchronized (mLock) {
843             throwIfNotConnectedLocked();
844         }
845         try {
846             if (DEBUG) {
847                 Log.i(LOG_TAG, "Clearing window animation frame stats");
848             }
849             // Calling out without a lock held.
850             mUiAutomationConnection.clearWindowAnimationFrameStats();
851         } catch (RemoteException re) {
852             Log.e(LOG_TAG, "Error clearing window animation frame stats!", re);
853         }
854     }
855 
856     /**
857      * Gets the window animation frame statistics. These statistics contain
858      * information about the most recently rendered window animation frames, i.e.
859      * for window transition animations.
860      *
861      * <p>
862      * A typical usage requires clearing the window animation frame statistics via
863      * {@link #clearWindowAnimationFrameStats()} followed by an interaction that causes
864      * a window transition which uses a window animation and finally getting the window
865      * animation frame statistics by calling this method.
866      * </p>
867      * <pre>
868      * // Start with a clean slate.
869      * uiAutimation.clearWindowAnimationFrameStats();
870      *
871      * // Do stuff to trigger a window transition.
872      *
873      * // Get the frame statistics.
874      * WindowAnimationFrameStats stats = uiAutomation.getWindowAnimationFrameStats();
875      * </pre>
876      *
877      * @return The window animation frame statistics.
878      *
879      * @see android.view.WindowAnimationFrameStats
880      * @see #clearWindowAnimationFrameStats()
881      * @see android.R.styleable#WindowAnimation
882      */
getWindowAnimationFrameStats()883     public WindowAnimationFrameStats getWindowAnimationFrameStats() {
884         synchronized (mLock) {
885             throwIfNotConnectedLocked();
886         }
887         try {
888             if (DEBUG) {
889                 Log.i(LOG_TAG, "Getting window animation frame stats");
890             }
891             // Calling out without a lock held.
892             return mUiAutomationConnection.getWindowAnimationFrameStats();
893         } catch (RemoteException re) {
894             Log.e(LOG_TAG, "Error getting window animation frame stats!", re);
895         }
896         return null;
897     }
898 
899     /**
900      * Grants a runtime permission to a package.
901      * @param packageName The package to which to grant.
902      * @param permission The permission to grant.
903      * @throws SecurityException if unable to grant the permission.
904      */
grantRuntimePermission(String packageName, String permission)905     public void grantRuntimePermission(String packageName, String permission) {
906         grantRuntimePermissionAsUser(packageName, permission, android.os.Process.myUserHandle());
907     }
908 
909     /**
910      * @deprecated replaced by
911      *             {@link #grantRuntimePermissionAsUser(String, String, UserHandle)}.
912      * @hide
913      */
914     @Deprecated
915     @TestApi
grantRuntimePermission(String packageName, String permission, UserHandle userHandle)916     public boolean grantRuntimePermission(String packageName, String permission,
917             UserHandle userHandle) {
918         grantRuntimePermissionAsUser(packageName, permission, userHandle);
919         return true;
920     }
921 
922     /**
923      * Grants a runtime permission to a package for a user.
924      * @param packageName The package to which to grant.
925      * @param permission The permission to grant.
926      * @throws SecurityException if unable to grant the permission.
927      */
grantRuntimePermissionAsUser(String packageName, String permission, UserHandle userHandle)928     public void grantRuntimePermissionAsUser(String packageName, String permission,
929             UserHandle userHandle) {
930         synchronized (mLock) {
931             throwIfNotConnectedLocked();
932         }
933         try {
934             if (DEBUG) {
935                 Log.i(LOG_TAG, "Granting runtime permission");
936             }
937             // Calling out without a lock held.
938             mUiAutomationConnection.grantRuntimePermission(packageName,
939                     permission, userHandle.getIdentifier());
940         } catch (Exception e) {
941             throw new SecurityException("Error granting runtime permission", e);
942         }
943     }
944 
945     /**
946      * Revokes a runtime permission from a package.
947      * @param packageName The package to which to grant.
948      * @param permission The permission to grant.
949      * @throws SecurityException if unable to revoke the permission.
950      */
revokeRuntimePermission(String packageName, String permission)951     public void revokeRuntimePermission(String packageName, String permission) {
952         revokeRuntimePermissionAsUser(packageName, permission, android.os.Process.myUserHandle());
953     }
954 
955     /**
956      * @deprecated replaced by
957      *             {@link #revokeRuntimePermissionAsUser(String, String, UserHandle)}.
958      * @hide
959      */
960     @Deprecated
961     @TestApi
revokeRuntimePermission(String packageName, String permission, UserHandle userHandle)962     public boolean revokeRuntimePermission(String packageName, String permission,
963             UserHandle userHandle) {
964         revokeRuntimePermissionAsUser(packageName, permission, userHandle);
965         return true;
966     }
967 
968     /**
969      * Revokes a runtime permission from a package.
970      * @param packageName The package to which to grant.
971      * @param permission The permission to grant.
972      * @throws SecurityException if unable to revoke the permission.
973      */
revokeRuntimePermissionAsUser(String packageName, String permission, UserHandle userHandle)974     public void revokeRuntimePermissionAsUser(String packageName, String permission,
975             UserHandle userHandle) {
976         synchronized (mLock) {
977             throwIfNotConnectedLocked();
978         }
979         try {
980             if (DEBUG) {
981                 Log.i(LOG_TAG, "Revoking runtime permission");
982             }
983             // Calling out without a lock held.
984             mUiAutomationConnection.revokeRuntimePermission(packageName,
985                     permission, userHandle.getIdentifier());
986         } catch (Exception e) {
987             throw new SecurityException("Error granting runtime permission", e);
988         }
989     }
990 
991     /**
992      * Executes a shell command. This method returns a file descriptor that points
993      * to the standard output stream. The command execution is similar to running
994      * "adb shell <command>" from a host connected to the device.
995      * <p>
996      * <strong>Note:</strong> It is your responsibility to close the returned file
997      * descriptor once you are done reading.
998      * </p>
999      *
1000      * @param command The command to execute.
1001      * @return A file descriptor to the standard output stream.
1002      */
executeShellCommand(String command)1003     public ParcelFileDescriptor executeShellCommand(String command) {
1004         synchronized (mLock) {
1005             throwIfNotConnectedLocked();
1006         }
1007         warnIfBetterCommand(command);
1008 
1009         ParcelFileDescriptor source = null;
1010         ParcelFileDescriptor sink = null;
1011 
1012         try {
1013             ParcelFileDescriptor[] pipe = ParcelFileDescriptor.createPipe();
1014             source = pipe[0];
1015             sink = pipe[1];
1016 
1017             // Calling out without a lock held.
1018             mUiAutomationConnection.executeShellCommand(command, sink, null);
1019         } catch (IOException ioe) {
1020             Log.e(LOG_TAG, "Error executing shell command!", ioe);
1021         } catch (RemoteException re) {
1022             Log.e(LOG_TAG, "Error executing shell command!", re);
1023         } finally {
1024             IoUtils.closeQuietly(sink);
1025         }
1026 
1027         return source;
1028     }
1029 
1030     /**
1031      * Executes a shell command. This method returns two file descriptors,
1032      * one that points to the standard output stream (element at index 0), and one that points
1033      * to the standard input stream (element at index 1). The command execution is similar
1034      * to running "adb shell <command>" from a host connected to the device.
1035      * <p>
1036      * <strong>Note:</strong> It is your responsibility to close the returned file
1037      * descriptors once you are done reading/writing.
1038      * </p>
1039      *
1040      * @param command The command to execute.
1041      * @return File descriptors (out, in) to the standard output/input streams.
1042      *
1043      * @hide
1044      */
1045     @TestApi
executeShellCommandRw(String command)1046     public ParcelFileDescriptor[] executeShellCommandRw(String command) {
1047         synchronized (mLock) {
1048             throwIfNotConnectedLocked();
1049         }
1050         warnIfBetterCommand(command);
1051 
1052         ParcelFileDescriptor source_read = null;
1053         ParcelFileDescriptor sink_read = null;
1054 
1055         ParcelFileDescriptor source_write = null;
1056         ParcelFileDescriptor sink_write = null;
1057 
1058         try {
1059             ParcelFileDescriptor[] pipe_read = ParcelFileDescriptor.createPipe();
1060             source_read = pipe_read[0];
1061             sink_read = pipe_read[1];
1062 
1063             ParcelFileDescriptor[] pipe_write = ParcelFileDescriptor.createPipe();
1064             source_write = pipe_write[0];
1065             sink_write = pipe_write[1];
1066 
1067             // Calling out without a lock held.
1068             mUiAutomationConnection.executeShellCommand(command, sink_read, source_write);
1069         } catch (IOException ioe) {
1070             Log.e(LOG_TAG, "Error executing shell command!", ioe);
1071         } catch (RemoteException re) {
1072             Log.e(LOG_TAG, "Error executing shell command!", re);
1073         } finally {
1074             IoUtils.closeQuietly(sink_read);
1075             IoUtils.closeQuietly(source_write);
1076         }
1077 
1078         ParcelFileDescriptor[] result = new ParcelFileDescriptor[2];
1079         result[0] = source_read;
1080         result[1] = sink_write;
1081         return result;
1082     }
1083 
getDegreesForRotation(int value)1084     private static float getDegreesForRotation(int value) {
1085         switch (value) {
1086             case Surface.ROTATION_90: {
1087                 return 360f - 90f;
1088             }
1089             case Surface.ROTATION_180: {
1090                 return 360f - 180f;
1091             }
1092             case Surface.ROTATION_270: {
1093                 return 360f - 270f;
1094             } default: {
1095                 return 0;
1096             }
1097         }
1098     }
1099 
isConnectedLocked()1100     private boolean isConnectedLocked() {
1101         return mConnectionId != CONNECTION_ID_UNDEFINED;
1102     }
1103 
throwIfConnectedLocked()1104     private void throwIfConnectedLocked() {
1105         if (mConnectionId != CONNECTION_ID_UNDEFINED) {
1106             throw new IllegalStateException("UiAutomation not connected!");
1107         }
1108     }
1109 
throwIfNotConnectedLocked()1110     private void throwIfNotConnectedLocked() {
1111         if (!isConnectedLocked()) {
1112             throw new IllegalStateException("UiAutomation not connected!");
1113         }
1114     }
1115 
warnIfBetterCommand(String cmd)1116     private void warnIfBetterCommand(String cmd) {
1117         if (cmd.startsWith("pm grant ")) {
1118             Log.w(LOG_TAG, "UiAutomation.grantRuntimePermission() "
1119                     + "is more robust and should be used instead of 'pm grant'");
1120         } else if (cmd.startsWith("pm revoke ")) {
1121             Log.w(LOG_TAG, "UiAutomation.revokeRuntimePermission() "
1122                     + "is more robust and should be used instead of 'pm revoke'");
1123         }
1124     }
1125 
1126     private class IAccessibilityServiceClientImpl extends IAccessibilityServiceClientWrapper {
1127 
IAccessibilityServiceClientImpl(Looper looper)1128         public IAccessibilityServiceClientImpl(Looper looper) {
1129             super(null, looper, new Callbacks() {
1130                 @Override
1131                 public void init(int connectionId, IBinder windowToken) {
1132                     synchronized (mLock) {
1133                         mConnectionId = connectionId;
1134                         mLock.notifyAll();
1135                     }
1136                 }
1137 
1138                 @Override
1139                 public void onServiceConnected() {
1140                     /* do nothing */
1141                 }
1142 
1143                 @Override
1144                 public void onInterrupt() {
1145                     /* do nothing */
1146                 }
1147 
1148                 @Override
1149                 public boolean onGesture(int gestureId) {
1150                     /* do nothing */
1151                     return false;
1152                 }
1153 
1154                 @Override
1155                 public void onAccessibilityEvent(AccessibilityEvent event) {
1156                     final OnAccessibilityEventListener listener;
1157                     synchronized (mLock) {
1158                         mLastEventTimeMillis = event.getEventTime();
1159                         if (mWaitingForEventDelivery) {
1160                             mEventQueue.add(AccessibilityEvent.obtain(event));
1161                         }
1162                         mLock.notifyAll();
1163                         listener = mOnAccessibilityEventListener;
1164                     }
1165                     if (listener != null) {
1166                         // Calling out only without a lock held.
1167                         mLocalCallbackHandler.post(PooledLambda.obtainRunnable(
1168                                 OnAccessibilityEventListener::onAccessibilityEvent,
1169                                 listener, AccessibilityEvent.obtain(event))
1170                                 .recycleOnUse());
1171                     }
1172                 }
1173 
1174                 @Override
1175                 public boolean onKeyEvent(KeyEvent event) {
1176                     return false;
1177                 }
1178 
1179                 @Override
1180                 public void onMagnificationChanged(@NonNull Region region,
1181                         float scale, float centerX, float centerY) {
1182                     /* do nothing */
1183                 }
1184 
1185                 @Override
1186                 public void onSoftKeyboardShowModeChanged(int showMode) {
1187                     /* do nothing */
1188                 }
1189 
1190                 @Override
1191                 public void onPerformGestureResult(int sequence, boolean completedSuccessfully) {
1192                     /* do nothing */
1193                 }
1194 
1195                 @Override
1196                 public void onFingerprintCapturingGesturesChanged(boolean active) {
1197                     /* do nothing */
1198                 }
1199 
1200                 @Override
1201                 public void onFingerprintGesture(int gesture) {
1202                     /* do nothing */
1203                 }
1204 
1205                 @Override
1206                 public void onAccessibilityButtonClicked() {
1207                     /* do nothing */
1208                 }
1209 
1210                 @Override
1211                 public void onAccessibilityButtonAvailabilityChanged(boolean available) {
1212                     /* do nothing */
1213                 }
1214             });
1215         }
1216     }
1217 }
1218