1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.uiautomator.core; 17 18 import android.app.UiAutomation.OnAccessibilityEventListener; 19 import android.os.SystemClock; 20 import android.util.Log; 21 import android.view.accessibility.AccessibilityEvent; 22 import android.view.accessibility.AccessibilityNodeInfo; 23 24 25 /** 26 * The QueryController main purpose is to translate a {@link UiSelector} selectors to 27 * {@link AccessibilityNodeInfo}. This is all this controller does. 28 */ 29 class QueryController { 30 31 private static final String LOG_TAG = QueryController.class.getSimpleName(); 32 33 private static final boolean DEBUG = Log.isLoggable(LOG_TAG, Log.DEBUG); 34 private static final boolean VERBOSE = Log.isLoggable(LOG_TAG, Log.VERBOSE); 35 36 private final UiAutomatorBridge mUiAutomatorBridge; 37 38 private final Object mLock = new Object(); 39 40 private String mLastActivityName = null; 41 42 // During a pattern selector search, the recursive pattern search 43 // methods will track their counts and indexes here. 44 private int mPatternCounter = 0; 45 private int mPatternIndexer = 0; 46 47 // These help show each selector's search context as it relates to the previous sub selector 48 // matched. When a compound selector fails, it is hard to tell which part of it is failing. 49 // Seeing how a selector is being parsed and which sub selector failed within a long list 50 // of compound selectors is very helpful. 51 private int mLogIndent = 0; 52 private int mLogParentIndent = 0; 53 54 private String mLastTraversedText = ""; 55 QueryController(UiAutomatorBridge bridge)56 public QueryController(UiAutomatorBridge bridge) { 57 mUiAutomatorBridge = bridge; 58 bridge.setOnAccessibilityEventListener(new OnAccessibilityEventListener() { 59 @Override 60 public void onAccessibilityEvent(AccessibilityEvent event) { 61 synchronized (mLock) { 62 switch(event.getEventType()) { 63 case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: 64 // don't trust event.getText(), check for nulls 65 if (event.getText() != null && event.getText().size() > 0) { 66 if(event.getText().get(0) != null) 67 mLastActivityName = event.getText().get(0).toString(); 68 } 69 break; 70 case AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY: 71 // don't trust event.getText(), check for nulls 72 if (event.getText() != null && event.getText().size() > 0) 73 if(event.getText().get(0) != null) 74 mLastTraversedText = event.getText().get(0).toString(); 75 if (DEBUG) 76 Log.d(LOG_TAG, "Last text selection reported: " + 77 mLastTraversedText); 78 break; 79 } 80 mLock.notifyAll(); 81 } 82 } 83 }); 84 } 85 86 /** 87 * Returns the last text selection reported by accessibility 88 * event TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY. One way to cause 89 * this event is using a DPad arrows to focus on UI elements. 90 */ getLastTraversedText()91 public String getLastTraversedText() { 92 mUiAutomatorBridge.waitForIdle(); 93 synchronized (mLock) { 94 if (mLastTraversedText.length() > 0) { 95 return mLastTraversedText; 96 } 97 } 98 return null; 99 } 100 101 /** 102 * Clears the last text selection value saved from the TYPE_VIEW_TEXT_SELECTION_CHANGED 103 * event 104 */ clearLastTraversedText()105 public void clearLastTraversedText() { 106 mUiAutomatorBridge.waitForIdle(); 107 synchronized (mLock) { 108 mLastTraversedText = ""; 109 } 110 } 111 initializeNewSearch()112 private void initializeNewSearch() { 113 mPatternCounter = 0; 114 mPatternIndexer = 0; 115 mLogIndent = 0; 116 mLogParentIndent = 0; 117 } 118 119 /** 120 * Counts the instances of the selector group. The selector must be in the following 121 * format: [container_selector, PATTERN=[INSTANCE=x, PATTERN=[the_pattern]] 122 * where the container_selector is used to find the containment region to search for patterns 123 * and the INSTANCE=x is the instance of the_pattern to return. 124 * @param selector 125 * @return number of pattern matches. Returns 0 for all other cases. 126 */ getPatternCount(UiSelector selector)127 public int getPatternCount(UiSelector selector) { 128 findAccessibilityNodeInfo(selector, true /*counting*/); 129 return mPatternCounter; 130 } 131 132 /** 133 * Main search method for translating By selectors to AccessibilityInfoNodes 134 * @param selector 135 * @return AccessibilityNodeInfo 136 */ findAccessibilityNodeInfo(UiSelector selector)137 public AccessibilityNodeInfo findAccessibilityNodeInfo(UiSelector selector) { 138 return findAccessibilityNodeInfo(selector, false); 139 } 140 findAccessibilityNodeInfo(UiSelector selector, boolean isCounting)141 protected AccessibilityNodeInfo findAccessibilityNodeInfo(UiSelector selector, 142 boolean isCounting) { 143 mUiAutomatorBridge.waitForIdle(); 144 initializeNewSearch(); 145 146 if (DEBUG) 147 Log.d(LOG_TAG, "Searching: " + selector); 148 149 synchronized (mLock) { 150 AccessibilityNodeInfo rootNode = getRootNode(); 151 if (rootNode == null) { 152 Log.e(LOG_TAG, "Cannot proceed when root node is null. Aborted search"); 153 return null; 154 } 155 156 // Copy so that we don't modify the original's sub selectors 157 UiSelector uiSelector = new UiSelector(selector); 158 return translateCompoundSelector(uiSelector, rootNode, isCounting); 159 } 160 } 161 162 /** 163 * Gets the root node from accessibility and if it fails to get one it will 164 * retry every 250ms for up to 1000ms. 165 * @return null if no root node is obtained 166 */ getRootNode()167 protected AccessibilityNodeInfo getRootNode() { 168 final int maxRetry = 4; 169 final long waitInterval = 250; 170 AccessibilityNodeInfo rootNode = null; 171 for(int x = 0; x < maxRetry; x++) { 172 rootNode = mUiAutomatorBridge.getRootInActiveWindow(); 173 if (rootNode != null) { 174 return rootNode; 175 } 176 if(x < maxRetry - 1) { 177 Log.e(LOG_TAG, "Got null root node from accessibility - Retrying..."); 178 SystemClock.sleep(waitInterval); 179 } 180 } 181 return rootNode; 182 } 183 184 /** 185 * A compoundSelector encapsulate both Regular and Pattern selectors. The formats follows: 186 * <p/> 187 * regular_selector = By[attributes... CHILD=By[attributes... CHILD=By[....]]] 188 * <br/> 189 * pattern_selector = ...CONTAINER=By[..] PATTERN=By[instance=x PATTERN=[regular_selector] 190 * <br/> 191 * compound_selector = [regular_selector [pattern_selector]] 192 * <p/> 193 * regular_selectors are the most common form of selectors and the search for them 194 * is straightforward. On the other hand pattern_selectors requires search to be 195 * performed as in regular_selector but where regular_selector search returns immediately 196 * upon a successful match, the search for pattern_selector continues until the 197 * requested matched _instance_ of that pattern is matched. 198 * <p/> 199 * Counting UI objects requires using pattern_selectors. The counting search is the same 200 * as a pattern_search however we're not looking to match an instance of the pattern but 201 * rather continuously walking the accessibility node hierarchy while counting matched 202 * patterns, until the end of the tree. 203 * <p/> 204 * If both present, order of parsing begins with CONTAINER followed by PATTERN then the 205 * top most selector is processed as regular_selector within the context of the previous 206 * CONTAINER and its PATTERN information. If neither is present then the top selector is 207 * directly treated as regular_selector. So the presence of a CONTAINER and PATTERN within 208 * a selector simply dictates that the selector matching will be constraint to the sub tree 209 * node where the CONTAINER and its child PATTERN have identified. 210 * @param selector 211 * @param fromNode 212 * @param isCounting 213 * @return AccessibilityNodeInfo 214 */ translateCompoundSelector(UiSelector selector, AccessibilityNodeInfo fromNode, boolean isCounting)215 private AccessibilityNodeInfo translateCompoundSelector(UiSelector selector, 216 AccessibilityNodeInfo fromNode, boolean isCounting) { 217 218 // Start translating compound selectors by translating the regular_selector first 219 // The regular_selector is then used as a container for any optional pattern_selectors 220 // that may or may not be specified. 221 if(selector.hasContainerSelector()) 222 // nested pattern selectors 223 if(selector.getContainerSelector().hasContainerSelector()) { 224 fromNode = translateCompoundSelector( 225 selector.getContainerSelector(), fromNode, false); 226 initializeNewSearch(); 227 } else 228 fromNode = translateReqularSelector(selector.getContainerSelector(), fromNode); 229 else 230 fromNode = translateReqularSelector(selector, fromNode); 231 232 if(fromNode == null) { 233 if (DEBUG) 234 Log.d(LOG_TAG, "Container selector not found: " + selector.dumpToString(false)); 235 return null; 236 } 237 238 if(selector.hasPatternSelector()) { 239 fromNode = translatePatternSelector(selector.getPatternSelector(), 240 fromNode, isCounting); 241 242 if (isCounting) { 243 Log.i(LOG_TAG, String.format( 244 "Counted %d instances of: %s", mPatternCounter, selector)); 245 return null; 246 } else { 247 if(fromNode == null) { 248 if (DEBUG) 249 Log.d(LOG_TAG, "Pattern selector not found: " + 250 selector.dumpToString(false)); 251 return null; 252 } 253 } 254 } 255 256 // translate any additions to the selector that may have been added by tests 257 // with getChild(By selector) after a container and pattern selectors 258 if(selector.hasContainerSelector() || selector.hasPatternSelector()) { 259 if(selector.hasChildSelector() || selector.hasParentSelector()) 260 fromNode = translateReqularSelector(selector, fromNode); 261 } 262 263 if(fromNode == null) { 264 if (DEBUG) 265 Log.d(LOG_TAG, "Object Not Found for selector " + selector); 266 return null; 267 } 268 Log.i(LOG_TAG, String.format("Matched selector: %s <<==>> [%s]", selector, fromNode)); 269 return fromNode; 270 } 271 272 /** 273 * Used by the {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 274 * to translate the regular_selector portion. It has the following format: 275 * <p/> 276 * regular_selector = By[attributes... CHILD=By[attributes... CHILD=By[....]]]<br/> 277 * <p/> 278 * regular_selectors are the most common form of selectors and the search for them 279 * is straightforward. This method will only look for CHILD or PARENT sub selectors. 280 * <p/> 281 * @param selector 282 * @param fromNode 283 * @return AccessibilityNodeInfo if found else null 284 */ translateReqularSelector(UiSelector selector, AccessibilityNodeInfo fromNode)285 private AccessibilityNodeInfo translateReqularSelector(UiSelector selector, 286 AccessibilityNodeInfo fromNode) { 287 288 return findNodeRegularRecursive(selector, fromNode, 0); 289 } 290 findNodeRegularRecursive(UiSelector subSelector, AccessibilityNodeInfo fromNode, int index)291 private AccessibilityNodeInfo findNodeRegularRecursive(UiSelector subSelector, 292 AccessibilityNodeInfo fromNode, int index) { 293 294 if (subSelector.isMatchFor(fromNode, index)) { 295 if (DEBUG) { 296 Log.d(LOG_TAG, formatLog(String.format("%s", 297 subSelector.dumpToString(false)))); 298 } 299 if(subSelector.isLeaf()) { 300 return fromNode; 301 } 302 if(subSelector.hasChildSelector()) { 303 mLogIndent++; // next selector 304 subSelector = subSelector.getChildSelector(); 305 if(subSelector == null) { 306 Log.e(LOG_TAG, "Error: A child selector without content"); 307 return null; // there is an implementation fault 308 } 309 } else if(subSelector.hasParentSelector()) { 310 mLogIndent++; // next selector 311 subSelector = subSelector.getParentSelector(); 312 if(subSelector == null) { 313 Log.e(LOG_TAG, "Error: A parent selector without content"); 314 return null; // there is an implementation fault 315 } 316 // the selector requested we start at this level from 317 // the parent node from the one we just matched 318 fromNode = fromNode.getParent(); 319 if(fromNode == null) 320 return null; 321 } 322 } 323 324 int childCount = fromNode.getChildCount(); 325 boolean hasNullChild = false; 326 for (int i = 0; i < childCount; i++) { 327 AccessibilityNodeInfo childNode = fromNode.getChild(i); 328 if (childNode == null) { 329 Log.w(LOG_TAG, String.format( 330 "AccessibilityNodeInfo returned a null child (%d of %d)", i, childCount)); 331 if (!hasNullChild) { 332 Log.w(LOG_TAG, String.format("parent = %s", fromNode.toString())); 333 } 334 hasNullChild = true; 335 continue; 336 } 337 if (!childNode.isVisibleToUser()) { 338 if (VERBOSE) 339 Log.v(LOG_TAG, 340 String.format("Skipping invisible child: %s", childNode.toString())); 341 continue; 342 } 343 AccessibilityNodeInfo retNode = findNodeRegularRecursive(subSelector, childNode, i); 344 if (retNode != null) { 345 return retNode; 346 } 347 } 348 return null; 349 } 350 351 /** 352 * Used by the {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 353 * to translate the pattern_selector portion. It has the following format: 354 * <p/> 355 * pattern_selector = ... PATTERN=By[instance=x PATTERN=[regular_selector]]<br/> 356 * <p/> 357 * pattern_selectors requires search to be performed as regular_selector but where 358 * regular_selector search returns immediately upon a successful match, the search for 359 * pattern_selector continues until the requested matched instance of that pattern is 360 * encountered. 361 * <p/> 362 * Counting UI objects requires using pattern_selectors. The counting search is the same 363 * as a pattern_search however we're not looking to match an instance of the pattern but 364 * rather continuously walking the accessibility node hierarchy while counting patterns 365 * until the end of the tree. 366 * @param subSelector 367 * @param fromNode 368 * @param isCounting 369 * @return null of node is not found or if counting mode is true. 370 * See {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 371 */ translatePatternSelector(UiSelector subSelector, AccessibilityNodeInfo fromNode, boolean isCounting)372 private AccessibilityNodeInfo translatePatternSelector(UiSelector subSelector, 373 AccessibilityNodeInfo fromNode, boolean isCounting) { 374 375 if(subSelector.hasPatternSelector()) { 376 // Since pattern_selectors are also the type of selectors used when counting, 377 // we check if this is a counting run or an indexing run 378 if(isCounting) 379 //since we're counting, we reset the indexer so to terminates the search when 380 // the end of tree is reached. The count will be in mPatternCount 381 mPatternIndexer = -1; 382 else 383 // terminates the search once we match the pattern's instance 384 mPatternIndexer = subSelector.getInstance(); 385 386 // A pattern is wrapped in a PATTERN[instance=x PATTERN[the_pattern]] 387 subSelector = subSelector.getPatternSelector(); 388 if(subSelector == null) { 389 Log.e(LOG_TAG, "Pattern portion of the selector is null or not defined"); 390 return null; // there is an implementation fault 391 } 392 // save the current indent level as parent indent before pattern searches 393 // begin under the current tree position. 394 mLogParentIndent = ++mLogIndent; 395 return findNodePatternRecursive(subSelector, fromNode, 0, subSelector); 396 } 397 398 Log.e(LOG_TAG, "Selector must have a pattern selector defined"); // implementation fault? 399 return null; 400 } 401 findNodePatternRecursive( UiSelector subSelector, AccessibilityNodeInfo fromNode, int index, UiSelector originalPattern)402 private AccessibilityNodeInfo findNodePatternRecursive( 403 UiSelector subSelector, AccessibilityNodeInfo fromNode, int index, 404 UiSelector originalPattern) { 405 406 if (subSelector.isMatchFor(fromNode, index)) { 407 if(subSelector.isLeaf()) { 408 if(mPatternIndexer == 0) { 409 if (DEBUG) 410 Log.d(LOG_TAG, formatLog( 411 String.format("%s", subSelector.dumpToString(false)))); 412 return fromNode; 413 } else { 414 if (DEBUG) 415 Log.d(LOG_TAG, formatLog( 416 String.format("%s", subSelector.dumpToString(false)))); 417 mPatternCounter++; //count the pattern matched 418 mPatternIndexer--; //decrement until zero for the instance requested 419 420 // At a leaf selector within a group and still not instance matched 421 // then reset the selector to continue search from current position 422 // in the accessibility tree for the next pattern match up until the 423 // pattern index hits 0. 424 subSelector = originalPattern; 425 // starting over with next pattern search so reset to parent level 426 mLogIndent = mLogParentIndent; 427 } 428 } else { 429 if (DEBUG) 430 Log.d(LOG_TAG, formatLog( 431 String.format("%s", subSelector.dumpToString(false)))); 432 433 if(subSelector.hasChildSelector()) { 434 mLogIndent++; // next selector 435 subSelector = subSelector.getChildSelector(); 436 if(subSelector == null) { 437 Log.e(LOG_TAG, "Error: A child selector without content"); 438 return null; 439 } 440 } else if(subSelector.hasParentSelector()) { 441 mLogIndent++; // next selector 442 subSelector = subSelector.getParentSelector(); 443 if(subSelector == null) { 444 Log.e(LOG_TAG, "Error: A parent selector without content"); 445 return null; 446 } 447 fromNode = fromNode.getParent(); 448 if(fromNode == null) 449 return null; 450 } 451 } 452 } 453 454 int childCount = fromNode.getChildCount(); 455 boolean hasNullChild = false; 456 for (int i = 0; i < childCount; i++) { 457 AccessibilityNodeInfo childNode = fromNode.getChild(i); 458 if (childNode == null) { 459 Log.w(LOG_TAG, String.format( 460 "AccessibilityNodeInfo returned a null child (%d of %d)", i, childCount)); 461 if (!hasNullChild) { 462 Log.w(LOG_TAG, String.format("parent = %s", fromNode.toString())); 463 } 464 hasNullChild = true; 465 continue; 466 } 467 if (!childNode.isVisibleToUser()) { 468 if (DEBUG) 469 Log.d(LOG_TAG, 470 String.format("Skipping invisible child: %s", childNode.toString())); 471 continue; 472 } 473 AccessibilityNodeInfo retNode = findNodePatternRecursive( 474 subSelector, childNode, i, originalPattern); 475 if (retNode != null) { 476 return retNode; 477 } 478 } 479 return null; 480 } 481 getAccessibilityRootNode()482 public AccessibilityNodeInfo getAccessibilityRootNode() { 483 return mUiAutomatorBridge.getRootInActiveWindow(); 484 } 485 486 /** 487 * Last activity to report accessibility events. 488 * @deprecated The results returned should be considered unreliable 489 * @return String name of activity 490 */ 491 @Deprecated getCurrentActivityName()492 public String getCurrentActivityName() { 493 mUiAutomatorBridge.waitForIdle(); 494 synchronized (mLock) { 495 return mLastActivityName; 496 } 497 } 498 499 /** 500 * Last package to report accessibility events 501 * @return String name of package 502 */ getCurrentPackageName()503 public String getCurrentPackageName() { 504 mUiAutomatorBridge.waitForIdle(); 505 AccessibilityNodeInfo rootNode = getRootNode(); 506 if (rootNode == null) 507 return null; 508 return rootNode.getPackageName() != null ? rootNode.getPackageName().toString() : null; 509 } 510 formatLog(String str)511 private String formatLog(String str) { 512 StringBuilder l = new StringBuilder(); 513 for(int space = 0; space < mLogIndent; space++) 514 l.append(". . "); 515 if(mLogIndent > 0) 516 l.append(String.format(". . [%d]: %s", mPatternCounter, str)); 517 else 518 l.append(String.format(". . [%d]: %s", mPatternCounter, str)); 519 return l.toString(); 520 } 521 } 522