1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16package com.android.uiautomator.core; 17 18import android.os.SystemClock; 19import android.util.Log; 20import android.view.accessibility.AccessibilityEvent; 21import android.view.accessibility.AccessibilityNodeInfo; 22 23import com.android.uiautomator.core.UiAutomatorBridge.AccessibilityEventListener; 24 25/** 26 * The QuertController main purpose is to translate a {@link UiSelector} selectors to 27 * {@link AccessibilityNodeInfo}. This is all this controller does. It is typically 28 * created in conjunction with a {@link InteractionController} by {@link UiAutomationContext} 29 * which owns both. {@link UiAutomationContext} is used by {@link UiBase} classes. 30 */ 31class QueryController { 32 33 private static final String LOG_TAG = QueryController.class.getSimpleName(); 34 35 private static final boolean DEBUG = false; 36 37 private final UiAutomatorBridge mUiAutomatorBridge; 38 39 private final Object mLock = new Object(); 40 41 private String mLastActivityName = null; 42 43 // During a pattern selector search, the recursive pattern search 44 // methods will track their counts and indexes here. 45 private int mPatternCounter = 0; 46 private int mPatternIndexer = 0; 47 48 // These help show each selector's search context as it relates to the previous sub selector 49 // matched. When a compound selector fails, it is hard to tell which part of it is failing. 50 // Seeing how a selector is being parsed and which sub selector failed within a long list 51 // of compound selectors is very helpful. 52 private int mLogIndent = 0; 53 private int mLogParentIndent = 0; 54 55 private String mLastTraversedText = ""; 56 57 public QueryController(UiAutomatorBridge bridge) { 58 mUiAutomatorBridge = bridge; 59 bridge.addAccessibilityEventListener(new AccessibilityEventListener() { 60 @Override 61 public void onAccessibilityEvent(AccessibilityEvent event) { 62 synchronized (mLock) { 63 switch(event.getEventType()) { 64 case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: 65 // don't trust event.getText(), check for nulls 66 if (event.getText() != null && event.getText().size() > 0) { 67 if(event.getText().get(0) != null) 68 mLastActivityName = event.getText().get(0).toString(); 69 } 70 break; 71 case AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY: 72 // don't trust event.getText(), check for nulls 73 if (event.getText() != null && event.getText().size() > 0) 74 if(event.getText().get(0) != null) 75 mLastTraversedText = event.getText().get(0).toString(); 76 if(DEBUG) 77 Log.i(LOG_TAG, "Last text selection reported: " + 78 mLastTraversedText); 79 break; 80 } 81 mLock.notifyAll(); 82 } 83 } 84 }); 85 } 86 87 /** 88 * Returns the last text selection reported by accessibility 89 * event TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY. One way to cause 90 * this event is using a DPad arrows to focus on UI elements. 91 * @return 92 */ 93 public String getLastTraversedText() { 94 mUiAutomatorBridge.waitForIdle(); 95 synchronized (mLock) { 96 if (mLastTraversedText.length() > 0) { 97 return mLastTraversedText; 98 } 99 } 100 return null; 101 } 102 103 /** 104 * Clears the last text selection value saved from the TYPE_VIEW_TEXT_SELECTION_CHANGED 105 * event 106 */ 107 public void clearLastTraversedText() { 108 mUiAutomatorBridge.waitForIdle(); 109 synchronized (mLock) { 110 mLastTraversedText = ""; 111 } 112 } 113 114 private void initializeNewSearch() { 115 mPatternCounter = 0; 116 mPatternIndexer = 0; 117 mLogIndent = 0; 118 mLogParentIndent = 0;; 119 } 120 121 /** 122 * Counts the instances of the selector group. The selector must be in the following 123 * format: [container_selector, PATTERN=[INSTANCE=x, PATTERN=[the_pattern]] 124 * where the container_selector is used to find the containment region to search for patterns 125 * and the INSTANCE=x is the instance of the_pattern to return. 126 * @param selector 127 * @return number of pattern matches. Returns 0 for all other cases. 128 */ 129 public int getPatternCount(UiSelector selector) { 130 findAccessibilityNodeInfo(selector, true /*counting*/); 131 return mPatternCounter; 132 } 133 134 /** 135 * Main search method for translating By selectors to AccessibilityInfoNodes 136 * @param selector 137 * @return 138 */ 139 public AccessibilityNodeInfo findAccessibilityNodeInfo(UiSelector selector) { 140 return findAccessibilityNodeInfo(selector, false); 141 } 142 143 protected AccessibilityNodeInfo findAccessibilityNodeInfo(UiSelector selector, 144 boolean isCounting) { 145 mUiAutomatorBridge.waitForIdle(); 146 initializeNewSearch(); 147 148 if(DEBUG) 149 Log.i(LOG_TAG, "Searching: " + selector); 150 151 synchronized (mLock) { 152 AccessibilityNodeInfo rootNode = getRootNode(); 153 if (rootNode == null) { 154 Log.e(LOG_TAG, "Cannot proceed when root node is null. Aborted search"); 155 return null; 156 } 157 158 // Copy so that we don't modify the original's sub selectors 159 UiSelector uiSelector = new UiSelector(selector); 160 return translateCompoundSelector(uiSelector, rootNode, isCounting); 161 } 162 } 163 164 /** 165 * Gets the root node from accessibility and if it fails to get one it will 166 * retry every 250ms for up to 1000ms. 167 * @return null if no root node is obtained 168 */ 169 protected AccessibilityNodeInfo getRootNode() { 170 final int maxRetry = 4; 171 final long waitInterval = 250; 172 AccessibilityNodeInfo rootNode = null; 173 for(int x = 0; x < maxRetry; x++) { 174 rootNode = mUiAutomatorBridge.getRootAccessibilityNodeInfoInActiveWindow(); 175 if (rootNode != null) { 176 return rootNode; 177 } 178 if(x < maxRetry - 1) { 179 Log.e(LOG_TAG, "Got null root node from accessibility - Retrying..."); 180 SystemClock.sleep(waitInterval); 181 } 182 } 183 return rootNode; 184 } 185 186 /** 187 * A compoundSelector encapsulate both Regular and Pattern selectors. The formats follows: 188 * <p/> 189 * regular_selector = By[attributes... CHILD=By[attributes... CHILD=By[....]]] 190 * <br/> 191 * pattern_selector = ...CONTAINER=By[..] PATTERN=By[instance=x PATTERN=[regular_selector] 192 * <br/> 193 * compound_selector = [regular_selector [pattern_selector]] 194 * <p/> 195 * regular_selectors are the most common form of selectors and the search for them 196 * is straightforward. On the other hand pattern_selectors requires search to be 197 * performed as in regular_selector but where regular_selector search returns immediately 198 * upon a successful match, the search for pattern_selector continues until the 199 * requested matched _instance_ of that pattern is matched. 200 * <p/> 201 * Counting UI objects requires using pattern_selectors. The counting search is the same 202 * as a pattern_search however we're not looking to match an instance of the pattern but 203 * rather continuously walking the accessibility node hierarchy while counting matched 204 * patterns, until the end of the tree. 205 * <p/> 206 * If both present, order of parsing begins with CONTAINER followed by PATTERN then the 207 * top most selector is processed as regular_selector within the context of the previous 208 * CONTAINER and its PATTERN information. If neither is present then the top selector is 209 * directly treated as regular_selector. So the presence of a CONTAINER and PATTERN within 210 * a selector simply dictates that the selector matching will be constraint to the sub tree 211 * node where the CONTAINER and its child PATTERN have identified. 212 * @param selector 213 * @param fromNode 214 * @param isCounting 215 * @return 216 */ 217 private AccessibilityNodeInfo translateCompoundSelector(UiSelector selector, 218 AccessibilityNodeInfo fromNode, boolean isCounting) { 219 220 // Start translating compound selectors by translating the regular_selector first 221 // The regular_selector is then used as a container for any optional pattern_selectors 222 // that may or may not be specified. 223 if(selector.hasContainerSelector()) 224 // nested pattern selectors 225 if(selector.getContainerSelector().hasContainerSelector()) { 226 fromNode = translateCompoundSelector( 227 selector.getContainerSelector(), fromNode, false); 228 initializeNewSearch(); 229 } else 230 fromNode = translateReqularSelector(selector.getContainerSelector(), fromNode); 231 else 232 fromNode = translateReqularSelector(selector, fromNode); 233 234 if(fromNode == null) { 235 if(DEBUG) 236 Log.i(LOG_TAG, "Container selector not found: " + selector.dumpToString(false)); 237 return null; 238 } 239 240 if(selector.hasPatternSelector()) { 241 fromNode = translatePatternSelector(selector.getPatternSelector(), 242 fromNode, isCounting); 243 244 if (isCounting) { 245 Log.i(LOG_TAG, String.format( 246 "Counted %d instances of: %s", mPatternCounter, selector)); 247 return null; 248 } else { 249 if(fromNode == null) { 250 if(DEBUG) 251 Log.i(LOG_TAG, "Pattern selector not found: " + 252 selector.dumpToString(false)); 253 return null; 254 } 255 } 256 } 257 258 // translate any additions to the selector that may have been added by tests 259 // with getChild(By selector) after a container and pattern selectors 260 if(selector.hasContainerSelector() || selector.hasPatternSelector()) { 261 if(selector.hasChildSelector() || selector.hasParentSelector()) 262 fromNode = translateReqularSelector(selector, fromNode); 263 } 264 265 if(fromNode == null) { 266 if(DEBUG) 267 Log.i(LOG_TAG, "Object Not Found for selector " + selector); 268 return null; 269 } 270 Log.i(LOG_TAG, String.format("Matched selector: %s <<==>> [%s]", selector, fromNode)); 271 return fromNode; 272 } 273 274 /** 275 * Used by the {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 276 * to translate the regular_selector portion. It has the following format: 277 * <p/> 278 * regular_selector = By[attributes... CHILD=By[attributes... CHILD=By[....]]]<br/> 279 * <p/> 280 * regular_selectors are the most common form of selectors and the search for them 281 * is straightforward. This method will only look for CHILD or PARENT sub selectors. 282 * <p/> 283 * @param selector 284 * @param fromNode 285 * @param index 286 * @return AccessibilityNodeInfo if found else null 287 */ 288 private AccessibilityNodeInfo translateReqularSelector(UiSelector selector, 289 AccessibilityNodeInfo fromNode) { 290 291 return findNodeRegularRecursive(selector, fromNode, 0); 292 } 293 294 private AccessibilityNodeInfo findNodeRegularRecursive(UiSelector subSelector, 295 AccessibilityNodeInfo fromNode, int index) { 296 297 if (subSelector.isMatchFor(fromNode, index)) { 298 if (DEBUG) { 299 Log.d(LOG_TAG, formatLog(String.format("%s", 300 subSelector.dumpToString(false)))); 301 } 302 if(subSelector.isLeaf()) { 303 return fromNode; 304 } 305 if(subSelector.hasChildSelector()) { 306 mLogIndent++; // next selector 307 subSelector = subSelector.getChildSelector(); 308 if(subSelector == null) { 309 Log.e(LOG_TAG, "Error: A child selector without content"); 310 return null; // there is an implementation fault 311 } 312 } else if(subSelector.hasParentSelector()) { 313 mLogIndent++; // next selector 314 subSelector = subSelector.getParentSelector(); 315 if(subSelector == null) { 316 Log.e(LOG_TAG, "Error: A parent selector without content"); 317 return null; // there is an implementation fault 318 } 319 // the selector requested we start at this level from 320 // the parent node from the one we just matched 321 fromNode = fromNode.getParent(); 322 if(fromNode == null) 323 return null; 324 } 325 } 326 327 int childCount = fromNode.getChildCount(); 328 boolean hasNullChild = false; 329 for (int i = 0; i < childCount; i++) { 330 AccessibilityNodeInfo childNode = fromNode.getChild(i); 331 if (childNode == null) { 332 Log.w(LOG_TAG, String.format( 333 "AccessibilityNodeInfo returned a null child (%d of %d)", i, childCount)); 334 if (!hasNullChild) { 335 Log.w(LOG_TAG, String.format("parent = %s", fromNode.toString())); 336 } 337 hasNullChild = true; 338 continue; 339 } 340 if (!childNode.isVisibleToUser()) { 341 // TODO: need to remove this or move it under if (DEBUG) 342 if(DEBUG) 343 Log.d(LOG_TAG, 344 String.format("Skipping invisible child: %s", childNode.toString())); 345 continue; 346 } 347 AccessibilityNodeInfo retNode = findNodeRegularRecursive(subSelector, childNode, i); 348 if (retNode != null) { 349 return retNode; 350 } 351 } 352 return null; 353 } 354 355 /** 356 * Used by the {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 357 * to translate the pattern_selector portion. It has the following format: 358 * <p/> 359 * pattern_selector = ... PATTERN=By[instance=x PATTERN=[regular_selector]]<br/> 360 * <p/> 361 * pattern_selectors requires search to be performed as regular_selector but where 362 * regular_selector search returns immediately upon a successful match, the search for 363 * pattern_selector continues until the requested matched instance of that pattern is 364 * encountered. 365 * <p/> 366 * Counting UI objects requires using pattern_selectors. The counting search is the same 367 * as a pattern_search however we're not looking to match an instance of the pattern but 368 * rather continuously walking the accessibility node hierarchy while counting patterns 369 * until the end of the tree. 370 * @param subSelector 371 * @param fromNode 372 * @param originalPattern 373 * @return null of node is not found or if counting mode is true. 374 * See {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 375 */ 376 private AccessibilityNodeInfo translatePatternSelector(UiSelector subSelector, 377 AccessibilityNodeInfo fromNode, boolean isCounting) { 378 379 if(subSelector.hasPatternSelector()) { 380 // Since pattern_selectors are also the type of selectors used when counting, 381 // we check if this is a counting run or an indexing run 382 if(isCounting) 383 //since we're counting, we reset the indexer so to terminates the search when 384 // the end of tree is reached. The count will be in mPatternCount 385 mPatternIndexer = -1; 386 else 387 // terminates the search once we match the pattern's instance 388 mPatternIndexer = subSelector.getInstance(); 389 390 // A pattern is wrapped in a PATTERN[instance=x PATTERN[the_pattern]] 391 subSelector = subSelector.getPatternSelector(); 392 if(subSelector == null) { 393 Log.e(LOG_TAG, "Pattern portion of the selector is null or not defined"); 394 return null; // there is an implementation fault 395 } 396 // save the current indent level as parent indent before pattern searches 397 // begin under the current tree position. 398 mLogParentIndent = ++mLogIndent; 399 return findNodePatternRecursive(subSelector, fromNode, 0, subSelector); 400 } 401 402 Log.e(LOG_TAG, "Selector must have a pattern selector defined"); // implementation fault? 403 return null; 404 } 405 406 private AccessibilityNodeInfo findNodePatternRecursive( 407 UiSelector subSelector, AccessibilityNodeInfo fromNode, int index, 408 UiSelector originalPattern) { 409 410 if (subSelector.isMatchFor(fromNode, index)) { 411 if(subSelector.isLeaf()) { 412 if(mPatternIndexer == 0) { 413 if (DEBUG) 414 Log.d(LOG_TAG, formatLog( 415 String.format("%s", subSelector.dumpToString(false)))); 416 return fromNode; 417 } else { 418 if(DEBUG) 419 Log.d(LOG_TAG, formatLog( 420 String.format("%s", subSelector.dumpToString(false)))); 421 mPatternCounter++; //count the pattern matched 422 mPatternIndexer--; //decrement until zero for the instance requested 423 424 // At a leaf selector within a group and still not instance matched 425 // then reset the selector to continue search from current position 426 // in the accessibility tree for the next pattern match up until the 427 // pattern index hits 0. 428 subSelector = originalPattern; 429 // starting over with next pattern search so reset to parent level 430 mLogIndent = mLogParentIndent; 431 } 432 } else { 433 if(DEBUG) 434 Log.d(LOG_TAG, formatLog( 435 String.format("%s", subSelector.dumpToString(false)))); 436 437 if(subSelector.hasChildSelector()) { 438 mLogIndent++; // next selector 439 subSelector = subSelector.getChildSelector(); 440 if(subSelector == null) { 441 Log.e(LOG_TAG, "Error: A child selector without content"); 442 return null; 443 } 444 } else if(subSelector.hasParentSelector()) { 445 mLogIndent++; // next selector 446 subSelector = subSelector.getParentSelector(); 447 if(subSelector == null) { 448 Log.e(LOG_TAG, "Error: A parent selector without content"); 449 return null; 450 } 451 fromNode = fromNode.getParent(); 452 if(fromNode == null) 453 return null; 454 } 455 } 456 } 457 458 int childCount = fromNode.getChildCount(); 459 boolean hasNullChild = false; 460 for (int i = 0; i < childCount; i++) { 461 AccessibilityNodeInfo childNode = fromNode.getChild(i); 462 if (childNode == null) { 463 Log.w(LOG_TAG, String.format( 464 "AccessibilityNodeInfo returned a null child (%d of %d)", i, childCount)); 465 if (!hasNullChild) { 466 Log.w(LOG_TAG, String.format("parent = %s", fromNode.toString())); 467 } 468 hasNullChild = true; 469 continue; 470 } 471 if (!childNode.isVisibleToUser()) { 472 if(DEBUG) 473 Log.d(LOG_TAG, 474 String.format("Skipping invisible child: %s", childNode.toString())); 475 continue; 476 } 477 AccessibilityNodeInfo retNode = findNodePatternRecursive( 478 subSelector, childNode, i, originalPattern); 479 if (retNode != null) { 480 return retNode; 481 } 482 } 483 return null; 484 } 485 486 public AccessibilityNodeInfo getAccessibilityRootNode() { 487 return mUiAutomatorBridge.getRootAccessibilityNodeInfoInActiveWindow(); 488 } 489 490 /** 491 * Last activity to report accessibility events. 492 * @deprecated The results returned should be considered unreliable 493 * @return String name of activity 494 */ 495 @Deprecated 496 public String getCurrentActivityName() { 497 mUiAutomatorBridge.waitForIdle(); 498 synchronized (mLock) { 499 return mLastActivityName; 500 } 501 } 502 503 /** 504 * Last package to report accessibility events 505 * @return String name of package 506 */ 507 public String getCurrentPackageName() { 508 mUiAutomatorBridge.waitForIdle(); 509 AccessibilityNodeInfo rootNode = getRootNode(); 510 if (rootNode == null) 511 return null; 512 return rootNode.getPackageName() != null ? rootNode.getPackageName().toString() : null; 513 } 514 515 private String formatLog(String str) { 516 StringBuilder l = new StringBuilder(); 517 for(int space = 0; space < mLogIndent; space++) 518 l.append(". . "); 519 if(mLogIndent > 0) 520 l.append(String.format(". . [%d]: %s", mPatternCounter, str)); 521 else 522 l.append(String.format(". . [%d]: %s", mPatternCounter, str)); 523 return l.toString(); 524 } 525} 526