1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16package com.android.uiautomator.core; 17 18import android.os.SystemClock; 19import android.util.Log; 20import android.view.accessibility.AccessibilityEvent; 21import android.view.accessibility.AccessibilityNodeInfo; 22 23import com.android.uiautomator.core.UiAutomatorBridge.AccessibilityEventListener; 24 25/** 26 * The QuertController main purpose is to translate a {@link UiSelector} selectors to 27 * {@link AccessibilityNodeInfo}. This is all this controller does. It is typically 28 * created in conjunction with a {@link InteractionController} by {@link UiAutomationContext} 29 * which owns both. {@link UiAutomationContext} is used by {@link UiBase} classes. 30 */ 31class QueryController { 32 33 private static final String LOG_TAG = QueryController.class.getSimpleName(); 34 35 private static final boolean DEBUG = false; 36 37 private final UiAutomatorBridge mUiAutomatorBridge; 38 39 private final Object mLock = new Object(); 40 41 private String mLastActivityName = null; 42 private String mLastPackageName = null; 43 44 // During a pattern selector search, the recursive pattern search 45 // methods will track their counts and indexes here. 46 private int mPatternCounter = 0; 47 private int mPatternIndexer = 0; 48 49 // These help show each selector's search context as it relates to the previous sub selector 50 // matched. When a compound selector fails, it is hard to tell which part of it is failing. 51 // Seeing how a selector is being parsed and which sub selector failed within a long list 52 // of compound selectors is very helpful. 53 private int mLogIndent = 0; 54 private int mLogParentIndent = 0; 55 56 private String mLastTraversedText = ""; 57 58 public QueryController(UiAutomatorBridge bridge) { 59 mUiAutomatorBridge = bridge; 60 bridge.addAccessibilityEventListener(new AccessibilityEventListener() { 61 @Override 62 public void onAccessibilityEvent(AccessibilityEvent event) { 63 synchronized (mLock) { 64 mLastPackageName = event.getPackageName().toString(); 65 // don't trust event.getText(), check for nulls 66 if (event.getText() != null && event.getText().size() > 0) { 67 if(event.getText().get(0) != null) 68 mLastActivityName = event.getText().get(0).toString(); 69 } 70 71 switch(event.getEventType()) { 72 case AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY: 73 // don't trust event.getText(), check for nulls 74 if (event.getText() != null && event.getText().size() > 0) 75 if(event.getText().get(0) != null) 76 mLastTraversedText = event.getText().get(0).toString(); 77 if(DEBUG) 78 Log.i(LOG_TAG, "Last text selection reported: " + mLastTraversedText); 79 break; 80 } 81 mLock.notifyAll(); 82 } 83 } 84 }); 85 } 86 87 /** 88 * Returns the last text selection reported by accessibility 89 * event TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY. One way to cause 90 * this event is using a DPad arrows to focus on UI elements. 91 * @return 92 */ 93 public String getLastTraversedText() { 94 mUiAutomatorBridge.waitForIdle(); 95 synchronized (mLock) { 96 if (mLastTraversedText.length() > 0) { 97 return mLastTraversedText; 98 } 99 } 100 return null; 101 } 102 103 /** 104 * Clears the last text selection value saved from the TYPE_VIEW_TEXT_SELECTION_CHANGED 105 * event 106 */ 107 public void clearLastTraversedText() { 108 mUiAutomatorBridge.waitForIdle(); 109 synchronized (mLock) { 110 mLastTraversedText = ""; 111 } 112 } 113 114 private void initializeNewSearch() { 115 mPatternCounter = 0; 116 mPatternIndexer = 0; 117 mLogIndent = 0; 118 mLogParentIndent = 0;; 119 } 120 121 /** 122 * Counts the instances of the selector group. The selector must be in the following 123 * format: [container_selector, PATTERN=[INSTANCE=x, PATTERN=[the_pattern]] 124 * where the container_selector is used to find the containment region to search for patterns 125 * and the INSTANCE=x is the instance of the_pattern to return. 126 * @param selector 127 * @return number of pattern matches. Returns 0 for all other cases. 128 */ 129 public int getPatternCount(UiSelector selector) { 130 findAccessibilityNodeInfo(selector, true /*counting*/); 131 return mPatternCounter; 132 } 133 134 /** 135 * Main search method for translating By selectors to AccessibilityInfoNodes 136 * @param selector 137 * @return 138 */ 139 public AccessibilityNodeInfo findAccessibilityNodeInfo(UiSelector selector) { 140 return findAccessibilityNodeInfo(selector, false); 141 } 142 143 protected AccessibilityNodeInfo findAccessibilityNodeInfo(UiSelector selector, 144 boolean isCounting) { 145 mUiAutomatorBridge.waitForIdle(); 146 initializeNewSearch(); 147 148 if(DEBUG) 149 Log.i(LOG_TAG, "Searching: " + selector); 150 151 synchronized (mLock) { 152 AccessibilityNodeInfo rootNode = getRootNode(); 153 if (rootNode == null) { 154 Log.e(LOG_TAG, "Cannot proceed when root node is null. Aborted search"); 155 return null; 156 } 157 158 // Copy so that we don't modify the original's sub selectors 159 UiSelector uiSelector = new UiSelector(selector); 160 return translateCompoundSelector(uiSelector, rootNode, isCounting); 161 } 162 } 163 164 /** 165 * Gets the root node from accessibility and if it fails to get one it will 166 * retry every 250ms for up to 1000ms. 167 * @return null if no root node is obtained 168 */ 169 protected AccessibilityNodeInfo getRootNode() { 170 final int maxRetry = 4; 171 final long waitInterval = 250; 172 AccessibilityNodeInfo rootNode = null; 173 for(int x = 0; x < maxRetry; x++) { 174 rootNode = mUiAutomatorBridge.getRootAccessibilityNodeInfoInActiveWindow(); 175 if (rootNode != null) { 176 return rootNode; 177 } 178 if(x < maxRetry - 1) { 179 Log.e(LOG_TAG, "Got null root node from accessibility - Retrying..."); 180 SystemClock.sleep(waitInterval); 181 } 182 } 183 return rootNode; 184 } 185 186 /** 187 * A compoundSelector encapsulate both Regular and Pattern selectors. The formats follows: 188 * <p/> 189 * regular_selector = By[attributes... CHILD=By[attributes... CHILD=By[....]]] 190 * <br/> 191 * pattern_selector = ...CONTAINER=By[..] PATTERN=By[instance=x PATTERN=[regular_selector] 192 * <br/> 193 * compound_selector = [regular_selector [pattern_selector]] 194 * <p/> 195 * regular_selectors are the most common form of selectors and the search for them 196 * is straightforward. On the other hand pattern_selectors requires search to be 197 * performed as in regular_selector but where regular_selector search returns immediately 198 * upon a successful match, the search for pattern_selector continues until the 199 * requested matched _instance_ of that pattern is matched. 200 * <p/> 201 * Counting UI objects requires using pattern_selectors. The counting search is the same 202 * as a pattern_search however we're not looking to match an instance of the pattern but 203 * rather continuously walking the accessibility node hierarchy while counting matched 204 * patterns, until the end of the tree. 205 * <p/> 206 * If both present, order of parsing begins with CONTAINER followed by PATTERN then the 207 * top most selector is processed as regular_selector within the context of the previous 208 * CONTAINER and its PATTERN information. If neither is present then the top selector is 209 * directly treated as regular_selector. So the presence of a CONTAINER and PATTERN within 210 * a selector simply dictates that the selector matching will be constraint to the sub tree 211 * node where the CONTAINER and its child PATTERN have identified. 212 * @param selector 213 * @param fromNode 214 * @param isCounting 215 * @return 216 */ 217 private AccessibilityNodeInfo translateCompoundSelector(UiSelector selector, 218 AccessibilityNodeInfo fromNode, boolean isCounting) { 219 220 // Start translating compound selectors by translating the regular_selector first 221 // The regular_selector is then used as a container for any optional pattern_selectors 222 // that may or may not be specified. 223 if(selector.hasContainerSelector()) 224 // nested pattern selectors 225 if(selector.getContainerSelector().hasContainerSelector()) { 226 fromNode = translateCompoundSelector( 227 selector.getContainerSelector(), fromNode, false); 228 initializeNewSearch(); 229 } else 230 fromNode = translateReqularSelector(selector.getContainerSelector(), fromNode); 231 else 232 fromNode = translateReqularSelector(selector, fromNode); 233 234 if(fromNode == null) { 235 if(DEBUG) 236 Log.i(LOG_TAG, "Container selector not found: " + selector.dumpToString(false)); 237 return null; 238 } 239 240 if(selector.hasPatternSelector()) { 241 fromNode = translatePatternSelector(selector.getPatternSelector(), 242 fromNode, isCounting); 243 244 if (isCounting) { 245 Log.i(LOG_TAG, String.format( 246 "Counted %d instances of: %s", mPatternCounter, selector)); 247 return null; 248 } else { 249 if(fromNode == null) { 250 if(DEBUG) 251 Log.i(LOG_TAG, "Pattern selector not found: " + 252 selector.dumpToString(false)); 253 return null; 254 } 255 } 256 } 257 258 // translate any additions to the selector that may have been added by tests 259 // with getChild(By selector) after a container and pattern selectors 260 if(selector.hasContainerSelector() || selector.hasPatternSelector()) { 261 if(selector.hasChildSelector() || selector.hasParentSelector()) 262 fromNode = translateReqularSelector(selector, fromNode); 263 } 264 265 if(fromNode == null) { 266 if(DEBUG) 267 Log.i(LOG_TAG, "Object Not Found for selector " + selector); 268 return null; 269 } 270 Log.i(LOG_TAG, String.format("Matched selector: %s <<==>> [%s]", selector, fromNode)); 271 return fromNode; 272 } 273 274 /** 275 * Used by the {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 276 * to translate the regular_selector portion. It has the following format: 277 * <p/> 278 * regular_selector = By[attributes... CHILD=By[attributes... CHILD=By[....]]]<br/> 279 * <p/> 280 * regular_selectors are the most common form of selectors and the search for them 281 * is straightforward. This method will only look for CHILD or PARENT sub selectors. 282 * <p/> 283 * @param selector 284 * @param fromNode 285 * @param index 286 * @return AccessibilityNodeInfo if found else null 287 */ 288 private AccessibilityNodeInfo translateReqularSelector(UiSelector selector, 289 AccessibilityNodeInfo fromNode) { 290 291 return findNodeRegularRecursive(selector, fromNode, 0); 292 } 293 294 private AccessibilityNodeInfo findNodeRegularRecursive(UiSelector subSelector, 295 AccessibilityNodeInfo fromNode, int index) { 296 297 if (subSelector.isMatchFor(fromNode, index)) { 298 if (DEBUG) { 299 Log.d(LOG_TAG, formatLog(String.format("%s", 300 subSelector.dumpToString(false)))); 301 } 302 if(subSelector.isLeaf()) { 303 return fromNode; 304 } 305 if(subSelector.hasChildSelector()) { 306 mLogIndent++; // next selector 307 subSelector = subSelector.getChildSelector(); 308 if(subSelector == null) { 309 Log.e(LOG_TAG, "Error: A child selector without content"); 310 return null; // there is an implementation fault 311 } 312 } else if(subSelector.hasParentSelector()) { 313 mLogIndent++; // next selector 314 subSelector = subSelector.getParentSelector(); 315 if(subSelector == null) { 316 Log.e(LOG_TAG, "Error: A parent selector without content"); 317 return null; // there is an implementation fault 318 } 319 // the selector requested we start at this level from 320 // the parent node from the one we just matched 321 fromNode = fromNode.getParent(); 322 if(fromNode == null) 323 return null; 324 } 325 } 326 327 int childCount = fromNode.getChildCount(); 328 boolean hasNullChild = false; 329 for (int i = 0; i < childCount; i++) { 330 AccessibilityNodeInfo childNode = fromNode.getChild(i); 331 if (childNode == null) { 332 Log.w(LOG_TAG, String.format( 333 "AccessibilityNodeInfo returned a null child (%d of %d)", i, childCount)); 334 if (!hasNullChild) { 335 Log.w(LOG_TAG, String.format("parent = %s", fromNode.toString())); 336 } 337 hasNullChild = true; 338 continue; 339 } 340 if (!childNode.isVisibleToUser()) { 341 // TODO: need to remove this or move it under if (DEBUG) 342 Log.d(LOG_TAG, String.format("Skipping invisible child: %s", childNode.toString())); 343 continue; 344 } 345 AccessibilityNodeInfo retNode = findNodeRegularRecursive(subSelector, childNode, i); 346 if (retNode != null) { 347 return retNode; 348 } 349 } 350 return null; 351 } 352 353 /** 354 * Used by the {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 355 * to translate the pattern_selector portion. It has the following format: 356 * <p/> 357 * pattern_selector = ... PATTERN=By[instance=x PATTERN=[regular_selector]]<br/> 358 * <p/> 359 * pattern_selectors requires search to be performed as regular_selector but where 360 * regular_selector search returns immediately upon a successful match, the search for 361 * pattern_selector continues until the requested matched instance of that pattern is 362 * encountered. 363 * <p/> 364 * Counting UI objects requires using pattern_selectors. The counting search is the same 365 * as a pattern_search however we're not looking to match an instance of the pattern but 366 * rather continuously walking the accessibility node hierarchy while counting patterns 367 * until the end of the tree. 368 * @param subSelector 369 * @param fromNode 370 * @param originalPattern 371 * @return null of node is not found or if counting mode is true. 372 * See {@link #translateCompoundSelector(UiSelector, AccessibilityNodeInfo, boolean)} 373 */ 374 private AccessibilityNodeInfo translatePatternSelector(UiSelector subSelector, 375 AccessibilityNodeInfo fromNode, boolean isCounting) { 376 377 if(subSelector.hasPatternSelector()) { 378 // Since pattern_selectors are also the type of selectors used when counting, 379 // we check if this is a counting run or an indexing run 380 if(isCounting) 381 //since we're counting, we reset the indexer so to terminates the search when 382 // the end of tree is reached. The count will be in mPatternCount 383 mPatternIndexer = -1; 384 else 385 // terminates the search once we match the pattern's instance 386 mPatternIndexer = subSelector.getInstance(); 387 388 // A pattern is wrapped in a PATTERN[instance=x PATTERN[the_pattern]] 389 subSelector = subSelector.getPatternSelector(); 390 if(subSelector == null) { 391 Log.e(LOG_TAG, "Pattern portion of the selector is null or not defined"); 392 return null; // there is an implementation fault 393 } 394 // save the current indent level as parent indent before pattern searches 395 // begin under the current tree position. 396 mLogParentIndent = ++mLogIndent; 397 return findNodePatternRecursive(subSelector, fromNode, 0, subSelector); 398 } 399 400 Log.e(LOG_TAG, "Selector must have a pattern selector defined"); // implementation fault? 401 return null; 402 } 403 404 private AccessibilityNodeInfo findNodePatternRecursive( 405 UiSelector subSelector, AccessibilityNodeInfo fromNode, int index, 406 UiSelector originalPattern) { 407 408 if (subSelector.isMatchFor(fromNode, index)) { 409 if(subSelector.isLeaf()) { 410 if(mPatternIndexer == 0) { 411 if (DEBUG) 412 Log.d(LOG_TAG, formatLog( 413 String.format("%s", subSelector.dumpToString(false)))); 414 return fromNode; 415 } else { 416 if(DEBUG) 417 Log.d(LOG_TAG, formatLog( 418 String.format("%s", subSelector.dumpToString(false)))); 419 mPatternCounter++; //count the pattern matched 420 mPatternIndexer--; //decrement until zero for the instance requested 421 422 // At a leaf selector within a group and still not instance matched 423 // then reset the selector to continue search from current position 424 // in the accessibility tree for the next pattern match up until the 425 // pattern index hits 0. 426 subSelector = originalPattern; 427 // starting over with next pattern search so reset to parent level 428 mLogIndent = mLogParentIndent; 429 } 430 } else { 431 if(DEBUG) 432 Log.d(LOG_TAG, formatLog( 433 String.format("%s", subSelector.dumpToString(false)))); 434 435 if(subSelector.hasChildSelector()) { 436 mLogIndent++; // next selector 437 subSelector = subSelector.getChildSelector(); 438 if(subSelector == null) { 439 Log.e(LOG_TAG, "Error: A child selector without content"); 440 return null; 441 } 442 } else if(subSelector.hasParentSelector()) { 443 mLogIndent++; // next selector 444 subSelector = subSelector.getParentSelector(); 445 if(subSelector == null) { 446 Log.e(LOG_TAG, "Error: A parent selector without content"); 447 return null; 448 } 449 fromNode = fromNode.getParent(); 450 if(fromNode == null) 451 return null; 452 } 453 } 454 } 455 456 int childCount = fromNode.getChildCount(); 457 boolean hasNullChild = false; 458 for (int i = 0; i < childCount; i++) { 459 AccessibilityNodeInfo childNode = fromNode.getChild(i); 460 if (childNode == null) { 461 Log.w(LOG_TAG, String.format( 462 "AccessibilityNodeInfo returned a null child (%d of %d)", i, childCount)); 463 if (!hasNullChild) { 464 Log.w(LOG_TAG, String.format("parent = %s", fromNode.toString())); 465 } 466 hasNullChild = true; 467 continue; 468 } 469 if (!childNode.isVisibleToUser()) { 470 // TODO: need to remove this or move it under if (DEBUG) 471 Log.d(LOG_TAG, 472 String.format("Skipping invisible child: %s", childNode.toString())); 473 continue; 474 } 475 AccessibilityNodeInfo retNode = findNodePatternRecursive( 476 subSelector, childNode, i, originalPattern); 477 if (retNode != null) { 478 return retNode; 479 } 480 } 481 return null; 482 } 483 484 public AccessibilityNodeInfo getAccessibilityRootNode() { 485 return mUiAutomatorBridge.getRootAccessibilityNodeInfoInActiveWindow(); 486 } 487 488 /** 489 * Last activity to report accessibility events 490 * @return String name of activity 491 */ 492 public String getCurrentActivityName() { 493 mUiAutomatorBridge.waitForIdle(); 494 synchronized (mLock) { 495 return mLastActivityName; 496 } 497 } 498 499 /** 500 * Last package to report accessibility events 501 * @return String name of package 502 */ 503 public String getCurrentPackageName() { 504 mUiAutomatorBridge.waitForIdle(); 505 synchronized (mLock) { 506 return mLastPackageName; 507 } 508 } 509 510 private String formatLog(String str) { 511 StringBuilder l = new StringBuilder(); 512 for(int space = 0; space < mLogIndent; space++) 513 l.append(". . "); 514 if(mLogIndent > 0) 515 l.append(String.format(". . [%d]: %s", mPatternCounter, str)); 516 else 517 l.append(String.format(". . [%d]: %s", mPatternCounter, str)); 518 return l.toString(); 519 } 520} 521