DeqpTestRunner.java revision 0b10cb3bb4eb918b270edacbe6b43f9aac283a89
1package com.android.cts.tradefed.testtype; 2 3import com.android.cts.tradefed.build.CtsBuildHelper; 4import com.android.cts.util.AbiUtils; 5import com.android.ddmlib.AdbCommandRejectedException; 6import com.android.ddmlib.IShellOutputReceiver; 7import com.android.ddmlib.MultiLineReceiver; 8import com.android.ddmlib.ShellCommandUnresponsiveException; 9import com.android.ddmlib.TimeoutException; 10import com.android.ddmlib.testrunner.TestIdentifier; 11import com.android.tradefed.build.IBuildInfo; 12import com.android.tradefed.device.DeviceNotAvailableException; 13import com.android.tradefed.device.ITestDevice; 14import com.android.tradefed.log.LogUtil.CLog; 15import com.android.tradefed.result.ByteArrayInputStreamSource; 16import com.android.tradefed.result.ITestInvocationListener; 17import com.android.tradefed.result.LogDataType; 18import com.android.tradefed.testtype.IAbi; 19import com.android.tradefed.testtype.IBuildReceiver; 20import com.android.tradefed.testtype.IDeviceTest; 21import com.android.tradefed.testtype.IRemoteTest; 22 23import java.io.File; 24import java.io.FileNotFoundException; 25import java.io.IOException; 26import java.lang.reflect.Method; 27import java.lang.reflect.InvocationTargetException; 28import java.util.ArrayList; 29import java.util.Collection; 30import java.util.Collections; 31import java.util.HashMap; 32import java.util.HashSet; 33import java.util.Iterator; 34import java.util.LinkedHashMap; 35import java.util.LinkedHashSet; 36import java.util.LinkedList; 37import java.util.List; 38import java.util.Map; 39import java.util.Set; 40import java.util.concurrent.TimeUnit; 41 42/** 43 * Test runner for dEQP tests 44 * 45 * Supports running drawElements Quality Program tests found under external/deqp. 46 */ 47public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest { 48 49 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; 50 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; 51 private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log"; 52 private static final String SKIPPED_INSTANCE_LOG_MESSAGE = "Configuration skipped"; 53 private static final String NOT_EXECUTABLE_LOG_MESSAGE = "Abort: Test cannot be executed"; 54 private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt"; 55 private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa"; 56 public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape"; 57 public static final String FEATURE_PORTRAIT = "android.hardware.screen.portrait"; 58 59 private static final int TESTCASE_BATCH_LIMIT = 1000; 60 private static final BatchRunConfiguration DEFAULT_CONFIG = 61 new BatchRunConfiguration("rgba8888d24s8", "unspecified", "window"); 62 63 private static final int UNRESPOSIVE_CMD_TIMEOUT_MS = 60000; // one minute 64 65 private final String mPackageName; 66 private final String mName; 67 private final Collection<TestIdentifier> mRemainingTests; 68 private final Map<TestIdentifier, Set<BatchRunConfiguration>> mTestInstances; 69 private final TestInstanceResultListener mInstanceListerner = new TestInstanceResultListener(); 70 private final Map<TestIdentifier, Integer> mTestInstabilityRatings; 71 private IAbi mAbi; 72 private CtsBuildHelper mCtsBuild; 73 private boolean mLogData = false; 74 private ITestDevice mDevice; 75 private Set<String> mDeviceFeatures; 76 private Map<String, Boolean> mConfigQuerySupportCache = new HashMap<>(); 77 78 private IRecovery mDeviceRecovery = new Recovery(); 79 { 80 mDeviceRecovery.setSleepProvider(new SleepProvider()); 81 } 82 83 public DeqpTestRunner(String packageName, String name, Collection<TestIdentifier> tests, 84 Map<TestIdentifier, List<Map<String,String>>> testInstances) { 85 mPackageName = packageName; 86 mName = name; 87 mRemainingTests = new LinkedList<>(tests); // avoid modifying arguments 88 mTestInstances = parseTestInstances(tests, testInstances); 89 mTestInstabilityRatings = new HashMap<>(); 90 } 91 92 /** 93 * @param abi the ABI to run the test on 94 */ 95 public void setAbi(IAbi abi) { 96 mAbi = abi; 97 } 98 99 /** 100 * {@inheritDoc} 101 */ 102 @Override 103 public void setBuild(IBuildInfo buildInfo) { 104 mCtsBuild = CtsBuildHelper.createBuildHelper(buildInfo); 105 } 106 107 /** 108 * Set the CTS build container. 109 * <p/> 110 * Exposed so unit tests can mock the provided build. 111 * 112 * @param buildHelper 113 */ 114 public void setBuildHelper(CtsBuildHelper buildHelper) { 115 mCtsBuild = buildHelper; 116 } 117 118 /** 119 * Enable or disable raw dEQP test log collection. 120 */ 121 public void setCollectLogs(boolean logData) { 122 mLogData = logData; 123 } 124 125 /** 126 * {@inheritDoc} 127 */ 128 @Override 129 public void setDevice(ITestDevice device) { 130 mDevice = device; 131 } 132 133 /** 134 * {@inheritDoc} 135 */ 136 @Override 137 public ITestDevice getDevice() { 138 return mDevice; 139 } 140 141 /** 142 * Set recovery handler. 143 * 144 * Exposed for unit testing. 145 */ 146 public void setRecovery(IRecovery deviceRecovery) { 147 mDeviceRecovery = deviceRecovery; 148 } 149 150 private static final class CapabilityQueryFailureException extends Exception { 151 }; 152 153 /** 154 * Test configuration of dEPQ test instance execution. 155 * Exposed for unit testing 156 */ 157 public static final class BatchRunConfiguration { 158 public static final String ROTATION_UNSPECIFIED = "unspecified"; 159 public static final String ROTATION_PORTRAIT = "0"; 160 public static final String ROTATION_LANDSCAPE = "90"; 161 public static final String ROTATION_REVERSE_PORTRAIT = "180"; 162 public static final String ROTATION_REVERSE_LANDSCAPE = "270"; 163 164 private final String mGlConfig; 165 private final String mRotation; 166 private final String mSurfaceType; 167 168 public BatchRunConfiguration(String glConfig, String rotation, String surfaceType) { 169 mGlConfig = glConfig; 170 mRotation = rotation; 171 mSurfaceType = surfaceType; 172 } 173 174 /** 175 * Get string that uniquely identifies this config 176 */ 177 public String getId() { 178 return String.format("{glformat=%s,rotation=%s,surfacetype=%s}", 179 mGlConfig, mRotation, mSurfaceType); 180 } 181 182 /** 183 * Get the GL config used in this configuration. 184 */ 185 public String getGlConfig() { 186 return mGlConfig; 187 } 188 189 /** 190 * Get the screen rotation used in this configuration. 191 */ 192 public String getRotation() { 193 return mRotation; 194 } 195 196 /** 197 * Get the surface type used in this configuration. 198 */ 199 public String getSurfaceType() { 200 return mSurfaceType; 201 } 202 203 @Override 204 public boolean equals(Object other) { 205 if (other == null) { 206 return false; 207 } else if (!(other instanceof BatchRunConfiguration)) { 208 return false; 209 } else { 210 return getId().equals(((BatchRunConfiguration)other).getId()); 211 } 212 } 213 214 @Override 215 public int hashCode() { 216 return getId().hashCode(); 217 } 218 } 219 220 /** 221 * dEQP test instance listerer and invocation result forwarded 222 */ 223 private class TestInstanceResultListener { 224 private ITestInvocationListener mSink; 225 private BatchRunConfiguration mRunConfig; 226 227 private TestIdentifier mCurrentTestId; 228 private boolean mGotTestResult; 229 private String mCurrentTestLog; 230 231 private class PendingResult 232 { 233 boolean allInstancesPassed; 234 Map<BatchRunConfiguration, String> testLogs; 235 Map<BatchRunConfiguration, String> errorMessages; 236 Set<BatchRunConfiguration> remainingConfigs; 237 }; 238 private final Map<TestIdentifier, PendingResult> mPendingResults = new HashMap<>(); 239 240 public void setSink(ITestInvocationListener sink) { 241 mSink = sink; 242 } 243 244 public void setCurrentConfig(BatchRunConfiguration runConfig) { 245 mRunConfig = runConfig; 246 } 247 248 /** 249 * Get currently processed test id, or null if not currently processing a test case 250 */ 251 public TestIdentifier getCurrentTestId() { 252 return mCurrentTestId; 253 } 254 255 /** 256 * Forward result to sink 257 */ 258 private void forwardFinalizedPendingResult(TestIdentifier testId) { 259 if (mRemainingTests.contains(testId)) { 260 final PendingResult result = mPendingResults.get(testId); 261 262 mPendingResults.remove(testId); 263 mRemainingTests.remove(testId); 264 265 // Forward results to the sink 266 mSink.testStarted(testId); 267 268 // Test Log 269 if (mLogData) { 270 for (Map.Entry<BatchRunConfiguration, String> entry : 271 result.testLogs.entrySet()) { 272 final ByteArrayInputStreamSource source 273 = new ByteArrayInputStreamSource(entry.getValue().getBytes()); 274 275 mSink.testLog(testId.getClassName() + "." + testId.getTestName() + "@" 276 + entry.getKey().getId(), LogDataType.XML, source); 277 278 source.cancel(); 279 } 280 } 281 282 // Error message 283 if (!result.allInstancesPassed) { 284 final StringBuilder errorLog = new StringBuilder(); 285 286 for (Map.Entry<BatchRunConfiguration, String> entry : 287 result.errorMessages.entrySet()) { 288 if (errorLog.length() > 0) { 289 errorLog.append('\n'); 290 } 291 errorLog.append(String.format("=== with config %s ===\n", 292 entry.getKey().getId())); 293 errorLog.append(entry.getValue()); 294 } 295 296 mSink.testFailed(testId, errorLog.toString()); 297 } 298 299 final Map<String, String> emptyMap = Collections.emptyMap(); 300 mSink.testEnded(testId, emptyMap); 301 } 302 } 303 304 /** 305 * Declare existence of a test and instances 306 */ 307 public void setTestInstances(TestIdentifier testId, Set<BatchRunConfiguration> configs) { 308 // Test instances cannot change at runtime, ignore if we have already set this 309 if (!mPendingResults.containsKey(testId)) { 310 final PendingResult pendingResult = new PendingResult(); 311 pendingResult.allInstancesPassed = true; 312 pendingResult.testLogs = new LinkedHashMap<>(); 313 pendingResult.errorMessages = new LinkedHashMap<>(); 314 pendingResult.remainingConfigs = new HashSet<>(configs); // avoid mutating argument 315 mPendingResults.put(testId, pendingResult); 316 } 317 } 318 319 /** 320 * Query if test instance has not yet been executed 321 */ 322 public boolean isPendingTestInstance(TestIdentifier testId, 323 BatchRunConfiguration config) { 324 final PendingResult result = mPendingResults.get(testId); 325 if (result == null) { 326 // test is not in the current working batch of the runner, i.e. it cannot be 327 // "partially" completed. 328 if (!mRemainingTests.contains(testId)) { 329 // The test has been fully executed. Not pending. 330 return false; 331 } else { 332 // Test has not yet been executed. Check if such instance exists 333 return mTestInstances.get(testId).contains(config); 334 } 335 } else { 336 // could be partially completed, check this particular config 337 return result.remainingConfigs.contains(config); 338 } 339 } 340 341 /** 342 * Fake execution of an instance with current config 343 */ 344 public void skipTest(TestIdentifier testId) { 345 final PendingResult result = mPendingResults.get(testId); 346 347 result.errorMessages.put(mRunConfig, SKIPPED_INSTANCE_LOG_MESSAGE); 348 result.remainingConfigs.remove(mRunConfig); 349 350 // Pending result finished, report result 351 if (result.remainingConfigs.isEmpty()) { 352 forwardFinalizedPendingResult(testId); 353 } 354 } 355 356 /** 357 * Fake failure of an instance with current config 358 */ 359 public void abortTest(TestIdentifier testId, String errorMessage) { 360 final PendingResult result = mPendingResults.get(testId); 361 362 // Mark as executed 363 result.allInstancesPassed = false; 364 result.errorMessages.put(mRunConfig, errorMessage); 365 result.remainingConfigs.remove(mRunConfig); 366 367 // Pending result finished, report result 368 if (result.remainingConfigs.isEmpty()) { 369 forwardFinalizedPendingResult(testId); 370 } 371 372 if (testId.equals(mCurrentTestId)) { 373 mCurrentTestId = null; 374 } 375 } 376 377 /** 378 * Handles beginning of dEQP session. 379 */ 380 private void handleBeginSession(Map<String, String> values) { 381 // ignore 382 } 383 384 /** 385 * Handles end of dEQP session. 386 */ 387 private void handleEndSession(Map<String, String> values) { 388 // ignore 389 } 390 391 /** 392 * Handles beginning of dEQP testcase. 393 */ 394 private void handleBeginTestCase(Map<String, String> values) { 395 mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath")); 396 mCurrentTestLog = ""; 397 mGotTestResult = false; 398 399 // mark instance as started 400 if (mPendingResults.get(mCurrentTestId) != null) { 401 mPendingResults.get(mCurrentTestId).remainingConfigs.remove(mRunConfig); 402 } else { 403 CLog.w("Got unexpected start of %s", mCurrentTestId); 404 } 405 } 406 407 /** 408 * Handles end of dEQP testcase. 409 */ 410 private void handleEndTestCase(Map<String, String> values) { 411 final PendingResult result = mPendingResults.get(mCurrentTestId); 412 413 if (result != null) { 414 if (!mGotTestResult) { 415 result.allInstancesPassed = false; 416 result.errorMessages.put(mRunConfig, INCOMPLETE_LOG_MESSAGE); 417 } 418 419 if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) { 420 result.testLogs.put(mRunConfig, mCurrentTestLog); 421 } 422 423 // Pending result finished, report result 424 if (result.remainingConfigs.isEmpty()) { 425 forwardFinalizedPendingResult(mCurrentTestId); 426 } 427 } else { 428 CLog.w("Got unexpected end of %s", mCurrentTestId); 429 } 430 mCurrentTestId = null; 431 } 432 433 /** 434 * Handles dEQP testcase result. 435 */ 436 private void handleTestCaseResult(Map<String, String> values) { 437 String code = values.get("dEQP-TestCaseResult-Code"); 438 String details = values.get("dEQP-TestCaseResult-Details"); 439 440 if (mPendingResults.get(mCurrentTestId) == null) { 441 CLog.w("Got unexpected result for %s", mCurrentTestId); 442 mGotTestResult = true; 443 return; 444 } 445 446 if (code.compareTo("Pass") == 0) { 447 mGotTestResult = true; 448 } else if (code.compareTo("NotSupported") == 0) { 449 mGotTestResult = true; 450 } else if (code.compareTo("QualityWarning") == 0) { 451 mGotTestResult = true; 452 } else if (code.compareTo("CompatibilityWarning") == 0) { 453 mGotTestResult = true; 454 } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0 455 || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0 456 || code.compareTo("Timeout") == 0) { 457 mPendingResults.get(mCurrentTestId).allInstancesPassed = false; 458 mPendingResults.get(mCurrentTestId) 459 .errorMessages.put(mRunConfig, code + ": " + details); 460 mGotTestResult = true; 461 } else { 462 String codeError = "Unknown result code: " + code; 463 mPendingResults.get(mCurrentTestId).allInstancesPassed = false; 464 mPendingResults.get(mCurrentTestId) 465 .errorMessages.put(mRunConfig, codeError + ": " + details); 466 mGotTestResult = true; 467 } 468 } 469 470 /** 471 * Handles terminated dEQP testcase. 472 */ 473 private void handleTestCaseTerminate(Map<String, String> values) { 474 final PendingResult result = mPendingResults.get(mCurrentTestId); 475 476 if (result != null) { 477 String reason = values.get("dEQP-TerminateTestCase-Reason"); 478 mPendingResults.get(mCurrentTestId).allInstancesPassed = false; 479 mPendingResults.get(mCurrentTestId) 480 .errorMessages.put(mRunConfig, "Terminated: " + reason); 481 482 // Pending result finished, report result 483 if (result.remainingConfigs.isEmpty()) { 484 forwardFinalizedPendingResult(mCurrentTestId); 485 } 486 } else { 487 CLog.w("Got unexpected termination of %s", mCurrentTestId); 488 } 489 490 mCurrentTestId = null; 491 mGotTestResult = true; 492 } 493 494 /** 495 * Handles dEQP testlog data. 496 */ 497 private void handleTestLogData(Map<String, String> values) { 498 mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log"); 499 } 500 501 /** 502 * Handles new instrumentation status message. 503 */ 504 public void handleStatus(Map<String, String> values) { 505 String eventType = values.get("dEQP-EventType"); 506 507 if (eventType == null) { 508 return; 509 } 510 511 if (eventType.compareTo("BeginSession") == 0) { 512 handleBeginSession(values); 513 } else if (eventType.compareTo("EndSession") == 0) { 514 handleEndSession(values); 515 } else if (eventType.compareTo("BeginTestCase") == 0) { 516 handleBeginTestCase(values); 517 } else if (eventType.compareTo("EndTestCase") == 0) { 518 handleEndTestCase(values); 519 } else if (eventType.compareTo("TestCaseResult") == 0) { 520 handleTestCaseResult(values); 521 } else if (eventType.compareTo("TerminateTestCase") == 0) { 522 handleTestCaseTerminate(values); 523 } else if (eventType.compareTo("TestLogData") == 0) { 524 handleTestLogData(values); 525 } 526 } 527 528 /** 529 * Signal listener that batch ended and forget incomplete results. 530 */ 531 public void endBatch() { 532 // end open test if when stream ends 533 if (mCurrentTestId != null) { 534 // Current instance was removed from remainingConfigs when case 535 // started. Mark current instance as pending. 536 if (mPendingResults.get(mCurrentTestId) != null) { 537 mPendingResults.get(mCurrentTestId).remainingConfigs.add(mRunConfig); 538 } else { 539 CLog.w("Got unexpected internal state of %s", mCurrentTestId); 540 } 541 } 542 mCurrentTestId = null; 543 } 544 } 545 546 /** 547 * dEQP instrumentation parser 548 */ 549 private static class InstrumentationParser extends MultiLineReceiver { 550 private TestInstanceResultListener mListener; 551 552 private Map<String, String> mValues; 553 private String mCurrentName; 554 private String mCurrentValue; 555 private int mResultCode; 556 private boolean mGotExitValue = false; 557 558 559 public InstrumentationParser(TestInstanceResultListener listener) { 560 mListener = listener; 561 } 562 563 /** 564 * {@inheritDoc} 565 */ 566 @Override 567 public void processNewLines(String[] lines) { 568 for (String line : lines) { 569 if (mValues == null) mValues = new HashMap<String, String>(); 570 571 if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) { 572 if (mCurrentName != null) { 573 mValues.put(mCurrentName, mCurrentValue); 574 575 mCurrentName = null; 576 mCurrentValue = null; 577 } 578 579 mListener.handleStatus(mValues); 580 mValues = null; 581 } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) { 582 if (mCurrentName != null) { 583 mValues.put(mCurrentName, mCurrentValue); 584 585 mCurrentValue = null; 586 mCurrentName = null; 587 } 588 589 String prefix = "INSTRUMENTATION_STATUS: "; 590 int nameBegin = prefix.length(); 591 int nameEnd = line.indexOf('='); 592 int valueBegin = nameEnd + 1; 593 594 mCurrentName = line.substring(nameBegin, nameEnd); 595 mCurrentValue = line.substring(valueBegin); 596 } else if (line.startsWith("INSTRUMENTATION_CODE: ")) { 597 try { 598 mResultCode = Integer.parseInt(line.substring(22)); 599 mGotExitValue = true; 600 } catch (NumberFormatException ex) { 601 CLog.w("Instrumentation code format unexpected"); 602 } 603 } else if (mCurrentValue != null) { 604 mCurrentValue = mCurrentValue + line; 605 } 606 } 607 } 608 609 /** 610 * {@inheritDoc} 611 */ 612 @Override 613 public void done() { 614 if (mCurrentName != null) { 615 mValues.put(mCurrentName, mCurrentValue); 616 617 mCurrentName = null; 618 mCurrentValue = null; 619 } 620 621 if (mValues != null) { 622 mListener.handleStatus(mValues); 623 mValues = null; 624 } 625 } 626 627 /** 628 * {@inheritDoc} 629 */ 630 @Override 631 public boolean isCancelled() { 632 return false; 633 } 634 635 /** 636 * Returns whether target instrumentation exited normally. 637 */ 638 public boolean wasSuccessful() { 639 return mGotExitValue; 640 } 641 642 /** 643 * Returns Instrumentation return code 644 */ 645 public int getResultCode() { 646 return mResultCode; 647 } 648 } 649 650 /** 651 * dEQP platfom query instrumentation parser 652 */ 653 private static class PlatformQueryInstrumentationParser extends MultiLineReceiver { 654 private Map<String,String> mResultMap = new LinkedHashMap<>(); 655 private int mResultCode; 656 private boolean mGotExitValue = false; 657 658 /** 659 * {@inheritDoc} 660 */ 661 @Override 662 public void processNewLines(String[] lines) { 663 for (String line : lines) { 664 if (line.startsWith("INSTRUMENTATION_RESULT: ")) { 665 final String parts[] = line.substring(24).split("=",2); 666 if (parts.length == 2) { 667 mResultMap.put(parts[0], parts[1]); 668 } else { 669 CLog.w("Instrumentation status format unexpected"); 670 } 671 } else if (line.startsWith("INSTRUMENTATION_CODE: ")) { 672 try { 673 mResultCode = Integer.parseInt(line.substring(22)); 674 mGotExitValue = true; 675 } catch (NumberFormatException ex) { 676 CLog.w("Instrumentation code format unexpected"); 677 } 678 } 679 } 680 } 681 682 /** 683 * {@inheritDoc} 684 */ 685 @Override 686 public boolean isCancelled() { 687 return false; 688 } 689 690 /** 691 * Returns whether target instrumentation exited normally. 692 */ 693 public boolean wasSuccessful() { 694 return mGotExitValue; 695 } 696 697 /** 698 * Returns Instrumentation return code 699 */ 700 public int getResultCode() { 701 return mResultCode; 702 } 703 704 public Map<String,String> getResultMap() { 705 return mResultMap; 706 } 707 } 708 709 /** 710 * Interface for sleeping. 711 * 712 * Exposed for unit testing 713 */ 714 public static interface ISleepProvider { 715 public void sleep(int milliseconds); 716 }; 717 718 private static class SleepProvider implements ISleepProvider { 719 public void sleep(int milliseconds) { 720 try { 721 Thread.sleep(milliseconds); 722 } catch (InterruptedException ex) { 723 } 724 } 725 }; 726 727 /** 728 * Interface for failure recovery. 729 * 730 * Exposed for unit testing 731 */ 732 public static interface IRecovery { 733 /** 734 * Sets the sleep provider IRecovery works on 735 */ 736 public void setSleepProvider(ISleepProvider sleepProvider); 737 738 /** 739 * Sets the device IRecovery works on 740 */ 741 public void setDevice(ITestDevice device); 742 743 /** 744 * Informs Recovery that test execution has progressed since the last recovery 745 */ 746 public void onExecutionProgressed(); 747 748 /** 749 * Tries to recover device after failed refused connection. 750 * 751 * @throws DeviceNotAvailableException if recovery did not succeed 752 */ 753 public void recoverConnectionRefused() throws DeviceNotAvailableException; 754 755 /** 756 * Tries to recover device after abnormal execution termination or link failure. 757 * 758 * @param progressedSinceLastCall true if test execution has progressed since last call 759 * @throws DeviceNotAvailableException if recovery did not succeed 760 */ 761 public void recoverComLinkKilled() throws DeviceNotAvailableException; 762 }; 763 764 /** 765 * State machine for execution failure recovery. 766 * 767 * Exposed for unit testing 768 */ 769 public static class Recovery implements IRecovery { 770 private int RETRY_COOLDOWN_MS = 6000; // 6 seconds 771 private int PROCESS_KILL_WAIT_MS = 1000; // 1 second 772 773 private static enum MachineState { 774 WAIT, // recover by waiting 775 RECOVER, // recover by calling recover() 776 REBOOT, // recover by rebooting 777 FAIL, // cannot recover 778 }; 779 780 private MachineState mState = MachineState.WAIT; 781 private ITestDevice mDevice; 782 private ISleepProvider mSleepProvider; 783 784 private static class ProcessKillFailureException extends Exception { 785 } 786 787 /** 788 * {@inheritDoc} 789 */ 790 public void setSleepProvider(ISleepProvider sleepProvider) { 791 mSleepProvider = sleepProvider; 792 } 793 794 /** 795 * {@inheritDoc} 796 */ 797 @Override 798 public void setDevice(ITestDevice device) { 799 mDevice = device; 800 } 801 802 /** 803 * {@inheritDoc} 804 */ 805 @Override 806 public void onExecutionProgressed() { 807 mState = MachineState.WAIT; 808 } 809 810 /** 811 * {@inheritDoc} 812 */ 813 @Override 814 public void recoverConnectionRefused() throws DeviceNotAvailableException { 815 switch (mState) { 816 case WAIT: // not a valid stratedy for connection refusal, fallthrough 817 case RECOVER: 818 // First failure, just try to recover 819 CLog.w("ADB connection failed, trying to recover"); 820 mState = MachineState.REBOOT; // the next step is to reboot 821 822 try { 823 recoverDevice(); 824 } catch (DeviceNotAvailableException ex) { 825 // chain forward 826 recoverConnectionRefused(); 827 } 828 break; 829 830 case REBOOT: 831 // Second failure in a row, try to reboot 832 CLog.w("ADB connection failed after recovery, rebooting device"); 833 mState = MachineState.FAIL; // the next step is to fail 834 835 try { 836 rebootDevice(); 837 } catch (DeviceNotAvailableException ex) { 838 // chain forward 839 recoverConnectionRefused(); 840 } 841 break; 842 843 case FAIL: 844 // Third failure in a row, just fail 845 CLog.w("Cannot recover ADB connection"); 846 throw new DeviceNotAvailableException("failed to connect after reboot"); 847 } 848 } 849 850 /** 851 * {@inheritDoc} 852 */ 853 @Override 854 public void recoverComLinkKilled() throws DeviceNotAvailableException { 855 switch (mState) { 856 case WAIT: 857 // First failure, just try to wait and try again 858 CLog.w("ADB link failed, retrying after a cooldown period"); 859 mState = MachineState.RECOVER; // the next step is to recover the device 860 861 waitCooldown(); 862 863 // even if the link to deqp on-device process was killed, the process might 864 // still be alive. Locate and terminate such unwanted processes. 865 try { 866 killDeqpProcess(); 867 } catch (DeviceNotAvailableException ex) { 868 // chain forward 869 recoverComLinkKilled(); 870 } catch (ProcessKillFailureException ex) { 871 // chain forward 872 recoverComLinkKilled(); 873 } 874 break; 875 876 case RECOVER: 877 // Second failure, just try to recover 878 CLog.w("ADB link failed, trying to recover"); 879 mState = MachineState.REBOOT; // the next step is to reboot 880 881 try { 882 recoverDevice(); 883 killDeqpProcess(); 884 } catch (DeviceNotAvailableException ex) { 885 // chain forward 886 recoverComLinkKilled(); 887 } catch (ProcessKillFailureException ex) { 888 // chain forward 889 recoverComLinkKilled(); 890 } 891 break; 892 893 case REBOOT: 894 // Third failure in a row, try to reboot 895 CLog.w("ADB link failed after recovery, rebooting device"); 896 mState = MachineState.FAIL; // the next step is to fail 897 898 try { 899 rebootDevice(); 900 } catch (DeviceNotAvailableException ex) { 901 // chain forward 902 recoverComLinkKilled(); 903 } 904 break; 905 906 case FAIL: 907 // Fourth failure in a row, just fail 908 CLog.w("Cannot recover ADB connection"); 909 throw new DeviceNotAvailableException("link killed after reboot"); 910 } 911 } 912 913 private void waitCooldown() { 914 mSleepProvider.sleep(RETRY_COOLDOWN_MS); 915 } 916 917 private Iterable<Integer> getDeqpProcessPids() throws DeviceNotAvailableException { 918 final List<Integer> pids = new ArrayList<Integer>(2); 919 final String processes = mDevice.executeShellCommand("ps | grep com.drawelements"); 920 final String[] lines = processes.split("(\\r|\\n)+"); 921 for (String line : lines) { 922 final String[] fields = line.split("\\s+"); 923 if (fields.length < 2) { 924 continue; 925 } 926 927 try { 928 final int processId = Integer.parseInt(fields[1], 10); 929 pids.add(processId); 930 } catch (NumberFormatException ex) { 931 continue; 932 } 933 } 934 return pids; 935 } 936 937 private void killDeqpProcess() throws DeviceNotAvailableException, 938 ProcessKillFailureException { 939 for (Integer processId : getDeqpProcessPids()) { 940 mDevice.executeShellCommand(String.format("kill -9 %d", processId)); 941 } 942 943 mSleepProvider.sleep(PROCESS_KILL_WAIT_MS); 944 945 // check that processes actually died 946 if (getDeqpProcessPids().iterator().hasNext()) { 947 // a process is still alive, killing failed 948 throw new ProcessKillFailureException(); 949 } 950 } 951 952 public void recoverDevice() throws DeviceNotAvailableException { 953 // Work around the API. We need to call recoverDevice() on the test device and 954 // we know that mDevice is a TestDevice. However even though the recoverDevice() 955 // method is public suggesting it should be publicly accessible, the class itself 956 // and its super-interface (IManagedTestDevice) are package-private. 957 final Method recoverDeviceMethod; 958 try { 959 recoverDeviceMethod = mDevice.getClass().getMethod("recoverDevice"); 960 recoverDeviceMethod.setAccessible(true); 961 } catch (NoSuchMethodException ex) { 962 throw new AssertionError("Test device must have recoverDevice()"); 963 } 964 965 try { 966 recoverDeviceMethod.invoke(mDevice); 967 } catch (InvocationTargetException ex) { 968 if (ex.getCause() instanceof DeviceNotAvailableException) { 969 throw (DeviceNotAvailableException)ex.getCause(); 970 } else if (ex.getCause() instanceof RuntimeException) { 971 throw (RuntimeException)ex.getCause(); 972 } else { 973 throw new AssertionError("unexpected throw", ex); 974 } 975 } catch (IllegalAccessException ex) { 976 throw new AssertionError("unexpected throw", ex); 977 } 978 } 979 980 private void rebootDevice() throws DeviceNotAvailableException { 981 mDevice.reboot(); 982 } 983 }; 984 985 /** 986 * Parse map of instance arguments to map of BatchRunConfigurations 987 */ 988 private static Map<TestIdentifier, Set<BatchRunConfiguration>> parseTestInstances( 989 Collection<TestIdentifier> tests, 990 Map<TestIdentifier, List<Map<String,String>>> testInstances) { 991 final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new HashMap<>(); 992 for (final TestIdentifier test : tests) { 993 final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>(); 994 if (testInstances.get(test).isEmpty()) { 995 // no instances defined, use default 996 testInstanceSet.add(DEFAULT_CONFIG); 997 } else { 998 for (Map<String, String> instanceArgs : testInstances.get(test)) { 999 testInstanceSet.add(parseRunConfig(instanceArgs)); 1000 } 1001 } 1002 instances.put(test, testInstanceSet); 1003 } 1004 return instances; 1005 } 1006 1007 private static BatchRunConfiguration parseRunConfig(Map<String,String> instanceArguments) { 1008 final String glConfig; 1009 final String rotation; 1010 final String surfaceType; 1011 1012 if (instanceArguments.containsKey("glconfig")) { 1013 glConfig = instanceArguments.get("glconfig"); 1014 } else { 1015 glConfig = DEFAULT_CONFIG.getGlConfig(); 1016 } 1017 if (instanceArguments.containsKey("rotation")) { 1018 rotation = instanceArguments.get("rotation"); 1019 } else { 1020 rotation = DEFAULT_CONFIG.getRotation(); 1021 } 1022 if (instanceArguments.containsKey("surfaceType")) { 1023 surfaceType = instanceArguments.get("surfaceType"); 1024 } else { 1025 surfaceType = DEFAULT_CONFIG.getSurfaceType(); 1026 } 1027 1028 return new BatchRunConfiguration(glConfig, rotation, surfaceType); 1029 } 1030 1031 private Set<BatchRunConfiguration> getTestRunConfigs (TestIdentifier testId) { 1032 return mTestInstances.get(testId); 1033 } 1034 1035 /** 1036 * Converts dEQP testcase path to TestIdentifier. 1037 */ 1038 private static TestIdentifier pathToIdentifier(String testPath) { 1039 String[] components = testPath.split("\\."); 1040 String name = components[components.length - 1]; 1041 String className = null; 1042 1043 for (int i = 0; i < components.length - 1; i++) { 1044 if (className == null) { 1045 className = components[i]; 1046 } else { 1047 className = className + "." + components[i]; 1048 } 1049 } 1050 1051 return new TestIdentifier(className, name); 1052 } 1053 1054 private String getId() { 1055 return AbiUtils.createId(mAbi.getName(), mPackageName); 1056 } 1057 1058 /** 1059 * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute. 1060 */ 1061 private static String generateTestCaseTrieFromPaths(Collection<String> tests) { 1062 String result = "{"; 1063 boolean first = true; 1064 1065 // Add testcases to results 1066 for (Iterator<String> iter = tests.iterator(); iter.hasNext();) { 1067 String test = iter.next(); 1068 String[] components = test.split("\\."); 1069 1070 if (components.length == 1) { 1071 if (!first) { 1072 result = result + ","; 1073 } 1074 first = false; 1075 1076 result += components[0]; 1077 iter.remove(); 1078 } 1079 } 1080 1081 if (!tests.isEmpty()) { 1082 HashMap<String, ArrayList<String> > testGroups = new HashMap<>(); 1083 1084 // Collect all sub testgroups 1085 for (String test : tests) { 1086 String[] components = test.split("\\."); 1087 ArrayList<String> testGroup = testGroups.get(components[0]); 1088 1089 if (testGroup == null) { 1090 testGroup = new ArrayList<String>(); 1091 testGroups.put(components[0], testGroup); 1092 } 1093 1094 testGroup.add(test.substring(components[0].length()+1)); 1095 } 1096 1097 for (String testGroup : testGroups.keySet()) { 1098 if (!first) { 1099 result = result + ","; 1100 } 1101 1102 first = false; 1103 result = result + testGroup 1104 + generateTestCaseTrieFromPaths(testGroups.get(testGroup)); 1105 } 1106 } 1107 1108 return result + "}"; 1109 } 1110 1111 /** 1112 * Generates testcase trie from TestIdentifiers. 1113 */ 1114 private static String generateTestCaseTrie(Collection<TestIdentifier> tests) { 1115 ArrayList<String> testPaths = new ArrayList<String>(); 1116 1117 for (TestIdentifier test : tests) { 1118 testPaths.add(test.getClassName() + "." + test.getTestName()); 1119 } 1120 1121 return generateTestCaseTrieFromPaths(testPaths); 1122 } 1123 1124 private static class TestBatch { 1125 public BatchRunConfiguration config; 1126 public List<TestIdentifier> tests; 1127 } 1128 1129 private TestBatch selectRunBatch() { 1130 return selectRunBatch(mRemainingTests, null); 1131 } 1132 1133 /** 1134 * Creates a TestBatch from the given tests or null if not tests remaining. 1135 * 1136 * @param pool List of tests to select from 1137 * @param requiredConfig Select only instances with pending requiredConfig, or null to select 1138 * any run configuration. 1139 */ 1140 private TestBatch selectRunBatch(Collection<TestIdentifier> pool, 1141 BatchRunConfiguration requiredConfig) { 1142 // select one test (leading test) that is going to be executed and then pack along as many 1143 // other compatible instances as possible. 1144 1145 TestIdentifier leadingTest = null; 1146 for (TestIdentifier test : pool) { 1147 if (!mRemainingTests.contains(test)) { 1148 continue; 1149 } 1150 if (requiredConfig != null && 1151 !mInstanceListerner.isPendingTestInstance(test, requiredConfig)) { 1152 continue; 1153 } 1154 leadingTest = test; 1155 break; 1156 } 1157 1158 // no remaining tests? 1159 if (leadingTest == null) { 1160 return null; 1161 } 1162 1163 BatchRunConfiguration leadingTestConfig = null; 1164 if (requiredConfig != null) { 1165 leadingTestConfig = requiredConfig; 1166 } else { 1167 for (BatchRunConfiguration runConfig : getTestRunConfigs(leadingTest)) { 1168 if (mInstanceListerner.isPendingTestInstance(leadingTest, runConfig)) { 1169 leadingTestConfig = runConfig; 1170 break; 1171 } 1172 } 1173 } 1174 1175 // test pending <=> test has a pending config 1176 if (leadingTestConfig == null) { 1177 throw new AssertionError("search postcondition failed"); 1178 } 1179 1180 final int leadingInstability = getTestInstabilityRating(leadingTest); 1181 1182 final TestBatch runBatch = new TestBatch(); 1183 runBatch.config = leadingTestConfig; 1184 runBatch.tests = new ArrayList<>(); 1185 runBatch.tests.add(leadingTest); 1186 1187 for (TestIdentifier test : pool) { 1188 if (test == leadingTest) { 1189 // do not re-select the leading tests 1190 continue; 1191 } 1192 if (!mInstanceListerner.isPendingTestInstance(test, leadingTestConfig)) { 1193 // select only compatible 1194 continue; 1195 } 1196 if (getTestInstabilityRating(test) != leadingInstability) { 1197 // pack along only cases in the same stability category. Packing more dangerous 1198 // tests along jeopardizes the stability of this run. Packing more stable tests 1199 // along jeopardizes their stability rating. 1200 continue; 1201 } 1202 if (runBatch.tests.size() >= getBatchSizeLimitForInstability(leadingInstability)) { 1203 // batch size is limited. 1204 break; 1205 } 1206 runBatch.tests.add(test); 1207 } 1208 1209 return runBatch; 1210 } 1211 1212 private int getBatchNumPendingCases(TestBatch batch) { 1213 int numPending = 0; 1214 for (TestIdentifier test : batch.tests) { 1215 if (mInstanceListerner.isPendingTestInstance(test, batch.config)) { 1216 ++numPending; 1217 } 1218 } 1219 return numPending; 1220 } 1221 1222 private int getBatchSizeLimitForInstability(int batchInstabilityRating) { 1223 // reduce group size exponentially down to one 1224 return Math.max(1, TESTCASE_BATCH_LIMIT / (1 << batchInstabilityRating)); 1225 } 1226 1227 private int getTestInstabilityRating(TestIdentifier testId) { 1228 if (mTestInstabilityRatings.containsKey(testId)) { 1229 return mTestInstabilityRatings.get(testId); 1230 } else { 1231 return 0; 1232 } 1233 } 1234 1235 private void recordTestInstability(TestIdentifier testId) { 1236 mTestInstabilityRatings.put(testId, getTestInstabilityRating(testId) + 1); 1237 } 1238 1239 private void clearTestInstability(TestIdentifier testId) { 1240 mTestInstabilityRatings.put(testId, 0); 1241 } 1242 1243 /** 1244 * Executes all tests on the device. 1245 */ 1246 private void runTests() throws DeviceNotAvailableException, CapabilityQueryFailureException { 1247 for (;;) { 1248 TestBatch batch = selectRunBatch(); 1249 1250 if (batch == null) { 1251 break; 1252 } 1253 1254 runTestRunBatch(batch); 1255 } 1256 } 1257 1258 /** 1259 * Runs a TestBatch by either faking it or executing it on a device. 1260 */ 1261 private void runTestRunBatch(TestBatch batch) throws DeviceNotAvailableException, 1262 CapabilityQueryFailureException { 1263 // prepare instance listener 1264 mInstanceListerner.setCurrentConfig(batch.config); 1265 for (TestIdentifier test : batch.tests) { 1266 mInstanceListerner.setTestInstances(test, getTestRunConfigs(test)); 1267 } 1268 1269 // execute only if config is executable, else fake results 1270 if (isSupportedRunConfiguration(batch.config)) { 1271 executeTestRunBatch(batch); 1272 } else { 1273 fakePassTestRunBatch(batch); 1274 } 1275 } 1276 1277 private boolean isSupportedRunConfiguration(BatchRunConfiguration runConfig) 1278 throws DeviceNotAvailableException, CapabilityQueryFailureException { 1279 // orientation support 1280 if (!BatchRunConfiguration.ROTATION_UNSPECIFIED.equals(runConfig.getRotation())) { 1281 final Set<String> features = getDeviceFeatures(mDevice); 1282 1283 if (isPortraitClassRotation(runConfig.getRotation()) && 1284 !features.contains(FEATURE_PORTRAIT)) { 1285 return false; 1286 } 1287 if (isLandscapeClassRotation(runConfig.getRotation()) && 1288 !features.contains(FEATURE_LANDSCAPE)) { 1289 return false; 1290 } 1291 } 1292 1293 if (isOpenGlEsPackage()) { 1294 // renderability support for OpenGL ES tests 1295 return isSupportedGlesRenderConfig(runConfig); 1296 } else { 1297 return true; 1298 } 1299 } 1300 1301 private static final class AdbComLinkOpenError extends Exception { 1302 public AdbComLinkOpenError(String description, Throwable inner) { 1303 super(description, inner); 1304 } 1305 }; 1306 private static final class AdbComLinkKilledError extends Exception { 1307 public AdbComLinkKilledError(String description, Throwable inner) { 1308 super(description, inner); 1309 } 1310 }; 1311 1312 /** 1313 * Executes a given command in adb shell 1314 * 1315 * @throws AdbComLinkOpenError if connection cannot be established. 1316 * @throws AdbComLinkKilledError if established connection is killed prematurely. 1317 */ 1318 private void executeShellCommandAndReadOutput(final String command, 1319 final IShellOutputReceiver receiver) 1320 throws AdbComLinkOpenError, AdbComLinkKilledError { 1321 try { 1322 mDevice.getIDevice().executeShellCommand(command, receiver, 1323 UNRESPOSIVE_CMD_TIMEOUT_MS, TimeUnit.MILLISECONDS); 1324 } catch (TimeoutException ex) { 1325 // Opening connection timed out 1326 throw new AdbComLinkOpenError("opening connection timed out", ex); 1327 } catch (AdbCommandRejectedException ex) { 1328 // Command rejected 1329 throw new AdbComLinkOpenError("command rejected", ex); 1330 } catch (IOException ex) { 1331 // shell command channel killed 1332 throw new AdbComLinkKilledError("command link killed", ex); 1333 } catch (ShellCommandUnresponsiveException ex) { 1334 // shell command halted 1335 throw new AdbComLinkKilledError("command link hung", ex); 1336 } 1337 } 1338 1339 /** 1340 * Executes given test batch on a device 1341 */ 1342 private void executeTestRunBatch(TestBatch batch) throws DeviceNotAvailableException { 1343 // attempt full run once 1344 executeTestRunBatchRun(batch); 1345 1346 // split remaining tests to two sub batches and execute both. This will terminate 1347 // since executeTestRunBatchRun will always progress for a batch of size 1. 1348 final ArrayList<TestIdentifier> pendingTests = new ArrayList<>(); 1349 1350 for (TestIdentifier test : batch.tests) { 1351 if (mInstanceListerner.isPendingTestInstance(test, batch.config)) { 1352 pendingTests.add(test); 1353 } 1354 } 1355 1356 final int divisorNdx = pendingTests.size() / 2; 1357 final List<TestIdentifier> headList = pendingTests.subList(0, divisorNdx); 1358 final List<TestIdentifier> tailList = pendingTests.subList(divisorNdx, pendingTests.size()); 1359 1360 // head 1361 for (;;) { 1362 TestBatch subBatch = selectRunBatch(headList, batch.config); 1363 1364 if (subBatch == null) { 1365 break; 1366 } 1367 1368 executeTestRunBatch(subBatch); 1369 } 1370 1371 // tail 1372 for (;;) { 1373 TestBatch subBatch = selectRunBatch(tailList, batch.config); 1374 1375 if (subBatch == null) { 1376 break; 1377 } 1378 1379 executeTestRunBatch(subBatch); 1380 } 1381 1382 if (getBatchNumPendingCases(batch) != 0) { 1383 throw new AssertionError("executeTestRunBatch postcondition failed"); 1384 } 1385 } 1386 1387 /** 1388 * Runs one execution pass over the given batch. 1389 * 1390 * Tries to run the batch. Always makes progress (executes instances or modifies stability 1391 * scores). 1392 */ 1393 private void executeTestRunBatchRun(TestBatch batch) throws DeviceNotAvailableException { 1394 if (getBatchNumPendingCases(batch) != batch.tests.size()) { 1395 throw new AssertionError("executeTestRunBatchRun precondition failed"); 1396 } 1397 1398 final String testCases = generateTestCaseTrie(batch.tests); 1399 1400 mDevice.executeShellCommand("rm " + CASE_LIST_FILE_NAME); 1401 mDevice.executeShellCommand("rm " + LOG_FILE_NAME); 1402 mDevice.pushString(testCases + "\n", CASE_LIST_FILE_NAME); 1403 1404 final String instrumentationName = 1405 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 1406 1407 final StringBuilder deqpCmdLine = new StringBuilder(); 1408 deqpCmdLine.append("--deqp-caselist-file="); 1409 deqpCmdLine.append(CASE_LIST_FILE_NAME); 1410 deqpCmdLine.append(" "); 1411 deqpCmdLine.append(getRunConfigDisplayCmdLine(batch.config)); 1412 1413 // If we are not logging data, do not bother outputting the images from the test exe. 1414 if (!mLogData) { 1415 deqpCmdLine.append(" --deqp-log-images=disable"); 1416 } 1417 1418 deqpCmdLine.append(" --deqp-watchdog=enable"); 1419 1420 final String command = String.format( 1421 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\"" 1422 + " -e deqpLogData \"%s\" %s", 1423 AbiUtils.createAbiFlag(mAbi.getName()), LOG_FILE_NAME, deqpCmdLine.toString(), 1424 mLogData, instrumentationName); 1425 1426 final int numRemainingInstancesBefore = getNumRemainingInstances(); 1427 final InstrumentationParser parser = new InstrumentationParser(mInstanceListerner); 1428 Throwable interruptingError = null; 1429 1430 try { 1431 executeShellCommandAndReadOutput(command, parser); 1432 } catch (Throwable ex) { 1433 interruptingError = ex; 1434 } finally { 1435 parser.flush(); 1436 } 1437 1438 final boolean progressedSinceLastCall = mInstanceListerner.getCurrentTestId() != null || 1439 getNumRemainingInstances() < numRemainingInstancesBefore; 1440 1441 if (progressedSinceLastCall) { 1442 mDeviceRecovery.onExecutionProgressed(); 1443 } 1444 1445 // interrupted, try to recover 1446 if (interruptingError != null) { 1447 if (interruptingError instanceof AdbComLinkOpenError) { 1448 mDeviceRecovery.recoverConnectionRefused(); 1449 } else if (interruptingError instanceof AdbComLinkKilledError) { 1450 mDeviceRecovery.recoverComLinkKilled(); 1451 } else { 1452 CLog.e(interruptingError); 1453 throw new RuntimeException(interruptingError); 1454 } 1455 1456 // recoverXXX did not throw => recovery succeeded 1457 } else if (!parser.wasSuccessful()) { 1458 mDeviceRecovery.recoverComLinkKilled(); 1459 // recoverXXX did not throw => recovery succeeded 1460 } 1461 1462 // Progress guarantees. 1463 if (batch.tests.size() == 1) { 1464 final TestIdentifier onlyTest = batch.tests.iterator().next(); 1465 final boolean wasTestExecuted = 1466 !mInstanceListerner.isPendingTestInstance(onlyTest, batch.config) && 1467 mInstanceListerner.getCurrentTestId() == null; 1468 final boolean wasLinkFailure = !parser.wasSuccessful() || interruptingError != null; 1469 1470 // Link failures can be caused by external events, require at least two observations 1471 // until bailing. 1472 if (!wasTestExecuted && (!wasLinkFailure || getTestInstabilityRating(onlyTest) > 0)) { 1473 recordTestInstability(onlyTest); 1474 // If we cannot finish the test, mark the case as a crash. 1475 // 1476 // If we couldn't even start the test, fail the test instance as non-executable. 1477 // This is required so that a consistently crashing or non-existent tests will 1478 // not cause futile (non-terminating) re-execution attempts. 1479 if (mInstanceListerner.getCurrentTestId() != null) { 1480 mInstanceListerner.abortTest(onlyTest, INCOMPLETE_LOG_MESSAGE); 1481 } else { 1482 mInstanceListerner.abortTest(onlyTest, NOT_EXECUTABLE_LOG_MESSAGE); 1483 } 1484 } else if (wasTestExecuted) { 1485 clearTestInstability(onlyTest); 1486 } 1487 } 1488 else 1489 { 1490 // Analyze results to update test stability ratings. If there is no interrupting test 1491 // logged, increase instability rating of all remaining tests. If there is a 1492 // interrupting test logged, increase only its instability rating. 1493 // 1494 // A successful run of tests clears instability rating. 1495 if (mInstanceListerner.getCurrentTestId() == null) { 1496 for (TestIdentifier test : batch.tests) { 1497 if (mInstanceListerner.isPendingTestInstance(test, batch.config)) { 1498 recordTestInstability(test); 1499 } else { 1500 clearTestInstability(test); 1501 } 1502 } 1503 } else { 1504 recordTestInstability(mInstanceListerner.getCurrentTestId()); 1505 for (TestIdentifier test : batch.tests) { 1506 // \note: isPendingTestInstance is false for getCurrentTestId. Current ID is 1507 // considered 'running' and will be restored to 'pending' in endBatch(). 1508 if (!test.equals(mInstanceListerner.getCurrentTestId()) && 1509 !mInstanceListerner.isPendingTestInstance(test, batch.config)) { 1510 clearTestInstability(test); 1511 } 1512 } 1513 } 1514 } 1515 1516 mInstanceListerner.endBatch(); 1517 } 1518 1519 private static String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) { 1520 final StringBuilder deqpCmdLine = new StringBuilder(); 1521 if (!runConfig.getGlConfig().isEmpty()) { 1522 deqpCmdLine.append("--deqp-gl-config-name="); 1523 deqpCmdLine.append(runConfig.getGlConfig()); 1524 } 1525 if (!runConfig.getRotation().isEmpty()) { 1526 if (deqpCmdLine.length() != 0) { 1527 deqpCmdLine.append(" "); 1528 } 1529 deqpCmdLine.append("--deqp-screen-rotation="); 1530 deqpCmdLine.append(runConfig.getRotation()); 1531 } 1532 if (!runConfig.getSurfaceType().isEmpty()) { 1533 if (deqpCmdLine.length() != 0) { 1534 deqpCmdLine.append(" "); 1535 } 1536 deqpCmdLine.append("--deqp-surface-type="); 1537 deqpCmdLine.append(runConfig.getSurfaceType()); 1538 } 1539 return deqpCmdLine.toString(); 1540 } 1541 1542 private int getNumRemainingInstances() { 1543 int retVal = 0; 1544 for (TestIdentifier testId : mRemainingTests) { 1545 // If case is in current working set, sum only not yet executed instances. 1546 // If case is not in current working set, sum all instances (since they are not yet 1547 // executed). 1548 if (mInstanceListerner.mPendingResults.containsKey(testId)) { 1549 retVal += mInstanceListerner.mPendingResults.get(testId).remainingConfigs.size(); 1550 } else { 1551 retVal += mTestInstances.get(testId).size(); 1552 } 1553 } 1554 return retVal; 1555 } 1556 1557 /** 1558 * Pass given batch tests without running it 1559 */ 1560 private void fakePassTestRunBatch(TestBatch batch) { 1561 for (TestIdentifier test : batch.tests) { 1562 CLog.d("Skipping test '%s' invocation in config '%s'", test.toString(), 1563 batch.config.getId()); 1564 mInstanceListerner.skipTest(test); 1565 } 1566 } 1567 1568 /** 1569 * Pass all remaining tests without running them 1570 */ 1571 private void fakePassTests(ITestInvocationListener listener) { 1572 Map <String, String> emptyMap = Collections.emptyMap(); 1573 for (TestIdentifier test : mRemainingTests) { 1574 CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString()); 1575 listener.testStarted(test); 1576 listener.testEnded(test, emptyMap); 1577 } 1578 mRemainingTests.clear(); 1579 } 1580 1581 /** 1582 * Check if device supports OpenGL ES version. 1583 */ 1584 private static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion, 1585 int requiredMinorVersion) throws DeviceNotAvailableException { 1586 String roOpenglesVersion = device.getProperty("ro.opengles.version"); 1587 1588 if (roOpenglesVersion == null) 1589 return false; 1590 1591 int intValue = Integer.parseInt(roOpenglesVersion); 1592 1593 int majorVersion = ((intValue & 0xffff0000) >> 16); 1594 int minorVersion = (intValue & 0xffff); 1595 1596 return (majorVersion > requiredMajorVersion) 1597 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion); 1598 } 1599 1600 /** 1601 * Query if rendertarget is supported 1602 */ 1603 private boolean isSupportedGlesRenderConfig(BatchRunConfiguration runConfig) 1604 throws DeviceNotAvailableException, CapabilityQueryFailureException { 1605 // query if configuration is supported 1606 final StringBuilder configCommandLine = 1607 new StringBuilder(getRunConfigDisplayCmdLine(runConfig)); 1608 if (configCommandLine.length() != 0) { 1609 configCommandLine.append(" "); 1610 } 1611 configCommandLine.append("--deqp-gl-major-version="); 1612 configCommandLine.append(getGlesMajorVersion()); 1613 configCommandLine.append(" --deqp-gl-minor-version="); 1614 configCommandLine.append(getGlesMinorVersion()); 1615 1616 final String commandLine = configCommandLine.toString(); 1617 1618 // check for cached result first 1619 if (mConfigQuerySupportCache.containsKey(commandLine)) { 1620 return mConfigQuerySupportCache.get(commandLine); 1621 } 1622 1623 final boolean supported = queryIsSupportedConfigCommandLine(commandLine); 1624 mConfigQuerySupportCache.put(commandLine, supported); 1625 return supported; 1626 } 1627 1628 private boolean queryIsSupportedConfigCommandLine(String deqpCommandLine) 1629 throws DeviceNotAvailableException, CapabilityQueryFailureException { 1630 final String instrumentationName = 1631 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 1632 final String command = String.format( 1633 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine \"%s\"" 1634 + " %s", 1635 AbiUtils.createAbiFlag(mAbi.getName()), deqpCommandLine, instrumentationName); 1636 1637 final PlatformQueryInstrumentationParser parser = new PlatformQueryInstrumentationParser(); 1638 mDevice.executeShellCommand(command, parser); 1639 parser.flush(); 1640 1641 if (parser.wasSuccessful() && parser.getResultCode() == 0 && 1642 parser.getResultMap().containsKey("Supported")) { 1643 if ("Yes".equals(parser.getResultMap().get("Supported"))) { 1644 return true; 1645 } else if ("No".equals(parser.getResultMap().get("Supported"))) { 1646 return false; 1647 } else { 1648 CLog.e("Capability query did not return a result"); 1649 throw new CapabilityQueryFailureException(); 1650 } 1651 } else if (parser.wasSuccessful()) { 1652 CLog.e("Failed to run capability query. Code: %d, Result: %s", 1653 parser.getResultCode(), parser.getResultMap().toString()); 1654 throw new CapabilityQueryFailureException(); 1655 } else { 1656 CLog.e("Failed to run capability query"); 1657 throw new CapabilityQueryFailureException(); 1658 } 1659 } 1660 1661 /** 1662 * Return feature set supported by the device 1663 */ 1664 private Set<String> getDeviceFeatures(ITestDevice device) 1665 throws DeviceNotAvailableException, CapabilityQueryFailureException { 1666 if (mDeviceFeatures == null) { 1667 mDeviceFeatures = queryDeviceFeatures(device); 1668 } 1669 return mDeviceFeatures; 1670 } 1671 1672 /** 1673 * Query feature set supported by the device 1674 */ 1675 private static Set<String> queryDeviceFeatures(ITestDevice device) 1676 throws DeviceNotAvailableException, CapabilityQueryFailureException { 1677 // NOTE: Almost identical code in BaseDevicePolicyTest#hasDeviceFeatures 1678 // TODO: Move this logic to ITestDevice. 1679 String command = "pm list features"; 1680 String commandOutput = device.executeShellCommand(command); 1681 1682 // Extract the id of the new user. 1683 HashSet<String> availableFeatures = new HashSet<>(); 1684 for (String feature: commandOutput.split("\\s+")) { 1685 // Each line in the output of the command has the format "feature:{FEATURE_VALUE}". 1686 String[] tokens = feature.split(":"); 1687 if (tokens.length < 2 || !"feature".equals(tokens[0])) { 1688 CLog.e("Failed parse features. Unexpect format on line \"%s\"", tokens[0]); 1689 throw new CapabilityQueryFailureException(); 1690 } 1691 availableFeatures.add(tokens[1]); 1692 } 1693 return availableFeatures; 1694 } 1695 1696 private boolean isPortraitClassRotation(String rotation) { 1697 return BatchRunConfiguration.ROTATION_PORTRAIT.equals(rotation) || 1698 BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT.equals(rotation); 1699 } 1700 1701 private boolean isLandscapeClassRotation(String rotation) { 1702 return BatchRunConfiguration.ROTATION_LANDSCAPE.equals(rotation) || 1703 BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE.equals(rotation); 1704 } 1705 1706 /** 1707 * Install dEQP OnDevice Package 1708 */ 1709 private void installTestApk() throws DeviceNotAvailableException { 1710 try { 1711 File apkFile = mCtsBuild.getTestApp(DEQP_ONDEVICE_APK); 1712 String[] options = {AbiUtils.createAbiFlag(mAbi.getName())}; 1713 String errorCode = getDevice().installPackage(apkFile, true, options); 1714 if (errorCode != null) { 1715 CLog.e("Failed to install %s. Reason: %s", DEQP_ONDEVICE_APK, errorCode); 1716 } 1717 } catch (FileNotFoundException e) { 1718 CLog.e("Could not find test apk %s", DEQP_ONDEVICE_APK); 1719 } 1720 } 1721 1722 /** 1723 * Uninstall dEQP OnDevice Package 1724 */ 1725 private void uninstallTestApk() throws DeviceNotAvailableException { 1726 getDevice().uninstallPackage(DEQP_ONDEVICE_PKG); 1727 } 1728 1729 /** 1730 * Parse gl nature from package name 1731 */ 1732 private boolean isOpenGlEsPackage() { 1733 if ("dEQP-GLES2".equals(mName) || "dEQP-GLES3".equals(mName) || 1734 "dEQP-GLES31".equals(mName)) { 1735 return true; 1736 } else if ("dEQP-EGL".equals(mName)) { 1737 return false; 1738 } else { 1739 throw new IllegalStateException("dEQP runner was created with illegal name"); 1740 } 1741 } 1742 1743 /** 1744 * Check GL support (based on package name) 1745 */ 1746 private boolean isSupportedGles() throws DeviceNotAvailableException { 1747 return isSupportedGles(mDevice, getGlesMajorVersion(), getGlesMinorVersion()); 1748 } 1749 1750 /** 1751 * Get GL major version (based on package name) 1752 */ 1753 private int getGlesMajorVersion() throws DeviceNotAvailableException { 1754 if ("dEQP-GLES2".equals(mName)) { 1755 return 2; 1756 } else if ("dEQP-GLES3".equals(mName)) { 1757 return 3; 1758 } else if ("dEQP-GLES31".equals(mName)) { 1759 return 3; 1760 } else { 1761 throw new IllegalStateException("getGlesMajorVersion called for non gles pkg"); 1762 } 1763 } 1764 1765 /** 1766 * Get GL minor version (based on package name) 1767 */ 1768 private int getGlesMinorVersion() throws DeviceNotAvailableException { 1769 if ("dEQP-GLES2".equals(mName)) { 1770 return 0; 1771 } else if ("dEQP-GLES3".equals(mName)) { 1772 return 0; 1773 } else if ("dEQP-GLES31".equals(mName)) { 1774 return 1; 1775 } else { 1776 throw new IllegalStateException("getGlesMinorVersion called for non gles pkg"); 1777 } 1778 } 1779 1780 /** 1781 * {@inheritDoc} 1782 */ 1783 @Override 1784 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 1785 final Map<String, String> emptyMap = Collections.emptyMap(); 1786 final boolean isSupportedApi = !isOpenGlEsPackage() || isSupportedGles(); 1787 1788 listener.testRunStarted(getId(), mRemainingTests.size()); 1789 1790 try { 1791 if (isSupportedApi) { 1792 // Make sure there is no pre-existing package form earlier interrupted test run. 1793 uninstallTestApk(); 1794 installTestApk(); 1795 1796 mInstanceListerner.setSink(listener); 1797 mDeviceRecovery.setDevice(mDevice); 1798 runTests(); 1799 1800 uninstallTestApk(); 1801 } else { 1802 // Pass all tests if OpenGL ES version is not supported 1803 fakePassTests(listener); 1804 } 1805 } catch (CapabilityQueryFailureException ex) { 1806 // Platform is not behaving correctly, for example crashing when trying to create 1807 // a window. Instead of silenty failing, signal failure by leaving the rest of the 1808 // test cases in "NotExecuted" state 1809 uninstallTestApk(); 1810 } 1811 1812 listener.testRunEnded(0, emptyMap); 1813 } 1814} 1815