1/* 2 * Copyright (C) 2007 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.test; 18 19import static android.test.suitebuilder.TestPredicates.REJECT_PERFORMANCE; 20 21import com.android.internal.util.Predicate; 22import com.android.internal.util.Predicates; 23 24import android.app.Activity; 25import android.app.Instrumentation; 26import android.os.Bundle; 27import android.os.Debug; 28import android.os.Looper; 29import android.os.Parcelable; 30import android.os.PerformanceCollector; 31import android.os.PerformanceCollector.PerformanceResultsWriter; 32import android.test.suitebuilder.TestMethod; 33import android.test.suitebuilder.TestPredicates; 34import android.test.suitebuilder.TestSuiteBuilder; 35import android.test.suitebuilder.annotation.HasAnnotation; 36import android.test.suitebuilder.annotation.LargeTest; 37import android.util.Log; 38 39import java.io.ByteArrayOutputStream; 40import java.io.File; 41import java.io.PrintStream; 42import java.lang.annotation.Annotation; 43import java.lang.reflect.InvocationTargetException; 44import java.lang.reflect.Method; 45import java.util.ArrayList; 46import java.util.List; 47 48import junit.framework.AssertionFailedError; 49import junit.framework.Test; 50import junit.framework.TestCase; 51import junit.framework.TestListener; 52import junit.framework.TestResult; 53import junit.framework.TestSuite; 54import junit.runner.BaseTestRunner; 55import junit.textui.ResultPrinter; 56 57/** 58 * An {@link Instrumentation} that runs various types of {@link junit.framework.TestCase}s against 59 * an Android package (application). Typical usage: 60 * <ol> 61 * <li>Write {@link junit.framework.TestCase}s that perform unit, functional, or performance tests 62 * against the classes in your package. Typically these are subclassed from: 63 * <ul><li>{@link android.test.ActivityInstrumentationTestCase2}</li> 64 * <li>{@link android.test.ActivityUnitTestCase}</li> 65 * <li>{@link android.test.AndroidTestCase}</li> 66 * <li>{@link android.test.ApplicationTestCase}</li> 67 * <li>{@link android.test.InstrumentationTestCase}</li> 68 * <li>{@link android.test.ProviderTestCase}</li> 69 * <li>{@link android.test.ServiceTestCase}</li> 70 * <li>{@link android.test.SingleLaunchActivityTestCase}</li></ul> 71 * <li>In an appropriate AndroidManifest.xml, define the this instrumentation with 72 * the appropriate android:targetPackage set. 73 * <li>Run the instrumentation using "adb shell am instrument -w", 74 * with no optional arguments, to run all tests (except performance tests). 75 * <li>Run the instrumentation using "adb shell am instrument -w", 76 * with the argument '-e func true' to run all functional tests. These are tests that derive from 77 * {@link android.test.InstrumentationTestCase}. 78 * <li>Run the instrumentation using "adb shell am instrument -w", 79 * with the argument '-e unit true' to run all unit tests. These are tests that <i>do not</i>derive 80 * from {@link android.test.InstrumentationTestCase} (and are not performance tests). 81 * <li>Run the instrumentation using "adb shell am instrument -w", 82 * with the argument '-e class' set to run an individual {@link junit.framework.TestCase}. 83 * </ol> 84 * <p/> 85 * <b>Running all tests:</b> adb shell am instrument -w 86 * com.android.foo/android.test.InstrumentationTestRunner 87 * <p/> 88 * <b>Running all small tests:</b> adb shell am instrument -w 89 * -e size small 90 * com.android.foo/android.test.InstrumentationTestRunner 91 * <p/> 92 * <b>Running all medium tests:</b> adb shell am instrument -w 93 * -e size medium 94 * com.android.foo/android.test.InstrumentationTestRunner 95 * <p/> 96 * <b>Running all large tests:</b> adb shell am instrument -w 97 * -e size large 98 * com.android.foo/android.test.InstrumentationTestRunner 99 * <p/> 100 * <b>Filter test run to tests with given annotation:</b> adb shell am instrument -w 101 * -e annotation com.android.foo.MyAnnotation 102 * com.android.foo/android.test.InstrumentationTestRunner 103 * <p/> 104 * If used with other options, the resulting test run will contain the union of the two options. 105 * e.g. "-e size large -e annotation com.android.foo.MyAnnotation" will run only tests with both 106 * the {@link LargeTest} and "com.android.foo.MyAnnotation" annotations. 107 * <p/> 108 * <b>Filter test run to tests <i>without</i> given annotation:</b> adb shell am instrument -w 109 * -e notAnnotation com.android.foo.MyAnnotation 110 * com.android.foo/android.test.InstrumentationTestRunner 111 * <p/> 112 * <b>Running a single testcase:</b> adb shell am instrument -w 113 * -e class com.android.foo.FooTest 114 * com.android.foo/android.test.InstrumentationTestRunner 115 * <p/> 116 * <b>Running a single test:</b> adb shell am instrument -w 117 * -e class com.android.foo.FooTest#testFoo 118 * com.android.foo/android.test.InstrumentationTestRunner 119 * <p/> 120 * <b>Running multiple tests:</b> adb shell am instrument -w 121 * -e class com.android.foo.FooTest,com.android.foo.TooTest 122 * com.android.foo/android.test.InstrumentationTestRunner 123 * <p/> 124 * <b>Running all tests in a java package:</b> adb shell am instrument -w 125 * -e package com.android.foo.subpkg 126 * com.android.foo/android.test.InstrumentationTestRunner 127 * <p/> 128 * <b>Including performance tests:</b> adb shell am instrument -w 129 * -e perf true 130 * com.android.foo/android.test.InstrumentationTestRunner 131 * <p/> 132 * <b>To debug your tests, set a break point in your code and pass:</b> 133 * -e debug true 134 * <p/> 135 * <b>To run in 'log only' mode</b> 136 * -e log true 137 * This option will load and iterate through all test classes and methods, but will bypass actual 138 * test execution. Useful for quickly obtaining info on the tests to be executed by an 139 * instrumentation command. 140 * <p/> 141 * <b>To generate EMMA code coverage:</b> 142 * -e coverage true 143 * Note: this requires an emma instrumented build. By default, the code coverage results file 144 * will be saved in a /data/<app>/coverage.ec file, unless overridden by coverageFile flag (see 145 * below) 146 * <p/> 147 * <b> To specify EMMA code coverage results file path:</b> 148 * -e coverageFile /sdcard/myFile.ec 149 * <br/> 150 * in addition to the other arguments. 151 */ 152 153/* (not JavaDoc) 154 * Although not necessary in most case, another way to use this class is to extend it and have the 155 * derived class return the desired test suite from the {@link #getTestSuite()} method. The test 156 * suite returned from this method will be used if no target class is defined in the meta-data or 157 * command line argument parameters. If a derived class is used it needs to be added as an 158 * instrumentation to the AndroidManifest.xml and the command to run it would look like: 159 * <p/> 160 * adb shell am instrument -w com.android.foo/<i>com.android.FooInstrumentationTestRunner</i> 161 * <p/> 162 * Where <i>com.android.FooInstrumentationTestRunner</i> is the derived class. 163 * 164 * This model is used by many existing app tests, but can probably be deprecated. 165 */ 166public class InstrumentationTestRunner extends Instrumentation implements TestSuiteProvider { 167 168 /** @hide */ 169 public static final String ARGUMENT_TEST_CLASS = "class"; 170 /** @hide */ 171 public static final String ARGUMENT_TEST_PACKAGE = "package"; 172 /** @hide */ 173 public static final String ARGUMENT_TEST_SIZE_PREDICATE = "size"; 174 /** @hide */ 175 public static final String ARGUMENT_INCLUDE_PERF = "perf"; 176 /** @hide */ 177 public static final String ARGUMENT_DELAY_MSEC = "delay_msec"; 178 179 private static final String SMALL_SUITE = "small"; 180 private static final String MEDIUM_SUITE = "medium"; 181 private static final String LARGE_SUITE = "large"; 182 183 private static final String ARGUMENT_LOG_ONLY = "log"; 184 /** @hide */ 185 static final String ARGUMENT_ANNOTATION = "annotation"; 186 /** @hide */ 187 static final String ARGUMENT_NOT_ANNOTATION = "notAnnotation"; 188 189 /** 190 * This constant defines the maximum allowed runtime (in ms) for a test included in the "small" 191 * suite. It is used to make an educated guess at what suite an unlabeled test belongs. 192 */ 193 private static final float SMALL_SUITE_MAX_RUNTIME = 100; 194 195 /** 196 * This constant defines the maximum allowed runtime (in ms) for a test included in the 197 * "medium" suite. It is used to make an educated guess at what suite an unlabeled test belongs. 198 */ 199 private static final float MEDIUM_SUITE_MAX_RUNTIME = 1000; 200 201 /** 202 * The following keys are used in the status bundle to provide structured reports to 203 * an IInstrumentationWatcher. 204 */ 205 206 /** 207 * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER}, 208 * identifies InstrumentationTestRunner as the source of the report. This is sent with all 209 * status messages. 210 */ 211 public static final String REPORT_VALUE_ID = "InstrumentationTestRunner"; 212 /** 213 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 214 * identifies the total number of tests that are being run. This is sent with all status 215 * messages. 216 */ 217 public static final String REPORT_KEY_NUM_TOTAL = "numtests"; 218 /** 219 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 220 * identifies the sequence number of the current test. This is sent with any status message 221 * describing a specific test being started or completed. 222 */ 223 public static final String REPORT_KEY_NUM_CURRENT = "current"; 224 /** 225 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 226 * identifies the name of the current test class. This is sent with any status message 227 * describing a specific test being started or completed. 228 */ 229 public static final String REPORT_KEY_NAME_CLASS = "class"; 230 /** 231 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 232 * identifies the name of the current test. This is sent with any status message 233 * describing a specific test being started or completed. 234 */ 235 public static final String REPORT_KEY_NAME_TEST = "test"; 236 /** 237 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 238 * reports the run time in seconds of the current test. 239 */ 240 private static final String REPORT_KEY_RUN_TIME = "runtime"; 241 /** 242 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 243 * reports the guessed suite assignment for the current test. 244 */ 245 private static final String REPORT_KEY_SUITE_ASSIGNMENT = "suiteassignment"; 246 /** 247 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 248 * identifies the path to the generated code coverage file. 249 */ 250 private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath"; 251 252 /** 253 * The test is starting. 254 */ 255 public static final int REPORT_VALUE_RESULT_START = 1; 256 /** 257 * The test completed successfully. 258 */ 259 public static final int REPORT_VALUE_RESULT_OK = 0; 260 /** 261 * The test completed with an error. 262 */ 263 public static final int REPORT_VALUE_RESULT_ERROR = -1; 264 /** 265 * The test completed with a failure. 266 */ 267 public static final int REPORT_VALUE_RESULT_FAILURE = -2; 268 /** 269 * If included in the status bundle sent to an IInstrumentationWatcher, this key 270 * identifies a stack trace describing an error or failure. This is sent with any status 271 * message describing a specific test being completed. 272 */ 273 public static final String REPORT_KEY_STACK = "stack"; 274 275 // Default file name for code coverage 276 private static final String DEFAULT_COVERAGE_FILE_NAME = "coverage.ec"; 277 278 private static final String LOG_TAG = "InstrumentationTestRunner"; 279 280 private final Bundle mResults = new Bundle(); 281 private AndroidTestRunner mTestRunner; 282 private boolean mDebug; 283 private boolean mJustCount; 284 private boolean mSuiteAssignmentMode; 285 private int mTestCount; 286 private String mPackageOfTests; 287 private boolean mCoverage; 288 private String mCoverageFilePath; 289 private int mDelayMsec; 290 291 @Override 292 public void onCreate(Bundle arguments) { 293 super.onCreate(arguments); 294 295 // Apk paths used to search for test classes when using TestSuiteBuilders. 296 String[] apkPaths = 297 {getTargetContext().getPackageCodePath(), getContext().getPackageCodePath()}; 298 ClassPathPackageInfoSource.setApkPaths(apkPaths); 299 300 Predicate<TestMethod> testSizePredicate = null; 301 Predicate<TestMethod> testAnnotationPredicate = null; 302 Predicate<TestMethod> testNotAnnotationPredicate = null; 303 boolean includePerformance = false; 304 String testClassesArg = null; 305 boolean logOnly = false; 306 307 if (arguments != null) { 308 // Test class name passed as an argument should override any meta-data declaration. 309 testClassesArg = arguments.getString(ARGUMENT_TEST_CLASS); 310 mDebug = getBooleanArgument(arguments, "debug"); 311 mJustCount = getBooleanArgument(arguments, "count"); 312 mSuiteAssignmentMode = getBooleanArgument(arguments, "suiteAssignment"); 313 mPackageOfTests = arguments.getString(ARGUMENT_TEST_PACKAGE); 314 testSizePredicate = getSizePredicateFromArg( 315 arguments.getString(ARGUMENT_TEST_SIZE_PREDICATE)); 316 testAnnotationPredicate = getAnnotationPredicate( 317 arguments.getString(ARGUMENT_ANNOTATION)); 318 testNotAnnotationPredicate = getNotAnnotationPredicate( 319 arguments.getString(ARGUMENT_NOT_ANNOTATION)); 320 321 includePerformance = getBooleanArgument(arguments, ARGUMENT_INCLUDE_PERF); 322 logOnly = getBooleanArgument(arguments, ARGUMENT_LOG_ONLY); 323 mCoverage = getBooleanArgument(arguments, "coverage"); 324 mCoverageFilePath = arguments.getString("coverageFile"); 325 326 try { 327 Object delay = arguments.get(ARGUMENT_DELAY_MSEC); // Accept either string or int 328 if (delay != null) mDelayMsec = Integer.parseInt(delay.toString()); 329 } catch (NumberFormatException e) { 330 Log.e(LOG_TAG, "Invalid delay_msec parameter", e); 331 } 332 } 333 334 TestSuiteBuilder testSuiteBuilder = new TestSuiteBuilder(getClass().getName(), 335 getTargetContext().getClassLoader()); 336 337 if (testSizePredicate != null) { 338 testSuiteBuilder.addRequirements(testSizePredicate); 339 } 340 if (testAnnotationPredicate != null) { 341 testSuiteBuilder.addRequirements(testAnnotationPredicate); 342 } 343 if (testNotAnnotationPredicate != null) { 344 testSuiteBuilder.addRequirements(testNotAnnotationPredicate); 345 } 346 if (!includePerformance) { 347 testSuiteBuilder.addRequirements(REJECT_PERFORMANCE); 348 } 349 350 if (testClassesArg == null) { 351 if (mPackageOfTests != null) { 352 testSuiteBuilder.includePackages(mPackageOfTests); 353 } else { 354 TestSuite testSuite = getTestSuite(); 355 if (testSuite != null) { 356 testSuiteBuilder.addTestSuite(testSuite); 357 } else { 358 // no package or class bundle arguments were supplied, and no test suite 359 // provided so add all tests in application 360 testSuiteBuilder.includePackages(""); 361 } 362 } 363 } else { 364 parseTestClasses(testClassesArg, testSuiteBuilder); 365 } 366 367 testSuiteBuilder.addRequirements(getBuilderRequirements()); 368 369 mTestRunner = getAndroidTestRunner(); 370 mTestRunner.setContext(getTargetContext()); 371 mTestRunner.setInstrumentation(this); 372 mTestRunner.setSkipExecution(logOnly); 373 mTestRunner.setTest(testSuiteBuilder.build()); 374 mTestCount = mTestRunner.getTestCases().size(); 375 if (mSuiteAssignmentMode) { 376 mTestRunner.addTestListener(new SuiteAssignmentPrinter()); 377 } else { 378 WatcherResultPrinter resultPrinter = new WatcherResultPrinter(mTestCount); 379 mTestRunner.addTestListener(new TestPrinter("TestRunner", false)); 380 mTestRunner.addTestListener(resultPrinter); 381 mTestRunner.setPerformanceResultsWriter(resultPrinter); 382 } 383 start(); 384 } 385 386 List<Predicate<TestMethod>> getBuilderRequirements() { 387 return new ArrayList<Predicate<TestMethod>>(); 388 } 389 390 /** 391 * Parses and loads the specified set of test classes 392 * 393 * @param testClassArg - comma-separated list of test classes and methods 394 * @param testSuiteBuilder - builder to add tests to 395 */ 396 private void parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder) { 397 String[] testClasses = testClassArg.split(","); 398 for (String testClass : testClasses) { 399 parseTestClass(testClass, testSuiteBuilder); 400 } 401 } 402 403 /** 404 * Parse and load the given test class and, optionally, method 405 * 406 * @param testClassName - full package name of test class and optionally method to add. 407 * Expected format: com.android.TestClass#testMethod 408 * @param testSuiteBuilder - builder to add tests to 409 */ 410 private void parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder) { 411 int methodSeparatorIndex = testClassName.indexOf('#'); 412 String testMethodName = null; 413 414 if (methodSeparatorIndex > 0) { 415 testMethodName = testClassName.substring(methodSeparatorIndex + 1); 416 testClassName = testClassName.substring(0, methodSeparatorIndex); 417 } 418 testSuiteBuilder.addTestClassByName(testClassName, testMethodName, getTargetContext()); 419 } 420 421 protected AndroidTestRunner getAndroidTestRunner() { 422 return new AndroidTestRunner(); 423 } 424 425 private boolean getBooleanArgument(Bundle arguments, String tag) { 426 String tagString = arguments.getString(tag); 427 return tagString != null && Boolean.parseBoolean(tagString); 428 } 429 430 /* 431 * Returns the size predicate object, corresponding to the "size" argument value. 432 */ 433 private Predicate<TestMethod> getSizePredicateFromArg(String sizeArg) { 434 435 if (SMALL_SUITE.equals(sizeArg)) { 436 return TestPredicates.SELECT_SMALL; 437 } else if (MEDIUM_SUITE.equals(sizeArg)) { 438 return TestPredicates.SELECT_MEDIUM; 439 } else if (LARGE_SUITE.equals(sizeArg)) { 440 return TestPredicates.SELECT_LARGE; 441 } else { 442 return null; 443 } 444 } 445 446 /** 447 * Returns the test predicate object, corresponding to the annotation class value provided via 448 * the {@link ARGUMENT_ANNOTATION} argument. 449 * 450 * @return the predicate or <code>null</code> 451 */ 452 private Predicate<TestMethod> getAnnotationPredicate(String annotationClassName) { 453 Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 454 if (annotationClass != null) { 455 return new HasAnnotation(annotationClass); 456 } 457 return null; 458 } 459 460 /** 461 * Returns the negative test predicate object, corresponding to the annotation class value 462 * provided via the {@link ARGUMENT_NOT_ANNOTATION} argument. 463 * 464 * @return the predicate or <code>null</code> 465 */ 466 private Predicate<TestMethod> getNotAnnotationPredicate(String annotationClassName) { 467 Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 468 if (annotationClass != null) { 469 return Predicates.not(new HasAnnotation(annotationClass)); 470 } 471 return null; 472 } 473 474 /** 475 * Helper method to return the annotation class with specified name 476 * 477 * @param annotationClassName the fully qualified name of the class 478 * @return the annotation class or <code>null</code> 479 */ 480 private Class<? extends Annotation> getAnnotationClass(String annotationClassName) { 481 if (annotationClassName == null) { 482 return null; 483 } 484 try { 485 Class<?> annotationClass = Class.forName(annotationClassName); 486 if (annotationClass.isAnnotation()) { 487 return (Class<? extends Annotation>)annotationClass; 488 } else { 489 Log.e(LOG_TAG, String.format("Provided annotation value %s is not an Annotation", 490 annotationClassName)); 491 } 492 } catch (ClassNotFoundException e) { 493 Log.e(LOG_TAG, String.format("Could not find class for specified annotation %s", 494 annotationClassName)); 495 } 496 return null; 497 } 498 499 @Override 500 public void onStart() { 501 Looper.prepare(); 502 503 if (mJustCount) { 504 mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 505 mResults.putInt(REPORT_KEY_NUM_TOTAL, mTestCount); 506 finish(Activity.RESULT_OK, mResults); 507 } else { 508 if (mDebug) { 509 Debug.waitForDebugger(); 510 } 511 512 ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 513 PrintStream writer = new PrintStream(byteArrayOutputStream); 514 try { 515 StringResultPrinter resultPrinter = new StringResultPrinter(writer); 516 517 mTestRunner.addTestListener(resultPrinter); 518 519 long startTime = System.currentTimeMillis(); 520 mTestRunner.runTest(); 521 long runTime = System.currentTimeMillis() - startTime; 522 523 resultPrinter.print(mTestRunner.getTestResult(), runTime); 524 } finally { 525 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 526 String.format("\nTest results for %s=%s", 527 mTestRunner.getTestClassName(), 528 byteArrayOutputStream.toString())); 529 530 if (mCoverage) { 531 generateCoverageReport(); 532 } 533 writer.close(); 534 535 finish(Activity.RESULT_OK, mResults); 536 } 537 } 538 } 539 540 public TestSuite getTestSuite() { 541 return getAllTests(); 542 } 543 544 /** 545 * Override this to define all of the tests to run in your package. 546 */ 547 public TestSuite getAllTests() { 548 return null; 549 } 550 551 /** 552 * Override this to provide access to the class loader of your package. 553 */ 554 public ClassLoader getLoader() { 555 return null; 556 } 557 558 private void generateCoverageReport() { 559 // use reflection to call emma dump coverage method, to avoid 560 // always statically compiling against emma jar 561 String coverageFilePath = getCoverageFilePath(); 562 java.io.File coverageFile = new java.io.File(coverageFilePath); 563 try { 564 Class<?> emmaRTClass = Class.forName("com.vladium.emma.rt.RT"); 565 Method dumpCoverageMethod = emmaRTClass.getMethod("dumpCoverageData", 566 coverageFile.getClass(), boolean.class, boolean.class); 567 568 dumpCoverageMethod.invoke(null, coverageFile, false, false); 569 // output path to generated coverage file so it can be parsed by a test harness if 570 // needed 571 mResults.putString(REPORT_KEY_COVERAGE_PATH, coverageFilePath); 572 // also output a more user friendly msg 573 final String currentStream = mResults.getString( 574 Instrumentation.REPORT_KEY_STREAMRESULT); 575 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 576 String.format("%s\nGenerated code coverage data to %s", currentStream, 577 coverageFilePath)); 578 } catch (ClassNotFoundException e) { 579 reportEmmaError("Is emma jar on classpath?", e); 580 } catch (SecurityException e) { 581 reportEmmaError(e); 582 } catch (NoSuchMethodException e) { 583 reportEmmaError(e); 584 } catch (IllegalArgumentException e) { 585 reportEmmaError(e); 586 } catch (IllegalAccessException e) { 587 reportEmmaError(e); 588 } catch (InvocationTargetException e) { 589 reportEmmaError(e); 590 } 591 } 592 593 private String getCoverageFilePath() { 594 if (mCoverageFilePath == null) { 595 return getTargetContext().getFilesDir().getAbsolutePath() + File.separator + 596 DEFAULT_COVERAGE_FILE_NAME; 597 } else { 598 return mCoverageFilePath; 599 } 600 } 601 602 private void reportEmmaError(Exception e) { 603 reportEmmaError("", e); 604 } 605 606 private void reportEmmaError(String hint, Exception e) { 607 String msg = "Failed to generate emma coverage. " + hint; 608 Log.e(LOG_TAG, msg, e); 609 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "\nError: " + msg); 610 } 611 612 // TODO kill this, use status() and prettyprint model for better output 613 private class StringResultPrinter extends ResultPrinter { 614 615 public StringResultPrinter(PrintStream writer) { 616 super(writer); 617 } 618 619 synchronized void print(TestResult result, long runTime) { 620 printHeader(runTime); 621 printFooter(result); 622 } 623 } 624 625 /** 626 * This class sends status reports back to the IInstrumentationWatcher about 627 * which suite each test belongs. 628 */ 629 private class SuiteAssignmentPrinter implements TestListener { 630 631 private Bundle mTestResult; 632 private long mStartTime; 633 private long mEndTime; 634 private boolean mTimingValid; 635 636 public SuiteAssignmentPrinter() { 637 } 638 639 /** 640 * send a status for the start of a each test, so long tests can be seen as "running" 641 */ 642 public void startTest(Test test) { 643 mTimingValid = true; 644 mStartTime = System.currentTimeMillis(); 645 } 646 647 /** 648 * @see junit.framework.TestListener#addError(Test, Throwable) 649 */ 650 public void addError(Test test, Throwable t) { 651 mTimingValid = false; 652 } 653 654 /** 655 * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 656 */ 657 public void addFailure(Test test, AssertionFailedError t) { 658 mTimingValid = false; 659 } 660 661 /** 662 * @see junit.framework.TestListener#endTest(Test) 663 */ 664 public void endTest(Test test) { 665 float runTime; 666 String assignmentSuite; 667 mEndTime = System.currentTimeMillis(); 668 mTestResult = new Bundle(); 669 670 if (!mTimingValid || mStartTime < 0) { 671 assignmentSuite = "NA"; 672 runTime = -1; 673 } else { 674 runTime = mEndTime - mStartTime; 675 if (runTime < SMALL_SUITE_MAX_RUNTIME 676 && !InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { 677 assignmentSuite = SMALL_SUITE; 678 } else if (runTime < MEDIUM_SUITE_MAX_RUNTIME) { 679 assignmentSuite = MEDIUM_SUITE; 680 } else { 681 assignmentSuite = LARGE_SUITE; 682 } 683 } 684 // Clear mStartTime so that we can verify that it gets set next time. 685 mStartTime = -1; 686 687 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 688 test.getClass().getName() + "#" + ((TestCase) test).getName() 689 + "\nin " + assignmentSuite + " suite\nrunTime: " 690 + String.valueOf(runTime) + "\n"); 691 mTestResult.putFloat(REPORT_KEY_RUN_TIME, runTime); 692 mTestResult.putString(REPORT_KEY_SUITE_ASSIGNMENT, assignmentSuite); 693 694 sendStatus(0, mTestResult); 695 } 696 } 697 698 /** 699 * This class sends status reports back to the IInstrumentationWatcher 700 */ 701 private class WatcherResultPrinter implements TestListener, PerformanceResultsWriter { 702 private final Bundle mResultTemplate; 703 Bundle mTestResult; 704 int mTestNum = 0; 705 int mTestResultCode = 0; 706 String mTestClass = null; 707 PerformanceCollector mPerfCollector = new PerformanceCollector(); 708 boolean mIsTimedTest = false; 709 boolean mIncludeDetailedStats = false; 710 711 public WatcherResultPrinter(int numTests) { 712 mResultTemplate = new Bundle(); 713 mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 714 mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests); 715 } 716 717 /** 718 * send a status for the start of a each test, so long tests can be seen 719 * as "running" 720 */ 721 public void startTest(Test test) { 722 String testClass = test.getClass().getName(); 723 String testName = ((TestCase)test).getName(); 724 mTestResult = new Bundle(mResultTemplate); 725 mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass); 726 mTestResult.putString(REPORT_KEY_NAME_TEST, testName); 727 mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum); 728 // pretty printing 729 if (testClass != null && !testClass.equals(mTestClass)) { 730 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 731 String.format("\n%s:", testClass)); 732 mTestClass = testClass; 733 } else { 734 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ""); 735 } 736 737 // The delay_msec parameter is normally used to provide buffers of idle time 738 // for power measurement purposes. To make sure there is a delay before and after 739 // every test in a suite, we delay *after* every test (see endTest below) and also 740 // delay *before* the first test. So, delay test1 delay test2 delay. 741 742 try { 743 if (mTestNum == 1) Thread.sleep(mDelayMsec); 744 } catch (InterruptedException e) { 745 throw new IllegalStateException(e); 746 } 747 748 sendStatus(REPORT_VALUE_RESULT_START, mTestResult); 749 mTestResultCode = 0; 750 751 mIsTimedTest = false; 752 mIncludeDetailedStats = false; 753 try { 754 // Look for TimedTest annotation on both test class and test method 755 if (test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class)) { 756 mIsTimedTest = true; 757 mIncludeDetailedStats = test.getClass().getMethod(testName).getAnnotation( 758 TimedTest.class).includeDetailedStats(); 759 } else if (test.getClass().isAnnotationPresent(TimedTest.class)) { 760 mIsTimedTest = true; 761 mIncludeDetailedStats = test.getClass().getAnnotation( 762 TimedTest.class).includeDetailedStats(); 763 } 764 } catch (SecurityException e) { 765 throw new IllegalStateException(e); 766 } catch (NoSuchMethodException e) { 767 throw new IllegalStateException(e); 768 } 769 770 if (mIsTimedTest && mIncludeDetailedStats) { 771 mPerfCollector.beginSnapshot(""); 772 } else if (mIsTimedTest) { 773 mPerfCollector.startTiming(""); 774 } 775 } 776 777 /** 778 * @see junit.framework.TestListener#addError(Test, Throwable) 779 */ 780 public void addError(Test test, Throwable t) { 781 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 782 mTestResultCode = REPORT_VALUE_RESULT_ERROR; 783 // pretty printing 784 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 785 String.format("\nError in %s:\n%s", 786 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 787 } 788 789 /** 790 * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 791 */ 792 public void addFailure(Test test, AssertionFailedError t) { 793 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 794 mTestResultCode = REPORT_VALUE_RESULT_FAILURE; 795 // pretty printing 796 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 797 String.format("\nFailure in %s:\n%s", 798 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 799 } 800 801 /** 802 * @see junit.framework.TestListener#endTest(Test) 803 */ 804 public void endTest(Test test) { 805 if (mIsTimedTest && mIncludeDetailedStats) { 806 mTestResult.putAll(mPerfCollector.endSnapshot()); 807 } else if (mIsTimedTest) { 808 writeStopTiming(mPerfCollector.stopTiming("")); 809 } 810 811 if (mTestResultCode == 0) { 812 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "."); 813 } 814 sendStatus(mTestResultCode, mTestResult); 815 816 try { // Sleep after every test, if specified 817 Thread.sleep(mDelayMsec); 818 } catch (InterruptedException e) { 819 throw new IllegalStateException(e); 820 } 821 } 822 823 public void writeBeginSnapshot(String label) { 824 // Do nothing 825 } 826 827 public void writeEndSnapshot(Bundle results) { 828 // Copy all snapshot data fields into mResults, which is outputted 829 // via Instrumentation.finish 830 mResults.putAll(results); 831 } 832 833 public void writeStartTiming(String label) { 834 // Do nothing 835 } 836 837 public void writeStopTiming(Bundle results) { 838 // Copy results into mTestResult by flattening list of iterations, 839 // which is outputted via WatcherResultPrinter.endTest 840 int i = 0; 841 for (Parcelable p : 842 results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) { 843 Bundle iteration = (Bundle)p; 844 String index = "iteration" + i + "."; 845 mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL, 846 iteration.getString(PerformanceCollector.METRIC_KEY_LABEL)); 847 mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME, 848 iteration.getLong(PerformanceCollector.METRIC_KEY_CPU_TIME)); 849 mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_EXECUTION_TIME, 850 iteration.getLong(PerformanceCollector.METRIC_KEY_EXECUTION_TIME)); 851 i++; 852 } 853 } 854 855 public void writeMeasurement(String label, long value) { 856 mTestResult.putLong(label, value); 857 } 858 859 public void writeMeasurement(String label, float value) { 860 mTestResult.putFloat(label, value); 861 } 862 863 public void writeMeasurement(String label, String value) { 864 mTestResult.putString(label, value); 865 } 866 867 // TODO report the end of the cycle 868 } 869} 870