19066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project/* 29066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Copyright (C) 2007 The Android Open Source Project 39066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * 49066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Licensed under the Apache License, Version 2.0 (the "License"); 59066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * you may not use this file except in compliance with the License. 69066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * You may obtain a copy of the License at 79066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * 89066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * http://www.apache.org/licenses/LICENSE-2.0 99066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * 109066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Unless required by applicable law or agreed to in writing, software 119066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * distributed under the License is distributed on an "AS IS" BASIS, 129066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 139066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * See the License for the specific language governing permissions and 149066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * limitations under the License. 159066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 169066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 179066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectpackage android.test; 189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 19ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport com.android.internal.util.Predicate; 2088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabotimport com.android.internal.util.Predicates; 21ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.app.Activity; 239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.app.Instrumentation; 249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.os.Bundle; 259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.os.Debug; 269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.os.Looper; 27ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport android.os.Parcelable; 28ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport android.os.PerformanceCollector; 29ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport android.os.PerformanceCollector.PerformanceResultsWriter; 309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.test.suitebuilder.TestMethod; 319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.test.suitebuilder.TestPredicates; 329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.test.suitebuilder.TestSuiteBuilder; 3388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabotimport android.test.suitebuilder.annotation.HasAnnotation; 340375fa12c7f94bfc9c840721dcc5215b6c7575d6Doug Zongkerimport android.test.suitebuilder.annotation.LargeTest; 359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.util.Log; 369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 37ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport java.io.ByteArrayOutputStream; 38ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport java.io.File; 39ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport java.io.PrintStream; 4088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabotimport java.lang.annotation.Annotation; 41ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport java.lang.reflect.InvocationTargetException; 42ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport java.lang.reflect.Method; 43ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport java.util.ArrayList; 44ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wangimport java.util.List; 459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.framework.AssertionFailedError; 479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.framework.Test; 489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.framework.TestCase; 499066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.framework.TestListener; 509066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.framework.TestResult; 519066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.framework.TestSuite; 529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.runner.BaseTestRunner; 539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport junit.textui.ResultPrinter; 549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project/** 569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * An {@link Instrumentation} that runs various types of {@link junit.framework.TestCase}s against 573aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * an Android package (application). 583aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * 593aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * <div class="special reference"> 603aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * <h3>Developer Guides</h3> 613aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * <p>For more information about application testing, read the 623aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * <a href="{@docRoot}guide/topics/testing/index.html">Testing</a> developer guide.</p> 633aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * </div> 643aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * 653aef8e1d1b2f0b87d470bcccf37ba4ebb6560c45Joe Fernandez * <h3>Typical Usage</h3> 669066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <ol> 679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>Write {@link junit.framework.TestCase}s that perform unit, functional, or performance tests 689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * against the classes in your package. Typically these are subclassed from: 69ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * <ul><li>{@link android.test.ActivityInstrumentationTestCase2}</li> 709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>{@link android.test.ActivityUnitTestCase}</li> 719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>{@link android.test.AndroidTestCase}</li> 729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>{@link android.test.ApplicationTestCase}</li> 739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>{@link android.test.InstrumentationTestCase}</li> 749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>{@link android.test.ProviderTestCase}</li> 759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>{@link android.test.ServiceTestCase}</li> 769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>{@link android.test.SingleLaunchActivityTestCase}</li></ul> 779066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>In an appropriate AndroidManifest.xml, define the this instrumentation with 789066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * the appropriate android:targetPackage set. 799066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>Run the instrumentation using "adb shell am instrument -w", 809066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * with no optional arguments, to run all tests (except performance tests). 819066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>Run the instrumentation using "adb shell am instrument -w", 829066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * with the argument '-e func true' to run all functional tests. These are tests that derive from 839066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * {@link android.test.InstrumentationTestCase}. 849066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>Run the instrumentation using "adb shell am instrument -w", 859066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * with the argument '-e unit true' to run all unit tests. These are tests that <i>do not</i>derive 869066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * from {@link android.test.InstrumentationTestCase} (and are not performance tests). 879066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <li>Run the instrumentation using "adb shell am instrument -w", 889066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * with the argument '-e class' set to run an individual {@link junit.framework.TestCase}. 899066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * </ol> 909066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 919066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Running all tests:</b> adb shell am instrument -w 929066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 939066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 949066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Running all small tests:</b> adb shell am instrument -w 959066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e size small 969066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 979066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 989066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Running all medium tests:</b> adb shell am instrument -w 999066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e size medium 1009066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 1019066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1029066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Running all large tests:</b> adb shell am instrument -w 1039066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e size large 1049066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 1059066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 10688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * <b>Filter test run to tests with given annotation:</b> adb shell am instrument -w 10788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * -e annotation com.android.foo.MyAnnotation 10888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * com.android.foo/android.test.InstrumentationTestRunner 10988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * <p/> 11088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * If used with other options, the resulting test run will contain the union of the two options. 11188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * e.g. "-e size large -e annotation com.android.foo.MyAnnotation" will run only tests with both 11288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * the {@link LargeTest} and "com.android.foo.MyAnnotation" annotations. 11388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * <p/> 11488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * <b>Filter test run to tests <i>without</i> given annotation:</b> adb shell am instrument -w 11588e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * -e notAnnotation com.android.foo.MyAnnotation 11688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * com.android.foo/android.test.InstrumentationTestRunner 11788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * <p/> 1189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Running a single testcase:</b> adb shell am instrument -w 1199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e class com.android.foo.FooTest 1209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 1219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Running a single test:</b> adb shell am instrument -w 1239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e class com.android.foo.FooTest#testFoo 1249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 1259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Running multiple tests:</b> adb shell am instrument -w 1279066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e class com.android.foo.FooTest,com.android.foo.TooTest 1289066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 1299066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 13089c0ef4aba57172bbc9288f2d22c943f220a3dd1Brett Chabot * <b>Running all tests in a java package:</b> adb shell am instrument -w 13189c0ef4aba57172bbc9288f2d22c943f220a3dd1Brett Chabot * -e package com.android.foo.subpkg 13289c0ef4aba57172bbc9288f2d22c943f220a3dd1Brett Chabot * com.android.foo/android.test.InstrumentationTestRunner 13389c0ef4aba57172bbc9288f2d22c943f220a3dd1Brett Chabot * <p/> 1349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>Including performance tests:</b> adb shell am instrument -w 1359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e perf true 1369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * com.android.foo/android.test.InstrumentationTestRunner 1379066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>To debug your tests, set a break point in your code and pass:</b> 1399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e debug true 1409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>To run in 'log only' mode</b> 1429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e log true 143ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * This option will load and iterate through all test classes and methods, but will bypass actual 144ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * test execution. Useful for quickly obtaining info on the tests to be executed by an 1459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * instrumentation command. 1469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b>To generate EMMA code coverage:</b> 1489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e coverage true 149ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * Note: this requires an emma instrumented build. By default, the code coverage results file 15051e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot * will be saved in a /data/<app>/coverage.ec file, unless overridden by coverageFile flag (see 15151e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot * below) 1529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <b> To specify EMMA code coverage results file path:</b> 1549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * -e coverageFile /sdcard/myFile.ec 1559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <br/> 1569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * in addition to the other arguments. 1579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 1589066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1599066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project/* (not JavaDoc) 1609066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Although not necessary in most case, another way to use this class is to extend it and have the 161ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * derived class return the desired test suite from the {@link #getTestSuite()} method. The test 162ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * suite returned from this method will be used if no target class is defined in the meta-data or 163ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * command line argument parameters. If a derived class is used it needs to be added as an 164ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * instrumentation to the AndroidManifest.xml and the command to run it would look like: 1659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1669066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * adb shell am instrument -w com.android.foo/<i>com.android.FooInstrumentationTestRunner</i> 1679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * <p/> 1689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Where <i>com.android.FooInstrumentationTestRunner</i> is the derived class. 1699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * 1709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * This model is used by many existing app tests, but can probably be deprecated. 1719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 1729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectpublic class InstrumentationTestRunner extends Instrumentation implements TestSuiteProvider { 1739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** @hide */ 1759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String ARGUMENT_TEST_CLASS = "class"; 1769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** @hide */ 1779066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String ARGUMENT_TEST_PACKAGE = "package"; 1789066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** @hide */ 1799066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String ARGUMENT_TEST_SIZE_PREDICATE = "size"; 1809066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** @hide */ 1819066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String ARGUMENT_DELAY_MSEC = "delay_msec"; 1829066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1839066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final String SMALL_SUITE = "small"; 184ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang private static final String MEDIUM_SUITE = "medium"; 1859066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final String LARGE_SUITE = "large"; 186ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 1879066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final String ARGUMENT_LOG_ONLY = "log"; 18888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot /** @hide */ 18988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot static final String ARGUMENT_ANNOTATION = "annotation"; 19088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot /** @hide */ 19188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot static final String ARGUMENT_NOT_ANNOTATION = "notAnnotation"; 1929066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1939066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 194ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * This constant defines the maximum allowed runtime (in ms) for a test included in the "small" 195ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * suite. It is used to make an educated guess at what suite an unlabeled test belongs. 1969066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 1979066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final float SMALL_SUITE_MAX_RUNTIME = 100; 198ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 1999066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 200ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * This constant defines the maximum allowed runtime (in ms) for a test included in the 201ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * "medium" suite. It is used to make an educated guess at what suite an unlabeled test belongs. 2029066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2039066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final float MEDIUM_SUITE_MAX_RUNTIME = 1000; 204ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 2059066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 206ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * The following keys are used in the status bundle to provide structured reports to 207ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * an IInstrumentationWatcher. 2089066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2099066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 2109066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 211ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER}, 2129066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * identifies InstrumentationTestRunner as the source of the report. This is sent with all 2139066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * status messages. 2149066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2159066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String REPORT_VALUE_ID = "InstrumentationTestRunner"; 2169066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 217ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 2189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * identifies the total number of tests that are being run. This is sent with all status 2199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * messages. 2209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String REPORT_KEY_NUM_TOTAL = "numtests"; 2229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 223ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 2249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * identifies the sequence number of the current test. This is sent with any status message 2259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * describing a specific test being started or completed. 2269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2279066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String REPORT_KEY_NUM_CURRENT = "current"; 2289066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 229ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 2309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * identifies the name of the current test class. This is sent with any status message 2319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * describing a specific test being started or completed. 2329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2339066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String REPORT_KEY_NAME_CLASS = "class"; 2349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 235ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 2369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * identifies the name of the current test. This is sent with any status message 2379066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * describing a specific test being started or completed. 2389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String REPORT_KEY_NAME_TEST = "test"; 2409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 241ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 2429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * reports the run time in seconds of the current test. 2439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final String REPORT_KEY_RUN_TIME = "runtime"; 2459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 246ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 2473fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang * reports the number of total iterations of the current test. 2483fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang */ 2493fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang private static final String REPORT_KEY_NUM_ITERATIONS = "numiterations"; 2503fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang /** 2513fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 2529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * reports the guessed suite assignment for the current test. 2539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final String REPORT_KEY_SUITE_ASSIGNMENT = "suiteassignment"; 2559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 25651e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 25751e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot * identifies the path to the generated code coverage file. 25851e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot */ 25951e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath"; 260ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 261ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang /** 2629066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * The test is starting. 2639066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2649066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final int REPORT_VALUE_RESULT_START = 1; 2659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 2669066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * The test completed successfully. 2679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final int REPORT_VALUE_RESULT_OK = 0; 2699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 2709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * The test completed with an error. 2719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final int REPORT_VALUE_RESULT_ERROR = -1; 2739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 2749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * The test completed with a failure. 2759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final int REPORT_VALUE_RESULT_FAILURE = -2; 2779066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 278ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * If included in the status bundle sent to an IInstrumentationWatcher, this key 279ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * identifies a stack trace describing an error or failure. This is sent with any status 2809066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * message describing a specific test being completed. 2819066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 2829066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public static final String REPORT_KEY_STACK = "stack"; 2839066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 28451e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot // Default file name for code coverage 28551e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot private static final String DEFAULT_COVERAGE_FILE_NAME = "coverage.ec"; 286ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 2879066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final String LOG_TAG = "InstrumentationTestRunner"; 2889066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 2899066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private final Bundle mResults = new Bundle(); 290cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu private Bundle mArguments; 2919066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private AndroidTestRunner mTestRunner; 2929066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean mDebug; 2939066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean mJustCount; 2949066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean mSuiteAssignmentMode; 2959066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private int mTestCount; 2969066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private String mPackageOfTests; 2979066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean mCoverage; 2989066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private String mCoverageFilePath; 2999066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private int mDelayMsec; 3009066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3019066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project @Override 3029066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void onCreate(Bundle arguments) { 3039066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project super.onCreate(arguments); 304cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu mArguments = arguments; 3059066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3069066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // Apk paths used to search for test classes when using TestSuiteBuilders. 3079066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String[] apkPaths = 3089066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project {getTargetContext().getPackageCodePath(), getContext().getPackageCodePath()}; 3099066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project ClassPathPackageInfoSource.setApkPaths(apkPaths); 3109066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3119066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project Predicate<TestMethod> testSizePredicate = null; 31288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Predicate<TestMethod> testAnnotationPredicate = null; 31388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Predicate<TestMethod> testNotAnnotationPredicate = null; 3149066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String testClassesArg = null; 3159066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project boolean logOnly = false; 3169066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3179066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (arguments != null) { 3189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // Test class name passed as an argument should override any meta-data declaration. 3199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project testClassesArg = arguments.getString(ARGUMENT_TEST_CLASS); 3209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mDebug = getBooleanArgument(arguments, "debug"); 3219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mJustCount = getBooleanArgument(arguments, "count"); 3229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mSuiteAssignmentMode = getBooleanArgument(arguments, "suiteAssignment"); 3239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mPackageOfTests = arguments.getString(ARGUMENT_TEST_PACKAGE); 3249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project testSizePredicate = getSizePredicateFromArg( 3259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project arguments.getString(ARGUMENT_TEST_SIZE_PREDICATE)); 32688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot testAnnotationPredicate = getAnnotationPredicate( 32788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot arguments.getString(ARGUMENT_ANNOTATION)); 32888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot testNotAnnotationPredicate = getNotAnnotationPredicate( 32988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot arguments.getString(ARGUMENT_NOT_ANNOTATION)); 33088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot 3319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project logOnly = getBooleanArgument(arguments, ARGUMENT_LOG_ONLY); 3329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mCoverage = getBooleanArgument(arguments, "coverage"); 3339066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mCoverageFilePath = arguments.getString("coverageFile"); 3349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 3369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project Object delay = arguments.get(ARGUMENT_DELAY_MSEC); // Accept either string or int 3379066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (delay != null) mDelayMsec = Integer.parseInt(delay.toString()); 3389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (NumberFormatException e) { 3399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project Log.e(LOG_TAG, "Invalid delay_msec parameter", e); 3409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 3419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 3429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project TestSuiteBuilder testSuiteBuilder = new TestSuiteBuilder(getClass().getName(), 3449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project getTargetContext().getClassLoader()); 3459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (testSizePredicate != null) { 3479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project testSuiteBuilder.addRequirements(testSizePredicate); 3489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 34988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot if (testAnnotationPredicate != null) { 35088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot testSuiteBuilder.addRequirements(testAnnotationPredicate); 35188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 35288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot if (testNotAnnotationPredicate != null) { 35388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot testSuiteBuilder.addRequirements(testNotAnnotationPredicate); 35488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 3559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (testClassesArg == null) { 3579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mPackageOfTests != null) { 3589066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project testSuiteBuilder.includePackages(mPackageOfTests); 3599066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 36061b10ac95de29c3ff826250cf12b04b932e0cfd7Brett Chabot TestSuite testSuite = getTestSuite(); 36161b10ac95de29c3ff826250cf12b04b932e0cfd7Brett Chabot if (testSuite != null) { 36261b10ac95de29c3ff826250cf12b04b932e0cfd7Brett Chabot testSuiteBuilder.addTestSuite(testSuite); 36361b10ac95de29c3ff826250cf12b04b932e0cfd7Brett Chabot } else { 364ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang // no package or class bundle arguments were supplied, and no test suite 36561b10ac95de29c3ff826250cf12b04b932e0cfd7Brett Chabot // provided so add all tests in application 36661b10ac95de29c3ff826250cf12b04b932e0cfd7Brett Chabot testSuiteBuilder.includePackages(""); 36761b10ac95de29c3ff826250cf12b04b932e0cfd7Brett Chabot } 3689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 3699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 3709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project parseTestClasses(testClassesArg, testSuiteBuilder); 3719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 372ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 373da13ef5dca5cac0e89b82a6d7360a575610b7bccUrs Grob testSuiteBuilder.addRequirements(getBuilderRequirements()); 3749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 3759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner = getAndroidTestRunner(); 3769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner.setContext(getTargetContext()); 3777aba54b2edcb94f43c991c555b23fc1de364835eJack Wang mTestRunner.setInstrumentation(this); 3789066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner.setSkipExecution(logOnly); 3799066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner.setTest(testSuiteBuilder.build()); 3809066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestCount = mTestRunner.getTestCases().size(); 3819066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mSuiteAssignmentMode) { 3829066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner.addTestListener(new SuiteAssignmentPrinter()); 3839066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 384ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang WatcherResultPrinter resultPrinter = new WatcherResultPrinter(mTestCount); 3859066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner.addTestListener(new TestPrinter("TestRunner", false)); 386ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestRunner.addTestListener(resultPrinter); 387ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestRunner.setPerformanceResultsWriter(resultPrinter); 3889066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 3899066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project start(); 3909066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 3919066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 392cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu /** 393cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu * Get the Bundle object that contains the arguments 394cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu * 395cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu * @return the Bundle object 396cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu * @hide 397cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu */ 398cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu public Bundle getBundle(){ 399cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu return mArguments; 400cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu } 401cbc584d7ebe332f78bf8012d21265ccb482c7cfcDavid Hu 4028cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu /** 4038cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu * Add a {@link TestListener} 4048cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu * @hide 4058cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu */ 4068cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu protected void addTestListener(TestListener listener){ 4078cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu if(mTestRunner!=null && listener!=null){ 4088cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu mTestRunner.addTestListener(listener); 4098cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu } 4108cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu } 4118cc9a8e132ebbf852e055ed17ab91f35b02fd974David Hu 412da13ef5dca5cac0e89b82a6d7360a575610b7bccUrs Grob List<Predicate<TestMethod>> getBuilderRequirements() { 413da13ef5dca5cac0e89b82a6d7360a575610b7bccUrs Grob return new ArrayList<Predicate<TestMethod>>(); 414da13ef5dca5cac0e89b82a6d7360a575610b7bccUrs Grob } 415da13ef5dca5cac0e89b82a6d7360a575610b7bccUrs Grob 4169066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 417ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * Parses and loads the specified set of test classes 418ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * 4199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @param testClassArg - comma-separated list of test classes and methods 4209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @param testSuiteBuilder - builder to add tests to 4219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 4229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private void parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder) { 4239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String[] testClasses = testClassArg.split(","); 4249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project for (String testClass : testClasses) { 4259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project parseTestClass(testClass, testSuiteBuilder); 4269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 4279066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 4289066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 4299066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 4309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Parse and load the given test class and, optionally, method 431ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * 432ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * @param testClassName - full package name of test class and optionally method to add. 433ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * Expected format: com.android.TestClass#testMethod 4349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @param testSuiteBuilder - builder to add tests to 4359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 4369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private void parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder) { 4379066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project int methodSeparatorIndex = testClassName.indexOf('#'); 4389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String testMethodName = null; 4399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 4409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (methodSeparatorIndex > 0) { 4419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project testMethodName = testClassName.substring(methodSeparatorIndex + 1); 4429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project testClassName = testClassName.substring(0, methodSeparatorIndex); 4439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 444ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang testSuiteBuilder.addTestClassByName(testClassName, testMethodName, getTargetContext()); 4459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 4469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 4479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project protected AndroidTestRunner getAndroidTestRunner() { 4489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return new AndroidTestRunner(); 4499066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 4509066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 4519066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean getBooleanArgument(Bundle arguments, String tag) { 4529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String tagString = arguments.getString(tag); 4539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return tagString != null && Boolean.parseBoolean(tagString); 4549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 455ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 4569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /* 4579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Returns the size predicate object, corresponding to the "size" argument value. 4589066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 4599066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private Predicate<TestMethod> getSizePredicateFromArg(String sizeArg) { 460ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 4619066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (SMALL_SUITE.equals(sizeArg)) { 4629066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return TestPredicates.SELECT_SMALL; 4639066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else if (MEDIUM_SUITE.equals(sizeArg)) { 4649066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return TestPredicates.SELECT_MEDIUM; 4659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else if (LARGE_SUITE.equals(sizeArg)) { 4669066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return TestPredicates.SELECT_LARGE; 4679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 4689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return null; 4699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 4709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 471ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 47288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot /** 47388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * Returns the test predicate object, corresponding to the annotation class value provided via 47488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * the {@link ARGUMENT_ANNOTATION} argument. 47588e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * 47688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * @return the predicate or <code>null</code> 47788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot */ 47888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot private Predicate<TestMethod> getAnnotationPredicate(String annotationClassName) { 47988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 48088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot if (annotationClass != null) { 48188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot return new HasAnnotation(annotationClass); 48288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 48388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot return null; 48488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 48588e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot 48688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot /** 48788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * Returns the negative test predicate object, corresponding to the annotation class value 48888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * provided via the {@link ARGUMENT_NOT_ANNOTATION} argument. 48988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * 49088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * @return the predicate or <code>null</code> 49188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot */ 49288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot private Predicate<TestMethod> getNotAnnotationPredicate(String annotationClassName) { 49388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 49488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot if (annotationClass != null) { 49588e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot return Predicates.not(new HasAnnotation(annotationClass)); 49688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 49788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot return null; 49888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 49988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot 50088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot /** 50188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * Helper method to return the annotation class with specified name 50288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * 50388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * @param annotationClassName the fully qualified name of the class 50488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot * @return the annotation class or <code>null</code> 50588e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot */ 50688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot private Class<? extends Annotation> getAnnotationClass(String annotationClassName) { 50788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot if (annotationClassName == null) { 50888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot return null; 50988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 51088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot try { 51188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Class<?> annotationClass = Class.forName(annotationClassName); 51288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot if (annotationClass.isAnnotation()) { 51388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot return (Class<? extends Annotation>)annotationClass; 51488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } else { 51588e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Log.e(LOG_TAG, String.format("Provided annotation value %s is not an Annotation", 51688e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot annotationClassName)); 51788e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 51888e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } catch (ClassNotFoundException e) { 51988e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Log.e(LOG_TAG, String.format("Could not find class for specified annotation %s", 52088e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot annotationClassName)); 52188e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 52288e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot return null; 52388e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot } 52488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot 52531e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot /** 52631e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot * Initialize the current thread as a looper. 52731e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot * <p/> 52831e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot * Exposed for unit testing. 52931e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot */ 53031e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot void prepareLooper() { 53131e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot Looper.prepare(); 53231e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot } 53331e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot 5349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project @Override 5359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void onStart() { 53631e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot prepareLooper(); 537ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 5389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mJustCount) { 5399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 5409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mResults.putInt(REPORT_KEY_NUM_TOTAL, mTestCount); 5419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project finish(Activity.RESULT_OK, mResults); 5429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 5439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mDebug) { 5449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project Debug.waitForDebugger(); 5459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 546ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 5479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 5489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project PrintStream writer = new PrintStream(byteArrayOutputStream); 5499066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 5509066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project StringResultPrinter resultPrinter = new StringResultPrinter(writer); 551ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 5529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner.addTestListener(resultPrinter); 553ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 5549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project long startTime = System.currentTimeMillis(); 5559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestRunner.runTest(); 5569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project long runTime = System.currentTimeMillis() - startTime; 557ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 5589066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project resultPrinter.print(mTestRunner.getTestResult(), runTime); 55931e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot } catch (Throwable t) { 56031e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot // catch all exceptions so a more verbose error message can be outputted 56131e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot writer.println(String.format("Test run aborted due to unexpected exception: %s", 56231e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot t.getMessage())); 56331e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot t.printStackTrace(writer); 5649066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } finally { 565ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 566ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang String.format("\nTest results for %s=%s", 567ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestRunner.getTestClassName(), 5689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project byteArrayOutputStream.toString())); 5699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 5709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mCoverage) { 5719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project generateCoverageReport(); 5729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 5739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project writer.close(); 574ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 5759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project finish(Activity.RESULT_OK, mResults); 5769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 5779066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 5789066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 5799066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 5809066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public TestSuite getTestSuite() { 5819066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return getAllTests(); 5829066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 5839066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 5849066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 5859066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Override this to define all of the tests to run in your package. 5869066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 5879066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public TestSuite getAllTests() { 5889066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return null; 5899066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 5909066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 5919066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 5929066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Override this to provide access to the class loader of your package. 5939066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 5949066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public ClassLoader getLoader() { 5959066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return null; 5969066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 597ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 5989066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private void generateCoverageReport() { 5999066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // use reflection to call emma dump coverage method, to avoid 6009066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // always statically compiling against emma jar 60151e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot String coverageFilePath = getCoverageFilePath(); 60251e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot java.io.File coverageFile = new java.io.File(coverageFilePath); 6039066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 60488e03a97366d08fc69d852cf2219f9d0b1916af4Brett Chabot Class<?> emmaRTClass = Class.forName("com.vladium.emma.rt.RT"); 605ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang Method dumpCoverageMethod = emmaRTClass.getMethod("dumpCoverageData", 6069066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project coverageFile.getClass(), boolean.class, boolean.class); 607ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 6089066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project dumpCoverageMethod.invoke(null, coverageFile, false, false); 60951e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot // output path to generated coverage file so it can be parsed by a test harness if 61051e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot // needed 61151e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot mResults.putString(REPORT_KEY_COVERAGE_PATH, coverageFilePath); 61251e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot // also output a more user friendly msg 61308d13c3cbb0b250b84ca861edb462060556aead9Brett Chabot final String currentStream = mResults.getString( 61408d13c3cbb0b250b84ca861edb462060556aead9Brett Chabot Instrumentation.REPORT_KEY_STREAMRESULT); 61551e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 61608d13c3cbb0b250b84ca861edb462060556aead9Brett Chabot String.format("%s\nGenerated code coverage data to %s", currentStream, 61708d13c3cbb0b250b84ca861edb462060556aead9Brett Chabot coverageFilePath)); 6189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (ClassNotFoundException e) { 6199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project reportEmmaError("Is emma jar on classpath?", e); 6209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (SecurityException e) { 6219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project reportEmmaError(e); 6229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (NoSuchMethodException e) { 6239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project reportEmmaError(e); 6249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (IllegalArgumentException e) { 6259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project reportEmmaError(e); 6269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (IllegalAccessException e) { 6279066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project reportEmmaError(e); 6289066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (InvocationTargetException e) { 6299066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project reportEmmaError(e); 6309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 6339066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private String getCoverageFilePath() { 6349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mCoverageFilePath == null) { 63551e03642cee5a695d6ff6f5b318925efcab05100Brett Chabot return getTargetContext().getFilesDir().getAbsolutePath() + File.separator + 636ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang DEFAULT_COVERAGE_FILE_NAME; 637ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } else { 6389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project return mCoverageFilePath; 6399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 6429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private void reportEmmaError(Exception e) { 643ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang reportEmmaError("", e); 6449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 6469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private void reportEmmaError(String hint, Exception e) { 6479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String msg = "Failed to generate emma coverage. " + hint; 6489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project Log.e(LOG_TAG, msg, e); 6499066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "\nError: " + msg); 6509066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6519066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 6529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // TODO kill this, use status() and prettyprint model for better output 6539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private class StringResultPrinter extends ResultPrinter { 6549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 6559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public StringResultPrinter(PrintStream writer) { 6569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project super(writer); 6579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6589066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 6599066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project synchronized void print(TestResult result, long runTime) { 6609066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project printHeader(runTime); 6619066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project printFooter(result); 6629066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6639066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 664ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 6659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 666ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * This class sends status reports back to the IInstrumentationWatcher about 6679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * which suite each test belongs. 6689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 669ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang private class SuiteAssignmentPrinter implements TestListener { 670ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 6719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private Bundle mTestResult; 6729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private long mStartTime; 6739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private long mEndTime; 6749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean mTimingValid; 675ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 6769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public SuiteAssignmentPrinter() { 6779066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 678ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 6799066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 6809066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * send a status for the start of a each test, so long tests can be seen as "running" 6819066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 6829066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void startTest(Test test) { 6839066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTimingValid = true; 684ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mStartTime = System.currentTimeMillis(); 6859066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 686ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 6879066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 6889066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @see junit.framework.TestListener#addError(Test, Throwable) 6899066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 6909066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void addError(Test test, Throwable t) { 6919066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTimingValid = false; 6929066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 6939066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 6949066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 6959066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 6969066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 6979066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void addFailure(Test test, AssertionFailedError t) { 6989066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTimingValid = false; 6999066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 7009066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 7019066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 7029066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @see junit.framework.TestListener#endTest(Test) 7039066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 7049066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void endTest(Test test) { 7059066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project float runTime; 7069066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String assignmentSuite; 7079066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mEndTime = System.currentTimeMillis(); 7089066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult = new Bundle(); 7099066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 7109066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (!mTimingValid || mStartTime < 0) { 7119066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project assignmentSuite = "NA"; 7129066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project runTime = -1; 7139066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 7149066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project runTime = mEndTime - mStartTime; 715ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang if (runTime < SMALL_SUITE_MAX_RUNTIME 7169066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project && !InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { 7179066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project assignmentSuite = SMALL_SUITE; 7189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else if (runTime < MEDIUM_SUITE_MAX_RUNTIME) { 7199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project assignmentSuite = MEDIUM_SUITE; 7209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 7219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project assignmentSuite = LARGE_SUITE; 7229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 7239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 7249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // Clear mStartTime so that we can verify that it gets set next time. 7259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mStartTime = -1; 7269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 727ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 728ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang test.getClass().getName() + "#" + ((TestCase) test).getName() 7299066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project + "\nin " + assignmentSuite + " suite\nrunTime: " 7309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project + String.valueOf(runTime) + "\n"); 7319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putFloat(REPORT_KEY_RUN_TIME, runTime); 7329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putString(REPORT_KEY_SUITE_ASSIGNMENT, assignmentSuite); 7339066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 7349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project sendStatus(0, mTestResult); 7359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 7369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 737ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 7389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 7399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * This class sends status reports back to the IInstrumentationWatcher 7409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 741ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang private class WatcherResultPrinter implements TestListener, PerformanceResultsWriter { 7429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private final Bundle mResultTemplate; 7439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project Bundle mTestResult; 7449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project int mTestNum = 0; 7459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project int mTestResultCode = 0; 7469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String mTestClass = null; 7474f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang PerformanceCollector mPerfCollector = new PerformanceCollector(); 748ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang boolean mIsTimedTest = false; 7494f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang boolean mIncludeDetailedStats = false; 750ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 7519066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public WatcherResultPrinter(int numTests) { 7529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mResultTemplate = new Bundle(); 7539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 7549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests); 7559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 756ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 7579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 758ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * send a status for the start of a each test, so long tests can be seen 759ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang * as "running" 7609066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 7619066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void startTest(Test test) { 7629066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String testClass = test.getClass().getName(); 763ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang String testName = ((TestCase)test).getName(); 7649066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult = new Bundle(mResultTemplate); 7659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass); 766ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putString(REPORT_KEY_NAME_TEST, testName); 7679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum); 7689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // pretty printing 7699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (testClass != null && !testClass.equals(mTestClass)) { 770ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 7719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String.format("\n%s:", testClass)); 7729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestClass = testClass; 7739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } else { 7749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ""); 7759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 7769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 7773fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang Method testMethod = null; 7783fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang try { 7793fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang testMethod = test.getClass().getMethod(testName); 7803fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang // Report total number of iterations, if test is repetitive 7813fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang if (testMethod.isAnnotationPresent(RepetitiveTest.class)) { 7823fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang int numIterations = testMethod.getAnnotation( 7833fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang RepetitiveTest.class).numIterations(); 7843fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang mTestResult.putInt(REPORT_KEY_NUM_ITERATIONS, numIterations); 7853fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang } 7863fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang } catch (NoSuchMethodException e) { 7873fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang // ignore- the test with given name does not exist. Will be handled during test 7883fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang // execution 7893fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang } 7903fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang 7919066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // The delay_msec parameter is normally used to provide buffers of idle time 792ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang // for power measurement purposes. To make sure there is a delay before and after 7939066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // every test in a suite, we delay *after* every test (see endTest below) and also 794ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang // delay *before* the first test. So, delay test1 delay test2 delay. 7959066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 7969066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 7979066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mTestNum == 1) Thread.sleep(mDelayMsec); 7989066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (InterruptedException e) { 7999066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project throw new IllegalStateException(e); 8009066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 8019066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 8029066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project sendStatus(REPORT_VALUE_RESULT_START, mTestResult); 8039066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResultCode = 0; 804ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 805ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mIsTimedTest = false; 8064f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mIncludeDetailedStats = false; 807ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang try { 8084f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang // Look for TimedTest annotation on both test class and test method 80948983959a34f67024bd0411c8353c196e6a87717Brian Muramatsu if (testMethod != null && testMethod.isAnnotationPresent(TimedTest.class)) { 8104f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mIsTimedTest = true; 8113fc03e619fb01678549b80e7a89af2c8e3f19968Jack Wang mIncludeDetailedStats = testMethod.getAnnotation( 8124f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang TimedTest.class).includeDetailedStats(); 8134f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang } else if (test.getClass().isAnnotationPresent(TimedTest.class)) { 8144f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mIsTimedTest = true; 8154f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mIncludeDetailedStats = test.getClass().getAnnotation( 8164f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang TimedTest.class).includeDetailedStats(); 8174f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang } 818ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } catch (SecurityException e) { 81931e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot // ignore - the test with given name cannot be accessed. Will be handled during 82031e7ce762ccbbb747fabb4581e42a0a2fe56e780Brett Chabot // test execution 821ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 822ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 8234f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang if (mIsTimedTest && mIncludeDetailedStats) { 8244f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mPerfCollector.beginSnapshot(""); 8254f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang } else if (mIsTimedTest) { 8264f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mPerfCollector.startTiming(""); 827ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 8289066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 829ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 8309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 8319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @see junit.framework.TestListener#addError(Test, Throwable) 8329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 8339066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void addError(Test test, Throwable t) { 8349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 8359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResultCode = REPORT_VALUE_RESULT_ERROR; 8369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // pretty printing 837ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 838ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang String.format("\nError in %s:\n%s", 839ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 8409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 8419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 8429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 8439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 8449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 8459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void addFailure(Test test, AssertionFailedError t) { 8469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 8479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResultCode = REPORT_VALUE_RESULT_FAILURE; 8489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // pretty printing 849ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 850ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang String.format("\nFailure in %s:\n%s", 851ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 8529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 8539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 8549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project /** 8559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * @see junit.framework.TestListener#endTest(Test) 8569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 8579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void endTest(Test test) { 8584f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang if (mIsTimedTest && mIncludeDetailedStats) { 8594f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mTestResult.putAll(mPerfCollector.endSnapshot()); 8604f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang } else if (mIsTimedTest) { 8614f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang writeStopTiming(mPerfCollector.stopTiming("")); 862ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 863ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 8649066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (mTestResultCode == 0) { 8659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "."); 8669066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 8679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project sendStatus(mTestResultCode, mTestResult); 8689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 869ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang try { // Sleep after every test, if specified 8709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project Thread.sleep(mDelayMsec); 8719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (InterruptedException e) { 8729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project throw new IllegalStateException(e); 8739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 8749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 8759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 876ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang public void writeBeginSnapshot(String label) { 877ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang // Do nothing 878ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 879ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 880ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang public void writeEndSnapshot(Bundle results) { 881075997f12e2fb2b646172a92926be0f26f739099Jack Wang // Copy all snapshot data fields into mResults, which is outputted 882075997f12e2fb2b646172a92926be0f26f739099Jack Wang // via Instrumentation.finish 883075997f12e2fb2b646172a92926be0f26f739099Jack Wang mResults.putAll(results); 884ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 885ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 886ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang public void writeStartTiming(String label) { 887ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang // Do nothing 888ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 889ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 890ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang public void writeStopTiming(Bundle results) { 891ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang // Copy results into mTestResult by flattening list of iterations, 892ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang // which is outputted via WatcherResultPrinter.endTest 893ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang int i = 0; 894ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang for (Parcelable p : 895ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) { 896ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang Bundle iteration = (Bundle)p; 8974f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang String index = "iteration" + i + "."; 898ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL, 899ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang iteration.getString(PerformanceCollector.METRIC_KEY_LABEL)); 900ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME, 901ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang iteration.getLong(PerformanceCollector.METRIC_KEY_CPU_TIME)); 902ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_EXECUTION_TIME, 903ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang iteration.getLong(PerformanceCollector.METRIC_KEY_EXECUTION_TIME)); 904ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang i++; 905ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 906ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang } 907ff1df69dd4835c177c724e1b5f1ba02d1f674047Jack Wang 908075997f12e2fb2b646172a92926be0f26f739099Jack Wang public void writeMeasurement(String label, long value) { 9094f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mTestResult.putLong(label, value); 910075997f12e2fb2b646172a92926be0f26f739099Jack Wang } 911075997f12e2fb2b646172a92926be0f26f739099Jack Wang 912075997f12e2fb2b646172a92926be0f26f739099Jack Wang public void writeMeasurement(String label, float value) { 9134f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mTestResult.putFloat(label, value); 914075997f12e2fb2b646172a92926be0f26f739099Jack Wang } 915075997f12e2fb2b646172a92926be0f26f739099Jack Wang 916075997f12e2fb2b646172a92926be0f26f739099Jack Wang public void writeMeasurement(String label, String value) { 9174f414bd7a76fc7ff063db70d1cdd09303c28cbe5Jack Wang mTestResult.putString(label, value); 918075997f12e2fb2b646172a92926be0f26f739099Jack Wang } 919075997f12e2fb2b646172a92926be0f26f739099Jack Wang 9209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // TODO report the end of the cycle 9219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 9229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project} 923