android/test/InstrumentationTestRunner.java

/*
 * Copyright (C) 2007 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.test;

import com.android.internal.util.Predicate;
import com.android.internal.util.Predicates;

import android.app.Activity;
import android.app.Instrumentation;
import android.os.Bundle;
import android.os.Debug;
import android.os.Looper;
import android.os.Parcelable;
import android.os.PerformanceCollector;
import android.os.PerformanceCollector.PerformanceResultsWriter;
import android.test.suitebuilder.TestMethod;
import android.test.suitebuilder.TestPredicates;
import android.test.suitebuilder.TestSuiteBuilder;
import android.test.suitebuilder.annotation.HasAnnotation;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;

import junit.framework.AssertionFailedError;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestListener;
import junit.framework.TestResult;
import junit.framework.TestSuite;
import junit.runner.BaseTestRunner;
import junit.textui.ResultPrinter;

/**
 * An {@link Instrumentation} that runs various types of {@link junit.framework.TestCase}s against
 * an Android package (application). Typical usage:
 * <ol>
 * <li>Write {@link junit.framework.TestCase}s that perform unit, functional, or performance tests
 * against the classes in your package.  Typically these are subclassed from:
 *   <ul><li>{@link android.test.ActivityInstrumentationTestCase2}</li>
 *   <li>{@link android.test.ActivityUnitTestCase}</li>
 *   <li>{@link android.test.AndroidTestCase}</li>
 *   <li>{@link android.test.ApplicationTestCase}</li>
 *   <li>{@link android.test.InstrumentationTestCase}</li>
 *   <li>{@link android.test.ProviderTestCase}</li>
 *   <li>{@link android.test.ServiceTestCase}</li>
 *   <li>{@link android.test.SingleLaunchActivityTestCase}</li></ul>
 *   <li>In an appropriate AndroidManifest.xml, define the this instrumentation with
 * the appropriate android:targetPackage set.
 * <li>Run the instrumentation using "adb shell am instrument -w",
 * with no optional arguments, to run all tests (except performance tests).
 * <li>Run the instrumentation using "adb shell am instrument -w",
 * with the argument '-e func true' to run all functional tests. These are tests that derive from
 * {@link android.test.InstrumentationTestCase}.
 * <li>Run the instrumentation using "adb shell am instrument -w",
 * with the argument '-e unit true' to run all unit tests. These are tests that <i>do not</i>derive
 * from {@link android.test.InstrumentationTestCase} (and are not performance tests).
 * <li>Run the instrumentation using "adb shell am instrument -w",
 * with the argument '-e class' set to run an individual {@link junit.framework.TestCase}.
 * </ol>
 * <p/>
 * <b>Running all tests:</b> adb shell am instrument -w
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Running all small tests:</b> adb shell am instrument -w
 * -e size small
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Running all medium tests:</b> adb shell am instrument -w
 * -e size medium
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Running all large tests:</b> adb shell am instrument -w
 * -e size large
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Filter test run to tests with given annotation:</b> adb shell am instrument -w
 * -e annotation com.android.foo.MyAnnotation
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * If used with other options, the resulting test run will contain the union of the two options.
 * e.g. "-e size large -e annotation com.android.foo.MyAnnotation" will run only tests with both
 * the {@link LargeTest} and "com.android.foo.MyAnnotation" annotations.
 * <p/>
 * <b>Filter test run to tests <i>without</i> given annotation:</b> adb shell am instrument -w
 * -e notAnnotation com.android.foo.MyAnnotation
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Running a single testcase:</b> adb shell am instrument -w
 * -e class com.android.foo.FooTest
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Running a single test:</b> adb shell am instrument -w
 * -e class com.android.foo.FooTest#testFoo
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Running multiple tests:</b> adb shell am instrument -w
 * -e class com.android.foo.FooTest,com.android.foo.TooTest
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Running all tests in a java package:</b> adb shell am instrument -w
 * -e package com.android.foo.subpkg
 *  com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>Including performance tests:</b> adb shell am instrument -w
 * -e perf true
 * com.android.foo/android.test.InstrumentationTestRunner
 * <p/>
 * <b>To debug your tests, set a break point in your code and pass:</b>
 * -e debug true
 * <p/>
 * <b>To run in 'log only' mode</b>
 * -e log true
 * This option will load and iterate through all test classes and methods, but will bypass actual
 * test execution. Useful for quickly obtaining info on the tests to be executed by an
 * instrumentation command.
 * <p/>
 * <b>To generate EMMA code coverage:</b>
 * -e coverage true
 * Note: this requires an emma instrumented build. By default, the code coverage results file
 * will be saved in a /data/<app>/coverage.ec file, unless overridden by coverageFile flag (see
 * below)
 * <p/>
 * <b> To specify EMMA code coverage results file path:</b>
 * -e coverageFile /sdcard/myFile.ec
 * <br/>
 * in addition to the other arguments.
 */

/* (not JavaDoc)
 * Although not necessary in most case, another way to use this class is to extend it and have the
 * derived class return the desired test suite from the {@link #getTestSuite()} method. The test
 * suite returned from this method will be used if no target class is defined in the meta-data or
 * command line argument parameters. If a derived class is used it needs to be added as an
 * instrumentation to the AndroidManifest.xml and the command to run it would look like:
 * <p/>
 * adb shell am instrument -w com.android.foo/<i>com.android.FooInstrumentationTestRunner</i>
 * <p/>
 * Where <i>com.android.FooInstrumentationTestRunner</i> is the derived class.
 *
 * This model is used by many existing app tests, but can probably be deprecated.
 */
public class InstrumentationTestRunner extends Instrumentation implements TestSuiteProvider {

    /** @hide */
    public static final String ARGUMENT_TEST_CLASS = "class";
    /** @hide */
    public static final String ARGUMENT_TEST_PACKAGE = "package";
    /** @hide */
    public static final String ARGUMENT_TEST_SIZE_PREDICATE = "size";
    /** @hide */
    public static final String ARGUMENT_DELAY_MSEC = "delay_msec";

    private static final String SMALL_SUITE = "small";
    private static final String MEDIUM_SUITE = "medium";
    private static final String LARGE_SUITE = "large";

    private static final String ARGUMENT_LOG_ONLY = "log";
    /** @hide */
    static final String ARGUMENT_ANNOTATION = "annotation";
    /** @hide */
    static final String ARGUMENT_NOT_ANNOTATION = "notAnnotation";

    /**
     * This constant defines the maximum allowed runtime (in ms) for a test included in the "small"
     * suite. It is used to make an educated guess at what suite an unlabeled test belongs.
     */
    private static final float SMALL_SUITE_MAX_RUNTIME = 100;

    /**
     * This constant defines the maximum allowed runtime (in ms) for a test included in the
     * "medium" suite. It is used to make an educated guess at what suite an unlabeled test belongs.
     */
    private static final float MEDIUM_SUITE_MAX_RUNTIME = 1000;

    /**
     * The following keys are used in the status bundle to provide structured reports to
     * an IInstrumentationWatcher.
     */

    /**
     * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER},
     * identifies InstrumentationTestRunner as the source of the report.  This is sent with all
     * status messages.
     */
    public static final String REPORT_VALUE_ID = "InstrumentationTestRunner";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * identifies the total number of tests that are being run.  This is sent with all status
     * messages.
     */
    public static final String REPORT_KEY_NUM_TOTAL = "numtests";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * identifies the sequence number of the current test.  This is sent with any status message
     * describing a specific test being started or completed.
     */
    public static final String REPORT_KEY_NUM_CURRENT = "current";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * identifies the name of the current test class.  This is sent with any status message
     * describing a specific test being started or completed.
     */
    public static final String REPORT_KEY_NAME_CLASS = "class";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * identifies the name of the current test.  This is sent with any status message
     * describing a specific test being started or completed.
     */
    public static final String REPORT_KEY_NAME_TEST = "test";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * reports the run time in seconds of the current test.
     */
    private static final String REPORT_KEY_RUN_TIME = "runtime";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * reports the number of total iterations of the current test.
     */
    private static final String REPORT_KEY_NUM_ITERATIONS = "numiterations";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * reports the guessed suite assignment for the current test.
     */
    private static final String REPORT_KEY_SUITE_ASSIGNMENT = "suiteassignment";
    /**
     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
     * identifies the path to the generated code coverage file.
     */
    private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath";

    /**
     * The test is starting.
     */
    public static final int REPORT_VALUE_RESULT_START = 1;
    /**
     * The test completed successfully.
     */
    public static final int REPORT_VALUE_RESULT_OK = 0;
    /**
     * The test completed with an error.
     */
    public static final int REPORT_VALUE_RESULT_ERROR = -1;
    /**
     * The test completed with a failure.
     */
    public static final int REPORT_VALUE_RESULT_FAILURE = -2;
    /**
     * If included in the status bundle sent to an IInstrumentationWatcher, this key
     * identifies a stack trace describing an error or failure.  This is sent with any status
     * message describing a specific test being completed.
     */
    public static final String REPORT_KEY_STACK = "stack";

    // Default file name for code coverage
    private static final String DEFAULT_COVERAGE_FILE_NAME = "coverage.ec";

    private static final String LOG_TAG = "InstrumentationTestRunner";

    private final Bundle mResults = new Bundle();
    private AndroidTestRunner mTestRunner;
    private boolean mDebug;
    private boolean mJustCount;
    private boolean mSuiteAssignmentMode;
    private int mTestCount;
    private String mPackageOfTests;
    private boolean mCoverage;
    private String mCoverageFilePath;
    private int mDelayMsec;

    @Override
    public void onCreate(Bundle arguments) {
        super.onCreate(arguments);

        // Apk paths used to search for test classes when using TestSuiteBuilders.
        String[] apkPaths =
                {getTargetContext().getPackageCodePath(), getContext().getPackageCodePath()};
        ClassPathPackageInfoSource.setApkPaths(apkPaths);

        Predicate<TestMethod> testSizePredicate = null;
        Predicate<TestMethod> testAnnotationPredicate = null;
        Predicate<TestMethod> testNotAnnotationPredicate = null;
        String testClassesArg = null;
        boolean logOnly = false;

        if (arguments != null) {
            // Test class name passed as an argument should override any meta-data declaration.
            testClassesArg = arguments.getString(ARGUMENT_TEST_CLASS);
            mDebug = getBooleanArgument(arguments, "debug");
            mJustCount = getBooleanArgument(arguments, "count");
            mSuiteAssignmentMode = getBooleanArgument(arguments, "suiteAssignment");
            mPackageOfTests = arguments.getString(ARGUMENT_TEST_PACKAGE);
            testSizePredicate = getSizePredicateFromArg(
                    arguments.getString(ARGUMENT_TEST_SIZE_PREDICATE));
            testAnnotationPredicate = getAnnotationPredicate(
                    arguments.getString(ARGUMENT_ANNOTATION));
            testNotAnnotationPredicate = getNotAnnotationPredicate(
                    arguments.getString(ARGUMENT_NOT_ANNOTATION));

            logOnly = getBooleanArgument(arguments, ARGUMENT_LOG_ONLY);
            mCoverage = getBooleanArgument(arguments, "coverage");
            mCoverageFilePath = arguments.getString("coverageFile");

            try {
                Object delay = arguments.get(ARGUMENT_DELAY_MSEC);  // Accept either string or int
                if (delay != null) mDelayMsec = Integer.parseInt(delay.toString());
            } catch (NumberFormatException e) {
                Log.e(LOG_TAG, "Invalid delay_msec parameter", e);
            }
        }

        TestSuiteBuilder testSuiteBuilder = new TestSuiteBuilder(getClass().getName(),
                getTargetContext().getClassLoader());

        if (testSizePredicate != null) {
            testSuiteBuilder.addRequirements(testSizePredicate);
        }
        if (testAnnotationPredicate != null) {
            testSuiteBuilder.addRequirements(testAnnotationPredicate);
        }
        if (testNotAnnotationPredicate != null) {
            testSuiteBuilder.addRequirements(testNotAnnotationPredicate);
        }

        if (testClassesArg == null) {
            if (mPackageOfTests != null) {
                testSuiteBuilder.includePackages(mPackageOfTests);
            } else {
                TestSuite testSuite = getTestSuite();
                if (testSuite != null) {
                    testSuiteBuilder.addTestSuite(testSuite);
                } else {
                    // no package or class bundle arguments were supplied, and no test suite
                    // provided so add all tests in application
                    testSuiteBuilder.includePackages("");
                }
            }
        } else {
            parseTestClasses(testClassesArg, testSuiteBuilder);
        }

        testSuiteBuilder.addRequirements(getBuilderRequirements());

        mTestRunner = getAndroidTestRunner();
        mTestRunner.setContext(getTargetContext());
        mTestRunner.setInstrumentation(this);
        mTestRunner.setSkipExecution(logOnly);
        mTestRunner.setTest(testSuiteBuilder.build());
        mTestCount = mTestRunner.getTestCases().size();
        if (mSuiteAssignmentMode) {
            mTestRunner.addTestListener(new SuiteAssignmentPrinter());
        } else {
            WatcherResultPrinter resultPrinter = new WatcherResultPrinter(mTestCount);
            mTestRunner.addTestListener(new TestPrinter("TestRunner", false));
            mTestRunner.addTestListener(resultPrinter);
            mTestRunner.setPerformanceResultsWriter(resultPrinter);
        }
        start();
    }

    List<Predicate<TestMethod>> getBuilderRequirements() {
        return new ArrayList<Predicate<TestMethod>>();
    }

    /**
     * Parses and loads the specified set of test classes
     *
     * @param testClassArg - comma-separated list of test classes and methods
     * @param testSuiteBuilder - builder to add tests to
     */
    private void parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder) {
        String[] testClasses = testClassArg.split(",");
        for (String testClass : testClasses) {
            parseTestClass(testClass, testSuiteBuilder);
        }
    }

    /**
     * Parse and load the given test class and, optionally, method
     *
     * @param testClassName - full package name of test class and optionally method to add.
     *        Expected format: com.android.TestClass#testMethod
     * @param testSuiteBuilder - builder to add tests to
     */
    private void parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder) {
        int methodSeparatorIndex = testClassName.indexOf('#');
        String testMethodName = null;

        if (methodSeparatorIndex > 0) {
            testMethodName = testClassName.substring(methodSeparatorIndex + 1);
            testClassName = testClassName.substring(0, methodSeparatorIndex);
        }
        testSuiteBuilder.addTestClassByName(testClassName, testMethodName, getTargetContext());
    }

    protected AndroidTestRunner getAndroidTestRunner() {
        return new AndroidTestRunner();
    }

    private boolean getBooleanArgument(Bundle arguments, String tag) {
        String tagString = arguments.getString(tag);
        return tagString != null && Boolean.parseBoolean(tagString);
    }

    /*
     * Returns the size predicate object, corresponding to the "size" argument value.
     */
    private Predicate<TestMethod> getSizePredicateFromArg(String sizeArg) {

        if (SMALL_SUITE.equals(sizeArg)) {
            return TestPredicates.SELECT_SMALL;
        } else if (MEDIUM_SUITE.equals(sizeArg)) {
            return TestPredicates.SELECT_MEDIUM;
        } else if (LARGE_SUITE.equals(sizeArg)) {
            return TestPredicates.SELECT_LARGE;
        } else {
            return null;
        }
    }

   /**
    * Returns the test predicate object, corresponding to the annotation class value provided via
    * the {@link ARGUMENT_ANNOTATION} argument.
    *
    * @return the predicate or <code>null</code>
    */
    private Predicate<TestMethod> getAnnotationPredicate(String annotationClassName) {
        Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName);
        if (annotationClass != null) {
            return new HasAnnotation(annotationClass);
        }
        return null;
    }

    /**
     * Returns the negative test predicate object, corresponding to the annotation class value
     * provided via the {@link ARGUMENT_NOT_ANNOTATION} argument.
     *
     * @return the predicate or <code>null</code>
     */
     private Predicate<TestMethod> getNotAnnotationPredicate(String annotationClassName) {
         Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName);
         if (annotationClass != null) {
             return Predicates.not(new HasAnnotation(annotationClass));
         }
         return null;
     }

    /**
     * Helper method to return the annotation class with specified name
     *
     * @param annotationClassName the fully qualified name of the class
     * @return the annotation class or <code>null</code>
     */
    private Class<? extends Annotation> getAnnotationClass(String annotationClassName) {
        if (annotationClassName == null) {
            return null;
        }
        try {
           Class<?> annotationClass = Class.forName(annotationClassName);
           if (annotationClass.isAnnotation()) {
               return (Class<? extends Annotation>)annotationClass;
           } else {
               Log.e(LOG_TAG, String.format("Provided annotation value %s is not an Annotation",
                       annotationClassName));
           }
        } catch (ClassNotFoundException e) {
            Log.e(LOG_TAG, String.format("Could not find class for specified annotation %s",
                    annotationClassName));
        }
        return null;
    }

    /**
     * Initialize the current thread as a looper.
     * <p/>
     * Exposed for unit testing.
     */
    void prepareLooper() {
        Looper.prepare();
    }

    @Override
    public void onStart() {
        prepareLooper();

        if (mJustCount) {
            mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID);
            mResults.putInt(REPORT_KEY_NUM_TOTAL, mTestCount);
            finish(Activity.RESULT_OK, mResults);
        } else {
            if (mDebug) {
                Debug.waitForDebugger();
            }

            ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
            PrintStream writer = new PrintStream(byteArrayOutputStream);
            try {
                StringResultPrinter resultPrinter = new StringResultPrinter(writer);

                mTestRunner.addTestListener(resultPrinter);

                long startTime = System.currentTimeMillis();
                mTestRunner.runTest();
                long runTime = System.currentTimeMillis() - startTime;

                resultPrinter.print(mTestRunner.getTestResult(), runTime);
            } catch (Throwable t) {
                // catch all exceptions so a more verbose error message can be outputted
                writer.println(String.format("Test run aborted due to unexpected exception: %s",
                                t.getMessage()));
                t.printStackTrace(writer);
            } finally {
                mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
                        String.format("\nTest results for %s=%s",
                        mTestRunner.getTestClassName(),
                        byteArrayOutputStream.toString()));

                if (mCoverage) {
                    generateCoverageReport();
                }
                writer.close();

                finish(Activity.RESULT_OK, mResults);
            }
        }
    }

    public TestSuite getTestSuite() {
        return getAllTests();
    }

    /**
     * Override this to define all of the tests to run in your package.
     */
    public TestSuite getAllTests() {
        return null;
    }

    /**
     * Override this to provide access to the class loader of your package.
     */
    public ClassLoader getLoader() {
        return null;
    }

    private void generateCoverageReport() {
        // use reflection to call emma dump coverage method, to avoid
        // always statically compiling against emma jar
        String coverageFilePath = getCoverageFilePath();
        java.io.File coverageFile = new java.io.File(coverageFilePath);
        try {
            Class<?> emmaRTClass = Class.forName("com.vladium.emma.rt.RT");
            Method dumpCoverageMethod = emmaRTClass.getMethod("dumpCoverageData",
                    coverageFile.getClass(), boolean.class, boolean.class);

            dumpCoverageMethod.invoke(null, coverageFile, false, false);
            // output path to generated coverage file so it can be parsed by a test harness if
            // needed
            mResults.putString(REPORT_KEY_COVERAGE_PATH, coverageFilePath);
            // also output a more user friendly msg
            final String currentStream = mResults.getString(
                    Instrumentation.REPORT_KEY_STREAMRESULT);
            mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
                String.format("%s\nGenerated code coverage data to %s", currentStream,
                coverageFilePath));
        } catch (ClassNotFoundException e) {
            reportEmmaError("Is emma jar on classpath?", e);
        } catch (SecurityException e) {
            reportEmmaError(e);
        } catch (NoSuchMethodException e) {
            reportEmmaError(e);
        } catch (IllegalArgumentException e) {
            reportEmmaError(e);
        } catch (IllegalAccessException e) {
            reportEmmaError(e);
        } catch (InvocationTargetException e) {
            reportEmmaError(e);
        }
    }

    private String getCoverageFilePath() {
        if (mCoverageFilePath == null) {
            return getTargetContext().getFilesDir().getAbsolutePath() + File.separator +
                   DEFAULT_COVERAGE_FILE_NAME;
        } else {
            return mCoverageFilePath;
        }
    }

    private void reportEmmaError(Exception e) {
        reportEmmaError("", e);
    }

    private void reportEmmaError(String hint, Exception e) {
        String msg = "Failed to generate emma coverage. " + hint;
        Log.e(LOG_TAG, msg, e);
        mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "\nError: " + msg);
    }

    // TODO kill this, use status() and prettyprint model for better output
    private class StringResultPrinter extends ResultPrinter {

        public StringResultPrinter(PrintStream writer) {
            super(writer);
        }

        synchronized void print(TestResult result, long runTime) {
            printHeader(runTime);
            printFooter(result);
        }
    }

    /**
     * This class sends status reports back to the IInstrumentationWatcher about
     * which suite each test belongs.
     */
    private class SuiteAssignmentPrinter implements TestListener {

        private Bundle mTestResult;
        private long mStartTime;
        private long mEndTime;
        private boolean mTimingValid;

        public SuiteAssignmentPrinter() {
        }

        /**
         * send a status for the start of a each test, so long tests can be seen as "running"
         */
        public void startTest(Test test) {
            mTimingValid = true;
            mStartTime = System.currentTimeMillis();
        }

        /**
         * @see junit.framework.TestListener#addError(Test, Throwable)
         */
        public void addError(Test test, Throwable t) {
            mTimingValid = false;
        }

        /**
         * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError)
         */
        public void addFailure(Test test, AssertionFailedError t) {
            mTimingValid = false;
        }

        /**
         * @see junit.framework.TestListener#endTest(Test)
         */
        public void endTest(Test test) {
            float runTime;
            String assignmentSuite;
            mEndTime = System.currentTimeMillis();
            mTestResult = new Bundle();

            if (!mTimingValid || mStartTime < 0) {
                assignmentSuite = "NA";
                runTime = -1;
            } else {
                runTime = mEndTime - mStartTime;
                if (runTime < SMALL_SUITE_MAX_RUNTIME
                        && !InstrumentationTestCase.class.isAssignableFrom(test.getClass())) {
                    assignmentSuite = SMALL_SUITE;
                } else if (runTime < MEDIUM_SUITE_MAX_RUNTIME) {
                    assignmentSuite = MEDIUM_SUITE;
                } else {
                    assignmentSuite = LARGE_SUITE;
                }
            }
            // Clear mStartTime so that we can verify that it gets set next time.
            mStartTime = -1;

            mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
                    test.getClass().getName() + "#" + ((TestCase) test).getName()
                    + "\nin " + assignmentSuite + " suite\nrunTime: "
                    + String.valueOf(runTime) + "\n");
            mTestResult.putFloat(REPORT_KEY_RUN_TIME, runTime);
            mTestResult.putString(REPORT_KEY_SUITE_ASSIGNMENT, assignmentSuite);

            sendStatus(0, mTestResult);
        }
    }

    /**
     * This class sends status reports back to the IInstrumentationWatcher
     */
    private class WatcherResultPrinter implements TestListener, PerformanceResultsWriter {
        private final Bundle mResultTemplate;
        Bundle mTestResult;
        int mTestNum = 0;
        int mTestResultCode = 0;
        String mTestClass = null;
        PerformanceCollector mPerfCollector = new PerformanceCollector();
        boolean mIsTimedTest = false;
        boolean mIncludeDetailedStats = false;

        public WatcherResultPrinter(int numTests) {
            mResultTemplate = new Bundle();
            mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID);
            mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests);
        }

        /**
         * send a status for the start of a each test, so long tests can be seen
         * as "running"
         */
        public void startTest(Test test) {
            String testClass = test.getClass().getName();
            String testName = ((TestCase)test).getName();
            mTestResult = new Bundle(mResultTemplate);
            mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass);
            mTestResult.putString(REPORT_KEY_NAME_TEST, testName);
            mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum);
            // pretty printing
            if (testClass != null && !testClass.equals(mTestClass)) {
                mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
                        String.format("\n%s:", testClass));
                mTestClass = testClass;
            } else {
                mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "");
            }

            Method testMethod = null;
            try {
                testMethod = test.getClass().getMethod(testName);
                // Report total number of iterations, if test is repetitive
                if (testMethod.isAnnotationPresent(RepetitiveTest.class)) {
                    int numIterations = testMethod.getAnnotation(
                        RepetitiveTest.class).numIterations();
                    mTestResult.putInt(REPORT_KEY_NUM_ITERATIONS, numIterations);
                }
            } catch (NoSuchMethodException e) {
                // ignore- the test with given name does not exist. Will be handled during test
                // execution
            }

            // The delay_msec parameter is normally used to provide buffers of idle time
            // for power measurement purposes. To make sure there is a delay before and after
            // every test in a suite, we delay *after* every test (see endTest below) and also
            // delay *before* the first test. So, delay test1 delay test2 delay.

            try {
                if (mTestNum == 1) Thread.sleep(mDelayMsec);
            } catch (InterruptedException e) {
                throw new IllegalStateException(e);
            }

            sendStatus(REPORT_VALUE_RESULT_START, mTestResult);
            mTestResultCode = 0;

            mIsTimedTest = false;
            mIncludeDetailedStats = false;
            try {
                // Look for TimedTest annotation on both test class and test method
                if (testMethod != null && testMethod.isAnnotationPresent(TimedTest.class)) {
                    mIsTimedTest = true;
                    mIncludeDetailedStats = testMethod.getAnnotation(
                            TimedTest.class).includeDetailedStats();
                } else if (test.getClass().isAnnotationPresent(TimedTest.class)) {
                    mIsTimedTest = true;
                    mIncludeDetailedStats = test.getClass().getAnnotation(
                            TimedTest.class).includeDetailedStats();
                }
            } catch (SecurityException e) {
                // ignore - the test with given name cannot be accessed. Will be handled during
                // test execution
            }

            if (mIsTimedTest && mIncludeDetailedStats) {
                mPerfCollector.beginSnapshot("");
            } else if (mIsTimedTest) {
                mPerfCollector.startTiming("");
            }
        }

        /**
         * @see junit.framework.TestListener#addError(Test, Throwable)
         */
        public void addError(Test test, Throwable t) {
            mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t));
            mTestResultCode = REPORT_VALUE_RESULT_ERROR;
            // pretty printing
            mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
                String.format("\nError in %s:\n%s",
                    ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t)));
        }

        /**
         * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError)
         */
        public void addFailure(Test test, AssertionFailedError t) {
            mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t));
            mTestResultCode = REPORT_VALUE_RESULT_FAILURE;
            // pretty printing
            mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
                String.format("\nFailure in %s:\n%s",
                    ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t)));
        }

        /**
         * @see junit.framework.TestListener#endTest(Test)
         */
        public void endTest(Test test) {
            if (mIsTimedTest && mIncludeDetailedStats) {
                mTestResult.putAll(mPerfCollector.endSnapshot());
            } else if (mIsTimedTest) {
                writeStopTiming(mPerfCollector.stopTiming(""));
            }

            if (mTestResultCode == 0) {
                mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ".");
            }
            sendStatus(mTestResultCode, mTestResult);

            try { // Sleep after every test, if specified
                Thread.sleep(mDelayMsec);
            } catch (InterruptedException e) {
                throw new IllegalStateException(e);
            }
        }

        public void writeBeginSnapshot(String label) {
            // Do nothing
        }

        public void writeEndSnapshot(Bundle results) {
            // Copy all snapshot data fields into mResults, which is outputted
            // via Instrumentation.finish
            mResults.putAll(results);
        }

        public void writeStartTiming(String label) {
            // Do nothing
        }

        public void writeStopTiming(Bundle results) {
            // Copy results into mTestResult by flattening list of iterations,
            // which is outputted via WatcherResultPrinter.endTest
            int i = 0;
            for (Parcelable p :
                    results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) {
                Bundle iteration = (Bundle)p;
                String index = "iteration" + i + ".";
                mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL,
                        iteration.getString(PerformanceCollector.METRIC_KEY_LABEL));
                mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME,
                        iteration.getLong(PerformanceCollector.METRIC_KEY_CPU_TIME));
                mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_EXECUTION_TIME,
                        iteration.getLong(PerformanceCollector.METRIC_KEY_EXECUTION_TIME));
                i++;
            }
        }

        public void writeMeasurement(String label, long value) {
            mTestResult.putLong(label, value);
        }

        public void writeMeasurement(String label, float value) {
            mTestResult.putFloat(label, value);
        }

        public void writeMeasurement(String label, String value) {
            mTestResult.putString(label, value);
        }

        // TODO report the end of the cycle
    }
}