1/*
2 * Copyright (C) 2008 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package android.test;
18
19import android.app.Instrumentation;
20import android.content.Context;
21import android.os.Bundle;
22import android.test.mock.MockContext;
23import android.test.suitebuilder.ListTestCaseNames;
24import android.test.suitebuilder.ListTestCaseNames.TestDescriptor;
25import android.test.suitebuilder.annotation.SmallTest;
26
27import junit.framework.Test;
28import junit.framework.TestCase;
29import junit.framework.TestSuite;
30
31import java.util.List;
32
33/**
34 * Tests for {@link InstrumentationTestRunner}
35 */
36@SmallTest
37public class InstrumentationTestRunnerTest extends TestCase {
38    private StubInstrumentationTestRunner mInstrumentationTestRunner;
39    private StubAndroidTestRunner mStubAndroidTestRunner;
40    private String mTargetContextPackageName;
41
42    protected void setUp() throws Exception {
43        super.setUp();
44        mStubAndroidTestRunner = new StubAndroidTestRunner();
45        mTargetContextPackageName = "android.test.suitebuilder.examples";
46        mInstrumentationTestRunner = new StubInstrumentationTestRunner(
47                new StubContext("com.google.foo.tests"),
48                new StubContext(mTargetContextPackageName), mStubAndroidTestRunner);
49    }
50
51    public void testOverrideTestToRunWithClassArgument() throws Exception {
52        String expectedTestClassName = PlaceHolderTest.class.getName();
53        mInstrumentationTestRunner.onCreate(createBundle(
54                InstrumentationTestRunner.ARGUMENT_TEST_CLASS, expectedTestClassName));
55
56        assertTestRunnerCalledWithExpectedParameters(expectedTestClassName, "testPlaceHolder");
57    }
58
59    public void testOverrideTestToRunWithClassAndMethodArgument() throws Exception {
60        String expectedTestClassName = PlaceHolderTest.class.getName();
61        String expectedTestMethodName = "testPlaceHolder";
62        String classAndMethod = expectedTestClassName + "#" + expectedTestMethodName;
63        mInstrumentationTestRunner.onCreate(createBundle(
64                InstrumentationTestRunner.ARGUMENT_TEST_CLASS, classAndMethod));
65
66        assertTestRunnerCalledWithExpectedParameters(expectedTestClassName,
67                expectedTestMethodName);
68    }
69
70    public void testUseSelfAsTestSuiteProviderWhenNoMetaDataOrClassArgument() throws Exception {
71        TestSuite testSuite = new TestSuite();
72        testSuite.addTestSuite(PlaceHolderTest.class);
73        mInstrumentationTestRunner.setAllTestsSuite(testSuite);
74        mInstrumentationTestRunner.onCreate(null);
75        assertTestRunnerCalledWithExpectedParameters(
76                PlaceHolderTest.class.getName(), "testPlaceHolder");
77    }
78
79    public void testMultipleTestClass() throws Exception {
80        String classArg = PlaceHolderTest.class.getName() + "," +
81            PlaceHolderTest2.class.getName();
82        mInstrumentationTestRunner.onCreate(createBundle(
83                InstrumentationTestRunner.ARGUMENT_TEST_CLASS, classArg));
84
85        Test test = mStubAndroidTestRunner.getTest();
86
87        assertContentsInOrder(ListTestCaseNames.getTestNames((TestSuite) test),
88            new TestDescriptor(PlaceHolderTest.class.getName(), "testPlaceHolder"),
89            new TestDescriptor(PlaceHolderTest2.class.getName(), "testPlaceHolder2"));
90
91    }
92
93    /**
94     * Test that runtime exceptions during runTest are handled gracefully
95     */
96    public void testUnhandledException() throws Exception {
97        StubAndroidTestRunner stubAndroidTestRunner = new StubAndroidTestRunner() {
98            @Override
99            public void runTest() {
100                throw new RuntimeException();
101            }
102        };
103        StubInstrumentationTestRunner instrumentationTestRunner = new StubInstrumentationTestRunner(
104                new StubContext("com.google.foo.tests"),
105                new StubContext(mTargetContextPackageName), stubAndroidTestRunner);
106        instrumentationTestRunner.onCreate(new Bundle());
107        instrumentationTestRunner.onStart();
108        assertTrue("Instrumentation did not finish", instrumentationTestRunner.isFinished());
109        // ensure a meaningful error message placed in results
110        String resultsData = instrumentationTestRunner.mResults.getString(
111                Instrumentation.REPORT_KEY_STREAMRESULT);
112        assertTrue("Instrumentation results is missing RuntimeException",
113                resultsData.contains("RuntimeException"));
114    }
115
116    /**
117     * Test that specifying a method which does not exist is handled gracefully
118     */
119    public void testBadMethodArgument() throws Exception {
120        String testClassName = PlaceHolderTest.class.getName();
121        String invalidMethodName = "testNoExist";
122        String classAndMethod = testClassName + "#" + invalidMethodName;
123        mInstrumentationTestRunner.onCreate(createBundle(
124                InstrumentationTestRunner.ARGUMENT_TEST_CLASS, classAndMethod));
125        assertTestRunnerCalledWithExpectedParameters(testClassName,
126                invalidMethodName);
127    }
128
129    public void testDelayParameter() throws Exception {
130        int delayMsec = 1000;
131        Bundle args = new Bundle();
132        args.putInt(InstrumentationTestRunner.ARGUMENT_DELAY_MSEC, delayMsec);
133        args.putString(InstrumentationTestRunner.ARGUMENT_TEST_CLASS,
134                PlaceHolderTest.class.getName() + "," +
135                PlaceHolderTest2.class.getName());
136        mInstrumentationTestRunner.onCreate(args);
137        Thread t = new Thread() { public void run() { mInstrumentationTestRunner.onStart(); } };
138
139        // Should delay three times: before, between, and after the two tests.
140        long beforeTest = System.currentTimeMillis();
141        t.start();
142        t.join();
143        assertTrue(System.currentTimeMillis() > beforeTest + delayMsec * 3);
144        assertTrue(mInstrumentationTestRunner.isStarted());
145        assertTrue(mInstrumentationTestRunner.isFinished());
146        assertTrue(mStubAndroidTestRunner.isRun());
147    }
148
149    /**
150     * Test that the -e {@link InstrumentationTestRunner.ARGUMENT_ANNOTATION} parameter properly
151     * selects tests.
152     */
153    public void testAnnotationParameter() throws Exception {
154        String expectedTestClassName = AnnotationTest.class.getName();
155        Bundle args = new Bundle();
156        args.putString(InstrumentationTestRunner.ARGUMENT_TEST_CLASS, expectedTestClassName);
157        args.putString(InstrumentationTestRunner.ARGUMENT_ANNOTATION, FlakyTest.class.getName());
158        mInstrumentationTestRunner.onCreate(args);
159        assertTestRunnerCalledWithExpectedParameters(expectedTestClassName, "testAnnotated");
160    }
161
162    /**
163     * Test that the -e {@link InstrumentationTestRunner.ARGUMENT_NOT_ANNOTATION} parameter
164     * properly excludes tests.
165     */
166    public void testNotAnnotationParameter() throws Exception {
167        String expectedTestClassName = AnnotationTest.class.getName();
168        Bundle args = new Bundle();
169        args.putString(InstrumentationTestRunner.ARGUMENT_TEST_CLASS, expectedTestClassName);
170        args.putString(InstrumentationTestRunner.ARGUMENT_NOT_ANNOTATION,
171                FlakyTest.class.getName());
172        mInstrumentationTestRunner.onCreate(args);
173        assertTestRunnerCalledWithExpectedParameters(expectedTestClassName, "testNotAnnotated");
174    }
175
176    private void assertContentsInOrder(List<TestDescriptor> actual, TestDescriptor... source) {
177        TestDescriptor[] clonedSource = source.clone();
178        assertEquals("Unexpected number of items.", clonedSource.length, actual.size());
179        for (int i = 0; i < actual.size(); i++) {
180            TestDescriptor actualItem = actual.get(i);
181            TestDescriptor sourceItem = clonedSource[i];
182            assertEquals("Unexpected item. Index: " + i, sourceItem, actualItem);
183        }
184    }
185
186    private void assertTestRunnerCalledWithExpectedParameters(
187            String expectedTestClassName, String expectedTestMethodName) {
188        Test test = mStubAndroidTestRunner.getTest();
189        assertContentsInOrder(ListTestCaseNames.getTestNames((TestSuite) test),
190                new TestDescriptor(expectedTestClassName, expectedTestMethodName));
191        assertTrue(mInstrumentationTestRunner.isStarted());
192        assertFalse(mInstrumentationTestRunner.isFinished());
193    }
194
195    private Bundle createBundle(String key, String value) {
196        Bundle bundle = new Bundle();
197        bundle.putString(key, value);
198        return bundle;
199    }
200
201    private static class StubInstrumentationTestRunner extends InstrumentationTestRunner {
202        private Context mContext;
203        private Context mTargetContext;
204        private boolean mStarted;
205        private boolean mFinished;
206        private AndroidTestRunner mAndroidTestRunner;
207        private TestSuite mTestSuite;
208        private TestSuite mDefaultTestSuite;
209        private String mPackageNameForDefaultTests;
210        private Bundle mResults;
211
212        public StubInstrumentationTestRunner(Context context, Context targetContext,
213                AndroidTestRunner androidTestRunner) {
214            this.mContext = context;
215            this.mTargetContext = targetContext;
216            this.mAndroidTestRunner = androidTestRunner;
217        }
218
219        public Context getContext() {
220            return mContext;
221        }
222
223        public TestSuite getAllTests() {
224            return mTestSuite;
225        }
226
227        public Context getTargetContext() {
228            return mTargetContext;
229        }
230
231        protected AndroidTestRunner getAndroidTestRunner() {
232            return mAndroidTestRunner;
233        }
234
235        public void start() {
236            mStarted = true;
237        }
238
239        public void finish(int resultCode, Bundle results) {
240            mFinished = true;
241            mResults = results;
242        }
243
244        public boolean isStarted() {
245            return mStarted;
246        }
247
248        public boolean isFinished() {
249            return mFinished;
250        }
251
252        public void setAllTestsSuite(TestSuite testSuite) {
253            mTestSuite = testSuite;
254        }
255
256        public void setDefaultTestsSuite(TestSuite testSuite) {
257            mDefaultTestSuite = testSuite;
258        }
259
260        public String getPackageNameForDefaultTests() {
261            return mPackageNameForDefaultTests;
262        }
263
264        @Override
265        void prepareLooper() {
266            // ignore
267        }
268    }
269
270    private static class StubContext extends MockContext {
271        private String mPackageName;
272
273        public StubContext(String packageName) {
274            this.mPackageName = packageName;
275        }
276
277        @Override
278        public String getPackageCodePath() {
279            return mPackageName;
280        }
281
282        @Override
283        public String getPackageName() {
284            return mPackageName;
285        }
286
287        @Override
288        public ClassLoader getClassLoader() {
289            return getClass().getClassLoader();
290        }
291    }
292
293    private static class StubAndroidTestRunner extends AndroidTestRunner {
294        private Test mTest;
295        private boolean mRun;
296
297        public boolean isRun() {
298            return mRun;
299        }
300
301        public void setTest(Test test) {
302            super.setTest(test);
303            mTest = test;
304        }
305
306        public Test getTest() {
307            return mTest;
308        }
309
310        public void runTest() {
311            super.runTest();
312            mRun = true;
313        }
314    }
315
316    /**
317     * Empty test used for validation
318     */
319    public static class PlaceHolderTest extends TestCase {
320
321        public PlaceHolderTest() {
322            super("testPlaceHolder");
323        }
324
325        public void testPlaceHolder() throws Exception {
326
327        }
328    }
329
330    /**
331     * Empty test used for validation
332     */
333    public static class PlaceHolderTest2 extends TestCase {
334
335        public PlaceHolderTest2() {
336            super("testPlaceHolder2");
337        }
338
339        public void testPlaceHolder2() throws Exception {
340
341        }
342    }
343
344    /**
345     * Annotated test used for validation.
346     */
347    public static class AnnotationTest extends TestCase {
348
349        public void testNotAnnotated() throws Exception {
350        }
351
352        @FlakyTest
353        public void testAnnotated() throws Exception {
354        }
355    }
356}
357