TestSuiteFactory.java revision f6c387128427e121477c1b32ad35cdcaa5101ba3 
1/*
2 * Copyright (C) 2008 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package tests;
18
19import dalvik.annotation.AndroidOnly;
20import dalvik.annotation.KnownFailure;
21
22import junit.extensions.TestSetup;
23import junit.framework.AssertionFailedError;
24import junit.framework.Protectable;
25import junit.framework.Test;
26import junit.framework.TestCase;
27import junit.framework.TestResult;
28import junit.framework.TestSuite;
29
30import java.io.PrintWriter;
31import java.io.StringWriter;
32import java.lang.annotation.Annotation;
33import java.lang.reflect.Method;
34import java.lang.reflect.Modifier;
35import java.util.Vector;
36
37/**
38 * an enhanced TestSuite e.g. for running RI tests.
39 *
40 * a sample command line:
41 *
42 *  /usr/lib/jvm/java-1.5.0-sun/bin/java -Xmx1024m -Dcts.listOnlyFailingTests=true
43 *  -Dcts.includeKnownFailure=false -Dcts.runOnDalvikVM=false
44 *  -Dcts.allowUnderscoreTests=false -Dcts.useEnhancedJunit=true
45 *  -Dcts.collectOnly=false
46 *  -cp
47 *  /tmp/cts_outjavac:
48 *  out/debug/host/linux-x86/product/sim/data/app/CtsCoreTests.apk:
49 *  out/debug/target/common/obj/APPS/CtsCoreTests_intermediates/classes.jar:
50 *  tools/cts/vm-tests/lib/junit.jar
51 *
52 *  junit.textui.TestRunner tests.AllTests
53 *
54 */
55public class TestSuiteFactory {
56
57
58
59    static boolean _collectOnly = false;
60    static boolean _useEnhancedJunit = false;
61    static boolean _allowUnderscoreTests = false;
62    static boolean _runOnDalvikVM = true;
63    static boolean _includeKnowFailure = false;
64    static boolean _listOnlyFailingTests = false;
65    static boolean _useSuppliedTestResult = false;
66    static int _maxRunningTimePerTest = 15000; // 15 seconds
67
68    static {
69        _useEnhancedJunit = System.getProperty("cts.useEnhancedJunit", "false").equals("true");
70        // next only applicable if _useEnhancedJunit
71        _collectOnly = System.getProperty("cts.collectOnly", "false").equals("true");
72        _allowUnderscoreTests= System.getProperty("cts.allowUnderscoreTests", "false").equals("true");
73        _runOnDalvikVM = System.getProperty("cts.runOnDalvikVM", "true").equals("true");
74        _includeKnowFailure = System.getProperty("cts.includeKnownFailure", "false").equals("true");
75        _maxRunningTimePerTest = Integer.parseInt(System.getProperty("cts.maxRunningTimePerTest", "15000"));
76        _listOnlyFailingTests = System.getProperty("cts.listOnlyFailingTests", "false").equals("true");
77        _useSuppliedTestResult = System.getProperty("cts.useSuppliedTestResult", "false").equals("true");
78
79        System.out.println("TestSuiteFactory: v0.97");
80        System.out.println("TestSuiteFactory: using cts.useEnhancedJunit: "+_useEnhancedJunit);
81        System.out.println("TestSuiteFactory: using cts.collectOnly: "+_collectOnly);
82        System.out.println("TestSuiteFactory: max allowed running time per test (using Thread.stop()) (cts.maxRunningTimePerTest): "+_maxRunningTimePerTest);
83        System.out.println("TestSuiteFactory: run tests on a dalvik vm (cts.runOnDalvikVM): "+_runOnDalvikVM);
84        System.out.println("TestSuiteFactory: include @KnowFailure when running on dalvik vm (cts.includeKnownFailure): "+_includeKnowFailure);
85        System.out.println("TestSuiteFactory: include '_test...' methods in test run (cts.allowUnderscoreTests): "+_allowUnderscoreTests);
86        System.out.println("TestSuiteFactory: list only failing tests (cts.listOnlyFailingTests): "+_listOnlyFailingTests);
87        System.out.println();
88    }
89
90    public static TestSuite createTestSuite(String name) {
91        return _useEnhancedJunit? new MyTestSuite(name): new TestSuite(name);
92    }
93
94    public static TestSuite createTestSuite() {
95        return _useEnhancedJunit? new MyTestSuite(): new TestSuite();
96    }
97
98}
99
100
101class MyTestSuite extends TestSuite {
102    private boolean allow_underscoretests = true;
103
104    public MyTestSuite() {
105    }
106
107    public MyTestSuite(String name) {
108        super(name);
109    }
110
111    public MyTestSuite(final Class theClass) {
112        String fName = theClass.getName();
113        try {
114            TestSuite.getTestConstructor(theClass); // Avoid generating multiple error
115                                          // messages
116        } catch (NoSuchMethodException e) {
117            addTest(warning("Class "
118                    + theClass.getName()
119                    + " has no public constructor TestCase(String name) or TestCase()"));
120            return;
121        }
122
123        if (!Modifier.isPublic(theClass.getModifiers())) {
124            addTest(warning("Class " + theClass.getName() + " is not public"));
125            return;
126        }
127
128        Class superClass = theClass;
129        Vector names = new Vector();
130        while (Test.class.isAssignableFrom(superClass)) {
131            Method[] methods = superClass.getDeclaredMethods();
132            for (int i = 0; i < methods.length; i++) {
133                addTestMethod(methods[i], names, theClass);
134            }
135            superClass = superClass.getSuperclass();
136        }
137        if (testCount() == 0)
138            addTest(warning("No tests found in " + theClass.getName()));
139    }
140
141    private void addTestMethod(Method m, Vector names, Class theClass) {
142        String name = m.getName();
143        if (names.contains(name)) return;
144        if (!isPublicTestMethod(m)) {
145            if (isTestMethod(m))
146                addTest(warning("Test method isn't public: " + m.getName()));
147            return;
148        }
149        names.addElement(name);
150        addTest(TestSuite.createTest(theClass, name));
151    }
152
153    private boolean isPublicTestMethod(Method m) {
154        return isTestMethod(m) && Modifier.isPublic(m.getModifiers());
155    }
156
157    private boolean isTestMethod(Method m) {
158        String name = m.getName();
159        Class[] parameters = m.getParameterTypes();
160        Class returnType = m.getReturnType();
161        return parameters.length == 0 &&
162        (name.startsWith("test") || (TestSuiteFactory._allowUnderscoreTests  && name.startsWith("_test")))
163        && returnType.equals(Void.TYPE);
164    }
165
166    public void addTestSuite(Class testClass) {
167        try {
168            addTest(new MyTestSuite(testClass));
169        } catch (Throwable e) {
170            System.err.println("---------------- error ----------------------");
171            System.err.println("error: could not add test suite: "
172                    + testClass.getName());
173            e.printStackTrace();
174            System.err.println("---------------- ----------------------------");
175        }
176    }
177
178    private static int testCnt = 0;
179
180    public void runTest(Test test, final TestResult dummy_result) {
181
182    	if (TestSuiteFactory._useSuppliedTestResult) {
183    		if (test instanceof TestSetup) {
184    			test = ((TestSetup)test).getTest();
185    		}
186    		test.run(dummy_result);
187    		return;
188    	}
189
190        TestResult eresult = new TestResult() {
191            private String msg;
192            private boolean error = false;
193
194            protected void run(final TestCase testcase) {
195                msg = "";
196                String testName = testcase.getClass().getName() + ":" + testcase.getName()+" (nr:"+(++testCnt)+")";
197                try {
198                    if (!TestSuiteFactory._listOnlyFailingTests) {
199                        System.out.print(testName+" ");
200                    }
201                    Annotation aKnownFailure = null;
202                    Annotation aAndroidOnly = null;
203                    if (true) { // handle annotations, allow them on both class (valid for all methods then) and method level
204                         //  @KnownFailure("Fails because of a defect in ...") if the test is correct but there is a bug in the core libraries implementation.
205                         //  @BrokenTest("This test is not implemented correctly because...") if there is a defect in the test method itself.
206                         //  @AndroidOnly("Because...") if the test is Android-specific, succeeds on Android but fails on the JDK.
207                        try {
208                            Annotation[] annosClass = testcase.getClass().getDeclaredAnnotations();
209                            Method runMethod= testcase.getClass().getMethod(testcase.getName());
210                            Annotation[] annosMethod = runMethod.getDeclaredAnnotations();
211                            Annotation[] annos = null;
212                            for (int i = 0; i < 2; i++) {
213                                annos = (i==0? annosClass : annosMethod);
214                                if (annos != null && annos.length > 0) {
215                                    for (Annotation anno : annos) {
216                                        Class<? extends Annotation> acla = anno.annotationType();
217                                        if (acla.getName().equals("dalvik.annotation.AndroidOnly")) {
218                                            aAndroidOnly = anno;
219                                        } else if (acla.getName().equals("dalvik.annotation.KnownFailure")) {
220                                            aKnownFailure = anno;
221                                        }
222                                    }
223                                }
224                            }
225                        } catch (NoSuchMethodException e) {
226                            error = true;
227                            msg +="::warning::unable to get test method to read annotations: testcase name="+testcase.getName();
228                        }
229                    }
230                    boolean androidOnly = aAndroidOnly != null;
231                    boolean knownFailure = aKnownFailure != null;
232
233                    if (
234                            !TestSuiteFactory._collectOnly
235                            && (
236                                    (TestSuiteFactory._runOnDalvikVM &&
237                                            (TestSuiteFactory._includeKnowFailure || !knownFailure)
238                                    )
239                                    ||
240                                    (!TestSuiteFactory._runOnDalvikVM && !androidOnly)
241                               )
242                       ) {
243
244                        msg += "[";
245                        long start = System.currentTimeMillis();
246                        // -----start the test ----
247                        startTest(testcase);
248                        final Protectable p= new Protectable() {
249                            public void protect() throws Throwable {
250                                testcase.runBare();
251                            }
252                        };
253                        boolean threadStopForced = false;
254                        if (!TestSuiteFactory._runOnDalvikVM) {
255                            // for jvm, start in a new thread, since we can stop() it for too-long running processes.
256                            Thread t = new Thread(new Runnable(){
257                                public void run() {
258                                    try {
259                                        p.protect();
260                                    }
261                                    catch (AssertionFailedError e) {
262                                        addFailure(testcase, e);
263                                    }
264                                    catch (ThreadDeath e) { // don't catch ThreadDeath by accident
265                                        throw e;
266                                    }
267                                    catch (Throwable e) {
268                                        addError(testcase, e);
269                                    }
270                                }});
271                            t.start();
272
273                            try {
274                                //System.out.println("joining...");
275                                t.join(TestSuiteFactory._maxRunningTimePerTest);
276                                //System.out.println("joining done...");
277                            } catch (InterruptedException e) {
278                                // ignore
279                            }
280                            if (t.isAlive()) {
281                                threadStopForced = true;
282                            }
283                            t.stop();
284
285                            // for RI vm : run in new thread and stop thread after a timeout
286                        } else {
287                            runProtected(testcase, p);
288                        }
289
290                        endTest(testcase);
291                        // ------------------------
292
293                        msg += "]";
294                        long stop = System.currentTimeMillis();
295                        if (threadStopForced) {
296                            error = true;
297                            msg += "::warning::slow test forced to stop since it took longer than "+ TestSuiteFactory._maxRunningTimePerTest;
298                        } else if (stop - start > TestSuiteFactory._maxRunningTimePerTest) {
299                            error = true;
300                            msg += "::warning::slow test took longer than "+ TestSuiteFactory._maxRunningTimePerTest+" milis: "+(stop-start)+" milis. ";
301                        }
302
303                    }
304                    if (!TestSuiteFactory._runOnDalvikVM && androidOnly) {
305                        msg+= "ignoring on RI since @AndroidOnly: "+((AndroidOnly)aAndroidOnly).value();
306                    }
307                    if (TestSuiteFactory._runOnDalvikVM && knownFailure && !TestSuiteFactory._includeKnowFailure) {
308                        msg += "ignoring on dalvik since @KnownFailure: "+((KnownFailure)aKnownFailure).value();
309                    }
310                }
311                finally {
312                    if (TestSuiteFactory._listOnlyFailingTests) {
313                        if (error) {
314                            // we have error / warnings
315                            msg = testName + msg;
316                            System.out.println(msg);
317                        } // else do not output anything
318                    } else {
319                        System.out.println(msg+(error? "": " cts-test-passed"));
320                    }
321                }
322            }
323
324            public synchronized void addError(Test test, Throwable t) {
325                error = true;
326                msg+= " ::error::err:"+exceptionToString(t);
327                super.addError(test, t);
328            }
329
330            public synchronized void addFailure(Test test,
331                    AssertionFailedError t) {
332                error = true;
333                msg+= " ::error::failure:"+exceptionToString(t);
334                super.addFailure(test, t);
335            }
336        };
337        test.run(eresult);
338    }
339
340    private static Test warning(final String message) {
341        return new TestCase("warning") {
342            protected void runTest() {
343                fail(message);
344            }
345        };
346    }
347
348    private static String exceptionToString(Throwable t) {
349        StringWriter stringWriter = new StringWriter();
350        PrintWriter writer = new PrintWriter(stringWriter);
351        t.printStackTrace(writer);
352        return stringWriter.toString();
353
354    }
355}
356