1/*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package vogar.target;
18
19import com.google.common.base.Function;
20import com.google.common.base.Functions;
21import java.util.List;
22import java.util.Properties;
23import java.util.concurrent.atomic.AtomicInteger;
24import org.junit.After;
25import org.junit.Before;
26import org.junit.Rule;
27import vogar.Result;
28import vogar.target.junit.JUnitUtils;
29import vogar.testing.InterceptOutputStreams;
30import vogar.testing.InterceptOutputStreams.Stream;
31
32import static org.junit.Assert.assertEquals;
33import static org.junit.Assert.assertTrue;
34
35/**
36 * Provides support for testing {@link TestRunner} class.
37 *
38 * <p>Subclasses provide the individual test methods, each test method has the following structure.
39 *
40 * <p>It is annotated with {@link TestRunnerProperties @TestRunnerProperties} that specifies the
41 * properties that the main Vogar process supplies to the client process that actually runs the
42 * tests. It must specify either a {@link TestRunnerProperties#testClass()} or
43 * {@link TestRunnerProperties#testClassOrPackage()}, all the remaining properties are optional.
44 *
45 * <p>It calls {@code testRunnerRule.createTestRunner(...)} to create a {@link TestRunner}; passing
46 * in any additional command line arguments that {@link TestRunner#TestRunner(Properties, List)}
47 * accepts.
48 *
49 * <p>It calls {@link TestRunner#run()} to actually run the tests.
50 *
51 * <p>It calls {@link #expectedResults()} to obtain a {@link ExpectedResults} instance that it uses
52 * to specify the expected results for the test. Once it has specified the expected results then it
53 * must call either {@link ExpectedResults#completedNormally()} or
54 * {@link ExpectedResults#aborted()}. They indicate whether the test process completed normally or
55 * would abort (due to a test timing out) and cause the actual results to be checked against the
56 * expected results.
57 */
58public abstract class AbstractTestRunnerTest {
59
60    @Rule
61    public InterceptOutputStreams ios = new InterceptOutputStreams(Stream.OUT);
62
63    @Rule
64    public TestRunnerRule testRunnerRule = new TestRunnerRule();
65
66    /**
67     * Keeps track of number of times {@link #expectedResults()} has been called without
68     * {@link ExpectedResults#checkFilteredOutput(String)}
69     * also being called. If it is {@code > 0} then the test is in error.
70     */
71    private AtomicInteger checkCount;
72
73    @Before
74    public void beforeTest() {
75        checkCount = new AtomicInteger();
76    }
77
78    @After
79    public void afterTest() {
80        if (checkCount.get() != 0) {
81            throw new IllegalStateException("Test called expectedResults() but failed to call"
82                    + "either aborted() or completedNormally()");
83        }
84    }
85
86    protected ExpectedResults expectedResults() {
87        checkCount.incrementAndGet();
88        return new ExpectedResults(testRunnerRule.testClass(), ios,
89                checkCount);
90    }
91
92    protected static class ExpectedResults {
93
94        private final StringBuilder builder = new StringBuilder();
95        private final InterceptOutputStreams ios;
96        private final AtomicInteger checkCount;
97        private String testClassName;
98        private Function<String, String> filter;
99
100        private ExpectedResults(
101                Class<?> testClass, InterceptOutputStreams ios, AtomicInteger checkCount) {
102            this.testClassName = testClass.getName();
103            this.checkCount = checkCount;
104            // Automatically strip out methods from a stack trace to avoid making tests dependent
105            // on either the call hierarchy or on source line numbers which would make the tests
106            // incredibly fragile. If a test fails then the unfiltered output containing the full
107            // stack trace will be output so this will not lose information needed to debug errors.
108            filter = new Function<String, String>() {
109                @Override
110                public String apply(String input) {
111                    // Remove stack trace from output.
112                    return input.replaceAll("\\t(at[^\\n]+|\\.\\.\\. [0-9]+ more)\\n", "");
113                }
114            };
115            this.ios = ios;
116        }
117
118        public ExpectedResults text(String message) {
119            builder.append(message);
120            return this;
121        }
122
123        private ExpectedResults addFilter(Function<String, String> function) {
124            filter = Functions.compose(filter, function);
125            return this;
126        }
127
128        public ExpectedResults ensureProfilingWasRequested() {
129            return addFilter(new Function<String, String>() {
130                @Override
131                public String apply(String input) {
132                    // Make sure that profiling is requested (even though it's not supported).
133                    assertTrue("Profiling was not requested",
134                            input.startsWith("Profiling is disabled: "));
135
136                    input = input.replaceAll("^Profiling is disabled:[^\n]+\\n", "");
137                    return input;
138                }
139            });
140        }
141
142        public ExpectedResults forTestClass(Class<?> testClass) {
143            this.testClassName = testClass.getName();
144            return this;
145        }
146
147        public ExpectedResults forTestClass(String testClassName) {
148            this.testClassName = testClassName;
149            return this;
150        }
151
152        public ExpectedResults failure(String methodName, String message) {
153            String output = outcome(testClassName, methodName, message, Result.EXEC_FAILED);
154            return text(output);
155        }
156
157        public ExpectedResults success(String methodName) {
158            String output = outcome(testClassName, methodName, null, Result.SUCCESS);
159            return text(output);
160        }
161
162        public ExpectedResults success(String methodName, String message) {
163            String output = outcome(testClassName, methodName, message, Result.SUCCESS);
164            return text(output);
165        }
166
167
168        public ExpectedResults unsupported() {
169            String output = outcome(
170                    testClassName, null,
171                    "Skipping " + testClassName + ": no associated runner class\n",
172                    Result.UNSUPPORTED);
173            return text(output);
174        }
175
176        public ExpectedResults noRunner() {
177            String message =
178                    String.format("Skipping %s: no associated runner class\n", testClassName);
179            String output = outcome(testClassName, null, message, Result.UNSUPPORTED);
180            return text(output);
181        }
182
183        public void completedNormally() {
184            text("//00xx{\"completedNormally\":true}\n");
185            checkFilteredOutput(builder.toString());
186        }
187
188        public void aborted() {
189            checkFilteredOutput(builder.toString());
190        }
191
192
193        private static String outcome(
194                String testClassName, String methodName, String message, Result result) {
195            String testName = JUnitUtils.getTestName(testClassName, methodName);
196
197            return String.format("//00xx{\"outcome\":\"%s\"}\n"
198                            + "%s"
199                            + "//00xx{\"result\":\"%s\"}\n",
200                    testName, message == null ? "" : message, result);
201        }
202
203        private void checkFilteredOutput(String expected) {
204            checkCount.decrementAndGet();
205            String output = ios.contents(Stream.OUT);
206            String filtered = filter.apply(output);
207            if (!expected.equals(filtered)) {
208                assertEquals(expected, output);
209            }
210        }
211    }
212}
213