1/* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package vogar.target; 18 19import com.google.common.base.Function; 20import com.google.common.base.Functions; 21import java.util.List; 22import java.util.Properties; 23import java.util.concurrent.atomic.AtomicInteger; 24import org.junit.After; 25import org.junit.Before; 26import org.junit.Rule; 27import vogar.Result; 28import vogar.target.junit.JUnitUtils; 29import vogar.testing.InterceptOutputStreams; 30import vogar.testing.InterceptOutputStreams.Stream; 31 32import static org.junit.Assert.assertEquals; 33import static org.junit.Assert.assertTrue; 34 35/** 36 * Provides support for testing {@link TestRunner} class. 37 * 38 * <p>Subclasses provide the individual test methods, each test method has the following structure. 39 * 40 * <p>It is annotated with {@link TestRunnerProperties @TestRunnerProperties} that specifies the 41 * properties that the main Vogar process supplies to the client process that actually runs the 42 * tests. It must specify either a {@link TestRunnerProperties#testClass()} or 43 * {@link TestRunnerProperties#testClassOrPackage()}, all the remaining properties are optional. 44 * 45 * <p>It calls {@code testRunnerRule.createTestRunner(...)} to create a {@link TestRunner}; passing 46 * in any additional command line arguments that {@link TestRunner#TestRunner(Properties, List)} 47 * accepts. 48 * 49 * <p>It calls {@link TestRunner#run()} to actually run the tests. 50 * 51 * <p>It calls {@link #expectedResults()} to obtain a {@link ExpectedResults} instance that it uses 52 * to specify the expected results for the test. Once it has specified the expected results then it 53 * must call either {@link ExpectedResults#completedNormally()} or 54 * {@link ExpectedResults#aborted()}. They indicate whether the test process completed normally or 55 * would abort (due to a test timing out) and cause the actual results to be checked against the 56 * expected results. 57 */ 58public abstract class AbstractTestRunnerTest { 59 60 @Rule 61 public InterceptOutputStreams ios = new InterceptOutputStreams(Stream.OUT); 62 63 @Rule 64 public TestRunnerRule testRunnerRule = new TestRunnerRule(); 65 66 /** 67 * Keeps track of number of times {@link #expectedResults()} has been called without 68 * {@link ExpectedResults#checkFilteredOutput(String)} 69 * also being called. If it is {@code > 0} then the test is in error. 70 */ 71 private AtomicInteger checkCount; 72 73 @Before 74 public void beforeTest() { 75 checkCount = new AtomicInteger(); 76 } 77 78 @After 79 public void afterTest() { 80 if (checkCount.get() != 0) { 81 throw new IllegalStateException("Test called expectedResults() but failed to call" 82 + "either aborted() or completedNormally()"); 83 } 84 } 85 86 protected ExpectedResults expectedResults() { 87 checkCount.incrementAndGet(); 88 return new ExpectedResults(testRunnerRule.testClass(), ios, 89 checkCount); 90 } 91 92 protected static class ExpectedResults { 93 94 private final StringBuilder builder = new StringBuilder(); 95 private final InterceptOutputStreams ios; 96 private final AtomicInteger checkCount; 97 private String testClassName; 98 private Function<String, String> filter; 99 100 private ExpectedResults( 101 Class<?> testClass, InterceptOutputStreams ios, AtomicInteger checkCount) { 102 this.testClassName = testClass.getName(); 103 this.checkCount = checkCount; 104 // Automatically strip out methods from a stack trace to avoid making tests dependent 105 // on either the call hierarchy or on source line numbers which would make the tests 106 // incredibly fragile. If a test fails then the unfiltered output containing the full 107 // stack trace will be output so this will not lose information needed to debug errors. 108 filter = new Function<String, String>() { 109 @Override 110 public String apply(String input) { 111 // Remove stack trace from output. 112 return input.replaceAll("\\t(at[^\\n]+|\\.\\.\\. [0-9]+ more)\\n", ""); 113 } 114 }; 115 this.ios = ios; 116 } 117 118 public ExpectedResults text(String message) { 119 builder.append(message); 120 return this; 121 } 122 123 private ExpectedResults addFilter(Function<String, String> function) { 124 filter = Functions.compose(filter, function); 125 return this; 126 } 127 128 public ExpectedResults ensureProfilingWasRequested() { 129 return addFilter(new Function<String, String>() { 130 @Override 131 public String apply(String input) { 132 // Make sure that profiling is requested (even though it's not supported). 133 assertTrue("Profiling was not requested", 134 input.startsWith("Profiling is disabled: ")); 135 136 input = input.replaceAll("^Profiling is disabled:[^\n]+\\n", ""); 137 return input; 138 } 139 }); 140 } 141 142 public ExpectedResults forTestClass(Class<?> testClass) { 143 this.testClassName = testClass.getName(); 144 return this; 145 } 146 147 public ExpectedResults forTestClass(String testClassName) { 148 this.testClassName = testClassName; 149 return this; 150 } 151 152 public ExpectedResults failure(String methodName, String message) { 153 String output = outcome(testClassName, methodName, message, Result.EXEC_FAILED); 154 return text(output); 155 } 156 157 public ExpectedResults success(String methodName) { 158 String output = outcome(testClassName, methodName, null, Result.SUCCESS); 159 return text(output); 160 } 161 162 public ExpectedResults success(String methodName, String message) { 163 String output = outcome(testClassName, methodName, message, Result.SUCCESS); 164 return text(output); 165 } 166 167 168 public ExpectedResults unsupported() { 169 String output = outcome( 170 testClassName, null, 171 "Skipping " + testClassName + ": no associated runner class\n", 172 Result.UNSUPPORTED); 173 return text(output); 174 } 175 176 public ExpectedResults noRunner() { 177 String message = 178 String.format("Skipping %s: no associated runner class\n", testClassName); 179 String output = outcome(testClassName, null, message, Result.UNSUPPORTED); 180 return text(output); 181 } 182 183 public void completedNormally() { 184 text("//00xx{\"completedNormally\":true}\n"); 185 checkFilteredOutput(builder.toString()); 186 } 187 188 public void aborted() { 189 checkFilteredOutput(builder.toString()); 190 } 191 192 193 private static String outcome( 194 String testClassName, String methodName, String message, Result result) { 195 String testName = JUnitUtils.getTestName(testClassName, methodName); 196 197 return String.format("//00xx{\"outcome\":\"%s\"}\n" 198 + "%s" 199 + "//00xx{\"result\":\"%s\"}\n", 200 testName, message == null ? "" : message, result); 201 } 202 203 private void checkFilteredOutput(String expected) { 204 checkCount.decrementAndGet(); 205 String output = ios.contents(Stream.OUT); 206 String filtered = filter.apply(output); 207 if (!expected.equals(filtered)) { 208 assertEquals(expected, output); 209 } 210 } 211 } 212} 213