1e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin/* 2e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * Copyright (C) 2009 Google Inc. 3e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * 4e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * Licensed under the Apache License, Version 2.0 (the "License"); 5e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * you may not use this file except in compliance with the License. 6e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * You may obtain a copy of the License at 7e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * 8e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * http://www.apache.org/licenses/LICENSE-2.0 9e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * 10e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * Unless required by applicable law or agreed to in writing, software 11e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * distributed under the License is distributed on an "AS IS" BASIS, 12e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * See the License for the specific language governing permissions and 14e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * limitations under the License. 15e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin */ 16e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 17e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinpackage com.google.caliper.runner; 18e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 19e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport static org.junit.Assert.assertEquals; 20e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport static org.junit.Assert.fail; 21e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 22e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport com.google.caliper.Benchmark; 23e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport com.google.caliper.Param; 24e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport com.google.caliper.config.InvalidConfigurationException; 25e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport com.google.caliper.util.InvalidCommandException; 26e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 27e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport junit.framework.AssertionFailedError; 28e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 29e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport org.junit.Test; 30e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport org.junit.runner.RunWith; 31e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport org.junit.runners.JUnit4; 32e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 33e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport java.io.PrintWriter; 34e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinimport java.io.StringWriter; 35e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 36e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin/** 37e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin * Unit test covering common user mistakes in benchmark classes. 38e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin */ 39e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin@RunWith(JUnit4.class) 40e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 41e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffinpublic class MalformedBenchmarksTest { 42e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin // Put the expected messages together here, which may promote some kind of 43e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin // consistency in their wording. :) 44e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 45e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String ABSTRACT = 46e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Class '%s' is abstract"; 47e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String NO_CONSTRUCTOR = 48e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Benchmark class %s does not have a publicly visible default constructor"; 49e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String NO_METHODS = 50e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "There were no experiments to be performed for the class %s using the instruments " + 51e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "[allocation, runtime]"; 52e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String STATIC_BENCHMARK = 53e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Benchmark methods must not be static: timeIt"; 54e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String WRONG_ARGUMENTS = 55e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Benchmark methods must have no arguments or accept a single int or long parameter: timeIt"; 56e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String STATIC_PARAM = 57e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Parameter field 'oops' must not be static"; 58e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String RESERVED_PARAM = 59e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Class '%s' uses reserved parameter name 'vm'"; 60e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String NO_CONVERSION = "Type 'Object' of parameter field 'oops' " 61e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin + "has no recognized String-converting method; see <TODO> for details"; 62e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private static final String CONVERT_FAILED = // granted this one's a little weird (and brittle) 63e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Cannot convert value 'oops' to type 'int': For input string: \"oops\""; 64e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 65e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void abstractBenchmark() throws Exception { 66e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(ABSTRACT, AbstractBenchmark.class); 67e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 68e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin abstract static class AbstractBenchmark {} 69e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 70e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void noSuitableConstructor() throws Exception { 71e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(String.format(NO_CONSTRUCTOR, BadConstructorBenchmark.class.getName()), 72e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin BadConstructorBenchmark.class); 73e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 74e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 75e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @SuppressWarnings("unused") 76e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class BadConstructorBenchmark { 77e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin BadConstructorBenchmark(String damnParam) {} 78e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark void timeIt(int reps) {} 79e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 80e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 81e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void noBenchmarkMethods() throws Exception { 82e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(NO_METHODS, NoMethodsBenchmark.class); 83e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 84e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 85e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @SuppressWarnings("unused") 86e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class NoMethodsBenchmark { 87e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin void timeIt(int reps) {} // not annotated 88e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 89e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 90e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void staticBenchmarkMethod() throws Exception { 91e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(STATIC_BENCHMARK, StaticBenchmarkMethodBenchmark.class); 92e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 93e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 94e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @SuppressWarnings("unused") 95e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class StaticBenchmarkMethodBenchmark { 96e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark public static void timeIt(int reps) {} 97e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 98e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 99e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void wrongSignature() throws Exception { 100e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(WRONG_ARGUMENTS, BoxedParamBenchmark.class); 101e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(WRONG_ARGUMENTS, ExtraParamBenchmark.class); 102e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 103e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 104e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @SuppressWarnings("unused") 105e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class BoxedParamBenchmark { 106e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark void timeIt(Integer reps) {} 107e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 108e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 109e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @SuppressWarnings("unused") 110e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class ExtraParamBenchmark { 111e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark void timeIt(int reps, int what) {} 112e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 113e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 114e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void hasBenchmarkOverloads() throws Exception { 115e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin // N.B. baz is fine since although it has an overload, its overload is not a benchmark method. 116e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException( 117e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin "Overloads are disallowed for benchmark methods, found overloads of [bar, foo] in " 118e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin + "benchmark OverloadsAnnotatedBenchmark", 119e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin OverloadsAnnotatedBenchmark.class); 120e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 121e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 122e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @SuppressWarnings("unused") 123e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class OverloadsAnnotatedBenchmark { 124e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark public void foo(long reps) {} 125e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark public void foo(int reps) {} 126e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark public void bar(long reps) {} 127e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark public void bar(int reps) {} 128e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Benchmark public void baz(int reps) {} 129e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin public void baz(long reps, boolean thing) {} 130e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin public void baz(long reps) {} 131e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 132e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 133e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void staticParam() throws Exception { 134e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(STATIC_PARAM, StaticParamBenchmark.class); 135e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 136e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class StaticParamBenchmark { 137e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Param static String oops; 138e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 139e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 140e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void reservedParameterName() throws Exception { 141e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(RESERVED_PARAM, ReservedParamBenchmark.class); 142e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 143e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class ReservedParamBenchmark { 144e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Param String vm; 145e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 146e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 147e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void unparsableParamType() throws Exception { 148e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(NO_CONVERSION, UnparsableParamTypeBenchmark.class); 149e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 150e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class UnparsableParamTypeBenchmark { 151e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Param Object oops; 152e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 153e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 154e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Test public void unparsableParamDefault() throws Exception { 155e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin expectException(CONVERT_FAILED, UnparsableParamDefaultBenchmark.class); 156e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 157e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin static class UnparsableParamDefaultBenchmark { 158e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin @Param({"1", "2", "oops"}) int number; 159e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 160e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 161e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin // end of tests 162e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 163e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin private void expectException(String expectedMessageFmt, Class<?> benchmarkClass) 164e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin throws InvalidCommandException, InvalidConfigurationException { 165e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin try { 166e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin CaliperMain.exitlessMain( 167e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin new String[] {"--instrument=allocation,runtime", "--dry-run", benchmarkClass.getName()}, 168e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin new PrintWriter(new StringWriter()), new PrintWriter(new StringWriter())); 169e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin fail("no exception thrown"); 170e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } catch (InvalidBenchmarkException e) { 171e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin try { 172e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin String expectedMessageText = 173e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin String.format(expectedMessageFmt, benchmarkClass.getSimpleName()); 174e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin assertEquals(expectedMessageText, e.getMessage()); 175e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin 176e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin // don't swallow our real stack trace 177e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } catch (AssertionFailedError afe) { 178e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin afe.initCause(e); 179e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin throw afe; 180e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 181e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 182e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin } 183e236301e5fc778bffe1748ed80d7936e6c807012Paul Duffin} 184