1/**
2 * @license
3 * Copyright 2013 Google Inc. All rights reserved.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *   http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.google.security.wycheproof;
18
19import java.lang.annotation.ElementType;
20import java.lang.annotation.Retention;
21import java.lang.annotation.RetentionPolicy;
22import java.lang.annotation.Target;
23import java.util.Arrays;
24import org.junit.runner.Description;
25import org.junit.runner.manipulation.Filter;
26import org.junit.runner.manipulation.NoTestsRemainException;
27import org.junit.runners.Suite;
28import org.junit.runners.model.InitializationError;
29import org.junit.runners.model.RunnerBuilder;
30
31/**
32 * <p>A custom JUnit4 runner that, with annotations, allows choosing tests to run on a specific
33 * provider. To use it, annotate a runner class with {@code RunWith(WycheproofRunner.class)}, and
34 * {@code SuiteClasses({AesGcmTest.class, ...})}. When you run this class, it will run all the tests
35 * in all the suite classes.
36 *
37 * <p>To exclude certain tests, a runner class should be annotated with {@code @Provider} which
38 * indicates the target provider. Test exclusion is defined as follows:
39 * <ul>@Fast test runners skip @SlowTest test functions.
40 * <ul>@Presubmit test runners skip @NoPresubmitTest test functions.
41 * <ul>All test runners skip @ExcludedTest test functions.
42 *
43 * @author thaidn@google.com (Thai Duong)
44 */
45public class WycheproofRunner extends Suite {
46
47  /** List of supported providers. */
48  public enum ProviderType {
49    BOUNCY_CASTLE,
50    CONSCRYPT,
51    OPENJDK,
52    SPONGY_CASTLE,
53  }
54
55  // Annotations for test runners.
56
57  /**
58   * Annotation to specify the target provider of a test runner.
59   *
60   * <p>Usage: @Provider(ProviderType.BOUNCY_CASTLE)
61   */
62  @Retention(RetentionPolicy.RUNTIME)
63  @Target({ElementType.TYPE})
64  public @interface Provider {
65    ProviderType value();
66  }
67
68  /**
69   * Annotation to specify presubmit test runners that exclude {@code @NoPresubmitTets} tests.
70   *
71   * <p>Usage: @Presubmit(ProviderType.BOUNCY_CASTLE)
72   */
73  @Retention(RetentionPolicy.RUNTIME)
74  @Target({ElementType.TYPE})
75  public @interface Presubmit {}
76
77  /**
78   * Annotation to specify fast test runners that exclude {@code @SlowTest} tests.
79   *
80   * <p>Usage: @Fast
81   */
82  @Retention(RetentionPolicy.RUNTIME)
83  @Target({ElementType.TYPE})
84  public @interface Fast {}
85
86  // Annotations for test functions
87
88  /**
89   * Tests that take too much time to run, should be excluded from TAP and wildcard target patterns
90   * like:..., :*, or :all.
91   *
92   * <p>Usage: @SlowTest(providers = {ProviderType.BOUNCY_CASTLE, ...})
93   */
94  @Retention(RetentionPolicy.RUNTIME)
95  @Target({ElementType.METHOD})
96  public @interface SlowTest {
97    ProviderType[] providers();
98  }
99
100  /**
101   * Tests that should be excluded from presubmit checks on specific providers.
102   *
103   * <p>Usage: @NoPresubmitTest(
104   *   providers = {ProviderType.BOUNCY_CASTLE, ...},
105   *   bugs = {"b/123456789"}
106   * )
107   */
108  @Retention(RetentionPolicy.RUNTIME)
109  @Target({ElementType.METHOD, ElementType.FIELD})
110  public @interface NoPresubmitTest {
111    /** List of providers that this test method should not run as presubmit check. */
112    ProviderType[] providers();
113
114    /** List of blocking bugs (and comments). */
115    String[] bugs();
116  }
117
118  /**
119   * Annotation to specify test functions that should be excluded on specific providers.
120   *
121   * <p>Usage: @ExcludedTest(providers = {ProviderType.BOUNCY_CASTLE, ProviderType.OPENJDK})
122   */
123  @Retention(RetentionPolicy.RUNTIME)
124  @Target({ElementType.METHOD})
125  public @interface ExcludedTest {
126    ProviderType[] providers();
127    String comment();
128  }
129
130  /**
131   * Custom filter to exclude certain test functions.
132   *
133   */
134  public static class ExcludeTestFilter extends Filter {
135
136    Class<?> runnerClass;
137    Provider targetProvider;
138    Fast fast;
139    Presubmit presubmit;
140
141    public ExcludeTestFilter(Class<?> runnerClass) {
142      this.runnerClass = runnerClass;
143      this.targetProvider = runnerClass.getAnnotation(Provider.class);
144      this.fast = runnerClass.getAnnotation(Fast.class);
145      this.presubmit = runnerClass.getAnnotation(Presubmit.class);
146    }
147
148    @Override
149    public String describe() {
150      return "exclude certain tests on specific providers";
151    }
152
153    @Override
154    public boolean shouldRun(Description description) {
155      return isOkayToRunTest(description);
156    }
157
158    private boolean isOkayToRunTest(Description description) {
159      if (targetProvider == null) {
160        // Run all test functions if the test runner is not annotated with {@code @Provider}.
161        return true;
162      }
163      // Skip @ExcludedTest tests
164      ExcludedTest excludedTest = description.getAnnotation(ExcludedTest.class);
165      if (excludedTest != null
166          && Arrays.asList(excludedTest.providers()).contains(targetProvider.value())) {
167        return false;
168      }
169
170      // If the runner class is annotated with @Presubmit, skip non-presubmit tests
171      if (presubmit != null) {
172        NoPresubmitTest ignoreOn = description.getAnnotation(NoPresubmitTest.class);
173        if (ignoreOn != null
174            && Arrays.asList(ignoreOn.providers()).contains(targetProvider.value())) {
175          return false;
176        }
177      }
178
179      // If the runner class is annotated with @Fast, skip slow tests
180      if (fast != null) {
181        SlowTest ignoreOn = description.getAnnotation(SlowTest.class);
182        if (ignoreOn != null
183            && Arrays.asList(ignoreOn.providers()).contains(targetProvider.value())) {
184          return false;
185        }
186      }
187
188      // run everything else
189      return true;
190    }
191  }
192
193  /** Required constructor: called by JUnit reflectively. */
194  public WycheproofRunner(Class<?> runnerClass, RunnerBuilder builder) throws InitializationError {
195    super(runnerClass, builder);
196    addFilter(new ExcludeTestFilter(runnerClass));
197  }
198
199  private void addFilter(Filter filter) {
200    try {
201      filter(filter);
202    } catch (NoTestsRemainException ex) {
203      System.out.println("No tests remain exception: " + ex);
204    }
205  }
206}
207