1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Run the first page of every benchmark that has a composable measurement.
6
7Ideally this test would be comprehensive, but the above serves as a
8kind of smoke test.
9"""
10
11import os
12import unittest
13
14from telemetry import benchmark as benchmark_module
15from telemetry.core import discover
16from telemetry.page import page_test
17from telemetry.unittest import options_for_unittests
18from telemetry.unittest import progress_reporter
19
20
21def SmokeTestGenerator(benchmark):
22  # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
23  #
24  # This smoke test dynamically tests all benchmarks. So disabling it for one
25  # failing or flaky benchmark would disable a much wider swath of coverage
26  # than is usally intended. Instead, if a particular benchmark is failing,
27  # disable it in tools/perf/benchmarks/*.
28  @benchmark_module.Disabled('chromeos')  # crbug.com/351114
29  def BenchmarkSmokeTest(self):
30    # Only measure a single page so that this test cycles reasonably quickly.
31    benchmark.options['pageset_repeat'] = 1
32    benchmark.options['page_repeat'] = 1
33
34    class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
35      def CreatePageSet(self, options):
36        # pylint: disable=E1002
37        ps = super(SinglePageBenchmark, self).CreatePageSet(options)
38        for p in ps.pages:
39          if not p.disabled:
40            p.skip_waits = True
41            ps.pages = [p]
42            break
43        return ps
44
45    # Set the benchmark's default arguments.
46    options = options_for_unittests.GetCopy()
47    options.output_format = 'none'
48    options.suppress_gtest_report = True
49    parser = options.CreateParser()
50
51    benchmark.AddCommandLineArgs(parser)
52    benchmark_module.AddCommandLineArgs(parser)
53    benchmark.SetArgumentDefaults(parser)
54    options.MergeDefaultValues(parser.get_default_values())
55
56    benchmark.ProcessCommandLineArgs(None, options)
57    benchmark_module.ProcessCommandLineArgs(None, options)
58
59    self.assertEqual(0, SinglePageBenchmark().Run(options),
60                     msg='Failed: %s' % benchmark)
61
62  return BenchmarkSmokeTest
63
64
65def load_tests(_, _2, _3):
66  suite = progress_reporter.TestSuite()
67
68  benchmarks_dir = os.path.dirname(__file__)
69  top_level_dir = os.path.dirname(benchmarks_dir)
70  measurements_dir = os.path.join(top_level_dir, 'measurements')
71
72  all_measurements = discover.DiscoverClasses(
73      measurements_dir, top_level_dir, page_test.PageTest,
74      pattern='*.py').values()
75  all_benchmarks = discover.DiscoverClasses(
76      benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
77      pattern='*.py').values()
78  for benchmark in all_benchmarks:
79    if benchmark.PageTestClass() not in all_measurements:
80      # If the benchmark is not in measurements, then it is not composable.
81      # Ideally we'd like to test these as well, but the non-composable
82      # benchmarks are usually long-running benchmarks.
83      continue
84
85    # TODO(tonyg): Smoke doesn't work with session_restore yet.
86    if benchmark.Name().startswith('session_restore'):
87      continue
88
89    if hasattr(benchmark, 'generated_profile_archive'):
90      # We'd like to test these, but don't know how yet.
91      continue
92
93    class BenchmarkSmokeTest(unittest.TestCase):
94      pass
95    setattr(BenchmarkSmokeTest, benchmark.Name(), SmokeTestGenerator(benchmark))
96    suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
97
98  return suite
99