benchmark_unittest.py revision 116680a4aac90f2aa7413d9095a592090648e557 
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Run the first page of every benchmark that has a composable measurement.
6
7Ideally this test would be comprehensive, but the above serves as a
8kind of smoke test.
9"""
10
11import os
12import unittest
13
14from telemetry import benchmark as benchmark_module
15from telemetry.core import discover
16from telemetry.page import page_measurement
17from telemetry.unittest import options_for_unittests
18from telemetry.unittest import output_formatter
19
20
21def SmokeTestGenerator(benchmark):
22  # In general you should @benchmark_module.Disabled individual benchmarks that
23  # fail, instead of this entire smoke test suite.
24  # TODO(achuith): Multiple tests failing on CrOS. crbug.com/351114
25  @benchmark_module.Disabled('chromeos')
26  def BenchmarkSmokeTest(self):
27    # Only measure a single page so that this test cycles reasonably quickly.
28    benchmark.options['pageset_repeat'] = 1
29    benchmark.options['page_repeat'] = 1
30
31    class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
32      def CreatePageSet(self, options):
33        # pylint: disable=E1002
34        ps = super(SinglePageBenchmark, self).CreatePageSet(options)
35        ps.pages = ps.pages[:1]
36        return ps
37
38    # Set the benchmark's default arguments.
39    options = options_for_unittests.GetCopy()
40    options.output_format = 'none'
41    parser = options.CreateParser()
42
43    benchmark.AddCommandLineArgs(parser)
44    benchmark_module.AddCommandLineArgs(parser)
45    benchmark.SetArgumentDefaults(parser)
46    options.MergeDefaultValues(parser.get_default_values())
47
48    benchmark.ProcessCommandLineArgs(None, options)
49    benchmark_module.ProcessCommandLineArgs(None, options)
50
51    self.assertEqual(0, SinglePageBenchmark().Run(options),
52                     msg='Failed: %s' % benchmark)
53
54  return BenchmarkSmokeTest
55
56
57def load_tests(_, _2, _3):
58  suite = output_formatter.TestSuite()
59
60  benchmarks_dir = os.path.dirname(__file__)
61  top_level_dir = os.path.dirname(benchmarks_dir)
62  measurements_dir = os.path.join(top_level_dir, 'measurements')
63
64  all_measurements = discover.DiscoverClasses(
65      measurements_dir, top_level_dir, page_measurement.PageMeasurement,
66      pattern='*.py').values()
67  all_benchmarks = discover.DiscoverClasses(
68      benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
69      pattern='*.py').values()
70  for benchmark in all_benchmarks:
71    if benchmark.PageTestClass() not in all_measurements:
72      # If the benchmark is not in measurements, then it is not composable.
73      # Ideally we'd like to test these as well, but the non-composable
74      # benchmarks are usually long-running benchmarks.
75      continue
76
77    # TODO(tonyg): Smoke doesn't work with session_restore yet.
78    if benchmark.Name().startswith('session_restore'):
79      continue
80
81    if hasattr(benchmark, 'generated_profile_archive'):
82      # We'd like to test these, but don't know how yet.
83      continue
84
85    class BenchmarkSmokeTest(unittest.TestCase):
86      pass
87    setattr(BenchmarkSmokeTest, benchmark.Name(), SmokeTestGenerator(benchmark))
88    suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
89
90  return suite
91