1cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# Copyright 2014 The Chromium Authors. All rights reserved. 2cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 3cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# found in the LICENSE file. 4cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 5cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)"""Run the first page of every benchmark that has a composable measurement. 6cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 7cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)Ideally this test would be comprehensive, but the above serves as a 8cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)kind of smoke test. 9cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)""" 10cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 11cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import os 12cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import unittest 13cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 14116680a4aac90f2aa7413d9095a592090648e557Ben Murdochfrom telemetry import benchmark as benchmark_module 15cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)from telemetry.core import discover 166e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)from telemetry.page import page_test 17cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)from telemetry.unittest import options_for_unittests 186e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)from telemetry.unittest import progress_reporter 19cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 20cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 21cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def SmokeTestGenerator(benchmark): 226e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST. 236e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # 246e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # This smoke test dynamically tests all benchmarks. So disabling it for one 256e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # failing or flaky benchmark would disable a much wider swath of coverage 266e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # than is usally intended. Instead, if a particular benchmark is failing, 276e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # disable it in tools/perf/benchmarks/*. 286e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) @benchmark_module.Disabled('chromeos') # crbug.com/351114 29cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def BenchmarkSmokeTest(self): 30cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Only measure a single page so that this test cycles reasonably quickly. 31cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) benchmark.options['pageset_repeat'] = 1 32cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) benchmark.options['page_repeat'] = 1 33cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 34cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) class SinglePageBenchmark(benchmark): # pylint: disable=W0232 35cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def CreatePageSet(self, options): 36cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # pylint: disable=E1002 37cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) ps = super(SinglePageBenchmark, self).CreatePageSet(options) 386e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) for p in ps.pages: 396e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if not p.disabled: 406e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) p.skip_waits = True 416e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) ps.pages = [p] 426e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) break 43cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return ps 44cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 45cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Set the benchmark's default arguments. 46cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) options = options_for_unittests.GetCopy() 47cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) options.output_format = 'none' 485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) options.suppress_gtest_report = True 49cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) parser = options.CreateParser() 50cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 51cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) benchmark.AddCommandLineArgs(parser) 52116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch benchmark_module.AddCommandLineArgs(parser) 53cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) benchmark.SetArgumentDefaults(parser) 54cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) options.MergeDefaultValues(parser.get_default_values()) 55cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 56cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) benchmark.ProcessCommandLineArgs(None, options) 57116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch benchmark_module.ProcessCommandLineArgs(None, options) 58cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 59cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.assertEqual(0, SinglePageBenchmark().Run(options), 60cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) msg='Failed: %s' % benchmark) 61cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 62cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return BenchmarkSmokeTest 63cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 64cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 65cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def load_tests(_, _2, _3): 666e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) suite = progress_reporter.TestSuite() 67cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 68cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) benchmarks_dir = os.path.dirname(__file__) 69cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) top_level_dir = os.path.dirname(benchmarks_dir) 70cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) measurements_dir = os.path.join(top_level_dir, 'measurements') 71cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 72cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) all_measurements = discover.DiscoverClasses( 736e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) measurements_dir, top_level_dir, page_test.PageTest, 74cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) pattern='*.py').values() 75cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) all_benchmarks = discover.DiscoverClasses( 76116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch benchmarks_dir, top_level_dir, benchmark_module.Benchmark, 77116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch pattern='*.py').values() 78cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for benchmark in all_benchmarks: 79cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if benchmark.PageTestClass() not in all_measurements: 80cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # If the benchmark is not in measurements, then it is not composable. 81cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Ideally we'd like to test these as well, but the non-composable 82cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # benchmarks are usually long-running benchmarks. 83cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) continue 84cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 85f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) # TODO(tonyg): Smoke doesn't work with session_restore yet. 86f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if benchmark.Name().startswith('session_restore'): 87f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) continue 88f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 89cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if hasattr(benchmark, 'generated_profile_archive'): 90cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # We'd like to test these, but don't know how yet. 91cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) continue 92cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 93cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) class BenchmarkSmokeTest(unittest.TestCase): 94cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) pass 95cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) setattr(BenchmarkSmokeTest, benchmark.Name(), SmokeTestGenerator(benchmark)) 96cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) suite.addTest(BenchmarkSmokeTest(benchmark.Name())) 97cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 98cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return suite 99