13551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)# Copyright 2013 The Chromium Authors. All rights reserved. 23551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 33551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)# found in the LICENSE file. 43551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 53551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)"""Generates test runner factory and tests for performance tests.""" 63551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 73551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import json 84e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)import fnmatch 93551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import logging 103551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import os 113551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import psutil 123551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import signal 138bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)import shutil 143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import time 153551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)from pylib import constants 173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from pylib import forwarder 18a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)from pylib.utils import test_environment 193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import test_runner 213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def Setup(test_options): 243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """Create and return the test runner factory and tests. 253551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) Args: 273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) test_options: A PerformanceOptions object. 283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) Returns: 303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) A tuple of (TestRunnerFactory, tests). 313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """ 32f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) # TODO(bulach): remove this once the bot side lands. BUG=318369 33f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) constants.SetBuildType('Release') 348bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if os.path.exists(constants.PERF_OUTPUT_DIR): 358bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) shutil.rmtree(constants.PERF_OUTPUT_DIR) 368bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) os.makedirs(constants.PERF_OUTPUT_DIR) 3758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # Before running the tests, kill any leftover server. 39a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) test_environment.CleanupLeftoverProcesses() 40a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) forwarder.Forwarder.UseMultiprocessing() 413551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 42f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if test_options.single_step: 43f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) # Running a single command, build the tests structure. 44f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) tests = [['single_step', test_options.single_step]] 45f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 46f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if test_options.steps: 47f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) with file(test_options.steps, 'r') as f: 48f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) tests = json.load(f) 493551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # The list is necessary to keep the steps order, but internally 514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # the format is squashed from a list of lists into a single dict: 524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # [["A", "cmd"], ["B", "cmd"]] into {"A": "cmd", "B": "cmd"} 534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) sorted_test_names = [i[0] for i in tests] 544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) tests_dict = dict(tests) 554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if test_options.test_filter: 574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) sorted_test_names = fnmatch.filter(sorted_test_names, 584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_options.test_filter) 594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) tests_dict = dict((k, v) for k, v in tests_dict.iteritems() 604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if k in sorted_test_names) 614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 623551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) flaky_steps = [] 633551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if test_options.flaky_steps: 643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) with file(test_options.flaky_steps, 'r') as f: 653551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) flaky_steps = json.load(f) 663551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 673551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) def TestRunnerFactory(device, shard_index): 683551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return test_runner.TestRunner( 694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_options, device, tests_dict, flaky_steps) 703551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 714e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return (TestRunnerFactory, sorted_test_names) 72