1b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Copyright 2012 the V8 project authors. All rights reserved. 2b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Redistribution and use in source and binary forms, with or without 3b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# modification, are permitted provided that the following conditions are 4b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# met: 5b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# 6b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Redistributions of source code must retain the above copyright 7b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# notice, this list of conditions and the following disclaimer. 8b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Redistributions in binary form must reproduce the above 9b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# copyright notice, this list of conditions and the following 10b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# disclaimer in the documentation and/or other materials provided 11b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# with the distribution. 12b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Neither the name of Google Inc. nor the names of its 13b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# contributors may be used to endorse or promote products derived 14b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# from this software without specific prior written permission. 15b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# 16b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 28b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 29014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochimport collections 30b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport os 31342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdochimport re 32b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport shutil 33014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochimport sys 34b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport time 35b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 36b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom pool import Pool 37b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom . import commands 38b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom . import perfdata 39958fae7ec3f466955f8e5b50fa5b8d38b9e91675Emily Bernierfrom . import statusfile 40014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochfrom . import testsuite 41b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom . import utils 42342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdochfrom ..objects import output 43b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 44b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 45014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch# Base dir of the v8 checkout. 46014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben MurdochBASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( 47014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch os.path.abspath(__file__))))) 48014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben MurdochTEST_DIR = os.path.join(BASE_DIR, "test") 49014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 50014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 51014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochclass Instructions(object): 521b268ca467c924004286c97bac133db489cf43d0Ben Murdoch def __init__(self, command, test_id, timeout, verbose): 53b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.command = command 54b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.id = test_id 55b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.timeout = timeout 56b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.verbose = verbose 57b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 58b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 59014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch# Structure that keeps global information per worker process. 60014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben MurdochProcessContext = collections.namedtuple( 61014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch "process_context", ["suites", "context"]) 62014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 63014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 64014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochdef MakeProcessContext(context): 65014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch """Generate a process-local context. 66014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 67014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch This reloads all suites per process and stores the global context. 68014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 69014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch Args: 70014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch context: The global context from the test runner. 71014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch """ 72014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch suite_paths = utils.GetSuitePaths(TEST_DIR) 73014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch suites = {} 74014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch for root in suite_paths: 75014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # Don't reinitialize global state as this is concurrently called from 76014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # different processes. 77014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch suite = testsuite.TestSuite.LoadTestSuite( 78014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch os.path.join(TEST_DIR, root), global_init=False) 79014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if suite: 80014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch suites[suite.name] = suite 81014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch return ProcessContext(suites, context) 82014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 83014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 84014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochdef GetCommand(test, context): 85014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch d8testflag = [] 86342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch shell = test.shell() 87014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if shell == "d8": 88014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch d8testflag = ["--test"] 89014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if utils.IsWindows(): 90014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch shell += ".exe" 91014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if context.random_seed: 92014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch d8testflag += ["--random-seed=%s" % context.random_seed] 93014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch cmd = (context.command_prefix + 94014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch [os.path.abspath(os.path.join(context.shell_dir, shell))] + 95014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch d8testflag + 96014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch test.suite.GetFlagsForTestCase(test, context) + 97014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch context.extra_flags) 98014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch return cmd 99014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 100014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 101014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochdef _GetInstructions(test, context): 102014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch command = GetCommand(test, context) 103014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch timeout = context.timeout 104014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if ("--stress-opt" in test.flags or 105014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch "--stress-opt" in context.mode_flags or 106014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch "--stress-opt" in context.extra_flags): 107014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch timeout *= 4 108014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if "--noenable-vfp3" in context.extra_flags: 109014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch timeout *= 2 110014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # FIXME(machenbach): Make this more OO. Don't expose default outcomes or 111014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # the like. 112014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if statusfile.IsSlow(test.outcomes or [statusfile.PASS]): 113014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch timeout *= 2 1141b268ca467c924004286c97bac133db489cf43d0Ben Murdoch return Instructions(command, test.id, timeout, context.verbose) 115014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 116014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 117014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochclass Job(object): 118014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch """Stores data to be sent over the multi-process boundary. 119014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 120014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch All contained fields will be pickled/unpickled. 121014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch """ 122014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 123014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch def Run(self, process_context): 124014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch """Executes the job. 125014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 126014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch Args: 127014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch process_context: Process-local information that is initialized by the 128014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch executing worker. 129014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch """ 130014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch raise NotImplementedError() 131014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 132014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 133342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdochdef SetupProblem(exception, test): 134342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch stderr = ">>> EXCEPTION: %s\n" % exception 135342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch match = re.match(r"^.*No such file or directory: '(.*)'$", str(exception)) 136342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch if match: 137342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch # Extra debuging information when files are claimed missing. 138342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch f = match.group(1) 139342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch stderr += ">>> File %s exists? -> %s\n" % (f, os.path.exists(f)) 1401b268ca467c924004286c97bac133db489cf43d0Ben Murdoch return test.id, output.Output(1, False, "", stderr, None), 0 141342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch 142342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch 143014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochclass TestJob(Job): 144014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch def __init__(self, test): 145014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.test = test 146014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 1471b268ca467c924004286c97bac133db489cf43d0Ben Murdoch def _rename_coverage_data(self, output, context): 1481b268ca467c924004286c97bac133db489cf43d0Ben Murdoch """Rename coverage data. 1491b268ca467c924004286c97bac133db489cf43d0Ben Murdoch 1501b268ca467c924004286c97bac133db489cf43d0Ben Murdoch Rename files with PIDs to files with unique test IDs, because the number 1511b268ca467c924004286c97bac133db489cf43d0Ben Murdoch of tests might be higher than pid_max. E.g.: 1521b268ca467c924004286c97bac133db489cf43d0Ben Murdoch d8.1234.sancov -> d8.test.1.sancov, where 1234 was the process' PID 1531b268ca467c924004286c97bac133db489cf43d0Ben Murdoch and 1 is the test ID. 1541b268ca467c924004286c97bac133db489cf43d0Ben Murdoch """ 1551b268ca467c924004286c97bac133db489cf43d0Ben Murdoch if context.sancov_dir and output.pid is not None: 1561b268ca467c924004286c97bac133db489cf43d0Ben Murdoch sancov_file = os.path.join( 1571b268ca467c924004286c97bac133db489cf43d0Ben Murdoch context.sancov_dir, "%s.%d.sancov" % (self.test.shell(), output.pid)) 1581b268ca467c924004286c97bac133db489cf43d0Ben Murdoch 1591b268ca467c924004286c97bac133db489cf43d0Ben Murdoch # Some tests are expected to fail and don't produce coverage data. 1601b268ca467c924004286c97bac133db489cf43d0Ben Murdoch if os.path.exists(sancov_file): 1611b268ca467c924004286c97bac133db489cf43d0Ben Murdoch parts = sancov_file.split(".") 1621b268ca467c924004286c97bac133db489cf43d0Ben Murdoch new_sancov_file = ".".join( 1631b268ca467c924004286c97bac133db489cf43d0Ben Murdoch parts[:-2] + ["test", str(self.test.id)] + parts[-1:]) 1641b268ca467c924004286c97bac133db489cf43d0Ben Murdoch assert not os.path.exists(new_sancov_file) 1651b268ca467c924004286c97bac133db489cf43d0Ben Murdoch os.rename(sancov_file, new_sancov_file) 1661b268ca467c924004286c97bac133db489cf43d0Ben Murdoch 167014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch def Run(self, process_context): 168342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch try: 169342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch # Retrieve a new suite object on the worker-process side. The original 170342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch # suite object isn't pickled. 171342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch self.test.SetSuiteObject(process_context.suites) 172342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch instr = _GetInstructions(self.test, process_context.context) 173342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch except Exception, e: 174342c50ce1624b485728b9a4fc41d8bbf37eb46cfBen Murdoch return SetupProblem(e, self.test) 175014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 176014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch start_time = time.time() 177014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch output = commands.Execute(instr.command, instr.verbose, instr.timeout) 1781b268ca467c924004286c97bac133db489cf43d0Ben Murdoch self._rename_coverage_data(output, process_context.context) 179014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch return (instr.id, output, time.time() - start_time) 180014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 181014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 182014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdochdef RunTest(job, process_context): 183014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch return job.Run(process_context) 184014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 185b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 186b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass Runner(object): 187b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 188b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def __init__(self, suites, progress_indicator, context): 189b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.datapath = os.path.join("out", "testrunner_data") 190014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.perf_data_manager = perfdata.GetPerfDataManager( 191014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch context, self.datapath) 192b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode) 193b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.perf_failures = False 194b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.printed_allocations = False 195b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.tests = [ t for s in suites for t in s.tests ] 196b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not context.no_sorting: 197b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for t in self.tests: 198b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch t.duration = self.perfdata.FetchPerfData(t) or 1.0 199014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch slow_key = lambda t: statusfile.IsSlow(t.outcomes) 200014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.tests.sort(key=slow_key, reverse=True) 201b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.tests.sort(key=lambda t: t.duration, reverse=True) 202014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self._CommonInit(suites, progress_indicator, context) 203b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 204014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch def _CommonInit(self, suites, progress_indicator, context): 205014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.total = 0 206014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch for s in suites: 207014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch for t in s.tests: 208014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch t.id = self.total 209014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.total += 1 210b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator = progress_indicator 211014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch progress_indicator.SetRunner(self) 212b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.context = context 213b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.succeeded = 0 214014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.remaining = self.total 215b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.failed = [] 216b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.crashed = 0 217b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.reran_tests = 0 218b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 219b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def _RunPerfSafe(self, fun): 220b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch try: 221b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch fun() 222b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch except Exception, e: 223b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("PerfData exception: %s" % e) 224b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.perf_failures = True 225b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 226b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def _MaybeRerun(self, pool, test): 227b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if test.run <= self.context.rerun_failures_count: 228b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Possibly rerun this test if its run count is below the maximum per 229b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # test. <= as the flag controls reruns not including the first run. 230b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if test.run == 1: 231b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Count the overall number of reran tests on the first rerun. 232b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if self.reran_tests < self.context.rerun_failures_max: 233b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.reran_tests += 1 234b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 235b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Don't rerun this if the overall number of rerun tests has been 236b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # reached. 237b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 238b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if test.run >= 2 and test.duration > self.context.timeout / 20.0: 239b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Rerun slow tests at most once. 240b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 241b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 242b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Rerun this test. 243b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.duration = None 244b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.output = None 245b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.run += 1 246014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch pool.add([TestJob(test)]) 247b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.remaining += 1 248014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.total += 1 249b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 250b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def _ProcessTestNormal(self, test, result, pool): 251b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.output = result[1] 252b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.duration = result[2] 253b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch has_unexpected_output = test.suite.HasUnexpectedOutput(test) 254b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if has_unexpected_output: 255b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.failed.append(test) 256b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if test.output.HasCrashed(): 257b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.crashed += 1 258b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 259b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.succeeded += 1 260b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.remaining -= 1 261b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # For the indicator, everything that happens after the first run is treated 262b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # as unexpected even if it flakily passes in order to include it in the 263b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # output. 264b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator.HasRun(test, has_unexpected_output or test.run > 1) 265b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if has_unexpected_output: 266b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Rerun test failures after the indicator has processed the results. 267014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self._VerbosePrint("Attempting to rerun test after failure.") 268b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self._MaybeRerun(pool, test) 269b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Update the perf database if the test succeeded. 270b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return not has_unexpected_output 271b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 272b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def _ProcessTestPredictable(self, test, result, pool): 273b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def HasDifferentAllocations(output1, output2): 274b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def AllocationStr(stdout): 275b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for line in reversed((stdout or "").splitlines()): 276b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if line.startswith("### Allocations = "): 277b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.printed_allocations = True 278b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return line 279b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return "" 280b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return (AllocationStr(output1.stdout) != AllocationStr(output2.stdout)) 281b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 282b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Always pass the test duration for the database update. 283b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.duration = result[2] 284b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if test.run == 1 and result[1].HasTimedOut(): 285b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # If we get a timeout in the first run, we are already in an 286b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # unpredictable state. Just report it as a failure and don't rerun. 287b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.output = result[1] 288b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.remaining -= 1 289b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.failed.append(test) 290b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator.HasRun(test, True) 291b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if test.run > 1 and HasDifferentAllocations(test.output, result[1]): 292b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # From the second run on, check for different allocations. If a 293b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # difference is found, call the indicator twice to report both tests. 294b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # All runs of each test are counted as one for the statistic. 295b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.remaining -= 1 296b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.failed.append(test) 297b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator.HasRun(test, True) 298b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.output = result[1] 299b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator.HasRun(test, True) 300b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif test.run >= 3: 301b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # No difference on the third run -> report a success. 302b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.remaining -= 1 303b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.succeeded += 1 304b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.output = result[1] 305b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator.HasRun(test, False) 306b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 307b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # No difference yet and less than three runs -> add another run and 308b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # remember the output for comparison. 309b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.run += 1 310b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test.output = result[1] 311014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch pool.add([TestJob(test)]) 312b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Always update the perf database. 313b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return True 314b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 315b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def Run(self, jobs): 316b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator.Starting() 317b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self._RunInternal(jobs) 318b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.indicator.Done() 319014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if self.failed: 320b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 1 321014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch elif self.remaining: 322014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch return 2 323b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 0 324b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 325b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def _RunInternal(self, jobs): 326b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch pool = Pool(jobs) 327b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test_map = {} 328014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch queued_exception = [None] 329014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch def gen_tests(): 330014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch for test in self.tests: 331014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch assert test.id >= 0 332014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch test_map[test.id] = test 333014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch try: 334014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch yield [TestJob(test)] 335014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch except Exception, e: 336014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # If this failed, save the exception and re-raise it later (after 337014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # all other tests have had a chance to run). 338014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch queued_exception[0] = e 339014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch continue 340b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch try: 341014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch it = pool.imap_unordered( 342014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch fn=RunTest, 343014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch gen=gen_tests(), 344014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch process_context_fn=MakeProcessContext, 345014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch process_context_args=[self.context], 346014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch ) 347b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for result in it: 348014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if result.heartbeat: 349014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.indicator.Heartbeat() 350014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch continue 351014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch test = test_map[result.value[0]] 352b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if self.context.predictable: 353014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch update_perf = self._ProcessTestPredictable(test, result.value, pool) 354b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 355014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch update_perf = self._ProcessTestNormal(test, result.value, pool) 356b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if update_perf: 357b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self._RunPerfSafe(lambda: self.perfdata.UpdatePerfData(test)) 358b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch finally: 359014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self._VerbosePrint("Closing process pool.") 360b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch pool.terminate() 361014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self._VerbosePrint("Closing database connection.") 362b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self._RunPerfSafe(lambda: self.perf_data_manager.close()) 363b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if self.perf_failures: 364b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Nuke perf data in case of failures. This might not work on windows as 365b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # some files might still be open. 366b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "Deleting perf test data due to db corruption." 367b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shutil.rmtree(self.datapath) 368014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if queued_exception[0]: 369014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch raise queued_exception[0] 370014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 371014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # Make sure that any allocations were printed in predictable mode (if we 372014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch # ran any tests). 373014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch assert ( 374014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch not self.total or 375014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch not self.context.predictable or 376014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch self.printed_allocations 377014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch ) 378014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch 379014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch def _VerbosePrint(self, text): 380014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch if self.context.verbose: 381014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch print text 382014dc512cdd3e367bee49a713fdc5ed92584a3e5Ben Murdoch sys.stdout.flush() 383b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 384b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 385b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass BreakNowException(Exception): 386b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def __init__(self, value): 387b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.value = value 388b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def __str__(self): 389b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return repr(self.value) 390