1bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org#!/usr/bin/env python2 2bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# Copyright 2013 Google Inc. All rights reserved. 3bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# 4bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# Licensed under the Apache License, Version 2.0 (the "License"); 5bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# you may not use this file except in compliance with the License. 6bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# You may obtain a copy of the License at 7bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# 8bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# http://www.apache.org/licenses/LICENSE-2.0 9bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# 10bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# Unless required by applicable law or agreed to in writing, software 11bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# distributed under the License is distributed on an "AS IS" BASIS, 12bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# See the License for the specific language governing permissions and 14bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# limitations under the License. 1502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmimport cPickle 16cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boströmimport errno 1702c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmimport gzip 1802c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmimport multiprocessing 19bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgimport optparse 2002c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmimport os 212f29d70184b791e7c6444a48b1239d0541ee787cpbosimport signal 22bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgimport subprocess 23bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgimport sys 24cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boströmimport tempfile 252f29d70184b791e7c6444a48b1239d0541ee787cpbosimport thread 26bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgimport threading 27bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgimport time 2802c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmimport zlib 29bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 302f29d70184b791e7c6444a48b1239d0541ee787cpbos# An object that catches SIGINT sent to the Python process and notices 312f29d70184b791e7c6444a48b1239d0541ee787cpbos# if processes passed to wait() die by SIGINT (we need to look for 322f29d70184b791e7c6444a48b1239d0541ee787cpbos# both of those cases, because pressing Ctrl+C can result in either 332f29d70184b791e7c6444a48b1239d0541ee787cpbos# the main process or one of the subprocesses getting the signal). 342f29d70184b791e7c6444a48b1239d0541ee787cpbos# 352f29d70184b791e7c6444a48b1239d0541ee787cpbos# Before a SIGINT is seen, wait(p) will simply call p.wait() and 362f29d70184b791e7c6444a48b1239d0541ee787cpbos# return the result. Once a SIGINT has been seen (in the main process 372f29d70184b791e7c6444a48b1239d0541ee787cpbos# or a subprocess, including the one the current call is waiting for), 382f29d70184b791e7c6444a48b1239d0541ee787cpbos# wait(p) will call p.terminate() and raise ProcessWasInterrupted. 392f29d70184b791e7c6444a48b1239d0541ee787cpbosclass SigintHandler(object): 402f29d70184b791e7c6444a48b1239d0541ee787cpbos class ProcessWasInterrupted(Exception): pass 412f29d70184b791e7c6444a48b1239d0541ee787cpbos sigint_returncodes = {-signal.SIGINT, # Unix 422f29d70184b791e7c6444a48b1239d0541ee787cpbos -1073741510, # Windows 432f29d70184b791e7c6444a48b1239d0541ee787cpbos } 442f29d70184b791e7c6444a48b1239d0541ee787cpbos def __init__(self): 452f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__lock = threading.Lock() 462f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__processes = set() 472f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__got_sigint = False 482f29d70184b791e7c6444a48b1239d0541ee787cpbos signal.signal(signal.SIGINT, self.__sigint_handler) 492f29d70184b791e7c6444a48b1239d0541ee787cpbos def __on_sigint(self): 502f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__got_sigint = True 512f29d70184b791e7c6444a48b1239d0541ee787cpbos while self.__processes: 522f29d70184b791e7c6444a48b1239d0541ee787cpbos try: 532f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__processes.pop().terminate() 542f29d70184b791e7c6444a48b1239d0541ee787cpbos except OSError: 552f29d70184b791e7c6444a48b1239d0541ee787cpbos pass 562f29d70184b791e7c6444a48b1239d0541ee787cpbos def __sigint_handler(self, signal_num, frame): 572f29d70184b791e7c6444a48b1239d0541ee787cpbos with self.__lock: 582f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__on_sigint() 592f29d70184b791e7c6444a48b1239d0541ee787cpbos def got_sigint(self): 602f29d70184b791e7c6444a48b1239d0541ee787cpbos with self.__lock: 612f29d70184b791e7c6444a48b1239d0541ee787cpbos return self.__got_sigint 622f29d70184b791e7c6444a48b1239d0541ee787cpbos def wait(self, p): 632f29d70184b791e7c6444a48b1239d0541ee787cpbos with self.__lock: 642f29d70184b791e7c6444a48b1239d0541ee787cpbos if self.__got_sigint: 652f29d70184b791e7c6444a48b1239d0541ee787cpbos p.terminate() 662f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__processes.add(p) 672f29d70184b791e7c6444a48b1239d0541ee787cpbos code = p.wait() 682f29d70184b791e7c6444a48b1239d0541ee787cpbos with self.__lock: 692f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__processes.discard(p) 702f29d70184b791e7c6444a48b1239d0541ee787cpbos if code in self.sigint_returncodes: 712f29d70184b791e7c6444a48b1239d0541ee787cpbos self.__on_sigint() 722f29d70184b791e7c6444a48b1239d0541ee787cpbos if self.__got_sigint: 732f29d70184b791e7c6444a48b1239d0541ee787cpbos raise self.ProcessWasInterrupted 742f29d70184b791e7c6444a48b1239d0541ee787cpbos return code 752f29d70184b791e7c6444a48b1239d0541ee787cpbossigint_handler = SigintHandler() 762f29d70184b791e7c6444a48b1239d0541ee787cpbos 77280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström# Return the width of the terminal, or None if it couldn't be 78280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström# determined (e.g. because we're not being run interactively). 79280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boströmdef term_width(out): 80280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström if not out.isatty(): 81280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström return None 82280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström try: 83280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström p = subprocess.Popen(["stty", "size"], 84280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström stdout=subprocess.PIPE, stderr=subprocess.PIPE) 85280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström (out, err) = p.communicate() 86280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström if p.returncode != 0 or err: 87280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström return None 88280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström return int(out.split()[1]) 89280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström except (IndexError, OSError, ValueError): 90280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström return None 91280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström 92280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström# Output transient and permanent lines of text. If several transient 93280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström# lines are written in sequence, the new will overwrite the old. We 94280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström# use this to ensure that lots of unimportant info (tests passing) 95280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström# won't drown out important info (tests failing). 96280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boströmclass Outputter(object): 97280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström def __init__(self, out_file): 98280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__out_file = out_file 99280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__previous_line_was_transient = False 100280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__width = term_width(out_file) # Line width, or None if not a tty. 101280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström def transient_line(self, msg): 102280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström if self.__width is None: 103280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__out_file.write(msg + "\n") 104280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström else: 105280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__out_file.write("\r" + msg[:self.__width].ljust(self.__width)) 106280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__previous_line_was_transient = True 107a0b9549b8802ea47516b2f76b149a95d7cbd5977pbos def flush_transient_output(self): 108280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström if self.__previous_line_was_transient: 109280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__out_file.write("\n") 110280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__previous_line_was_transient = False 111a0b9549b8802ea47516b2f76b149a95d7cbd5977pbos def permanent_line(self, msg): 112a0b9549b8802ea47516b2f76b149a95d7cbd5977pbos self.flush_transient_output() 113280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.__out_file.write(msg + "\n") 114280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström 1150fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.orgstdout_lock = threading.Lock() 116280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström 117bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgclass FilterFormat: 118cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström if sys.stdout.isatty(): 119cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström # stdout needs to be unbuffered since the output is interactive. 120cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) 121cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström 122280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström out = Outputter(sys.stdout) 123bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org total_tests = 0 124bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org finished_tests = 0 125bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 126bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org tests = {} 127bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org outputs = {} 128bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org failures = [] 129bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 130bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org def print_test_status(self, last_finished_test, time_ms): 131280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.out.transient_line("[%d/%d] %s (%d ms)" 132280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström % (self.finished_tests, self.total_tests, 133280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström last_finished_test, time_ms)) 134bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 135bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org def handle_meta(self, job_id, args): 136bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org (command, arg) = args.split(' ', 1) 137bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org if command == "TEST": 138bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org (binary, test) = arg.split(' ', 1) 139bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org self.tests[job_id] = (binary, test.strip()) 140bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org elif command == "EXIT": 141bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org (exit_code, time_ms) = [int(x) for x in arg.split(' ', 1)] 142bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org self.finished_tests += 1 143bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org (binary, test) = self.tests[job_id] 144bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org self.print_test_status(test, time_ms) 145bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org if exit_code != 0: 146bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org self.failures.append(self.tests[job_id]) 147cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström with open(self.outputs[job_id]) as f: 148cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström for line in f.readlines(): 149cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström self.out.permanent_line(line.rstrip()) 150280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.out.permanent_line( 151280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström "[%d/%d] %s returned/aborted with exit code %d (%d ms)" 152280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström % (self.finished_tests, self.total_tests, test, exit_code, time_ms)) 1530fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org elif command == "TESTCNT": 1540fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org self.total_tests = int(arg.split(' ', 1)[1]) 155280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.out.transient_line("[0/%d] Running tests..." % self.total_tests) 156bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 157cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström def logfile(self, job_id, name): 158cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström self.outputs[job_id] = name 159bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 1600fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org def log(self, line): 1610fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org stdout_lock.acquire() 1620fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org (prefix, output) = line.split(' ', 1) 1630fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org 164cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström assert prefix[-1] == ':' 165cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström self.handle_meta(int(prefix[:-1]), output) 1660fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org stdout_lock.release() 1670fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org 1680fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org def end(self): 169bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org if self.failures: 170280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.out.permanent_line("FAILED TESTS (%d/%d):" 171280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström % (len(self.failures), self.total_tests)) 172bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org for (binary, test) in self.failures: 173280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström self.out.permanent_line(" " + binary + ": " + test) 174a0b9549b8802ea47516b2f76b149a95d7cbd5977pbos self.out.flush_transient_output() 175bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 176bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgclass RawFormat: 1770fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org def log(self, line): 1780fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org stdout_lock.acquire() 1790fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org sys.stdout.write(line + "\n") 1800fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org sys.stdout.flush() 1810fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org stdout_lock.release() 182cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström def logfile(self, job_id, name): 183cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström with open(self.outputs[job_id]) as f: 184cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström for line in f.readlines(): 185cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström self.log(str(job_id) + '> ' + line.rstrip()) 1860fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org def end(self): 1870fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org pass 188bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 18902c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström# Record of test runtimes. Has built-in locking. 19002c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmclass TestTimes(object): 19102c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström def __init__(self, save_file): 19202c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström "Create new object seeded with saved test times from the given file." 19302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström self.__times = {} # (test binary, test name) -> runtime in ms 19402c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 19502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström # Protects calls to record_test_time(); other calls are not 19602c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström # expected to be made concurrently. 19702c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström self.__lock = threading.Lock() 19802c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 19902c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström try: 20002c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström with gzip.GzipFile(save_file, "rb") as f: 20102c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström times = cPickle.load(f) 20202c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström except (EOFError, IOError, cPickle.UnpicklingError, zlib.error): 20302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström # File doesn't exist, isn't readable, is malformed---whatever. 20402c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström # Just ignore it. 20502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström return 20602c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 20702c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström # Discard saved times if the format isn't right. 20802c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström if type(times) is not dict: 20902c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström return 21002c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström for ((test_binary, test_name), runtime) in times.items(): 21102c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström if (type(test_binary) is not str or type(test_name) is not str 212be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos or type(runtime) not in {int, long, type(None)}): 21302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström return 21402c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 21502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström self.__times = times 21602c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 21702c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström def get_test_time(self, binary, testname): 218be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos """Return the last duration for the given test as an integer number of 219be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos milliseconds, or None if the test failed or if there's no record for it.""" 220be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos return self.__times.get((binary, testname), None) 22102c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 22202c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström def record_test_time(self, binary, testname, runtime_ms): 223be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos """Record that the given test ran in the specified number of 224be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos milliseconds. If the test failed, runtime_ms should be None.""" 22502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström with self.__lock: 22602c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström self.__times[(binary, testname)] = runtime_ms 22702c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 22802c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström def write_to_file(self, save_file): 22902c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström "Write all the times to file." 23002c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström try: 23102c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström with open(save_file, "wb") as f: 23202c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström with gzip.GzipFile("", "wb", 9, f) as gzf: 23302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström cPickle.dump(self.__times, gzf, cPickle.HIGHEST_PROTOCOL) 23402c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström except IOError: 23502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström pass # ignore errors---saving the times isn't that important 23602c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 2371777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org# Remove additional arguments (anything after --). 2381777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.orgadditional_args = [] 2391777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org 2401777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.orgfor i in range(len(sys.argv)): 2411777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org if sys.argv[i] == '--': 2421777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org additional_args = sys.argv[i+1:] 2431777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org sys.argv = sys.argv[:i] 2441777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org break 2451777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org 246bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgparser = optparse.OptionParser( 2471777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org usage = 'usage: %prog [options] binary [binary ...] -- [additional args]') 248bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 249cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boströmparser.add_option('-d', '--output_dir', type='string', 250cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström default=os.path.join(tempfile.gettempdir(), "gtest-parallel"), 251cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström help='output directory for test logs') 252bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgparser.add_option('-r', '--repeat', type='int', default=1, 253bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org help='repeat tests') 25402c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmparser.add_option('-w', '--workers', type='int', 25502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström default=multiprocessing.cpu_count(), 256bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org help='number of workers to spawn') 257bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgparser.add_option('--gtest_color', type='string', default='yes', 258bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org help='color output') 259bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgparser.add_option('--gtest_filter', type='string', default='', 260bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org help='test filter') 261bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgparser.add_option('--gtest_also_run_disabled_tests', action='store_true', 262bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org default=False, help='run disabled tests too') 263bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgparser.add_option('--format', type='string', default='filter', 264bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org help='output format (raw,filter)') 265be26c07cb5e5fd1cb86d0886726e4aba17afc834pbosparser.add_option('--print_test_times', action='store_true', default=False, 266be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos help='When done, list the run time of each test') 267bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 268bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org(options, binaries) = parser.parse_args() 269bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 270bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgif binaries == []: 271bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org parser.print_usage() 272bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org sys.exit(1) 273bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 274bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orglogger = RawFormat() 275bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgif options.format == 'raw': 276bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org pass 277bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgelif options.format == 'filter': 278bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org logger = FilterFormat() 279bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgelse: 280bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org sys.exit("Unknown output format: " + options.format) 281bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 282bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# Find tests. 28302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmsave_file = os.path.join(os.path.expanduser("~"), ".gtest-parallel-times") 28402c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmtimes = TestTimes(save_file) 285bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgtests = [] 286bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgfor test_binary in binaries: 287bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org command = [test_binary] 288bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org if options.gtest_also_run_disabled_tests: 289bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org command += ['--gtest_also_run_disabled_tests'] 290bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 2918786f637b292d3f91c771f6e08a510591ac51bdcPeter Boström list_command = list(command) 2928786f637b292d3f91c771f6e08a510591ac51bdcPeter Boström if options.gtest_filter != '': 2938786f637b292d3f91c771f6e08a510591ac51bdcPeter Boström list_command += ['--gtest_filter=' + options.gtest_filter] 2948786f637b292d3f91c771f6e08a510591ac51bdcPeter Boström 29502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström try: 29602c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström test_list = subprocess.Popen(list_command + ['--gtest_list_tests'], 29702c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström stdout=subprocess.PIPE).communicate()[0] 29802c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström except OSError as e: 29902c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström sys.exit("%s: %s" % (test_binary, str(e))) 300bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 3011777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org command += additional_args 3021777880f54acce46df7d1cf207e3a70ee3d6beacpbos@webrtc.org 303bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org test_group = '' 304bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org for line in test_list.split('\n'): 305bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org if not line.strip(): 306bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org continue 307bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org if line[0] != " ": 308bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org test_group = line.strip() 309bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org continue 3106d3f11c36604c124e0ab79c37921464e7dd7dfe7pbos # Remove comments for parameterized tests and strip whitespace. 3118786f637b292d3f91c771f6e08a510591ac51bdcPeter Boström line = line.split('#')[0].strip() 3128786f637b292d3f91c771f6e08a510591ac51bdcPeter Boström if not line: 3138786f637b292d3f91c771f6e08a510591ac51bdcPeter Boström continue 314bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 315bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org test = test_group + line 3166d3f11c36604c124e0ab79c37921464e7dd7dfe7pbos if not options.gtest_also_run_disabled_tests and 'DISABLED_' in test: 3176d3f11c36604c124e0ab79c37921464e7dd7dfe7pbos continue 31802c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström tests.append((times.get_test_time(test_binary, test), 31902c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström test_binary, test, command)) 320be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos 321be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos# Sort tests by falling runtime (with None, which is what we get for 322be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos# new and failing tests, being considered larger than any real 323be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos# runtime). 324be26c07cb5e5fd1cb86d0886726e4aba17afc834pbostests.sort(reverse=True, key=lambda x: ((1 if x[0] is None else 0), x)) 325bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 326bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org# Repeat tests (-r flag). 327bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgtests *= options.repeat 3280fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.orgtest_lock = threading.Lock() 3290fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.orgjob_id = 0 3300fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.orglogger.log(str(-1) + ': TESTCNT ' + ' ' + str(len(tests))) 331bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 332bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgexit_code = 0 33302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström 334cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström# Create directory for test log output. 335cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boströmtry: 336cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström os.makedirs(options.output_dir) 337cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boströmexcept OSError as e: 338cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström # Ignore errors if this directory already exists. 339cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström if e.errno != errno.EEXIST or not os.path.isdir(options.output_dir): 340cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström raise e 341cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström# Remove files from old test runs. 342cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boströmfor logfile in os.listdir(options.output_dir): 343cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström os.remove(os.path.join(options.output_dir, logfile)) 344cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström 345280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström# Run the specified job. Return the elapsed time in milliseconds if 346be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos# the job succeeds, or None if the job fails. (This ensures that 347be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos# failing tests will run first the next time.) 348bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgdef run_job((command, job_id, test)): 349bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org begin = time.time() 350bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 351cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström with tempfile.NamedTemporaryFile(dir=options.output_dir, delete=False) as log: 352cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström sub = subprocess.Popen(command + ['--gtest_filter=' + test] + 353cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström ['--gtest_color=' + options.gtest_color], 354cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström stdout=log.file, 355cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström stderr=log.file) 3562f29d70184b791e7c6444a48b1239d0541ee787cpbos try: 3572f29d70184b791e7c6444a48b1239d0541ee787cpbos code = sigint_handler.wait(sub) 3582f29d70184b791e7c6444a48b1239d0541ee787cpbos except sigint_handler.ProcessWasInterrupted: 3592f29d70184b791e7c6444a48b1239d0541ee787cpbos thread.exit() 360cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström runtime_ms = int(1000 * (time.time() - begin)) 361cf890bc58eb28d5f1f6ce3f90d4e541983042369Peter Boström logger.logfile(job_id, log.name) 362bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 36302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström logger.log("%s: EXIT %s %d" % (job_id, code, runtime_ms)) 364280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström if code == 0: 365280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström return runtime_ms 366280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström global exit_code 367280ed1149312a479c2186bc2f4dfabcb49f7a593Peter Boström exit_code = code 368be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos return None 369bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 370bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgdef worker(): 3710fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org global job_id 372bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org while True: 3730fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org job = None 3740fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org test_lock.acquire() 3750fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org if job_id < len(tests): 37602c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström (_, test_binary, test, command) = tests[job_id] 3770fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org logger.log(str(job_id) + ': TEST ' + test_binary + ' ' + test) 3780fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org job = (command, job_id, test) 3790fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org job_id += 1 3800fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org test_lock.release() 3810fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.org if job is None: 382bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org return 38302c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boström times.record_test_time(test_binary, test, run_job(job)) 384bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 385bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgdef start_daemon(func): 386bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org t = threading.Thread(target=func) 387bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org t.daemon = True 388bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org t.start() 389bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org return t 390bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 391bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.orgworkers = [start_daemon(worker) for i in range(options.workers)] 392bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org 393bc524ae41adb1304088c5d595381422fd0e28460pbos@webrtc.org[t.join() for t in workers] 3940fa04755afe626c94b6d17e08a8b3355e8d851d6pbos@webrtc.orglogger.end() 39502c9b3673327d7e03f8e3ea787105c63a322c29ePeter Boströmtimes.write_to_file(save_file) 396be26c07cb5e5fd1cb86d0886726e4aba17afc834pbosif options.print_test_times: 397be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos ts = sorted((times.get_test_time(test_binary, test), test_binary, test) 398be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos for (_, test_binary, test, _) in tests 399be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos if times.get_test_time(test_binary, test) is not None) 400be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos for (time_ms, test_binary, test) in ts: 401be26c07cb5e5fd1cb86d0886726e4aba17afc834pbos print "%8s %s" % ("%dms" % time_ms, test) 4022f29d70184b791e7c6444a48b1239d0541ee787cpbossys.exit(-signal.SIGINT if sigint_handler.got_sigint() else exit_code) 403