1691925f304f07de950da5a7b45c797f38c542bb8Dan Shi#! /usr/bin/python 2691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 3691925f304f07de950da5a7b45c797f38c542bb8Dan Shi# The tool gathers the time used by special tasks/test jobs to a tab-separated 4691925f304f07de950da5a7b45c797f38c542bb8Dan Shi# output. 5691925f304f07de950da5a7b45c797f38c542bb8Dan Shi# Import the output to google spreadsheet can generate a sheet like this: 6691925f304f07de950da5a7b45c797f38c542bb8Dan Shi# https://docs.google.com/a/google.com/spreadsheets/d/ 7691925f304f07de950da5a7b45c797f38c542bb8Dan Shi# 1iLPSRAgSVz9rGVusTb2yaHJ88iv0QY3hZI_ZI-RdatI/edit#gid=51630294 8691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 9691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 10691925f304f07de950da5a7b45c797f38c542bb8Dan Shiimport os 11691925f304f07de950da5a7b45c797f38c542bb8Dan Shiimport argparse 12691925f304f07de950da5a7b45c797f38c542bb8Dan Shifrom datetime import datetime 13691925f304f07de950da5a7b45c797f38c542bb8Dan Shiimport re 14322f4d1060633bb4d463ce7880e36587d5792a8cDan Shiimport subprocess 15691925f304f07de950da5a7b45c797f38c542bb8Dan Shiimport sys 16322f4d1060633bb4d463ce7880e36587d5792a8cDan Shiimport urllib2 17691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 18691925f304f07de950da5a7b45c797f38c542bb8Dan Shiimport common 19691925f304f07de950da5a7b45c797f38c542bb8Dan Shifrom autotest_lib.client.common_lib import global_config 20691925f304f07de950da5a7b45c797f38c542bb8Dan Shifrom autotest_lib.frontend import setup_django_environment 21691925f304f07de950da5a7b45c797f38c542bb8Dan Shifrom autotest_lib.frontend.afe import models 22691925f304f07de950da5a7b45c797f38c542bb8Dan Shifrom autotest_lib.frontend.tko import models as tko_models 23691925f304f07de950da5a7b45c797f38c542bb8Dan Shifrom autotest_lib.server.cros.dynamic_suite import job_status 24691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 25691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 26691925f304f07de950da5a7b45c797f38c542bb8Dan ShiCONFIG = global_config.global_config 27691925f304f07de950da5a7b45c797f38c542bb8Dan ShiAUTOTEST_SERVER = CONFIG.get_config_value('SERVER', 'hostname', type=str) 28691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 29691925f304f07de950da5a7b45c797f38c542bb8Dan ShiLOG_BASE_URL = 'http://%s/tko/retrieve_logs.cgi?job=/results/' % AUTOTEST_SERVER 30691925f304f07de950da5a7b45c797f38c542bb8Dan ShiJOB_URL = LOG_BASE_URL + '%(job_id)s-%(owner)s/%(hostname)s' 31691925f304f07de950da5a7b45c797f38c542bb8Dan ShiLOG_PATH_FMT = 'hosts/%(hostname)s/%(task_id)d-%(taskname)s' 32691925f304f07de950da5a7b45c797f38c542bb8Dan ShiTASK_URL = LOG_BASE_URL + LOG_PATH_FMT 33691925f304f07de950da5a7b45c797f38c542bb8Dan ShiAUTOSERV_DEBUG_LOG = 'debug/autoserv.DEBUG' 34322f4d1060633bb4d463ce7880e36587d5792a8cDan ShiHYPERLINK = '=HYPERLINK("%s","%0.1f")' 35322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi 36322f4d1060633bb4d463ce7880e36587d5792a8cDan ShiGS_URI = 'gs://chromeos-autotest-results' 37691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 38691925f304f07de950da5a7b45c797f38c542bb8Dan Shi# A cache of special tasks, hostname:[task] 39691925f304f07de950da5a7b45c797f38c542bb8Dan Shitasks_cache = {} 40691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 41322f4d1060633bb4d463ce7880e36587d5792a8cDan Shidef get_debug_log(autoserv_log_url, autoserv_log_path): 42322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi """Get a list of strings or a stream for autoserv.DEBUG log file. 43322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi 44322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi @param autoserv_log_url: Url to the repair job's autoserv.DEBUG log. 45322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi @param autoserv_log_path: Path to autoserv.DEBUG log, e.g., 46322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi hosts/hostname/job_id-repair/debug/autoserv.DEBUG. 47322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi @return: A list of string if debug log was moved to GS already, or a 48322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi stream of the autoserv.DEBUG file. 49322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi """ 50322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi url = urllib2.urlopen(autoserv_log_url).geturl() 51322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi if not 'accounts.google.com' in url: 52322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi return urllib2.urlopen(url) 53322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi else: 54322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi # The file was moved to Google storage already, read the file from GS. 55322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi debug_log_link = '%s/%s' % (GS_URI, autoserv_log_path) 56322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi cmd = 'gsutil cat %s' % debug_log_link 57322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 58322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi stderr=subprocess.PIPE) 59322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi stdout, stderr = proc.communicate() 60322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi if proc.returncode == 0: 61322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi return stdout.split(os.linesep) 62322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi else: 63322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi print 'Failed to read %s: %s' % (debug_log_link, stderr) 64322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi 65322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi 66691925f304f07de950da5a7b45c797f38c542bb8Dan Shiclass task_runtime(object): 67691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """Details about the task run time. 68691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 69691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def __init__(self, task): 70691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """Init task_runtime 71691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 72691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Special task ID 73691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.id = task.id 74691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # AFE special_task model 75691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.task = task 76691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.hostname = task.host.hostname 77691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 78691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Link to log 79691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_info = {'task_id': task.id, 'taskname': task.task.lower(), 80691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 'hostname': self.hostname} 81691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.log = TASK_URL % task_info 82691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 83691925f304f07de950da5a7b45c797f38c542bb8Dan Shi autoserv_log_url = '%s/%s' % (self.log, AUTOSERV_DEBUG_LOG) 84691925f304f07de950da5a7b45c797f38c542bb8Dan Shi log_path = LOG_PATH_FMT % task_info 85691925f304f07de950da5a7b45c797f38c542bb8Dan Shi autoserv_log_path = '%s/%s' % (log_path, AUTOSERV_DEBUG_LOG) 86322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi debug_log = get_debug_log(autoserv_log_url, autoserv_log_path) 87691925f304f07de950da5a7b45c797f38c542bb8Dan Shi lines = [line for line in debug_log if line] 88691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Task start time 89691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.start_time = _get_timestamp(lines[0]) 90691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Task end time 91691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.end_time = _get_timestamp(lines[-1]) 92691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Run time across end of a year. 93691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if self.end_time < self.start_time: 94691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.end_time = self.end_time.replace(year=self.end_time.year+1) 95691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.time_used = self.end_time - self.start_time 96691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 97691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Start/end time from afe_special_tasks table. 98691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # To support old special tasks, the value falls back to the ones from 99691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # debug log. 100691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.start_time_db = (self.task.time_started if self.task.time_started 101691925f304f07de950da5a7b45c797f38c542bb8Dan Shi else self.start_time) 102691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.end_time_db = (self.task.time_finished if self.task.time_finished 103691925f304f07de950da5a7b45c797f38c542bb8Dan Shi else self.end_time) 104691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.time_used_db = self.end_time_db - self.start_time_db 105691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 106691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 107691925f304f07de950da5a7b45c797f38c542bb8Dan Shiclass job_runtime(object): 108691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """Details about the job run time, including time spent on special tasks. 109691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 110691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def __init__(self, job, suite_start_time=None, suite_end_time=None): 111691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """Init job_run_time 112691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 113691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # AFE job ID 114691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.id = job.id 115691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # AFE job model 116691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.job = job 117691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Name of the job, strip all build and suite info. 118691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.name = job.name.split('/')[-1] 119691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 120691925f304f07de950da5a7b45c797f38c542bb8Dan Shi try: 121691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.tko_job = tko_models.Job.objects.filter(afe_job_id=self.id)[0] 122691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.host_id = self.tko_job.machine_id 123691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.hostname = self.tko_job.machine.hostname 124691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Job start time 125691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.start_time = self.tko_job.started_time 126691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Job end time 127691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.end_time = self.tko_job.finished_time 128691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.time_used = self.end_time - self.start_time 129691925f304f07de950da5a7b45c797f38c542bb8Dan Shi except IndexError: 130691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.tko_job = None 131691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.host_id = 0 132691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.time_used = 0 133691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 134691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Link to log 135691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.log = JOB_URL % {'job_id': job.id, 'owner': job.owner, 136691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 'hostname': self.hostname} 137691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 138691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Special tasks run before job starts. 139691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.tasks_before = [] 140691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Special tasks run after job finished. 141691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.tasks_after = [] 142691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 143691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # If job time used is not calculated, skip locating special tasks. 144691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if not self.time_used: 145691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return 146691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 147691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # suite job has no special tasks 148691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if not self.hostname: 149691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return 150691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 151691925f304f07de950da5a7b45c797f38c542bb8Dan Shi tasks = tasks_cache.get(self.hostname, None) 152691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if not tasks: 153691925f304f07de950da5a7b45c797f38c542bb8Dan Shi tasks = models.SpecialTask.objects.filter( 154691925f304f07de950da5a7b45c797f38c542bb8Dan Shi host__hostname=self.hostname, 155691925f304f07de950da5a7b45c797f38c542bb8Dan Shi time_started__gte=suite_start_time, 156691925f304f07de950da5a7b45c797f38c542bb8Dan Shi time_started__lte=suite_end_time) 157691925f304f07de950da5a7b45c797f38c542bb8Dan Shi tasks_cache[self.hostname] = tasks 158691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for task in tasks: 159691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if not task.queue_entry or task.queue_entry.job_id != self.id: 160691925f304f07de950da5a7b45c797f38c542bb8Dan Shi continue 161691925f304f07de950da5a7b45c797f38c542bb8Dan Shi t_runtime = task_runtime(task) 162691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.time_started < self.start_time: 163691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.tasks_before.append(t_runtime) 164691925f304f07de950da5a7b45c797f38c542bb8Dan Shi else: 165691925f304f07de950da5a7b45c797f38c542bb8Dan Shi self.tasks_after.append(t_runtime) 166691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 167691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 168691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_all_tasks(self): 169691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return self.tasks_before + self.tasks_after 170691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 171691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 172691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_first_task_start_time(self): 173691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """Get the start time of first task, return test job start time if 174691925f304f07de950da5a7b45c797f38c542bb8Dan Shi there is no tasks_before 175691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 176691925f304f07de950da5a7b45c797f38c542bb8Dan Shi start = self.start_time 177691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for task in self.tasks_before: 178691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.start_time_db < start: 179691925f304f07de950da5a7b45c797f38c542bb8Dan Shi start = task.start_time_db 180691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return start 181691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 182691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 183691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_last_task_end_time(self): 184691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """Get the end time of last task, return test job end time if there is 185691925f304f07de950da5a7b45c797f38c542bb8Dan Shi no tasks_after. 186691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 187691925f304f07de950da5a7b45c797f38c542bb8Dan Shi end = self.end_time 188691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for task in self.tasks_after: 189691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if end < task.end_time_db: 190691925f304f07de950da5a7b45c797f38c542bb8Dan Shi end = task.end_time_db 191691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return end 192691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 193691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 194691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_total_time(self): 195322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi return (self.get_last_task_end_time() - 196322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi self.get_first_task_start_time()).total_seconds() 197691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 198691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 199691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_time_on_tasks(self): 200322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi time = 0 201322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi for task in self.tasks_before + self.tasks_after: 202322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi time += task.time_used.total_seconds() 203322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi return time 204691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 205691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 206691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_time_on_tasks_from_db(self): 207322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi time = 0 208322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi for task in self.tasks_before + self.tasks_after: 209322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi time += task.time_used_db.total_seconds() 210322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi return time 211691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 212691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 213691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_time_on_wait(self): 214322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi return (self.get_total_time() - 215322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi self.get_time_on_tasks() - 216322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi self.time_used.total_seconds()) 217691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 218691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 219691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_time_on_wait_from_db(self): 220322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi return (self.get_total_time() - 221322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi self.get_time_on_tasks_from_db() - 222322f4d1060633bb4d463ce7880e36587d5792a8cDan Shi self.time_used.total_seconds()) 223691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 224691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 225691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_time_per_task_type(self, task_type): 226691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """only used for suite job 227691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 228691925f304f07de950da5a7b45c797f38c542bb8Dan Shi tasks = [] 229691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for job in self.test_jobs: 230691925f304f07de950da5a7b45c797f38c542bb8Dan Shi tasks += [task for task in job.get_all_tasks() 231691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.task.task == task_type] 232691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if not tasks: 233691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return None 234691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 235691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_min = tasks[0] 236691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_max = tasks[0] 237691925f304f07de950da5a7b45c797f38c542bb8Dan Shi total = 0 238691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for task in tasks: 239691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.time_used < task_min.time_used: 240691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_min = task 241691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.time_used > task_max.time_used: 242691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_max = task 243691925f304f07de950da5a7b45c797f38c542bb8Dan Shi total += task.time_used.total_seconds() 244691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return (task_min, task_max, total/len(tasks), len(tasks)) 245691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 246691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 247691925f304f07de950da5a7b45c797f38c542bb8Dan Shi def get_time_per_task_type_from_db(self, task_type): 248691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """only used for suite job 249691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 250691925f304f07de950da5a7b45c797f38c542bb8Dan Shi tasks = [] 251691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for job in self.test_jobs: 252691925f304f07de950da5a7b45c797f38c542bb8Dan Shi tasks += [task for task in job.get_all_tasks() 253691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.task.task == task_type] 254691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if not tasks: 255691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return None 256691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 257691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_min = tasks[0] 258691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_max = tasks[0] 259691925f304f07de950da5a7b45c797f38c542bb8Dan Shi total = 0 260691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for task in tasks: 261691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.time_used_db < task_min.time_used_db: 262691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_min = task 263691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if task.time_used_db > task_max.time_used_db: 264691925f304f07de950da5a7b45c797f38c542bb8Dan Shi task_max = task 265691925f304f07de950da5a7b45c797f38c542bb8Dan Shi total += task.time_used_db.total_seconds() 266691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return (task_min, task_max, total/len(tasks), len(tasks)) 267691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 268691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 269691925f304f07de950da5a7b45c797f38c542bb8Dan Shidef _get_timestamp(line): 270691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """Get the time from the beginning of a log entry. 271691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 272691925f304f07de950da5a7b45c797f38c542bb8Dan Shi @param line: A line of log entry, e.g., "06/19 19:56:53.602 INFO |" 273691925f304f07de950da5a7b45c797f38c542bb8Dan Shi @return: A datetime value of the log entry. 274691925f304f07de950da5a7b45c797f38c542bb8Dan Shi """ 275691925f304f07de950da5a7b45c797f38c542bb8Dan Shi try: 276691925f304f07de950da5a7b45c797f38c542bb8Dan Shi time = re.match( '^(\d\d\/\d\d \d\d:\d\d:\d\d.\d+).*', line).group(1) 277691925f304f07de950da5a7b45c797f38c542bb8Dan Shi time = '%d/%s' % (datetime.now().year, time) 278691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return datetime.strptime(time, '%Y/%m/%d %H:%M:%S.%f') 279691925f304f07de950da5a7b45c797f38c542bb8Dan Shi except: 280691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return None 281691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 282691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 283691925f304f07de950da5a7b45c797f38c542bb8Dan Shidef _parse_args(args): 284691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if not args: 285691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print 'Try ./contrib/compare_suite.py --jobs 51,52,53' 286691925f304f07de950da5a7b45c797f38c542bb8Dan Shi sys.exit(0) 287691925f304f07de950da5a7b45c797f38c542bb8Dan Shi parser = argparse.ArgumentParser( 288691925f304f07de950da5a7b45c797f38c542bb8Dan Shi description=('A script to compare the performance of multiple suite' 289691925f304f07de950da5a7b45c797f38c542bb8Dan Shi ' runs.')) 290691925f304f07de950da5a7b45c797f38c542bb8Dan Shi parser.add_argument('--jobs', 291691925f304f07de950da5a7b45c797f38c542bb8Dan Shi help='A list of job IDs.') 292691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return parser.parse_args(args) 293691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 294691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 295691925f304f07de950da5a7b45c797f38c542bb8Dan Shidef merge_time_dict(time_dict_list): 296691925f304f07de950da5a7b45c797f38c542bb8Dan Shi merged = {} 297691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for time_dict in time_dict_list: 298691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for key,val in time_dict.items(): 299691925f304f07de950da5a7b45c797f38c542bb8Dan Shi time_used = merged.get(key, 0) 300691925f304f07de950da5a7b45c797f38c542bb8Dan Shi merged[key] = time_used + val 301691925f304f07de950da5a7b45c797f38c542bb8Dan Shi return merged 302691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 303691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 304691925f304f07de950da5a7b45c797f38c542bb8Dan Shidef print_task_data(all_jobs, time_data): 305691925f304f07de950da5a7b45c797f38c542bb8Dan Shi percent_string = delimiter.join( 306691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [str(100.0*data[2]*data[3]/suite_job.total_time) 307691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if data else '_' for (data, suite_job) in 308691925f304f07de950da5a7b45c797f38c542bb8Dan Shi zip(time_data, all_jobs.keys())]) 309691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print '%% on %s %s%s' % (task_type, delimiter, percent_string) 310691925f304f07de950da5a7b45c797f38c542bb8Dan Shi min_string = delimiter.join( 311691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [(HYPERLINK % (data[0].log, data[0].time_used.total_seconds())) 312691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if data else '_' for data in time_data]) 313691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ' %s min (seconds)%s%s' % (task_type, delimiter, min_string) 314691925f304f07de950da5a7b45c797f38c542bb8Dan Shi max_string = delimiter.join( 315691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [HYPERLINK % (data[1].log, data[1].time_used.total_seconds()) 316691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if data else '_' for data in time_data]) 317691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ' %s max (seconds)%s%s' % (task_type, delimiter, max_string) 318691925f304f07de950da5a7b45c797f38c542bb8Dan Shi ave_string = delimiter.join( 319691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [str(data[2]) if data else '_' for data in time_data]) 320691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ' %s average (seconds)%s%s' % (task_type, delimiter, ave_string) 321691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 322691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 323691925f304f07de950da5a7b45c797f38c542bb8Dan Shiif __name__ == '__main__': 324691925f304f07de950da5a7b45c797f38c542bb8Dan Shi args = _parse_args(sys.argv[1:]) 325691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print 'Comparing jobs: %s' % args.jobs 326691925f304f07de950da5a7b45c797f38c542bb8Dan Shi job_ids = [int(id) for id in args.jobs.split(',')] 327691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 328691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Make sure all jobs are suite jobs 329691925f304f07de950da5a7b45c797f38c542bb8Dan Shi parent_jobs = [job.parent_job_id for job in 330691925f304f07de950da5a7b45c797f38c542bb8Dan Shi models.Job.objects.filter(id__in=job_ids)] 331691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if any(parent_jobs): 332691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('Some jobs are not suite job. Please provide a list of suite job' 333691925f304f07de950da5a7b45c797f38c542bb8Dan Shi ' IDs.') 334691925f304f07de950da5a7b45c797f38c542bb8Dan Shi sys.exit(1) 335691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 336691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # A dictionary of suite_job_runtime:{test_job_name:test_job_runtime} 337691925f304f07de950da5a7b45c797f38c542bb8Dan Shi all_jobs = {} 338691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for job_id in job_ids: 339691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job = models.Job.objects.filter(id=job_id)[0] 340691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime = job_runtime(suite_job) 341691925f304f07de950da5a7b45c797f38c542bb8Dan Shi test_jobs = models.Job.objects.filter(parent_job_id=job_id) 342691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if len(test_jobs) == 0: 343691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print 'No child job found for suite job: %d' % job_id 344691925f304f07de950da5a7b45c797f38c542bb8Dan Shi sys.exit(1) 345691925f304f07de950da5a7b45c797f38c542bb8Dan Shi test_job_runtimes = [job_runtime(job, suite_job_runtime.start_time, 346691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.end_time) 347691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for job in test_jobs] 348691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.test_jobs = test_job_runtimes 349691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.hosts = set([job.host_id 350691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for job in test_job_runtimes 351691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if job.host_id != 0]) 352691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time = sum( 353691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [job.get_total_time() for job in test_job_runtimes]) 354691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_tasks = sum( 355691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [job.get_time_on_tasks() for job in test_job_runtimes]) 356691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_tasks_from_db = sum( 357691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [job.get_time_on_tasks_from_db() for job in test_job_runtimes]) 358691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_wait = sum( 359691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [job.get_time_on_wait() for job in test_job_runtimes]) 360691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_wait_from_db = sum( 361691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [job.get_time_on_wait_from_db() for job in test_job_runtimes]) 362691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.percent_on_tasks = 100*( 363691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_tasks / 364691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time) 365691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.percent_on_wait = 100*( 366691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_wait / 367691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time) 368691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.percent_on_tasks_from_db = 100*( 369691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_tasks_from_db / 370691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time) 371691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.percent_on_wait_from_db = 100*( 372691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time_on_wait_from_db / 373691925f304f07de950da5a7b45c797f38c542bb8Dan Shi suite_job_runtime.total_time) 374691925f304f07de950da5a7b45c797f38c542bb8Dan Shi all_jobs[suite_job_runtime] = {r.name:r for r in test_job_runtimes} 375691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 376691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter = '\t' 377691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print a row of suite job IDs. 378691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('job ID%s' % delimiter + 379691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([str(suite_job.id) 380691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 381691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 382691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print a row of platforms, e.g., lumpy-release. 383691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('platform%s' % delimiter + 384691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([suite_job.job.name.split('/')[0] 385691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 386691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 387691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print time to run suite 388691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('time(mins)%s' % delimiter + 389691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([str(suite_job.time_used.total_seconds()/60) 390691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 391691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 392691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print percent of time on tasks 393691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('%% on special tasks%s' % delimiter + 394691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([str(suite_job.percent_on_tasks) 395691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 396691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 397691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print percent of time on tasks based on time data from afe_special_tasks 398691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('%% on special tasks with data from DB%s' % delimiter + 399691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([str(suite_job.percent_on_tasks_from_db) 400691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 401691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 402691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print percent of time on tasks per task type 403691925f304f07de950da5a7b45c797f38c542bb8Dan Shi all_task_types = set() 404691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys(): 405691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for job in suite_job.test_jobs: 406691925f304f07de950da5a7b45c797f38c542bb8Dan Shi all_task_types.update( 407691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [task.task.task for task in job.get_all_tasks()]) 408691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for task_type in all_task_types: 409691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print 'Time data retrieved from debug log.' 410691925f304f07de950da5a7b45c797f38c542bb8Dan Shi time_data = [suite_job.get_time_per_task_type(task_type) 411691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()] 412691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print_task_data(all_jobs, time_data) 413691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 414691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print 'Time data retrieved from database' 415691925f304f07de950da5a7b45c797f38c542bb8Dan Shi time_data = [suite_job.get_time_per_task_type_from_db(task_type) 416691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()] 417691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print_task_data(all_jobs, time_data) 418691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 419691925f304f07de950da5a7b45c797f38c542bb8Dan Shi count_string = delimiter.join( 420691925f304f07de950da5a7b45c797f38c542bb8Dan Shi [str(data[3]) if data else '_' for data in time_data]) 421691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ' %s count%s%s' % (task_type, delimiter, count_string) 422691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 423691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print percent of time on wait 424691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('%% on wait%s' % delimiter + 425691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([str(suite_job.percent_on_wait) 426691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 427691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 428691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print percent of time on wait based on special task time data from 429691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # database 430691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('%% on wait based on time data from db%s' % delimiter + 431691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([str(suite_job.percent_on_wait_from_db) 432691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 433691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 434691925f304f07de950da5a7b45c797f38c542bb8Dan Shi # Print the number of duts used for suite. 435691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print ('dut #%s' % delimiter + 436691925f304f07de950da5a7b45c797f38c542bb8Dan Shi delimiter.join([str(len(suite_job.hosts)) 437691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys()])) 438691925f304f07de950da5a7b45c797f38c542bb8Dan Shi 439691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for test_name in [job for job in all_jobs.values()[0].keys()]: 440691925f304f07de950da5a7b45c797f38c542bb8Dan Shi line = '%s%s' % (test_name, delimiter) 441691925f304f07de950da5a7b45c797f38c542bb8Dan Shi for suite_job in all_jobs.keys(): 442691925f304f07de950da5a7b45c797f38c542bb8Dan Shi test_job = all_jobs[suite_job].get(test_name) 443691925f304f07de950da5a7b45c797f38c542bb8Dan Shi if test_job: 444691925f304f07de950da5a7b45c797f38c542bb8Dan Shi line += (HYPERLINK % 445691925f304f07de950da5a7b45c797f38c542bb8Dan Shi (test_job.log, test_job.time_used.total_seconds()) 446691925f304f07de950da5a7b45c797f38c542bb8Dan Shi + delimiter) 447691925f304f07de950da5a7b45c797f38c542bb8Dan Shi else: 448691925f304f07de950da5a7b45c797f38c542bb8Dan Shi line += '_%s' % delimiter 449691925f304f07de950da5a7b45c797f38c542bb8Dan Shi print line 450