1#!/usr/bin/env python 2 3# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7# This script is used to compare the performance of duts when running the same 8# test/special task. For example: 9# 10# python compare_dut_perf.py -l 240 --board stumpy 11# 12# compares the test runtime of all stumpy for the last 10 days. Sample output: 13# ============================================================================== 14# Test hardware_MemoryTotalSize 15# ============================================================================== 16# chromeos2-row2-rack8-host8 : min= 479, max= 479, mean= 479, med= 479, cnt= 1 17# chromeos2-row2-rack8-host12 : min= 440, max= 440, mean= 440, med= 440, cnt= 1 18# chromeos2-row2-rack8-host11 : min= 504, max= 504, mean= 504, med= 504, cnt= 1 19# 20# At the end of each row, it also lists the last 5 jobs running in the dut. 21 22 23import argparse 24import datetime 25import multiprocessing.pool 26import pprint 27import time 28from itertools import groupby 29 30import common 31import numpy 32from autotest_lib.frontend import setup_django_environment 33from autotest_lib.frontend.afe import models 34from autotest_lib.frontend.afe import rpc_utils 35from autotest_lib.frontend.tko import models as tko_models 36from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 37 38 39def get_matched_duts(hostnames=None, board=None, pool=None, other_labels=None): 40 """Get duts with matching board and pool labels from given autotest instance 41 42 @param hostnames: A list of hostnames. 43 @param board: board of DUT, set to None if board doesn't need to match. 44 Default is None. 45 @param pool: pool of DUT, set to None if pool doesn't need to match. Default 46 is None. 47 @param other_labels: Other labels to filter duts. 48 @return: A list of duts that match the specified board and pool. 49 """ 50 if hostnames: 51 hosts = models.Host.objects.filter(hostname__in=hostnames) 52 else: 53 multiple_labels = () 54 if pool: 55 multiple_labels += ('pool:%s' % pool,) 56 if board: 57 multiple_labels += ('board:%s' % board,) 58 if other_labels: 59 for label in other_labels: 60 multiple_labels += (label,) 61 hosts = rpc_utils.get_host_query(multiple_labels, 62 exclude_only_if_needed_labels=False, 63 exclude_atomic_group_hosts=False, 64 valid_only=True, filter_data={}) 65 return [host_obj.get_object_dict() for host_obj in hosts] 66 67 68def get_job_runtime(input): 69 """Get all test jobs and special tasks' runtime for a given host during 70 a give time period. 71 72 @param input: input arguments, including: 73 start_time: Start time of the search interval. 74 end_time: End time of the search interval. 75 host_id: id of the dut. 76 hostname: Name of the dut. 77 @return: A list of records, e.g., 78 [{'job_name':'dummy_Pass', 'time_used': 3, 'id': 12313, 79 'hostname': '1.2.3.4'}, 80 {'task_name':'Cleanup', 'time_used': 30, 'id': 5687, 81 'hostname': '1.2.3.4'}] 82 """ 83 start_time = input['start_time'] 84 end_time = input['end_time'] 85 host_id = input['host_id'] 86 hostname = input['hostname'] 87 records = [] 88 special_tasks = models.SpecialTask.objects.filter( 89 host_id=host_id, 90 time_started__gte=start_time, 91 time_started__lte=end_time, 92 time_started__isnull=False, 93 time_finished__isnull=False).values('task', 'id', 'time_started', 94 'time_finished') 95 for task in special_tasks: 96 time_used = task['time_finished'] - task['time_started'] 97 records.append({'name': task['task'], 98 'id': task['id'], 99 'time_used': time_used.total_seconds(), 100 'hostname': hostname}) 101 hqes = models.HostQueueEntry.objects.filter( 102 host_id=host_id, 103 started_on__gte=start_time, 104 started_on__lte=end_time, 105 started_on__isnull=False, 106 finished_on__isnull=False) 107 for hqe in hqes: 108 time_used = (hqe.finished_on - hqe.started_on).total_seconds() 109 records.append({'name': hqe.job.name.split('/')[-1], 110 'id': hqe.job.id, 111 'time_used': time_used, 112 'hostname': hostname}) 113 return records 114 115def get_job_stats(jobs): 116 """Get the stats of a list of jobs. 117 118 @param jobs: A list of jobs. 119 @return: Stats of the jobs' runtime, including: 120 t_min: minimum runtime. 121 t_max: maximum runtime. 122 t_average: average runtime. 123 t_median: median runtime. 124 """ 125 runtimes = [job['time_used'] for job in jobs] 126 t_min = min(runtimes) 127 t_max = max(runtimes) 128 t_mean = numpy.mean(runtimes) 129 t_median = numpy.median(runtimes) 130 return t_min, t_max, t_mean, t_median, len(runtimes) 131 132 133def process_results(results): 134 """Compare the results. 135 136 @param results: A list of a list of job/task information. 137 """ 138 # Merge list of all results. 139 all_results = [] 140 for result in results: 141 all_results.extend(result) 142 all_results = sorted(all_results, key=lambda r: r['name']) 143 for name,jobs_for_test in groupby(all_results, lambda r: r['name']): 144 print '='*80 145 print 'Test %s' % name 146 print '='*80 147 for hostname,jobs_for_dut in groupby(jobs_for_test, 148 lambda j: j['hostname']): 149 jobs = list(jobs_for_dut) 150 t_min, t_max, t_mean, t_median, count = get_job_stats(jobs) 151 ids = [str(job['id']) for job in jobs] 152 print ('%-28s: min= %-3.0f max= %-3.0f mean= %-3.0f med= %-3.0f ' 153 'cnt= %-3s IDs: %s' % 154 (hostname, t_min, t_max, t_mean, t_median, count, 155 ','.join(sorted(ids)[-5:]))) 156 157 158def main(): 159 """main script. """ 160 t_now = time.time() 161 t_now_minus_one_day = t_now - 3600 * 24 162 parser = argparse.ArgumentParser() 163 parser.add_argument('-l', type=float, dest='last', 164 help='last hours to search results across', 165 default=24) 166 parser.add_argument('--board', type=str, dest='board', 167 help='restrict query by board', 168 default=None) 169 parser.add_argument('--pool', type=str, dest='pool', 170 help='restrict query by pool', 171 default=None) 172 parser.add_argument('--hosts', nargs='+', dest='hosts', 173 help='Enter space deliminated hostnames', 174 default=[]) 175 parser.add_argument('--start', type=str, dest='start', 176 help=('Enter start time as: yyyy-mm-dd hh-mm-ss,' 177 'defualts to 24h ago.')) 178 parser.add_argument('--end', type=str, dest='end', 179 help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,' 180 'defualts to current time.')) 181 options = parser.parse_args() 182 183 if not options.start or not options.end: 184 end_time = datetime.datetime.now() 185 start_time = end_time - datetime.timedelta(seconds=3600 * options.last) 186 else: 187 start_time = time_utils.time_string_to_datetime(options.start) 188 end_time = time_utils.time_string_to_datetime(options.end) 189 190 hosts = get_matched_duts(hostnames=options.hosts, board=options.board, 191 pool=options.pool) 192 if not hosts: 193 raise Exception('No host found to search for history.') 194 print 'Found %d duts.' % len(hosts) 195 print 'Start time: %s' % start_time 196 print 'End time: %s' % end_time 197 args = [] 198 for host in hosts: 199 args.append({'start_time': start_time, 200 'end_time': end_time, 201 'host_id': host['id'], 202 'hostname': host['hostname']}) 203 get_job_runtime(args[0]) 204 # Parallizing this process. 205 pool = multiprocessing.pool.ThreadPool() 206 results = pool.imap_unordered(get_job_runtime, args) 207 process_results(results) 208 209 210if __name__ == '__main__': 211 main() 212