1#!/usr/bin/env python
2
3# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# This script is used to compare the performance of duts when running the same
8# test/special task. For example:
9#
10# python compare_dut_perf.py -l 240 --board stumpy
11#
12# compares the test runtime of all stumpy for the last 10 days. Sample output:
13# ==============================================================================
14# Test hardware_MemoryTotalSize
15# ==============================================================================
16# chromeos2-row2-rack8-host8  : min= 479, max= 479, mean= 479, med= 479, cnt= 1
17# chromeos2-row2-rack8-host12 : min= 440, max= 440, mean= 440, med= 440, cnt= 1
18# chromeos2-row2-rack8-host11 : min= 504, max= 504, mean= 504, med= 504, cnt= 1
19#
20# At the end of each row, it also lists the last 5 jobs running in the dut.
21
22
23import argparse
24import datetime
25import multiprocessing.pool
26import pprint
27import time
28from itertools import groupby
29
30import common
31import numpy
32from autotest_lib.frontend import setup_django_environment
33from autotest_lib.frontend.afe import models
34from autotest_lib.frontend.afe import rpc_utils
35from autotest_lib.frontend.tko import models as tko_models
36from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
37
38
39def get_matched_duts(hostnames=None, board=None, pool=None, other_labels=None):
40    """Get duts with matching board and pool labels from given autotest instance
41
42    @param hostnames: A list of hostnames.
43    @param board: board of DUT, set to None if board doesn't need to match.
44                  Default is None.
45    @param pool: pool of DUT, set to None if pool doesn't need to match. Default
46                 is None.
47    @param other_labels: Other labels to filter duts.
48    @return: A list of duts that match the specified board and pool.
49    """
50    if hostnames:
51        hosts = models.Host.objects.filter(hostname__in=hostnames)
52    else:
53        multiple_labels = ()
54        if pool:
55            multiple_labels += ('pool:%s' % pool,)
56        if board:
57            multiple_labels += ('board:%s' % board,)
58        if other_labels:
59            for label in other_labels:
60                multiple_labels += (label,)
61        hosts = rpc_utils.get_host_query(multiple_labels,
62                                         exclude_only_if_needed_labels=False,
63                                         exclude_atomic_group_hosts=False,
64                                         valid_only=True, filter_data={})
65    return [host_obj.get_object_dict() for host_obj in hosts]
66
67
68def get_job_runtime(input):
69    """Get all test jobs and special tasks' runtime for a given host during
70    a give time period.
71
72    @param input: input arguments, including:
73                  start_time: Start time of the search interval.
74                  end_time: End time of the search interval.
75                  host_id: id of the dut.
76                  hostname: Name of the dut.
77    @return: A list of records, e.g.,
78                     [{'job_name':'dummy_Pass', 'time_used': 3, 'id': 12313,
79                       'hostname': '1.2.3.4'},
80                      {'task_name':'Cleanup', 'time_used': 30, 'id': 5687,
81                       'hostname': '1.2.3.4'}]
82    """
83    start_time = input['start_time']
84    end_time = input['end_time']
85    host_id = input['host_id']
86    hostname = input['hostname']
87    records = []
88    special_tasks = models.SpecialTask.objects.filter(
89            host_id=host_id,
90            time_started__gte=start_time,
91            time_started__lte=end_time,
92            time_started__isnull=False,
93            time_finished__isnull=False).values('task', 'id', 'time_started',
94                                                'time_finished')
95    for task in special_tasks:
96        time_used = task['time_finished'] - task['time_started']
97        records.append({'name': task['task'],
98                        'id': task['id'],
99                        'time_used': time_used.total_seconds(),
100                        'hostname': hostname})
101    hqes = models.HostQueueEntry.objects.filter(
102            host_id=host_id,
103            started_on__gte=start_time,
104            started_on__lte=end_time,
105            started_on__isnull=False,
106            finished_on__isnull=False)
107    for hqe in hqes:
108        time_used = (hqe.finished_on - hqe.started_on).total_seconds()
109        records.append({'name': hqe.job.name.split('/')[-1],
110                        'id': hqe.job.id,
111                        'time_used': time_used,
112                        'hostname': hostname})
113    return records
114
115def get_job_stats(jobs):
116    """Get the stats of a list of jobs.
117
118    @param jobs: A list of jobs.
119    @return: Stats of the jobs' runtime, including:
120             t_min: minimum runtime.
121             t_max: maximum runtime.
122             t_average: average runtime.
123             t_median: median runtime.
124    """
125    runtimes = [job['time_used'] for job in jobs]
126    t_min = min(runtimes)
127    t_max = max(runtimes)
128    t_mean = numpy.mean(runtimes)
129    t_median = numpy.median(runtimes)
130    return t_min, t_max, t_mean, t_median, len(runtimes)
131
132
133def process_results(results):
134    """Compare the results.
135
136    @param results: A list of a list of job/task information.
137    """
138    # Merge list of all results.
139    all_results = []
140    for result in results:
141        all_results.extend(result)
142    all_results = sorted(all_results, key=lambda r: r['name'])
143    for name,jobs_for_test in groupby(all_results, lambda r: r['name']):
144        print '='*80
145        print 'Test %s' % name
146        print '='*80
147        for hostname,jobs_for_dut in groupby(jobs_for_test,
148                                             lambda j: j['hostname']):
149            jobs = list(jobs_for_dut)
150            t_min, t_max, t_mean, t_median, count = get_job_stats(jobs)
151            ids = [str(job['id']) for job in jobs]
152            print ('%-28s: min= %-3.0f max= %-3.0f mean= %-3.0f med= %-3.0f '
153                   'cnt= %-3s IDs: %s' %
154                   (hostname, t_min, t_max, t_mean, t_median, count,
155                    ','.join(sorted(ids)[-5:])))
156
157
158def main():
159    """main script. """
160    t_now = time.time()
161    t_now_minus_one_day = t_now - 3600 * 24
162    parser = argparse.ArgumentParser()
163    parser.add_argument('-l', type=float, dest='last',
164                        help='last hours to search results across',
165                        default=24)
166    parser.add_argument('--board', type=str, dest='board',
167                        help='restrict query by board',
168                        default=None)
169    parser.add_argument('--pool', type=str, dest='pool',
170                        help='restrict query by pool',
171                        default=None)
172    parser.add_argument('--hosts', nargs='+', dest='hosts',
173                        help='Enter space deliminated hostnames',
174                        default=[])
175    parser.add_argument('--start', type=str, dest='start',
176                        help=('Enter start time as: yyyy-mm-dd hh-mm-ss,'
177                              'defualts to 24h ago.'))
178    parser.add_argument('--end', type=str, dest='end',
179                        help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,'
180                              'defualts to current time.'))
181    options = parser.parse_args()
182
183    if not options.start or not options.end:
184        end_time = datetime.datetime.now()
185        start_time = end_time - datetime.timedelta(seconds=3600 * options.last)
186    else:
187        start_time = time_utils.time_string_to_datetime(options.start)
188        end_time = time_utils.time_string_to_datetime(options.end)
189
190    hosts = get_matched_duts(hostnames=options.hosts, board=options.board,
191                             pool=options.pool)
192    if not hosts:
193        raise Exception('No host found to search for history.')
194    print 'Found %d duts.' % len(hosts)
195    print 'Start time: %s' % start_time
196    print 'End time:   %s' % end_time
197    args = []
198    for host in hosts:
199        args.append({'start_time': start_time,
200                     'end_time': end_time,
201                     'host_id': host['id'],
202                     'hostname': host['hostname']})
203    get_job_runtime(args[0])
204    # Parallizing this process.
205    pool = multiprocessing.pool.ThreadPool()
206    results = pool.imap_unordered(get_job_runtime, args)
207    process_results(results)
208
209
210if __name__ == '__main__':
211    main()
212