1#!/usr/bin/env python2.7
2# Copyright 2015 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Script to pull chromium.perf step timings from chrome-infra-stats API.
7
8Currently this pulls the list of steps per builder. For each step, if it is not
9a setup step, we get the step stats for the last 20 runs for that builder.
10
11The API documentation for chrome-infra-stats is at:
12https://apis-explorer.appspot.com/apis-explorer/?
13   base=https://chrome-infra-stats.appspot.com/_ah/api#p/
14"""
15
16import csv
17import datetime
18import json
19import sys
20import urllib
21import urllib2
22
23
24BUILDER_STEPS_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/'
25                     'masters/chromium.perf/%s')
26
27
28STEP_ACTIVE_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/'
29                   'steps/last/chromium.perf/%s/%s/1')
30
31
32STEP_STATS_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/'
33                  'stats/last/chromium.perf/%s/%s/20')
34
35
36IGNORED_STEPS = [
37    'List Perf Tests',
38    'Sharded Perf Tests',
39    'authorize_adb_devices',
40    'bot_update',
41    'build__schedule__time__',
42    'clean local files',
43    'cleanup_temp',
44    'device_status_check',
45    'extract build',
46    'gclient runhooks',
47    'get compile targets for scripts',
48    'get perf test list',
49    'gsutil download_build_product',
50    'host_info',
51    'install ChromeShell.apk',
52    'json.output cache',
53    'json.output cache',
54    'overall__build__result__',
55    'overall__queued__time__',
56    'provision_devices',
57    'read test spec',
58    'rmtree build directory',
59    'setup_build',
60    'spawn_logcat_monitor',
61    'stack_tool_for_tombstones',
62    'stack_tool_with_logcat_dump',
63    'steps',
64    'test_report',
65    'unzip_build_product',
66    'update_scripts'
67]
68
69KNOWN_TESTERS_LIST = [
70    'Android Nexus4 Perf',
71    'Android Nexus5 Perf',
72    'Android Nexus6 Perf',
73    'Android Nexus10 Perf',
74    'Android Nexus7v2 Perf',
75    'Android One Perf',
76    'Linux Perf (1)',
77    'Linux Perf (2)',
78    'Linux Perf (3)',
79    'Linux Perf (4)',
80    'Linux Perf (5)',
81    'Mac 10.8 Perf (1)',
82    'Mac 10.8 Perf (2)',
83    'Mac 10.8 Perf (3)',
84    'Mac 10.8 Perf (4)',
85    'Mac 10.8 Perf (5)',
86    'Mac 10.9 Perf (1)',
87    'Mac 10.9 Perf (2)',
88    'Mac 10.9 Perf (3)',
89    'Mac 10.9 Perf (4)',
90    'Mac 10.9 Perf (5)',
91    'Win 7 ATI GPU Perf',
92    'Win 7 Intel GPU Perf',
93    'Win 7 Low-End Perf (1)',
94    'Win 7 Low-End Perf (2)',
95    'Win 7 Nvidia GPU Perf',
96    'Win 7 Perf (1)',
97    'Win 7 Perf (2)',
98    'Win 7 Perf (3)',
99    'Win 7 Perf (4)',
100    'Win 7 Perf (5)',
101    'Win 7 x64 Perf (1)',
102    'Win 7 x64 Perf (2)',
103    'Win 8 Perf (1)',
104    'Win 8 Perf (2)',
105    'Win XP Perf (1)',
106    'Win XP Perf (2)',
107    'Win XP Perf (3)',
108    'Win XP Perf (4)',
109    'Win XP Perf (5)'
110]
111
112
113USAGE = 'Usage: chrome-perf-step-timings.py <outfilename>'
114
115
116def main():
117  if len(sys.argv) != 2:
118    print USAGE
119    sys.exit(0)
120  outfilename = sys.argv[1]
121
122  threshold_time = datetime.datetime.now() - datetime.timedelta(days=2)
123
124  col_names = [('builder', 'step', 'run_count', 'stddev', 'mean', 'maximum',
125                'median', 'seventyfive', 'ninety', 'ninetynine')]
126  with open(outfilename, 'wb') as f:
127    writer = csv.writer(f)
128    writer.writerows(col_names)
129
130  for builder in KNOWN_TESTERS_LIST:
131    step_timings = []
132    url = BUILDER_STEPS_URL % urllib.quote(builder)
133    response = urllib2.urlopen(url)
134    results = json.load(response)
135    steps = results['steps']
136    steps.sort()  # to group tests and their references together.
137    for step in steps:
138      if step in IGNORED_STEPS:
139        continue
140      url = STEP_ACTIVE_URL % (urllib.quote(builder), urllib.quote(step))
141      response = urllib2.urlopen(url)
142      results = json.load(response)
143      if ('step_records' not in results.keys() or
144          len(results['step_records']) == 0):
145        continue
146      first_record = results['step_records'][0]
147      last_step_time = datetime.datetime.strptime(
148          first_record['step_start'], "%Y-%m-%dT%H:%M:%S.%f")
149      # ignore steps that did not run for more than 2 days
150      if last_step_time < threshold_time:
151        continue
152      url = STEP_STATS_URL % (urllib.quote(builder), urllib.quote(step))
153      response = urllib2.urlopen(url)
154      results = json.load(response)
155      step_timings.append(
156          [builder, step, results['count'], results['stddev'],
157           results['mean'], results['maximum'], results['median'],
158           results['seventyfive'], results['ninety'],
159           results['ninetynine']])
160    with open(outfilename, 'ab') as f:
161      writer = csv.writer(f)
162      writer.writerows(step_timings)
163
164
165if __name__ == '__main__':
166  main()
167