15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#!/usr/bin/env python 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# found in the LICENSE file. 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# For instructions see: 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# http://www.chromium.org/developers/tree-sheriffs/perf-sheriffs 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import hashlib 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import math 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import optparse 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import os 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import re 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import subprocess 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sys 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import time 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import urllib2 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)try: 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) import json 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)except ImportError: 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) import simplejson as json 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__version__ = '1.0' 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)EXPECTATIONS_DIR = os.path.dirname(os.path.abspath(__file__)) 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFAULT_CONFIG_FILE = os.path.join(EXPECTATIONS_DIR, 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'chromium_perf_expectations.cfg') 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFAULT_TOLERANCE = 0.05 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)USAGE = '' 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def ReadFile(filename): 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 36a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) file = open(filename, 'rb') 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except IOError, e: 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print >> sys.stderr, ('I/O Error reading file %s(%s): %s' % 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (filename, e.errno, e.strerror)) 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise e 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) contents = file.read() 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) file.close() 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return contents 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def ConvertJsonIntoDict(string): 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Read a JSON string and convert its contents into a Python datatype.""" 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(string) == 0: 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print >> sys.stderr, ('Error could not parse empty string') 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise Exception('JSON data missing') 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) jsondata = json.loads(string) 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except ValueError, e: 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print >> sys.stderr, ('Error parsing string: "%s"' % string) 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise e 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return jsondata 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Floating point representation of last time we fetched a URL. 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)last_fetched_at = None 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def FetchUrlContents(url): 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) global last_fetched_at 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if last_fetched_at and ((time.time() - last_fetched_at) <= 0.5): 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Sleep for half a second to avoid overloading the server. 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) time.sleep(0.5) 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) last_fetched_at = time.time() 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) connection = urllib2.urlopen(url) 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except urllib2.HTTPError, e: 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if e.code == 404: 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise e 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) text = connection.read().strip() 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) connection.close() 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return text 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetRowData(data, key): 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rowdata = [] 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # reva and revb always come first. 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for subkey in ['reva', 'revb']: 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if subkey in data[key]: 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rowdata.append('"%s": %s' % (subkey, data[key][subkey])) 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Strings, like type, come next. 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for subkey in ['type', 'better']: 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if subkey in data[key]: 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rowdata.append('"%s": "%s"' % (subkey, data[key][subkey])) 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Finally the main numbers come last. 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for subkey in ['improve', 'regress', 'tolerance']: 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if subkey in data[key]: 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rowdata.append('"%s": %s' % (subkey, data[key][subkey])) 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rowdata 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetRowDigest(rowdata, key): 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sha1 = hashlib.sha1() 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rowdata = [str(possibly_unicode_string).encode('ascii') 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for possibly_unicode_string in rowdata] 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sha1.update(str(rowdata) + key) 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return sha1.hexdigest()[0:8] 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def WriteJson(filename, data, keys, calculate_sha1=True): 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Write a list of |keys| in |data| to the file specified in |filename|.""" 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 107a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) file = open(filename, 'wb') 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except IOError, e: 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print >> sys.stderr, ('I/O Error writing file %s(%s): %s' % 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (filename, e.errno, e.strerror)) 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) jsondata = [] 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for key in keys: 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rowdata = GetRowData(data, key) 1152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if calculate_sha1: 1162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Include an updated checksum. 1172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) rowdata.append('"sha1": "%s"' % GetRowDigest(rowdata, key)) 1182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) else: 1192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if 'sha1' in data[key]: 1202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) rowdata.append('"sha1": "%s"' % (data[key]['sha1'])) 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) jsondata.append('"%s": {%s}' % (key, ', '.join(rowdata))) 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) jsondata.append('"load": true') 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) jsontext = '{%s\n}' % ',\n '.join(jsondata) 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) file.write(jsontext + '\n') 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) file.close() 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def FloatIsInt(f): 1302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) epsilon = 1.0e-10 1312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return abs(f - int(f)) <= epsilon 1322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)last_key_printed = None 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def Main(args): 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def OutputMessage(message, verbose_message=True): 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) global last_key_printed 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not options.verbose and verbose_message: 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if key != last_key_printed: 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) last_key_printed = key 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print '\n' + key + ':' 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print ' %s' % message 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parser = optparse.OptionParser(usage=USAGE, version=__version__) 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parser.add_option('-v', '--verbose', action='store_true', default=False, 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) help='enable verbose output') 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parser.add_option('-s', '--checksum', action='store_true', 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) help='test if any changes are pending') 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parser.add_option('-c', '--config', dest='config_file', 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) default=DEFAULT_CONFIG_FILE, 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) help='set the config file to FILE', metavar='FILE') 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) options, args = parser.parse_args(args) 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if options.verbose: 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Verbose output enabled.' 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) config = ConvertJsonIntoDict(ReadFile(options.config_file)) 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Get the list of summaries for a test. 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base_url = config['base_url'] 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Make the perf expectations file relative to the path of the config file. 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perf_file = os.path.join( 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) os.path.dirname(options.config_file), config['perf_file']) 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perf = ConvertJsonIntoDict(ReadFile(perf_file)) 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Fetch graphs.dat for this combination. 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perfkeys = perf.keys() 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # In perf_expectations.json, ignore the 'load' key. 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perfkeys.remove('load') 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perfkeys.sort() 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) write_new_expectations = False 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) found_checksum_mismatch = False 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for key in perfkeys: 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) value = perf[key] 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tolerance = value.get('tolerance', DEFAULT_TOLERANCE) 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) better = value.get('better', None) 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Verify the checksum. 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) original_checksum = value.get('sha1', '') 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if 'sha1' in value: 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) del value['sha1'] 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rowdata = GetRowData(perf, key) 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) computed_checksum = GetRowDigest(rowdata, key) 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if original_checksum == computed_checksum: 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('checksum matches, skipping') 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif options.checksum: 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) found_checksum_mismatch = True 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Skip expectations that are missing a reva or revb. We can't generate 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # expectations for those. 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not(value.has_key('reva') and value.has_key('revb')): 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('missing revision range, skipping') 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) revb = int(value['revb']) 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reva = int(value['reva']) 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Ensure that reva is less than revb. 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if reva > revb: 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) temp = reva 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reva = revb 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) revb = temp 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Get the system/test/graph/tracename and reftracename for the current key. 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) matchData = re.match(r'^([^/]+)\/([^/]+)\/([^/]+)\/([^/]+)$', key) 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not matchData: 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('cannot parse key, skipping') 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) system = matchData.group(1) 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) test = matchData.group(2) 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) graph = matchData.group(3) 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tracename = matchData.group(4) 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reftracename = tracename + '_ref' 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Create the summary_url and get the json data for that URL. 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # FetchUrlContents() may sleep to avoid overloading the server with 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # requests. 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) summary_url = '%s/%s/%s/%s-summary.dat' % (base_url, system, test, graph) 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) summaryjson = FetchUrlContents(summary_url) 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not summaryjson: 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('ERROR: cannot find json data, please verify', 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) verbose_message=False) 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Set value's type to 'relative' by default. 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) value_type = value.get('type', 'relative') 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) summarylist = summaryjson.split('\n') 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) trace_values = {} 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) traces = [tracename] 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if value_type == 'relative': 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) traces += [reftracename] 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for trace in traces: 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) trace_values.setdefault(trace, {}) 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Find the high and low values for each of the traces. 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scanning = False 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for line in summarylist: 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) jsondata = ConvertJsonIntoDict(line) 2445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # TODO(iannucci): Remove this once http://crbug.com/336471 is resolved. 2465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if 'Force the Chro' in jsondata['rev']: 2475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) continue 2485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if int(jsondata['rev']) <= revb: 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scanning = True 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if int(jsondata['rev']) < reva: 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # We found the upper revision in the range. Scan for trace data until we 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # find the lower revision in the range. 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if scanning: 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for trace in traces: 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if trace not in jsondata['traces']: 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('trace %s missing' % trace) 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if type(jsondata['traces'][trace]) != type([]): 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('trace %s format not recognized' % trace) 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tracevalue = float(jsondata['traces'][trace][0]) 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except ValueError: 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('trace %s value error: %s' % ( 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) trace, str(jsondata['traces'][trace][0]))) 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for bound in ['high', 'low']: 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) trace_values[trace].setdefault(bound, tracevalue) 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) trace_values[trace]['high'] = max(trace_values[trace]['high'], 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tracevalue) 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) trace_values[trace]['low'] = min(trace_values[trace]['low'], 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tracevalue) 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if 'high' not in trace_values[tracename]: 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('no suitable traces matched, skipping') 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if value_type == 'relative': 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Calculate assuming high deltas are regressions and low deltas are 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # improvements. 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) regress = (float(trace_values[tracename]['high']) - 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) float(trace_values[reftracename]['low'])) 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) improve = (float(trace_values[tracename]['low']) - 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) float(trace_values[reftracename]['high'])) 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif value_type == 'absolute': 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Calculate assuming high absolutes are regressions and low absolutes are 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # improvements. 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) regress = float(trace_values[tracename]['high']) 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) improve = float(trace_values[tracename]['low']) 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # So far we've assumed better is lower (regress > improve). If the actual 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # values for regress and improve are equal, though, and better was not 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # specified, alert the user so we don't let them create a new file with 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # ambiguous rules. 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if better == None and regress == improve: 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('regress (%s) is equal to improve (%s), and "better" is ' 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'unspecified, please fix by setting "better": "lower" or ' 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '"better": "higher" in this perf trace\'s expectation' % ( 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) regress, improve), verbose_message=False) 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 1 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # If the existing values assume regressions are low deltas relative to 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # improvements, swap our regress and improve. This value must be a 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # scores-like result. 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if 'regress' in perf[key] and 'improve' in perf[key]: 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if perf[key]['regress'] < perf[key]['improve']: 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert(better != 'lower') 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) better = 'higher' 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) temp = regress 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) regress = improve 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) improve = temp 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Sometimes values are equal, e.g., when they are both 0, 3192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # 'better' may still be set to 'higher'. 3202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert(better != 'higher' or 3212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) perf[key]['regress'] == perf[key]['improve']) 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) better = 'lower' 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # If both were ints keep as int, otherwise use the float version. 3252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) originally_ints = False 3262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if FloatIsInt(regress) and FloatIsInt(improve): 3272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) originally_ints = True 3282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if better == 'higher': 3302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if originally_ints: 3312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) regress = int(math.floor(regress - abs(regress*tolerance))) 3322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) improve = int(math.ceil(improve + abs(improve*tolerance))) 3332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) else: 3342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) regress = regress - abs(regress*tolerance) 3352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) improve = improve + abs(improve*tolerance) 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if originally_ints: 3382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) improve = int(math.floor(improve - abs(improve*tolerance))) 3392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) regress = int(math.ceil(regress + abs(regress*tolerance))) 3402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) else: 3412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) improve = improve - abs(improve*tolerance) 3422a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) regress = regress + abs(regress*tolerance) 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Calculate the new checksum to test if this is the only thing that may have 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # changed. 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) checksum_rowdata = GetRowData(perf, key) 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_checksum = GetRowDigest(checksum_rowdata, key) 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if ('regress' in perf[key] and 'improve' in perf[key] and 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perf[key]['regress'] == regress and perf[key]['improve'] == improve and 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) original_checksum == new_checksum): 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('no change') 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) write_new_expectations = True 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('traces: %s' % trace_values, verbose_message=False) 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('before: %s' % perf[key], verbose_message=False) 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perf[key]['regress'] = regress 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) perf[key]['improve'] = improve 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OutputMessage('after: %s' % perf[key], verbose_message=False) 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if options.checksum: 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if found_checksum_mismatch: 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 1 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if write_new_expectations: 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print '\nWriting expectations... ', 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteJson(perf_file, perf, perfkeys) 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'done' 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if options.verbose: 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print '' 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'No changes.' 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)if __name__ == '__main__': 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sys.exit(Main(sys.argv)) 381