158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger#!/usr/bin/python 258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger''' 458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerCopyright 2013 Google Inc. 558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerUse of this source code is governed by a BSD-style license that can be 758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerfound in the LICENSE file. 858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger''' 958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 1058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger''' 1158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerGathers diffs between 2 JSON expectations files, or between actual and 1258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerexpected results within a single JSON actual-results file, 1358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerand generates an old-vs-new diff dictionary. 1458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 1558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerTODO(epoger): Fix indentation in this file (2-space indents, not 4-space). 1658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger''' 1758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 1858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# System-level imports 1958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerimport argparse 2058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerimport json 2158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerimport os 2258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerimport sys 2358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerimport urllib2 2458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 2558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# Imports from within Skia 2658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# 2758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# We need to add the 'gm' directory, so that we can import gm_json.py within 2858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# that directory. That script allows us to parse the actual-results.json file 2958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# written out by the GM tool. 3058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 3158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# so any dirs that are already in the PYTHONPATH will be preferred. 3258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# 3358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# This assumes that the 'gm' directory has been checked out as a sibling of 3458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# the 'tools' directory containing this script, which will be the case if 3558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# 'trunk' was checked out as a single unit. 3658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerGM_DIRECTORY = os.path.realpath( 3758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) 3858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerif GM_DIRECTORY not in sys.path: 3958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger sys.path.append(GM_DIRECTORY) 4058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerimport gm_json 4158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 4258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 4358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# Object that generates diffs between two JSON gm result files. 4458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerclass GMDiffer(object): 4558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 4658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger def __init__(self): 4758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger pass 4858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 4958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger def _GetFileContentsAsString(self, filepath): 5058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """Returns the full contents of a file, as a single string. 51e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger If the filename looks like a URL, download its contents. 52e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger If the filename is None, return None.""" 53e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger if filepath is None: 54e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger return None 55e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger elif filepath.startswith('http:') or filepath.startswith('https:'): 5658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger return urllib2.urlopen(filepath).read() 5758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger else: 5858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger return open(filepath, 'r').read() 5958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 60e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger def _GetExpectedResults(self, contents): 61e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger """Returns the dictionary of expected results from a JSON string, 6258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger in this form: 6358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 6458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger { 6558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'test1' : 14760033689012826769, 6658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'test2' : 9151974350149210736, 6758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger ... 6858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger } 6958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 7058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger We make these simplifying assumptions: 7158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 1. Each test has either 0 or 1 allowed results. 7258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 2. All expectations are of type JSONKEY_HASHTYPE_BITMAP_64BITMD5. 7358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 7458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger Any tests which violate those assumptions will cause an exception to 7558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger be raised. 7658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 7758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger Any tests for which we have no expectations will be left out of the 7858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger returned dictionary. 7958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """ 8058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger result_dict = {} 8158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger json_dict = gm_json.LoadFromString(contents) 8258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger all_expectations = json_dict[gm_json.JSONKEY_EXPECTEDRESULTS] 830a657bbc2c6fc9daf699942e023050536d5ec95fDerek Sollenberger 840a657bbc2c6fc9daf699942e023050536d5ec95fDerek Sollenberger # Prevent https://code.google.com/p/skia/issues/detail?id=1588 850a657bbc2c6fc9daf699942e023050536d5ec95fDerek Sollenberger # ('svndiff.py: 'NoneType' object has no attribute 'keys'') 860a657bbc2c6fc9daf699942e023050536d5ec95fDerek Sollenberger if not all_expectations: 870a657bbc2c6fc9daf699942e023050536d5ec95fDerek Sollenberger return result_dict 880a657bbc2c6fc9daf699942e023050536d5ec95fDerek Sollenberger 8958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger for test_name in all_expectations.keys(): 9058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger test_expectations = all_expectations[test_name] 9158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger allowed_digests = test_expectations[ 9258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] 9358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if allowed_digests: 9458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger num_allowed_digests = len(allowed_digests) 9558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if num_allowed_digests > 1: 9658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger raise ValueError( 97e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger 'test %s has %d allowed digests' % ( 98e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger test_name, num_allowed_digests)) 9958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger digest_pair = allowed_digests[0] 10058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if digest_pair[0] != gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5: 10158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger raise ValueError( 102e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger 'test %s has unsupported hashtype %s' % ( 103e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger test_name, digest_pair[0])) 10458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger result_dict[test_name] = digest_pair[1] 10558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger return result_dict 10658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 107e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger def _GetActualResults(self, contents): 108e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger """Returns the dictionary of actual results from a JSON string, 10958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger in this form: 11058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 11158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger { 11258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'test1' : 14760033689012826769, 11358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'test2' : 9151974350149210736, 11458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger ... 11558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger } 11658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 11758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger We make these simplifying assumptions: 11858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 1. All results are of type JSONKEY_HASHTYPE_BITMAP_64BITMD5. 11958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 12058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger Any tests which violate those assumptions will cause an exception to 12158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger be raised. 12258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 12358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger Any tests for which we have no actual results will be left out of the 12458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger returned dictionary. 12558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """ 12658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger result_dict = {} 12758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger json_dict = gm_json.LoadFromString(contents) 12858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger all_result_types = json_dict[gm_json.JSONKEY_ACTUALRESULTS] 12958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger for result_type in all_result_types.keys(): 13058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger results_of_this_type = all_result_types[result_type] 13158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if results_of_this_type: 13258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger for test_name in results_of_this_type.keys(): 13358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger digest_pair = results_of_this_type[test_name] 13458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if digest_pair[0] != gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5: 13558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger raise ValueError( 136e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger 'test %s has unsupported hashtype %s' % ( 137e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger test_name, digest_pair[0])) 13858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger result_dict[test_name] = digest_pair[1] 13958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger return result_dict 14058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 14158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger def _DictionaryDiff(self, old_dict, new_dict): 14258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """Generate a dictionary showing the diffs between old_dict and new_dict. 14358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger Any entries which are identical across them will be left out.""" 14458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger diff_dict = {} 14558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger all_keys = set(old_dict.keys() + new_dict.keys()) 14658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger for key in all_keys: 14758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if old_dict.get(key) != new_dict.get(key): 14858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger new_entry = {} 14958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger new_entry['old'] = old_dict.get(key) 15058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger new_entry['new'] = new_dict.get(key) 15158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger diff_dict[key] = new_entry 15258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger return diff_dict 15358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 15458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger def GenerateDiffDict(self, oldfile, newfile=None): 15558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """Generate a dictionary showing the diffs: 15658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger old = expectations within oldfile 15758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger new = expectations within newfile 15858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 15958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger If newfile is not specified, then 'new' is the actual results within 16058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger oldfile. 16158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """ 162e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger return self.GenerateDiffDictFromStrings(self._GetFileContentsAsString(oldfile), 163e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger self._GetFileContentsAsString(newfile)) 164e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger 165e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger def GenerateDiffDictFromStrings(self, oldjson, newjson=None): 166e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger """Generate a dictionary showing the diffs: 167e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger old = expectations within oldjson 168e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger new = expectations within newjson 169e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger 170e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger If newfile is not specified, then 'new' is the actual results within 171e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger oldfile. 172e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger """ 173e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger old_results = self._GetExpectedResults(oldjson) 174e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger if newjson: 175e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger new_results = self._GetExpectedResults(newjson) 17658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger else: 177e27eefc4844477cee5d32f51ab45ff62020cdb36Derek Sollenberger new_results = self._GetActualResults(oldjson) 17858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger return self._DictionaryDiff(old_results, new_results) 17958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 18058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 18158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerdef _Main(): 18258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger parser = argparse.ArgumentParser() 18358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger parser.add_argument( 18458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'old', 18558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger help='Path to JSON file whose expectations to display on ' + 18658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'the "old" side of the diff. This can be a filepath on ' + 18758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'local storage, or a URL.') 18858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger parser.add_argument( 18958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'new', nargs='?', 19058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger help='Path to JSON file whose expectations to display on ' + 19158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'the "new" side of the diff; if not specified, uses the ' + 19258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'ACTUAL results from the "old" JSON file. This can be a ' + 19358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 'filepath on local storage, or a URL.') 19458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger args = parser.parse_args() 19558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger differ = GMDiffer() 19658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger diffs = differ.GenerateDiffDict(oldfile=args.old, newfile=args.new) 19758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger json.dump(diffs, sys.stdout, sort_keys=True, indent=2) 19858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 19958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger 20058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerif __name__ == '__main__': 20158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger _Main() 202