1#!/usr/bin/python 2''' 3Copyright 2012 Google Inc. 4 5Use of this source code is governed by a BSD-style license that can be 6found in the LICENSE file. 7''' 8 9''' 10Generates a visual diff of all pending changes in the local SVN (or git!) 11checkout. 12 13Launch with --help to see more information. 14 15TODO(epoger): Now that this tool supports either git or svn, rename it. 16TODO(epoger): Fix indentation in this file (2-space indents, not 4-space). 17''' 18 19# common Python modules 20import optparse 21import os 22import re 23import shutil 24import subprocess 25import sys 26import tempfile 27import urllib2 28 29# Imports from within Skia 30# 31# We need to add the 'gm' directory, so that we can import gm_json.py within 32# that directory. That script allows us to parse the actual-results.json file 33# written out by the GM tool. 34# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 35# so any dirs that are already in the PYTHONPATH will be preferred. 36# 37# This assumes that the 'gm' directory has been checked out as a sibling of 38# the 'tools' directory containing this script, which will be the case if 39# 'trunk' was checked out as a single unit. 40GM_DIRECTORY = os.path.realpath( 41 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) 42if GM_DIRECTORY not in sys.path: 43 sys.path.append(GM_DIRECTORY) 44import gm_json 45import jsondiff 46import svn 47 48USAGE_STRING = 'Usage: %s [options]' 49HELP_STRING = ''' 50 51Generates a visual diff of all pending changes in the local SVN/git checkout. 52 53This includes a list of all files that have been added, deleted, or modified 54(as far as SVN/git knows about). For any image modifications, pixel diffs will 55be generated. 56 57''' 58 59IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 60 61TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir) 62 63OPTION_DEST_DIR = '--dest-dir' 64OPTION_PATH_TO_SKDIFF = '--path-to-skdiff' 65OPTION_SOURCE_DIR = '--source-dir' 66 67def RunCommand(command): 68 """Run a command, raising an exception if it fails. 69 70 @param command the command as a single string 71 """ 72 print 'running command [%s]...' % command 73 retval = os.system(command) 74 if retval is not 0: 75 raise Exception('command [%s] failed' % command) 76 77def FindPathToSkDiff(user_set_path=None): 78 """Return path to an existing skdiff binary, or raise an exception if we 79 cannot find one. 80 81 @param user_set_path if None, the user did not specify a path, so look in 82 some likely places; otherwise, only check at this path 83 """ 84 if user_set_path is not None: 85 if os.path.isfile(user_set_path): 86 return user_set_path 87 raise Exception('unable to find skdiff at user-set path %s' % 88 user_set_path) 89 trunk_path = os.path.join(os.path.dirname(__file__), os.pardir) 90 possible_paths = [os.path.join(trunk_path, 'out', 'Release', 'skdiff'), 91 os.path.join(trunk_path, 'out', 'Debug', 'skdiff')] 92 for try_path in possible_paths: 93 if os.path.isfile(try_path): 94 return try_path 95 raise Exception('cannot find skdiff in paths %s; maybe you need to ' 96 'specify the %s option or build skdiff?' % ( 97 possible_paths, OPTION_PATH_TO_SKDIFF)) 98 99def _DownloadUrlToFile(source_url, dest_path): 100 """Download source_url, and save its contents to dest_path. 101 Raises an exception if there were any problems.""" 102 try: 103 reader = urllib2.urlopen(source_url) 104 writer = open(dest_path, 'wb') 105 writer.write(reader.read()) 106 writer.close() 107 except BaseException as e: 108 raise Exception( 109 '%s: unable to download source_url %s to dest_path %s' % ( 110 e, source_url, dest_path)) 111 112def _CreateGSUrl(imagename, hash_type, hash_digest): 113 """Return the HTTP URL we can use to download this particular version of 114 the actually-generated GM image with this imagename. 115 116 imagename: name of the test image, e.g. 'perlinnoise_msaa4.png' 117 hash_type: string indicating the hash type used to generate hash_digest, 118 e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 119 hash_digest: the hash digest of the image to retrieve 120 """ 121 return gm_json.CreateGmActualUrl( 122 test_name=IMAGE_FILENAME_RE.match(imagename).group(1), 123 hash_type=hash_type, 124 hash_digest=hash_digest) 125 126def _CallJsonDiff(old_json_path, new_json_path, 127 old_flattened_dir, new_flattened_dir, 128 filename_prefix): 129 """Using jsondiff.py, write the images that differ between two GM 130 expectations summary files (old and new) into old_flattened_dir and 131 new_flattened_dir. 132 133 filename_prefix: prefix to prepend to filenames of all images we write 134 into the flattened directories 135 """ 136 json_differ = jsondiff.GMDiffer() 137 diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path, 138 newfile=new_json_path) 139 print 'Downloading %d before-and-after image pairs...' % len(diff_dict) 140 for (imagename, results) in diff_dict.iteritems(): 141 # TODO(epoger): Currently, this assumes that all images have been 142 # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 143 144 old_checksum = results['old'] 145 if old_checksum: 146 old_image_url = _CreateGSUrl( 147 imagename=imagename, 148 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 149 hash_digest=old_checksum) 150 _DownloadUrlToFile( 151 source_url=old_image_url, 152 dest_path=os.path.join(old_flattened_dir, 153 filename_prefix + imagename)) 154 155 new_checksum = results['new'] 156 if new_checksum: 157 new_image_url = _CreateGSUrl( 158 imagename=imagename, 159 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 160 hash_digest=new_checksum) 161 _DownloadUrlToFile( 162 source_url=new_image_url, 163 dest_path=os.path.join(new_flattened_dir, 164 filename_prefix + imagename)) 165 166def _RunCommand(args): 167 """Run a command (from self._directory) and return stdout as a single 168 string. 169 170 @param args a list of arguments 171 """ 172 proc = subprocess.Popen(args, 173 stdout=subprocess.PIPE, 174 stderr=subprocess.PIPE) 175 (stdout, stderr) = proc.communicate() 176 if proc.returncode is not 0: 177 raise Exception('command "%s" failed: %s' % (args, stderr)) 178 return stdout 179 180def _GitGetModifiedFiles(): 181 """Returns a list of locally modified files within the current working dir. 182 183 TODO(epoger): Move this into a git utility package? 184 """ 185 return _RunCommand(['git', 'ls-files', '-m']).splitlines() 186 187def _GitExportBaseVersionOfFile(file_within_repo, dest_path): 188 """Retrieves a copy of the base version of a file within the repository. 189 190 @param file_within_repo path to the file within the repo whose base 191 version you wish to obtain 192 @param dest_path destination to which to write the base content 193 194 TODO(epoger): Move this into a git utility package? 195 """ 196 # TODO(epoger): Replace use of "git show" command with lower-level git 197 # commands? senorblanco points out that "git show" is a "porcelain" 198 # command, intended for human use, as opposed to the "plumbing" commands 199 # generally more suitable for scripting. (See 200 # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain ) 201 # 202 # For now, though, "git show" is the most straightforward implementation 203 # I could come up with. I tried using "git cat-file", but I had trouble 204 # getting it to work as desired. 205 args = ['git', 'show', os.path.join('HEAD:.', file_within_repo)] 206 with open(dest_path, 'wb') as outfile: 207 proc = subprocess.Popen(args, stdout=outfile) 208 proc.communicate() 209 if proc.returncode is not 0: 210 raise Exception('command "%s" failed' % args) 211 212def SvnDiff(path_to_skdiff, dest_dir, source_dir): 213 """Generates a visual diff of all pending changes in source_dir. 214 215 @param path_to_skdiff 216 @param dest_dir existing directory within which to write results 217 @param source_dir 218 """ 219 # Validate parameters, filling in default values if necessary and possible. 220 path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff)) 221 if not dest_dir: 222 dest_dir = tempfile.mkdtemp() 223 dest_dir = os.path.abspath(dest_dir) 224 225 os.chdir(source_dir) 226 using_svn = os.path.isdir('.svn') 227 228 # Prepare temporary directories. 229 modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened') 230 original_flattened_dir = os.path.join(dest_dir, 'original_flattened') 231 diff_dir = os.path.join(dest_dir, 'diffs') 232 for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] : 233 shutil.rmtree(dir, ignore_errors=True) 234 os.mkdir(dir) 235 236 # Get a list of all locally modified (including added/deleted) files, 237 # descending subdirectories. 238 if using_svn: 239 svn_repo = svn.Svn('.') 240 modified_file_paths = svn_repo.GetFilesWithStatus( 241 svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED) 242 else: 243 modified_file_paths = _GitGetModifiedFiles() 244 245 # For each modified file: 246 # 1. copy its current contents into modified_flattened_dir 247 # 2. copy its original contents into original_flattened_dir 248 for modified_file_path in modified_file_paths: 249 if modified_file_path.endswith('.json'): 250 # Special handling for JSON files, in the hopes that they 251 # contain GM result summaries. 252 (_unused, original_file_path) = tempfile.mkstemp() 253 if using_svn: 254 svn_repo.ExportBaseVersionOfFile( 255 modified_file_path, original_file_path) 256 else: 257 _GitExportBaseVersionOfFile( 258 modified_file_path, original_file_path) 259 platform_prefix = re.sub(os.sep, '__', 260 os.path.dirname(modified_file_path)) + '__' 261 _CallJsonDiff(old_json_path=original_file_path, 262 new_json_path=modified_file_path, 263 old_flattened_dir=original_flattened_dir, 264 new_flattened_dir=modified_flattened_dir, 265 filename_prefix=platform_prefix) 266 os.remove(original_file_path) 267 else: 268 dest_filename = re.sub(os.sep, '__', modified_file_path) 269 # If the file had STATUS_DELETED, it won't exist anymore... 270 if os.path.isfile(modified_file_path): 271 shutil.copyfile(modified_file_path, 272 os.path.join(modified_flattened_dir, 273 dest_filename)) 274 if using_svn: 275 svn_repo.ExportBaseVersionOfFile( 276 modified_file_path, 277 os.path.join(original_flattened_dir, dest_filename)) 278 else: 279 _GitExportBaseVersionOfFile( 280 modified_file_path, 281 os.path.join(original_flattened_dir, dest_filename)) 282 283 # Run skdiff: compare original_flattened_dir against modified_flattened_dir 284 RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir, 285 modified_flattened_dir, diff_dir)) 286 print '\nskdiff results are ready in file://%s/index.html' % diff_dir 287 288def RaiseUsageException(): 289 raise Exception('%s\nRun with --help for more detail.' % ( 290 USAGE_STRING % __file__)) 291 292def Main(options, args): 293 """Allow other scripts to call this script with fake command-line args. 294 """ 295 num_args = len(args) 296 if num_args != 0: 297 RaiseUsageException() 298 SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir, 299 source_dir=options.source_dir) 300 301if __name__ == '__main__': 302 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) 303 parser.add_option(OPTION_DEST_DIR, 304 action='store', type='string', default=None, 305 help='existing directory within which to write results; ' 306 'if not set, will create a temporary directory which ' 307 'will remain in place after this script completes') 308 parser.add_option(OPTION_PATH_TO_SKDIFF, 309 action='store', type='string', default=None, 310 help='path to already-built skdiff tool; if not set, ' 311 'will search for it in typical directories near this ' 312 'script') 313 parser.add_option(OPTION_SOURCE_DIR, 314 action='store', type='string', 315 default=os.path.join('expectations', 'gm'), 316 help='root directory within which to compare all ' + 317 'files; defaults to "%default"') 318 (options, args) = parser.parse_args() 319 Main(options, args) 320