1#!/usr/bin/python 2''' 3Copyright 2012 Google Inc. 4 5Use of this source code is governed by a BSD-style license that can be 6found in the LICENSE file. 7''' 8 9''' 10Generates a visual diff of all pending changes in the local SVN (or git!) 11checkout. 12 13Launch with --help to see more information. 14 15TODO(epoger): Now that this tool supports either git or svn, rename it. 16TODO(epoger): Fix indentation in this file (2-space indents, not 4-space). 17''' 18 19# common Python modules 20import optparse 21import os 22import posixpath 23import re 24import shutil 25import subprocess 26import sys 27import tempfile 28import urllib2 29 30# Imports from within Skia 31# 32# We need to add the 'gm' directory, so that we can import gm_json.py within 33# that directory. That script allows us to parse the actual-results.json file 34# written out by the GM tool. 35# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 36# so any dirs that are already in the PYTHONPATH will be preferred. 37# 38# This assumes that the 'gm' directory has been checked out as a sibling of 39# the 'tools' directory containing this script, which will be the case if 40# 'trunk' was checked out as a single unit. 41GM_DIRECTORY = os.path.realpath( 42 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) 43if GM_DIRECTORY not in sys.path: 44 sys.path.append(GM_DIRECTORY) 45import gm_json 46import jsondiff 47import svn 48 49USAGE_STRING = 'Usage: %s [options]' 50HELP_STRING = ''' 51 52Generates a visual diff of all pending changes in the local SVN/git checkout. 53 54This includes a list of all files that have been added, deleted, or modified 55(as far as SVN/git knows about). For any image modifications, pixel diffs will 56be generated. 57 58''' 59 60IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 61 62TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir) 63 64OPTION_DEST_DIR = '--dest-dir' 65OPTION_PATH_TO_SKDIFF = '--path-to-skdiff' 66OPTION_SOURCE_DIR = '--source-dir' 67 68def RunCommand(command): 69 """Run a command, raising an exception if it fails. 70 71 @param command the command as a single string 72 """ 73 print 'running command [%s]...' % command 74 retval = os.system(command) 75 if retval is not 0: 76 raise Exception('command [%s] failed' % command) 77 78def FindPathToSkDiff(user_set_path=None): 79 """Return path to an existing skdiff binary, or raise an exception if we 80 cannot find one. 81 82 @param user_set_path if None, the user did not specify a path, so look in 83 some likely places; otherwise, only check at this path 84 """ 85 if user_set_path is not None: 86 if os.path.isfile(user_set_path): 87 return user_set_path 88 raise Exception('unable to find skdiff at user-set path %s' % 89 user_set_path) 90 trunk_path = os.path.join(os.path.dirname(__file__), os.pardir) 91 92 extension = '' 93 if os.name is 'nt': 94 extension = '.exe' 95 96 possible_paths = [os.path.join(trunk_path, 'out', 'Release', 97 'skdiff' + extension), 98 os.path.join(trunk_path, 'out', 'Debug', 99 'skdiff' + extension)] 100 for try_path in possible_paths: 101 if os.path.isfile(try_path): 102 return try_path 103 raise Exception('cannot find skdiff in paths %s; maybe you need to ' 104 'specify the %s option or build skdiff?' % ( 105 possible_paths, OPTION_PATH_TO_SKDIFF)) 106 107def _DownloadUrlToFile(source_url, dest_path): 108 """Download source_url, and save its contents to dest_path. 109 Raises an exception if there were any problems.""" 110 try: 111 reader = urllib2.urlopen(source_url) 112 writer = open(dest_path, 'wb') 113 writer.write(reader.read()) 114 writer.close() 115 except BaseException as e: 116 raise Exception( 117 '%s: unable to download source_url %s to dest_path %s' % ( 118 e, source_url, dest_path)) 119 120def _CreateGSUrl(imagename, hash_type, hash_digest): 121 """Return the HTTP URL we can use to download this particular version of 122 the actually-generated GM image with this imagename. 123 124 imagename: name of the test image, e.g. 'perlinnoise_msaa4.png' 125 hash_type: string indicating the hash type used to generate hash_digest, 126 e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 127 hash_digest: the hash digest of the image to retrieve 128 """ 129 return gm_json.CreateGmActualUrl( 130 test_name=IMAGE_FILENAME_RE.match(imagename).group(1), 131 hash_type=hash_type, 132 hash_digest=hash_digest) 133 134def _CallJsonDiff(old_json_path, new_json_path, 135 old_flattened_dir, new_flattened_dir, 136 filename_prefix): 137 """Using jsondiff.py, write the images that differ between two GM 138 expectations summary files (old and new) into old_flattened_dir and 139 new_flattened_dir. 140 141 filename_prefix: prefix to prepend to filenames of all images we write 142 into the flattened directories 143 """ 144 json_differ = jsondiff.GMDiffer() 145 diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path, 146 newfile=new_json_path) 147 print 'Downloading %d before-and-after image pairs...' % len(diff_dict) 148 for (imagename, results) in diff_dict.iteritems(): 149 # TODO(epoger): Currently, this assumes that all images have been 150 # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 151 152 old_checksum = results['old'] 153 if old_checksum: 154 old_image_url = _CreateGSUrl( 155 imagename=imagename, 156 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 157 hash_digest=old_checksum) 158 _DownloadUrlToFile( 159 source_url=old_image_url, 160 dest_path=os.path.join(old_flattened_dir, 161 filename_prefix + imagename)) 162 163 new_checksum = results['new'] 164 if new_checksum: 165 new_image_url = _CreateGSUrl( 166 imagename=imagename, 167 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 168 hash_digest=new_checksum) 169 _DownloadUrlToFile( 170 source_url=new_image_url, 171 dest_path=os.path.join(new_flattened_dir, 172 filename_prefix + imagename)) 173 174def _RunCommand(args): 175 """Run a command (from self._directory) and return stdout as a single 176 string. 177 178 @param args a list of arguments 179 """ 180 proc = subprocess.Popen(args, 181 stdout=subprocess.PIPE, 182 stderr=subprocess.PIPE) 183 (stdout, stderr) = proc.communicate() 184 if proc.returncode is not 0: 185 raise Exception('command "%s" failed: %s' % (args, stderr)) 186 return stdout 187 188def _GitGetModifiedFiles(): 189 """Returns a list of locally modified files within the current working dir. 190 191 TODO(epoger): Move this into a git utility package? 192 """ 193 return _RunCommand(['git', 'ls-files', '-m']).splitlines() 194 195def _GitExportBaseVersionOfFile(file_within_repo, dest_path): 196 """Retrieves a copy of the base version of a file within the repository. 197 198 @param file_within_repo path to the file within the repo whose base 199 version you wish to obtain 200 @param dest_path destination to which to write the base content 201 202 TODO(epoger): Move this into a git utility package? 203 """ 204 # TODO(epoger): Replace use of "git show" command with lower-level git 205 # commands? senorblanco points out that "git show" is a "porcelain" 206 # command, intended for human use, as opposed to the "plumbing" commands 207 # generally more suitable for scripting. (See 208 # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain ) 209 # 210 # For now, though, "git show" is the most straightforward implementation 211 # I could come up with. I tried using "git cat-file", but I had trouble 212 # getting it to work as desired. 213 # Note that git expects / rather than \ as a path separator even on 214 # windows. 215 args = ['git', 'show', posixpath.join('HEAD:.', file_within_repo)] 216 with open(dest_path, 'wb') as outfile: 217 proc = subprocess.Popen(args, stdout=outfile) 218 proc.communicate() 219 if proc.returncode is not 0: 220 raise Exception('command "%s" failed' % args) 221 222def SvnDiff(path_to_skdiff, dest_dir, source_dir): 223 """Generates a visual diff of all pending changes in source_dir. 224 225 @param path_to_skdiff 226 @param dest_dir existing directory within which to write results 227 @param source_dir 228 """ 229 # Validate parameters, filling in default values if necessary and possible. 230 path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff)) 231 if not dest_dir: 232 dest_dir = tempfile.mkdtemp() 233 dest_dir = os.path.abspath(dest_dir) 234 235 os.chdir(source_dir) 236 svn_repo = svn.Svn('.') 237 using_svn = True 238 try: 239 svn_repo.GetInfo() 240 except: 241 using_svn = False 242 243 # Prepare temporary directories. 244 modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened') 245 original_flattened_dir = os.path.join(dest_dir, 'original_flattened') 246 diff_dir = os.path.join(dest_dir, 'diffs') 247 for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] : 248 shutil.rmtree(dir, ignore_errors=True) 249 os.mkdir(dir) 250 251 # Get a list of all locally modified (including added/deleted) files, 252 # descending subdirectories. 253 if using_svn: 254 modified_file_paths = svn_repo.GetFilesWithStatus( 255 svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED) 256 else: 257 modified_file_paths = _GitGetModifiedFiles() 258 259 # For each modified file: 260 # 1. copy its current contents into modified_flattened_dir 261 # 2. copy its original contents into original_flattened_dir 262 for modified_file_path in modified_file_paths: 263 if modified_file_path.endswith('.json'): 264 # Special handling for JSON files, in the hopes that they 265 # contain GM result summaries. 266 original_file = tempfile.NamedTemporaryFile(delete = False) 267 original_file.close() 268 if using_svn: 269 svn_repo.ExportBaseVersionOfFile( 270 modified_file_path, original_file.name) 271 else: 272 _GitExportBaseVersionOfFile( 273 modified_file_path, original_file.name) 274 modified_dir = os.path.dirname(modified_file_path) 275 platform_prefix = (re.sub(re.escape(os.sep), '__', 276 os.path.splitdrive(modified_dir)[1]) 277 + '__') 278 _CallJsonDiff(old_json_path=original_file.name, 279 new_json_path=modified_file_path, 280 old_flattened_dir=original_flattened_dir, 281 new_flattened_dir=modified_flattened_dir, 282 filename_prefix=platform_prefix) 283 os.remove(original_file.name) 284 else: 285 dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path) 286 # If the file had STATUS_DELETED, it won't exist anymore... 287 if os.path.isfile(modified_file_path): 288 shutil.copyfile(modified_file_path, 289 os.path.join(modified_flattened_dir, 290 dest_filename)) 291 if using_svn: 292 svn_repo.ExportBaseVersionOfFile( 293 modified_file_path, 294 os.path.join(original_flattened_dir, dest_filename)) 295 else: 296 _GitExportBaseVersionOfFile( 297 modified_file_path, 298 os.path.join(original_flattened_dir, dest_filename)) 299 300 # Run skdiff: compare original_flattened_dir against modified_flattened_dir 301 RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir, 302 modified_flattened_dir, diff_dir)) 303 print '\nskdiff results are ready in file://%s/index.html' % diff_dir 304 305def RaiseUsageException(): 306 raise Exception('%s\nRun with --help for more detail.' % ( 307 USAGE_STRING % __file__)) 308 309def Main(options, args): 310 """Allow other scripts to call this script with fake command-line args. 311 """ 312 num_args = len(args) 313 if num_args != 0: 314 RaiseUsageException() 315 SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir, 316 source_dir=options.source_dir) 317 318if __name__ == '__main__': 319 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) 320 parser.add_option(OPTION_DEST_DIR, 321 action='store', type='string', default=None, 322 help='existing directory within which to write results; ' 323 'if not set, will create a temporary directory which ' 324 'will remain in place after this script completes') 325 parser.add_option(OPTION_PATH_TO_SKDIFF, 326 action='store', type='string', default=None, 327 help='path to already-built skdiff tool; if not set, ' 328 'will search for it in typical directories near this ' 329 'script') 330 parser.add_option(OPTION_SOURCE_DIR, 331 action='store', type='string', 332 default=os.path.join('expectations', 'gm'), 333 help='root directory within which to compare all ' + 334 'files; defaults to "%default"') 335 (options, args) = parser.parse_args() 336 Main(options, args) 337