1#!/usr/bin/python
2'''
3Copyright 2012 Google Inc.
4
5Use of this source code is governed by a BSD-style license that can be
6found in the LICENSE file.
7'''
8
9'''
10Generates a visual diff of all pending changes in the local SVN (or git!)
11checkout.
12
13Launch with --help to see more information.
14
15TODO(epoger): Now that this tool supports either git or svn, rename it.
16TODO(epoger): Fix indentation in this file (2-space indents, not 4-space).
17'''
18
19# common Python modules
20import optparse
21import os
22import re
23import shutil
24import subprocess
25import sys
26import tempfile
27import urllib2
28
29# Imports from within Skia
30#
31# We need to add the 'gm' directory, so that we can import gm_json.py within
32# that directory.  That script allows us to parse the actual-results.json file
33# written out by the GM tool.
34# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
35# so any dirs that are already in the PYTHONPATH will be preferred.
36#
37# This assumes that the 'gm' directory has been checked out as a sibling of
38# the 'tools' directory containing this script, which will be the case if
39# 'trunk' was checked out as a single unit.
40GM_DIRECTORY = os.path.realpath(
41    os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
42if GM_DIRECTORY not in sys.path:
43    sys.path.append(GM_DIRECTORY)
44import gm_json
45import jsondiff
46import svn
47
48USAGE_STRING = 'Usage: %s [options]'
49HELP_STRING = '''
50
51Generates a visual diff of all pending changes in the local SVN/git checkout.
52
53This includes a list of all files that have been added, deleted, or modified
54(as far as SVN/git knows about).  For any image modifications, pixel diffs will
55be generated.
56
57'''
58
59IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
60
61TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir)
62
63OPTION_DEST_DIR = '--dest-dir'
64OPTION_PATH_TO_SKDIFF = '--path-to-skdiff'
65OPTION_SOURCE_DIR = '--source-dir'
66
67def RunCommand(command):
68    """Run a command, raising an exception if it fails.
69
70    @param command the command as a single string
71    """
72    print 'running command [%s]...' % command
73    retval = os.system(command)
74    if retval is not 0:
75        raise Exception('command [%s] failed' % command)
76
77def FindPathToSkDiff(user_set_path=None):
78    """Return path to an existing skdiff binary, or raise an exception if we
79    cannot find one.
80
81    @param user_set_path if None, the user did not specify a path, so look in
82           some likely places; otherwise, only check at this path
83    """
84    if user_set_path is not None:
85        if os.path.isfile(user_set_path):
86            return user_set_path
87        raise Exception('unable to find skdiff at user-set path %s' %
88                        user_set_path)
89    trunk_path = os.path.join(os.path.dirname(__file__), os.pardir)
90    possible_paths = [os.path.join(trunk_path, 'out', 'Release', 'skdiff'),
91                      os.path.join(trunk_path, 'out', 'Debug', 'skdiff')]
92    for try_path in possible_paths:
93        if os.path.isfile(try_path):
94            return try_path
95    raise Exception('cannot find skdiff in paths %s; maybe you need to '
96                    'specify the %s option or build skdiff?' % (
97                        possible_paths, OPTION_PATH_TO_SKDIFF))
98
99def _DownloadUrlToFile(source_url, dest_path):
100    """Download source_url, and save its contents to dest_path.
101    Raises an exception if there were any problems."""
102    try:
103        reader = urllib2.urlopen(source_url)
104        writer = open(dest_path, 'wb')
105        writer.write(reader.read())
106        writer.close()
107    except BaseException as e:
108        raise Exception(
109            '%s: unable to download source_url %s to dest_path %s' % (
110                e, source_url, dest_path))
111
112def _CreateGSUrl(imagename, hash_type, hash_digest):
113    """Return the HTTP URL we can use to download this particular version of
114    the actually-generated GM image with this imagename.
115
116    imagename: name of the test image, e.g. 'perlinnoise_msaa4.png'
117    hash_type: string indicating the hash type used to generate hash_digest,
118               e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
119    hash_digest: the hash digest of the image to retrieve
120    """
121    return gm_json.CreateGmActualUrl(
122        test_name=IMAGE_FILENAME_RE.match(imagename).group(1),
123        hash_type=hash_type,
124        hash_digest=hash_digest)
125
126def _CallJsonDiff(old_json_path, new_json_path,
127                  old_flattened_dir, new_flattened_dir,
128                  filename_prefix):
129    """Using jsondiff.py, write the images that differ between two GM
130    expectations summary files (old and new) into old_flattened_dir and
131    new_flattened_dir.
132
133    filename_prefix: prefix to prepend to filenames of all images we write
134        into the flattened directories
135    """
136    json_differ = jsondiff.GMDiffer()
137    diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path,
138                                             newfile=new_json_path)
139    print 'Downloading %d before-and-after image pairs...' % len(diff_dict)
140    for (imagename, results) in diff_dict.iteritems():
141        # TODO(epoger): Currently, this assumes that all images have been
142        # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
143
144        old_checksum = results['old']
145        if old_checksum:
146            old_image_url = _CreateGSUrl(
147                imagename=imagename,
148                hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
149                hash_digest=old_checksum)
150            _DownloadUrlToFile(
151                source_url=old_image_url,
152                dest_path=os.path.join(old_flattened_dir,
153                                       filename_prefix + imagename))
154
155        new_checksum = results['new']
156        if new_checksum:
157            new_image_url = _CreateGSUrl(
158                imagename=imagename,
159                hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
160                hash_digest=new_checksum)
161            _DownloadUrlToFile(
162                source_url=new_image_url,
163                dest_path=os.path.join(new_flattened_dir,
164                                       filename_prefix + imagename))
165
166def _RunCommand(args):
167    """Run a command (from self._directory) and return stdout as a single
168    string.
169
170    @param args a list of arguments
171    """
172    proc = subprocess.Popen(args,
173                            stdout=subprocess.PIPE,
174                            stderr=subprocess.PIPE)
175    (stdout, stderr) = proc.communicate()
176    if proc.returncode is not 0:
177        raise Exception('command "%s" failed: %s' % (args, stderr))
178    return stdout
179
180def _GitGetModifiedFiles():
181    """Returns a list of locally modified files within the current working dir.
182
183    TODO(epoger): Move this into a git utility package?
184    """
185    return _RunCommand(['git', 'ls-files', '-m']).splitlines()
186
187def _GitExportBaseVersionOfFile(file_within_repo, dest_path):
188    """Retrieves a copy of the base version of a file within the repository.
189
190    @param file_within_repo path to the file within the repo whose base
191           version you wish to obtain
192    @param dest_path destination to which to write the base content
193
194    TODO(epoger): Move this into a git utility package?
195    """
196    # TODO(epoger): Replace use of "git show" command with lower-level git
197    # commands?  senorblanco points out that "git show" is a "porcelain"
198    # command, intended for human use, as opposed to the "plumbing" commands
199    # generally more suitable for scripting.  (See
200    # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain )
201    #
202    # For now, though, "git show" is the most straightforward implementation
203    # I could come up with.  I tried using "git cat-file", but I had trouble
204    # getting it to work as desired.
205    args = ['git', 'show', os.path.join('HEAD:.', file_within_repo)]
206    with open(dest_path, 'wb') as outfile:
207        proc = subprocess.Popen(args, stdout=outfile)
208        proc.communicate()
209        if proc.returncode is not 0:
210            raise Exception('command "%s" failed' % args)
211
212def SvnDiff(path_to_skdiff, dest_dir, source_dir):
213    """Generates a visual diff of all pending changes in source_dir.
214
215    @param path_to_skdiff
216    @param dest_dir existing directory within which to write results
217    @param source_dir
218    """
219    # Validate parameters, filling in default values if necessary and possible.
220    path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff))
221    if not dest_dir:
222        dest_dir = tempfile.mkdtemp()
223    dest_dir = os.path.abspath(dest_dir)
224
225    os.chdir(source_dir)
226    using_svn = os.path.isdir('.svn')
227
228    # Prepare temporary directories.
229    modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened')
230    original_flattened_dir = os.path.join(dest_dir, 'original_flattened')
231    diff_dir = os.path.join(dest_dir, 'diffs')
232    for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] :
233        shutil.rmtree(dir, ignore_errors=True)
234        os.mkdir(dir)
235
236    # Get a list of all locally modified (including added/deleted) files,
237    # descending subdirectories.
238    if using_svn:
239        svn_repo = svn.Svn('.')
240        modified_file_paths = svn_repo.GetFilesWithStatus(
241            svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
242    else:
243        modified_file_paths = _GitGetModifiedFiles()
244
245    # For each modified file:
246    # 1. copy its current contents into modified_flattened_dir
247    # 2. copy its original contents into original_flattened_dir
248    for modified_file_path in modified_file_paths:
249        if modified_file_path.endswith('.json'):
250            # Special handling for JSON files, in the hopes that they
251            # contain GM result summaries.
252            (_unused, original_file_path) = tempfile.mkstemp()
253            if using_svn:
254                svn_repo.ExportBaseVersionOfFile(
255                    modified_file_path, original_file_path)
256            else:
257                _GitExportBaseVersionOfFile(
258                    modified_file_path, original_file_path)
259            platform_prefix = re.sub(os.sep, '__',
260                                     os.path.dirname(modified_file_path)) + '__'
261            _CallJsonDiff(old_json_path=original_file_path,
262                          new_json_path=modified_file_path,
263                          old_flattened_dir=original_flattened_dir,
264                          new_flattened_dir=modified_flattened_dir,
265                          filename_prefix=platform_prefix)
266            os.remove(original_file_path)
267        else:
268            dest_filename = re.sub(os.sep, '__', modified_file_path)
269            # If the file had STATUS_DELETED, it won't exist anymore...
270            if os.path.isfile(modified_file_path):
271                shutil.copyfile(modified_file_path,
272                                os.path.join(modified_flattened_dir,
273                                             dest_filename))
274            if using_svn:
275                svn_repo.ExportBaseVersionOfFile(
276                    modified_file_path,
277                    os.path.join(original_flattened_dir, dest_filename))
278            else:
279                _GitExportBaseVersionOfFile(
280                    modified_file_path,
281                    os.path.join(original_flattened_dir, dest_filename))
282
283    # Run skdiff: compare original_flattened_dir against modified_flattened_dir
284    RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir,
285                                modified_flattened_dir, diff_dir))
286    print '\nskdiff results are ready in file://%s/index.html' % diff_dir
287
288def RaiseUsageException():
289    raise Exception('%s\nRun with --help for more detail.' % (
290        USAGE_STRING % __file__))
291
292def Main(options, args):
293    """Allow other scripts to call this script with fake command-line args.
294    """
295    num_args = len(args)
296    if num_args != 0:
297        RaiseUsageException()
298    SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir,
299            source_dir=options.source_dir)
300
301if __name__ == '__main__':
302    parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
303    parser.add_option(OPTION_DEST_DIR,
304                      action='store', type='string', default=None,
305                      help='existing directory within which to write results; '
306                      'if not set, will create a temporary directory which '
307                      'will remain in place after this script completes')
308    parser.add_option(OPTION_PATH_TO_SKDIFF,
309                      action='store', type='string', default=None,
310                      help='path to already-built skdiff tool; if not set, '
311                      'will search for it in typical directories near this '
312                      'script')
313    parser.add_option(OPTION_SOURCE_DIR,
314                      action='store', type='string',
315                      default=os.path.join('expectations', 'gm'),
316                      help='root directory within which to compare all ' +
317                      'files; defaults to "%default"')
318    (options, args) = parser.parse_args()
319    Main(options, args)
320