1#!/usr/bin/python
2'''
3Copyright 2012 Google Inc.
4
5Use of this source code is governed by a BSD-style license that can be
6found in the LICENSE file.
7'''
8
9'''
10Generates a visual diff of all pending changes in the local SVN (or git!)
11checkout.
12
13Launch with --help to see more information.
14
15TODO(epoger): Now that this tool supports either git or svn, rename it.
16TODO(epoger): Fix indentation in this file (2-space indents, not 4-space).
17'''
18
19# common Python modules
20import optparse
21import os
22import posixpath
23import re
24import shutil
25import subprocess
26import sys
27import tempfile
28import urllib2
29
30# Imports from within Skia
31#
32# We need to add the 'gm' directory, so that we can import gm_json.py within
33# that directory.  That script allows us to parse the actual-results.json file
34# written out by the GM tool.
35# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
36# so any dirs that are already in the PYTHONPATH will be preferred.
37#
38# This assumes that the 'gm' directory has been checked out as a sibling of
39# the 'tools' directory containing this script, which will be the case if
40# 'trunk' was checked out as a single unit.
41GM_DIRECTORY = os.path.realpath(
42    os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
43if GM_DIRECTORY not in sys.path:
44    sys.path.append(GM_DIRECTORY)
45import gm_json
46import jsondiff
47import svn
48
49USAGE_STRING = 'Usage: %s [options]'
50HELP_STRING = '''
51
52Generates a visual diff of all pending changes in the local SVN/git checkout.
53
54This includes a list of all files that have been added, deleted, or modified
55(as far as SVN/git knows about).  For any image modifications, pixel diffs will
56be generated.
57
58'''
59
60IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
61
62TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir)
63
64OPTION_DEST_DIR = '--dest-dir'
65OPTION_PATH_TO_SKDIFF = '--path-to-skdiff'
66OPTION_SOURCE_DIR = '--source-dir'
67
68def RunCommand(command):
69    """Run a command, raising an exception if it fails.
70
71    @param command the command as a single string
72    """
73    print 'running command [%s]...' % command
74    retval = os.system(command)
75    if retval is not 0:
76        raise Exception('command [%s] failed' % command)
77
78def FindPathToSkDiff(user_set_path=None):
79    """Return path to an existing skdiff binary, or raise an exception if we
80    cannot find one.
81
82    @param user_set_path if None, the user did not specify a path, so look in
83           some likely places; otherwise, only check at this path
84    """
85    if user_set_path is not None:
86        if os.path.isfile(user_set_path):
87            return user_set_path
88        raise Exception('unable to find skdiff at user-set path %s' %
89                        user_set_path)
90    trunk_path = os.path.join(os.path.dirname(__file__), os.pardir)
91
92    extension = ''
93    if os.name is 'nt':
94        extension = '.exe'
95
96    possible_paths = [os.path.join(trunk_path, 'out', 'Release',
97                                    'skdiff' + extension),
98                      os.path.join(trunk_path, 'out', 'Debug',
99                                   'skdiff' + extension)]
100    for try_path in possible_paths:
101        if os.path.isfile(try_path):
102            return try_path
103    raise Exception('cannot find skdiff in paths %s; maybe you need to '
104                    'specify the %s option or build skdiff?' % (
105                        possible_paths, OPTION_PATH_TO_SKDIFF))
106
107def _DownloadUrlToFile(source_url, dest_path):
108    """Download source_url, and save its contents to dest_path.
109    Raises an exception if there were any problems."""
110    try:
111        reader = urllib2.urlopen(source_url)
112        writer = open(dest_path, 'wb')
113        writer.write(reader.read())
114        writer.close()
115    except BaseException as e:
116        raise Exception(
117            '%s: unable to download source_url %s to dest_path %s' % (
118                e, source_url, dest_path))
119
120def _CreateGSUrl(imagename, hash_type, hash_digest):
121    """Return the HTTP URL we can use to download this particular version of
122    the actually-generated GM image with this imagename.
123
124    imagename: name of the test image, e.g. 'perlinnoise_msaa4.png'
125    hash_type: string indicating the hash type used to generate hash_digest,
126               e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
127    hash_digest: the hash digest of the image to retrieve
128    """
129    return gm_json.CreateGmActualUrl(
130        test_name=IMAGE_FILENAME_RE.match(imagename).group(1),
131        hash_type=hash_type,
132        hash_digest=hash_digest)
133
134def _CallJsonDiff(old_json_path, new_json_path,
135                  old_flattened_dir, new_flattened_dir,
136                  filename_prefix):
137    """Using jsondiff.py, write the images that differ between two GM
138    expectations summary files (old and new) into old_flattened_dir and
139    new_flattened_dir.
140
141    filename_prefix: prefix to prepend to filenames of all images we write
142        into the flattened directories
143    """
144    json_differ = jsondiff.GMDiffer()
145    diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path,
146                                             newfile=new_json_path)
147    print 'Downloading %d before-and-after image pairs...' % len(diff_dict)
148    for (imagename, results) in diff_dict.iteritems():
149        # TODO(epoger): Currently, this assumes that all images have been
150        # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
151
152        old_checksum = results['old']
153        if old_checksum:
154            old_image_url = _CreateGSUrl(
155                imagename=imagename,
156                hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
157                hash_digest=old_checksum)
158            _DownloadUrlToFile(
159                source_url=old_image_url,
160                dest_path=os.path.join(old_flattened_dir,
161                                       filename_prefix + imagename))
162
163        new_checksum = results['new']
164        if new_checksum:
165            new_image_url = _CreateGSUrl(
166                imagename=imagename,
167                hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
168                hash_digest=new_checksum)
169            _DownloadUrlToFile(
170                source_url=new_image_url,
171                dest_path=os.path.join(new_flattened_dir,
172                                       filename_prefix + imagename))
173
174def _RunCommand(args):
175    """Run a command (from self._directory) and return stdout as a single
176    string.
177
178    @param args a list of arguments
179    """
180    proc = subprocess.Popen(args,
181                            stdout=subprocess.PIPE,
182                            stderr=subprocess.PIPE)
183    (stdout, stderr) = proc.communicate()
184    if proc.returncode is not 0:
185        raise Exception('command "%s" failed: %s' % (args, stderr))
186    return stdout
187
188def _GitGetModifiedFiles():
189    """Returns a list of locally modified files within the current working dir.
190
191    TODO(epoger): Move this into a git utility package?
192    """
193    return _RunCommand(['git', 'ls-files', '-m']).splitlines()
194
195def _GitExportBaseVersionOfFile(file_within_repo, dest_path):
196    """Retrieves a copy of the base version of a file within the repository.
197
198    @param file_within_repo path to the file within the repo whose base
199           version you wish to obtain
200    @param dest_path destination to which to write the base content
201
202    TODO(epoger): Move this into a git utility package?
203    """
204    # TODO(epoger): Replace use of "git show" command with lower-level git
205    # commands?  senorblanco points out that "git show" is a "porcelain"
206    # command, intended for human use, as opposed to the "plumbing" commands
207    # generally more suitable for scripting.  (See
208    # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain )
209    #
210    # For now, though, "git show" is the most straightforward implementation
211    # I could come up with.  I tried using "git cat-file", but I had trouble
212    # getting it to work as desired.
213    # Note that git expects / rather than \ as a path separator even on
214    # windows.
215    args = ['git', 'show', posixpath.join('HEAD:.', file_within_repo)]
216    with open(dest_path, 'wb') as outfile:
217        proc = subprocess.Popen(args, stdout=outfile)
218        proc.communicate()
219        if proc.returncode is not 0:
220            raise Exception('command "%s" failed' % args)
221
222def SvnDiff(path_to_skdiff, dest_dir, source_dir):
223    """Generates a visual diff of all pending changes in source_dir.
224
225    @param path_to_skdiff
226    @param dest_dir existing directory within which to write results
227    @param source_dir
228    """
229    # Validate parameters, filling in default values if necessary and possible.
230    path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff))
231    if not dest_dir:
232        dest_dir = tempfile.mkdtemp()
233    dest_dir = os.path.abspath(dest_dir)
234
235    os.chdir(source_dir)
236    svn_repo = svn.Svn('.')
237    using_svn = True
238    try:
239      svn_repo.GetInfo()
240    except:
241      using_svn = False
242
243    # Prepare temporary directories.
244    modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened')
245    original_flattened_dir = os.path.join(dest_dir, 'original_flattened')
246    diff_dir = os.path.join(dest_dir, 'diffs')
247    for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] :
248        shutil.rmtree(dir, ignore_errors=True)
249        os.mkdir(dir)
250
251    # Get a list of all locally modified (including added/deleted) files,
252    # descending subdirectories.
253    if using_svn:
254        modified_file_paths = svn_repo.GetFilesWithStatus(
255            svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
256    else:
257        modified_file_paths = _GitGetModifiedFiles()
258
259    # For each modified file:
260    # 1. copy its current contents into modified_flattened_dir
261    # 2. copy its original contents into original_flattened_dir
262    for modified_file_path in modified_file_paths:
263        if modified_file_path.endswith('.json'):
264            # Special handling for JSON files, in the hopes that they
265            # contain GM result summaries.
266            original_file = tempfile.NamedTemporaryFile(delete = False)
267            original_file.close()
268            if using_svn:
269                svn_repo.ExportBaseVersionOfFile(
270                    modified_file_path, original_file.name)
271            else:
272                _GitExportBaseVersionOfFile(
273                    modified_file_path, original_file.name)
274            modified_dir = os.path.dirname(modified_file_path)
275            platform_prefix = (re.sub(re.escape(os.sep), '__',
276                                      os.path.splitdrive(modified_dir)[1])
277                              + '__')
278            _CallJsonDiff(old_json_path=original_file.name,
279                          new_json_path=modified_file_path,
280                          old_flattened_dir=original_flattened_dir,
281                          new_flattened_dir=modified_flattened_dir,
282                          filename_prefix=platform_prefix)
283            os.remove(original_file.name)
284        else:
285            dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path)
286            # If the file had STATUS_DELETED, it won't exist anymore...
287            if os.path.isfile(modified_file_path):
288                shutil.copyfile(modified_file_path,
289                                os.path.join(modified_flattened_dir,
290                                             dest_filename))
291            if using_svn:
292                svn_repo.ExportBaseVersionOfFile(
293                    modified_file_path,
294                    os.path.join(original_flattened_dir, dest_filename))
295            else:
296                _GitExportBaseVersionOfFile(
297                    modified_file_path,
298                    os.path.join(original_flattened_dir, dest_filename))
299
300    # Run skdiff: compare original_flattened_dir against modified_flattened_dir
301    RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir,
302                                modified_flattened_dir, diff_dir))
303    print '\nskdiff results are ready in file://%s/index.html' % diff_dir
304
305def RaiseUsageException():
306    raise Exception('%s\nRun with --help for more detail.' % (
307        USAGE_STRING % __file__))
308
309def Main(options, args):
310    """Allow other scripts to call this script with fake command-line args.
311    """
312    num_args = len(args)
313    if num_args != 0:
314        RaiseUsageException()
315    SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir,
316            source_dir=options.source_dir)
317
318if __name__ == '__main__':
319    parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
320    parser.add_option(OPTION_DEST_DIR,
321                      action='store', type='string', default=None,
322                      help='existing directory within which to write results; '
323                      'if not set, will create a temporary directory which '
324                      'will remain in place after this script completes')
325    parser.add_option(OPTION_PATH_TO_SKDIFF,
326                      action='store', type='string', default=None,
327                      help='path to already-built skdiff tool; if not set, '
328                      'will search for it in typical directories near this '
329                      'script')
330    parser.add_option(OPTION_SOURCE_DIR,
331                      action='store', type='string',
332                      default=os.path.join('expectations', 'gm'),
333                      help='root directory within which to compare all ' +
334                      'files; defaults to "%default"')
335    (options, args) = parser.parse_args()
336    Main(options, args)
337