1'''
2Downloads the actual gm results most recently generated by the Skia buildbots,
3and adds any new ones to SVN control.
4
5Launch with --help to see more information.
6
7
8Copyright 2011 Google Inc.
9
10Use of this source code is governed by a BSD-style license that can be
11found in the LICENSE file.
12'''
13
14# common Python modules
15import fnmatch
16import optparse
17import os
18import re
19import shutil
20import sys
21import tempfile
22
23# modules declared within this same directory
24import compare_baselines
25import svn
26
27USAGE_STRING = 'Usage: %s [options] [baseline_subdir]...'
28HELP_STRING = '''
29
30Downloads the actual gm results most recently generated by the Skia buildbots,
31and adds any new ones to SVN control.
32
33If no baseline_subdir is given, then this tool will download the most-recently
34generated actual gm results for ALL platforms.
35
36''' + compare_baselines.HOWTO_STRING
37
38# Base URL of SVN repository where buildbots store actual gm image results.
39GM_ACTUAL_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
40
41# GM baseline image URL in regular Skia SVN repository
42GM_BASELINE_URL = 'https://skia.googlecode.com/svn/gm-expected'
43
44GM_EXPECTED_DIR = 'gm-expected'
45
46OPTION_ADD_NEW_FILES = '--add-new-files'
47OPTION_BUILDER_SUFFIX = '--builder-suffix'
48DEFAULT_BUILDER_SUFFIX = '32'
49OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods'
50
51def GetLatestResultsSvnUrl(svn, baseline_subdir, builder_suffix):
52    """Return SVN URL from which we can check out the MOST RECENTLY generated images for this
53    baseline type.
54
55    @param svn an Svn object we can use to call ListSubdirs()
56    @param baseline_subdir indicates which platform we want images for
57    @param builder_suffix if multiple builders uploaded actual GM images for this baseline type,
58           choose the one whose builder_name matches this suffix
59    """
60    root_url = '%s/%s' % (GM_ACTUAL_URL, baseline_subdir)
61    subdirs = sorted(svn.ListSubdirs(root_url))
62    num_subdirs = len(subdirs)
63    print('Within actual-results root URL %s, found these %d subdirs (presumably builder_names): %s'
64          % (root_url, num_subdirs, subdirs))
65
66    selected_subdir = None
67    if num_subdirs == 0:
68        print 'Found no builder_name subdirs, so reading actual images from the root_url itself.'
69        return root_url
70    elif num_subdirs == 1:
71        selected_subdir = subdirs[0]
72        print 'Found exactly one subdir in actual-results root_url: %s' % selected_subdir
73    else:
74        for possible_subdir in subdirs:
75            if possible_subdir.endswith(builder_suffix):
76                selected_subdir = possible_subdir
77                print 'Selected the first subdir ending in "%s": %s' % (
78                    builder_suffix, selected_subdir)
79                break
80
81    if selected_subdir:
82        return '%s/%s/%s' % (root_url, selected_subdir, baseline_subdir)
83    else:
84        raise Exception('none of these subdirs of %s ended in "%s": %s' % (
85            root_url, builder_suffix, subdirs))
86
87def GetBaselineSvnUrl(baseline_subdir):
88    """Return SVN URL from which we can check out the baseline images for this
89    baseline type.
90
91    @param baseline_subdir indicates which platform we want baselines for
92    """
93    return '%s/%s' % (GM_BASELINE_URL, baseline_subdir)
94
95def CopyMatchingFiles(source_dir, dest_dir, filename_pattern, only_copy_updates=False):
96    """Copy all files from source_dir that match filename_pattern, and save them (with their
97    original filenames) in dest_dir.
98
99    @param source_dir
100    @param dest_dir where to save the copied files
101    @param filename_pattern only copy files that match this Unix-style filename
102           pattern (e.g., '*.jpg')
103    @param only_copy_updates if True, only copy files that are already present in dest_dir
104    """
105    all_filenames = os.listdir(source_dir)
106    matching_filenames = fnmatch.filter(all_filenames, filename_pattern)
107    for filename in matching_filenames:
108        source_path = os.path.join(source_dir, filename)
109        dest_path = os.path.join(dest_dir, filename)
110        if only_copy_updates and not os.path.isfile(dest_path):
111            continue
112        shutil.copyfile(source_path, dest_path)
113
114def DownloadBaselinesForOnePlatform(baseline_subdir):
115    """Download most recently generated baseline images for a single platform,
116    and add any new ones to SVN control.
117
118    @param baseline_subdir
119    """
120    # Create repo_to_modify to handle the SVN repository we will add files to.
121    gm_dir = os.path.join(os.pardir, GM_EXPECTED_DIR) # Shouldn't assume we're in trunk...
122    try:
123        os.makedirs(gm_dir)
124    except:
125        pass
126    repo_to_modify = svn.Svn(gm_dir)
127    repo_to_modify.Checkout(GetBaselineSvnUrl(baseline_subdir), baseline_subdir)
128
129    # If there are any locally modified files in that directory, exit
130    # (so that we don't risk overwriting the user's previous work).
131    new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
132    if not options.ignore_local_mods:
133        if new_and_modified_files:
134            raise Exception('Exiting because there are already new and/or '
135                            'modified files in %s.  To continue in spite of '
136                            'that, run with %s option.' % (
137                                baseline_subdir, OPTION_IGNORE_LOCAL_MODS))
138
139    # Download actual gm images into a separate repo in a temporary directory.
140    actual_dir = tempfile.mkdtemp()
141    actual_repo = svn.Svn(actual_dir)
142    print 'Using %s as a temp dir' % actual_dir
143    actual_url = GetLatestResultsSvnUrl(svn=actual_repo, baseline_subdir=baseline_subdir,
144                                        builder_suffix=options.builder_suffix)
145    print 'Reading actual buildbot GM results from %s' % actual_url
146    actual_repo.Checkout(actual_url, '.')
147
148    # Copy any of those files we are interested in into repo_to_modify,
149    # and then delete the temporary directory.
150    CopyMatchingFiles(source_dir=actual_dir,
151                      dest_dir=os.path.join(gm_dir, baseline_subdir),
152                      filename_pattern='*.png',
153                      only_copy_updates=(not options.add_new_files))
154    shutil.rmtree(actual_dir)
155    actual_repo = None
156
157    # Add any new files to SVN control (if we are running with add_new_files).
158    if options.add_new_files:
159        new_files = repo_to_modify.GetNewFiles()
160        if new_files:
161            repo_to_modify.AddFiles(sorted(new_files))
162
163    # Set the mimetype property on any new/modified image files in
164    # baseline_subdir.  (We used to set the mimetype property on *all* image
165    # files in the directory, even those whose content wasn't changing,
166    # but that caused confusion.  See
167    # http://code.google.com/p/skia/issues/detail?id=618 .)
168    modified_files = repo_to_modify.GetNewAndModifiedFiles()
169    repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')),
170                               svn.PROPERTY_MIMETYPE, 'image/png')
171    repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')),
172                               svn.PROPERTY_MIMETYPE, 'application/pdf')
173
174def RaiseUsageException():
175    raise Exception('%s\nRun with --help for more detail.' % (
176        USAGE_STRING % __file__))
177
178def Main(options, args):
179    """Allow other scripts to call this script with fake command-line args.
180    """
181    # If no platforms are specified, do 'em all.
182    num_args = len(args)
183    if num_args == 0:
184        # TODO(epoger): automate the default set of platforms. We want to ensure
185        # that the user gets all of the platforms that the bots are running,
186        # not just whatever subdirectories he happens to have checked out...
187        # See http://code.google.com/p/skia/issues/detail?id=678
188        # Now that we have added Svn.ListSubdirs(), we should be able to do this
189        # pretty easily...
190        #
191        # For now, I generate this list using these Unix commands:
192        # svn ls http://skia.googlecode.com/svn/gm-expected | grep ^base | sort >/tmp/baselines
193        # svn ls http://skia-autogen.googlecode.com/svn/gm-actual | grep ^base | sort >/tmp/actual
194        # comm -1 -2 /tmp/baselines /tmp/actual
195        args = [
196            'base-android-galaxy-nexus',
197            'base-android-nexus-7',
198            'base-android-nexus-s',
199            'base-android-xoom',
200            'base-macmini',
201            'base-macmini-lion-float',
202            'base-shuttle-win7-intel-float',
203            'base-shuttle_ubuntu12_ati5770',
204            'base-shuttle-win7-intel-angle',
205            'base-shuttle-win7-intel-directwrite',
206            ]
207
208    # Trim all subdir names.
209    baseline_subdirs = []
210    for arg in args:
211        baseline_subdirs.append(arg.rstrip(os.sep))
212
213    # Process the subdirs, one at a time.
214    for baseline_subdir in baseline_subdirs:
215        DownloadBaselinesForOnePlatform(baseline_subdir=baseline_subdir)
216
217if __name__ == '__main__':
218    parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
219    parser.add_option(OPTION_ADD_NEW_FILES,
220                      action='store_true', default=False,
221                      help='in addition to downloading new versions of '
222                      'existing baselines, also download baselines that are '
223                      'not under SVN control yet')
224    parser.add_option(OPTION_BUILDER_SUFFIX,
225                      action='store', type='string', default=DEFAULT_BUILDER_SUFFIX,
226                      help='if multiple builders have uploaded actual GM images '
227                      'for this platform, download the images uploaded by the '
228                      'builder whose name ends in this suffix; defaults to '
229                      '"%s".' % DEFAULT_BUILDER_SUFFIX)
230    parser.add_option(OPTION_IGNORE_LOCAL_MODS,
231                      action='store_true', default=False,
232                      help='allow tool to run even if there are already '
233                      'local modifications in the baseline_subdir')
234    (options, args) = parser.parse_args()
235    Main(options, args)
236