1# Copyright (C) 2013 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29"""Moves a directory of LayoutTests.
30
31Given a path to a directory of LayoutTests, moves that directory, including all recursive children,
32to the specified destination path. Updates all references in tests and resources to reflect the new
33location. Also moves any corresponding platform-specific expected results and updates the test
34expectations to reflect the move.
35
36If the destination directory does not exist, it and any missing parent directories are created. If
37the destination directory already exists, the child members of the origin directory are added to the
38destination directory. If any of the child members clash with existing members of the destination
39directory, the move fails.
40
41Note that when new entries are added to the test expectations, no attempt is made to group or merge
42them with existing entries. This should be be done manually and with lint-test-expectations.
43"""
44
45import copy
46import logging
47import optparse
48import os
49import re
50import urlparse
51
52from webkitpy.common.checkout.scm.detection import SCMDetector
53from webkitpy.common.host import Host
54from webkitpy.common.system.executive import Executive
55from webkitpy.common.system.filesystem import FileSystem
56from webkitpy.layout_tests.port.base import Port
57from webkitpy.layout_tests.models.test_expectations import TestExpectations
58
59
60logging.basicConfig()
61_log = logging.getLogger(__name__)
62_log.setLevel(logging.INFO)
63
64PLATFORM_DIRECTORY = 'platform'
65
66class LayoutTestsMover(object):
67
68    def __init__(self, port=None):
69        self._port = port
70        if not self._port:
71            host = Host()
72            # Given that we use include_overrides=False and model_all_expectations=True when
73            # constructing the TestExpectations object, it doesn't matter which Port object we use.
74            self._port = host.port_factory.get()
75            self._port.host.initialize_scm()
76        self._filesystem = self._port.host.filesystem
77        self._scm = self._port.host.scm()
78        self._layout_tests_root = self._port.layout_tests_dir()
79
80    def _scm_path(self, *paths):
81        return self._filesystem.join('LayoutTests', *paths)
82
83    def _is_child_path(self, parent, possible_child):
84        normalized_parent = self._filesystem.normpath(parent)
85        normalized_child = self._filesystem.normpath(possible_child)
86        # We need to add a trailing separator to parent to avoid returning true for cases like
87        # parent='/foo/b', and possible_child='/foo/bar/baz'.
88        return normalized_parent == normalized_child or normalized_child.startswith(normalized_parent + self._filesystem.sep)
89
90    def _move_path(self, path, origin, destination):
91        if not self._is_child_path(origin, path):
92            return path
93        return self._filesystem.normpath(self._filesystem.join(destination, self._filesystem.relpath(path, origin)))
94
95    def _validate_input(self):
96        if not self._filesystem.isdir(self._absolute_origin):
97            raise Exception('Source path %s is not a directory' % self._origin)
98        if not self._is_child_path(self._layout_tests_root, self._absolute_origin):
99            raise Exception('Source path %s is not in LayoutTests directory' % self._origin)
100        if self._filesystem.isfile(self._absolute_destination):
101            raise Exception('Destination path %s is a file' % self._destination)
102        if not self._is_child_path(self._layout_tests_root, self._absolute_destination):
103            raise Exception('Destination path %s is not in LayoutTests directory' % self._destination)
104
105        # If destination is an existing directory, we move the children of origin into destination.
106        # However, if any of the children of origin would clash with existing children of
107        # destination, we fail.
108        # FIXME: Consider adding support for recursively moving into an existing directory.
109        if self._filesystem.isdir(self._absolute_destination):
110            for file_path in self._filesystem.listdir(self._absolute_origin):
111                if self._filesystem.exists(self._filesystem.join(self._absolute_destination, file_path)):
112                    raise Exception('Origin path %s clashes with existing destination path %s' %
113                            (self._filesystem.join(self._origin, file_path), self._filesystem.join(self._destination, file_path)))
114
115    def _get_expectations_for_test(self, model, test_path):
116        """Given a TestExpectationsModel object, finds all expectations that match the specified
117        test, specified as a relative path. Handles the fact that expectations may be keyed by
118        directory.
119        """
120        expectations = set()
121        if model.has_test(test_path):
122            expectations.add(model.get_expectation_line(test_path))
123        test_path = self._filesystem.dirname(test_path)
124        while not test_path == '':
125            # The model requires a trailing slash for directories.
126            test_path_for_model = test_path + '/'
127            if model.has_test(test_path_for_model):
128                expectations.add(model.get_expectation_line(test_path_for_model))
129            test_path = self._filesystem.dirname(test_path)
130        return expectations
131
132    def _get_expectations(self, model, path):
133        """Given a TestExpectationsModel object, finds all expectations for all tests under the
134        specified relative path.
135        """
136        expectations = set()
137        for test in self._filesystem.files_under(self._filesystem.join(self._layout_tests_root, path), dirs_to_skip=['script-tests', 'resources'],
138                                                 file_filter=Port.is_test_file):
139            expectations = expectations.union(self._get_expectations_for_test(model, self._filesystem.relpath(test, self._layout_tests_root)))
140        return expectations
141
142    @staticmethod
143    def _clone_expectation_line_for_path(expectation_line, path):
144        """Clones a TestExpectationLine object and updates the clone to apply to the specified
145        relative path.
146        """
147        clone = copy.copy(expectation_line)
148        clone.original_string = re.compile(expectation_line.name).sub(path, expectation_line.original_string)
149        clone.name = path
150        clone.path = path
151        # FIXME: Should we search existing expectations for matches, like in
152        # TestExpectationsParser._collect_matching_tests()?
153        clone.matching_tests = [path]
154        return clone
155
156    def _update_expectations(self):
157        """Updates all test expectations that are affected by the move.
158        """
159        _log.info('Updating expectations')
160        test_expectations = TestExpectations(self._port, include_overrides=False, model_all_expectations=True)
161
162        for expectation in self._get_expectations(test_expectations.model(), self._origin):
163            path = expectation.path
164            if self._is_child_path(self._origin, path):
165                # If the existing expectation is a child of the moved path, we simply replace it
166                # with an expectation for the updated path.
167                new_path = self._move_path(path, self._origin, self._destination)
168                _log.debug('Updating expectation for %s to %s' % (path, new_path))
169                test_expectations.remove_expectation_line(path)
170                test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path))
171            else:
172                # If the existing expectation is not a child of the moved path, we have to leave it
173                # in place. But we also add a new expectation for the destination path.
174                new_path = self._destination
175                _log.warning('Copying expectation for %s to %s. You should check that these expectations are still correct.' %
176                             (path, new_path))
177                test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path))
178
179        expectations_file = self._port.path_to_generic_test_expectations_file()
180        self._filesystem.write_text_file(expectations_file,
181                                         TestExpectations.list_to_string(test_expectations._expectations, reconstitute_only_these=[]))
182        self._scm.add(self._filesystem.relpath(expectations_file, self._scm.checkout_root))
183
184    def _find_references(self, input_files):
185        """Attempts to find all references to other files in the supplied list of files. Returns a
186        dictionary that maps from an absolute file path to an array of reference strings.
187        """
188        reference_regex = re.compile(r'(?:(?:src=|href=|importScripts\(|url\()(?:"([^"]+)"|\'([^\']+)\')|url\(([^\)\'"]+)\))')
189        references = {}
190        for input_file in input_files:
191            matches = reference_regex.findall(self._filesystem.read_binary_file(input_file))
192            if matches:
193                references[input_file] = [filter(None, match)[0] for match in matches]
194        return references
195
196    def _get_updated_reference(self, root, reference):
197        """For a reference <reference> in a directory <root>, determines the updated reference.
198        Returns the the updated reference, or None if no update is required.
199        """
200        # If the reference is an absolute path or url, it's safe.
201        if reference.startswith('/') or urlparse.urlparse(reference).scheme:
202            return None
203
204        # Both the root path and the target of the reference my be subject to the move, so there are
205        # four cases to consider. In the case where both or neither are subject to the move, the
206        # reference doesn't need updating.
207        #
208        # This is true even if the reference includes superfluous dot segments which mention a moved
209        # directory, as dot segments are collapsed during URL normalization. For example, if
210        # foo.html contains a reference 'bar/../script.js', this remains valid (though ugly) even if
211        # bar/ is moved to baz/, because the reference is always normalized to 'script.js'.
212        absolute_reference = self._filesystem.normpath(self._filesystem.join(root, reference))
213        if self._is_child_path(self._absolute_origin, root) == self._is_child_path(self._absolute_origin, absolute_reference):
214            return None;
215
216        new_root = self._move_path(root, self._absolute_origin, self._absolute_destination)
217        new_absolute_reference = self._move_path(absolute_reference, self._absolute_origin, self._absolute_destination)
218        return self._filesystem.relpath(new_absolute_reference, new_root)
219
220    def _get_all_updated_references(self, references):
221        """Determines the updated references due to the move. Returns a dictionary that maps from an
222        absolute file path to a dictionary that maps from a reference string to the corresponding
223        updated reference.
224        """
225        updates = {}
226        for file_path in references.keys():
227            root = self._filesystem.dirname(file_path)
228            # sript-tests/TEMPLATE.html files contain references which are written as if the file
229            # were in the parent directory. This special-casing is ugly, but there are plans to
230            # remove script-tests.
231            if root.endswith('script-tests') and file_path.endswith('TEMPLATE.html'):
232                root = self._filesystem.dirname(root)
233            local_updates = {}
234            for reference in references[file_path]:
235                update = self._get_updated_reference(root, reference)
236                if update:
237                    local_updates[reference] = update
238            if local_updates:
239                updates[file_path] = local_updates
240        return updates
241
242    def _update_file(self, path, updates):
243        contents = self._filesystem.read_binary_file(path)
244        # Note that this regex isn't quite as strict as that used to find the references, but this
245        # avoids the need for alternative match groups, which simplifies things.
246        for target in updates.keys():
247            regex = re.compile(r'((?:src=|href=|importScripts\(|url\()["\']?)%s(["\']?)' % target)
248            contents = regex.sub(r'\1%s\2' % updates[target], contents)
249        self._filesystem.write_binary_file(path, contents)
250        self._scm.add(path)
251
252    def _update_test_source_files(self):
253        def is_test_source_file(filesystem, dirname, basename):
254            pass_regex = re.compile(r'\.(css|js)$')
255            fail_regex = re.compile(r'-expected\.')
256            return (Port.is_test_file(filesystem, dirname, basename) or pass_regex.search(basename)) and not fail_regex.search(basename)
257
258        test_source_files = self._filesystem.files_under(self._layout_tests_root, file_filter=is_test_source_file)
259        _log.info('Considering %s test source files for references' % len(test_source_files))
260        references = self._find_references(test_source_files)
261        _log.info('Considering references in %s files' % len(references))
262        updates = self._get_all_updated_references(references)
263        _log.info('Updating references in %s files' % len(updates))
264        count = 0
265        for file_path in updates.keys():
266            self._update_file(file_path, updates[file_path])
267            count += 1
268            if count % 1000 == 0 or count == len(updates):
269                _log.debug('Updated references in %s files' % count)
270
271    def _move_directory(self, origin, destination):
272        """Moves the directory <origin> to <destination>. If <destination> is a directory, moves the
273        children of <origin> into <destination>. Uses relative paths.
274        """
275        absolute_origin = self._filesystem.join(self._layout_tests_root, origin)
276        if not self._filesystem.isdir(absolute_origin):
277            return
278        _log.info('Moving directory %s to %s' % (origin, destination))
279        # Note that FileSystem.move() may silently overwrite existing files, but we
280        # check for this in _validate_input().
281        absolute_destination = self._filesystem.join(self._layout_tests_root, destination)
282        self._filesystem.maybe_make_directory(absolute_destination)
283        for directory in self._filesystem.listdir(absolute_origin):
284            self._scm.move(self._scm_path(origin, directory), self._scm_path(destination, directory))
285        self._filesystem.rmtree(absolute_origin)
286
287    def _move_files(self):
288        """Moves the all files that correspond to the move, including platform-specific expected
289        results.
290        """
291        self._move_directory(self._origin, self._destination)
292        for directory in self._filesystem.listdir(self._filesystem.join(self._layout_tests_root, PLATFORM_DIRECTORY)):
293            self._move_directory(self._filesystem.join(PLATFORM_DIRECTORY, directory, self._origin),
294                           self._filesystem.join(PLATFORM_DIRECTORY, directory, self._destination))
295
296    def _commit_changes(self):
297        if not self._scm.supports_local_commits():
298            return
299        title = 'Move LayoutTests directory %s to %s' % (self._origin, self._destination)
300        _log.info('Committing change \'%s\'' % title)
301        self._scm.commit_locally_with_message('%s\n\nThis commit was automatically generated by move-layout-tests.' % title,
302                                              commit_all_working_directory_changes=False)
303
304    def move(self, origin, destination):
305        self._origin = origin
306        self._destination = destination
307        self._absolute_origin = self._filesystem.join(self._layout_tests_root, self._origin)
308        self._absolute_destination = self._filesystem.join(self._layout_tests_root, self._destination)
309        self._validate_input()
310        self._update_expectations()
311        self._update_test_source_files()
312        self._move_files()
313        # FIXME: Handle virtual test suites.
314        self._commit_changes()
315
316def main(argv):
317    parser = optparse.OptionParser(description=__doc__)
318    parser.add_option('--origin',
319                      help=('The directory of tests to move, as a relative path from the LayoutTests directory.'))
320    parser.add_option('--destination',
321                      help=('The new path for the directory of tests, as a relative path from the LayoutTests directory.'))
322    options, _ = parser.parse_args()
323    LayoutTestsMover().move(options.origin, options.destination)
324