153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# Copyright (C) 2013 Google Inc. All rights reserved. 253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# 353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# Redistribution and use in source and binary forms, with or without 453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# modification, are permitted provided that the following conditions are 553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# met: 653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# 753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# * Redistributions of source code must retain the above copyright 853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# notice, this list of conditions and the following disclaimer. 953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# * Redistributions in binary form must reproduce the above 1053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 1153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# in the documentation and/or other materials provided with the 1253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# distribution. 1353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 1453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# contributors may be used to endorse or promote products derived from 1553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# this software without specific prior written permission. 1653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# 1753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 2953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)"""Moves a directory of LayoutTests. 3053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 3153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)Given a path to a directory of LayoutTests, moves that directory, including all recursive children, 3253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)to the specified destination path. Updates all references in tests and resources to reflect the new 3353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)location. Also moves any corresponding platform-specific expected results and updates the test 3453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)expectations to reflect the move. 3553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 3653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)If the destination directory does not exist, it and any missing parent directories are created. If 3753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)the destination directory already exists, the child members of the origin directory are added to the 3853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)destination directory. If any of the child members clash with existing members of the destination 3953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)directory, the move fails. 4053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 4153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)Note that when new entries are added to the test expectations, no attempt is made to group or merge 4253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)them with existing entries. This should be be done manually and with lint-test-expectations. 4353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)""" 4453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 4553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)import copy 4653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)import logging 475267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)import optparse 4853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)import os 4953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)import re 5053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)import urlparse 5153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 5253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)from webkitpy.common.checkout.scm.detection import SCMDetector 5353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)from webkitpy.common.host import Host 5453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)from webkitpy.common.system.executive import Executive 5553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)from webkitpy.common.system.filesystem import FileSystem 5653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)from webkitpy.layout_tests.port.base import Port 5753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)from webkitpy.layout_tests.models.test_expectations import TestExpectations 5853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 5953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 6053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)logging.basicConfig() 6153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)_log = logging.getLogger(__name__) 6253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)_log.setLevel(logging.INFO) 6353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 6453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)PLATFORM_DIRECTORY = 'platform' 6553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 6653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)class LayoutTestsMover(object): 6753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 6853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def __init__(self, port=None): 6953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._port = port 7053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if not self._port: 7153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) host = Host() 7253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # Given that we use include_overrides=False and model_all_expectations=True when 7353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # constructing the TestExpectations object, it doesn't matter which Port object we use. 7453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._port = host.port_factory.get() 7553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._port.host.initialize_scm() 7653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._filesystem = self._port.host.filesystem 7753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._scm = self._port.host.scm() 7853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._layout_tests_root = self._port.layout_tests_dir() 7953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 8053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _scm_path(self, *paths): 8153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return self._filesystem.join('LayoutTests', *paths) 8253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 8353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _is_child_path(self, parent, possible_child): 8453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) normalized_parent = self._filesystem.normpath(parent) 8553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) normalized_child = self._filesystem.normpath(possible_child) 8653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # We need to add a trailing separator to parent to avoid returning true for cases like 8753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # parent='/foo/b', and possible_child='/foo/bar/baz'. 8853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return normalized_parent == normalized_child or normalized_child.startswith(normalized_parent + self._filesystem.sep) 8953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 9053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _move_path(self, path, origin, destination): 9153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if not self._is_child_path(origin, path): 9253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return path 9353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return self._filesystem.normpath(self._filesystem.join(destination, self._filesystem.relpath(path, origin))) 9453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 9553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _validate_input(self): 9653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if not self._filesystem.isdir(self._absolute_origin): 9753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) raise Exception('Source path %s is not a directory' % self._origin) 9853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if not self._is_child_path(self._layout_tests_root, self._absolute_origin): 9953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) raise Exception('Source path %s is not in LayoutTests directory' % self._origin) 10053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if self._filesystem.isfile(self._absolute_destination): 10153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) raise Exception('Destination path %s is a file' % self._destination) 10253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if not self._is_child_path(self._layout_tests_root, self._absolute_destination): 10353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) raise Exception('Destination path %s is not in LayoutTests directory' % self._destination) 10453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 10553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # If destination is an existing directory, we move the children of origin into destination. 10653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # However, if any of the children of origin would clash with existing children of 10753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # destination, we fail. 10853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # FIXME: Consider adding support for recursively moving into an existing directory. 10953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if self._filesystem.isdir(self._absolute_destination): 11053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for file_path in self._filesystem.listdir(self._absolute_origin): 11153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if self._filesystem.exists(self._filesystem.join(self._absolute_destination, file_path)): 11253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) raise Exception('Origin path %s clashes with existing destination path %s' % 11353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) (self._filesystem.join(self._origin, file_path), self._filesystem.join(self._destination, file_path))) 11453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 11553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _get_expectations_for_test(self, model, test_path): 11653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Given a TestExpectationsModel object, finds all expectations that match the specified 11753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test, specified as a relative path. Handles the fact that expectations may be keyed by 11853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) directory. 11953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 12053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) expectations = set() 12153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if model.has_test(test_path): 12253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) expectations.add(model.get_expectation_line(test_path)) 12353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_path = self._filesystem.dirname(test_path) 12453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) while not test_path == '': 12553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # The model requires a trailing slash for directories. 12653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_path_for_model = test_path + '/' 12753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if model.has_test(test_path_for_model): 12853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) expectations.add(model.get_expectation_line(test_path_for_model)) 12953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_path = self._filesystem.dirname(test_path) 13053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return expectations 13153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 13253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _get_expectations(self, model, path): 13353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Given a TestExpectationsModel object, finds all expectations for all tests under the 13453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) specified relative path. 13553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 13653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) expectations = set() 13753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for test in self._filesystem.files_under(self._filesystem.join(self._layout_tests_root, path), dirs_to_skip=['script-tests', 'resources'], 13853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) file_filter=Port.is_test_file): 13953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) expectations = expectations.union(self._get_expectations_for_test(model, self._filesystem.relpath(test, self._layout_tests_root))) 14053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return expectations 14153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 14253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) @staticmethod 14353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _clone_expectation_line_for_path(expectation_line, path): 14453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Clones a TestExpectationLine object and updates the clone to apply to the specified 14553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) relative path. 14653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 14753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) clone = copy.copy(expectation_line) 14853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) clone.original_string = re.compile(expectation_line.name).sub(path, expectation_line.original_string) 14953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) clone.name = path 15053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) clone.path = path 15153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # FIXME: Should we search existing expectations for matches, like in 15253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # TestExpectationsParser._collect_matching_tests()? 15353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) clone.matching_tests = [path] 15453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return clone 15553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 15653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _update_expectations(self): 15753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Updates all test expectations that are affected by the move. 15853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 15953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.info('Updating expectations') 16053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_expectations = TestExpectations(self._port, include_overrides=False, model_all_expectations=True) 16153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 16253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for expectation in self._get_expectations(test_expectations.model(), self._origin): 16353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) path = expectation.path 16453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if self._is_child_path(self._origin, path): 16553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # If the existing expectation is a child of the moved path, we simply replace it 16653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # with an expectation for the updated path. 16753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) new_path = self._move_path(path, self._origin, self._destination) 16853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.debug('Updating expectation for %s to %s' % (path, new_path)) 16953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_expectations.remove_expectation_line(path) 17053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path)) 17153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) else: 17253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # If the existing expectation is not a child of the moved path, we have to leave it 17353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # in place. But we also add a new expectation for the destination path. 17453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) new_path = self._destination 17553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.warning('Copying expectation for %s to %s. You should check that these expectations are still correct.' % 17653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) (path, new_path)) 17753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path)) 17853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 17953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) expectations_file = self._port.path_to_generic_test_expectations_file() 18053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._filesystem.write_text_file(expectations_file, 18153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) TestExpectations.list_to_string(test_expectations._expectations, reconstitute_only_these=[])) 18253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._scm.add(self._filesystem.relpath(expectations_file, self._scm.checkout_root)) 18353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 18453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _find_references(self, input_files): 18553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Attempts to find all references to other files in the supplied list of files. Returns a 18653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) dictionary that maps from an absolute file path to an array of reference strings. 18753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 188e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) reference_regex = re.compile(r'(?:(?:src=|href=|importScripts\(|url\()(?:"([^"]+)"|\'([^\']+)\')|url\(([^\)\'"]+)\))') 18953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) references = {} 19053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for input_file in input_files: 19153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) matches = reference_regex.findall(self._filesystem.read_binary_file(input_file)) 19253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if matches: 193e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) references[input_file] = [filter(None, match)[0] for match in matches] 19453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return references 19553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 19653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _get_updated_reference(self, root, reference): 19753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """For a reference <reference> in a directory <root>, determines the updated reference. 19853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) Returns the the updated reference, or None if no update is required. 19953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 200e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) # If the reference is an absolute path or url, it's safe. 201e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) if reference.startswith('/') or urlparse.urlparse(reference).scheme: 20253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return None 20353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 20453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # Both the root path and the target of the reference my be subject to the move, so there are 20553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # four cases to consider. In the case where both or neither are subject to the move, the 20653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # reference doesn't need updating. 20753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # 20853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # This is true even if the reference includes superfluous dot segments which mention a moved 20953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # directory, as dot segments are collapsed during URL normalization. For example, if 21053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # foo.html contains a reference 'bar/../script.js', this remains valid (though ugly) even if 21153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # bar/ is moved to baz/, because the reference is always normalized to 'script.js'. 21253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) absolute_reference = self._filesystem.normpath(self._filesystem.join(root, reference)) 21353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if self._is_child_path(self._absolute_origin, root) == self._is_child_path(self._absolute_origin, absolute_reference): 21453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return None; 21553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 21653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) new_root = self._move_path(root, self._absolute_origin, self._absolute_destination) 21753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) new_absolute_reference = self._move_path(absolute_reference, self._absolute_origin, self._absolute_destination) 21853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return self._filesystem.relpath(new_absolute_reference, new_root) 21953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 22053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _get_all_updated_references(self, references): 22153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Determines the updated references due to the move. Returns a dictionary that maps from an 22253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) absolute file path to a dictionary that maps from a reference string to the corresponding 22353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) updated reference. 22453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 22553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) updates = {} 22653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for file_path in references.keys(): 22753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) root = self._filesystem.dirname(file_path) 22853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # sript-tests/TEMPLATE.html files contain references which are written as if the file 22953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # were in the parent directory. This special-casing is ugly, but there are plans to 23053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # remove script-tests. 23153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if root.endswith('script-tests') and file_path.endswith('TEMPLATE.html'): 23253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) root = self._filesystem.dirname(root) 23353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) local_updates = {} 23453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for reference in references[file_path]: 23553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) update = self._get_updated_reference(root, reference) 23653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if update: 23753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) local_updates[reference] = update 23853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if local_updates: 23953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) updates[file_path] = local_updates 24053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return updates 24153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 24253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _update_file(self, path, updates): 24353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) contents = self._filesystem.read_binary_file(path) 244e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) # Note that this regex isn't quite as strict as that used to find the references, but this 245e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) # avoids the need for alternative match groups, which simplifies things. 24653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for target in updates.keys(): 247e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) regex = re.compile(r'((?:src=|href=|importScripts\(|url\()["\']?)%s(["\']?)' % target) 24853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) contents = regex.sub(r'\1%s\2' % updates[target], contents) 24953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._filesystem.write_binary_file(path, contents) 25053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._scm.add(path) 25153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 25253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _update_test_source_files(self): 25353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def is_test_source_file(filesystem, dirname, basename): 25453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) pass_regex = re.compile(r'\.(css|js)$') 25553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) fail_regex = re.compile(r'-expected\.') 25653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return (Port.is_test_file(filesystem, dirname, basename) or pass_regex.search(basename)) and not fail_regex.search(basename) 25753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 25853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) test_source_files = self._filesystem.files_under(self._layout_tests_root, file_filter=is_test_source_file) 25953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.info('Considering %s test source files for references' % len(test_source_files)) 26053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) references = self._find_references(test_source_files) 26153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.info('Considering references in %s files' % len(references)) 26253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) updates = self._get_all_updated_references(references) 26353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.info('Updating references in %s files' % len(updates)) 26453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) count = 0 26553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for file_path in updates.keys(): 26653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._update_file(file_path, updates[file_path]) 26753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) count += 1 26853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if count % 1000 == 0 or count == len(updates): 26953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.debug('Updated references in %s files' % count) 27053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 27153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _move_directory(self, origin, destination): 27253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Moves the directory <origin> to <destination>. If <destination> is a directory, moves the 27353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) children of <origin> into <destination>. Uses relative paths. 27453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 27553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) absolute_origin = self._filesystem.join(self._layout_tests_root, origin) 27653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if not self._filesystem.isdir(absolute_origin): 27753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return 27853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.info('Moving directory %s to %s' % (origin, destination)) 27953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # Note that FileSystem.move() may silently overwrite existing files, but we 28053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # check for this in _validate_input(). 28153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) absolute_destination = self._filesystem.join(self._layout_tests_root, destination) 28253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._filesystem.maybe_make_directory(absolute_destination) 28353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for directory in self._filesystem.listdir(absolute_origin): 28453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._scm.move(self._scm_path(origin, directory), self._scm_path(destination, directory)) 28553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._filesystem.rmtree(absolute_origin) 28653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 28753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _move_files(self): 28853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """Moves the all files that correspond to the move, including platform-specific expected 28953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) results. 29053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) """ 29153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._move_directory(self._origin, self._destination) 29253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) for directory in self._filesystem.listdir(self._filesystem.join(self._layout_tests_root, PLATFORM_DIRECTORY)): 29353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._move_directory(self._filesystem.join(PLATFORM_DIRECTORY, directory, self._origin), 29453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._filesystem.join(PLATFORM_DIRECTORY, directory, self._destination)) 29553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 29653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def _commit_changes(self): 29753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) if not self._scm.supports_local_commits(): 29853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return 29953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) title = 'Move LayoutTests directory %s to %s' % (self._origin, self._destination) 30053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) _log.info('Committing change \'%s\'' % title) 30153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._scm.commit_locally_with_message('%s\n\nThis commit was automatically generated by move-layout-tests.' % title, 30253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) commit_all_working_directory_changes=False) 30353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 30453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def move(self, origin, destination): 30553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._origin = origin 30653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._destination = destination 30753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._absolute_origin = self._filesystem.join(self._layout_tests_root, self._origin) 30853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._absolute_destination = self._filesystem.join(self._layout_tests_root, self._destination) 30953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._validate_input() 31053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._update_expectations() 31153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._update_test_source_files() 31253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._move_files() 31353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # FIXME: Handle virtual test suites. 31453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._commit_changes() 31553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 31653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)def main(argv): 3175267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) parser = optparse.OptionParser(description=__doc__) 3185267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) parser.add_option('--origin', 3195267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) help=('The directory of tests to move, as a relative path from the LayoutTests directory.')) 3205267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) parser.add_option('--destination', 3215267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) help=('The new path for the directory of tests, as a relative path from the LayoutTests directory.')) 3225267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) options, _ = parser.parse_args() 3235267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) LayoutTestsMover().move(options.origin, options.destination) 324