filesystem.py revision ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddb
1dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# Copyright (C) 2010 Google Inc. All rights reserved. 2dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# 3dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# Redistribution and use in source and binary forms, with or without 4dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# modification, are permitted provided that the following conditions are 5dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# met: 6dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# 7dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# * Redistributions of source code must retain the above copyright 8dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# notice, this list of conditions and the following disclaimer. 9dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# * Redistributions in binary form must reproduce the above 10dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# copyright notice, this list of conditions and the following disclaimer 11dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# in the documentation and/or other materials provided with the 12dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# distribution. 13dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# * Neither the name of Google Inc. nor the names of its 14dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# contributors may be used to endorse or promote products derived from 15dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# this software without specific prior written permission. 16dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# 17dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 29dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty"""Wrapper object for the file system / source tree.""" 30dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 31dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyfrom __future__ import with_statement 32dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 33dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport codecs 34dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport errno 35dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport exceptions 36dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport glob 37dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport os 38dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport shutil 39dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport tempfile 40dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyimport time 41dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 42dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Doughertyclass FileSystem(object): 43dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """FileSystem interface for webkitpy. 44dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 45dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty Unless otherwise noted, all paths are allowed to be either absolute 46dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty or relative.""" 47dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def __init__(self): 48dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty self.sep = os.sep 49dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 50dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def abspath(self, path): 51dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.abspath(path) 52dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 53dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def basename(self, path): 54dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Wraps os.path.basename().""" 55dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.basename(path) 56dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 577b756e62462b6f6b0d35a878debe0891a386b523Dirk Dougherty def copyfile(self, source, destination): 58dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Copies the contents of the file at the given path to the destination 59dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty path.""" 60dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty shutil.copyfile(source, destination) 61dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 62dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def dirname(self, path): 63dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Wraps os.path.dirname().""" 64dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.dirname(path) 65dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 66dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def exists(self, path): 67dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Return whether the path exists in the filesystem.""" 68dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.exists(path) 69dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 70dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def files_under(self, path, dirs_to_skip=[], file_filter=None): 71dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Return the list of all files under the given path in topdown order. 72dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 73dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty Args: 74dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty dirs_to_skip: a list of directories to skip over during the 75dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty traversal (e.g., .svn, resources, etc.) 76dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty file_filter: if not None, the filter will be invoked 77dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty with the filesystem object and the dirname and basename of 78dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty each file found. The file is included in the result if the 79dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty callback returns True. 80dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """ 81dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def filter_all(fs, dirpath, basename): 82dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return True 83dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 84dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty file_filter = file_filter or filter_all 85dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty files = [] 86dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty if self.isfile(path): 87dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty if file_filter(self, self.dirname(path), self.basename(path)): 88dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty files.append(path) 89dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return files 90dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 91dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty if self.basename(path) in dirs_to_skip: 92dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return [] 93dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 94dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty for (dirpath, dirnames, filenames) in os.walk(path): 95dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty for d in dirs_to_skip: 96dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty if d in dirnames: 97dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty dirnames.remove(d) 98dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 99dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty for filename in filenames: 100dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty if file_filter(self, dirpath, filename): 101dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty files.append(self.join(dirpath, filename)) 102dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return files 103dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 104dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def glob(self, path): 105dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Wraps glob.glob().""" 106dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return glob.glob(path) 107dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 108dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def isabs(self, path): 109dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Return whether the path is an absolute path.""" 110dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.isabs(path) 111b4c6c1b218dc0c044b40c5132ae890a3a6910f69Dirk Dougherty 112dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def isfile(self, path): 113dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Return whether the path refers to a file.""" 114dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.isfile(path) 115dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 116dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def isdir(self, path): 117dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Return whether the path refers to a directory.""" 118dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.isdir(path) 119dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty 120dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty def join(self, *comps): 121dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty """Return the path formed by joining the components.""" 122dbccc9012e3637ad08e8ae62771b0f9fae1d9b13Dirk Dougherty return os.path.join(*comps) 123 124 def listdir(self, path): 125 """Return the contents of the directory pointed to by path.""" 126 return os.listdir(path) 127 128 def mkdtemp(self, **kwargs): 129 """Create and return a uniquely named directory. 130 131 This is like tempfile.mkdtemp, but if used in a with statement 132 the directory will self-delete at the end of the block (if the 133 directory is empty; non-empty directories raise errors). The 134 directory can be safely deleted inside the block as well, if so 135 desired. 136 137 Note that the object returned is not a string and does not support all of the string 138 methods. If you need a string, coerce the object to a string and go from there. 139 """ 140 class TemporaryDirectory(object): 141 def __init__(self, **kwargs): 142 self._kwargs = kwargs 143 self._directory_path = tempfile.mkdtemp(**self._kwargs) 144 145 def __str__(self): 146 return self._directory_path 147 148 def __enter__(self): 149 return self._directory_path 150 151 def __exit__(self, type, value, traceback): 152 # Only self-delete if necessary. 153 154 # FIXME: Should we delete non-empty directories? 155 if os.path.exists(self._directory_path): 156 os.rmdir(self._directory_path) 157 158 return TemporaryDirectory(**kwargs) 159 160 def maybe_make_directory(self, *path): 161 """Create the specified directory if it doesn't already exist.""" 162 try: 163 os.makedirs(self.join(*path)) 164 except OSError, e: 165 if e.errno != errno.EEXIST: 166 raise 167 168 def move(self, src, dest): 169 shutil.move(src, dest) 170 171 def mtime(self, path): 172 return os.stat(path).st_mtime 173 174 def normpath(self, path): 175 """Wraps os.path.normpath().""" 176 return os.path.normpath(path) 177 178 def open_binary_tempfile(self, suffix): 179 """Create, open, and return a binary temp file. Returns a tuple of the file and the name.""" 180 temp_fd, temp_name = tempfile.mkstemp(suffix) 181 f = os.fdopen(temp_fd, 'wb') 182 return f, temp_name 183 184 def open_text_file_for_writing(self, path, append=False): 185 """Returns a file handle suitable for writing to.""" 186 mode = 'w' 187 if append: 188 mode = 'a' 189 return codecs.open(path, mode, 'utf8') 190 191 def read_binary_file(self, path): 192 """Return the contents of the file at the given path as a byte string.""" 193 with file(path, 'rb') as f: 194 return f.read() 195 196 def read_text_file(self, path): 197 """Return the contents of the file at the given path as a Unicode string. 198 199 The file is read assuming it is a UTF-8 encoded file with no BOM.""" 200 with codecs.open(path, 'r', 'utf8') as f: 201 return f.read() 202 203 class _WindowsError(exceptions.OSError): 204 """Fake exception for Linux and Mac.""" 205 pass 206 207 def remove(self, path, osremove=os.remove): 208 """On Windows, if a process was recently killed and it held on to a 209 file, the OS will hold on to the file for a short while. This makes 210 attempts to delete the file fail. To work around that, this method 211 will retry for a few seconds until Windows is done with the file.""" 212 try: 213 exceptions.WindowsError 214 except AttributeError: 215 exceptions.WindowsError = FileSystem._WindowsError 216 217 retry_timeout_sec = 3.0 218 sleep_interval = 0.1 219 while True: 220 try: 221 osremove(path) 222 return True 223 except exceptions.WindowsError, e: 224 time.sleep(sleep_interval) 225 retry_timeout_sec -= sleep_interval 226 if retry_timeout_sec < 0: 227 raise e 228 229 def rmtree(self, path): 230 """Delete the directory rooted at path, empty or no.""" 231 shutil.rmtree(path, ignore_errors=True) 232 233 def read_binary_file(self, path): 234 """Return the contents of the file at the given path as a byte string.""" 235 with file(path, 'rb') as f: 236 return f.read() 237 238 def read_text_file(self, path): 239 """Return the contents of the file at the given path as a Unicode string. 240 241 The file is read assuming it is a UTF-8 encoded file with no BOM.""" 242 with codecs.open(path, 'r', 'utf8') as f: 243 return f.read() 244 245 def splitext(self, path): 246 """Return (dirname + os.sep + basename, '.' + ext)""" 247 return os.path.splitext(path) 248 249 def write_binary_file(self, path, contents): 250 """Write the contents to the file at the given location.""" 251 with file(path, 'wb') as f: 252 f.write(contents) 253 254 def write_text_file(self, path, contents): 255 """Write the contents to the file at the given location. 256 257 The file is written encoded as UTF-8 with no BOM.""" 258 with codecs.open(path, 'w', 'utf8') as f: 259 f.write(contents) 260