build_py.py revision 73a22f0500ce713d46fce864e95402c34a45afd9
1"""distutils.command.build_py 2 3Implements the Distutils 'build_py' command.""" 4 5__revision__ = "$Id$" 6 7import sys, os 8from glob import glob 9 10from distutils.core import Command 11from distutils.errors import * 12from distutils.util import convert_path 13from distutils import log 14 15class build_py (Command): 16 17 description = "\"build\" pure Python modules (copy to build directory)" 18 19 user_options = [ 20 ('build-lib=', 'd', "directory to \"build\" (copy) to"), 21 ('compile', 'c', "compile .py to .pyc"), 22 ('no-compile', None, "don't compile .py files [default]"), 23 ('optimize=', 'O', 24 "also compile with optimization: -O1 for \"python -O\", " 25 "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), 26 ('force', 'f', "forcibly build everything (ignore file timestamps)"), 27 ] 28 29 boolean_options = ['compile', 'force'] 30 negative_opt = {'no-compile' : 'compile'} 31 32 def initialize_options(self): 33 self.build_lib = None 34 self.py_modules = None 35 self.package = None 36 self.package_data = None 37 self.package_dir = None 38 self.compile = 0 39 self.optimize = 0 40 self.force = None 41 42 def finalize_options(self): 43 self.set_undefined_options('build', 44 ('build_lib', 'build_lib'), 45 ('force', 'force')) 46 47 # Get the distribution options that are aliases for build_py 48 # options -- list of packages and list of modules. 49 self.packages = self.distribution.packages 50 self.py_modules = self.distribution.py_modules 51 self.package_data = self.distribution.package_data 52 self.package_dir = {} 53 if self.distribution.package_dir: 54 for name, path in self.distribution.package_dir.items(): 55 self.package_dir[name] = convert_path(path) 56 self.data_files = self.get_data_files() 57 58 # Ick, copied straight from install_lib.py (fancy_getopt needs a 59 # type system! Hell, *everything* needs a type system!!!) 60 if not isinstance(self.optimize, int): 61 try: 62 self.optimize = int(self.optimize) 63 assert 0 <= self.optimize <= 2 64 except (ValueError, AssertionError): 65 raise DistutilsOptionError("optimize must be 0, 1, or 2") 66 67 def run(self): 68 # XXX copy_file by default preserves atime and mtime. IMHO this is 69 # the right thing to do, but perhaps it should be an option -- in 70 # particular, a site administrator might want installed files to 71 # reflect the time of installation rather than the last 72 # modification time before the installed release. 73 74 # XXX copy_file by default preserves mode, which appears to be the 75 # wrong thing to do: if a file is read-only in the working 76 # directory, we want it to be installed read/write so that the next 77 # installation of the same module distribution can overwrite it 78 # without problems. (This might be a Unix-specific issue.) Thus 79 # we turn off 'preserve_mode' when copying to the build directory, 80 # since the build directory is supposed to be exactly what the 81 # installation will look like (ie. we preserve mode when 82 # installing). 83 84 # Two options control which modules will be installed: 'packages' 85 # and 'py_modules'. The former lets us work with whole packages, not 86 # specifying individual modules at all; the latter is for 87 # specifying modules one-at-a-time. 88 89 if self.py_modules: 90 self.build_modules() 91 if self.packages: 92 self.build_packages() 93 self.build_package_data() 94 95 self.byte_compile(self.get_outputs(include_bytecode=0)) 96 97 def get_data_files(self): 98 """Generate list of '(package,src_dir,build_dir,filenames)' tuples""" 99 data = [] 100 if not self.packages: 101 return data 102 for package in self.packages: 103 # Locate package source directory 104 src_dir = self.get_package_dir(package) 105 106 # Compute package build directory 107 build_dir = os.path.join(*([self.build_lib] + package.split('.'))) 108 109 # Length of path to strip from found files 110 plen = 0 111 if src_dir: 112 plen = len(src_dir)+1 113 114 # Strip directory from globbed filenames 115 filenames = [ 116 file[plen:] for file in self.find_data_files(package, src_dir) 117 ] 118 data.append((package, src_dir, build_dir, filenames)) 119 return data 120 121 def find_data_files(self, package, src_dir): 122 """Return filenames for package's data files in 'src_dir'""" 123 globs = (self.package_data.get('', []) 124 + self.package_data.get(package, [])) 125 files = [] 126 for pattern in globs: 127 # Each pattern has to be converted to a platform-specific path 128 filelist = glob(os.path.join(src_dir, convert_path(pattern))) 129 # Files that match more than one pattern are only added once 130 files.extend([fn for fn in filelist if fn not in files]) 131 return files 132 133 def build_package_data(self): 134 """Copy data files into build directory""" 135 lastdir = None 136 for package, src_dir, build_dir, filenames in self.data_files: 137 for filename in filenames: 138 target = os.path.join(build_dir, filename) 139 self.mkpath(os.path.dirname(target)) 140 self.copy_file(os.path.join(src_dir, filename), target, 141 preserve_mode=False) 142 143 def get_package_dir(self, package): 144 """Return the directory, relative to the top of the source 145 distribution, where package 'package' should be found 146 (at least according to the 'package_dir' option, if any).""" 147 path = package.split('.') 148 149 if not self.package_dir: 150 if path: 151 return os.path.join(*path) 152 else: 153 return '' 154 else: 155 tail = [] 156 while path: 157 try: 158 pdir = self.package_dir['.'.join(path)] 159 except KeyError: 160 tail.insert(0, path[-1]) 161 del path[-1] 162 else: 163 tail.insert(0, pdir) 164 return os.path.join(*tail) 165 else: 166 # Oops, got all the way through 'path' without finding a 167 # match in package_dir. If package_dir defines a directory 168 # for the root (nameless) package, then fallback on it; 169 # otherwise, we might as well have not consulted 170 # package_dir at all, as we just use the directory implied 171 # by 'tail' (which should be the same as the original value 172 # of 'path' at this point). 173 pdir = self.package_dir.get('') 174 if pdir is not None: 175 tail.insert(0, pdir) 176 177 if tail: 178 return os.path.join(*tail) 179 else: 180 return '' 181 182 def check_package(self, package, package_dir): 183 # Empty dir name means current directory, which we can probably 184 # assume exists. Also, os.path.exists and isdir don't know about 185 # my "empty string means current dir" convention, so we have to 186 # circumvent them. 187 if package_dir != "": 188 if not os.path.exists(package_dir): 189 raise DistutilsFileError( 190 "package directory '%s' does not exist" % package_dir) 191 if not os.path.isdir(package_dir): 192 raise DistutilsFileError( 193 "supposed package directory '%s' exists, " 194 "but is not a directory" % package_dir) 195 196 # Require __init__.py for all but the "root package" 197 if package: 198 init_py = os.path.join(package_dir, "__init__.py") 199 if os.path.isfile(init_py): 200 return init_py 201 else: 202 log.warn(("package init file '%s' not found " + 203 "(or not a regular file)"), init_py) 204 205 # Either not in a package at all (__init__.py not expected), or 206 # __init__.py doesn't exist -- so don't return the filename. 207 return None 208 209 def check_module(self, module, module_file): 210 if not os.path.isfile(module_file): 211 log.warn("file %s (for module %s) not found", module_file, module) 212 return False 213 else: 214 return True 215 216 def find_package_modules(self, package, package_dir): 217 self.check_package(package, package_dir) 218 module_files = glob(os.path.join(package_dir, "*.py")) 219 modules = [] 220 setup_script = os.path.abspath(self.distribution.script_name) 221 222 for f in module_files: 223 abs_f = os.path.abspath(f) 224 if abs_f != setup_script: 225 module = os.path.splitext(os.path.basename(f))[0] 226 modules.append((package, module, f)) 227 else: 228 self.debug_print("excluding %s" % setup_script) 229 return modules 230 231 def find_modules(self): 232 """Finds individually-specified Python modules, ie. those listed by 233 module name in 'self.py_modules'. Returns a list of tuples (package, 234 module_base, filename): 'package' is a tuple of the path through 235 package-space to the module; 'module_base' is the bare (no 236 packages, no dots) module name, and 'filename' is the path to the 237 ".py" file (relative to the distribution root) that implements the 238 module. 239 """ 240 # Map package names to tuples of useful info about the package: 241 # (package_dir, checked) 242 # package_dir - the directory where we'll find source files for 243 # this package 244 # checked - true if we have checked that the package directory 245 # is valid (exists, contains __init__.py, ... ?) 246 packages = {} 247 248 # List of (package, module, filename) tuples to return 249 modules = [] 250 251 # We treat modules-in-packages almost the same as toplevel modules, 252 # just the "package" for a toplevel is empty (either an empty 253 # string or empty list, depending on context). Differences: 254 # - don't check for __init__.py in directory for empty package 255 for module in self.py_modules: 256 path = module.split('.') 257 package = '.'.join(path[0:-1]) 258 module_base = path[-1] 259 260 try: 261 (package_dir, checked) = packages[package] 262 except KeyError: 263 package_dir = self.get_package_dir(package) 264 checked = 0 265 266 if not checked: 267 init_py = self.check_package(package, package_dir) 268 packages[package] = (package_dir, 1) 269 if init_py: 270 modules.append((package, "__init__", init_py)) 271 272 # XXX perhaps we should also check for just .pyc files 273 # (so greedy closed-source bastards can distribute Python 274 # modules too) 275 module_file = os.path.join(package_dir, module_base + ".py") 276 if not self.check_module(module, module_file): 277 continue 278 279 modules.append((package, module_base, module_file)) 280 281 return modules 282 283 def find_all_modules(self): 284 """Compute the list of all modules that will be built, whether 285 they are specified one-module-at-a-time ('self.py_modules') or 286 by whole packages ('self.packages'). Return a list of tuples 287 (package, module, module_file), just like 'find_modules()' and 288 'find_package_modules()' do.""" 289 modules = [] 290 if self.py_modules: 291 modules.extend(self.find_modules()) 292 if self.packages: 293 for package in self.packages: 294 package_dir = self.get_package_dir(package) 295 m = self.find_package_modules(package, package_dir) 296 modules.extend(m) 297 return modules 298 299 def get_source_files(self): 300 return [module[-1] for module in self.find_all_modules()] 301 302 def get_module_outfile(self, build_dir, package, module): 303 outfile_path = [build_dir] + list(package) + [module + ".py"] 304 return os.path.join(*outfile_path) 305 306 def get_outputs(self, include_bytecode=1): 307 modules = self.find_all_modules() 308 outputs = [] 309 for (package, module, module_file) in modules: 310 package = package.split('.') 311 filename = self.get_module_outfile(self.build_lib, package, module) 312 outputs.append(filename) 313 if include_bytecode: 314 if self.compile: 315 outputs.append(filename + "c") 316 if self.optimize > 0: 317 outputs.append(filename + "o") 318 319 outputs += [ 320 os.path.join(build_dir, filename) 321 for package, src_dir, build_dir, filenames in self.data_files 322 for filename in filenames 323 ] 324 325 return outputs 326 327 def build_module(self, module, module_file, package): 328 if isinstance(package, str): 329 package = package.split('.') 330 elif not isinstance(package, (list, tuple)): 331 raise TypeError( 332 "'package' must be a string (dot-separated), list, or tuple") 333 334 # Now put the module source file into the "build" area -- this is 335 # easy, we just copy it somewhere under self.build_lib (the build 336 # directory for Python source). 337 outfile = self.get_module_outfile(self.build_lib, package, module) 338 dir = os.path.dirname(outfile) 339 self.mkpath(dir) 340 return self.copy_file(module_file, outfile, preserve_mode=0) 341 342 def build_modules(self): 343 modules = self.find_modules() 344 for (package, module, module_file) in modules: 345 # Now "build" the module -- ie. copy the source file to 346 # self.build_lib (the build directory for Python source). 347 # (Actually, it gets copied to the directory for this package 348 # under self.build_lib.) 349 self.build_module(module, module_file, package) 350 351 def build_packages(self): 352 for package in self.packages: 353 # Get list of (package, module, module_file) tuples based on 354 # scanning the package directory. 'package' is only included 355 # in the tuple so that 'find_modules()' and 356 # 'find_package_tuples()' have a consistent interface; it's 357 # ignored here (apart from a sanity check). Also, 'module' is 358 # the *unqualified* module name (ie. no dots, no package -- we 359 # already know its package!), and 'module_file' is the path to 360 # the .py file, relative to the current directory 361 # (ie. including 'package_dir'). 362 package_dir = self.get_package_dir(package) 363 modules = self.find_package_modules(package, package_dir) 364 365 # Now loop over the modules we found, "building" each one (just 366 # copy it to self.build_lib). 367 for (package_, module, module_file) in modules: 368 assert package == package_ 369 self.build_module(module, module_file, package) 370 371 def byte_compile(self, files): 372 from distutils.util import byte_compile 373 prefix = self.build_lib 374 if prefix[-1] != os.sep: 375 prefix = prefix + os.sep 376 377 # XXX this code is essentially the same as the 'byte_compile() 378 # method of the "install_lib" command, except for the determination 379 # of the 'prefix' string. Hmmm. 380 if self.compile: 381 byte_compile(files, optimize=0, 382 force=self.force, prefix=prefix, dry_run=self.dry_run) 383 if self.optimize > 0: 384 byte_compile(files, optimize=self.optimize, 385 force=self.force, prefix=prefix, dry_run=self.dry_run) 386 387class build_py_2to3(build_py): 388 def run(self): 389 from lib2to3.refactor import RefactoringTool 390 self.updated_files = [] 391 build_py.run(self) 392 class Options: 393 pass 394 o = Options() 395 o.doctests_only = False 396 o.fix = [] 397 o.list_fixes = [] 398 o.print_function = False 399 o.verbose = False 400 o.write = True 401 r = RefactoringTool(o) 402 r.refactor_args(self.updated_files) 403 404 def build_module(self, module, module_file, package): 405 res = build_py.build_module(self, module, module_file, package) 406 if res[1]: 407 # file was copied 408 self.updated_files.append(res[0]) 409 return res 410