build_ext.py revision 09e532bcecfb891b8aee0f9730fcbfccda2f8767
1"""distutils.command.build_ext 2 3Implements the Distutils 'build_ext' command, for building extension 4modules (currently limited to C extensions, should accommodate C++ 5extensions ASAP).""" 6 7# created 1999/08/09, Greg Ward 8 9__revision__ = "$Id$" 10 11import sys, os, string, re 12from types import * 13from distutils.core import Command 14from distutils.errors import * 15from distutils.sysconfig import customize_compiler 16from distutils.dep_util import newer_group 17from distutils.extension import Extension 18from distutils import log 19 20# An extension name is just a dot-separated list of Python NAMEs (ie. 21# the same as a fully-qualified module name). 22extension_name_re = re.compile \ 23 (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') 24 25 26def show_compilers (): 27 from distutils.ccompiler import show_compilers 28 show_compilers() 29 30 31class build_ext (Command): 32 33 description = "build C/C++ extensions (compile/link to build directory)" 34 35 # XXX thoughts on how to deal with complex command-line options like 36 # these, i.e. how to make it so fancy_getopt can suck them off the 37 # command line and make it look like setup.py defined the appropriate 38 # lists of tuples of what-have-you. 39 # - each command needs a callback to process its command-line options 40 # - Command.__init__() needs access to its share of the whole 41 # command line (must ultimately come from 42 # Distribution.parse_command_line()) 43 # - it then calls the current command class' option-parsing 44 # callback to deal with weird options like -D, which have to 45 # parse the option text and churn out some custom data 46 # structure 47 # - that data structure (in this case, a list of 2-tuples) 48 # will then be present in the command object by the time 49 # we get to finalize_options() (i.e. the constructor 50 # takes care of both command-line and client options 51 # in between initialize_options() and finalize_options()) 52 53 sep_by = " (separated by '%s')" % os.pathsep 54 user_options = [ 55 ('build-lib=', 'b', 56 "directory for compiled extension modules"), 57 ('build-temp=', 't', 58 "directory for temporary files (build by-products)"), 59 ('inplace', 'i', 60 "ignore build-lib and put compiled extensions into the source " + 61 "directory alongside your pure Python modules"), 62 ('include-dirs=', 'I', 63 "list of directories to search for header files" + sep_by), 64 ('define=', 'D', 65 "C preprocessor macros to define"), 66 ('undef=', 'U', 67 "C preprocessor macros to undefine"), 68 ('libraries=', 'l', 69 "external C libraries to link with"), 70 ('library-dirs=', 'L', 71 "directories to search for external C libraries" + sep_by), 72 ('rpath=', 'R', 73 "directories to search for shared C libraries at runtime"), 74 ('link-objects=', 'O', 75 "extra explicit link objects to include in the link"), 76 ('debug', 'g', 77 "compile/link with debugging information"), 78 ('force', 'f', 79 "forcibly build everything (ignore file timestamps)"), 80 ('compiler=', 'c', 81 "specify the compiler type"), 82 ('swig-cpp', None, 83 "make SWIG create C++ files (default is C)"), 84 ] 85 86 boolean_options = ['inplace', 'debug', 'force', 'swig-cpp'] 87 88 help_options = [ 89 ('help-compiler', None, 90 "list available compilers", show_compilers), 91 ] 92 93 def initialize_options (self): 94 self.extensions = None 95 self.build_lib = None 96 self.build_temp = None 97 self.inplace = 0 98 self.package = None 99 100 self.include_dirs = None 101 self.define = None 102 self.undef = None 103 self.libraries = None 104 self.library_dirs = None 105 self.rpath = None 106 self.link_objects = None 107 self.debug = None 108 self.force = None 109 self.compiler = None 110 self.swig_cpp = None 111 112 113 def finalize_options (self): 114 from distutils import sysconfig 115 116 self.set_undefined_options('build', 117 ('build_lib', 'build_lib'), 118 ('build_temp', 'build_temp'), 119 ('compiler', 'compiler'), 120 ('debug', 'debug'), 121 ('force', 'force')) 122 123 if self.package is None: 124 self.package = self.distribution.ext_package 125 126 self.extensions = self.distribution.ext_modules 127 128 129 # Make sure Python's include directories (for Python.h, pyconfig.h, 130 # etc.) are in the include search path. 131 py_include = sysconfig.get_python_inc() 132 plat_py_include = sysconfig.get_python_inc(plat_specific=1) 133 if self.include_dirs is None: 134 self.include_dirs = self.distribution.include_dirs or [] 135 if type(self.include_dirs) is StringType: 136 self.include_dirs = string.split(self.include_dirs, os.pathsep) 137 138 # Put the Python "system" include dir at the end, so that 139 # any local include dirs take precedence. 140 self.include_dirs.append(py_include) 141 if plat_py_include != py_include: 142 self.include_dirs.append(plat_py_include) 143 144 if type(self.libraries) is StringType: 145 self.libraries = [self.libraries] 146 147 # Life is easier if we're not forever checking for None, so 148 # simplify these options to empty lists if unset 149 if self.libraries is None: 150 self.libraries = [] 151 if self.library_dirs is None: 152 self.library_dirs = [] 153 elif type(self.library_dirs) is StringType: 154 self.library_dirs = string.split(self.library_dirs, os.pathsep) 155 156 if self.rpath is None: 157 self.rpath = [] 158 elif type(self.rpath) is StringType: 159 self.rpath = string.split(self.rpath, os.pathsep) 160 161 # for extensions under windows use different directories 162 # for Release and Debug builds. 163 # also Python's library directory must be appended to library_dirs 164 if os.name == 'nt': 165 self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) 166 if self.debug: 167 self.build_temp = os.path.join(self.build_temp, "Debug") 168 else: 169 self.build_temp = os.path.join(self.build_temp, "Release") 170 171 # Append the source distribution include and library directories, 172 # this allows distutils on windows to work in the source tree 173 self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) 174 self.library_dirs.append(os.path.join(sys.exec_prefix, 'PCBuild')) 175 176 # OS/2 (EMX) doesn't support Debug vs Release builds, but has the 177 # import libraries in its "Config" subdirectory 178 if os.name == 'os2': 179 self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config')) 180 181 # for extensions under Cygwin and AtheOS Python's library directory must be 182 # appended to library_dirs 183 if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': 184 if string.find(sys.executable, sys.exec_prefix) != -1: 185 # building third party extensions 186 self.library_dirs.append(os.path.join(sys.prefix, "lib", 187 "python" + sys.version[:3], 188 "config")) 189 else: 190 # building python standard extensions 191 self.library_dirs.append('.') 192 193 # The argument parsing will result in self.define being a string, but 194 # it has to be a list of 2-tuples. All the preprocessor symbols 195 # specified by the 'define' option will be set to '1'. Multiple 196 # symbols can be separated with commas. 197 198 if self.define: 199 defines = string.split(self.define, ',') 200 self.define = map(lambda symbol: (symbol, '1'), defines) 201 202 # The option for macros to undefine is also a string from the 203 # option parsing, but has to be a list. Multiple symbols can also 204 # be separated with commas here. 205 if self.undef: 206 self.undef = string.split(self.undef, ',') 207 208 # finalize_options () 209 210 211 def run (self): 212 213 from distutils.ccompiler import new_compiler 214 215 # 'self.extensions', as supplied by setup.py, is a list of 216 # Extension instances. See the documentation for Extension (in 217 # distutils.extension) for details. 218 # 219 # For backwards compatibility with Distutils 0.8.2 and earlier, we 220 # also allow the 'extensions' list to be a list of tuples: 221 # (ext_name, build_info) 222 # where build_info is a dictionary containing everything that 223 # Extension instances do except the name, with a few things being 224 # differently named. We convert these 2-tuples to Extension 225 # instances as needed. 226 227 if not self.extensions: 228 return 229 230 # If we were asked to build any C/C++ libraries, make sure that the 231 # directory where we put them is in the library search path for 232 # linking extensions. 233 if self.distribution.has_c_libraries(): 234 build_clib = self.get_finalized_command('build_clib') 235 self.libraries.extend(build_clib.get_library_names() or []) 236 self.library_dirs.append(build_clib.build_clib) 237 238 # Setup the CCompiler object that we'll use to do all the 239 # compiling and linking 240 self.compiler = new_compiler(compiler=self.compiler, 241 verbose=self.verbose, 242 dry_run=self.dry_run, 243 force=self.force) 244 customize_compiler(self.compiler) 245 246 # And make sure that any compile/link-related options (which might 247 # come from the command-line or from the setup script) are set in 248 # that CCompiler object -- that way, they automatically apply to 249 # all compiling and linking done here. 250 if self.include_dirs is not None: 251 self.compiler.set_include_dirs(self.include_dirs) 252 if self.define is not None: 253 # 'define' option is a list of (name,value) tuples 254 for (name,value) in self.define: 255 self.compiler.define_macro(name, value) 256 if self.undef is not None: 257 for macro in self.undef: 258 self.compiler.undefine_macro(macro) 259 if self.libraries is not None: 260 self.compiler.set_libraries(self.libraries) 261 if self.library_dirs is not None: 262 self.compiler.set_library_dirs(self.library_dirs) 263 if self.rpath is not None: 264 self.compiler.set_runtime_library_dirs(self.rpath) 265 if self.link_objects is not None: 266 self.compiler.set_link_objects(self.link_objects) 267 268 # Now actually compile and link everything. 269 self.build_extensions() 270 271 # run () 272 273 274 def check_extensions_list (self, extensions): 275 """Ensure that the list of extensions (presumably provided as a 276 command option 'extensions') is valid, i.e. it is a list of 277 Extension objects. We also support the old-style list of 2-tuples, 278 where the tuples are (ext_name, build_info), which are converted to 279 Extension instances here. 280 281 Raise DistutilsSetupError if the structure is invalid anywhere; 282 just returns otherwise. 283 """ 284 if type(extensions) is not ListType: 285 raise DistutilsSetupError, \ 286 "'ext_modules' option must be a list of Extension instances" 287 288 for i in range(len(extensions)): 289 ext = extensions[i] 290 if isinstance(ext, Extension): 291 continue # OK! (assume type-checking done 292 # by Extension constructor) 293 294 (ext_name, build_info) = ext 295 log.warn(("old-style (ext_name, build_info) tuple found in " 296 "ext_modules for extension '%s'" 297 "-- please convert to Extension instance" % ext_name)) 298 if type(ext) is not TupleType and len(ext) != 2: 299 raise DistutilsSetupError, \ 300 ("each element of 'ext_modules' option must be an " 301 "Extension instance or 2-tuple") 302 303 if not (type(ext_name) is StringType and 304 extension_name_re.match(ext_name)): 305 raise DistutilsSetupError, \ 306 ("first element of each tuple in 'ext_modules' " 307 "must be the extension name (a string)") 308 309 if type(build_info) is not DictionaryType: 310 raise DistutilsSetupError, \ 311 ("second element of each tuple in 'ext_modules' " 312 "must be a dictionary (build info)") 313 314 # OK, the (ext_name, build_info) dict is type-safe: convert it 315 # to an Extension instance. 316 ext = Extension(ext_name, build_info['sources']) 317 318 # Easy stuff: one-to-one mapping from dict elements to 319 # instance attributes. 320 for key in ('include_dirs', 321 'library_dirs', 322 'libraries', 323 'extra_objects', 324 'extra_compile_args', 325 'extra_link_args'): 326 val = build_info.get(key) 327 if val is not None: 328 setattr(ext, key, val) 329 330 # Medium-easy stuff: same syntax/semantics, different names. 331 ext.runtime_library_dirs = build_info.get('rpath') 332 if build_info.has_key('def_file'): 333 log.warn("'def_file' element of build info dict " 334 "no longer supported") 335 336 # Non-trivial stuff: 'macros' split into 'define_macros' 337 # and 'undef_macros'. 338 macros = build_info.get('macros') 339 if macros: 340 ext.define_macros = [] 341 ext.undef_macros = [] 342 for macro in macros: 343 if not (type(macro) is TupleType and 344 1 <= len(macro) <= 2): 345 raise DistutilsSetupError, \ 346 ("'macros' element of build info dict " 347 "must be 1- or 2-tuple") 348 if len(macro) == 1: 349 ext.undef_macros.append(macro[0]) 350 elif len(macro) == 2: 351 ext.define_macros.append(macro) 352 353 extensions[i] = ext 354 355 # for extensions 356 357 # check_extensions_list () 358 359 360 def get_source_files (self): 361 self.check_extensions_list(self.extensions) 362 filenames = [] 363 364 # Wouldn't it be neat if we knew the names of header files too... 365 for ext in self.extensions: 366 filenames.extend(ext.sources) 367 368 return filenames 369 370 371 def get_outputs (self): 372 373 # Sanity check the 'extensions' list -- can't assume this is being 374 # done in the same run as a 'build_extensions()' call (in fact, we 375 # can probably assume that it *isn't*!). 376 self.check_extensions_list(self.extensions) 377 378 # And build the list of output (built) filenames. Note that this 379 # ignores the 'inplace' flag, and assumes everything goes in the 380 # "build" tree. 381 outputs = [] 382 for ext in self.extensions: 383 fullname = self.get_ext_fullname(ext.name) 384 outputs.append(os.path.join(self.build_lib, 385 self.get_ext_filename(fullname))) 386 return outputs 387 388 # get_outputs () 389 390 def build_extensions(self): 391 392 # First, sanity-check the 'extensions' list 393 self.check_extensions_list(self.extensions) 394 395 for ext in self.extensions: 396 self.build_extension(ext) 397 398 def build_extension(self, ext): 399 400 sources = ext.sources 401 if sources is None or type(sources) not in (ListType, TupleType): 402 raise DistutilsSetupError, \ 403 ("in 'ext_modules' option (extension '%s'), " + 404 "'sources' must be present and must be " + 405 "a list of source filenames") % ext.name 406 sources = list(sources) 407 408 fullname = self.get_ext_fullname(ext.name) 409 if self.inplace: 410 # ignore build-lib -- put the compiled extension into 411 # the source tree along with pure Python modules 412 413 modpath = string.split(fullname, '.') 414 package = string.join(modpath[0:-1], '.') 415 base = modpath[-1] 416 417 build_py = self.get_finalized_command('build_py') 418 package_dir = build_py.get_package_dir(package) 419 ext_filename = os.path.join(package_dir, 420 self.get_ext_filename(base)) 421 else: 422 ext_filename = os.path.join(self.build_lib, 423 self.get_ext_filename(fullname)) 424 425 depends = sources + ext.depends 426 if not (self.force or newer_group(depends, ext_filename, 'newer')): 427 log.debug("skipping '%s' extension (up-to-date)", ext.name) 428 return 429 else: 430 log.info("building '%s' extension", ext.name) 431 432 # First, scan the sources for SWIG definition files (.i), run 433 # SWIG on 'em to create .c files, and modify the sources list 434 # accordingly. 435 sources = self.swig_sources(sources) 436 437 # Next, compile the source code to object files. 438 439 # XXX not honouring 'define_macros' or 'undef_macros' -- the 440 # CCompiler API needs to change to accommodate this, and I 441 # want to do one thing at a time! 442 443 # Two possible sources for extra compiler arguments: 444 # - 'extra_compile_args' in Extension object 445 # - CFLAGS environment variable (not particularly 446 # elegant, but people seem to expect it and I 447 # guess it's useful) 448 # The environment variable should take precedence, and 449 # any sensible compiler will give precedence to later 450 # command line args. Hence we combine them in order: 451 extra_args = ext.extra_compile_args or [] 452 453 macros = ext.define_macros[:] 454 for undef in ext.undef_macros: 455 macros.append((undef,)) 456 457 # XXX and if we support CFLAGS, why not CC (compiler 458 # executable), CPPFLAGS (pre-processor options), and LDFLAGS 459 # (linker options) too? 460 # XXX should we use shlex to properly parse CFLAGS? 461 462 if os.environ.has_key('CFLAGS'): 463 extra_args.extend(string.split(os.environ['CFLAGS'])) 464 465 objects = self.compiler.compile(sources, 466 output_dir=self.build_temp, 467 macros=macros, 468 include_dirs=ext.include_dirs, 469 debug=self.debug, 470 extra_postargs=extra_args) 471 472 # XXX -- this is a Vile HACK! 473 # 474 # The setup.py script for Python on Unix needs to be able to 475 # get this list so it can perform all the clean up needed to 476 # avoid keeping object files around when cleaning out a failed 477 # build of an extension module. Since Distutils does not 478 # track dependencies, we have to get rid of intermediates to 479 # ensure all the intermediates will be properly re-built. 480 # 481 self._built_objects = objects[:] 482 483 # Now link the object files together into a "shared object" -- 484 # of course, first we have to figure out all the other things 485 # that go into the mix. 486 if ext.extra_objects: 487 objects.extend(ext.extra_objects) 488 extra_args = ext.extra_link_args or [] 489 490 491 self.compiler.link_shared_object( 492 objects, ext_filename, 493 libraries=self.get_libraries(ext), 494 library_dirs=ext.library_dirs, 495 runtime_library_dirs=ext.runtime_library_dirs, 496 extra_postargs=extra_args, 497 export_symbols=self.get_export_symbols(ext), 498 debug=self.debug, 499 build_temp=self.build_temp) 500 501 502 def swig_sources (self, sources): 503 504 """Walk the list of source files in 'sources', looking for SWIG 505 interface (.i) files. Run SWIG on all that are found, and 506 return a modified 'sources' list with SWIG source files replaced 507 by the generated C (or C++) files. 508 """ 509 510 new_sources = [] 511 swig_sources = [] 512 swig_targets = {} 513 514 # XXX this drops generated C/C++ files into the source tree, which 515 # is fine for developers who want to distribute the generated 516 # source -- but there should be an option to put SWIG output in 517 # the temp dir. 518 519 if self.swig_cpp: 520 target_ext = '.cpp' 521 else: 522 target_ext = '.c' 523 524 for source in sources: 525 (base, ext) = os.path.splitext(source) 526 if ext == ".i": # SWIG interface file 527 new_sources.append(base + '_wrap' + target_ext) 528 swig_sources.append(source) 529 swig_targets[source] = new_sources[-1] 530 else: 531 new_sources.append(source) 532 533 if not swig_sources: 534 return new_sources 535 536 swig = self.find_swig() 537 swig_cmd = [swig, "-python"] 538 if self.swig_cpp: 539 swig_cmd.append("-c++") 540 541 for source in swig_sources: 542 target = swig_targets[source] 543 log.info("swigging %s to %s", source, target) 544 self.spawn(swig_cmd + ["-o", target, source]) 545 546 return new_sources 547 548 # swig_sources () 549 550 def find_swig (self): 551 """Return the name of the SWIG executable. On Unix, this is 552 just "swig" -- it should be in the PATH. Tries a bit harder on 553 Windows. 554 """ 555 556 if os.name == "posix": 557 return "swig" 558 elif os.name == "nt": 559 560 # Look for SWIG in its standard installation directory on 561 # Windows (or so I presume!). If we find it there, great; 562 # if not, act like Unix and assume it's in the PATH. 563 for vers in ("1.3", "1.2", "1.1"): 564 fn = os.path.join("c:\\swig%s" % vers, "swig.exe") 565 if os.path.isfile(fn): 566 return fn 567 else: 568 return "swig.exe" 569 570 elif os.name == "os2": 571 # assume swig available in the PATH. 572 return "swig.exe" 573 574 else: 575 raise DistutilsPlatformError, \ 576 ("I don't know how to find (much less run) SWIG " 577 "on platform '%s'") % os.name 578 579 # find_swig () 580 581 # -- Name generators ----------------------------------------------- 582 # (extension names, filenames, whatever) 583 584 def get_ext_fullname (self, ext_name): 585 if self.package is None: 586 return ext_name 587 else: 588 return self.package + '.' + ext_name 589 590 def get_ext_filename (self, ext_name): 591 r"""Convert the name of an extension (eg. "foo.bar") into the name 592 of the file from which it will be loaded (eg. "foo/bar.so", or 593 "foo\bar.pyd"). 594 """ 595 596 from distutils.sysconfig import get_config_var 597 ext_path = string.split(ext_name, '.') 598 # OS/2 has an 8 character module (extension) limit :-( 599 if os.name == "os2": 600 ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8] 601 # extensions in debug_mode are named 'module_d.pyd' under windows 602 so_ext = get_config_var('SO') 603 if os.name == 'nt' and self.debug: 604 return apply(os.path.join, ext_path) + '_d' + so_ext 605 return apply(os.path.join, ext_path) + so_ext 606 607 def get_export_symbols (self, ext): 608 """Return the list of symbols that a shared extension has to 609 export. This either uses 'ext.export_symbols' or, if it's not 610 provided, "init" + module_name. Only relevant on Windows, where 611 the .pyd file (DLL) must export the module "init" function. 612 """ 613 614 initfunc_name = "init" + string.split(ext.name,'.')[-1] 615 if initfunc_name not in ext.export_symbols: 616 ext.export_symbols.append(initfunc_name) 617 return ext.export_symbols 618 619 def get_libraries (self, ext): 620 """Return the list of libraries to link against when building a 621 shared extension. On most platforms, this is just 'ext.libraries'; 622 on Windows and OS/2, we add the Python library (eg. python20.dll). 623 """ 624 # The python library is always needed on Windows. For MSVC, this 625 # is redundant, since the library is mentioned in a pragma in 626 # pyconfig.h that MSVC groks. The other Windows compilers all seem 627 # to need it mentioned explicitly, though, so that's what we do. 628 # Append '_d' to the python import library on debug builds. 629 from distutils.msvccompiler import MSVCCompiler 630 if sys.platform == "win32" and \ 631 not isinstance(self.compiler, MSVCCompiler): 632 template = "python%d%d" 633 if self.debug: 634 template = template + '_d' 635 pythonlib = (template % 636 (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) 637 # don't extend ext.libraries, it may be shared with other 638 # extensions, it is a reference to the original list 639 return ext.libraries + [pythonlib] 640 elif sys.platform == "os2emx": 641 # EMX/GCC requires the python library explicitly, and I 642 # believe VACPP does as well (though not confirmed) - AIM Apr01 643 template = "python%d%d" 644 # debug versions of the main DLL aren't supported, at least 645 # not at this time - AIM Apr01 646 #if self.debug: 647 # template = template + '_d' 648 pythonlib = (template % 649 (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) 650 # don't extend ext.libraries, it may be shared with other 651 # extensions, it is a reference to the original list 652 return ext.libraries + [pythonlib] 653 elif sys.platform[:6] == "cygwin": 654 template = "python%d.%d" 655 pythonlib = (template % 656 (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) 657 # don't extend ext.libraries, it may be shared with other 658 # extensions, it is a reference to the original list 659 return ext.libraries + [pythonlib] 660 elif sys.platform[:6] == "atheos": 661 from distutils import sysconfig 662 663 template = "python%d.%d" 664 pythonlib = (template % 665 (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) 666 # Get SHLIBS from Makefile 667 extra = [] 668 for lib in sysconfig.get_config_var('SHLIBS').split(): 669 if lib.startswith('-l'): 670 extra.append(lib[2:]) 671 else: 672 extra.append(lib) 673 # don't extend ext.libraries, it may be shared with other 674 # extensions, it is a reference to the original list 675 return ext.libraries + [pythonlib, "m"] + extra 676 else: 677 return ext.libraries 678 679# class build_ext 680