1# 2# Cython Top Level 3# 4 5import os, sys, re, codecs 6if sys.version_info[:2] < (2, 4): 7 sys.stderr.write("Sorry, Cython requires Python 2.4 or later\n") 8 sys.exit(1) 9 10import Errors 11# Do not import Parsing here, import it when needed, because Parsing imports 12# Nodes, which globally needs debug command line options initialized to set a 13# conditional metaclass. These options are processed by CmdLine called from 14# main() in this file. 15# import Parsing 16import Version 17from Scanning import PyrexScanner, FileSourceDescriptor 18from Errors import PyrexError, CompileError, error, warning 19from Symtab import ModuleScope 20from Cython import Utils 21import Options 22 23module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$") 24 25verbose = 0 26 27class CompilationData(object): 28 # Bundles the information that is passed from transform to transform. 29 # (For now, this is only) 30 31 # While Context contains every pxd ever loaded, path information etc., 32 # this only contains the data related to a single compilation pass 33 # 34 # pyx ModuleNode Main code tree of this compilation. 35 # pxds {string : ModuleNode} Trees for the pxds used in the pyx. 36 # codewriter CCodeWriter Where to output final code. 37 # options CompilationOptions 38 # result CompilationResult 39 pass 40 41class Context(object): 42 # This class encapsulates the context needed for compiling 43 # one or more Cython implementation files along with their 44 # associated and imported declaration files. It includes 45 # the root of the module import namespace and the list 46 # of directories to search for include files. 47 # 48 # modules {string : ModuleScope} 49 # include_directories [string] 50 # future_directives [object] 51 # language_level int currently 2 or 3 for Python 2/3 52 53 cython_scope = None 54 55 def __init__(self, include_directories, compiler_directives, cpp=False, 56 language_level=2, options=None, create_testscope=True): 57 # cython_scope is a hack, set to False by subclasses, in order to break 58 # an infinite loop. 59 # Better code organization would fix it. 60 61 import Builtin, CythonScope 62 self.modules = {"__builtin__" : Builtin.builtin_scope} 63 self.cython_scope = CythonScope.create_cython_scope(self) 64 self.modules["cython"] = self.cython_scope 65 self.include_directories = include_directories 66 self.future_directives = set() 67 self.compiler_directives = compiler_directives 68 self.cpp = cpp 69 self.options = options 70 71 self.pxds = {} # full name -> node tree 72 73 standard_include_path = os.path.abspath(os.path.normpath( 74 os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) 75 self.include_directories = include_directories + [standard_include_path] 76 77 self.set_language_level(language_level) 78 79 self.gdb_debug_outputwriter = None 80 81 def set_language_level(self, level): 82 self.language_level = level 83 if level >= 3: 84 from Future import print_function, unicode_literals, absolute_import 85 self.future_directives.update([print_function, unicode_literals, absolute_import]) 86 self.modules['builtins'] = self.modules['__builtin__'] 87 88 # pipeline creation functions can now be found in Pipeline.py 89 90 def process_pxd(self, source_desc, scope, module_name): 91 import Pipeline 92 if isinstance(source_desc, FileSourceDescriptor) and source_desc._file_type == 'pyx': 93 source = CompilationSource(source_desc, module_name, os.getcwd()) 94 result_sink = create_default_resultobj(source, self.options) 95 pipeline = Pipeline.create_pyx_as_pxd_pipeline(self, result_sink) 96 result = Pipeline.run_pipeline(pipeline, source) 97 else: 98 pipeline = Pipeline.create_pxd_pipeline(self, scope, module_name) 99 result = Pipeline.run_pipeline(pipeline, source_desc) 100 return result 101 102 def nonfatal_error(self, exc): 103 return Errors.report_error(exc) 104 105 def find_module(self, module_name, 106 relative_to = None, pos = None, need_pxd = 1, check_module_name = True): 107 # Finds and returns the module scope corresponding to 108 # the given relative or absolute module name. If this 109 # is the first time the module has been requested, finds 110 # the corresponding .pxd file and process it. 111 # If relative_to is not None, it must be a module scope, 112 # and the module will first be searched for relative to 113 # that module, provided its name is not a dotted name. 114 debug_find_module = 0 115 if debug_find_module: 116 print("Context.find_module: module_name = %s, relative_to = %s, pos = %s, need_pxd = %s" % ( 117 module_name, relative_to, pos, need_pxd)) 118 119 scope = None 120 pxd_pathname = None 121 if check_module_name and not module_name_pattern.match(module_name): 122 if pos is None: 123 pos = (module_name, 0, 0) 124 raise CompileError(pos, 125 "'%s' is not a valid module name" % module_name) 126 if "." not in module_name and relative_to: 127 if debug_find_module: 128 print("...trying relative import") 129 scope = relative_to.lookup_submodule(module_name) 130 if not scope: 131 qualified_name = relative_to.qualify_name(module_name) 132 pxd_pathname = self.find_pxd_file(qualified_name, pos) 133 if pxd_pathname: 134 scope = relative_to.find_submodule(module_name) 135 if not scope: 136 if debug_find_module: 137 print("...trying absolute import") 138 scope = self 139 for name in module_name.split("."): 140 scope = scope.find_submodule(name) 141 if debug_find_module: 142 print("...scope =", scope) 143 if not scope.pxd_file_loaded: 144 if debug_find_module: 145 print("...pxd not loaded") 146 scope.pxd_file_loaded = 1 147 if not pxd_pathname: 148 if debug_find_module: 149 print("...looking for pxd file") 150 pxd_pathname = self.find_pxd_file(module_name, pos) 151 if debug_find_module: 152 print("......found ", pxd_pathname) 153 if not pxd_pathname and need_pxd: 154 package_pathname = self.search_include_directories(module_name, ".py", pos) 155 if package_pathname and package_pathname.endswith('__init__.py'): 156 pass 157 else: 158 error(pos, "'%s.pxd' not found" % module_name) 159 if pxd_pathname: 160 try: 161 if debug_find_module: 162 print("Context.find_module: Parsing %s" % pxd_pathname) 163 rel_path = module_name.replace('.', os.sep) + os.path.splitext(pxd_pathname)[1] 164 if not pxd_pathname.endswith(rel_path): 165 rel_path = pxd_pathname # safety measure to prevent printing incorrect paths 166 source_desc = FileSourceDescriptor(pxd_pathname, rel_path) 167 err, result = self.process_pxd(source_desc, scope, module_name) 168 if err: 169 raise err 170 (pxd_codenodes, pxd_scope) = result 171 self.pxds[module_name] = (pxd_codenodes, pxd_scope) 172 except CompileError: 173 pass 174 return scope 175 176 def find_pxd_file(self, qualified_name, pos): 177 # Search include path for the .pxd file corresponding to the 178 # given fully-qualified module name. 179 # Will find either a dotted filename or a file in a 180 # package directory. If a source file position is given, 181 # the directory containing the source file is searched first 182 # for a dotted filename, and its containing package root 183 # directory is searched first for a non-dotted filename. 184 pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=True) 185 if pxd is None: # XXX Keep this until Includes/Deprecated is removed 186 if (qualified_name.startswith('python') or 187 qualified_name in ('stdlib', 'stdio', 'stl')): 188 standard_include_path = os.path.abspath(os.path.normpath( 189 os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) 190 deprecated_include_path = os.path.join(standard_include_path, 'Deprecated') 191 self.include_directories.append(deprecated_include_path) 192 try: 193 pxd = self.search_include_directories(qualified_name, ".pxd", pos) 194 finally: 195 self.include_directories.pop() 196 if pxd: 197 name = qualified_name 198 if name.startswith('python'): 199 warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1) 200 elif name in ('stdlib', 'stdio'): 201 warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1) 202 elif name in ('stl'): 203 warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1) 204 if pxd is None and Options.cimport_from_pyx: 205 return self.find_pyx_file(qualified_name, pos) 206 return pxd 207 208 def find_pyx_file(self, qualified_name, pos): 209 # Search include path for the .pyx file corresponding to the 210 # given fully-qualified module name, as for find_pxd_file(). 211 return self.search_include_directories(qualified_name, ".pyx", pos) 212 213 def find_include_file(self, filename, pos): 214 # Search list of include directories for filename. 215 # Reports an error and returns None if not found. 216 path = self.search_include_directories(filename, "", pos, 217 include=True) 218 if not path: 219 error(pos, "'%s' not found" % filename) 220 return path 221 222 def search_include_directories(self, qualified_name, suffix, pos, 223 include=False, sys_path=False): 224 return Utils.search_include_directories( 225 tuple(self.include_directories), qualified_name, suffix, pos, include, sys_path) 226 227 def find_root_package_dir(self, file_path): 228 return Utils.find_root_package_dir(file_path) 229 230 def check_package_dir(self, dir, package_names): 231 return Utils.check_package_dir(dir, tuple(package_names)) 232 233 def c_file_out_of_date(self, source_path): 234 c_path = Utils.replace_suffix(source_path, ".c") 235 if not os.path.exists(c_path): 236 return 1 237 c_time = Utils.modification_time(c_path) 238 if Utils.file_newer_than(source_path, c_time): 239 return 1 240 pos = [source_path] 241 pxd_path = Utils.replace_suffix(source_path, ".pxd") 242 if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time): 243 return 1 244 for kind, name in self.read_dependency_file(source_path): 245 if kind == "cimport": 246 dep_path = self.find_pxd_file(name, pos) 247 elif kind == "include": 248 dep_path = self.search_include_directories(name, pos) 249 else: 250 continue 251 if dep_path and Utils.file_newer_than(dep_path, c_time): 252 return 1 253 return 0 254 255 def find_cimported_module_names(self, source_path): 256 return [ name for kind, name in self.read_dependency_file(source_path) 257 if kind == "cimport" ] 258 259 def is_package_dir(self, dir_path): 260 return Utils.is_package_dir(dir_path) 261 262 def read_dependency_file(self, source_path): 263 dep_path = Utils.replace_suffix(source_path, ".dep") 264 if os.path.exists(dep_path): 265 f = open(dep_path, "rU") 266 chunks = [ line.strip().split(" ", 1) 267 for line in f.readlines() 268 if " " in line.strip() ] 269 f.close() 270 return chunks 271 else: 272 return () 273 274 def lookup_submodule(self, name): 275 # Look up a top-level module. Returns None if not found. 276 return self.modules.get(name, None) 277 278 def find_submodule(self, name): 279 # Find a top-level module, creating a new one if needed. 280 scope = self.lookup_submodule(name) 281 if not scope: 282 scope = ModuleScope(name, 283 parent_module = None, context = self) 284 self.modules[name] = scope 285 return scope 286 287 def parse(self, source_desc, scope, pxd, full_module_name): 288 if not isinstance(source_desc, FileSourceDescriptor): 289 raise RuntimeError("Only file sources for code supported") 290 source_filename = source_desc.filename 291 scope.cpp = self.cpp 292 # Parse the given source file and return a parse tree. 293 num_errors = Errors.num_errors 294 try: 295 f = Utils.open_source_file(source_filename, "rU") 296 try: 297 import Parsing 298 s = PyrexScanner(f, source_desc, source_encoding = f.encoding, 299 scope = scope, context = self) 300 tree = Parsing.p_module(s, pxd, full_module_name) 301 finally: 302 f.close() 303 except UnicodeDecodeError, e: 304 #import traceback 305 #traceback.print_exc() 306 line = 1 307 column = 0 308 msg = e.args[-1] 309 position = e.args[2] 310 encoding = e.args[0] 311 312 f = open(source_filename, "rb") 313 try: 314 byte_data = f.read() 315 finally: 316 f.close() 317 318 # FIXME: make this at least a little less inefficient 319 for idx, c in enumerate(byte_data): 320 if c in (ord('\n'), '\n'): 321 line += 1 322 column = 0 323 if idx == position: 324 break 325 326 column += 1 327 328 error((source_desc, line, column), 329 "Decoding error, missing or incorrect coding=<encoding-name> " 330 "at top of source (cannot decode with encoding %r: %s)" % (encoding, msg)) 331 332 if Errors.num_errors > num_errors: 333 raise CompileError() 334 return tree 335 336 def extract_module_name(self, path, options): 337 # Find fully_qualified module name from the full pathname 338 # of a source file. 339 dir, filename = os.path.split(path) 340 module_name, _ = os.path.splitext(filename) 341 if "." in module_name: 342 return module_name 343 names = [module_name] 344 while self.is_package_dir(dir): 345 parent, package_name = os.path.split(dir) 346 if parent == dir: 347 break 348 names.append(package_name) 349 dir = parent 350 names.reverse() 351 return ".".join(names) 352 353 def setup_errors(self, options, result): 354 Errors.reset() # clear any remaining error state 355 if options.use_listing_file: 356 result.listing_file = Utils.replace_suffix(source, ".lis") 357 path = result.listing_file 358 else: 359 path = None 360 Errors.open_listing_file(path=path, 361 echo_to_stderr=options.errors_to_stderr) 362 363 def teardown_errors(self, err, options, result): 364 source_desc = result.compilation_source.source_desc 365 if not isinstance(source_desc, FileSourceDescriptor): 366 raise RuntimeError("Only file sources for code supported") 367 Errors.close_listing_file() 368 result.num_errors = Errors.num_errors 369 if result.num_errors > 0: 370 err = True 371 if err and result.c_file: 372 try: 373 Utils.castrate_file(result.c_file, os.stat(source_desc.filename)) 374 except EnvironmentError: 375 pass 376 result.c_file = None 377 378def create_default_resultobj(compilation_source, options): 379 result = CompilationResult() 380 result.main_source_file = compilation_source.source_desc.filename 381 result.compilation_source = compilation_source 382 source_desc = compilation_source.source_desc 383 if options.output_file: 384 result.c_file = os.path.join(compilation_source.cwd, options.output_file) 385 else: 386 if options.cplus: 387 c_suffix = ".cpp" 388 else: 389 c_suffix = ".c" 390 result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix) 391 return result 392 393def run_pipeline(source, options, full_module_name=None, context=None): 394 import Pipeline 395 396 source_ext = os.path.splitext(source)[1] 397 options.configure_language_defaults(source_ext[1:]) # py/pyx 398 if context is None: 399 context = options.create_context() 400 401 # Set up source object 402 cwd = os.getcwd() 403 abs_path = os.path.abspath(source) 404 full_module_name = full_module_name or context.extract_module_name(source, options) 405 406 if options.relative_path_in_code_position_comments: 407 rel_path = full_module_name.replace('.', os.sep) + source_ext 408 if not abs_path.endswith(rel_path): 409 rel_path = source # safety measure to prevent printing incorrect paths 410 else: 411 rel_path = abs_path 412 source_desc = FileSourceDescriptor(abs_path, rel_path) 413 source = CompilationSource(source_desc, full_module_name, cwd) 414 415 # Set up result object 416 result = create_default_resultobj(source, options) 417 418 if options.annotate is None: 419 # By default, decide based on whether an html file already exists. 420 html_filename = os.path.splitext(result.c_file)[0] + ".html" 421 if os.path.exists(html_filename): 422 line = codecs.open(html_filename, "r", encoding="UTF-8").readline() 423 if line.startswith(u'<!-- Generated by Cython'): 424 options.annotate = True 425 426 # Get pipeline 427 if source_ext.lower() == '.py' or not source_ext: 428 pipeline = Pipeline.create_py_pipeline(context, options, result) 429 else: 430 pipeline = Pipeline.create_pyx_pipeline(context, options, result) 431 432 context.setup_errors(options, result) 433 err, enddata = Pipeline.run_pipeline(pipeline, source) 434 context.teardown_errors(err, options, result) 435 return result 436 437 438#------------------------------------------------------------------------ 439# 440# Main Python entry points 441# 442#------------------------------------------------------------------------ 443 444class CompilationSource(object): 445 """ 446 Contains the data necesarry to start up a compilation pipeline for 447 a single compilation unit. 448 """ 449 def __init__(self, source_desc, full_module_name, cwd): 450 self.source_desc = source_desc 451 self.full_module_name = full_module_name 452 self.cwd = cwd 453 454class CompilationOptions(object): 455 """ 456 Options to the Cython compiler: 457 458 show_version boolean Display version number 459 use_listing_file boolean Generate a .lis file 460 errors_to_stderr boolean Echo errors to stderr when using .lis 461 include_path [string] Directories to search for include files 462 output_file string Name of generated .c file 463 generate_pxi boolean Generate .pxi file for public declarations 464 capi_reexport_cincludes 465 boolean Add cincluded headers to any auto-generated 466 header files. 467 timestamps boolean Only compile changed source files. 468 verbose boolean Always print source names being compiled 469 compiler_directives dict Overrides for pragma options (see Options.py) 470 evaluate_tree_assertions boolean Test support: evaluate parse tree assertions 471 language_level integer The Python language level: 2 or 3 472 473 cplus boolean Compile as c++ code 474 """ 475 476 def __init__(self, defaults = None, **kw): 477 self.include_path = [] 478 if defaults: 479 if isinstance(defaults, CompilationOptions): 480 defaults = defaults.__dict__ 481 else: 482 defaults = default_options 483 484 options = dict(defaults) 485 options.update(kw) 486 487 directives = dict(options['compiler_directives']) # copy mutable field 488 options['compiler_directives'] = directives 489 if 'language_level' in directives and 'language_level' not in kw: 490 options['language_level'] = int(directives['language_level']) 491 if 'cache' in options: 492 if options['cache'] is True: 493 options['cache'] = os.path.expanduser("~/.cycache") 494 elif options['cache'] in (False, None): 495 del options['cache'] 496 497 self.__dict__.update(options) 498 499 def configure_language_defaults(self, source_extension): 500 if source_extension == 'py': 501 if self.compiler_directives.get('binding') is None: 502 self.compiler_directives['binding'] = True 503 504 def create_context(self): 505 return Context(self.include_path, self.compiler_directives, 506 self.cplus, self.language_level, options=self) 507 508 509class CompilationResult(object): 510 """ 511 Results from the Cython compiler: 512 513 c_file string or None The generated C source file 514 h_file string or None The generated C header file 515 i_file string or None The generated .pxi file 516 api_file string or None The generated C API .h file 517 listing_file string or None File of error messages 518 object_file string or None Result of compiling the C file 519 extension_file string or None Result of linking the object file 520 num_errors integer Number of compilation errors 521 compilation_source CompilationSource 522 """ 523 524 def __init__(self): 525 self.c_file = None 526 self.h_file = None 527 self.i_file = None 528 self.api_file = None 529 self.listing_file = None 530 self.object_file = None 531 self.extension_file = None 532 self.main_source_file = None 533 534 535class CompilationResultSet(dict): 536 """ 537 Results from compiling multiple Pyrex source files. A mapping 538 from source file paths to CompilationResult instances. Also 539 has the following attributes: 540 541 num_errors integer Total number of compilation errors 542 """ 543 544 num_errors = 0 545 546 def add(self, source, result): 547 self[source] = result 548 self.num_errors += result.num_errors 549 550 551def compile_single(source, options, full_module_name = None): 552 """ 553 compile_single(source, options, full_module_name) 554 555 Compile the given Pyrex implementation file and return a CompilationResult. 556 Always compiles a single file; does not perform timestamp checking or 557 recursion. 558 """ 559 return run_pipeline(source, options, full_module_name) 560 561 562def compile_multiple(sources, options): 563 """ 564 compile_multiple(sources, options) 565 566 Compiles the given sequence of Pyrex implementation files and returns 567 a CompilationResultSet. Performs timestamp checking and/or recursion 568 if these are specified in the options. 569 """ 570 # run_pipeline creates the context 571 # context = options.create_context() 572 sources = [os.path.abspath(source) for source in sources] 573 processed = set() 574 results = CompilationResultSet() 575 timestamps = options.timestamps 576 verbose = options.verbose 577 context = None 578 for source in sources: 579 if source not in processed: 580 if context is None: 581 context = options.create_context() 582 if not timestamps or context.c_file_out_of_date(source): 583 if verbose: 584 sys.stderr.write("Compiling %s\n" % source) 585 586 result = run_pipeline(source, options, context=context) 587 results.add(source, result) 588 # Compiling multiple sources in one context doesn't quite 589 # work properly yet. 590 context = None 591 processed.add(source) 592 return results 593 594def compile(source, options = None, full_module_name = None, **kwds): 595 """ 596 compile(source [, options], [, <option> = <value>]...) 597 598 Compile one or more Pyrex implementation files, with optional timestamp 599 checking and recursing on dependecies. The source argument may be a string 600 or a sequence of strings If it is a string and no recursion or timestamp 601 checking is requested, a CompilationResult is returned, otherwise a 602 CompilationResultSet is returned. 603 """ 604 options = CompilationOptions(defaults = options, **kwds) 605 if isinstance(source, basestring) and not options.timestamps: 606 return compile_single(source, options, full_module_name) 607 else: 608 return compile_multiple(source, options) 609 610#------------------------------------------------------------------------ 611# 612# Main command-line entry point 613# 614#------------------------------------------------------------------------ 615def setuptools_main(): 616 return main(command_line = 1) 617 618def main(command_line = 0): 619 args = sys.argv[1:] 620 any_failures = 0 621 if command_line: 622 from CmdLine import parse_command_line 623 options, sources = parse_command_line(args) 624 else: 625 options = CompilationOptions(default_options) 626 sources = args 627 628 if options.show_version: 629 sys.stderr.write("Cython version %s\n" % Version.version) 630 if options.working_path!="": 631 os.chdir(options.working_path) 632 try: 633 result = compile(sources, options) 634 if result.num_errors > 0: 635 any_failures = 1 636 except (EnvironmentError, PyrexError), e: 637 sys.stderr.write(str(e) + '\n') 638 any_failures = 1 639 if any_failures: 640 sys.exit(1) 641 642 643 644#------------------------------------------------------------------------ 645# 646# Set the default options depending on the platform 647# 648#------------------------------------------------------------------------ 649 650default_options = dict( 651 show_version = 0, 652 use_listing_file = 0, 653 errors_to_stderr = 1, 654 cplus = 0, 655 output_file = None, 656 annotate = None, 657 generate_pxi = 0, 658 capi_reexport_cincludes = 0, 659 working_path = "", 660 timestamps = None, 661 verbose = 0, 662 quiet = 0, 663 compiler_directives = {}, 664 evaluate_tree_assertions = False, 665 emit_linenums = False, 666 relative_path_in_code_position_comments = True, 667 c_line_in_traceback = True, 668 language_level = 2, 669 gdb_debug = False, 670 compile_time_env = None, 671 common_utility_include_dir = None, 672) 673