15f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from TreeFragment import parse_from_strings, StringParseContext 25f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import Symtab 35f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import Naming 45f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import Code 55f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 65f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)class NonManglingModuleScope(Symtab.ModuleScope): 75f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 85f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def __init__(self, prefix, *args, **kw): 95f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.prefix = prefix 105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.cython_scope = None 115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Symtab.ModuleScope.__init__(self, *args, **kw) 125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def add_imported_entry(self, name, entry, pos): 145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entry.used = True 155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return super(NonManglingModuleScope, self).add_imported_entry( 165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) name, entry, pos) 175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def mangle(self, prefix, name=None): 195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if name: 205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if prefix in (Naming.typeobj_prefix, Naming.func_prefix, Naming.var_prefix, Naming.pyfunc_prefix): 215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Functions, classes etc. gets a manually defined prefix easily 225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # manually callable instead (the one passed to CythonUtilityCode) 235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) prefix = self.prefix 245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return "%s%s" % (prefix, name) 255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return Symtab.ModuleScope.mangle(self, prefix) 275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)class CythonUtilityCodeContext(StringParseContext): 295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) scope = None 305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def find_module(self, module_name, relative_to = None, pos = None, 325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) need_pxd = 1): 335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if module_name != self.module_name: 355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if module_name not in self.modules: 365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) raise AssertionError("Only the cython cimport is supported.") 375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return self.modules[module_name] 395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if self.scope is None: 415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.scope = NonManglingModuleScope(self.prefix, 425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) module_name, 435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) parent_module=None, 445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) context=self) 455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return self.scope 475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)class CythonUtilityCode(Code.UtilityCodeBase): 505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Utility code written in the Cython language itself. 525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The @cname decorator can set the cname for a function, method of cdef class. 545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Functions decorated with @cname('c_func_name') get the given cname. 555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) For cdef classes the rules are as follows: 575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) obj struct -> <cname>_obj 585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) obj type ptr -> <cname>_type 595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) methods -> <class_cname>_<method_cname> 605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) For methods the cname decorator is optional, but without the decorator the 625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) methods will not be prototyped. See Cython.Compiler.CythonScope and 635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tests/run/cythonscope.pyx for examples. 645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) is_cython_utility = True 675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def __init__(self, impl, name="__pyxutil", prefix="", requires=None, 695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) file=None, from_scope=None, context=None): 705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # 1) We need to delay the parsing/processing, so that all modules can be 715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # imported without import loops 725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # 2) The same utility code object can be used for multiple source files; 735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # while the generated node trees can be altered in the compilation of a 745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # single file. 755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Hence, delay any processing until later. 765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if context is not None: 775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) impl = Code.sub_tempita(impl, context, file, name) 785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.impl = impl 795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.name = name 805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.file = file 815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.prefix = prefix 825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.requires = requires or [] 835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.from_scope = from_scope 845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def get_tree(self, entries_only=False, cython_scope=None): 865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from AnalysedTreeTransforms import AutoTestDictTransform 875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # The AutoTestDictTransform creates the statement "__test__ = {}", 885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # which when copied into the main ModuleNode overwrites 895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # any __test__ in user code; not desired 905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) excludes = [AutoTestDictTransform] 915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) import Pipeline, ParseTreeTransforms 935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) context = CythonUtilityCodeContext(self.name) 945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) context.prefix = self.prefix 955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) context.cython_scope = cython_scope 965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) #context = StringParseContext(self.name) 975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree = parse_from_strings(self.name, self.impl, context=context, 985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) allow_struct_enum_decorator=True) 995f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline = Pipeline.create_pipeline(context, 'pyx', exclude_classes=excludes) 1005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if entries_only: 1025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) p = [] 1035f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for t in pipeline: 1045f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) p.append(t) 1055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if isinstance(p, ParseTreeTransforms.AnalyseDeclarationsTransform): 1065f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) break 1075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1085f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline = p 1095f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) transform = ParseTreeTransforms.CnameDirectivesTransform(context) 1115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # InterpretCompilerDirectives already does a cdef declarator check 1125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) #before = ParseTreeTransforms.DecoratorTransform 1135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) before = ParseTreeTransforms.InterpretCompilerDirectives 1145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline = Pipeline.insert_into_pipeline(pipeline, transform, 1155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) before=before) 1165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if self.from_scope: 1185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def scope_transform(module_node): 1195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) module_node.scope.merge_in(self.from_scope) 1205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return module_node 1215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) transform = ParseTreeTransforms.AnalyseDeclarationsTransform 1235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline = Pipeline.insert_into_pipeline(pipeline, scope_transform, 1245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) before=transform) 1255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) (err, tree) = Pipeline.run_pipeline(pipeline, tree, printtree=False) 1275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) assert not err, err 1285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return tree 1295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def put_code(self, output): 1315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pass 1325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) @classmethod 1345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def load_as_string(cls, util_code_name, from_file=None, **kwargs): 1355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 1365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Load a utility code as a string. Returns (proto, implementation) 1375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 1385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) util = cls.load(util_code_name, from_file, **kwargs) 1395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return util.proto, util.impl # keep line numbers => no lstrip() 1405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def declare_in_scope(self, dest_scope, used=False, cython_scope=None, 1425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) whitelist=None): 1435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 1445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Declare all entries from the utility code in dest_scope. Code will only 1455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) be included for used entries. If module_name is given, declare the 1465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) type entries with that name. 1475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 1485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree = self.get_tree(entries_only=True, cython_scope=cython_scope) 1495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entries = tree.scope.entries 1515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entries.pop('__name__') 1525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entries.pop('__file__') 1535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entries.pop('__builtins__') 1545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entries.pop('__doc__') 1555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for name, entry in entries.iteritems(): 1575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entry.utility_code_definition = self 1585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entry.used = used 1595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) original_scope = tree.scope 1615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) dest_scope.merge_in(original_scope, merge_unused=True, 1625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) whitelist=whitelist) 1635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree.scope = dest_scope 1645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for dep in self.requires: 1665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if dep.is_cython_utility: 1675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) dep.declare_in_scope(dest_scope) 1685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return original_scope 1705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def declare_declarations_in_scope(declaration_string, env, private_type=True, 1725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) *args, **kwargs): 1735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 1745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Declare some declarations given as Cython code in declaration_string 1755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) in scope env. 1765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 1775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) CythonUtilityCode(declaration_string, *args, **kwargs).declare_in_scope(env) 178