15f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import itertools 25f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from time import time 35f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 45f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import Errors 55f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import DebugFlags 65f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import Options 75f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from Visitor import CythonTransform 85f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from Errors import CompileError, InternalError, AbortError 95f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import Naming 105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# 125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# Really small pipeline stages 135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# 145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def dumptree(t): 155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # For quick debugging in pipelines 165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print t.dump() 175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return t 185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def abort_on_errors(node): 205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Stop the pipeline if there are any errors. 215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if Errors.num_errors != 0: 225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) raise AbortError("pipeline break") 235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return node 245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def parse_stage_factory(context): 265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def parse(compsrc): 275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) source_desc = compsrc.source_desc 285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) full_module_name = compsrc.full_module_name 295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) initial_pos = (source_desc, 1, 0) 305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) saved_cimport_from_pyx, Options.cimport_from_pyx = Options.cimport_from_pyx, False 315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0, 325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) check_module_name = not Options.embed) 335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Options.cimport_from_pyx = saved_cimport_from_pyx 345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name) 355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree.compilation_source = compsrc 365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree.scope = scope 375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree.is_pxd = False 385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return tree 395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return parse 405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def parse_pxd_stage_factory(context, scope, module_name): 425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def parse(source_desc): 435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree = context.parse(source_desc, scope, pxd=True, 445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) full_module_name=module_name) 455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree.scope = scope 465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree.is_pxd = True 475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return tree 485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return parse 495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def generate_pyx_code_stage_factory(options, result): 515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def generate_pyx_code_stage(module_node): 525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) module_node.process_implementation(options, result) 535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) result.compilation_source = module_node.compilation_source 545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return result 555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return generate_pyx_code_stage 565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def inject_pxd_code_stage_factory(context): 585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def inject_pxd_code_stage(module_node): 595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from textwrap import dedent 605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) stats = module_node.body.stats 615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for name, (statlistnode, scope) in context.pxds.iteritems(): 625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) module_node.merge_in(statlistnode, scope) 635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return module_node 645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return inject_pxd_code_stage 655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def use_utility_code_definitions(scope, target, seen=None): 675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if seen is None: 685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) seen = set() 695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for entry in scope.entries.itervalues(): 715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if entry in seen: 725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) continue 735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) seen.add(entry) 755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if entry.used and entry.utility_code_definition: 765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) target.use_utility_code(entry.utility_code_definition) 775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for required_utility in entry.utility_code_definition.requires: 785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) target.use_utility_code(required_utility) 795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) elif entry.as_module: 805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) use_utility_code_definitions(entry.as_module, target, seen) 815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def inject_utility_code_stage_factory(context): 835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def inject_utility_code_stage(module_node): 845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) use_utility_code_definitions(context.cython_scope, module_node.scope) 855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) added = [] 865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Note: the list might be extended inside the loop (if some utility code 875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # pulls in other utility code, explicitly or implicitly) 885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for utilcode in module_node.scope.utility_code_list: 895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if utilcode in added: continue 905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) added.append(utilcode) 915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if utilcode.requires: 925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for dep in utilcode.requires: 935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not dep in added and not dep in module_node.scope.utility_code_list: 945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) module_node.scope.utility_code_list.append(dep) 955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) tree = utilcode.get_tree() 965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if tree: 975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) module_node.merge_in(tree.body, tree.scope, merge_scope=True) 985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return module_node 995f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return inject_utility_code_stage 1005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)class UseUtilityCodeDefinitions(CythonTransform): 1025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Temporary hack to use any utility code in nodes' "utility_code_definitions". 1035f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # This should be moved to the code generation phase of the relevant nodes once 1045f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # it is safe to generate CythonUtilityCode at code generation time. 1055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def __call__(self, node): 1065f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.scope = node.scope 1075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return super(UseUtilityCodeDefinitions, self).__call__(node) 1085f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1095f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def process_entry(self, entry): 1105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if entry: 1115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for utility_code in (entry.utility_code, entry.utility_code_definition): 1125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if utility_code: 1135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.scope.use_utility_code(utility_code) 1145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def visit_AttributeNode(self, node): 1165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.process_entry(node.entry) 1175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return node 1185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def visit_NameNode(self, node): 1205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.process_entry(node.entry) 1215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self.process_entry(node.type_entry) 1225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return node 1235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# 1255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# Pipeline factories 1265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# 1275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def create_pipeline(context, mode, exclude_classes=()): 1295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) assert mode in ('pyx', 'py', 'pxd') 1305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Visitor import PrintTree 1315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse 1325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform 1335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import AnalyseExpressionsTransform, FindInvalidUseOfFusedTypes 1345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform 1355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods 1365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform 1375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import CalculateQualifiedNamesTransform 1385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic 1395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions 1405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import RemoveUnreachableCode, GilCheck 1415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from FlowControl import ControlFlowAnalysis 1425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from AnalysedTreeTransforms import AutoTestDictTransform 1435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from AutoDocTransforms import EmbedSignature 1445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform 1455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls 1465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Optimize import InlineDefNodeCalls 1475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Optimize import ConstantFolding, FinalOptimizePhase 1485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Optimize import DropRefcountingTransform 1495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Optimize import ConsolidateOverflowCheck 1505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Buffer import IntroduceBufferAuxiliaryVars 1515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ModuleNode import check_c_declarations, check_c_declarations_pxd 1525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if mode == 'pxd': 1555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _check_c_declarations = check_c_declarations_pxd 1565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _specific_post_parse = PxdPostParse(context) 1575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 1585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _check_c_declarations = check_c_declarations 1595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _specific_post_parse = None 1605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if mode == 'py': 1625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _align_function_definitions = AlignFunctionDefinitions(context) 1635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 1645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _align_function_definitions = None 1655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # NOTE: This is the "common" parts of the pipeline, which is also 1675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # code in pxd files. So it will be run multiple times in a 1685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # compilation stage. 1695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) stages = [ 1705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) NormalizeTree(context), 1715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) PostParse(context), 1725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _specific_post_parse, 1735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) InterpretCompilerDirectives(context, context.compiler_directives), 1745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ParallelRangeTransform(context), 1755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) AdjustDefByDirectives(context), 1765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) MarkClosureVisitor(context), 1775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _align_function_definitions, 1785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) RemoveUnreachableCode(context), 1795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ConstantFolding(), 1805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) FlattenInListTransform(), 1815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) WithTransform(context), 1825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) DecoratorTransform(context), 1835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ForwardDeclareTypes(context), 1845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) AnalyseDeclarationsTransform(context), 1855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) AutoTestDictTransform(context), 1865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) EmbedSignature(context), 1875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) EarlyReplaceBuiltinCalls(context), ## Necessary? 1885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) TransformBuiltinMethods(context), ## Necessary? 1895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) MarkParallelAssignments(context), 1905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ControlFlowAnalysis(context), 1915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) RemoveUnreachableCode(context), 1925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # MarkParallelAssignments(context), 1935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) MarkOverflowingArithmetic(context), 1945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) IntroduceBufferAuxiliaryVars(context), 1955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _check_c_declarations, 1965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) InlineDefNodeCalls(context), 1975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) AnalyseExpressionsTransform(context), 1985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) FindInvalidUseOfFusedTypes(context), 1995f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ExpandInplaceOperators(context), 2005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) OptimizeBuiltinCalls(context), ## Necessary? 2015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) CreateClosureClasses(context), ## After all lookups and type inference 2025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) CalculateQualifiedNamesTransform(context), 2035f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ConsolidateOverflowCheck(context), 2045f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) IterationTransform(context), 2055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) SwitchTransform(), 2065f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) DropRefcountingTransform(), 2075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) FinalOptimizePhase(context), 2085f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) GilCheck(), 2095f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) UseUtilityCodeDefinitions(context), 2105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ] 2115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) filtered_stages = [] 2125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for s in stages: 2135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if s.__class__ not in exclude_classes: 2145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) filtered_stages.append(s) 2155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return filtered_stages 2165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): 2185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if py: 2195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) mode = 'py' 2205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 2215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) mode = 'pyx' 2225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) test_support = [] 2235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if options.evaluate_tree_assertions: 2245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Cython.TestUtils import TreeAssertVisitor 2255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) test_support.append(TreeAssertVisitor()) 2265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if options.gdb_debug: 2285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Cython.Debugger import DebugWriter # requires Py2.5+ 2295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import DebugTransform 2305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter( 2315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) options.output_dir) 2325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) debug_transform = [DebugTransform(context, options, result)] 2335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 2345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) debug_transform = [] 2355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return list(itertools.chain( 2375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) [parse_stage_factory(context)], 2385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) create_pipeline(context, mode, exclude_classes=exclude_classes), 2395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) test_support, 2405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) [inject_pxd_code_stage_factory(context), 2415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) inject_utility_code_stage_factory(context), 2425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) abort_on_errors], 2435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) debug_transform, 2445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) [generate_pyx_code_stage_factory(options, result)])) 2455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def create_pxd_pipeline(context, scope, module_name): 2475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from CodeGeneration import ExtractPxdCode 2485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # The pxd pipeline ends up with a CCodeWriter containing the 2505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # code of the pxd, as well as a pxd scope. 2515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return [ 2525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) parse_pxd_stage_factory(context, scope, module_name) 2535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ] + create_pipeline(context, 'pxd') + [ 2545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ExtractPxdCode() 2555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ] 2565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def create_py_pipeline(context, options, result): 2585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return create_pyx_pipeline(context, options, result, py=True) 2595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def create_pyx_as_pxd_pipeline(context, result): 2615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from ParseTreeTransforms import AlignFunctionDefinitions, \ 2625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) MarkClosureVisitor, WithTransform, AnalyseDeclarationsTransform 2635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Optimize import ConstantFolding, FlattenInListTransform 2645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Nodes import StatListNode 2655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline = [] 2665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pyx_pipeline = create_pyx_pipeline(context, context.options, result, 2675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) exclude_classes=[ 2685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) AlignFunctionDefinitions, 2695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) MarkClosureVisitor, 2705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ConstantFolding, 2715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) FlattenInListTransform, 2725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) WithTransform 2735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ]) 2745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for stage in pyx_pipeline: 2755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline.append(stage) 2765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if isinstance(stage, AnalyseDeclarationsTransform): 2775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # This is the last stage we need. 2785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) break 2795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def fake_pxd(root): 2805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for entry in root.scope.entries.values(): 2815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not entry.in_cinclude: 2825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entry.defined_in_pxd = 1 2835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if entry.name == entry.cname and entry.visibility != 'extern': 2845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Always mangle non-extern cimported entries. 2855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) entry.cname = entry.scope.mangle(Naming.func_prefix, entry.name) 2865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return StatListNode(root.pos, stats=[]), root.scope 2875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline.append(fake_pxd) 2885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return pipeline 2895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def insert_into_pipeline(pipeline, transform, before=None, after=None): 2915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 2925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Insert a new transform into the pipeline after or before an instance of 2935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) the given class. e.g. 2945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pipeline = insert_into_pipeline(pipeline, transform, 2965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) after=AnalyseDeclarationsTransform) 2975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 2985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) assert before or after 2995f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 3005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) cls = before or after 3015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for i, t in enumerate(pipeline): 3025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if isinstance(t, cls): 3035f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) break 3045f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 3055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if after: 3065f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) i += 1 3075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 3085f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return pipeline[:i] + [transform] + pipeline[i:] 3095f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 3105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# 3115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# Running a pipeline 3125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# 3135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 3145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def run_pipeline(pipeline, source, printtree=True): 3155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) from Cython.Compiler.Visitor import PrintTree 3165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 3175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) error = None 3185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) data = source 3195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) try: 3205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) try: 3215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for phase in pipeline: 3225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if phase is not None: 3235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if DebugFlags.debug_verbose_pipeline: 3245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) t = time() 3255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print "Entering pipeline phase %r" % phase 3265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not printtree and isinstance(phase, PrintTree): 3275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) continue 3285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) data = phase(data) 3295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if DebugFlags.debug_verbose_pipeline: 3305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print " %.3f seconds" % (time() - t) 3315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) except CompileError, err: 3325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # err is set 3335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Errors.report_error(err) 3345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) error = err 3355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) except InternalError, err: 3365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Only raise if there was not an earlier error 3375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if Errors.num_errors == 0: 3385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) raise 3395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) error = err 3405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) except AbortError, err: 3415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) error = err 3425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return (error, data) 343