1"""
2Import hooks; when installed with the install() function, these hooks
3allow importing .pyx files as if they were Python modules.
4
5If you want the hook installed every time you run Python
6you can add it to your Python version by adding these lines to
7sitecustomize.py (which you can create from scratch in site-packages
8if it doesn't exist there or somewhere else on your python path)::
9
10    import pyximport
11    pyximport.install()
12
13For instance on the Mac with a non-system Python 2.3, you could create
14sitecustomize.py with only those two lines at
15/usr/local/lib/python2.3/site-packages/sitecustomize.py .
16
17A custom distutils.core.Extension instance and setup() args
18(Distribution) for for the build can be defined by a <modulename>.pyxbld
19file like:
20
21# examplemod.pyxbld
22def make_ext(modname, pyxfilename):
23    from distutils.extension import Extension
24    return Extension(name = modname,
25                     sources=[pyxfilename, 'hello.c'],
26                     include_dirs=['/myinclude'] )
27def make_setup_args():
28    return dict(script_args=["--compiler=mingw32"])
29
30Extra dependencies can be defined by a <modulename>.pyxdep .
31See README.
32
33Since Cython 0.11, the :mod:`pyximport` module also has experimental
34compilation support for normal Python modules.  This allows you to
35automatically run Cython on every .pyx and .py module that Python
36imports, including parts of the standard library and installed
37packages.  Cython will still fail to compile a lot of Python modules,
38in which case the import mechanism will fall back to loading the
39Python source modules instead.  The .py import mechanism is installed
40like this::
41
42    pyximport.install(pyimport = True)
43
44Running this module as a top-level script will run a test and then print
45the documentation.
46
47This code is based on the Py2.3+ import protocol as described in PEP 302.
48"""
49
50import sys
51import os
52import glob
53import imp
54
55mod_name = "pyximport"
56
57assert sys.hexversion >= 0x2030000, "need Python 2.3 or later"
58
59PYX_EXT = ".pyx"
60PYXDEP_EXT = ".pyxdep"
61PYXBLD_EXT = ".pyxbld"
62
63DEBUG_IMPORT = False
64
65def _print(message, args):
66    if args:
67        message = message % args
68    print(message)
69
70def _debug(message, *args):
71    if DEBUG_IMPORT:
72        _print(message, args)
73
74def _info(message, *args):
75    _print(message, args)
76
77# Performance problem: for every PYX file that is imported, we will
78# invoke the whole distutils infrastructure even if the module is
79# already built. It might be more efficient to only do it when the
80# mod time of the .pyx is newer than the mod time of the .so but
81# the question is how to get distutils to tell me the name of the .so
82# before it builds it. Maybe it is easy...but maybe the peformance
83# issue isn't real.
84def _load_pyrex(name, filename):
85    "Load a pyrex file given a name and filename."
86
87def get_distutils_extension(modname, pyxfilename, language_level=None):
88#    try:
89#        import hashlib
90#    except ImportError:
91#        import md5 as hashlib
92#    extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
93#    modname = modname + extra
94    extension_mod,setup_args = handle_special_build(modname, pyxfilename)
95    if not extension_mod:
96        if not isinstance(pyxfilename, str):
97            # distutils is stupid in Py2 and requires exactly 'str'
98            # => encode accidentally coerced unicode strings back to str
99            pyxfilename = pyxfilename.encode(sys.getfilesystemencoding())
100        from distutils.extension import Extension
101        extension_mod = Extension(name = modname, sources=[pyxfilename])
102        if language_level is not None:
103            extension_mod.cython_directives = {'language_level': language_level}
104    return extension_mod,setup_args
105
106def handle_special_build(modname, pyxfilename):
107    special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
108    ext = None
109    setup_args={}
110    if os.path.exists(special_build):
111        # globls = {}
112        # locs = {}
113        # execfile(special_build, globls, locs)
114        # ext = locs["make_ext"](modname, pyxfilename)
115        mod = imp.load_source("XXXX", special_build, open(special_build))
116        make_ext = getattr(mod,'make_ext',None)
117        if make_ext:
118            ext = make_ext(modname, pyxfilename)
119            assert ext and ext.sources, ("make_ext in %s did not return Extension"
120                                         % special_build)
121        make_setup_args = getattr(mod,'make_setup_args',None)
122        if make_setup_args:
123            setup_args = make_setup_args()
124            assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
125                                         % special_build)
126        assert set or setup_args, ("neither make_ext nor make_setup_args %s"
127                                         % special_build)
128        ext.sources = [os.path.join(os.path.dirname(special_build), source)
129                       for source in ext.sources]
130    return ext, setup_args
131
132def handle_dependencies(pyxfilename):
133    testing = '_test_files' in globals()
134    dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
135
136    # by default let distutils decide whether to rebuild on its own
137    # (it has a better idea of what the output file will be)
138
139    # but we know more about dependencies so force a rebuild if
140    # some of the dependencies are newer than the pyxfile.
141    if os.path.exists(dependfile):
142        depends = open(dependfile).readlines()
143        depends = [depend.strip() for depend in depends]
144
145        # gather dependencies in the "files" variable
146        # the dependency file is itself a dependency
147        files = [dependfile]
148        for depend in depends:
149            fullpath = os.path.join(os.path.dirname(dependfile),
150                                    depend)
151            files.extend(glob.glob(fullpath))
152
153        # only for unit testing to see we did the right thing
154        if testing:
155            _test_files[:] = []  #$pycheck_no
156
157        # if any file that the pyxfile depends upon is newer than
158        # the pyx file, 'touch' the pyx file so that distutils will
159        # be tricked into rebuilding it.
160        for file in files:
161            from distutils.dep_util import newer
162            if newer(file, pyxfilename):
163                _debug("Rebuilding %s because of %s", pyxfilename, file)
164                filetime = os.path.getmtime(file)
165                os.utime(pyxfilename, (filetime, filetime))
166                if testing:
167                    _test_files.append(file)
168
169def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None):
170    assert os.path.exists(pyxfilename), (
171        "Path does not exist: %s" % pyxfilename)
172    handle_dependencies(pyxfilename)
173
174    extension_mod,setup_args = get_distutils_extension(name, pyxfilename, language_level)
175    build_in_temp=pyxargs.build_in_temp
176    sargs=pyxargs.setup_args.copy()
177    sargs.update(setup_args)
178    build_in_temp=sargs.pop('build_in_temp',build_in_temp)
179
180    import pyxbuild
181    so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
182                                  build_in_temp=build_in_temp,
183                                  pyxbuild_dir=pyxbuild_dir,
184                                  setup_args=sargs,
185                                  inplace=inplace,
186                                  reload_support=pyxargs.reload_support)
187    assert os.path.exists(so_path), "Cannot find: %s" % so_path
188
189    junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;)
190    junkstuff = glob.glob(junkpath)
191    for path in junkstuff:
192        if path!=so_path:
193            try:
194                os.remove(path)
195            except IOError:
196                _info("Couldn't remove %s", path)
197
198    return so_path
199
200def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False,
201                build_inplace=False, language_level=None, so_path=None):
202    try:
203        if so_path is None:
204            if is_package:
205                module_name = name + '.__init__'
206            else:
207                module_name = name
208            so_path = build_module(module_name, pyxfilename, pyxbuild_dir,
209                                   inplace=build_inplace, language_level=language_level)
210        mod = imp.load_dynamic(name, so_path)
211        if is_package and not hasattr(mod, '__path__'):
212            mod.__path__ = [os.path.dirname(so_path)]
213        assert mod.__file__ == so_path, (mod.__file__, so_path)
214    except Exception:
215        if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'):
216            # try to fall back to normal import
217            mod = imp.load_source(name, pyxfilename)
218            assert mod.__file__ in (pyxfilename, pyxfilename+'c', pyxfilename+'o'), (mod.__file__, pyxfilename)
219        else:
220            import traceback
221            raise ImportError("Building module %s failed: %s" %
222                              (name,
223                               traceback.format_exception_only(*sys.exc_info()[:2]))), None, sys.exc_info()[2]
224    return mod
225
226
227# import hooks
228
229class PyxImporter(object):
230    """A meta-path importer for .pyx files.
231    """
232    def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False,
233                 language_level=None):
234        self.extension = extension
235        self.pyxbuild_dir = pyxbuild_dir
236        self.inplace = inplace
237        self.language_level = language_level
238
239    def find_module(self, fullname, package_path=None):
240        if fullname in sys.modules  and  not pyxargs.reload_support:
241            return None  # only here when reload()
242        try:
243            fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
244            if fp: fp.close()  # Python should offer a Default-Loader to avoid this double find/open!
245            if pathname and ty == imp.PKG_DIRECTORY:
246                pkg_file = os.path.join(pathname, '__init__'+self.extension)
247                if os.path.isfile(pkg_file):
248                    return PyxLoader(fullname, pathname,
249                        init_path=pkg_file,
250                        pyxbuild_dir=self.pyxbuild_dir,
251                        inplace=self.inplace,
252                        language_level=self.language_level)
253            if pathname and pathname.endswith(self.extension):
254                return PyxLoader(fullname, pathname,
255                                 pyxbuild_dir=self.pyxbuild_dir,
256                                 inplace=self.inplace,
257                                 language_level=self.language_level)
258            if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
259                return None
260
261            # find .pyx fast, when .so/.pyd exist --inplace
262            pyxpath = os.path.splitext(pathname)[0]+self.extension
263            if os.path.isfile(pyxpath):
264                return PyxLoader(fullname, pyxpath,
265                                 pyxbuild_dir=self.pyxbuild_dir,
266                                 inplace=self.inplace,
267                                 language_level=self.language_level)
268
269            # .so/.pyd's on PATH should not be remote from .pyx's
270            # think no need to implement PyxArgs.importer_search_remote here?
271
272        except ImportError:
273            pass
274
275        # searching sys.path ...
276
277        #if DEBUG_IMPORT:  print "SEARCHING", fullname, package_path
278        if '.' in fullname: # only when package_path anyway?
279            mod_parts = fullname.split('.')
280            module_name = mod_parts[-1]
281        else:
282            module_name = fullname
283        pyx_module_name = module_name + self.extension
284        # this may work, but it returns the file content, not its path
285        #import pkgutil
286        #pyx_source = pkgutil.get_data(package, pyx_module_name)
287
288        if package_path:
289            paths = package_path
290        else:
291            paths = sys.path
292        join_path = os.path.join
293        is_file = os.path.isfile
294        is_abs = os.path.isabs
295        abspath = os.path.abspath
296        #is_dir = os.path.isdir
297        sep = os.path.sep
298        for path in paths:
299            if not path:
300                path = os.getcwd()
301            elif not is_abs(path):
302                path = abspath(path)
303            if is_file(path+sep+pyx_module_name):
304                return PyxLoader(fullname, join_path(path, pyx_module_name),
305                                 pyxbuild_dir=self.pyxbuild_dir,
306                                 inplace=self.inplace,
307                                 language_level=self.language_level)
308
309        # not found, normal package, not a .pyx file, none of our business
310        _debug("%s not found" % fullname)
311        return None
312
313class PyImporter(PyxImporter):
314    """A meta-path importer for normal .py files.
315    """
316    def __init__(self, pyxbuild_dir=None, inplace=False, language_level=None):
317        if language_level is None:
318            language_level = sys.version_info[0]
319        self.super = super(PyImporter, self)
320        self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir, inplace=inplace,
321                            language_level=language_level)
322        self.uncompilable_modules = {}
323        self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild',
324                                'distutils.extension', 'distutils.sysconfig']
325
326    def find_module(self, fullname, package_path=None):
327        if fullname in sys.modules:
328            return None
329        if fullname.startswith('Cython.'):
330            return None
331        if fullname in self.blocked_modules:
332            # prevent infinite recursion
333            return None
334        if _lib_loader.knows(fullname):
335            return _lib_loader
336        _debug("trying import of module '%s'", fullname)
337        if fullname in self.uncompilable_modules:
338            path, last_modified = self.uncompilable_modules[fullname]
339            try:
340                new_last_modified = os.stat(path).st_mtime
341                if new_last_modified > last_modified:
342                    # import would fail again
343                    return None
344            except OSError:
345                # module is no longer where we found it, retry the import
346                pass
347
348        self.blocked_modules.append(fullname)
349        try:
350            importer = self.super.find_module(fullname, package_path)
351            if importer is not None:
352                if importer.init_path:
353                    path = importer.init_path
354                    real_name = fullname + '.__init__'
355                else:
356                    path = importer.path
357                    real_name = fullname
358                _debug("importer found path %s for module %s", path, real_name)
359                try:
360                    so_path = build_module(
361                        real_name, path,
362                        pyxbuild_dir=self.pyxbuild_dir,
363                        language_level=self.language_level,
364                        inplace=self.inplace)
365                    _lib_loader.add_lib(fullname, path, so_path,
366                                        is_package=bool(importer.init_path))
367                    return _lib_loader
368                except Exception:
369                    if DEBUG_IMPORT:
370                        import traceback
371                        traceback.print_exc()
372                    # build failed, not a compilable Python module
373                    try:
374                        last_modified = os.stat(path).st_mtime
375                    except OSError:
376                        last_modified = 0
377                    self.uncompilable_modules[fullname] = (path, last_modified)
378                    importer = None
379        finally:
380            self.blocked_modules.pop()
381        return importer
382
383class LibLoader(object):
384    def __init__(self):
385        self._libs = {}
386
387    def load_module(self, fullname):
388        try:
389            source_path, so_path, is_package = self._libs[fullname]
390        except KeyError:
391            raise ValueError("invalid module %s" % fullname)
392        _debug("Loading shared library module '%s' from %s", fullname, so_path)
393        return load_module(fullname, source_path, so_path=so_path, is_package=is_package)
394
395    def add_lib(self, fullname, path, so_path, is_package):
396        self._libs[fullname] = (path, so_path, is_package)
397
398    def knows(self, fullname):
399        return fullname in self._libs
400
401_lib_loader = LibLoader()
402
403class PyxLoader(object):
404    def __init__(self, fullname, path, init_path=None, pyxbuild_dir=None,
405                 inplace=False, language_level=None):
406        _debug("PyxLoader created for loading %s from %s (init path: %s)",
407               fullname, path, init_path)
408        self.fullname = fullname
409        self.path, self.init_path = path, init_path
410        self.pyxbuild_dir = pyxbuild_dir
411        self.inplace = inplace
412        self.language_level = language_level
413
414    def load_module(self, fullname):
415        assert self.fullname == fullname, (
416            "invalid module, expected %s, got %s" % (
417            self.fullname, fullname))
418        if self.init_path:
419            # package
420            #print "PACKAGE", fullname
421            module = load_module(fullname, self.init_path,
422                                 self.pyxbuild_dir, is_package=True,
423                                 build_inplace=self.inplace,
424                                 language_level=self.language_level)
425            module.__path__ = [self.path]
426        else:
427            #print "MODULE", fullname
428            module = load_module(fullname, self.path,
429                                 self.pyxbuild_dir,
430                                 build_inplace=self.inplace,
431                                 language_level=self.language_level)
432        return module
433
434
435#install args
436class PyxArgs(object):
437    build_dir=True
438    build_in_temp=True
439    setup_args={}   #None
440
441##pyxargs=None
442
443def _have_importers():
444    has_py_importer = False
445    has_pyx_importer = False
446    for importer in sys.meta_path:
447        if isinstance(importer, PyxImporter):
448            if isinstance(importer, PyImporter):
449                has_py_importer = True
450            else:
451                has_pyx_importer = True
452
453    return has_py_importer, has_pyx_importer
454
455def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
456            setup_args={}, reload_support=False,
457            load_py_module_on_import_failure=False, inplace=False,
458            language_level=None):
459    """Main entry point. Call this to install the .pyx import hook in
460    your meta-path for a single Python process.  If you want it to be
461    installed whenever you use Python, add it to your sitecustomize
462    (as described above).
463
464    You can pass ``pyimport=True`` to also install the .py import hook
465    in your meta-path.  Note, however, that it is highly experimental,
466    will not work for most .py files, and will therefore only slow
467    down your imports.  Use at your own risk.
468
469    By default, compiled modules will end up in a ``.pyxbld``
470    directory in the user's home directory.  Passing a different path
471    as ``build_dir`` will override this.
472
473    ``build_in_temp=False`` will produce the C files locally. Working
474    with complex dependencies and debugging becomes more easy. This
475    can principally interfere with existing files of the same name.
476    build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args()
477    by a dict item of 'build_in_temp'
478
479    ``setup_args``: dict of arguments for Distribution - see
480    distutils.core.setup() . They are extended/overriden by those of
481    <modulename>.pyxbld/make_setup_args()
482
483    ``reload_support``:  Enables support for dynamic
484    reload(<pyxmodulename>), e.g. after a change in the Cython code.
485    Additional files <so_path>.reloadNN may arise on that account, when
486    the previously loaded module file cannot be overwritten.
487
488    ``load_py_module_on_import_failure``: If the compilation of a .py
489    file succeeds, but the subsequent import fails for some reason,
490    retry the import with the normal .py module instead of the
491    compiled module.  Note that this may lead to unpredictable results
492    for modules that change the system state during their import, as
493    the second import will rerun these modifications in whatever state
494    the system was left after the import of the compiled module
495    failed.
496
497    ``inplace``: Install the compiled module next to the source file.
498
499    ``language_level``: The source language level to use: 2 or 3.
500    The default is to use the language level of the current Python
501    runtime for .py files and Py2 for .pyx files.
502    """
503    if not build_dir:
504        build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld')
505
506    global pyxargs
507    pyxargs = PyxArgs()  #$pycheck_no
508    pyxargs.build_dir = build_dir
509    pyxargs.build_in_temp = build_in_temp
510    pyxargs.setup_args = (setup_args or {}).copy()
511    pyxargs.reload_support = reload_support
512    pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure
513
514    has_py_importer, has_pyx_importer = _have_importers()
515    py_importer, pyx_importer = None, None
516
517    if pyimport and not has_py_importer:
518        py_importer = PyImporter(pyxbuild_dir=build_dir, inplace=inplace,
519                                 language_level=language_level)
520        # make sure we import Cython before we install the import hook
521        import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize
522        sys.meta_path.insert(0, py_importer)
523
524    if pyximport and not has_pyx_importer:
525        pyx_importer = PyxImporter(pyxbuild_dir=build_dir, inplace=inplace,
526                                   language_level=language_level)
527        sys.meta_path.append(pyx_importer)
528
529    return py_importer, pyx_importer
530
531def uninstall(py_importer, pyx_importer):
532    """
533    Uninstall an import hook.
534    """
535    try:
536        sys.meta_path.remove(py_importer)
537    except ValueError:
538        pass
539
540    try:
541        sys.meta_path.remove(pyx_importer)
542    except ValueError:
543        pass
544
545# MAIN
546
547def show_docs():
548    import __main__
549    __main__.__name__ = mod_name
550    for name in dir(__main__):
551        item = getattr(__main__, name)
552        try:
553            setattr(item, "__module__", mod_name)
554        except (AttributeError, TypeError):
555            pass
556    help(__main__)
557
558if __name__ == '__main__':
559    show_docs()
560