1# -*- coding: utf-8 -*-
2"""
3    jinja2.bccache
4    ~~~~~~~~~~~~~~
5
6    This module implements the bytecode cache system Jinja is optionally
7    using.  This is useful if you have very complex template situations and
8    the compiliation of all those templates slow down your application too
9    much.
10
11    Situations where this is useful are often forking web applications that
12    are initialized on the first request.
13
14    :copyright: (c) 2010 by the Jinja Team.
15    :license: BSD.
16"""
17from os import path, listdir
18import sys
19import marshal
20import tempfile
21import fnmatch
22from hashlib import sha1
23from jinja2.utils import open_if_exists
24from jinja2._compat import BytesIO, pickle, PY2, text_type
25
26
27# marshal works better on 3.x, one hack less required
28if not PY2:
29    marshal_dump = marshal.dump
30    marshal_load = marshal.load
31else:
32
33    def marshal_dump(code, f):
34        if isinstance(f, file):
35            marshal.dump(code, f)
36        else:
37            f.write(marshal.dumps(code))
38
39    def marshal_load(f):
40        if isinstance(f, file):
41            return marshal.load(f)
42        return marshal.loads(f.read())
43
44
45bc_version = 2
46
47# magic version used to only change with new jinja versions.  With 2.6
48# we change this to also take Python version changes into account.  The
49# reason for this is that Python tends to segfault if fed earlier bytecode
50# versions because someone thought it would be a good idea to reuse opcodes
51# or make Python incompatible with earlier versions.
52bc_magic = 'j2'.encode('ascii') + \
53    pickle.dumps(bc_version, 2) + \
54    pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
55
56
57class Bucket(object):
58    """Buckets are used to store the bytecode for one template.  It's created
59    and initialized by the bytecode cache and passed to the loading functions.
60
61    The buckets get an internal checksum from the cache assigned and use this
62    to automatically reject outdated cache material.  Individual bytecode
63    cache subclasses don't have to care about cache invalidation.
64    """
65
66    def __init__(self, environment, key, checksum):
67        self.environment = environment
68        self.key = key
69        self.checksum = checksum
70        self.reset()
71
72    def reset(self):
73        """Resets the bucket (unloads the bytecode)."""
74        self.code = None
75
76    def load_bytecode(self, f):
77        """Loads bytecode from a file or file like object."""
78        # make sure the magic header is correct
79        magic = f.read(len(bc_magic))
80        if magic != bc_magic:
81            self.reset()
82            return
83        # the source code of the file changed, we need to reload
84        checksum = pickle.load(f)
85        if self.checksum != checksum:
86            self.reset()
87            return
88        self.code = marshal_load(f)
89
90    def write_bytecode(self, f):
91        """Dump the bytecode into the file or file like object passed."""
92        if self.code is None:
93            raise TypeError('can\'t write empty bucket')
94        f.write(bc_magic)
95        pickle.dump(self.checksum, f, 2)
96        marshal_dump(self.code, f)
97
98    def bytecode_from_string(self, string):
99        """Load bytecode from a string."""
100        self.load_bytecode(BytesIO(string))
101
102    def bytecode_to_string(self):
103        """Return the bytecode as string."""
104        out = BytesIO()
105        self.write_bytecode(out)
106        return out.getvalue()
107
108
109class BytecodeCache(object):
110    """To implement your own bytecode cache you have to subclass this class
111    and override :meth:`load_bytecode` and :meth:`dump_bytecode`.  Both of
112    these methods are passed a :class:`~jinja2.bccache.Bucket`.
113
114    A very basic bytecode cache that saves the bytecode on the file system::
115
116        from os import path
117
118        class MyCache(BytecodeCache):
119
120            def __init__(self, directory):
121                self.directory = directory
122
123            def load_bytecode(self, bucket):
124                filename = path.join(self.directory, bucket.key)
125                if path.exists(filename):
126                    with open(filename, 'rb') as f:
127                        bucket.load_bytecode(f)
128
129            def dump_bytecode(self, bucket):
130                filename = path.join(self.directory, bucket.key)
131                with open(filename, 'wb') as f:
132                    bucket.write_bytecode(f)
133
134    A more advanced version of a filesystem based bytecode cache is part of
135    Jinja2.
136    """
137
138    def load_bytecode(self, bucket):
139        """Subclasses have to override this method to load bytecode into a
140        bucket.  If they are not able to find code in the cache for the
141        bucket, it must not do anything.
142        """
143        raise NotImplementedError()
144
145    def dump_bytecode(self, bucket):
146        """Subclasses have to override this method to write the bytecode
147        from a bucket back to the cache.  If it unable to do so it must not
148        fail silently but raise an exception.
149        """
150        raise NotImplementedError()
151
152    def clear(self):
153        """Clears the cache.  This method is not used by Jinja2 but should be
154        implemented to allow applications to clear the bytecode cache used
155        by a particular environment.
156        """
157
158    def get_cache_key(self, name, filename=None):
159        """Returns the unique hash key for this template name."""
160        hash = sha1(name.encode('utf-8'))
161        if filename is not None:
162            filename = '|' + filename
163            if isinstance(filename, text_type):
164                filename = filename.encode('utf-8')
165            hash.update(filename)
166        return hash.hexdigest()
167
168    def get_source_checksum(self, source):
169        """Returns a checksum for the source."""
170        return sha1(source.encode('utf-8')).hexdigest()
171
172    def get_bucket(self, environment, name, filename, source):
173        """Return a cache bucket for the given template.  All arguments are
174        mandatory but filename may be `None`.
175        """
176        key = self.get_cache_key(name, filename)
177        checksum = self.get_source_checksum(source)
178        bucket = Bucket(environment, key, checksum)
179        self.load_bytecode(bucket)
180        return bucket
181
182    def set_bucket(self, bucket):
183        """Put the bucket into the cache."""
184        self.dump_bytecode(bucket)
185
186
187class FileSystemBytecodeCache(BytecodeCache):
188    """A bytecode cache that stores bytecode on the filesystem.  It accepts
189    two arguments: The directory where the cache items are stored and a
190    pattern string that is used to build the filename.
191
192    If no directory is specified the system temporary items folder is used.
193
194    The pattern can be used to have multiple separate caches operate on the
195    same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
196    is replaced with the cache key.
197
198    >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
199
200    This bytecode cache supports clearing of the cache using the clear method.
201    """
202
203    def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
204        if directory is None:
205            directory = tempfile.gettempdir()
206        self.directory = directory
207        self.pattern = pattern
208
209    def _get_cache_filename(self, bucket):
210        return path.join(self.directory, self.pattern % bucket.key)
211
212    def load_bytecode(self, bucket):
213        f = open_if_exists(self._get_cache_filename(bucket), 'rb')
214        if f is not None:
215            try:
216                bucket.load_bytecode(f)
217            finally:
218                f.close()
219
220    def dump_bytecode(self, bucket):
221        f = open(self._get_cache_filename(bucket), 'wb')
222        try:
223            bucket.write_bytecode(f)
224        finally:
225            f.close()
226
227    def clear(self):
228        # imported lazily here because google app-engine doesn't support
229        # write access on the file system and the function does not exist
230        # normally.
231        from os import remove
232        files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
233        for filename in files:
234            try:
235                remove(path.join(self.directory, filename))
236            except OSError:
237                pass
238
239
240class MemcachedBytecodeCache(BytecodeCache):
241    """This class implements a bytecode cache that uses a memcache cache for
242    storing the information.  It does not enforce a specific memcache library
243    (tummy's memcache or cmemcache) but will accept any class that provides
244    the minimal interface required.
245
246    Libraries compatible with this class:
247
248    -   `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
249    -   `python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
250    -   `cmemcache <http://gijsbert.org/cmemcache/>`_
251
252    (Unfortunately the django cache interface is not compatible because it
253    does not support storing binary data, only unicode.  You can however pass
254    the underlying cache client to the bytecode cache which is available
255    as `django.core.cache.cache._client`.)
256
257    The minimal interface for the client passed to the constructor is this:
258
259    .. class:: MinimalClientInterface
260
261        .. method:: set(key, value[, timeout])
262
263            Stores the bytecode in the cache.  `value` is a string and
264            `timeout` the timeout of the key.  If timeout is not provided
265            a default timeout or no timeout should be assumed, if it's
266            provided it's an integer with the number of seconds the cache
267            item should exist.
268
269        .. method:: get(key)
270
271            Returns the value for the cache key.  If the item does not
272            exist in the cache the return value must be `None`.
273
274    The other arguments to the constructor are the prefix for all keys that
275    is added before the actual cache key and the timeout for the bytecode in
276    the cache system.  We recommend a high (or no) timeout.
277
278    This bytecode cache does not support clearing of used items in the cache.
279    The clear method is a no-operation function.
280
281    .. versionadded:: 2.7
282       Added support for ignoring memcache errors through the
283       `ignore_memcache_errors` parameter.
284    """
285
286    def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
287                 ignore_memcache_errors=True):
288        self.client = client
289        self.prefix = prefix
290        self.timeout = timeout
291        self.ignore_memcache_errors = ignore_memcache_errors
292
293    def load_bytecode(self, bucket):
294        try:
295            code = self.client.get(self.prefix + bucket.key)
296        except Exception:
297            if not self.ignore_memcache_errors:
298                raise
299            code = None
300        if code is not None:
301            bucket.bytecode_from_string(code)
302
303    def dump_bytecode(self, bucket):
304        args = (self.prefix + bucket.key, bucket.bytecode_to_string())
305        if self.timeout is not None:
306            args += (self.timeout,)
307        try:
308            self.client.set(*args)
309        except Exception:
310            if not self.ignore_memcache_errors:
311                raise
312