1b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)# -*- coding: utf-8 -*- 2b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)""" 3b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) jinja2.ext 4b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ~~~~~~~~~~ 5b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 6b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) Jinja extensions allow to add custom tags similar to the way django custom 7b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tags work. By default two example extensions exist: an i18n and a cache 8b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) extension. 9b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 10b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :copyright: (c) 2010 by the Jinja Team. 11b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :license: BSD. 12b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)""" 13b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from collections import deque 14b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from jinja2 import nodes 15b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from jinja2.defaults import * 16b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from jinja2.environment import Environment 17b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from jinja2.runtime import Undefined, concat 18b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError 19b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from jinja2.utils import contextfunction, import_string, Markup, next 20b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 21b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 22b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)# the only real useful gettext functions for a Jinja template. Note 23b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)# that ugettext must be assigned to gettext as Jinja doesn't support 24b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)# non unicode strings. 25b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext') 26b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 27b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 28b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class ExtensionRegistry(type): 29b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Gives the extension an unique identifier.""" 30b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 31b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def __new__(cls, name, bases, d): 32b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv = type.__new__(cls, name, bases, d) 33b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv.identifier = rv.__module__ + '.' + rv.__name__ 34b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return rv 35b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 36b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 37b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class Extension(object): 38b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Extensions can be used to add extra functionality to the Jinja template 39b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) system at the parser level. Custom extensions are bound to an environment 40b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) but may not store environment specific data on `self`. The reason for 41b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) this is that an extension can be bound to another environment (for 42b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) overlays) by creating a copy and reassigning the `environment` attribute. 43b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 44b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) As extensions are created by the environment they cannot accept any 45b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) arguments for configuration. One may want to work around that by using 46b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) a factory function, but that is not possible as extensions are identified 47b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) by their import name. The correct way to configure the extension is 48b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) storing the configuration values on the environment. Because this way the 49b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) environment ends up acting as central configuration storage the 50b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) attributes may clash which is why extensions have to ensure that the names 51b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) they choose for configuration are not too generic. ``prefix`` for example 52b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) is a terrible name, ``fragment_cache_prefix`` on the other hand is a good 53b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) name as includes the name of the extension (fragment cache). 54b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 55b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) __metaclass__ = ExtensionRegistry 56b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 57b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) #: if this extension parses this is the list of tags it's listening to. 58b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tags = set() 59b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 60b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) #: the priority of that extension. This is especially useful for 61b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) #: extensions that preprocess values. A lower value means higher 62b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) #: priority. 63b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) #: 64b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) #: .. versionadded:: 2.4 65b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) priority = 100 66b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 67b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def __init__(self, environment): 68b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.environment = environment 69b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 70b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def bind(self, environment): 71b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Create a copy of this extension bound to another environment.""" 72b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv = object.__new__(self.__class__) 73b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv.__dict__.update(self.__dict__) 74b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv.environment = environment 75b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return rv 76b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 77b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def preprocess(self, source, name, filename=None): 78b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """This method is called before the actual lexing and can be used to 79b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) preprocess the source. The `filename` is optional. The return value 80b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) must be the preprocessed source. 81b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 82b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return source 83b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 84b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def filter_stream(self, stream): 85b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used 86b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) to filter tokens returned. This method has to return an iterable of 87b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :class:`~jinja2.lexer.Token`\s, but it doesn't have to return a 88b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :class:`~jinja2.lexer.TokenStream`. 89b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 90b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) In the `ext` folder of the Jinja2 source distribution there is a file 91b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) called `inlinegettext.py` which implements a filter that utilizes this 92b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) method. 93b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 94b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return stream 95b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 96b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def parse(self, parser): 97b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """If any of the :attr:`tags` matched this method is called with the 98b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser as first argument. The token the parser stream is pointing at 99b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) is the name token that matched. This method has to return one or a 100b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) list of multiple nodes. 101b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 102b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) raise NotImplementedError() 103b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 104b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def attr(self, name, lineno=None): 105b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Return an attribute node for the current extension. This is useful 106b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) to pass constants on extensions to generated template code. 107b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 108b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :: 109b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 110b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.attr('_my_attribute', lineno=lineno) 111b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 112b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) 113b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 114b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def call_method(self, name, args=None, kwargs=None, dyn_args=None, 115b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) dyn_kwargs=None, lineno=None): 116b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Call a method of the extension. This is a shortcut for 117b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :meth:`attr` + :class:`jinja2.nodes.Call`. 118b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 119b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if args is None: 120b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) args = [] 121b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if kwargs is None: 122b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) kwargs = [] 123b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return nodes.Call(self.attr(name, lineno=lineno), args, kwargs, 124b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) dyn_args, dyn_kwargs, lineno=lineno) 125b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 126b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 127b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)@contextfunction 128b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)def _gettext_alias(__context, *args, **kwargs): 129b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return __context.call(__context.resolve('gettext'), *args, **kwargs) 130b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 131b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 132b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)def _make_new_gettext(func): 133b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) @contextfunction 134b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def gettext(__context, __string, **variables): 135b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv = __context.call(func, __string) 136b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if __context.eval_ctx.autoescape: 137b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv = Markup(rv) 138b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return rv % variables 139b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return gettext 140b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 141b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 142b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)def _make_new_ngettext(func): 143b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) @contextfunction 144b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def ngettext(__context, __singular, __plural, __num, **variables): 145b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) variables.setdefault('num', __num) 146b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv = __context.call(func, __singular, __plural, __num) 147b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if __context.eval_ctx.autoescape: 148b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) rv = Markup(rv) 149b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return rv % variables 150b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return ngettext 151b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 152b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 153b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class InternationalizationExtension(Extension): 154b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """This extension adds gettext support to Jinja2.""" 155b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tags = set(['trans']) 156b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 157b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # TODO: the i18n extension is currently reevaluating values in a few 158b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # situations. Take this example: 159b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # {% trans count=something() %}{{ count }} foo{% pluralize 160b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # %}{{ count }} fooss{% endtrans %} 161b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # something is called twice here. One time for the gettext value and 162b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # the other time for the n-parameter of the ngettext function. 163b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 164b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def __init__(self, environment): 165b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) Extension.__init__(self, environment) 166b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) environment.globals['_'] = _gettext_alias 167b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) environment.extend( 168b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) install_gettext_translations=self._install, 169b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) install_null_translations=self._install_null, 170b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) install_gettext_callables=self._install_callables, 171b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) uninstall_gettext_translations=self._uninstall, 172b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) extract_translations=self._extract, 173b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) newstyle_gettext=False 174b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ) 175b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 176b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _install(self, translations, newstyle=None): 177b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext = getattr(translations, 'ugettext', None) 178b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if gettext is None: 179b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext = translations.gettext 180b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ngettext = getattr(translations, 'ungettext', None) 181b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if ngettext is None: 182b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ngettext = translations.ngettext 183b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self._install_callables(gettext, ngettext, newstyle) 184b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 185b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _install_null(self, newstyle=None): 186b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self._install_callables( 187b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) lambda x: x, 188b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) lambda s, p, n: (n != 1 and (p,) or (s,))[0], 189b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) newstyle 190b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ) 191b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 192b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _install_callables(self, gettext, ngettext, newstyle=None): 193b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if newstyle is not None: 194b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.environment.newstyle_gettext = newstyle 195b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if self.environment.newstyle_gettext: 196b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext = _make_new_gettext(gettext) 197b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ngettext = _make_new_ngettext(ngettext) 198b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.environment.globals.update( 199b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext=gettext, 200b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ngettext=ngettext 201b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ) 202b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 203b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _uninstall(self, translations): 204b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for key in 'gettext', 'ngettext': 205b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.environment.globals.pop(key, None) 206b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 207b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): 208b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if isinstance(source, basestring): 209b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) source = self.environment.parse(source) 210b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return extract_from_ast(source, gettext_functions) 211b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 212b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def parse(self, parser): 213b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Parse a translatable tag.""" 214b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) lineno = next(parser.stream).lineno 215b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) num_called_num = False 216b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 217b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # find all the variables referenced. Additionally a variable can be 218b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # defined in the body of the trans block too, but this is checked at 219b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # a later state. 220b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural_expr = None 221b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) variables = {} 222b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) while parser.stream.current.type != 'block_end': 223b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if variables: 224b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.stream.expect('comma') 225b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 226b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # skip colon for python compatibility 227b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if parser.stream.skip_if('colon'): 228b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) break 229b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 230b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) name = parser.stream.expect('name') 231b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if name.value in variables: 232b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.fail('translatable variable %r defined twice.' % 233b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) name.value, name.lineno, 234b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) exc=TemplateAssertionError) 235b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 236b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # expressions 237b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if parser.stream.current.type == 'assign': 238b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) next(parser.stream) 239b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) variables[name.value] = var = parser.parse_expression() 240b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 241b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) variables[name.value] = var = nodes.Name(name.value, 'load') 242b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 243b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if plural_expr is None: 244b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural_expr = var 245b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) num_called_num = name.value == 'num' 246b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 247b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.stream.expect('block_end') 248b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 249b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural = plural_names = None 250b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) have_plural = False 251b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) referenced = set() 252b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 253b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # now parse until endtrans or pluralize 254b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) singular_names, singular = self._parse_block(parser, True) 255b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if singular_names: 256b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) referenced.update(singular_names) 257b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if plural_expr is None: 258b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural_expr = nodes.Name(singular_names[0], 'load') 259b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) num_called_num = singular_names[0] == 'num' 260b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 261b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # if we have a pluralize block, we parse that too 262b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if parser.stream.current.test('name:pluralize'): 263b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) have_plural = True 264b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) next(parser.stream) 265b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if parser.stream.current.type != 'block_end': 266b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) name = parser.stream.expect('name') 267b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if name.value not in variables: 268b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.fail('unknown variable %r for pluralization' % 269b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) name.value, name.lineno, 270b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) exc=TemplateAssertionError) 271b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural_expr = variables[name.value] 272b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) num_called_num = name.value == 'num' 273b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.stream.expect('block_end') 274b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural_names, plural = self._parse_block(parser, False) 275b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) next(parser.stream) 276b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) referenced.update(plural_names) 277b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 278b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) next(parser.stream) 279b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 280b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # register free names as simple name expressions 281b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for var in referenced: 282b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if var not in variables: 283b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) variables[var] = nodes.Name(var, 'load') 284b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 285b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if not have_plural: 286b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural_expr = None 287b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) elif plural_expr is None: 288b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.fail('pluralize without variables', lineno) 289b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 290b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = self._make_node(singular, plural, variables, plural_expr, 291b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) bool(referenced), 292b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) num_called_num and have_plural) 293b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node.set_lineno(lineno) 294b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return node 295b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 296b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _parse_block(self, parser, allow_pluralize): 297b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Parse until the next block tag with a given name.""" 298b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) referenced = [] 299b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) buf = [] 300b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) while 1: 301b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if parser.stream.current.type == 'data': 302b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) buf.append(parser.stream.current.value.replace('%', '%%')) 303b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) next(parser.stream) 304b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) elif parser.stream.current.type == 'variable_begin': 305b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) next(parser.stream) 306b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) name = parser.stream.expect('name').value 307b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) referenced.append(name) 308b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) buf.append('%%(%s)s' % name) 309b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.stream.expect('variable_end') 310b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) elif parser.stream.current.type == 'block_begin': 311b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) next(parser.stream) 312b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if parser.stream.current.test('name:endtrans'): 313b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) break 314b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) elif parser.stream.current.test('name:pluralize'): 315b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if allow_pluralize: 316b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) break 317b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.fail('a translatable section can have only one ' 318b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 'pluralize section') 319b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.fail('control structures in translatable sections are ' 320b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 'not allowed') 321b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) elif parser.stream.eos: 322b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.fail('unclosed translation block') 323b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 324b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) assert False, 'internal parser error' 325b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 326b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return referenced, concat(buf) 327b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 328b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _make_node(self, singular, plural, variables, plural_expr, 329b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) vars_referenced, num_called_num): 330b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Generates a useful node from the data provided.""" 331b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # no variables referenced? no need to escape for old style 332b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # gettext invocations only if there are vars. 333b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if not vars_referenced and not self.environment.newstyle_gettext: 334b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) singular = singular.replace('%%', '%') 335b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if plural: 336b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural = plural.replace('%%', '%') 337b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 338b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # singular only: 339b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if plural_expr is None: 340b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext = nodes.Name('gettext', 'load') 341b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = nodes.Call(gettext, [nodes.Const(singular)], 342b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) [], None, None) 343b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 344b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # singular and plural 345b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 346b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ngettext = nodes.Name('ngettext', 'load') 347b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = nodes.Call(ngettext, [ 348b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) nodes.Const(singular), 349b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) nodes.Const(plural), 350b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) plural_expr 351b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ], [], None, None) 352b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 353b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # in case newstyle gettext is used, the method is powerful 354b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # enough to handle the variable expansion and autoescape 355b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # handling itself 356b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if self.environment.newstyle_gettext: 357b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for key, value in variables.iteritems(): 358b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # the function adds that later anyways in case num was 359b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # called num, so just skip it. 360b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if num_called_num and key == 'num': 361b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) continue 362b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node.kwargs.append(nodes.Keyword(key, value)) 363b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 364b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # otherwise do that here 365b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 366b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # mark the return value as safe if we are in an 367b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # environment with autoescaping turned on 368b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = nodes.MarkSafeIfAutoescape(node) 369b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if variables: 370b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = nodes.Mod(node, nodes.Dict([ 371b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) nodes.Pair(nodes.Const(key), value) 372b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for key, value in variables.items() 373b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ])) 374b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return nodes.Output([node]) 375b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 376b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 377b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class ExprStmtExtension(Extension): 378b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Adds a `do` tag to Jinja2 that works like the print statement just 379b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) that it doesn't print the return value. 380b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 381b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tags = set(['do']) 382b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 383b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def parse(self, parser): 384b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = nodes.ExprStmt(lineno=next(parser.stream).lineno) 385b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node.node = parser.parse_tuple() 386b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return node 387b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 388b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 389b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class LoopControlExtension(Extension): 390b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Adds break and continue to the template engine.""" 391b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tags = set(['break', 'continue']) 392b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 393b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def parse(self, parser): 394b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) token = next(parser.stream) 395b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if token.value == 'break': 396b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return nodes.Break(lineno=token.lineno) 397b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return nodes.Continue(lineno=token.lineno) 398b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 399b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 400b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class WithExtension(Extension): 401b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Adds support for a django-like with block.""" 402b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tags = set(['with']) 403b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 404b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def parse(self, parser): 405b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = nodes.Scope(lineno=next(parser.stream).lineno) 406b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) assignments = [] 407b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) while parser.stream.current.type != 'block_end': 408b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) lineno = parser.stream.current.lineno 409b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if assignments: 410b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.stream.expect('comma') 411b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) target = parser.parse_assign_target() 412b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parser.stream.expect('assign') 413b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) expr = parser.parse_expression() 414b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) assignments.append(nodes.Assign(target, expr, lineno=lineno)) 415b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node.body = assignments + \ 416b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) list(parser.parse_statements(('name:endwith',), 417b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) drop_needle=True)) 418b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return node 419b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 420b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 421b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class AutoEscapeExtension(Extension): 422b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Changes auto escape rules for a scope.""" 423b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tags = set(['autoescape']) 424b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 425b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def parse(self, parser): 426b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno) 427b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node.options = [ 428b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) nodes.Keyword('autoescape', parser.parse_expression()) 429b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ] 430b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node.body = parser.parse_statements(('name:endautoescape',), 431b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) drop_needle=True) 432b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return nodes.Scope([node]) 433b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 434b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 435b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, 436b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) babel_style=True): 437b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Extract localizable strings from the given template node. Per 438b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) default this function returns matches in babel style that means non string 439b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) parameters as well as keyword arguments are returned as `None`. This 440b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) allows Babel to figure out what you really meant if you are using 441b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext functions that allow keyword arguments for placeholder expansion. 442b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) If you don't want that behavior set the `babel_style` parameter to `False` 443b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) which causes only strings to be returned and parameters are always stored 444b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) in tuples. As a consequence invalid gettext calls (calls without a single 445b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) string parameter or string parameters after non-string parameters) are 446b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) skipped. 447b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 448b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) This example explains the behavior: 449b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 450b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) >>> from jinja2 import Environment 451b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) >>> env = Environment() 452b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}') 453b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) >>> list(extract_from_ast(node)) 454b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))] 455b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) >>> list(extract_from_ast(node, babel_style=False)) 456b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))] 457b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 458b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) For every string found this function yields a ``(lineno, function, 459b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) message)`` tuple, where: 460b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 461b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) * ``lineno`` is the number of the line on which the string was found, 462b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) * ``function`` is the name of the ``gettext`` function used (if the 463b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) string was extracted from embedded Python code), and 464b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) * ``message`` is the string itself (a ``unicode`` object, or a tuple 465b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) of ``unicode`` objects for functions with multiple string arguments). 466b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 467b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) This extraction function operates on the AST and is because of that unable 468b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) to extract any comments. For comment support you have to use the babel 469b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) extraction interface or extract comments yourself. 470b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 471b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for node in node.find_all(nodes.Call): 472b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if not isinstance(node.node, nodes.Name) or \ 473b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node.node.name not in gettext_functions: 474b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) continue 475b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 476b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings = [] 477b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for arg in node.args: 478b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if isinstance(arg, nodes.Const) and \ 479b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) isinstance(arg.value, basestring): 480b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings.append(arg.value) 481b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 482b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings.append(None) 483b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 484b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for arg in node.kwargs: 485b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings.append(None) 486b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if node.dyn_args is not None: 487b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings.append(None) 488b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if node.dyn_kwargs is not None: 489b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings.append(None) 490b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 491b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if not babel_style: 492b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings = tuple(x for x in strings if x is not None) 493b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if not strings: 494b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) continue 495b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 496b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if len(strings) == 1: 497b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings = strings[0] 498b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) else: 499b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) strings = tuple(strings) 500b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) yield node.lineno, node.node.name, strings 501b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 502b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 503b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)class _CommentFinder(object): 504b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Helper class to find comments in a token stream. Can only 505b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) find comments for gettext calls forwards. Once the comment 506b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) from line 4 is found, a comment for line 1 will not return a 507b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) usable value. 508b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 509b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 510b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def __init__(self, tokens, comment_tags): 511b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.tokens = tokens 512b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.comment_tags = comment_tags 513b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.offset = 0 514b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.last_lineno = 0 515b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 516b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def find_backwards(self, offset): 517b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) try: 518b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for _, token_type, token_value in \ 519b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) reversed(self.tokens[self.offset:offset]): 520b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if token_type in ('comment', 'linecomment'): 521b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) try: 522b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) prefix, comment = token_value.split(None, 1) 523b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) except ValueError: 524b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) continue 525b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if prefix in self.comment_tags: 526b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return [comment.rstrip()] 527b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return [] 528b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) finally: 529b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.offset = offset 530b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 531b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def find_comments(self, lineno): 532b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if not self.comment_tags or self.last_lineno > lineno: 533b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return [] 534b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]): 535b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if token_lineno > lineno: 536b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return self.find_backwards(self.offset + idx) 537b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return self.find_backwards(len(self.tokens)) 538b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 539b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 540b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)def babel_extract(fileobj, keywords, comment_tags, options): 541b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """Babel extraction method for Jinja templates. 542b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 543b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) .. versionchanged:: 2.3 544b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) Basic support for translation comments was added. If `comment_tags` 545b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) is now set to a list of keywords for extraction, the extractor will 546b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) try to find the best preceeding comment that begins with one of the 547b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) keywords. For best results, make sure to not have more than one 548b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext call in one line of code and the matching comment in the 549b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) same line or the line before. 550b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 551b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) .. versionchanged:: 2.5.1 552b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) The `newstyle_gettext` flag can be set to `True` to enable newstyle 553b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) gettext calls. 554b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 555b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :param fileobj: the file-like object the messages should be extracted from 556b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :param keywords: a list of keywords (i.e. function names) that should be 557b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) recognized as translation functions 558b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :param comment_tags: a list of translator tags to search for and include 559b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) in the results. 560b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :param options: a dictionary of additional options (optional) 561b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples. 562b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) (comments will be empty currently) 563b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) """ 564b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) extensions = set() 565b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for extension in options.get('extensions', '').split(','): 566b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) extension = extension.strip() 567b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if not extension: 568b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) continue 569b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) extensions.add(import_string(extension)) 570b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if InternationalizationExtension not in extensions: 571b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) extensions.add(InternationalizationExtension) 572b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 573b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def getbool(options, key, default=False): 574b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get(key, str(default)).lower() in ('1', 'on', 'yes', 'true') 575b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 576b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) environment = Environment( 577b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('block_start_string', BLOCK_START_STRING), 578b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('block_end_string', BLOCK_END_STRING), 579b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('variable_start_string', VARIABLE_START_STRING), 580b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('variable_end_string', VARIABLE_END_STRING), 581b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('comment_start_string', COMMENT_START_STRING), 582b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('comment_end_string', COMMENT_END_STRING), 583b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, 584b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, 585b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) getbool(options, 'trim_blocks', TRIM_BLOCKS), 586b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) NEWLINE_SEQUENCE, frozenset(extensions), 587b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) cache_size=0, 588b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) auto_reload=False 589b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ) 590b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 591b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) if getbool(options, 'newstyle_gettext'): 592b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) environment.newstyle_gettext = True 593b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 594b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) source = fileobj.read().decode(options.get('encoding', 'utf-8')) 595b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) try: 596b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) node = environment.parse(source) 597b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) tokens = list(environment.lex(environment.preprocess(source))) 598b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) except TemplateSyntaxError, e: 599b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) # skip templates with syntax errors 600b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return 601b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 602b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) finder = _CommentFinder(tokens, comment_tags) 603b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) for lineno, func, message in extract_from_ast(node, keywords): 604b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) yield lineno, func, message, finder.find_comments(lineno) 605b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 606b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 607b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)#: nicer import names 608b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)i18n = InternationalizationExtension 609b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)do = ExprStmtExtension 610b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)loopcontrols = LoopControlExtension 611b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)with_ = WithExtension 612b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)autoescape = AutoEscapeExtension 613