1"""
2A small templating language
3
4This implements a small templating language.  This language implements
5if/elif/else, for/continue/break, expressions, and blocks of Python
6code.  The syntax is::
7
8  {{any expression (function calls etc)}}
9  {{any expression | filter}}
10  {{for x in y}}...{{endfor}}
11  {{if x}}x{{elif y}}y{{else}}z{{endif}}
12  {{py:x=1}}
13  {{py:
14  def foo(bar):
15      return 'baz'
16  }}
17  {{default var = default_value}}
18  {{# comment}}
19
20You use this with the ``Template`` class or the ``sub`` shortcut.
21The ``Template`` class takes the template string and the name of
22the template (for errors) and a default namespace.  Then (like
23``string.Template``) you can call the ``tmpl.substitute(**kw)``
24method to make a substitution (or ``tmpl.substitute(a_dict)``).
25
26``sub(content, **kw)`` substitutes the template immediately.  You
27can use ``__name='tmpl.html'`` to set the name of the template.
28
29If there are syntax errors ``TemplateError`` will be raised.
30"""
31
32import re
33import sys
34import cgi
35try:
36    from urllib import quote as url_quote
37except ImportError:  # Py3
38    from urllib.parse import quote as url_quote
39import os
40import tokenize
41try:
42    from io import StringIO
43except ImportError:
44    from cStringIO import StringIO
45from Cython.Tempita._looper import looper
46from Cython.Tempita.compat3 import bytes, basestring_, next, is_unicode, coerce_text
47
48__all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate',
49           'sub_html', 'html', 'bunch']
50
51in_re = re.compile(r'\s+in\s+')
52var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I)
53
54
55class TemplateError(Exception):
56    """Exception raised while parsing a template
57    """
58
59    def __init__(self, message, position, name=None):
60        Exception.__init__(self, message)
61        self.position = position
62        self.name = name
63
64    def __str__(self):
65        msg = ' '.join(self.args)
66        if self.position:
67            msg = '%s at line %s column %s' % (
68                msg, self.position[0], self.position[1])
69        if self.name:
70            msg += ' in %s' % self.name
71        return msg
72
73
74class _TemplateContinue(Exception):
75    pass
76
77
78class _TemplateBreak(Exception):
79    pass
80
81
82def get_file_template(name, from_template):
83    path = os.path.join(os.path.dirname(from_template.name), name)
84    return from_template.__class__.from_filename(
85        path, namespace=from_template.namespace,
86        get_template=from_template.get_template)
87
88
89class Template(object):
90
91    default_namespace = {
92        'start_braces': '{{',
93        'end_braces': '}}',
94        'looper': looper,
95        }
96
97    default_encoding = 'utf8'
98    default_inherit = None
99
100    def __init__(self, content, name=None, namespace=None, stacklevel=None,
101                 get_template=None, default_inherit=None, line_offset=0,
102                 delimeters=None):
103        self.content = content
104
105        # set delimeters
106        if delimeters is None:
107            delimeters = (self.default_namespace['start_braces'],
108                          self.default_namespace['end_braces'])
109        else:
110            #assert len(delimeters) == 2 and all([isinstance(delimeter, basestring)
111            #                                     for delimeter in delimeters])
112            self.default_namespace = self.__class__.default_namespace.copy()
113            self.default_namespace['start_braces'] = delimeters[0]
114            self.default_namespace['end_braces'] = delimeters[1]
115        self.delimeters = delimeters
116
117        self._unicode = is_unicode(content)
118        if name is None and stacklevel is not None:
119            try:
120                caller = sys._getframe(stacklevel)
121            except ValueError:
122                pass
123            else:
124                globals = caller.f_globals
125                lineno = caller.f_lineno
126                if '__file__' in globals:
127                    name = globals['__file__']
128                    if name.endswith('.pyc') or name.endswith('.pyo'):
129                        name = name[:-1]
130                elif '__name__' in globals:
131                    name = globals['__name__']
132                else:
133                    name = '<string>'
134                if lineno:
135                    name += ':%s' % lineno
136        self.name = name
137        self._parsed = parse(content, name=name, line_offset=line_offset, delimeters=self.delimeters)
138        if namespace is None:
139            namespace = {}
140        self.namespace = namespace
141        self.get_template = get_template
142        if default_inherit is not None:
143            self.default_inherit = default_inherit
144
145    def from_filename(cls, filename, namespace=None, encoding=None,
146                      default_inherit=None, get_template=get_file_template):
147        f = open(filename, 'rb')
148        c = f.read()
149        f.close()
150        if encoding:
151            c = c.decode(encoding)
152        return cls(content=c, name=filename, namespace=namespace,
153                   default_inherit=default_inherit, get_template=get_template)
154
155    from_filename = classmethod(from_filename)
156
157    def __repr__(self):
158        return '<%s %s name=%r>' % (
159            self.__class__.__name__,
160            hex(id(self))[2:], self.name)
161
162    def substitute(self, *args, **kw):
163        if args:
164            if kw:
165                raise TypeError(
166                    "You can only give positional *or* keyword arguments")
167            if len(args) > 1:
168                raise TypeError(
169                    "You can only give one positional argument")
170            if not hasattr(args[0], 'items'):
171                raise TypeError(
172                    "If you pass in a single argument, you must pass in a dictionary-like object (with a .items() method); you gave %r"
173                    % (args[0],))
174            kw = args[0]
175        ns = kw
176        ns['__template_name__'] = self.name
177        if self.namespace:
178            ns.update(self.namespace)
179        result, defs, inherit = self._interpret(ns)
180        if not inherit:
181            inherit = self.default_inherit
182        if inherit:
183            result = self._interpret_inherit(result, defs, inherit, ns)
184        return result
185
186    def _interpret(self, ns):
187        __traceback_hide__ = True
188        parts = []
189        defs = {}
190        self._interpret_codes(self._parsed, ns, out=parts, defs=defs)
191        if '__inherit__' in defs:
192            inherit = defs.pop('__inherit__')
193        else:
194            inherit = None
195        return ''.join(parts), defs, inherit
196
197    def _interpret_inherit(self, body, defs, inherit_template, ns):
198        __traceback_hide__ = True
199        if not self.get_template:
200            raise TemplateError(
201                'You cannot use inheritance without passing in get_template',
202                position=None, name=self.name)
203        templ = self.get_template(inherit_template, self)
204        self_ = TemplateObject(self.name)
205        for name, value in defs.iteritems():
206            setattr(self_, name, value)
207        self_.body = body
208        ns = ns.copy()
209        ns['self'] = self_
210        return templ.substitute(ns)
211
212    def _interpret_codes(self, codes, ns, out, defs):
213        __traceback_hide__ = True
214        for item in codes:
215            if isinstance(item, basestring_):
216                out.append(item)
217            else:
218                self._interpret_code(item, ns, out, defs)
219
220    def _interpret_code(self, code, ns, out, defs):
221        __traceback_hide__ = True
222        name, pos = code[0], code[1]
223        if name == 'py':
224            self._exec(code[2], ns, pos)
225        elif name == 'continue':
226            raise _TemplateContinue()
227        elif name == 'break':
228            raise _TemplateBreak()
229        elif name == 'for':
230            vars, expr, content = code[2], code[3], code[4]
231            expr = self._eval(expr, ns, pos)
232            self._interpret_for(vars, expr, content, ns, out, defs)
233        elif name == 'cond':
234            parts = code[2:]
235            self._interpret_if(parts, ns, out, defs)
236        elif name == 'expr':
237            parts = code[2].split('|')
238            base = self._eval(parts[0], ns, pos)
239            for part in parts[1:]:
240                func = self._eval(part, ns, pos)
241                base = func(base)
242            out.append(self._repr(base, pos))
243        elif name == 'default':
244            var, expr = code[2], code[3]
245            if var not in ns:
246                result = self._eval(expr, ns, pos)
247                ns[var] = result
248        elif name == 'inherit':
249            expr = code[2]
250            value = self._eval(expr, ns, pos)
251            defs['__inherit__'] = value
252        elif name == 'def':
253            name = code[2]
254            signature = code[3]
255            parts = code[4]
256            ns[name] = defs[name] = TemplateDef(self, name, signature, body=parts, ns=ns,
257                                                pos=pos)
258        elif name == 'comment':
259            return
260        else:
261            assert 0, "Unknown code: %r" % name
262
263    def _interpret_for(self, vars, expr, content, ns, out, defs):
264        __traceback_hide__ = True
265        for item in expr:
266            if len(vars) == 1:
267                ns[vars[0]] = item
268            else:
269                if len(vars) != len(item):
270                    raise ValueError(
271                        'Need %i items to unpack (got %i items)'
272                        % (len(vars), len(item)))
273                for name, value in zip(vars, item):
274                    ns[name] = value
275            try:
276                self._interpret_codes(content, ns, out, defs)
277            except _TemplateContinue:
278                continue
279            except _TemplateBreak:
280                break
281
282    def _interpret_if(self, parts, ns, out, defs):
283        __traceback_hide__ = True
284        # @@: if/else/else gets through
285        for part in parts:
286            assert not isinstance(part, basestring_)
287            name, pos = part[0], part[1]
288            if name == 'else':
289                result = True
290            else:
291                result = self._eval(part[2], ns, pos)
292            if result:
293                self._interpret_codes(part[3], ns, out, defs)
294                break
295
296    def _eval(self, code, ns, pos):
297        __traceback_hide__ = True
298        try:
299            try:
300                value = eval(code, self.default_namespace, ns)
301            except SyntaxError, e:
302                raise SyntaxError(
303                    'invalid syntax in expression: %s' % code)
304            return value
305        except:
306            exc_info = sys.exc_info()
307            e = exc_info[1]
308            if getattr(e, 'args', None):
309                arg0 = e.args[0]
310            else:
311                arg0 = coerce_text(e)
312            e.args = (self._add_line_info(arg0, pos),)
313            raise exc_info[0], e, exc_info[2]
314
315    def _exec(self, code, ns, pos):
316        __traceback_hide__ = True
317        try:
318            exec code in self.default_namespace, ns
319        except:
320            exc_info = sys.exc_info()
321            e = exc_info[1]
322            if e.args:
323                e.args = (self._add_line_info(e.args[0], pos),)
324            else:
325                e.args = (self._add_line_info(None, pos),)
326            raise exc_info[0], e, exc_info[2]
327
328    def _repr(self, value, pos):
329        __traceback_hide__ = True
330        try:
331            if value is None:
332                return ''
333            if self._unicode:
334                try:
335                    value = unicode(value)
336                except UnicodeDecodeError:
337                    value = bytes(value)
338            else:
339                if not isinstance(value, basestring_):
340                    value = coerce_text(value)
341                if (is_unicode(value)
342                    and self.default_encoding):
343                    value = value.encode(self.default_encoding)
344        except:
345            exc_info = sys.exc_info()
346            e = exc_info[1]
347            e.args = (self._add_line_info(e.args[0], pos),)
348            raise exc_info[0], e, exc_info[2]
349        else:
350            if self._unicode and isinstance(value, bytes):
351                if not self.default_encoding:
352                    raise UnicodeDecodeError(
353                        'Cannot decode bytes value %r into unicode '
354                        '(no default_encoding provided)' % value)
355                try:
356                    value = value.decode(self.default_encoding)
357                except UnicodeDecodeError, e:
358                    raise UnicodeDecodeError(
359                        e.encoding,
360                        e.object,
361                        e.start,
362                        e.end,
363                        e.reason + ' in string %r' % value)
364            elif not self._unicode and is_unicode(value):
365                if not self.default_encoding:
366                    raise UnicodeEncodeError(
367                        'Cannot encode unicode value %r into bytes '
368                        '(no default_encoding provided)' % value)
369                value = value.encode(self.default_encoding)
370            return value
371
372    def _add_line_info(self, msg, pos):
373        msg = "%s at line %s column %s" % (
374            msg, pos[0], pos[1])
375        if self.name:
376            msg += " in file %s" % self.name
377        return msg
378
379
380def sub(content, delimeters=None, **kw):
381    name = kw.get('__name')
382    tmpl = Template(content, name=name, delimeters=delimeters)
383    return tmpl.substitute(kw)
384
385
386def paste_script_template_renderer(content, vars, filename=None):
387    tmpl = Template(content, name=filename)
388    return tmpl.substitute(vars)
389
390
391class bunch(dict):
392
393    def __init__(self, **kw):
394        for name, value in kw.iteritems():
395            setattr(self, name, value)
396
397    def __setattr__(self, name, value):
398        self[name] = value
399
400    def __getattr__(self, name):
401        try:
402            return self[name]
403        except KeyError:
404            raise AttributeError(name)
405
406    def __getitem__(self, key):
407        if 'default' in self:
408            try:
409                return dict.__getitem__(self, key)
410            except KeyError:
411                return dict.__getitem__(self, 'default')
412        else:
413            return dict.__getitem__(self, key)
414
415    def __repr__(self):
416        items = [
417            (k, v) for k, v in self.iteritems()]
418        items.sort()
419        return '<%s %s>' % (
420            self.__class__.__name__,
421            ' '.join(['%s=%r' % (k, v) for k, v in items]))
422
423############################################################
424## HTML Templating
425############################################################
426
427
428class html(object):
429
430    def __init__(self, value):
431        self.value = value
432
433    def __str__(self):
434        return self.value
435
436    def __html__(self):
437        return self.value
438
439    def __repr__(self):
440        return '<%s %r>' % (
441            self.__class__.__name__, self.value)
442
443
444def html_quote(value, force=True):
445    if not force and hasattr(value, '__html__'):
446        return value.__html__()
447    if value is None:
448        return ''
449    if not isinstance(value, basestring_):
450        value = coerce_text(value)
451    if sys.version >= "3" and isinstance(value, bytes):
452        value = cgi.escape(value.decode('latin1'), 1)
453        value = value.encode('latin1')
454    else:
455        value = cgi.escape(value, 1)
456    if sys.version < "3":
457        if is_unicode(value):
458            value = value.encode('ascii', 'xmlcharrefreplace')
459    return value
460
461
462def url(v):
463    v = coerce_text(v)
464    if is_unicode(v):
465        v = v.encode('utf8')
466    return url_quote(v)
467
468
469def attr(**kw):
470    kw = list(kw.iteritems())
471    kw.sort()
472    parts = []
473    for name, value in kw:
474        if value is None:
475            continue
476        if name.endswith('_'):
477            name = name[:-1]
478        parts.append('%s="%s"' % (html_quote(name), html_quote(value)))
479    return html(' '.join(parts))
480
481
482class HTMLTemplate(Template):
483
484    default_namespace = Template.default_namespace.copy()
485    default_namespace.update(dict(
486        html=html,
487        attr=attr,
488        url=url,
489        html_quote=html_quote,
490        ))
491
492    def _repr(self, value, pos):
493        if hasattr(value, '__html__'):
494            value = value.__html__()
495            quote = False
496        else:
497            quote = True
498        plain = Template._repr(self, value, pos)
499        if quote:
500            return html_quote(plain)
501        else:
502            return plain
503
504
505def sub_html(content, **kw):
506    name = kw.get('__name')
507    tmpl = HTMLTemplate(content, name=name)
508    return tmpl.substitute(kw)
509
510
511class TemplateDef(object):
512    def __init__(self, template, func_name, func_signature,
513                 body, ns, pos, bound_self=None):
514        self._template = template
515        self._func_name = func_name
516        self._func_signature = func_signature
517        self._body = body
518        self._ns = ns
519        self._pos = pos
520        self._bound_self = bound_self
521
522    def __repr__(self):
523        return '<tempita function %s(%s) at %s:%s>' % (
524            self._func_name, self._func_signature,
525            self._template.name, self._pos)
526
527    def __str__(self):
528        return self()
529
530    def __call__(self, *args, **kw):
531        values = self._parse_signature(args, kw)
532        ns = self._ns.copy()
533        ns.update(values)
534        if self._bound_self is not None:
535            ns['self'] = self._bound_self
536        out = []
537        subdefs = {}
538        self._template._interpret_codes(self._body, ns, out, subdefs)
539        return ''.join(out)
540
541    def __get__(self, obj, type=None):
542        if obj is None:
543            return self
544        return self.__class__(
545            self._template, self._func_name, self._func_signature,
546            self._body, self._ns, self._pos, bound_self=obj)
547
548    def _parse_signature(self, args, kw):
549        values = {}
550        sig_args, var_args, var_kw, defaults = self._func_signature
551        extra_kw = {}
552        for name, value in kw.iteritems():
553            if not var_kw and name not in sig_args:
554                raise TypeError(
555                    'Unexpected argument %s' % name)
556            if name in sig_args:
557                values[sig_args] = value
558            else:
559                extra_kw[name] = value
560        args = list(args)
561        sig_args = list(sig_args)
562        while args:
563            while sig_args and sig_args[0] in values:
564                sig_args.pop(0)
565            if sig_args:
566                name = sig_args.pop(0)
567                values[name] = args.pop(0)
568            elif var_args:
569                values[var_args] = tuple(args)
570                break
571            else:
572                raise TypeError(
573                    'Extra position arguments: %s'
574                    % ', '.join([repr(v) for v in args]))
575        for name, value_expr in defaults.iteritems():
576            if name not in values:
577                values[name] = self._template._eval(
578                    value_expr, self._ns, self._pos)
579        for name in sig_args:
580            if name not in values:
581                raise TypeError(
582                    'Missing argument: %s' % name)
583        if var_kw:
584            values[var_kw] = extra_kw
585        return values
586
587
588class TemplateObject(object):
589
590    def __init__(self, name):
591        self.__name = name
592        self.get = TemplateObjectGetter(self)
593
594    def __repr__(self):
595        return '<%s %s>' % (self.__class__.__name__, self.__name)
596
597
598class TemplateObjectGetter(object):
599
600    def __init__(self, template_obj):
601        self.__template_obj = template_obj
602
603    def __getattr__(self, attr):
604        return getattr(self.__template_obj, attr, Empty)
605
606    def __repr__(self):
607        return '<%s around %r>' % (self.__class__.__name__, self.__template_obj)
608
609
610class _Empty(object):
611    def __call__(self, *args, **kw):
612        return self
613
614    def __str__(self):
615        return ''
616
617    def __repr__(self):
618        return 'Empty'
619
620    def __unicode__(self):
621        return u''
622
623    def __iter__(self):
624        return iter(())
625
626    def __bool__(self):
627        return False
628
629    if sys.version < "3":
630        __nonzero__ = __bool__
631
632Empty = _Empty()
633del _Empty
634
635############################################################
636## Lexing and Parsing
637############################################################
638
639
640def lex(s, name=None, trim_whitespace=True, line_offset=0, delimeters=None):
641    """
642    Lex a string into chunks:
643
644        >>> lex('hey')
645        ['hey']
646        >>> lex('hey {{you}}')
647        ['hey ', ('you', (1, 7))]
648        >>> lex('hey {{')
649        Traceback (most recent call last):
650            ...
651        TemplateError: No }} to finish last expression at line 1 column 7
652        >>> lex('hey }}')
653        Traceback (most recent call last):
654            ...
655        TemplateError: }} outside expression at line 1 column 7
656        >>> lex('hey {{ {{')
657        Traceback (most recent call last):
658            ...
659        TemplateError: {{ inside expression at line 1 column 10
660
661    """
662    if delimeters is None:
663        delimeters = ( Template.default_namespace['start_braces'],
664                       Template.default_namespace['end_braces'] )
665    in_expr = False
666    chunks = []
667    last = 0
668    last_pos = (line_offset + 1, 1)
669
670    token_re = re.compile(r'%s|%s' % (re.escape(delimeters[0]),
671                                      re.escape(delimeters[1])))
672    for match in token_re.finditer(s):
673        expr = match.group(0)
674        pos = find_position(s, match.end(), last, last_pos)
675        if expr == delimeters[0] and in_expr:
676            raise TemplateError('%s inside expression' % delimeters[0],
677                                position=pos,
678                                name=name)
679        elif expr == delimeters[1] and not in_expr:
680            raise TemplateError('%s outside expression' % delimeters[1],
681                                position=pos,
682                                name=name)
683        if expr == delimeters[0]:
684            part = s[last:match.start()]
685            if part:
686                chunks.append(part)
687            in_expr = True
688        else:
689            chunks.append((s[last:match.start()], last_pos))
690            in_expr = False
691        last = match.end()
692        last_pos = pos
693    if in_expr:
694        raise TemplateError('No %s to finish last expression' % delimeters[1],
695                            name=name, position=last_pos)
696    part = s[last:]
697    if part:
698        chunks.append(part)
699    if trim_whitespace:
700        chunks = trim_lex(chunks)
701    return chunks
702
703statement_re = re.compile(r'^(?:if |elif |for |def |inherit |default |py:)')
704single_statements = ['else', 'endif', 'endfor', 'enddef', 'continue', 'break']
705trail_whitespace_re = re.compile(r'\n\r?[\t ]*$')
706lead_whitespace_re = re.compile(r'^[\t ]*\n')
707
708
709def trim_lex(tokens):
710    r"""
711    Takes a lexed set of tokens, and removes whitespace when there is
712    a directive on a line by itself:
713
714       >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False)
715       >>> tokens
716       [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny']
717       >>> trim_lex(tokens)
718       [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y']
719    """
720    last_trim = None
721    for i, current in enumerate(tokens):
722        if isinstance(current, basestring_):
723            # we don't trim this
724            continue
725        item = current[0]
726        if not statement_re.search(item) and item not in single_statements:
727            continue
728        if not i:
729            prev = ''
730        else:
731            prev = tokens[i - 1]
732        if i + 1 >= len(tokens):
733            next_chunk = ''
734        else:
735            next_chunk = tokens[i + 1]
736        if (not isinstance(next_chunk, basestring_)
737            or not isinstance(prev, basestring_)):
738            continue
739        prev_ok = not prev or trail_whitespace_re.search(prev)
740        if i == 1 and not prev.strip():
741            prev_ok = True
742        if last_trim is not None and last_trim + 2 == i and not prev.strip():
743            prev_ok = 'last'
744        if (prev_ok
745            and (not next_chunk or lead_whitespace_re.search(next_chunk)
746                 or (i == len(tokens) - 2 and not next_chunk.strip()))):
747            if prev:
748                if ((i == 1 and not prev.strip())
749                    or prev_ok == 'last'):
750                    tokens[i - 1] = ''
751                else:
752                    m = trail_whitespace_re.search(prev)
753                    # +1 to leave the leading \n on:
754                    prev = prev[:m.start() + 1]
755                    tokens[i - 1] = prev
756            if next_chunk:
757                last_trim = i
758                if i == len(tokens) - 2 and not next_chunk.strip():
759                    tokens[i + 1] = ''
760                else:
761                    m = lead_whitespace_re.search(next_chunk)
762                    next_chunk = next_chunk[m.end():]
763                    tokens[i + 1] = next_chunk
764    return tokens
765
766
767def find_position(string, index, last_index, last_pos):
768    """Given a string and index, return (line, column)"""
769    lines = string.count('\n', last_index, index)
770    if lines > 0:
771        column = index - string.rfind('\n', last_index, index)
772    else:
773        column = last_pos[1] + (index - last_index)
774    return (last_pos[0] + lines, column)
775
776
777def parse(s, name=None, line_offset=0, delimeters=None):
778    r"""
779    Parses a string into a kind of AST
780
781        >>> parse('{{x}}')
782        [('expr', (1, 3), 'x')]
783        >>> parse('foo')
784        ['foo']
785        >>> parse('{{if x}}test{{endif}}')
786        [('cond', (1, 3), ('if', (1, 3), 'x', ['test']))]
787        >>> parse('series->{{for x in y}}x={{x}}{{endfor}}')
788        ['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])]
789        >>> parse('{{for x, y in z:}}{{continue}}{{endfor}}')
790        [('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])]
791        >>> parse('{{py:x=1}}')
792        [('py', (1, 3), 'x=1')]
793        >>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}')
794        [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))]
795
796    Some exceptions::
797
798        >>> parse('{{continue}}')
799        Traceback (most recent call last):
800            ...
801        TemplateError: continue outside of for loop at line 1 column 3
802        >>> parse('{{if x}}foo')
803        Traceback (most recent call last):
804            ...
805        TemplateError: No {{endif}} at line 1 column 3
806        >>> parse('{{else}}')
807        Traceback (most recent call last):
808            ...
809        TemplateError: else outside of an if block at line 1 column 3
810        >>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}')
811        Traceback (most recent call last):
812            ...
813        TemplateError: Unexpected endif at line 1 column 25
814        >>> parse('{{if}}{{endif}}')
815        Traceback (most recent call last):
816            ...
817        TemplateError: if with no expression at line 1 column 3
818        >>> parse('{{for x y}}{{endfor}}')
819        Traceback (most recent call last):
820            ...
821        TemplateError: Bad for (no "in") in 'x y' at line 1 column 3
822        >>> parse('{{py:x=1\ny=2}}')
823        Traceback (most recent call last):
824            ...
825        TemplateError: Multi-line py blocks must start with a newline at line 1 column 3
826    """
827    if delimeters is None:
828        delimeters = ( Template.default_namespace['start_braces'],
829                       Template.default_namespace['end_braces'] )
830    tokens = lex(s, name=name, line_offset=line_offset, delimeters=delimeters)
831    result = []
832    while tokens:
833        next_chunk, tokens = parse_expr(tokens, name)
834        result.append(next_chunk)
835    return result
836
837
838def parse_expr(tokens, name, context=()):
839    if isinstance(tokens[0], basestring_):
840        return tokens[0], tokens[1:]
841    expr, pos = tokens[0]
842    expr = expr.strip()
843    if expr.startswith('py:'):
844        expr = expr[3:].lstrip(' \t')
845        if expr.startswith('\n') or expr.startswith('\r'):
846            expr = expr.lstrip('\r\n')
847            if '\r' in expr:
848                expr = expr.replace('\r\n', '\n')
849                expr = expr.replace('\r', '')
850            expr += '\n'
851        else:
852            if '\n' in expr:
853                raise TemplateError(
854                    'Multi-line py blocks must start with a newline',
855                    position=pos, name=name)
856        return ('py', pos, expr), tokens[1:]
857    elif expr in ('continue', 'break'):
858        if 'for' not in context:
859            raise TemplateError(
860                'continue outside of for loop',
861                position=pos, name=name)
862        return (expr, pos), tokens[1:]
863    elif expr.startswith('if '):
864        return parse_cond(tokens, name, context)
865    elif (expr.startswith('elif ')
866          or expr == 'else'):
867        raise TemplateError(
868            '%s outside of an if block' % expr.split()[0],
869            position=pos, name=name)
870    elif expr in ('if', 'elif', 'for'):
871        raise TemplateError(
872            '%s with no expression' % expr,
873            position=pos, name=name)
874    elif expr in ('endif', 'endfor', 'enddef'):
875        raise TemplateError(
876            'Unexpected %s' % expr,
877            position=pos, name=name)
878    elif expr.startswith('for '):
879        return parse_for(tokens, name, context)
880    elif expr.startswith('default '):
881        return parse_default(tokens, name, context)
882    elif expr.startswith('inherit '):
883        return parse_inherit(tokens, name, context)
884    elif expr.startswith('def '):
885        return parse_def(tokens, name, context)
886    elif expr.startswith('#'):
887        return ('comment', pos, tokens[0][0]), tokens[1:]
888    return ('expr', pos, tokens[0][0]), tokens[1:]
889
890
891def parse_cond(tokens, name, context):
892    start = tokens[0][1]
893    pieces = []
894    context = context + ('if',)
895    while 1:
896        if not tokens:
897            raise TemplateError(
898                'Missing {{endif}}',
899                position=start, name=name)
900        if (isinstance(tokens[0], tuple)
901            and tokens[0][0] == 'endif'):
902            return ('cond', start) + tuple(pieces), tokens[1:]
903        next_chunk, tokens = parse_one_cond(tokens, name, context)
904        pieces.append(next_chunk)
905
906
907def parse_one_cond(tokens, name, context):
908    (first, pos), tokens = tokens[0], tokens[1:]
909    content = []
910    if first.endswith(':'):
911        first = first[:-1]
912    if first.startswith('if '):
913        part = ('if', pos, first[3:].lstrip(), content)
914    elif first.startswith('elif '):
915        part = ('elif', pos, first[5:].lstrip(), content)
916    elif first == 'else':
917        part = ('else', pos, None, content)
918    else:
919        assert 0, "Unexpected token %r at %s" % (first, pos)
920    while 1:
921        if not tokens:
922            raise TemplateError(
923                'No {{endif}}',
924                position=pos, name=name)
925        if (isinstance(tokens[0], tuple)
926            and (tokens[0][0] == 'endif'
927                 or tokens[0][0].startswith('elif ')
928                 or tokens[0][0] == 'else')):
929            return part, tokens
930        next_chunk, tokens = parse_expr(tokens, name, context)
931        content.append(next_chunk)
932
933
934def parse_for(tokens, name, context):
935    first, pos = tokens[0]
936    tokens = tokens[1:]
937    context = ('for',) + context
938    content = []
939    assert first.startswith('for ')
940    if first.endswith(':'):
941        first = first[:-1]
942    first = first[3:].strip()
943    match = in_re.search(first)
944    if not match:
945        raise TemplateError(
946            'Bad for (no "in") in %r' % first,
947            position=pos, name=name)
948    vars = first[:match.start()]
949    if '(' in vars:
950        raise TemplateError(
951            'You cannot have () in the variable section of a for loop (%r)'
952            % vars, position=pos, name=name)
953    vars = tuple([
954        v.strip() for v in first[:match.start()].split(',')
955        if v.strip()])
956    expr = first[match.end():]
957    while 1:
958        if not tokens:
959            raise TemplateError(
960                'No {{endfor}}',
961                position=pos, name=name)
962        if (isinstance(tokens[0], tuple)
963            and tokens[0][0] == 'endfor'):
964            return ('for', pos, vars, expr, content), tokens[1:]
965        next_chunk, tokens = parse_expr(tokens, name, context)
966        content.append(next_chunk)
967
968
969def parse_default(tokens, name, context):
970    first, pos = tokens[0]
971    assert first.startswith('default ')
972    first = first.split(None, 1)[1]
973    parts = first.split('=', 1)
974    if len(parts) == 1:
975        raise TemplateError(
976            "Expression must be {{default var=value}}; no = found in %r" % first,
977            position=pos, name=name)
978    var = parts[0].strip()
979    if ',' in var:
980        raise TemplateError(
981            "{{default x, y = ...}} is not supported",
982            position=pos, name=name)
983    if not var_re.search(var):
984        raise TemplateError(
985            "Not a valid variable name for {{default}}: %r"
986            % var, position=pos, name=name)
987    expr = parts[1].strip()
988    return ('default', pos, var, expr), tokens[1:]
989
990
991def parse_inherit(tokens, name, context):
992    first, pos = tokens[0]
993    assert first.startswith('inherit ')
994    expr = first.split(None, 1)[1]
995    return ('inherit', pos, expr), tokens[1:]
996
997
998def parse_def(tokens, name, context):
999    first, start = tokens[0]
1000    tokens = tokens[1:]
1001    assert first.startswith('def ')
1002    first = first.split(None, 1)[1]
1003    if first.endswith(':'):
1004        first = first[:-1]
1005    if '(' not in first:
1006        func_name = first
1007        sig = ((), None, None, {})
1008    elif not first.endswith(')'):
1009        raise TemplateError("Function definition doesn't end with ): %s" % first,
1010                            position=start, name=name)
1011    else:
1012        first = first[:-1]
1013        func_name, sig_text = first.split('(', 1)
1014        sig = parse_signature(sig_text, name, start)
1015    context = context + ('def',)
1016    content = []
1017    while 1:
1018        if not tokens:
1019            raise TemplateError(
1020                'Missing {{enddef}}',
1021                position=start, name=name)
1022        if (isinstance(tokens[0], tuple)
1023            and tokens[0][0] == 'enddef'):
1024            return ('def', start, func_name, sig, content), tokens[1:]
1025        next_chunk, tokens = parse_expr(tokens, name, context)
1026        content.append(next_chunk)
1027
1028
1029def parse_signature(sig_text, name, pos):
1030    tokens = tokenize.generate_tokens(StringIO(sig_text).readline)
1031    sig_args = []
1032    var_arg = None
1033    var_kw = None
1034    defaults = {}
1035
1036    def get_token(pos=False):
1037        try:
1038            tok_type, tok_string, (srow, scol), (erow, ecol), line = next(tokens)
1039        except StopIteration:
1040            return tokenize.ENDMARKER, ''
1041        if pos:
1042            return tok_type, tok_string, (srow, scol), (erow, ecol)
1043        else:
1044            return tok_type, tok_string
1045    while 1:
1046        var_arg_type = None
1047        tok_type, tok_string = get_token()
1048        if tok_type == tokenize.ENDMARKER:
1049            break
1050        if tok_type == tokenize.OP and (tok_string == '*' or tok_string == '**'):
1051            var_arg_type = tok_string
1052            tok_type, tok_string = get_token()
1053        if tok_type != tokenize.NAME:
1054            raise TemplateError('Invalid signature: (%s)' % sig_text,
1055                                position=pos, name=name)
1056        var_name = tok_string
1057        tok_type, tok_string = get_token()
1058        if tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','):
1059            if var_arg_type == '*':
1060                var_arg = var_name
1061            elif var_arg_type == '**':
1062                var_kw = var_name
1063            else:
1064                sig_args.append(var_name)
1065            if tok_type == tokenize.ENDMARKER:
1066                break
1067            continue
1068        if var_arg_type is not None:
1069            raise TemplateError('Invalid signature: (%s)' % sig_text,
1070                                position=pos, name=name)
1071        if tok_type == tokenize.OP and tok_string == '=':
1072            nest_type = None
1073            unnest_type = None
1074            nest_count = 0
1075            start_pos = end_pos = None
1076            parts = []
1077            while 1:
1078                tok_type, tok_string, s, e = get_token(True)
1079                if start_pos is None:
1080                    start_pos = s
1081                end_pos = e
1082                if tok_type == tokenize.ENDMARKER and nest_count:
1083                    raise TemplateError('Invalid signature: (%s)' % sig_text,
1084                                        position=pos, name=name)
1085                if (not nest_count and
1086                    (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))):
1087                    default_expr = isolate_expression(sig_text, start_pos, end_pos)
1088                    defaults[var_name] = default_expr
1089                    sig_args.append(var_name)
1090                    break
1091                parts.append((tok_type, tok_string))
1092                if nest_count and tok_type == tokenize.OP and tok_string == nest_type:
1093                    nest_count += 1
1094                elif nest_count and tok_type == tokenize.OP and tok_string == unnest_type:
1095                    nest_count -= 1
1096                    if not nest_count:
1097                        nest_type = unnest_type = None
1098                elif not nest_count and tok_type == tokenize.OP and tok_string in ('(', '[', '{'):
1099                    nest_type = tok_string
1100                    nest_count = 1
1101                    unnest_type = {'(': ')', '[': ']', '{': '}'}[nest_type]
1102    return sig_args, var_arg, var_kw, defaults
1103
1104
1105def isolate_expression(string, start_pos, end_pos):
1106    srow, scol = start_pos
1107    srow -= 1
1108    erow, ecol = end_pos
1109    erow -= 1
1110    lines = string.splitlines(True)
1111    if srow == erow:
1112        return lines[srow][scol:ecol]
1113    parts = [lines[srow][scol:]]
1114    parts.extend(lines[srow+1:erow])
1115    if erow < len(lines):
1116        # It'll sometimes give (end_row_past_finish, 0)
1117        parts.append(lines[erow][:ecol])
1118    return ''.join(parts)
1119
1120_fill_command_usage = """\
1121%prog [OPTIONS] TEMPLATE arg=value
1122
1123Use py:arg=value to set a Python value; otherwise all values are
1124strings.
1125"""
1126
1127
1128def fill_command(args=None):
1129    import sys
1130    import optparse
1131    import pkg_resources
1132    import os
1133    if args is None:
1134        args = sys.argv[1:]
1135    dist = pkg_resources.get_distribution('Paste')
1136    parser = optparse.OptionParser(
1137        version=coerce_text(dist),
1138        usage=_fill_command_usage)
1139    parser.add_option(
1140        '-o', '--output',
1141        dest='output',
1142        metavar="FILENAME",
1143        help="File to write output to (default stdout)")
1144    parser.add_option(
1145        '--html',
1146        dest='use_html',
1147        action='store_true',
1148        help="Use HTML style filling (including automatic HTML quoting)")
1149    parser.add_option(
1150        '--env',
1151        dest='use_env',
1152        action='store_true',
1153        help="Put the environment in as top-level variables")
1154    options, args = parser.parse_args(args)
1155    if len(args) < 1:
1156        print('You must give a template filename')
1157        sys.exit(2)
1158    template_name = args[0]
1159    args = args[1:]
1160    vars = {}
1161    if options.use_env:
1162        vars.update(os.environ)
1163    for value in args:
1164        if '=' not in value:
1165            print('Bad argument: %r' % value)
1166            sys.exit(2)
1167        name, value = value.split('=', 1)
1168        if name.startswith('py:'):
1169            name = name[:3]
1170            value = eval(value)
1171        vars[name] = value
1172    if template_name == '-':
1173        template_content = sys.stdin.read()
1174        template_name = '<stdin>'
1175    else:
1176        f = open(template_name, 'rb')
1177        template_content = f.read()
1178        f.close()
1179    if options.use_html:
1180        TemplateClass = HTMLTemplate
1181    else:
1182        TemplateClass = Template
1183    template = TemplateClass(template_content, name=template_name)
1184    result = template.substitute(vars)
1185    if options.output:
1186        f = open(options.output, 'wb')
1187        f.write(result)
1188        f.close()
1189    else:
1190        sys.stdout.write(result)
1191
1192if __name__ == '__main__':
1193    fill_command()
1194