1# Copyright 2012 Benjamin Kalman
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15# TODO: Escaping control characters somehow. e.g. \{{, \{{-.
16
17import json
18import re
19
20'''Motemplate templates are data binding templates more-than-loosely inspired by
21ctemplate. Use like:
22
23  from motemplate import Motemplate
24
25  template = Motemplate('hello {{#foo bar/}} world')
26  input = {
27    'foo': [
28      { 'bar': 1 },
29      { 'bar': 2 },
30      { 'bar': 3 }
31    ]
32  }
33  print(template.render(input).text)
34
35Motemplate will use get() on contexts to return values, so to create custom
36getters (for example, something that populates values lazily from keys), just
37provide an object with a get() method.
38
39  class CustomContext(object):
40    def get(self, key):
41      return 10
42  print(Motemplate('hello {{world}}').render(CustomContext()).text)
43
44will print 'hello 10'.
45'''
46
47class ParseException(Exception):
48  '''The exception thrown while parsing a template.
49  '''
50  def __init__(self, error):
51    Exception.__init__(self, error)
52
53class RenderResult(object):
54  '''The result of a render operation.
55  '''
56  def __init__(self, text, errors):
57    self.text = text;
58    self.errors = errors
59
60  def __repr__(self):
61    return '%s(text=%s, errors=%s)' % (type(self).__name__,
62                                       self.text,
63                                       self.errors)
64
65  def __str__(self):
66    return repr(self)
67
68class _StringBuilder(object):
69  '''Efficiently builds strings.
70  '''
71  def __init__(self):
72    self._buf = []
73
74  def __len__(self):
75    self._Collapse()
76    return len(self._buf[0])
77
78  def Append(self, string):
79    if not isinstance(string, basestring):
80      string = str(string)
81    self._buf.append(string)
82
83  def ToString(self):
84    self._Collapse()
85    return self._buf[0]
86
87  def _Collapse(self):
88    self._buf = [u''.join(self._buf)]
89
90  def __repr__(self):
91    return self.ToString()
92
93  def __str__(self):
94    return repr(self)
95
96class _Contexts(object):
97  '''Tracks a stack of context objects, providing efficient key/value retrieval.
98  '''
99  class _Node(object):
100    '''A node within the stack. Wraps a real context and maintains the key/value
101    pairs seen so far.
102    '''
103    def __init__(self, value):
104      self._value = value
105      self._value_has_get = hasattr(value, 'get')
106      self._found = {}
107
108    def GetKeys(self):
109      '''Returns the list of keys that |_value| contains.
110      '''
111      return self._found.keys()
112
113    def Get(self, key):
114      '''Returns the value for |key|, or None if not found (including if
115      |_value| doesn't support key retrieval).
116      '''
117      if not self._value_has_get:
118        return None
119      value = self._found.get(key)
120      if value is not None:
121        return value
122      value = self._value.get(key)
123      if value is not None:
124        self._found[key] = value
125      return value
126
127    def __repr__(self):
128      return 'Node(value=%s, found=%s)' % (self._value, self._found)
129
130    def __str__(self):
131      return repr(self)
132
133  def __init__(self, globals_):
134    '''Initializes with the initial global contexts, listed in order from most
135    to least important.
136    '''
137    self._nodes = map(_Contexts._Node, globals_)
138    self._first_local = len(self._nodes)
139    self._value_info = {}
140
141  def CreateFromGlobals(self):
142    new = _Contexts([])
143    new._nodes = self._nodes[:self._first_local]
144    new._first_local = self._first_local
145    return new
146
147  def Push(self, context):
148    self._nodes.append(_Contexts._Node(context))
149
150  def Pop(self):
151    node = self._nodes.pop()
152    assert len(self._nodes) >= self._first_local
153    for found_key in node.GetKeys():
154      # [0] is the stack of nodes that |found_key| has been found in.
155      self._value_info[found_key][0].pop()
156
157  def FirstLocal(self):
158    if len(self._nodes) == self._first_local:
159      return None
160    return self._nodes[-1]._value
161
162  def Resolve(self, path):
163    # This method is only efficient at finding |key|; if |tail| has a value (and
164    # |key| evaluates to an indexable value) we'll need to descend into that.
165    key, tail = path.split('.', 1) if '.' in path else (path, None)
166    found = self._FindNodeValue(key)
167    if tail is None:
168      return found
169    for part in tail.split('.'):
170      if not hasattr(found, 'get'):
171        return None
172      found = found.get(part)
173    return found
174
175  def Scope(self, context, fn, *args):
176    self.Push(context)
177    try:
178      return fn(*args)
179    finally:
180      self.Pop()
181
182  def _FindNodeValue(self, key):
183    # |found_node_list| will be all the nodes that |key| has been found in.
184    # |checked_node_set| are those that have been checked.
185    info = self._value_info.get(key)
186    if info is None:
187      info = ([], set())
188      self._value_info[key] = info
189    found_node_list, checked_node_set = info
190
191    # Check all the nodes not yet checked for |key|.
192    newly_found = []
193    for node in reversed(self._nodes):
194      if node in checked_node_set:
195        break
196      value = node.Get(key)
197      if value is not None:
198        newly_found.append(node)
199      checked_node_set.add(node)
200
201    # The nodes will have been found in reverse stack order. After extending
202    # the found nodes, the freshest value will be at the tip of the stack.
203    found_node_list.extend(reversed(newly_found))
204    if not found_node_list:
205      return None
206
207    return found_node_list[-1]._value.get(key)
208
209class _Stack(object):
210  class Entry(object):
211    def __init__(self, name, id_):
212      self.name = name
213      self.id_ = id_
214
215  def __init__(self, entries=[]):
216    self.entries = entries
217
218  def Descend(self, name, id_):
219    descended = list(self.entries)
220    descended.append(_Stack.Entry(name, id_))
221    return _Stack(entries=descended)
222
223class _InternalContext(object):
224  def __init__(self):
225    self._render_state = None
226
227  def SetRenderState(self, render_state):
228    self._render_state = render_state
229
230  def get(self, key):
231    if key == 'errors':
232      errors = self._render_state._errors
233      return '\n'.join(errors) if errors else None
234    return None
235
236class _RenderState(object):
237  '''The state of a render call.
238  '''
239  def __init__(self, name, contexts, _stack=_Stack()):
240    self.text = _StringBuilder()
241    self.contexts = contexts
242    self._name = name
243    self._errors = []
244    self._stack = _stack
245
246  def AddResolutionError(self, id_, description=None):
247    message = id_.CreateResolutionErrorMessage(self._name, stack=self._stack)
248    if description is not None:
249      message = '%s (%s)' % (message, description)
250    self._errors.append(message)
251
252  def Copy(self):
253    return _RenderState(
254        self._name, self.contexts, _stack=self._stack)
255
256  def ForkPartial(self, custom_name, id_):
257    name = custom_name or id_.name
258    return _RenderState(name,
259                        self.contexts.CreateFromGlobals(),
260                        _stack=self._stack.Descend(name, id_))
261
262  def Merge(self, render_state, text_transform=None):
263    self._errors.extend(render_state._errors)
264    text = render_state.text.ToString()
265    if text_transform is not None:
266      text = text_transform(text)
267    self.text.Append(text)
268
269  def GetResult(self):
270    return RenderResult(self.text.ToString(), self._errors);
271
272class _Identifier(object):
273  '''An identifier of the form 'foo', 'foo.bar.baz', 'foo-bar.baz', etc.
274  '''
275  _VALID_ID_MATCHER = re.compile(r'^[a-zA-Z0-9@_/-]+$')
276
277  def __init__(self, name, line, column):
278    self.name = name
279    self.line = line
280    self.column = column
281    if name == '':
282      raise ParseException('Empty identifier %s' % self.GetDescription())
283    for part in name.split('.'):
284      if not _Identifier._VALID_ID_MATCHER.match(part):
285        raise ParseException('Invalid identifier %s' % self.GetDescription())
286
287  def GetDescription(self):
288    return '\'%s\' at line %s column %s' % (self.name, self.line, self.column)
289
290  def CreateResolutionErrorMessage(self, name, stack=None):
291    message = _StringBuilder()
292    message.Append('Failed to resolve %s in %s\n' % (self.GetDescription(),
293                                                     name))
294    if stack is not None:
295      for entry in reversed(stack.entries):
296        message.Append('  included as %s in %s\n' % (entry.id_.GetDescription(),
297                                                     entry.name))
298    return message.ToString().strip()
299
300  def __repr__(self):
301    return self.name
302
303  def __str__(self):
304    return repr(self)
305
306class _Node(object): pass
307
308class _LeafNode(_Node):
309  def __init__(self, start_line, end_line):
310    self._start_line = start_line
311    self._end_line = end_line
312
313  def StartsWithNewLine(self):
314    return False
315
316  def TrimStartingNewLine(self):
317    pass
318
319  def TrimEndingSpaces(self):
320    return 0
321
322  def TrimEndingNewLine(self):
323    pass
324
325  def EndsWithEmptyLine(self):
326    return False
327
328  def GetStartLine(self):
329    return self._start_line
330
331  def GetEndLine(self):
332    return self._end_line
333
334  def __str__(self):
335    return repr(self)
336
337class _DecoratorNode(_Node):
338  def __init__(self, content):
339    self._content = content
340
341  def StartsWithNewLine(self):
342    return self._content.StartsWithNewLine()
343
344  def TrimStartingNewLine(self):
345    self._content.TrimStartingNewLine()
346
347  def TrimEndingSpaces(self):
348    return self._content.TrimEndingSpaces()
349
350  def TrimEndingNewLine(self):
351    self._content.TrimEndingNewLine()
352
353  def EndsWithEmptyLine(self):
354    return self._content.EndsWithEmptyLine()
355
356  def GetStartLine(self):
357    return self._content.GetStartLine()
358
359  def GetEndLine(self):
360    return self._content.GetEndLine()
361
362  def __repr__(self):
363    return str(self._content)
364
365  def __str__(self):
366    return repr(self)
367
368class _InlineNode(_DecoratorNode):
369  def __init__(self, content):
370    _DecoratorNode.__init__(self, content)
371
372  def Render(self, render_state):
373    content_render_state = render_state.Copy()
374    self._content.Render(content_render_state)
375    render_state.Merge(content_render_state,
376                       text_transform=lambda text: text.replace('\n', ''))
377
378class _IndentedNode(_DecoratorNode):
379  def __init__(self, content, indentation):
380    _DecoratorNode.__init__(self, content)
381    self._indent_str = ' ' * indentation
382
383  def Render(self, render_state):
384    if isinstance(self._content, _CommentNode):
385      return
386    def inlinify(text):
387      if len(text) == 0:  # avoid rendering a blank line
388        return ''
389      buf = _StringBuilder()
390      buf.Append(self._indent_str)
391      buf.Append(text.replace('\n', '\n%s' % self._indent_str))
392      if not text.endswith('\n'):  # partials will often already end in a \n
393        buf.Append('\n')
394      return buf.ToString()
395    content_render_state = render_state.Copy()
396    self._content.Render(content_render_state)
397    render_state.Merge(content_render_state, text_transform=inlinify)
398
399class _BlockNode(_DecoratorNode):
400  def __init__(self, content):
401    _DecoratorNode.__init__(self, content)
402    content.TrimStartingNewLine()
403    content.TrimEndingSpaces()
404
405  def Render(self, render_state):
406    self._content.Render(render_state)
407
408class _NodeCollection(_Node):
409  def __init__(self, nodes):
410    assert nodes
411    self._nodes = nodes
412
413  def Render(self, render_state):
414    for node in self._nodes:
415      node.Render(render_state)
416
417  def StartsWithNewLine(self):
418    return self._nodes[0].StartsWithNewLine()
419
420  def TrimStartingNewLine(self):
421    self._nodes[0].TrimStartingNewLine()
422
423  def TrimEndingSpaces(self):
424    return self._nodes[-1].TrimEndingSpaces()
425
426  def TrimEndingNewLine(self):
427    self._nodes[-1].TrimEndingNewLine()
428
429  def EndsWithEmptyLine(self):
430    return self._nodes[-1].EndsWithEmptyLine()
431
432  def GetStartLine(self):
433    return self._nodes[0].GetStartLine()
434
435  def GetEndLine(self):
436    return self._nodes[-1].GetEndLine()
437
438  def __repr__(self):
439    return ''.join(str(node) for node in self._nodes)
440
441class _StringNode(_Node):
442  '''Just a string.
443  '''
444  def __init__(self, string, start_line, end_line):
445    self._string = string
446    self._start_line = start_line
447    self._end_line = end_line
448
449  def Render(self, render_state):
450    render_state.text.Append(self._string)
451
452  def StartsWithNewLine(self):
453    return self._string.startswith('\n')
454
455  def TrimStartingNewLine(self):
456    if self.StartsWithNewLine():
457      self._string = self._string[1:]
458
459  def TrimEndingSpaces(self):
460    original_length = len(self._string)
461    self._string = self._string[:self._LastIndexOfSpaces()]
462    return original_length - len(self._string)
463
464  def TrimEndingNewLine(self):
465    if self._string.endswith('\n'):
466      self._string = self._string[:len(self._string) - 1]
467
468  def EndsWithEmptyLine(self):
469    index = self._LastIndexOfSpaces()
470    return index == 0 or self._string[index - 1] == '\n'
471
472  def _LastIndexOfSpaces(self):
473    index = len(self._string)
474    while index > 0 and self._string[index - 1] == ' ':
475      index -= 1
476    return index
477
478  def GetStartLine(self):
479    return self._start_line
480
481  def GetEndLine(self):
482    return self._end_line
483
484  def __repr__(self):
485    return self._string
486
487class _EscapedVariableNode(_LeafNode):
488  '''{{foo}}
489  '''
490  def __init__(self, id_):
491    _LeafNode.__init__(self, id_.line, id_.line)
492    self._id = id_
493
494  def Render(self, render_state):
495    value = render_state.contexts.Resolve(self._id.name)
496    if value is None:
497      render_state.AddResolutionError(self._id)
498      return
499    string = value if isinstance(value, basestring) else str(value)
500    render_state.text.Append(string.replace('&', '&')
501                                   .replace('<', '&lt;')
502                                   .replace('>', '&gt;'))
503
504  def __repr__(self):
505    return '{{%s}}' % self._id
506
507class _UnescapedVariableNode(_LeafNode):
508  '''{{{foo}}}
509  '''
510  def __init__(self, id_):
511    _LeafNode.__init__(self, id_.line, id_.line)
512    self._id = id_
513
514  def Render(self, render_state):
515    value = render_state.contexts.Resolve(self._id.name)
516    if value is None:
517      render_state.AddResolutionError(self._id)
518      return
519    string = value if isinstance(value, basestring) else str(value)
520    render_state.text.Append(string)
521
522  def __repr__(self):
523    return '{{{%s}}}' % self._id
524
525class _CommentNode(_LeafNode):
526  '''{{- This is a comment -}}
527  An empty placeholder node for correct indented rendering behaviour.
528  '''
529  def __init__(self, start_line, end_line):
530    _LeafNode.__init__(self, start_line, end_line)
531
532  def Render(self, render_state):
533    pass
534
535  def __repr__(self):
536    return '<comment>'
537
538class _SectionNode(_DecoratorNode):
539  '''{{#var:foo}} ... {{/foo}}
540  '''
541  def __init__(self, bind_to, id_, content):
542    _DecoratorNode.__init__(self, content)
543    self._bind_to = bind_to
544    self._id = id_
545
546  def Render(self, render_state):
547    value = render_state.contexts.Resolve(self._id.name)
548    if isinstance(value, list):
549      for item in value:
550        if self._bind_to is not None:
551          render_state.contexts.Scope({self._bind_to.name: item},
552                                      self._content.Render, render_state)
553        else:
554          self._content.Render(render_state)
555    elif hasattr(value, 'get'):
556      if self._bind_to is not None:
557        render_state.contexts.Scope({self._bind_to.name: value},
558                                    self._content.Render, render_state)
559      else:
560        render_state.contexts.Scope(value, self._content.Render, render_state)
561    else:
562      render_state.AddResolutionError(self._id)
563
564  def __repr__(self):
565    return '{{#%s}}%s{{/%s}}' % (
566        self._id, _DecoratorNode.__repr__(self), self._id)
567
568class _VertedSectionNode(_DecoratorNode):
569  '''{{?var:foo}} ... {{/foo}}
570  '''
571  def __init__(self, bind_to, id_, content):
572    _DecoratorNode.__init__(self, content)
573    self._bind_to = bind_to
574    self._id = id_
575
576  def Render(self, render_state):
577    value = render_state.contexts.Resolve(self._id.name)
578    if _VertedSectionNode.ShouldRender(value):
579      if self._bind_to is not None:
580        render_state.contexts.Scope({self._bind_to.name: value},
581                                    self._content.Render, render_state)
582      else:
583        self._content.Render(render_state)
584
585  def __repr__(self):
586    return '{{?%s}}%s{{/%s}}' % (
587        self._id, _DecoratorNode.__repr__(self), self._id)
588
589  @staticmethod
590  def ShouldRender(value):
591    if value is None:
592      return False
593    if isinstance(value, bool):
594      return value
595    if isinstance(value, list):
596      return len(value) > 0
597    return True
598
599class _InvertedSectionNode(_DecoratorNode):
600  '''{{^foo}} ... {{/foo}}
601  '''
602  def __init__(self, bind_to, id_, content):
603    _DecoratorNode.__init__(self, content)
604    if bind_to is not None:
605      raise ParseException('{{^%s:%s}} does not support variable binding'
606                           % (bind_to, id_))
607    self._id = id_
608
609  def Render(self, render_state):
610    value = render_state.contexts.Resolve(self._id.name)
611    if not _VertedSectionNode.ShouldRender(value):
612      self._content.Render(render_state)
613
614  def __repr__(self):
615    return '{{^%s}}%s{{/%s}}' % (
616        self._id, _DecoratorNode.__repr__(self), self._id)
617
618class _AssertionNode(_LeafNode):
619  '''{{!foo Some comment about foo}}
620  '''
621  def __init__(self, id_, description):
622    _LeafNode.__init__(self, id_.line, id_.line)
623    self._id = id_
624    self._description = description
625
626  def Render(self, render_state):
627    if render_state.contexts.Resolve(self._id.name) is None:
628      render_state.AddResolutionError(self._id, description=self._description)
629
630  def __repr__(self):
631    return '{{!%s %s}}' % (self._id, self._description)
632
633class _JsonNode(_LeafNode):
634  '''{{*foo}}
635  '''
636  def __init__(self, id_):
637    _LeafNode.__init__(self, id_.line, id_.line)
638    self._id = id_
639
640  def Render(self, render_state):
641    value = render_state.contexts.Resolve(self._id.name)
642    if value is None:
643      render_state.AddResolutionError(self._id)
644      return
645    render_state.text.Append(json.dumps(value, separators=(',',':')))
646
647  def __repr__(self):
648    return '{{*%s}}' % self._id
649
650class _PartialNodeWithArguments(_DecoratorNode):
651  def __init__(self, partial, args):
652    if isinstance(partial, Motemplate):
653      # Preserve any get() method that the caller has added.
654      if hasattr(partial, 'get'):
655        self.get = partial.get
656      partial = partial._top_node
657    _DecoratorNode.__init__(self, partial)
658    self._partial = partial
659    self._args = args
660
661  def Render(self, render_state):
662    render_state.contexts.Scope(self._args, self._partial.Render, render_state)
663
664class _PartialNodeInContext(_DecoratorNode):
665  def __init__(self, partial, context):
666    if isinstance(partial, Motemplate):
667      # Preserve any get() method that the caller has added.
668      if hasattr(partial, 'get'):
669        self.get = partial.get
670      partial = partial._top_node
671    _DecoratorNode.__init__(self, partial)
672    self._partial = partial
673    self._context = context
674
675  def Render(self, render_state):
676    original_contexts = render_state.contexts
677    try:
678      render_state.contexts = self._context
679      render_state.contexts.Scope(
680          # The first local context of |original_contexts| will be the
681          # arguments that were passed to the partial, if any.
682          original_contexts.FirstLocal() or {},
683          self._partial.Render, render_state)
684    finally:
685      render_state.contexts = original_contexts
686
687class _PartialNode(_LeafNode):
688  '''{{+var:foo}} ... {{/foo}}
689  '''
690  def __init__(self, bind_to, id_, content):
691    _LeafNode.__init__(self, id_.line, id_.line)
692    self._bind_to = bind_to
693    self._id = id_
694    self._content = content
695    self._args = None
696    self._pass_through_id = None
697
698  @classmethod
699  def Inline(cls, id_):
700    return cls(None, id_, None)
701
702  def Render(self, render_state):
703    value = render_state.contexts.Resolve(self._id.name)
704    if value is None:
705      render_state.AddResolutionError(self._id)
706      return
707    if not isinstance(value, (Motemplate, _Node)):
708      render_state.AddResolutionError(self._id, description='not a partial')
709      return
710
711    if isinstance(value, Motemplate):
712      node, name = value._top_node, value._name
713    else:
714      node, name = value, None
715
716    partial_render_state = render_state.ForkPartial(name, self._id)
717
718    arg_context = {}
719    if self._pass_through_id is not None:
720      context = render_state.contexts.Resolve(self._pass_through_id.name)
721      if context is not None:
722        arg_context[self._pass_through_id.name] = context
723    if self._args is not None:
724      def resolve_args(args):
725        resolved = {}
726        for key, value in args.iteritems():
727          if isinstance(value, dict):
728            assert len(value.keys()) == 1
729            id_of_partial, partial_args = value.items()[0]
730            partial = render_state.contexts.Resolve(id_of_partial.name)
731            if partial is not None:
732              resolved[key] = _PartialNodeWithArguments(
733                  partial, resolve_args(partial_args))
734          else:
735            context = render_state.contexts.Resolve(value.name)
736            if context is not None:
737              resolved[key] = context
738        return resolved
739      arg_context.update(resolve_args(self._args))
740    if self._bind_to and self._content:
741      arg_context[self._bind_to.name] = _PartialNodeInContext(
742          self._content, render_state.contexts)
743    if arg_context:
744      partial_render_state.contexts.Push(arg_context)
745
746    node.Render(partial_render_state)
747
748    render_state.Merge(
749        partial_render_state,
750        text_transform=lambda text: text[:-1] if text.endswith('\n') else text)
751
752  def SetArguments(self, args):
753    self._args = args
754
755  def PassThroughArgument(self, id_):
756    self._pass_through_id = id_
757
758  def __repr__(self):
759    return '{{+%s}}' % self._id
760
761_TOKENS = {}
762
763class _Token(object):
764  '''The tokens that can appear in a template.
765  '''
766  class Data(object):
767    def __init__(self, name, text, clazz):
768      self.name = name
769      self.text = text
770      self.clazz = clazz
771      _TOKENS[text] = self
772
773    def ElseNodeClass(self):
774      if self.clazz == _VertedSectionNode:
775        return _InvertedSectionNode
776      if self.clazz == _InvertedSectionNode:
777        return _VertedSectionNode
778      return None
779
780    def __repr__(self):
781      return self.name
782
783    def __str__(self):
784      return repr(self)
785
786  OPEN_START_SECTION = Data(
787      'OPEN_START_SECTION'         , '{{#', _SectionNode)
788  OPEN_START_VERTED_SECTION = Data(
789      'OPEN_START_VERTED_SECTION'  , '{{?', _VertedSectionNode)
790  OPEN_START_INVERTED_SECTION = Data(
791      'OPEN_START_INVERTED_SECTION', '{{^', _InvertedSectionNode)
792  OPEN_ASSERTION = Data(
793      'OPEN_ASSERTION'             , '{{!', _AssertionNode)
794  OPEN_JSON = Data(
795      'OPEN_JSON'                  , '{{*', _JsonNode)
796  OPEN_PARTIAL = Data(
797      'OPEN_PARTIAL'               , '{{+', _PartialNode)
798  OPEN_ELSE = Data(
799      'OPEN_ELSE'                  , '{{:', None)
800  OPEN_END_SECTION = Data(
801      'OPEN_END_SECTION'           , '{{/', None)
802  INLINE_END_SECTION = Data(
803      'INLINE_END_SECTION'         , '/}}', None)
804  OPEN_UNESCAPED_VARIABLE = Data(
805      'OPEN_UNESCAPED_VARIABLE'    , '{{{', _UnescapedVariableNode)
806  CLOSE_MUSTACHE3 = Data(
807      'CLOSE_MUSTACHE3'            , '}}}', None)
808  OPEN_COMMENT = Data(
809      'OPEN_COMMENT'               , '{{-', _CommentNode)
810  CLOSE_COMMENT = Data(
811      'CLOSE_COMMENT'              , '-}}', None)
812  OPEN_VARIABLE = Data(
813      'OPEN_VARIABLE'              , '{{' , _EscapedVariableNode)
814  CLOSE_MUSTACHE = Data(
815      'CLOSE_MUSTACHE'             , '}}' , None)
816  CHARACTER = Data(
817      'CHARACTER'                  , '.'  , None)
818
819class _TokenStream(object):
820  '''Tokeniser for template parsing.
821  '''
822  def __init__(self, string):
823    self.next_token = None
824    self.next_line = 1
825    self.next_column = 0
826    self._string = string
827    self._cursor = 0
828    self.Advance()
829
830  def HasNext(self):
831    return self.next_token is not None
832
833  def NextCharacter(self):
834    if self.next_token is _Token.CHARACTER:
835      return self._string[self._cursor - 1]
836    return None
837
838  def Advance(self):
839    if self._cursor > 0 and self._string[self._cursor - 1] == '\n':
840      self.next_line += 1
841      self.next_column = 0
842    elif self.next_token is not None:
843      self.next_column += len(self.next_token.text)
844
845    self.next_token = None
846
847    if self._cursor == len(self._string):
848      return None
849    assert self._cursor < len(self._string)
850
851    if (self._cursor + 1 < len(self._string) and
852        self._string[self._cursor + 1] in '{}'):
853      self.next_token = (
854          _TOKENS.get(self._string[self._cursor:self._cursor+3]) or
855          _TOKENS.get(self._string[self._cursor:self._cursor+2]))
856
857    if self.next_token is None:
858      self.next_token = _Token.CHARACTER
859
860    self._cursor += len(self.next_token.text)
861    return self
862
863  def AdvanceOver(self, token, description=None):
864    parse_error = None
865    if not self.next_token:
866      parse_error = 'Reached EOF but expected %s' % token.name
867    elif self.next_token is not token:
868      parse_error = 'Expecting token %s but got %s at line %s' % (
869                     token.name, self.next_token.name, self.next_line)
870    if parse_error:
871      parse_error += ' %s' % description or ''
872      raise ParseException(parse_error)
873    return self.Advance()
874
875  def AdvanceOverSeparator(self, char, description=None):
876    self.SkipWhitespace()
877    next_char = self.NextCharacter()
878    if next_char != char:
879      parse_error = 'Expected \'%s\'. got \'%s\'' % (char, next_char)
880      if description is not None:
881        parse_error += ' (%s)' % description
882      raise ParseException(parse_error)
883    self.AdvanceOver(_Token.CHARACTER)
884    self.SkipWhitespace()
885
886  def AdvanceOverNextString(self, excluded=''):
887    start = self._cursor - len(self.next_token.text)
888    while (self.next_token is _Token.CHARACTER and
889           # Can use -1 here because token length of CHARACTER is 1.
890           self._string[self._cursor - 1] not in excluded):
891      self.Advance()
892    end = self._cursor - (len(self.next_token.text) if self.next_token else 0)
893    return self._string[start:end]
894
895  def AdvanceToNextWhitespace(self):
896    return self.AdvanceOverNextString(excluded=' \n\r\t')
897
898  def SkipWhitespace(self):
899    while (self.next_token is _Token.CHARACTER and
900           # Can use -1 here because token length of CHARACTER is 1.
901           self._string[self._cursor - 1] in ' \n\r\t'):
902      self.Advance()
903
904  def __repr__(self):
905    return '%s(next_token=%s, remainder=%s)' % (type(self).__name__,
906                                                self.next_token,
907                                                self._string[self._cursor:])
908
909  def __str__(self):
910    return repr(self)
911
912class Motemplate(object):
913  '''A motemplate template.
914  '''
915  def __init__(self, template, name=None):
916    self.source = template
917    self._name = name
918    tokens = _TokenStream(template)
919    self._top_node = self._ParseSection(tokens)
920    if not self._top_node:
921      raise ParseException('Template is empty')
922    if tokens.HasNext():
923      raise ParseException('There are still tokens remaining at %s, '
924                           'was there an end-section without a start-section?' %
925                           tokens.next_line)
926
927  def _ParseSection(self, tokens):
928    nodes = []
929    while tokens.HasNext():
930      if tokens.next_token in (_Token.OPEN_END_SECTION,
931                               _Token.OPEN_ELSE):
932        # Handled after running parseSection within the SECTION cases, so this
933        # is a terminating condition. If there *is* an orphaned
934        # OPEN_END_SECTION, it will be caught by noticing that there are
935        # leftover tokens after termination.
936        break
937      elif tokens.next_token in (_Token.CLOSE_MUSTACHE,
938                                 _Token.CLOSE_MUSTACHE3):
939        raise ParseException('Orphaned %s at line %s' % (tokens.next_token.name,
940                                                         tokens.next_line))
941      nodes += self._ParseNextOpenToken(tokens)
942
943    for i, node in enumerate(nodes):
944      if isinstance(node, _StringNode):
945        continue
946
947      previous_node = nodes[i - 1] if i > 0 else None
948      next_node = nodes[i + 1] if i < len(nodes) - 1 else None
949      rendered_node = None
950
951      if node.GetStartLine() != node.GetEndLine():
952        rendered_node = _BlockNode(node)
953        if previous_node:
954          previous_node.TrimEndingSpaces()
955        if next_node:
956          next_node.TrimStartingNewLine()
957      elif ((not previous_node or previous_node.EndsWithEmptyLine()) and
958            (not next_node or next_node.StartsWithNewLine())):
959        indentation = 0
960        if previous_node:
961          indentation = previous_node.TrimEndingSpaces()
962        if next_node:
963          next_node.TrimStartingNewLine()
964        rendered_node = _IndentedNode(node, indentation)
965      else:
966        rendered_node = _InlineNode(node)
967
968      nodes[i] = rendered_node
969
970    if len(nodes) == 0:
971      return None
972    if len(nodes) == 1:
973      return nodes[0]
974    return _NodeCollection(nodes)
975
976  def _ParseNextOpenToken(self, tokens):
977    next_token = tokens.next_token
978
979    if next_token is _Token.CHARACTER:
980      # Plain strings.
981      start_line = tokens.next_line
982      string = tokens.AdvanceOverNextString()
983      return [_StringNode(string, start_line, tokens.next_line)]
984    elif next_token in (_Token.OPEN_VARIABLE,
985                        _Token.OPEN_UNESCAPED_VARIABLE,
986                        _Token.OPEN_JSON):
987      # Inline nodes that don't take arguments.
988      tokens.Advance()
989      close_token = (_Token.CLOSE_MUSTACHE3
990                     if next_token is _Token.OPEN_UNESCAPED_VARIABLE else
991                     _Token.CLOSE_MUSTACHE)
992      id_ = self._NextIdentifier(tokens)
993      tokens.AdvanceOver(close_token)
994      return [next_token.clazz(id_)]
995    elif next_token is _Token.OPEN_ASSERTION:
996      # Inline nodes that take arguments.
997      tokens.Advance()
998      id_ = self._NextIdentifier(tokens)
999      node = next_token.clazz(id_, tokens.AdvanceOverNextString())
1000      tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1001      return [node]
1002    elif next_token in (_Token.OPEN_PARTIAL,
1003                        _Token.OPEN_START_SECTION,
1004                        _Token.OPEN_START_VERTED_SECTION,
1005                        _Token.OPEN_START_INVERTED_SECTION):
1006      # Block nodes, though they may have inline syntax like {{#foo bar /}}.
1007      tokens.Advance()
1008      bind_to, id_ = None, self._NextIdentifier(tokens)
1009      if tokens.NextCharacter() == ':':
1010        # This section has the format {{#bound:id}} as opposed to just {{id}}.
1011        # That is, |id_| is actually the identifier to bind what the section
1012        # is producing, not the identifier of where to find that content.
1013        tokens.AdvanceOverSeparator(':')
1014        bind_to, id_ = id_, self._NextIdentifier(tokens)
1015      partial_args = None
1016      if next_token is _Token.OPEN_PARTIAL:
1017        partial_args = self._ParsePartialNodeArgs(tokens)
1018        if tokens.next_token is not _Token.CLOSE_MUSTACHE:
1019          # Inline syntax for partial types.
1020          if bind_to is not None:
1021            raise ParseException(
1022                'Cannot bind %s to a self-closing partial' % bind_to)
1023          tokens.AdvanceOver(_Token.INLINE_END_SECTION)
1024          partial_node = _PartialNode.Inline(id_)
1025          partial_node.SetArguments(partial_args)
1026          return [partial_node]
1027      elif tokens.next_token is not _Token.CLOSE_MUSTACHE:
1028        # Inline syntax for non-partial types. Support select node types:
1029        # variables, partials, JSON.
1030        line, column = tokens.next_line, (tokens.next_column + 1)
1031        name = tokens.AdvanceToNextWhitespace()
1032        clazz = _UnescapedVariableNode
1033        if name.startswith('*'):
1034          clazz = _JsonNode
1035        elif name.startswith('+'):
1036          clazz = _PartialNode.Inline
1037        if clazz is not _UnescapedVariableNode:
1038          name = name[1:]
1039          column += 1
1040        inline_node = clazz(_Identifier(name, line, column))
1041        if isinstance(inline_node, _PartialNode):
1042          inline_node.SetArguments(self._ParsePartialNodeArgs(tokens))
1043          if bind_to is not None:
1044            inline_node.PassThroughArgument(bind_to)
1045        tokens.SkipWhitespace()
1046        tokens.AdvanceOver(_Token.INLINE_END_SECTION)
1047        return [next_token.clazz(bind_to, id_, inline_node)]
1048      # Block syntax.
1049      tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1050      section = self._ParseSection(tokens)
1051      else_node_class = next_token.ElseNodeClass()  # may not have one
1052      else_section = None
1053      if (else_node_class is not None and
1054          tokens.next_token is _Token.OPEN_ELSE):
1055        self._OpenElse(tokens, id_)
1056        else_section = self._ParseSection(tokens)
1057      self._CloseSection(tokens, id_)
1058      nodes = []
1059      if section is not None:
1060        node = next_token.clazz(bind_to, id_, section)
1061        if partial_args:
1062          node.SetArguments(partial_args)
1063        nodes.append(node)
1064      if else_section is not None:
1065        nodes.append(else_node_class(bind_to, id_, else_section))
1066      return nodes
1067    elif next_token is _Token.OPEN_COMMENT:
1068      # Comments.
1069      start_line = tokens.next_line
1070      self._AdvanceOverComment(tokens)
1071      return [_CommentNode(start_line, tokens.next_line)]
1072
1073  def _AdvanceOverComment(self, tokens):
1074    tokens.AdvanceOver(_Token.OPEN_COMMENT)
1075    depth = 1
1076    while tokens.HasNext() and depth > 0:
1077      if tokens.next_token is _Token.OPEN_COMMENT:
1078        depth += 1
1079      elif tokens.next_token is _Token.CLOSE_COMMENT:
1080        depth -= 1
1081      tokens.Advance()
1082
1083  def _CloseSection(self, tokens, id_):
1084    tokens.AdvanceOver(_Token.OPEN_END_SECTION,
1085                       description='to match %s' % id_.GetDescription())
1086    next_string = tokens.AdvanceOverNextString()
1087    if next_string != '' and next_string != id_.name:
1088      raise ParseException(
1089          'Start section %s doesn\'t match end %s' % (id_, next_string))
1090    tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1091
1092  def _OpenElse(self, tokens, id_):
1093    tokens.AdvanceOver(_Token.OPEN_ELSE)
1094    next_string = tokens.AdvanceOverNextString()
1095    if next_string != '' and next_string != id_.name:
1096      raise ParseException(
1097          'Start section %s doesn\'t match else %s' % (id_, next_string))
1098    tokens.AdvanceOver(_Token.CLOSE_MUSTACHE)
1099
1100  def _ParsePartialNodeArgs(self, tokens):
1101    args = {}
1102    tokens.SkipWhitespace()
1103    while (tokens.next_token is _Token.CHARACTER and
1104           tokens.NextCharacter() != ')'):
1105      key = tokens.AdvanceOverNextString(excluded=':')
1106      tokens.AdvanceOverSeparator(':')
1107      if tokens.NextCharacter() == '(':
1108        tokens.AdvanceOverSeparator('(')
1109        inner_id = self._NextIdentifier(tokens)
1110        inner_args = self._ParsePartialNodeArgs(tokens)
1111        tokens.AdvanceOverSeparator(')')
1112        args[key] = {inner_id: inner_args}
1113      else:
1114        args[key] = self._NextIdentifier(tokens)
1115    return args or None
1116
1117  def _NextIdentifier(self, tokens):
1118    tokens.SkipWhitespace()
1119    column_start = tokens.next_column + 1
1120    id_ = _Identifier(tokens.AdvanceOverNextString(excluded=' \n\r\t:()'),
1121                      tokens.next_line,
1122                      column_start)
1123    tokens.SkipWhitespace()
1124    return id_
1125
1126  def Render(self, *user_contexts):
1127    '''Renders this template given a variable number of contexts to read out
1128    values from (such as those appearing in {{foo}}).
1129    '''
1130    internal_context = _InternalContext()
1131    contexts = list(user_contexts)
1132    contexts.append({
1133      '_': internal_context,
1134      'false': False,
1135      'true': True,
1136    })
1137    render_state = _RenderState(self._name or '<root>', _Contexts(contexts))
1138    internal_context.SetRenderState(render_state)
1139    self._top_node.Render(render_state)
1140    return render_state.GetResult()
1141
1142  def render(self, *contexts):
1143    return self.Render(*contexts)
1144
1145  def __eq__(self, other):
1146    return self.source == other.source and self._name == other._name
1147
1148  def __ne__(self, other):
1149    return not (self == other)
1150
1151  def __repr__(self):
1152    return str('%s(%s)' % (type(self).__name__, self._top_node))
1153
1154  def __str__(self):
1155    return repr(self)
1156