1# markdown is released under the BSD license
2# Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later)
3# Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b)
4# Copyright 2004 Manfred Stienstra (the original version)
5#
6# All rights reserved.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# *   Redistributions of source code must retain the above copyright
12#     notice, this list of conditions and the following disclaimer.
13# *   Redistributions in binary form must reproduce the above copyright
14#     notice, this list of conditions and the following disclaimer in the
15#     documentation and/or other materials provided with the distribution.
16# *   Neither the name of the <organization> nor the
17#     names of its contributors may be used to endorse or promote products
18#     derived from this software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE PYTHON MARKDOWN PROJECT ''AS IS'' AND ANY
21# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
22# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23# DISCLAIMED. IN NO EVENT SHALL ANY CONTRIBUTORS TO THE PYTHON MARKDOWN PROJECT
24# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30# POSSIBILITY OF SUCH DAMAGE.
31
32
33"""
34CORE MARKDOWN BLOCKPARSER
35===========================================================================
36
37This parser handles basic parsing of Markdown blocks.  It doesn't concern itself
38with inline elements such as **bold** or *italics*, but rather just catches
39blocks, lists, quotes, etc.
40
41The BlockParser is made up of a bunch of BlockProssors, each handling a
42different type of block. Extensions may add/replace/remove BlockProcessors
43as they need to alter how markdown blocks are parsed.
44"""
45
46from __future__ import absolute_import
47from __future__ import division
48from __future__ import unicode_literals
49import logging
50import re
51from . import util
52from .blockparser import BlockParser
53
54logger =  logging.getLogger('MARKDOWN')
55
56
57def build_block_parser(md_instance, **kwargs):
58    """ Build the default block parser used by Markdown. """
59    parser = BlockParser(md_instance)
60    parser.blockprocessors['empty'] = EmptyBlockProcessor(parser)
61    parser.blockprocessors['indent'] = ListIndentProcessor(parser)
62    parser.blockprocessors['code'] = CodeBlockProcessor(parser)
63    parser.blockprocessors['hashheader'] = HashHeaderProcessor(parser)
64    parser.blockprocessors['setextheader'] = SetextHeaderProcessor(parser)
65    parser.blockprocessors['hr'] = HRProcessor(parser)
66    parser.blockprocessors['olist'] = OListProcessor(parser)
67    parser.blockprocessors['ulist'] = UListProcessor(parser)
68    parser.blockprocessors['quote'] = BlockQuoteProcessor(parser)
69    parser.blockprocessors['paragraph'] = ParagraphProcessor(parser)
70    return parser
71
72
73class BlockProcessor:
74    """ Base class for block processors.
75
76    Each subclass will provide the methods below to work with the source and
77    tree. Each processor will need to define it's own ``test`` and ``run``
78    methods. The ``test`` method should return True or False, to indicate
79    whether the current block should be processed by this processor. If the
80    test passes, the parser will call the processors ``run`` method.
81
82    """
83
84    def __init__(self, parser):
85        self.parser = parser
86        self.tab_length = parser.markdown.tab_length
87
88    def lastChild(self, parent):
89        """ Return the last child of an etree element. """
90        if len(parent):
91            return parent[-1]
92        else:
93            return None
94
95    def detab(self, text):
96        """ Remove a tab from the front of each line of the given text. """
97        newtext = []
98        lines = text.split('\n')
99        for line in lines:
100            if line.startswith(' '*self.tab_length):
101                newtext.append(line[self.tab_length:])
102            elif not line.strip():
103                newtext.append('')
104            else:
105                break
106        return '\n'.join(newtext), '\n'.join(lines[len(newtext):])
107
108    def looseDetab(self, text, level=1):
109        """ Remove a tab from front of lines but allowing dedented lines. """
110        lines = text.split('\n')
111        for i in range(len(lines)):
112            if lines[i].startswith(' '*self.tab_length*level):
113                lines[i] = lines[i][self.tab_length*level:]
114        return '\n'.join(lines)
115
116    def test(self, parent, block):
117        """ Test for block type. Must be overridden by subclasses.
118
119        As the parser loops through processors, it will call the ``test`` method
120        on each to determine if the given block of text is of that type. This
121        method must return a boolean ``True`` or ``False``. The actual method of
122        testing is left to the needs of that particular block type. It could
123        be as simple as ``block.startswith(some_string)`` or a complex regular
124        expression. As the block type may be different depending on the parent
125        of the block (i.e. inside a list), the parent etree element is also
126        provided and may be used as part of the test.
127
128        Keywords:
129
130        * ``parent``: A etree element which will be the parent of the block.
131        * ``block``: A block of text from the source which has been split at
132            blank lines.
133        """
134        pass
135
136    def run(self, parent, blocks):
137        """ Run processor. Must be overridden by subclasses.
138
139        When the parser determines the appropriate type of a block, the parser
140        will call the corresponding processor's ``run`` method. This method
141        should parse the individual lines of the block and append them to
142        the etree.
143
144        Note that both the ``parent`` and ``etree`` keywords are pointers
145        to instances of the objects which should be edited in place. Each
146        processor must make changes to the existing objects as there is no
147        mechanism to return new/different objects to replace them.
148
149        This means that this method should be adding SubElements or adding text
150        to the parent, and should remove (``pop``) or add (``insert``) items to
151        the list of blocks.
152
153        Keywords:
154
155        * ``parent``: A etree element which is the parent of the current block.
156        * ``blocks``: A list of all remaining blocks of the document.
157        """
158        pass
159
160
161class ListIndentProcessor(BlockProcessor):
162    """ Process children of list items.
163
164    Example:
165        * a list item
166            process this part
167
168            or this part
169
170    """
171
172    ITEM_TYPES = ['li']
173    LIST_TYPES = ['ul', 'ol']
174
175    def __init__(self, *args):
176        BlockProcessor.__init__(self, *args)
177        self.INDENT_RE = re.compile(r'^(([ ]{%s})+)'% self.tab_length)
178
179    def test(self, parent, block):
180        return block.startswith(' '*self.tab_length) and \
181                not self.parser.state.isstate('detabbed') and  \
182                (parent.tag in self.ITEM_TYPES or \
183                    (len(parent) and parent[-1] and \
184                        (parent[-1].tag in self.LIST_TYPES)
185                    )
186                )
187
188    def run(self, parent, blocks):
189        block = blocks.pop(0)
190        level, sibling = self.get_level(parent, block)
191        block = self.looseDetab(block, level)
192
193        self.parser.state.set('detabbed')
194        if parent.tag in self.ITEM_TYPES:
195            # It's possible that this parent has a 'ul' or 'ol' child list
196            # with a member.  If that is the case, then that should be the
197            # parent.  This is intended to catch the edge case of an indented
198            # list whose first member was parsed previous to this point
199            # see OListProcessor
200            if len(parent) and parent[-1].tag in self.LIST_TYPES:
201                self.parser.parseBlocks(parent[-1], [block])
202            else:
203                # The parent is already a li. Just parse the child block.
204                self.parser.parseBlocks(parent, [block])
205        elif sibling.tag in self.ITEM_TYPES:
206            # The sibling is a li. Use it as parent.
207            self.parser.parseBlocks(sibling, [block])
208        elif len(sibling) and sibling[-1].tag in self.ITEM_TYPES:
209            # The parent is a list (``ol`` or ``ul``) which has children.
210            # Assume the last child li is the parent of this block.
211            if sibling[-1].text:
212                # If the parent li has text, that text needs to be moved to a p
213                # The p must be 'inserted' at beginning of list in the event
214                # that other children already exist i.e.; a nested sublist.
215                p = util.etree.Element('p')
216                p.text = sibling[-1].text
217                sibling[-1].text = ''
218                sibling[-1].insert(0, p)
219            self.parser.parseChunk(sibling[-1], block)
220        else:
221            self.create_item(sibling, block)
222        self.parser.state.reset()
223
224    def create_item(self, parent, block):
225        """ Create a new li and parse the block with it as the parent. """
226        li = util.etree.SubElement(parent, 'li')
227        self.parser.parseBlocks(li, [block])
228
229    def get_level(self, parent, block):
230        """ Get level of indent based on list level. """
231        # Get indent level
232        m = self.INDENT_RE.match(block)
233        if m:
234            indent_level = len(m.group(1))/self.tab_length
235        else:
236            indent_level = 0
237        if self.parser.state.isstate('list'):
238            # We're in a tightlist - so we already are at correct parent.
239            level = 1
240        else:
241            # We're in a looselist - so we need to find parent.
242            level = 0
243        # Step through children of tree to find matching indent level.
244        while indent_level > level:
245            child = self.lastChild(parent)
246            if child and (child.tag in self.LIST_TYPES or child.tag in self.ITEM_TYPES):
247                if child.tag in self.LIST_TYPES:
248                    level += 1
249                parent = child
250            else:
251                # No more child levels. If we're short of indent_level,
252                # we have a code block. So we stop here.
253                break
254        return level, parent
255
256
257class CodeBlockProcessor(BlockProcessor):
258    """ Process code blocks. """
259
260    def test(self, parent, block):
261        return block.startswith(' '*self.tab_length)
262
263    def run(self, parent, blocks):
264        sibling = self.lastChild(parent)
265        block = blocks.pop(0)
266        theRest = ''
267        if sibling and sibling.tag == "pre" and len(sibling) \
268                    and sibling[0].tag == "code":
269            # The previous block was a code block. As blank lines do not start
270            # new code blocks, append this block to the previous, adding back
271            # linebreaks removed from the split into a list.
272            code = sibling[0]
273            block, theRest = self.detab(block)
274            code.text = util.AtomicString('%s\n%s\n' % (code.text, block.rstrip()))
275        else:
276            # This is a new codeblock. Create the elements and insert text.
277            pre = util.etree.SubElement(parent, 'pre')
278            code = util.etree.SubElement(pre, 'code')
279            block, theRest = self.detab(block)
280            code.text = util.AtomicString('%s\n' % block.rstrip())
281        if theRest:
282            # This block contained unindented line(s) after the first indented
283            # line. Insert these lines as the first block of the master blocks
284            # list for future processing.
285            blocks.insert(0, theRest)
286
287
288class BlockQuoteProcessor(BlockProcessor):
289
290    RE = re.compile(r'(^|\n)[ ]{0,3}>[ ]?(.*)')
291
292    def test(self, parent, block):
293        return bool(self.RE.search(block))
294
295    def run(self, parent, blocks):
296        block = blocks.pop(0)
297        m = self.RE.search(block)
298        if m:
299            before = block[:m.start()] # Lines before blockquote
300            # Pass lines before blockquote in recursively for parsing forst.
301            self.parser.parseBlocks(parent, [before])
302            # Remove ``> `` from begining of each line.
303            block = '\n'.join([self.clean(line) for line in
304                            block[m.start():].split('\n')])
305        sibling = self.lastChild(parent)
306        if sibling and sibling.tag == "blockquote":
307            # Previous block was a blockquote so set that as this blocks parent
308            quote = sibling
309        else:
310            # This is a new blockquote. Create a new parent element.
311            quote = util.etree.SubElement(parent, 'blockquote')
312        # Recursively parse block with blockquote as parent.
313        # change parser state so blockquotes embedded in lists use p tags
314        self.parser.state.set('blockquote')
315        self.parser.parseChunk(quote, block)
316        self.parser.state.reset()
317
318    def clean(self, line):
319        """ Remove ``>`` from beginning of a line. """
320        m = self.RE.match(line)
321        if line.strip() == ">":
322            return ""
323        elif m:
324            return m.group(2)
325        else:
326            return line
327
328class OListProcessor(BlockProcessor):
329    """ Process ordered list blocks. """
330
331    TAG = 'ol'
332    # Detect an item (``1. item``). ``group(1)`` contains contents of item.
333    RE = re.compile(r'^[ ]{0,3}\d+\.[ ]+(.*)')
334    # Detect items on secondary lines. they can be of either list type.
335    CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.)|[*+-])[ ]+(.*)')
336    # Detect indented (nested) items of either type
337    INDENT_RE = re.compile(r'^[ ]{4,7}((\d+\.)|[*+-])[ ]+.*')
338    # The integer (python string) with which the lists starts (default=1)
339    # Eg: If list is intialized as)
340    #   3. Item
341    # The ol tag will get starts="3" attribute
342    STARTSWITH = '1'
343    # List of allowed sibling tags.
344    SIBLING_TAGS = ['ol', 'ul']
345
346    def test(self, parent, block):
347        return bool(self.RE.match(block))
348
349    def run(self, parent, blocks):
350        # Check fr multiple items in one block.
351        items = self.get_items(blocks.pop(0))
352        sibling = self.lastChild(parent)
353
354        if sibling and sibling.tag in self.SIBLING_TAGS:
355            # Previous block was a list item, so set that as parent
356            lst = sibling
357            # make sure previous item is in a p- if the item has text, then it
358            # it isn't in a p
359            if lst[-1].text:
360                # since it's possible there are other children for this sibling,
361                # we can't just SubElement the p, we need to insert it as the
362                # first item
363                p = util.etree.Element('p')
364                p.text = lst[-1].text
365                lst[-1].text = ''
366                lst[-1].insert(0, p)
367            # if the last item has a tail, then the tail needs to be put in a p
368            # likely only when a header is not followed by a blank line
369            lch = self.lastChild(lst[-1])
370            if lch is not None and lch.tail:
371                p = util.etree.SubElement(lst[-1], 'p')
372                p.text = lch.tail.lstrip()
373                lch.tail = ''
374
375            # parse first block differently as it gets wrapped in a p.
376            li = util.etree.SubElement(lst, 'li')
377            self.parser.state.set('looselist')
378            firstitem = items.pop(0)
379            self.parser.parseBlocks(li, [firstitem])
380            self.parser.state.reset()
381        elif parent.tag in ['ol', 'ul']:
382            # this catches the edge case of a multi-item indented list whose
383            # first item is in a blank parent-list item:
384            # * * subitem1
385            #     * subitem2
386            # see also ListIndentProcessor
387            lst = parent
388        else:
389            # This is a new list so create parent with appropriate tag.
390            lst = util.etree.SubElement(parent, self.TAG)
391            # Check if a custom start integer is set
392            if not self.parser.markdown.lazy_ol and self.STARTSWITH !='1':
393                lst.attrib['start'] = self.STARTSWITH
394
395        self.parser.state.set('list')
396        # Loop through items in block, recursively parsing each with the
397        # appropriate parent.
398        for item in items:
399            if item.startswith(' '*self.tab_length):
400                # Item is indented. Parse with last item as parent
401                self.parser.parseBlocks(lst[-1], [item])
402            else:
403                # New item. Create li and parse with it as parent
404                li = util.etree.SubElement(lst, 'li')
405                self.parser.parseBlocks(li, [item])
406        self.parser.state.reset()
407
408    def get_items(self, block):
409        """ Break a block into list items. """
410        items = []
411        for line in block.split('\n'):
412            m = self.CHILD_RE.match(line)
413            if m:
414                # This is a new list item
415                # Check first item for the start index
416                if not items and self.TAG=='ol':
417                    # Detect the integer value of first list item
418                    INTEGER_RE = re.compile('(\d+)')
419                    self.STARTSWITH = INTEGER_RE.match(m.group(1)).group()
420                # Append to the list
421                items.append(m.group(3))
422            elif self.INDENT_RE.match(line):
423                # This is an indented (possibly nested) item.
424                if items[-1].startswith(' '*self.tab_length):
425                    # Previous item was indented. Append to that item.
426                    items[-1] = '%s\n%s' % (items[-1], line)
427                else:
428                    items.append(line)
429            else:
430                # This is another line of previous item. Append to that item.
431                items[-1] = '%s\n%s' % (items[-1], line)
432        return items
433
434
435class UListProcessor(OListProcessor):
436    """ Process unordered list blocks. """
437
438    TAG = 'ul'
439    RE = re.compile(r'^[ ]{0,3}[*+-][ ]+(.*)')
440
441
442class HashHeaderProcessor(BlockProcessor):
443    """ Process Hash Headers. """
444
445    # Detect a header at start of any line in block
446    RE = re.compile(r'(^|\n)(?P<level>#{1,6})(?P<header>.*?)#*(\n|$)')
447
448    def test(self, parent, block):
449        return bool(self.RE.search(block))
450
451    def run(self, parent, blocks):
452        block = blocks.pop(0)
453        m = self.RE.search(block)
454        if m:
455            before = block[:m.start()] # All lines before header
456            after = block[m.end():]    # All lines after header
457            if before:
458                # As the header was not the first line of the block and the
459                # lines before the header must be parsed first,
460                # recursively parse this lines as a block.
461                self.parser.parseBlocks(parent, [before])
462            # Create header using named groups from RE
463            h = util.etree.SubElement(parent, 'h%d' % len(m.group('level')))
464            h.text = m.group('header').strip()
465            if after:
466                # Insert remaining lines as first block for future parsing.
467                blocks.insert(0, after)
468        else:
469            # This should never happen, but just in case...
470            logger.warn("We've got a problem header: %r" % block)
471
472
473class SetextHeaderProcessor(BlockProcessor):
474    """ Process Setext-style Headers. """
475
476    # Detect Setext-style header. Must be first 2 lines of block.
477    RE = re.compile(r'^.*?\n[=-]+[ ]*(\n|$)', re.MULTILINE)
478
479    def test(self, parent, block):
480        return bool(self.RE.match(block))
481
482    def run(self, parent, blocks):
483        lines = blocks.pop(0).split('\n')
484        # Determine level. ``=`` is 1 and ``-`` is 2.
485        if lines[1].startswith('='):
486            level = 1
487        else:
488            level = 2
489        h = util.etree.SubElement(parent, 'h%d' % level)
490        h.text = lines[0].strip()
491        if len(lines) > 2:
492            # Block contains additional lines. Add to  master blocks for later.
493            blocks.insert(0, '\n'.join(lines[2:]))
494
495
496class HRProcessor(BlockProcessor):
497    """ Process Horizontal Rules. """
498
499    RE = r'^[ ]{0,3}((-+[ ]{0,2}){3,}|(_+[ ]{0,2}){3,}|(\*+[ ]{0,2}){3,})[ ]*'
500    # Detect hr on any line of a block.
501    SEARCH_RE = re.compile(RE, re.MULTILINE)
502
503    def test(self, parent, block):
504        m = self.SEARCH_RE.search(block)
505        # No atomic grouping in python so we simulate it here for performance.
506        # The regex only matches what would be in the atomic group - the HR.
507        # Then check if we are at end of block or if next char is a newline.
508        if m and (m.end() == len(block) or block[m.end()] == '\n'):
509            # Save match object on class instance so we can use it later.
510            self.match = m
511            return True
512        return False
513
514    def run(self, parent, blocks):
515        block = blocks.pop(0)
516        # Check for lines in block before hr.
517        prelines = block[:self.match.start()].rstrip('\n')
518        if prelines:
519            # Recursively parse lines before hr so they get parsed first.
520            self.parser.parseBlocks(parent, [prelines])
521        # create hr
522        util.etree.SubElement(parent, 'hr')
523        # check for lines in block after hr.
524        postlines = block[self.match.end():].lstrip('\n')
525        if postlines:
526            # Add lines after hr to master blocks for later parsing.
527            blocks.insert(0, postlines)
528
529
530
531class EmptyBlockProcessor(BlockProcessor):
532    """ Process blocks that are empty or start with an empty line. """
533
534    def test(self, parent, block):
535        return not block or block.startswith('\n')
536
537    def run(self, parent, blocks):
538        block = blocks.pop(0)
539        filler = '\n\n'
540        if block:
541            # Starts with empty line
542            # Only replace a single line.
543            filler = '\n'
544            # Save the rest for later.
545            theRest = block[1:]
546            if theRest:
547                # Add remaining lines to master blocks for later.
548                blocks.insert(0, theRest)
549        sibling = self.lastChild(parent)
550        if sibling and sibling.tag == 'pre' and len(sibling) and sibling[0].tag == 'code':
551            # Last block is a codeblock. Append to preserve whitespace.
552            sibling[0].text = util.AtomicString('%s%s' % (sibling[0].text, filler))
553
554
555class ParagraphProcessor(BlockProcessor):
556    """ Process Paragraph blocks. """
557
558    def test(self, parent, block):
559        return True
560
561    def run(self, parent, blocks):
562        block = blocks.pop(0)
563        if block.strip():
564            # Not a blank block. Add to parent, otherwise throw it away.
565            if self.parser.state.isstate('list'):
566                # The parent is a tight-list.
567                #
568                # Check for any children. This will likely only happen in a
569                # tight-list when a header isn't followed by a blank line.
570                # For example:
571                #
572                #     * # Header
573                #     Line 2 of list item - not part of header.
574                sibling = self.lastChild(parent)
575                if sibling is not None:
576                    # Insetrt after sibling.
577                    if sibling.tail:
578                        sibling.tail = '%s\n%s' % (sibling.tail, block)
579                    else:
580                        sibling.tail = '\n%s' % block
581                else:
582                    # Append to parent.text
583                    if parent.text:
584                        parent.text = '%s\n%s' % (parent.text, block)
585                    else:
586                        parent.text = block.lstrip()
587            else:
588                # Create a regular paragraph
589                p = util.etree.SubElement(parent, 'p')
590                p.text = block.lstrip()
591