1# Copyright (C) 2004-2006 Python Software Foundation
2# Authors: Baxter, Wouters and Warsaw
3# Contact: email-sig@python.org
4
5"""FeedParser - An email feed parser.
6
7The feed parser implements an interface for incrementally parsing an email
8message, line by line.  This has advantages for certain applications, such as
9those reading email messages off a socket.
10
11FeedParser.feed() is the primary interface for pushing new data into the
12parser.  It returns when there's nothing more it can do with the available
13data.  When you have no more data to push into the parser, call .close().
14This completes the parsing and returns the root message object.
15
16The other advantage of this parser is that it will never raise a parsing
17exception.  Instead, when it finds something unexpected, it adds a 'defect' to
18the current message.  Defects are just instances that live on the message
19object's .defects attribute.
20"""
21
22__all__ = ['FeedParser']
23
24import re
25
26from email import errors
27from email import message
28
29NLCRE = re.compile('\r\n|\r|\n')
30NLCRE_bol = re.compile('(\r\n|\r|\n)')
31NLCRE_eol = re.compile('(\r\n|\r|\n)\Z')
32NLCRE_crack = re.compile('(\r\n|\r|\n)')
33# RFC 2822 $3.6.8 Optional fields.  ftext is %d33-57 / %d59-126, Any character
34# except controls, SP, and ":".
35headerRE = re.compile(r'^(From |[\041-\071\073-\176]{1,}:|[\t ])')
36EMPTYSTRING = ''
37NL = '\n'
38
39NeedMoreData = object()
40
41
42
43class BufferedSubFile(object):
44    """A file-ish object that can have new data loaded into it.
45
46    You can also push and pop line-matching predicates onto a stack.  When the
47    current predicate matches the current line, a false EOF response
48    (i.e. empty string) is returned instead.  This lets the parser adhere to a
49    simple abstraction -- it parses until EOF closes the current message.
50    """
51    def __init__(self):
52        # The last partial line pushed into this object.
53        self._partial = ''
54        # The list of full, pushed lines, in reverse order
55        self._lines = []
56        # The stack of false-EOF checking predicates.
57        self._eofstack = []
58        # A flag indicating whether the file has been closed or not.
59        self._closed = False
60
61    def push_eof_matcher(self, pred):
62        self._eofstack.append(pred)
63
64    def pop_eof_matcher(self):
65        return self._eofstack.pop()
66
67    def close(self):
68        # Don't forget any trailing partial line.
69        self._lines.append(self._partial)
70        self._partial = ''
71        self._closed = True
72
73    def readline(self):
74        if not self._lines:
75            if self._closed:
76                return ''
77            return NeedMoreData
78        # Pop the line off the stack and see if it matches the current
79        # false-EOF predicate.
80        line = self._lines.pop()
81        # RFC 2046, section 5.1.2 requires us to recognize outer level
82        # boundaries at any level of inner nesting.  Do this, but be sure it's
83        # in the order of most to least nested.
84        for ateof in self._eofstack[::-1]:
85            if ateof(line):
86                # We're at the false EOF.  But push the last line back first.
87                self._lines.append(line)
88                return ''
89        return line
90
91    def unreadline(self, line):
92        # Let the consumer push a line back into the buffer.
93        assert line is not NeedMoreData
94        self._lines.append(line)
95
96    def push(self, data):
97        """Push some new data into this object."""
98        # Handle any previous leftovers
99        data, self._partial = self._partial + data, ''
100        # Crack into lines, but preserve the newlines on the end of each
101        parts = NLCRE_crack.split(data)
102        # The *ahem* interesting behaviour of re.split when supplied grouping
103        # parentheses is that the last element of the resulting list is the
104        # data after the final RE.  In the case of a NL/CR terminated string,
105        # this is the empty string.
106        self._partial = parts.pop()
107        #GAN 29Mar09  bugs 1555570, 1721862  Confusion at 8K boundary ending with \r:
108        # is there a \n to follow later?
109        if not self._partial and parts and parts[-1].endswith('\r'):
110            self._partial = parts.pop(-2)+parts.pop()
111        # parts is a list of strings, alternating between the line contents
112        # and the eol character(s).  Gather up a list of lines after
113        # re-attaching the newlines.
114        lines = []
115        for i in range(len(parts) // 2):
116            lines.append(parts[i*2] + parts[i*2+1])
117        self.pushlines(lines)
118
119    def pushlines(self, lines):
120        # Reverse and insert at the front of the lines.
121        self._lines[:0] = lines[::-1]
122
123    def is_closed(self):
124        return self._closed
125
126    def __iter__(self):
127        return self
128
129    def next(self):
130        line = self.readline()
131        if line == '':
132            raise StopIteration
133        return line
134
135
136
137class FeedParser:
138    """A feed-style parser of email."""
139
140    def __init__(self, _factory=message.Message):
141        """_factory is called with no arguments to create a new message obj"""
142        self._factory = _factory
143        self._input = BufferedSubFile()
144        self._msgstack = []
145        self._parse = self._parsegen().next
146        self._cur = None
147        self._last = None
148        self._headersonly = False
149
150    # Non-public interface for supporting Parser's headersonly flag
151    def _set_headersonly(self):
152        self._headersonly = True
153
154    def feed(self, data):
155        """Push more data into the parser."""
156        self._input.push(data)
157        self._call_parse()
158
159    def _call_parse(self):
160        try:
161            self._parse()
162        except StopIteration:
163            pass
164
165    def close(self):
166        """Parse all remaining data and return the root message object."""
167        self._input.close()
168        self._call_parse()
169        root = self._pop_message()
170        assert not self._msgstack
171        # Look for final set of defects
172        if root.get_content_maintype() == 'multipart' \
173               and not root.is_multipart():
174            root.defects.append(errors.MultipartInvariantViolationDefect())
175        return root
176
177    def _new_message(self):
178        msg = self._factory()
179        if self._cur and self._cur.get_content_type() == 'multipart/digest':
180            msg.set_default_type('message/rfc822')
181        if self._msgstack:
182            self._msgstack[-1].attach(msg)
183        self._msgstack.append(msg)
184        self._cur = msg
185        self._last = msg
186
187    def _pop_message(self):
188        retval = self._msgstack.pop()
189        if self._msgstack:
190            self._cur = self._msgstack[-1]
191        else:
192            self._cur = None
193        return retval
194
195    def _parsegen(self):
196        # Create a new message and start by parsing headers.
197        self._new_message()
198        headers = []
199        # Collect the headers, searching for a line that doesn't match the RFC
200        # 2822 header or continuation pattern (including an empty line).
201        for line in self._input:
202            if line is NeedMoreData:
203                yield NeedMoreData
204                continue
205            if not headerRE.match(line):
206                # If we saw the RFC defined header/body separator
207                # (i.e. newline), just throw it away. Otherwise the line is
208                # part of the body so push it back.
209                if not NLCRE.match(line):
210                    self._input.unreadline(line)
211                break
212            headers.append(line)
213        # Done with the headers, so parse them and figure out what we're
214        # supposed to see in the body of the message.
215        self._parse_headers(headers)
216        # Headers-only parsing is a backwards compatibility hack, which was
217        # necessary in the older parser, which could raise errors.  All
218        # remaining lines in the input are thrown into the message body.
219        if self._headersonly:
220            lines = []
221            while True:
222                line = self._input.readline()
223                if line is NeedMoreData:
224                    yield NeedMoreData
225                    continue
226                if line == '':
227                    break
228                lines.append(line)
229            self._cur.set_payload(EMPTYSTRING.join(lines))
230            return
231        if self._cur.get_content_type() == 'message/delivery-status':
232            # message/delivery-status contains blocks of headers separated by
233            # a blank line.  We'll represent each header block as a separate
234            # nested message object, but the processing is a bit different
235            # than standard message/* types because there is no body for the
236            # nested messages.  A blank line separates the subparts.
237            while True:
238                self._input.push_eof_matcher(NLCRE.match)
239                for retval in self._parsegen():
240                    if retval is NeedMoreData:
241                        yield NeedMoreData
242                        continue
243                    break
244                msg = self._pop_message()
245                # We need to pop the EOF matcher in order to tell if we're at
246                # the end of the current file, not the end of the last block
247                # of message headers.
248                self._input.pop_eof_matcher()
249                # The input stream must be sitting at the newline or at the
250                # EOF.  We want to see if we're at the end of this subpart, so
251                # first consume the blank line, then test the next line to see
252                # if we're at this subpart's EOF.
253                while True:
254                    line = self._input.readline()
255                    if line is NeedMoreData:
256                        yield NeedMoreData
257                        continue
258                    break
259                while True:
260                    line = self._input.readline()
261                    if line is NeedMoreData:
262                        yield NeedMoreData
263                        continue
264                    break
265                if line == '':
266                    break
267                # Not at EOF so this is a line we're going to need.
268                self._input.unreadline(line)
269            return
270        if self._cur.get_content_maintype() == 'message':
271            # The message claims to be a message/* type, then what follows is
272            # another RFC 2822 message.
273            for retval in self._parsegen():
274                if retval is NeedMoreData:
275                    yield NeedMoreData
276                    continue
277                break
278            self._pop_message()
279            return
280        if self._cur.get_content_maintype() == 'multipart':
281            boundary = self._cur.get_boundary()
282            if boundary is None:
283                # The message /claims/ to be a multipart but it has not
284                # defined a boundary.  That's a problem which we'll handle by
285                # reading everything until the EOF and marking the message as
286                # defective.
287                self._cur.defects.append(errors.NoBoundaryInMultipartDefect())
288                lines = []
289                for line in self._input:
290                    if line is NeedMoreData:
291                        yield NeedMoreData
292                        continue
293                    lines.append(line)
294                self._cur.set_payload(EMPTYSTRING.join(lines))
295                return
296            # Create a line match predicate which matches the inter-part
297            # boundary as well as the end-of-multipart boundary.  Don't push
298            # this onto the input stream until we've scanned past the
299            # preamble.
300            separator = '--' + boundary
301            boundaryre = re.compile(
302                '(?P<sep>' + re.escape(separator) +
303                r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
304            capturing_preamble = True
305            preamble = []
306            linesep = False
307            while True:
308                line = self._input.readline()
309                if line is NeedMoreData:
310                    yield NeedMoreData
311                    continue
312                if line == '':
313                    break
314                mo = boundaryre.match(line)
315                if mo:
316                    # If we're looking at the end boundary, we're done with
317                    # this multipart.  If there was a newline at the end of
318                    # the closing boundary, then we need to initialize the
319                    # epilogue with the empty string (see below).
320                    if mo.group('end'):
321                        linesep = mo.group('linesep')
322                        break
323                    # We saw an inter-part boundary.  Were we in the preamble?
324                    if capturing_preamble:
325                        if preamble:
326                            # According to RFC 2046, the last newline belongs
327                            # to the boundary.
328                            lastline = preamble[-1]
329                            eolmo = NLCRE_eol.search(lastline)
330                            if eolmo:
331                                preamble[-1] = lastline[:-len(eolmo.group(0))]
332                            self._cur.preamble = EMPTYSTRING.join(preamble)
333                        capturing_preamble = False
334                        self._input.unreadline(line)
335                        continue
336                    # We saw a boundary separating two parts.  Consume any
337                    # multiple boundary lines that may be following.  Our
338                    # interpretation of RFC 2046 BNF grammar does not produce
339                    # body parts within such double boundaries.
340                    while True:
341                        line = self._input.readline()
342                        if line is NeedMoreData:
343                            yield NeedMoreData
344                            continue
345                        mo = boundaryre.match(line)
346                        if not mo:
347                            self._input.unreadline(line)
348                            break
349                    # Recurse to parse this subpart; the input stream points
350                    # at the subpart's first line.
351                    self._input.push_eof_matcher(boundaryre.match)
352                    for retval in self._parsegen():
353                        if retval is NeedMoreData:
354                            yield NeedMoreData
355                            continue
356                        break
357                    # Because of RFC 2046, the newline preceding the boundary
358                    # separator actually belongs to the boundary, not the
359                    # previous subpart's payload (or epilogue if the previous
360                    # part is a multipart).
361                    if self._last.get_content_maintype() == 'multipart':
362                        epilogue = self._last.epilogue
363                        if epilogue == '':
364                            self._last.epilogue = None
365                        elif epilogue is not None:
366                            mo = NLCRE_eol.search(epilogue)
367                            if mo:
368                                end = len(mo.group(0))
369                                self._last.epilogue = epilogue[:-end]
370                    else:
371                        payload = self._last.get_payload()
372                        if isinstance(payload, basestring):
373                            mo = NLCRE_eol.search(payload)
374                            if mo:
375                                payload = payload[:-len(mo.group(0))]
376                                self._last.set_payload(payload)
377                    self._input.pop_eof_matcher()
378                    self._pop_message()
379                    # Set the multipart up for newline cleansing, which will
380                    # happen if we're in a nested multipart.
381                    self._last = self._cur
382                else:
383                    # I think we must be in the preamble
384                    assert capturing_preamble
385                    preamble.append(line)
386            # We've seen either the EOF or the end boundary.  If we're still
387            # capturing the preamble, we never saw the start boundary.  Note
388            # that as a defect and store the captured text as the payload.
389            # Everything from here to the EOF is epilogue.
390            if capturing_preamble:
391                self._cur.defects.append(errors.StartBoundaryNotFoundDefect())
392                self._cur.set_payload(EMPTYSTRING.join(preamble))
393                epilogue = []
394                for line in self._input:
395                    if line is NeedMoreData:
396                        yield NeedMoreData
397                        continue
398                self._cur.epilogue = EMPTYSTRING.join(epilogue)
399                return
400            # If the end boundary ended in a newline, we'll need to make sure
401            # the epilogue isn't None
402            if linesep:
403                epilogue = ['']
404            else:
405                epilogue = []
406            for line in self._input:
407                if line is NeedMoreData:
408                    yield NeedMoreData
409                    continue
410                epilogue.append(line)
411            # Any CRLF at the front of the epilogue is not technically part of
412            # the epilogue.  Also, watch out for an empty string epilogue,
413            # which means a single newline.
414            if epilogue:
415                firstline = epilogue[0]
416                bolmo = NLCRE_bol.match(firstline)
417                if bolmo:
418                    epilogue[0] = firstline[len(bolmo.group(0)):]
419            self._cur.epilogue = EMPTYSTRING.join(epilogue)
420            return
421        # Otherwise, it's some non-multipart type, so the entire rest of the
422        # file contents becomes the payload.
423        lines = []
424        for line in self._input:
425            if line is NeedMoreData:
426                yield NeedMoreData
427                continue
428            lines.append(line)
429        self._cur.set_payload(EMPTYSTRING.join(lines))
430
431    def _parse_headers(self, lines):
432        # Passed a list of lines that make up the headers for the current msg
433        lastheader = ''
434        lastvalue = []
435        for lineno, line in enumerate(lines):
436            # Check for continuation
437            if line[0] in ' \t':
438                if not lastheader:
439                    # The first line of the headers was a continuation.  This
440                    # is illegal, so let's note the defect, store the illegal
441                    # line, and ignore it for purposes of headers.
442                    defect = errors.FirstHeaderLineIsContinuationDefect(line)
443                    self._cur.defects.append(defect)
444                    continue
445                lastvalue.append(line)
446                continue
447            if lastheader:
448                # XXX reconsider the joining of folded lines
449                lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n')
450                self._cur[lastheader] = lhdr
451                lastheader, lastvalue = '', []
452            # Check for envelope header, i.e. unix-from
453            if line.startswith('From '):
454                if lineno == 0:
455                    # Strip off the trailing newline
456                    mo = NLCRE_eol.search(line)
457                    if mo:
458                        line = line[:-len(mo.group(0))]
459                    self._cur.set_unixfrom(line)
460                    continue
461                elif lineno == len(lines) - 1:
462                    # Something looking like a unix-from at the end - it's
463                    # probably the first line of the body, so push back the
464                    # line and stop.
465                    self._input.unreadline(line)
466                    return
467                else:
468                    # Weirdly placed unix-from line.  Note this as a defect
469                    # and ignore it.
470                    defect = errors.MisplacedEnvelopeHeaderDefect(line)
471                    self._cur.defects.append(defect)
472                    continue
473            # Split the line on the colon separating field name from value.
474            i = line.find(':')
475            if i < 0:
476                defect = errors.MalformedHeaderDefect(line)
477                self._cur.defects.append(defect)
478                continue
479            lastheader = line[:i]
480            lastvalue = [line[i+1:].lstrip()]
481        # Done with all the lines, so handle the last header.
482        if lastheader:
483            # XXX reconsider the joining of folded lines
484            self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n')
485