xref: /external/antlr/antlr-3.4/runtime/Python/unittests/teststreams.py

# -*- coding: utf-8 -*-

import os
import unittest
from StringIO import StringIO
import antlr3


class TestStringStream(unittest.TestCase):
    """Test case for the StringStream class."""

    def testSize(self):
        """StringStream.size()"""

        stream = antlr3.StringStream('foo')

        self.failUnlessEqual(stream.size(), 3)


    def testIndex(self):
        """StringStream.index()"""

        stream = antlr3.StringStream('foo')

        self.failUnlessEqual(stream.index(), 0)


    def testConsume(self):
        """StringStream.consume()"""

        stream = antlr3.StringStream('foo\nbar')

        stream.consume() # f
        self.failUnlessEqual(stream.index(), 1)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.line, 1)

        stream.consume() # o
        self.failUnlessEqual(stream.index(), 2)
        self.failUnlessEqual(stream.charPositionInLine, 2)
        self.failUnlessEqual(stream.line, 1)

        stream.consume() # o
        self.failUnlessEqual(stream.index(), 3)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 1)

        stream.consume() # \n
        self.failUnlessEqual(stream.index(), 4)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # b
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # a
        self.failUnlessEqual(stream.index(), 6)
        self.failUnlessEqual(stream.charPositionInLine, 2)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # r
        self.failUnlessEqual(stream.index(), 7)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # EOF
        self.failUnlessEqual(stream.index(), 7)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 2)

        stream.consume() # EOF
        self.failUnlessEqual(stream.index(), 7)
        self.failUnlessEqual(stream.charPositionInLine, 3)
        self.failUnlessEqual(stream.line, 2)


    def testReset(self):
        """StringStream.reset()"""

        stream = antlr3.StringStream('foo')

        stream.consume()
        stream.consume()

        stream.reset()
        self.failUnlessEqual(stream.index(), 0)
        self.failUnlessEqual(stream.line, 1)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.LT(1), 'f')


    def testLA(self):
        """StringStream.LA()"""

        stream = antlr3.StringStream('foo')

        self.failUnlessEqual(stream.LT(1), 'f')
        self.failUnlessEqual(stream.LT(2), 'o')
        self.failUnlessEqual(stream.LT(3), 'o')

        stream.consume()
        stream.consume()

        self.failUnlessEqual(stream.LT(1), 'o')
        self.failUnlessEqual(stream.LT(2), antlr3.EOF)
        self.failUnlessEqual(stream.LT(3), antlr3.EOF)


    def testSubstring(self):
        """StringStream.substring()"""

        stream = antlr3.StringStream('foobar')

        self.failUnlessEqual(stream.substring(0, 0), 'f')
        self.failUnlessEqual(stream.substring(0, 1), 'fo')
        self.failUnlessEqual(stream.substring(0, 5), 'foobar')
        self.failUnlessEqual(stream.substring(3, 5), 'bar')


    def testSeekForward(self):
        """StringStream.seek(): forward"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)

        self.failUnlessEqual(stream.index(), 4)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.LT(1), 'b')


##     # not yet implemented
##     def testSeekBackward(self):
##         """StringStream.seek(): backward"""

##         stream = antlr3.StringStream('foo\nbar')

##         stream.seek(4)
##         stream.seek(1)

##         self.failUnlessEqual(stream.index(), 1)
##         self.failUnlessEqual(stream.line, 1)
##         self.failUnlessEqual(stream.charPositionInLine, 1)
##         self.failUnlessEqual(stream.LA(1), 'o')


    def testMark(self):
        """StringStream.mark()"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)

        marker = stream.mark()
        self.failUnlessEqual(marker, 1)
        self.failUnlessEqual(stream.markDepth, 1)

        stream.consume()
        marker = stream.mark()
        self.failUnlessEqual(marker, 2)
        self.failUnlessEqual(stream.markDepth, 2)


    def testReleaseLast(self):
        """StringStream.release(): last marker"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)
        marker1 = stream.mark()

        stream.consume()
        marker2 = stream.mark()

        stream.release()
        self.failUnlessEqual(stream.markDepth, 1)

        # release same marker again, nothing has changed
        stream.release()
        self.failUnlessEqual(stream.markDepth, 1)


    def testReleaseNested(self):
        """StringStream.release(): nested"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)
        marker1 = stream.mark()

        stream.consume()
        marker2 = stream.mark()

        stream.consume()
        marker3 = stream.mark()

        stream.release(marker2)
        self.failUnlessEqual(stream.markDepth, 1)


    def testRewindLast(self):
        """StringStream.rewind(): last marker"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)

        marker = stream.mark()
        stream.consume()
        stream.consume()

        stream.rewind()
        self.failUnlessEqual(stream.markDepth, 0)
        self.failUnlessEqual(stream.index(), 4)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 0)
        self.failUnlessEqual(stream.LT(1), 'b')


    def testRewindNested(self):
        """StringStream.rewind(): nested"""

        stream = antlr3.StringStream('foo\nbar')

        stream.seek(4)
        marker1 = stream.mark()

        stream.consume()
        marker2 = stream.mark()

        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LT(1), 'a')


class TestFileStream(unittest.TestCase):
    """Test case for the FileStream class."""


    def testNoEncoding(self):
        path = os.path.join(os.path.dirname(__file__), 'teststreams.input1')

        stream = antlr3.FileStream(path)

        stream.seek(4)
        marker1 = stream.mark()

        stream.consume()
        marker2 = stream.mark()

        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LT(1), 'a')
        self.failUnlessEqual(stream.LA(1), ord('a'))


    def testEncoded(self):
        path = os.path.join(os.path.dirname(__file__), 'teststreams.input2')

        stream = antlr3.FileStream(path, 'utf-8')

        stream.seek(4)
        marker1 = stream.mark()

        stream.consume()
        marker2 = stream.mark()

        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LT(1), u'ä')
        self.failUnlessEqual(stream.LA(1), ord(u'ä'))



class TestInputStream(unittest.TestCase):
    """Test case for the InputStream class."""

    def testNoEncoding(self):
        file = StringIO('foo\nbar')

        stream = antlr3.InputStream(file)

        stream.seek(4)
        marker1 = stream.mark()

        stream.consume()
        marker2 = stream.mark()

        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LT(1), 'a')
        self.failUnlessEqual(stream.LA(1), ord('a'))


    def testEncoded(self):
        file = StringIO(u'foo\nbär'.encode('utf-8'))

        stream = antlr3.InputStream(file, 'utf-8')

        stream.seek(4)
        marker1 = stream.mark()

        stream.consume()
        marker2 = stream.mark()

        stream.consume()
        marker3 = stream.mark()

        stream.rewind(marker2)
        self.failUnlessEqual(stream.markDepth, 1)
        self.failUnlessEqual(stream.index(), 5)
        self.failUnlessEqual(stream.line, 2)
        self.failUnlessEqual(stream.charPositionInLine, 1)
        self.failUnlessEqual(stream.LT(1), u'ä')
        self.failUnlessEqual(stream.LA(1), ord(u'ä'))


class TestCommonTokenStream(unittest.TestCase):
    """Test case for the StringStream class."""

    def setUp(self):
        """Setup test fixure

        The constructor of CommonTokenStream needs a token source. This
        is a simple mock class providing just the nextToken() method.

        """

        class MockSource(object):
            def __init__(self):
                self.tokens = []

            def makeEOFToken(self):
                return antlr3.CommonToken(type=antlr3.EOF)

            def nextToken(self):
                try:
                    return self.tokens.pop(0)
                except IndexError:
                    return None

        self.source = MockSource()


    def testInit(self):
        """CommonTokenStream.__init__()"""

        stream = antlr3.CommonTokenStream(self.source)
        self.failUnlessEqual(stream.index(), -1)


    def testSetTokenSource(self):
        """CommonTokenStream.setTokenSource()"""

        stream = antlr3.CommonTokenStream(None)
        stream.setTokenSource(self.source)
        self.failUnlessEqual(stream.index(), -1)
        self.failUnlessEqual(stream.channel, antlr3.DEFAULT_CHANNEL)


    def testLTEmptySource(self):
        """CommonTokenStream.LT(): EOF (empty source)"""

        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(1)
        self.failUnlessEqual(lt1.type, antlr3.EOF)


    def testLT1(self):
        """CommonTokenStream.LT(1)"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(1)
        self.failUnlessEqual(lt1.type, 12)


    def testLT1WithHidden(self):
        """CommonTokenStream.LT(1): with hidden tokens"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(1)
        self.failUnlessEqual(lt1.type, 13)


    def testLT2BeyondEnd(self):
        """CommonTokenStream.LT(2): beyond end"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL)
            )

        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(2)
        self.failUnlessEqual(lt1.type, antlr3.EOF)


    # not yet implemented
    def testLTNegative(self):
        """CommonTokenStream.LT(-1): look back"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()
        stream.consume()

        lt1 = stream.LT(-1)
        self.failUnlessEqual(lt1.type, 12)


    def testLB1(self):
        """CommonTokenStream.LB(1)"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()
        stream.consume()

        self.failUnlessEqual(stream.LB(1).type, 12)


    def testLTZero(self):
        """CommonTokenStream.LT(0)"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        stream = antlr3.CommonTokenStream(self.source)

        lt1 = stream.LT(0)
        self.failUnless(lt1 is None)


    def testLBBeyondBegin(self):
        """CommonTokenStream.LB(-1): beyond begin"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        stream = antlr3.CommonTokenStream(self.source)
        self.failUnless(stream.LB(1) is None)

        stream.consume()
        stream.consume()
        self.failUnless(stream.LB(3) is None)


    def testFillBuffer(self):
        """CommonTokenStream.fillBuffer()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=14)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )

        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()

        self.failUnlessEqual(len(stream.tokens), 3)
        self.failUnlessEqual(stream.tokens[0].type, 12)
        self.failUnlessEqual(stream.tokens[1].type, 13)
        self.failUnlessEqual(stream.tokens[2].type, 14)


    def testConsume(self):
        """CommonTokenStream.consume()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )

        stream = antlr3.CommonTokenStream(self.source)
        self.failUnlessEqual(stream.LA(1), 12)

        stream.consume()
        self.failUnlessEqual(stream.LA(1), 13)

        stream.consume()
        self.failUnlessEqual(stream.LA(1), antlr3.EOF)

        stream.consume()
        self.failUnlessEqual(stream.LA(1), antlr3.EOF)


    def testSeek(self):
        """CommonTokenStream.seek()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )

        stream = antlr3.CommonTokenStream(self.source)
        self.failUnlessEqual(stream.LA(1), 12)

        stream.seek(2)
        self.failUnlessEqual(stream.LA(1), antlr3.EOF)

        stream.seek(0)
        self.failUnlessEqual(stream.LA(1), 12)


    def testMarkRewind(self):
        """CommonTokenStream.mark()/rewind()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13)
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=antlr3.EOF)
            )

        stream = antlr3.CommonTokenStream(self.source)
        stream.fillBuffer()

        stream.consume()
        marker = stream.mark()

        stream.consume()
        stream.rewind(marker)

        self.failUnlessEqual(stream.LA(1), 13)


    def testToString(self):
        """CommonTokenStream.toString()"""

        self.source.tokens.append(
            antlr3.CommonToken(type=12, text="foo")
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=13, text="bar")
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=14, text="gnurz")
            )

        self.source.tokens.append(
            antlr3.CommonToken(type=15, text="blarz")
            )

        stream = antlr3.CommonTokenStream(self.source)

        assert stream.toString() == "foobargnurzblarz"
        assert stream.toString(1, 2) == "bargnurz"
        assert stream.toString(stream.tokens[1], stream.tokens[-2]) == "bargnurz"


if __name__ == "__main__":
    unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))