testbase.py revision 324c4644fee44b9898524c09511bd33c3f12e2df 
1import unittest
2import imp
3import os
4import errno
5import sys
6import glob
7import re
8import tempfile
9import shutil
10import inspect
11import hashlib
12from distutils.errors import *
13import antlr3
14
15def unlink(path):
16    try:
17        os.unlink(path)
18    except OSError, exc:
19        if exc.errno != errno.ENOENT:
20            raise
21
22
23class GrammarCompileError(Exception):
24  """Grammar failed to compile."""
25  pass
26
27
28# At least on MacOSX tempdir (/tmp) is a symlink. It's sometimes dereferences,
29# sometimes not, breaking the inspect.getmodule() function.
30testbasedir = os.path.join(
31    os.path.realpath(tempfile.gettempdir()),
32    'antlr3-test')
33
34
35class BrokenTest(unittest.TestCase.failureException):
36    def __repr__(self):
37        name, reason = self.args
38        return '%s: %s: %s works now' % (
39            (self.__class__.__name__, name, reason))
40
41
42def broken(reason, *exceptions):
43    '''Indicates a failing (or erroneous) test case fails that should succeed.
44    If the test fails with an exception, list the exception type in args'''
45    def wrapper(test_method):
46        def replacement(*args, **kwargs):
47            try:
48                test_method(*args, **kwargs)
49            except exceptions or unittest.TestCase.failureException:
50                pass
51            else:
52                raise BrokenTest(test_method.__name__, reason)
53        replacement.__doc__ = test_method.__doc__
54        replacement.__name__ = 'XXX_' + test_method.__name__
55        replacement.todo = reason
56        return replacement
57    return wrapper
58
59
60dependencyCache = {}
61compileErrorCache = {}
62
63# setup java CLASSPATH
64if 'CLASSPATH' not in os.environ:
65    cp = []
66
67    baseDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
68    libDir = os.path.join(baseDir, 'lib')
69
70    jar = os.path.join(libDir, 'ST-4.0.1.jar')
71    if not os.path.isfile(jar):
72        raise DistutilsFileError(
73            "Missing file '%s'. Grap it from a distribution package."
74            % jar,
75            )
76    cp.append(jar)
77
78    jar = os.path.join(libDir, 'antlr-2.7.7.jar')
79    if not os.path.isfile(jar):
80        raise DistutilsFileError(
81            "Missing file '%s'. Grap it from a distribution package."
82            % jar,
83            )
84    cp.append(jar)
85
86    jar = os.path.join(libDir, 'junit-4.2.jar')
87    if not os.path.isfile(jar):
88        raise DistutilsFileError(
89            "Missing file '%s'. Grap it from a distribution package."
90            % jar,
91            )
92    cp.append(jar)
93
94    cp.append(os.path.join(baseDir, 'runtime', 'Python', 'build'))
95
96    classpath = '-cp "' + ':'.join([os.path.abspath(p) for p in cp]) + '"'
97
98else:
99    classpath = ''
100
101
102class ANTLRTest(unittest.TestCase):
103    def __init__(self, *args, **kwargs):
104        unittest.TestCase.__init__(self, *args, **kwargs)
105
106        self.moduleName = os.path.splitext(os.path.basename(sys.modules[self.__module__].__file__))[0]
107        self.className = self.__class__.__name__
108        self._baseDir = None
109
110        self.lexerModule = None
111        self.parserModule = None
112
113        self.grammarName = None
114        self.grammarType = None
115
116
117    def assertListEqual(self, a, b):
118        if a == b:
119            return
120
121        import difflib
122        a = [str(l) + '\n' for l in a]
123        b = [str(l) + '\n' for l in b]
124
125        raise AssertionError(''.join(difflib.unified_diff(a, b)))
126
127
128    @property
129    def baseDir(self):
130        if self._baseDir is None:
131            testName = 'unknownTest'
132            for frame in inspect.stack():
133                code = frame[0].f_code
134                codeMod = inspect.getmodule(code)
135                if codeMod is None:
136                    continue
137
138                # skip frames not in requested module
139                if codeMod is not sys.modules[self.__module__]:
140                    continue
141
142                # skip some unwanted names
143                if code.co_name in ('nextToken', '<module>'):
144                    continue
145
146                if code.co_name.startswith('test'):
147                    testName = code.co_name
148                    break
149
150            self._baseDir = os.path.join(
151                testbasedir,
152                self.moduleName, self.className, testName)
153            if not os.path.isdir(self._baseDir):
154                os.makedirs(self._baseDir)
155
156        return self._baseDir
157
158
159    def _invokeantlr(self, dir, file, options, javaOptions=''):
160        cmd = 'cd %s; java %s %s org.antlr.Tool -o . %s %s 2>&1' % (
161            dir, javaOptions, classpath, options, file
162            )
163        fp = os.popen(cmd)
164        output = ''
165        failed = False
166        for line in fp:
167            output += line
168
169            if line.startswith('error('):
170                failed = True
171
172        rc = fp.close()
173        if rc is not None:
174            failed = True
175
176        if failed:
177            raise GrammarCompileError(
178                "Failed to compile grammar '%s':\n%s\n\n" % (file, cmd)
179                + output
180                )
181
182
183    def compileGrammar(self, grammarName=None, options='', javaOptions=''):
184        if grammarName is None:
185            grammarName = self.moduleName + '.g'
186
187        self._baseDir = os.path.join(
188            testbasedir,
189            self.moduleName)
190        if not os.path.isdir(self._baseDir):
191            os.makedirs(self._baseDir)
192
193        if self.grammarName is None:
194            self.grammarName = os.path.splitext(grammarName)[0]
195
196        grammarPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), grammarName)
197
198        # get type and name from first grammar line
199        grammar = open(grammarPath, 'r').read()
200        m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE)
201        assert m is not None, grammar
202        self.grammarType = m.group(2)
203        if self.grammarType is None:
204            self.grammarType = 'combined'
205
206        if self.grammarType is None:
207            assert self.grammarType in ('lexer', 'parser', 'tree', 'combined'), self.grammarType
208
209        # don't try to rebuild grammar, if it already failed
210        if grammarName in compileErrorCache:
211            return
212
213        try:
214        #     # get dependencies from antlr
215        #     if grammarName in dependencyCache:
216        #         dependencies = dependencyCache[grammarName]
217
218        #     else:
219        #         dependencies = []
220        #         cmd = ('cd %s; java %s %s org.antlr.Tool -o . -depend %s 2>&1'
221        #                % (self.baseDir, javaOptions, classpath, grammarPath))
222
223        #         output = ""
224        #         failed = False
225
226        #         fp = os.popen(cmd)
227        #         for line in fp:
228        #             output += line
229
230        #             if line.startswith('error('):
231        #                 failed = True
232        #             elif ':' in line:
233        #                 a, b = line.strip().split(':', 1)
234        #                 dependencies.append(
235        #                     (os.path.join(self.baseDir, a.strip()),
236        #                      [os.path.join(self.baseDir, b.strip())])
237        #                     )
238
239        #         rc = fp.close()
240        #         if rc is not None:
241        #             failed = True
242
243        #         if failed:
244        #             raise GrammarCompileError(
245        #                 "antlr -depend failed with code %s on grammar '%s':\n\n"
246        #                 % (rc, grammarName)
247        #                 + cmd
248        #                 + "\n"
249        #                 + output
250        #                 )
251
252        #         # add dependencies to my .stg files
253        #         templateDir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tool', 'src', 'main', 'resources', 'org', 'antlr', 'codegen', 'templates', 'Python'))
254        #         templates = glob.glob(os.path.join(templateDir, '*.stg'))
255
256        #         for dst, src in dependencies:
257        #             src.extend(templates)
258
259        #         dependencyCache[grammarName] = dependencies
260
261        #     rebuild = False
262        #     for dest, sources in dependencies:
263        #         if not os.path.isfile(dest):
264        #             rebuild = True
265        #             break
266
267        #         for source in sources:
268        #             if os.path.getmtime(source) > os.path.getmtime(dest):
269        #                 rebuild = True
270        #                 break
271
272
273        #     if rebuild:
274        #         self._invokeantlr(self.baseDir, grammarPath, options, javaOptions)
275
276            self._invokeantlr(self.baseDir, grammarPath, options, javaOptions)
277
278        except:
279            # mark grammar as broken
280            compileErrorCache[grammarName] = True
281            raise
282
283
284    def lexerClass(self, base):
285        """Optionally build a subclass of generated lexer class"""
286
287        return base
288
289
290    def parserClass(self, base):
291        """Optionally build a subclass of generated parser class"""
292
293        return base
294
295
296    def walkerClass(self, base):
297        """Optionally build a subclass of generated walker class"""
298
299        return base
300
301
302    def __load_module(self, name):
303        modFile, modPathname, modDescription \
304                 = imp.find_module(name, [self.baseDir])
305
306        return imp.load_module(
307            name, modFile, modPathname, modDescription
308            )
309
310
311    def getLexer(self, *args, **kwargs):
312        """Build lexer instance. Arguments are passed to lexer.__init__()."""
313
314        if self.grammarType == 'lexer':
315            self.lexerModule = self.__load_module(self.grammarName)
316            cls = getattr(self.lexerModule, self.grammarName)
317        else:
318            self.lexerModule = self.__load_module(self.grammarName + 'Lexer')
319            cls = getattr(self.lexerModule, self.grammarName + 'Lexer')
320
321        cls = self.lexerClass(cls)
322
323        lexer = cls(*args, **kwargs)
324
325        return lexer
326
327
328    def getParser(self, *args, **kwargs):
329        """Build parser instance. Arguments are passed to parser.__init__()."""
330
331        if self.grammarType == 'parser':
332            self.lexerModule = self.__load_module(self.grammarName)
333            cls = getattr(self.lexerModule, self.grammarName)
334        else:
335            self.parserModule = self.__load_module(self.grammarName + 'Parser')
336            cls = getattr(self.parserModule, self.grammarName + 'Parser')
337        cls = self.parserClass(cls)
338
339        parser = cls(*args, **kwargs)
340
341        return parser
342
343
344    def getWalker(self, *args, **kwargs):
345        """Build walker instance. Arguments are passed to walker.__init__()."""
346
347        self.walkerModule = self.__load_module(self.grammarName + 'Walker')
348        cls = getattr(self.walkerModule, self.grammarName + 'Walker')
349        cls = self.walkerClass(cls)
350
351        walker = cls(*args, **kwargs)
352
353        return walker
354
355
356    def writeInlineGrammar(self, grammar):
357        # Create a unique ID for this test and use it as the grammar name,
358        # to avoid class name reuse. This kinda sucks. Need to find a way so
359        # tests can use the same grammar name without messing up the namespace.
360        # Well, first I should figure out what the exact problem is...
361        id = hashlib.md5(self.baseDir).hexdigest()[-8:]
362        grammar = grammar.replace('$TP', 'TP' + id)
363        grammar = grammar.replace('$T', 'T' + id)
364
365        # get type and name from first grammar line
366        m = re.match(r'\s*((lexer|parser|tree)\s+|)grammar\s+(\S+);', grammar, re.MULTILINE)
367        assert m is not None, grammar
368        grammarType = m.group(2)
369        if grammarType is None:
370            grammarType = 'combined'
371        grammarName = m.group(3)
372
373        assert grammarType in ('lexer', 'parser', 'tree', 'combined'), grammarType
374
375        grammarPath = os.path.join(self.baseDir, grammarName + '.g')
376
377        # dump temp grammar file
378        fp = open(grammarPath, 'w')
379        fp.write(grammar)
380        fp.close()
381
382        return grammarName, grammarPath, grammarType
383
384
385    def writeFile(self, name, contents):
386        testDir = os.path.dirname(os.path.abspath(__file__))
387        path = os.path.join(self.baseDir, name)
388
389        fp = open(path, 'w')
390        fp.write(contents)
391        fp.close()
392
393        return path
394
395
396    def compileInlineGrammar(self, grammar, options='', javaOptions='',
397                             returnModule=False):
398        # write grammar file
399        grammarName, grammarPath, grammarType = self.writeInlineGrammar(grammar)
400
401        # compile it
402        self._invokeantlr(
403            os.path.dirname(grammarPath),
404            os.path.basename(grammarPath),
405            options,
406            javaOptions
407            )
408
409        if grammarType == 'combined':
410            lexerMod = self.__load_module(grammarName + 'Lexer')
411            parserMod = self.__load_module(grammarName + 'Parser')
412            if returnModule:
413                return lexerMod, parserMod
414
415            lexerCls = getattr(lexerMod, grammarName + 'Lexer')
416            lexerCls = self.lexerClass(lexerCls)
417            parserCls = getattr(parserMod, grammarName + 'Parser')
418            parserCls = self.parserClass(parserCls)
419
420            return lexerCls, parserCls
421
422        if grammarType == 'lexer':
423            lexerMod = self.__load_module(grammarName)
424            if returnModule:
425                return lexerMod
426
427            lexerCls = getattr(lexerMod, grammarName)
428            lexerCls = self.lexerClass(lexerCls)
429
430            return lexerCls
431
432        if grammarType == 'parser':
433            parserMod = self.__load_module(grammarName)
434            if returnModule:
435                return parserMod
436
437            parserCls = getattr(parserMod, grammarName)
438            parserCls = self.parserClass(parserCls)
439
440            return parserCls
441
442        if grammarType == 'tree':
443            walkerMod = self.__load_module(grammarName)
444            if returnModule:
445                return walkerMod
446
447            walkerCls = getattr(walkerMod, grammarName)
448            walkerCls = self.walkerClass(walkerCls)
449
450            return walkerCls
451