1# Copyright (C) 2011 Google Inc.  All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions
5# are met:
6# 1. Redistributions of source code must retain the above copyright
7#    notice, this list of conditions and the following disclaimer.
8# 2. Redistributions in binary form must reproduce the above copyright
9#    notice, this list of conditions and the following disclaimer in the
10#    documentation and/or other materials provided with the distribution.
11#
12# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
13# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
14# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
16# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
17# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
18# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
19# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
20# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23#
24
25from contextlib import contextmanager
26import filecmp
27import fnmatch
28import os
29import re
30import shutil
31import sys
32import tempfile
33
34from webkitpy.common.system.executive import Executive
35
36# Source/ path is needed both to find input IDL files, and to import other
37# Python modules.
38module_path = os.path.dirname(__file__)
39source_path = os.path.normpath(os.path.join(module_path, os.pardir, os.pardir,
40                                            os.pardir, os.pardir, 'Source'))
41sys.path.append(source_path)  # for Source/bindings imports
42
43import bindings.scripts.compute_interfaces_info_individual
44from bindings.scripts.compute_interfaces_info_individual import compute_info_individual, info_individual
45import bindings.scripts.compute_interfaces_info_overall
46from bindings.scripts.compute_interfaces_info_overall import compute_interfaces_info_overall, interfaces_info
47from bindings.scripts.idl_compiler import IdlCompilerDictionaryImpl, IdlCompilerV8
48
49
50PASS_MESSAGE = 'All tests PASS!'
51FAIL_MESSAGE = """Some tests FAIL!
52To update the reference files, execute:
53    run-bindings-tests --reset-results
54
55If the failures are not due to your changes, test results may be out of sync;
56please rebaseline them in a separate CL, after checking that tests fail in ToT.
57In CL, please set:
58NOTRY=true
59TBR=(someone in Source/bindings/OWNERS or WATCHLISTS:bindings)
60"""
61
62DEPENDENCY_IDL_FILES = frozenset([
63    'TestImplements.idl',
64    'TestImplements2.idl',
65    'TestImplements3.idl',
66    'TestPartialInterface.idl',
67    'TestPartialInterface2.idl',
68    'TestPartialInterface3.idl',
69])
70
71COMPONENT_DIRECTORY = frozenset(['core', 'modules'])
72
73test_input_directory = os.path.join(source_path, 'bindings', 'tests', 'idls')
74reference_directory = os.path.join(source_path, 'bindings', 'tests', 'results')
75
76PLY_LEX_YACC_FILES = frozenset([
77    'lextab.py',  # PLY lex
78    'lextab.pyc',
79    'parsetab.pickle',  # PLY yacc
80])
81
82@contextmanager
83def TemporaryDirectory():
84    """Wrapper for tempfile.mkdtemp() so it's usable with 'with' statement.
85
86    Simple backport of tempfile.TemporaryDirectory from Python 3.2.
87    """
88    name = tempfile.mkdtemp()
89    try:
90        yield name
91    finally:
92        shutil.rmtree(name)
93
94
95def generate_interface_dependencies():
96    def idl_paths_recursive(directory):
97        # This is slow, especially on Windows, due to os.walk making
98        # excess stat() calls. Faster versions may appear in Python 3.5 or
99        # later:
100        # https://github.com/benhoyt/scandir
101        # http://bugs.python.org/issue11406
102        idl_paths = []
103        for dirpath, _, files in os.walk(directory):
104            idl_paths.extend(os.path.join(dirpath, filename)
105                             for filename in fnmatch.filter(files, '*.idl'))
106        return idl_paths
107
108    # We compute interfaces info for *all* IDL files, not just test IDL
109    # files, as code generator output depends on inheritance (both ancestor
110    # chain and inherited extended attributes), and some real interfaces
111    # are special-cased, such as Node.
112    #
113    # For example, when testing the behavior of interfaces that inherit
114    # from Node, we also need to know that these inherit from EventTarget,
115    # since this is also special-cased and Node inherits from EventTarget,
116    # but this inheritance information requires computing dependencies for
117    # the real Node.idl file.
118
119    # 2-stage computation: individual, then overall
120    #
121    # Properly should compute separately by component (currently test
122    # includes are invalid), but that's brittle (would need to update this file
123    # for each new component) and doesn't test the code generator any better
124    # than using a single component.
125    for idl_filename in idl_paths_recursive(source_path):
126        compute_info_individual(idl_filename)
127    info_individuals = [info_individual()]
128    # TestDictionary.{h,cpp} are placed under Source/bindings/tests/idls/core.
129    # However, IdlCompiler generates TestDictionary.{h,cpp} by using relative_dir.
130    # So the files will be generated under output_dir/core/bindings/tests/idls/core.
131    # To avoid this issue, we need to clear relative_dir here.
132    for info in info_individuals:
133        for value in info['interfaces_info'].itervalues():
134            value['relative_dir'] = ''
135    compute_interfaces_info_overall(info_individuals)
136
137
138def bindings_tests(output_directory, verbose):
139    executive = Executive()
140
141    def list_files(directory):
142        files = []
143        for component in os.listdir(directory):
144            if component not in COMPONENT_DIRECTORY:
145                continue
146            directory_with_component = os.path.join(directory, component)
147            for filename in os.listdir(directory_with_component):
148                files.append(os.path.join(directory_with_component, filename))
149        return files
150
151    def diff(filename1, filename2):
152        # Python's difflib module is too slow, especially on long output, so
153        # run external diff(1) command
154        cmd = ['diff',
155               '-u',  # unified format
156               '-N',  # treat absent files as empty
157               filename1,
158               filename2]
159        # Return output and don't raise exception, even though diff(1) has
160        # non-zero exit if files differ.
161        return executive.run_command(cmd, error_handler=lambda x: None)
162
163    def is_cache_file(filename):
164        if filename in PLY_LEX_YACC_FILES:
165            return True
166        if filename.endswith('.cache'):  # Jinja
167            return True
168        return False
169
170    def delete_cache_files():
171        # FIXME: Instead of deleting cache files, don't generate them.
172        cache_files = [path for path in list_files(output_directory)
173                       if is_cache_file(os.path.basename(path))]
174        for cache_file in cache_files:
175            os.remove(cache_file)
176
177    def identical_file(reference_filename, output_filename):
178        reference_basename = os.path.basename(reference_filename)
179
180        if not os.path.isfile(reference_filename):
181            print 'Missing reference file!'
182            print '(if adding new test, update reference files)'
183            print reference_basename
184            print
185            return False
186
187        if not filecmp.cmp(reference_filename, output_filename):
188            # cmp is much faster than diff, and usual case is "no differance",
189            # so only run diff if cmp detects a difference
190            print 'FAIL: %s' % reference_basename
191            print diff(reference_filename, output_filename)
192            return False
193
194        if verbose:
195            print 'PASS: %s' % reference_basename
196        return True
197
198    def identical_output_files(output_files):
199        reference_files = [os.path.join(reference_directory,
200                                        os.path.relpath(path, output_directory))
201                           for path in output_files]
202        return all([identical_file(reference_filename, output_filename)
203                    for (reference_filename, output_filename) in zip(reference_files, output_files)])
204
205    def no_excess_files(output_files):
206        generated_files = set([os.path.relpath(path, output_directory)
207                               for path in output_files])
208        # Add subversion working copy directories in core and modules.
209        for component in COMPONENT_DIRECTORY:
210            generated_files.add(os.path.join(component, '.svn'))
211
212        excess_files = []
213        for path in list_files(reference_directory):
214            relpath = os.path.relpath(path, reference_directory)
215            if relpath not in generated_files:
216                excess_files.append(relpath)
217        if excess_files:
218            print ('Excess reference files! '
219                  '(probably cruft from renaming or deleting):\n' +
220                  '\n'.join(excess_files))
221            return False
222        return True
223
224    try:
225        generate_interface_dependencies()
226        for component in COMPONENT_DIRECTORY:
227            output_dir = os.path.join(output_directory, component)
228            if not os.path.exists(output_dir):
229                os.makedirs(output_dir)
230
231            idl_compiler = IdlCompilerV8(output_dir,
232                                         interfaces_info=interfaces_info,
233                                         only_if_changed=True)
234            dictionary_impl_compiler = IdlCompilerDictionaryImpl(
235                output_dir, interfaces_info=interfaces_info,
236                only_if_changed=True)
237
238            idl_filenames = []
239            input_directory = os.path.join(test_input_directory, component)
240            for filename in os.listdir(input_directory):
241                if (filename.endswith('.idl') and
242                    # Dependencies aren't built
243                    # (they are used by the dependent)
244                    filename not in DEPENDENCY_IDL_FILES):
245                    idl_filenames.append(
246                        os.path.realpath(
247                            os.path.join(input_directory, filename)))
248            for idl_path in idl_filenames:
249                idl_basename = os.path.basename(idl_path)
250                idl_compiler.compile_file(idl_path)
251                definition_name, _ = os.path.splitext(idl_basename)
252                if (definition_name in interfaces_info and interfaces_info[definition_name]['is_dictionary']):
253                    dictionary_impl_compiler.compile_file(idl_path)
254                if verbose:
255                    print 'Compiled: %s' % idl_path
256    finally:
257        delete_cache_files()
258
259    # Detect all changes
260    output_files = list_files(output_directory)
261    passed = identical_output_files(output_files)
262    passed &= no_excess_files(output_files)
263
264    if passed:
265        if verbose:
266            print
267            print PASS_MESSAGE
268        return 0
269    print
270    print FAIL_MESSAGE
271    return 1
272
273
274def run_bindings_tests(reset_results, verbose):
275    # Generate output into the reference directory if resetting results, or
276    # a temp directory if not.
277    if reset_results:
278        print 'Resetting results'
279        return bindings_tests(reference_directory, verbose)
280    with TemporaryDirectory() as temp_dir:
281        return bindings_tests(temp_dir, verbose)
282