1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Test harness for chromium clang tools."""
7
8import difflib
9import glob
10import json
11import os
12import os.path
13import shutil
14import subprocess
15import sys
16
17
18def _RunGit(args):
19  if sys.platform == 'win32':
20    args = ['git.bat'] + args
21  else:
22    args = ['git'] + args
23  subprocess.check_call(args)
24
25
26def _GenerateCompileCommands(files, include_paths):
27  """Returns a JSON string containing a compilation database for the input."""
28  # Note: in theory, backslashes in the compile DB should work but the tools
29  # that write compile DBs and the tools that read them don't agree on the
30  # escaping convention: https://llvm.org/bugs/show_bug.cgi?id=19687
31  files = [f.replace('\\', '/') for f in files]
32  include_path_flags = ' '.join('-I %s' % include_path.replace('\\', '/')
33                                for include_path in include_paths)
34  return json.dumps([{'directory': '.',
35                      'command': 'clang++ -std=c++11 -fsyntax-only %s -c %s' % (
36                          include_path_flags, f),
37                      'file': f} for f in files], indent=2)
38
39
40def _NumberOfTestsToString(tests):
41  """Returns an English describing the number of tests."""
42  return '%d test%s' % (tests, 's' if tests != 1 else '')
43
44
45def main(argv):
46  if len(argv) < 1:
47    print 'Usage: test_tool.py <clang tool>'
48    print '  <clang tool> is the clang tool to be tested.'
49    sys.exit(1)
50
51  tool_to_test = argv[0]
52  tools_clang_scripts_directory = os.path.dirname(os.path.realpath(__file__))
53  tools_clang_directory = os.path.dirname(tools_clang_scripts_directory)
54  test_directory_for_tool = os.path.join(
55      tools_clang_directory, tool_to_test, 'tests')
56  compile_database = os.path.join(test_directory_for_tool,
57                                  'compile_commands.json')
58  source_files = glob.glob(os.path.join(test_directory_for_tool,
59                                        '*-original.cc'))
60  actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc'])
61                  for source_file in source_files]
62  expected_files = ['-'.join([source_file.rsplit('-', 1)[0], 'expected.cc'])
63                    for source_file in source_files]
64  include_paths = []
65  include_paths.append(
66      os.path.realpath(os.path.join(tools_clang_directory, '../..')))
67  # Many gtest headers expect to have testing/gtest/include in the include
68  # search path.
69  include_paths.append(
70      os.path.realpath(os.path.join(tools_clang_directory,
71                                    '../..',
72                                    'testing/gtest/include')))
73
74  try:
75    # Set up the test environment.
76    for source, actual in zip(source_files, actual_files):
77      shutil.copyfile(source, actual)
78    # Stage the test files in the git index. If they aren't staged, then
79    # run_tools.py will skip them when applying replacements.
80    args = ['add']
81    args.extend(actual_files)
82    _RunGit(args)
83    # Generate a temporary compilation database to run the tool over.
84    with open(compile_database, 'w') as f:
85      f.write(_GenerateCompileCommands(actual_files, include_paths))
86
87    args = ['python',
88            os.path.join(tools_clang_scripts_directory, 'run_tool.py'),
89            tool_to_test,
90            test_directory_for_tool]
91    args.extend(actual_files)
92    run_tool = subprocess.Popen(args, stdout=subprocess.PIPE)
93    stdout, _ = run_tool.communicate()
94    if run_tool.returncode != 0:
95      print 'run_tool failed:\n%s' % stdout
96      sys.exit(1)
97
98    args = ['cl', 'format']
99    args.extend(actual_files)
100    _RunGit(args)
101
102    passed = 0
103    failed = 0
104    for expected, actual in zip(expected_files, actual_files):
105      print '[ RUN      ] %s' % os.path.relpath(actual)
106      expected_output = actual_output = None
107      with open(expected, 'r') as f:
108        expected_output = f.readlines()
109      with open(actual, 'r') as f:
110        actual_output = f.readlines()
111      if actual_output != expected_output:
112        failed += 1
113        for line in difflib.unified_diff(expected_output, actual_output,
114                                         fromfile=os.path.relpath(expected),
115                                         tofile=os.path.relpath(actual)):
116          sys.stdout.write(line)
117        print '[  FAILED  ] %s' % os.path.relpath(actual)
118        # Don't clean up the file on failure, so the results can be referenced
119        # more easily.
120        continue
121      print '[       OK ] %s' % os.path.relpath(actual)
122      passed += 1
123      os.remove(actual)
124
125    if failed == 0:
126      os.remove(compile_database)
127
128    print '[==========] %s ran.' % _NumberOfTestsToString(len(source_files))
129    if passed > 0:
130      print '[  PASSED  ] %s.' % _NumberOfTestsToString(passed)
131    if failed > 0:
132      print '[  FAILED  ] %s.' % _NumberOfTestsToString(failed)
133  finally:
134    # No matter what, unstage the git changes we made earlier to avoid polluting
135    # the index.
136    args = ['reset', '--quiet', 'HEAD']
137    args.extend(actual_files)
138    _RunGit(args)
139
140
141if __name__ == '__main__':
142  sys.exit(main(sys.argv[1:]))
143