test_tool.py revision 1320f92c476a1ad9d19dba2a48c72b75566198e9
1#!/usr/bin/env python 2# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Test harness for chromium clang tools.""" 7 8import difflib 9import glob 10import json 11import os 12import os.path 13import subprocess 14import shutil 15import sys 16 17 18def _GenerateCompileCommands(files, include_paths): 19 """Returns a JSON string containing a compilation database for the input.""" 20 include_path_flags = ' '.join('-I %s' % include_path 21 for include_path in include_paths) 22 return json.dumps([{'directory': '.', 23 'command': 'clang++ -fsyntax-only %s -c %s' % ( 24 include_path_flags, f), 25 'file': f} for f in files], indent=2) 26 27 28def _NumberOfTestsToString(tests): 29 """Returns an English describing the number of tests.""" 30 return "%d test%s" % (tests, 's' if tests != 1 else '') 31 32 33def main(argv): 34 if len(argv) < 1: 35 print 'Usage: test_tool.py <clang tool>' 36 print ' <clang tool> is the clang tool to be tested.' 37 sys.exit(1) 38 39 tool_to_test = argv[0] 40 tools_clang_scripts_directory = os.path.dirname(os.path.realpath(__file__)) 41 tools_clang_directory = os.path.dirname(tools_clang_scripts_directory) 42 test_directory_for_tool = os.path.join( 43 tools_clang_directory, tool_to_test, 'tests') 44 compile_database = os.path.join(test_directory_for_tool, 45 'compile_commands.json') 46 source_files = glob.glob(os.path.join(test_directory_for_tool, 47 '*-original.cc')) 48 actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc']) 49 for source_file in source_files] 50 expected_files = ['-'.join([source_file.rsplit('-', 1)[0], 'expected.cc']) 51 for source_file in source_files] 52 include_paths = [] 53 include_paths.append( 54 os.path.realpath(os.path.join(tools_clang_directory, '../..'))) 55 # Many gtest headers expect to have testing/gtest/include in the include 56 # search path. 57 include_paths.append( 58 os.path.realpath(os.path.join(tools_clang_directory, 59 '../..', 60 'testing/gtest/include'))) 61 62 try: 63 # Set up the test environment. 64 for source, actual in zip(source_files, actual_files): 65 shutil.copyfile(source, actual) 66 # Stage the test files in the git index. If they aren't staged, then 67 # run_tools.py will skip them when applying replacements. 68 args = ['git', 'add'] 69 args.extend(actual_files) 70 subprocess.check_call(args) 71 # Generate a temporary compilation database to run the tool over. 72 with open(compile_database, 'w') as f: 73 f.write(_GenerateCompileCommands(actual_files, include_paths)) 74 75 args = ['python', 76 os.path.join(tools_clang_scripts_directory, 'run_tool.py'), 77 tool_to_test, 78 test_directory_for_tool] 79 args.extend(actual_files) 80 run_tool = subprocess.Popen(args, stdout=subprocess.PIPE) 81 stdout, _ = run_tool.communicate() 82 if run_tool.returncode != 0: 83 print 'run_tool failed:\n%s' % stdout 84 sys.exit(1) 85 86 passed = 0 87 failed = 0 88 for expected, actual in zip(expected_files, actual_files): 89 print '[ RUN ] %s' % os.path.relpath(actual) 90 expected_output = actual_output = None 91 with open(expected, 'r') as f: 92 expected_output = f.readlines() 93 with open(actual, 'r') as f: 94 actual_output = f.readlines() 95 if actual_output != expected_output: 96 failed += 1 97 for line in difflib.unified_diff(expected_output, actual_output, 98 fromfile=os.path.relpath(expected), 99 tofile=os.path.relpath(actual)): 100 sys.stdout.write(line) 101 print '[ FAILED ] %s' % os.path.relpath(actual) 102 # Don't clean up the file on failure, so the results can be referenced 103 # more easily. 104 continue 105 print '[ OK ] %s' % os.path.relpath(actual) 106 passed += 1 107 os.remove(actual) 108 109 if failed == 0: 110 os.remove(compile_database) 111 112 print '[==========] %s ran.' % _NumberOfTestsToString(len(source_files)) 113 if passed > 0: 114 print '[ PASSED ] %s.' % _NumberOfTestsToString(passed) 115 if failed > 0: 116 print '[ FAILED ] %s.' % _NumberOfTestsToString(failed) 117 finally: 118 # No matter what, unstage the git changes we made earlier to avoid polluting 119 # the index. 120 args = ['git', 'reset', '--quiet', 'HEAD'] 121 args.extend(actual_files) 122 subprocess.call(args) 123 124 125if __name__ == '__main__': 126 sys.exit(main(sys.argv[1:])) 127