SATestBuild.py revision 1b41716abf6856af893fd2f13cd0272dffa0e312
11b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks#!/usr/bin/env python 21b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 31b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks""" 41b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksStatic Analyzer qualification infrastructure. 51b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 61b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksThe goal is to test the analyzer against different projects, check for failures, 71b41716abf6856af893fd2f13cd0272dffa0e312Anna Zakscompare results, and measure performance. 81b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 91b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksRepository Directory will contain sources of the projects as well as the 101b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksinformation on how to build them and the expected output. 111b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksRepository Directory structure: 121b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - ProjectMap file 131b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - Historical Performance Data 141b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - Project Dir1 151b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - ReferenceOutput 161b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - Project Dir2 171b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - ReferenceOutput 181b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks .. 191b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 201b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksTo test the build of the analyzer one would: 211b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that 221b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks the build directory does not pollute the repository to min network traffic). 231b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - Build all projects, until error. Produce logs to report errors. 241b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks - Compare results. 251b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 261b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksThe files which should be kept around for failure investigations: 271b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RepositoryCopy/Project DirI/ScanBuildResults 281b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RepositoryCopy/Project DirI/run_static_analyzer.log 291b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 301b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksAssumptions (TODO: shouldn't need to assume these.): 311b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks The script is being run from the Repository Directory. 321b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks The compiler for scan-build is in the PATH. 331b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH 341b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 351b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksFor more logging, set the env variables: 361b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks zaks:TI zaks$ export CCC_ANALYZER_LOG=1 371b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1 381b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks""" 391b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport CmpRuns 401b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 411b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport os 421b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport csv 431b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport sys 441b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport glob 451b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport shutil 461b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport time 471b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksimport plistlib 481b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksfrom subprocess import check_call 491b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 501b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Project map stores info about all the "registered" projects. 511b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksProjectMapFile = "projectMap.csv" 521b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 531b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Names of the project specific scripts. 541b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# The script that needs to be executed before the build can start. 551b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksPreprocessScript = "pre_run_static_analyzer.sh" 561b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# This is a file containing commands for scan-build. 571b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksBuildScript = "run_static_analyzer.cmd" 581b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 591b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# The log file name. 601b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksBuildLogName = "run_static_analyzer.log" 611b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Summary file - contains the summary of the failures. Ex: This info can be be 621b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# displayed when buildbot detects a build failure. 631b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksNumOfFailuresInSummary = 10 641b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksFailuresSummaryFileName = "failures.txt" 651b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Summary of the result diffs. 661b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksDiffsSummaryFileName = "diffs.txt" 671b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 681b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# The scan-build result directory. 691b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksSBOutputDirName = "ScanBuildResults" 701b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksSBOutputDirReferencePrefix = "Ref" 711b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 721b41716abf6856af893fd2f13cd0272dffa0e312Anna ZaksVerbose = 1 731b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 741b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef getProjectMapPath(): 751b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks ProjectMapPath = os.path.join(os.path.abspath(os.curdir), 761b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks ProjectMapFile) 771b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if not os.path.exists(ProjectMapPath): 781b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Error: Cannot find the Project Map file " + ProjectMapPath +\ 791b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks "\nRunning script for the wrong directory?" 801b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks sys.exit(-1) 811b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks return ProjectMapPath 821b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 831b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef getProjectDir(ID): 841b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks return os.path.join(os.path.abspath(os.curdir), ID) 851b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 861b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Run pre-processing script if any. 871b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef runPreProcessingScript(Dir, PBuildLogFile): 881b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks ScriptPath = os.path.join(Dir, PreprocessScript) 891b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if os.path.exists(ScriptPath): 901b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks try: 911b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Verbose == 1: 921b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " Executing: %s" % (ScriptPath,) 931b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks check_call("chmod +x %s" % ScriptPath, cwd = Dir, 941b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks stderr=PBuildLogFile, 951b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks stdout=PBuildLogFile, 961b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks shell=True) 971b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile, 981b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks stdout=PBuildLogFile, 991b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks shell=True) 1001b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks except: 1011b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Error: The pre-processing step failed. See ", \ 1021b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks PBuildLogFile.name, " for details." 1031b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks sys.exit(-1) 1041b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1051b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Build the project with scan-build by reading in the commands and 1061b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# prefixing them with the scan-build options. 1071b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef runScanBuild(Dir, SBOutputDir, PBuildLogFile): 1081b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks BuildScriptPath = os.path.join(Dir, BuildScript) 1091b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if not os.path.exists(BuildScriptPath): 1101b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Error: build script is not defined: %s" % BuildScriptPath 1111b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks sys.exit(-1) 1121b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBOptions = "-plist -o " + SBOutputDir + " " 1131b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBOptions += "-enable-checker core,deadcode.DeadStores" 1141b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks try: 1151b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBCommandFile = open(BuildScriptPath, "r") 1161b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBPrefix = "scan-build " + SBOptions + " " 1171b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks for Command in SBCommandFile: 1181b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBCommand = SBPrefix + Command 1191b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Verbose == 1: 1201b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " Executing: %s" % (SBCommand,) 1211b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile, 1221b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks stdout=PBuildLogFile, 1231b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks shell=True) 1241b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks except: 1251b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Error: scan-build failed. See ",PBuildLogFile.name,\ 1261b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks " for details." 1271b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks sys.exit(-1) 1281b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1291b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef buildProject(Dir, SBOutputDir): 1301b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks TBegin = time.time() 1311b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1321b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks BuildLogPath = os.path.join(Dir, BuildLogName) 1331b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Log file: %s" % (BuildLogPath,) 1341b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1351b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Clean up the log file. 1361b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if (os.path.exists(BuildLogPath)) : 1371b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RmCommand = "rm " + BuildLogPath 1381b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Verbose == 1: 1391b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " Executing: %s." % (RmCommand,) 1401b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks check_call(RmCommand, shell=True) 1411b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1421b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Open the log file. 1431b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks PBuildLogFile = open(BuildLogPath, "wb+") 1441b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks try: 1451b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Clean up scan build results. 1461b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if (os.path.exists(SBOutputDir)) : 1471b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RmCommand = "rm -r " + SBOutputDir 1481b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Verbose == 1: 1491b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " Executing: %s" % (RmCommand,) 1501b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks check_call(RmCommand, stderr=PBuildLogFile, 1511b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks stdout=PBuildLogFile, shell=True) 1521b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1531b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks runPreProcessingScript(Dir, PBuildLogFile) 1541b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks runScanBuild(Dir, SBOutputDir, PBuildLogFile) 1551b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks finally: 1561b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks PBuildLogFile.close() 1571b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1581b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Build complete (time: %.2f). See the log for more details: %s" % \ 1591b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks ((time.time()-TBegin), BuildLogPath) 1601b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1611b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# A plist file is created for each call to the analyzer(each source file). 1621b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# We are only interested on the once that have bug reports, so delete the rest. 1631b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef CleanUpEmptyPlists(SBOutputDir): 1641b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks for F in glob.glob(SBOutputDir + "/*/*.plist"): 1651b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks P = os.path.join(SBOutputDir, F) 1661b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1671b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks Data = plistlib.readPlist(P) 1681b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Delete empty reports. 1691b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if not Data['files']: 1701b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks os.remove(P) 1711b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks continue 1721b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1731b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Given the scan-build output directory, checks if the build failed 1741b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# (by searching for the failures directories). If there are failures, it 1751b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# creates a summary file in the output directory. 1761b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef checkBuild(SBOutputDir): 1771b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Check if there are failures. 1781b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt") 1791b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks TotalFailed = len(Failures); 1801b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if TotalFailed == 0: 1811b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks CleanUpEmptyPlists(SBOutputDir) 1821b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks Plists = glob.glob(SBOutputDir + "/*/*.plist") 1831b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Number of bug reports (non empty plist files) produced: %d" %\ 1841b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks len(Plists) 1851b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks return; 1861b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1871b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Create summary file to display when the build fails. 1881b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SummaryPath = os.path.join(SBOutputDir, FailuresSummaryFileName); 1891b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if (Verbose > 0): 1901b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " Creating the failures summary file %s." % (SummaryPath,) 1911b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 1921b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SummaryLog = open(SummaryPath, "w+") 1931b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks try: 1941b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,)) 1951b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if TotalFailed > NumOfFailuresInSummary: 1961b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SummaryLog.write("See the first %d below.\n" 1971b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks % (NumOfFailuresInSummary,)) 1981b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # TODO: Add a line "See the results folder for more." 1991b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2001b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks FailuresCopied = NumOfFailuresInSummary 2011b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks Idx = 0 2021b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks for FailLogPathI in glob.glob(SBOutputDir + "/*/failures/*.stderr.txt"): 2031b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Idx >= NumOfFailuresInSummary: 2041b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks break; 2051b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks Idx += 1 2061b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,)); 2071b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks FailLogI = open(FailLogPathI, "r"); 2081b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks try: 2091b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks shutil.copyfileobj(FailLogI, SummaryLog); 2101b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks finally: 2111b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks FailLogI.close() 2121b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks finally: 2131b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SummaryLog.close() 2141b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2151b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Error: Scan-build failed. See ", \ 2161b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks os.path.join(SBOutputDir, FailuresSummaryFileName) 2171b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks sys.exit(-1) 2181b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2191b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Auxiliary object to discard stdout. 2201b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksclass Discarder(object): 2211b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks def write(self, text): 2221b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks pass # do nothing 2231b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2241b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks# Compare the warnings produced by scan-build. 2251b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef runCmpResults(Dir): 2261b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks TBegin = time.time() 2271b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2281b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName) 2291b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks NewDir = os.path.join(Dir, SBOutputDirName) 2301b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2311b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # We have to go one level down the directory tree. 2321b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RefList = glob.glob(RefDir + "/*") 2331b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks NewList = glob.glob(NewDir + "/*") 2341b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if len(RefList) == 0 or len(NewList) == 0: 2351b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks return False 2361b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks assert(len(RefList) == len(NewList)) 2371b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2381b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # There might be more then one folder underneath - one per each scan-build 2391b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # command (Ex: one for configure and one for make). 2401b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if (len(RefList) > 1): 2411b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Assume that the corresponding folders have the same names. 2421b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RefList.sort() 2431b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks NewList.sort() 2441b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2451b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Iterate and find the differences. 2461b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks HaveDiffs = False 2471b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks PairList = zip(RefList, NewList) 2481b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks for P in PairList: 2491b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks RefDir = P[0] 2501b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks NewDir = P[1] 2511b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2521b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks assert(RefDir != NewDir) 2531b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Verbose == 1: 2541b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " Comparing Results: %s %s" % (RefDir, NewDir) 2551b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2561b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks DiffsPath = os.path.join(NewDir, DiffsSummaryFileName) 2571b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks Opts = CmpRuns.CmpOptions(DiffsPath) 2581b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Discard everything coming out of stdout (CmpRun produces a lot of them). 2591b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks OLD_STDOUT = sys.stdout 2601b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks sys.stdout = Discarder() 2611b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Scan the results, delete empty plist files. 2621b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False) 2631b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks sys.stdout = OLD_STDOUT 2641b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if HaveDiffs: 2651b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Warning: difference in diagnostics. See %s" % (DiffsPath,) 2661b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks HaveDiffs=True 2671b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2681b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin) 2691b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks return HaveDiffs 2701b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2711b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef testProject(ID, IsReferenceBuild, Dir=None): 2721b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks TBegin = time.time() 2731b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2741b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Dir is None : 2751b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks Dir = getProjectDir(ID) 2761b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if Verbose == 1: 2771b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " Build directory: %s." % (Dir,) 2781b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2791b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks # Set the build results directory. 2801b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if IsReferenceBuild == True : 2811b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBOutputDir = os.path.join(Dir, SBOutputDirReferencePrefix + \ 2821b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBOutputDirName) 2831b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks else : 2841b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks SBOutputDir = os.path.join(Dir, SBOutputDirName) 2851b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2861b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks buildProject(Dir, SBOutputDir) 2871b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2881b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks checkBuild(SBOutputDir) 2891b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2901b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks if IsReferenceBuild == False: 2911b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks runCmpResults(Dir) 2921b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2931b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print "Completed tests for project %s (time: %.2f)." % \ 2941b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks (ID, (time.time()-TBegin)) 2951b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 2961b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksdef testAll(IsReferenceBuild=False): 2971b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks PMapFile = open(getProjectMapPath(), "rb") 2981b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks try: 2991b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks PMapReader = csv.reader(PMapFile) 3001b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks for I in PMapReader: 3011b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks print " --- Building project %s" % (I[0],) 3021b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks testProject(I[0], IsReferenceBuild) 3031b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks finally: 3041b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks PMapFile.close() 3051b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks 3061b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaksif __name__ == '__main__': 3071b41716abf6856af893fd2f13cd0272dffa0e312Anna Zaks testAll() 308