1# -*- coding: utf-8 -*- 2 3from build.common import * 4from build.config import * 5from build.build import * 6 7import os 8import sys 9import string 10import socket 11import fnmatch 12from datetime import datetime 13 14BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly")) 15BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build") 16BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs") 17BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs") 18 19EXECUTOR_PATH = "executor/executor" 20LOG_TO_CSV_PATH = "executor/testlog-to-csv" 21EXECSERVER_PATH = "execserver/execserver" 22 23CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data") 24 25COMPARE_NUM_RESULTS = 4 26COMPARE_REPORT_NAME = "nightly-report.html" 27 28COMPARE_REPORT_TMPL = ''' 29<html> 30<head> 31<title>${TITLE}</title> 32<style type="text/css"> 33<!-- 34body { font: serif; font-size: 1em; } 35table { border-spacing: 0; border-collapse: collapse; } 36td { border-width: 1px; border-style: solid; border-color: #808080; } 37.Header { font-weight: bold; font-size: 1em; border-style: none; } 38.CasePath { } 39.Pass { background: #80ff80; } 40.Fail { background: #ff4040; } 41.QualityWarning { background: #ffff00; } 42.CompabilityWarning { background: #ffff00; } 43.Pending { background: #808080; } 44.Running { background: #d3d3d3; } 45.NotSupported { background: #ff69b4; } 46.ResourceError { background: #ff4040; } 47.InternalError { background: #ff1493; } 48.Canceled { background: #808080; } 49.Crash { background: #ffa500; } 50.Timeout { background: #ffa500; } 51.Disabled { background: #808080; } 52.Missing { background: #808080; } 53.Ignored { opacity: 0.5; } 54--> 55</style> 56</head> 57<body> 58<h1>${TITLE}</h1> 59<table> 60${RESULTS} 61</table> 62</body> 63</html> 64''' 65 66class NightlyRunConfig: 67 def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []): 68 self.name = name 69 self.buildConfig = buildConfig 70 self.generator = generator 71 self.binaryName = binaryName 72 self.testset = testset 73 self.args = args 74 self.exclude = exclude 75 self.ignore = ignore 76 77 def getBinaryPath(self, basePath): 78 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath)) 79 80class NightlyBuildConfig(BuildConfig): 81 def __init__(self, name, buildType, args): 82 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args) 83 84class TestCaseResult: 85 def __init__ (self, name, statusCode): 86 self.name = name 87 self.statusCode = statusCode 88 89class MultiResult: 90 def __init__ (self, name, statusCodes): 91 self.name = name 92 self.statusCodes = statusCodes 93 94class BatchResult: 95 def __init__ (self, name): 96 self.name = name 97 self.results = [] 98 99def parseResultCsv (data): 100 lines = data.splitlines()[1:] 101 results = [] 102 103 for line in lines: 104 items = line.split(",") 105 results.append(TestCaseResult(items[0], items[1])) 106 107 return results 108 109def readTestCaseResultsFromCSV (filename): 110 return parseResultCsv(readFile(filename)) 111 112def readBatchResultFromCSV (filename, batchResultName = None): 113 batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename)) 114 batchResult.results = readTestCaseResultsFromCSV(filename) 115 return batchResult 116 117def getResultTimestamp (): 118 return datetime.now().strftime("%Y-%m-%d-%H-%M") 119 120def getCompareFilenames (logsDir): 121 files = [] 122 for file in os.listdir(logsDir): 123 fullPath = os.path.join(logsDir, file) 124 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"): 125 files.append(fullPath) 126 files.sort() 127 128 return files[-COMPARE_NUM_RESULTS:] 129 130def parseAsCSV (logPath, config): 131 args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath] 132 proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 133 out, err = proc.communicate() 134 return out 135 136def computeUnifiedTestCaseList (batchResults): 137 caseList = [] 138 caseSet = set() 139 140 for batchResult in batchResults: 141 for result in batchResult.results: 142 if not result.name in caseSet: 143 caseList.append(result.name) 144 caseSet.add(result.name) 145 146 return caseList 147 148def computeUnifiedResults (batchResults): 149 150 def genResultMap (batchResult): 151 resMap = {} 152 for result in batchResult.results: 153 resMap[result.name] = result 154 return resMap 155 156 resultMap = [genResultMap(r) for r in batchResults] 157 caseList = computeUnifiedTestCaseList(batchResults) 158 results = [] 159 160 for caseName in caseList: 161 statusCodes = [] 162 163 for i in range(0, len(batchResults)): 164 result = resultMap[i][caseName] if caseName in resultMap[i] else None 165 statusCode = result.statusCode if result != None else 'Missing' 166 statusCodes.append(statusCode) 167 168 results.append(MultiResult(caseName, statusCodes)) 169 170 return results 171 172def allStatusCodesEqual (result): 173 firstCode = result.statusCodes[0] 174 for i in range(1, len(result.statusCodes)): 175 if result.statusCodes[i] != firstCode: 176 return False 177 return True 178 179def computeDiffResults (unifiedResults): 180 diff = [] 181 for result in unifiedResults: 182 if not allStatusCodesEqual(result): 183 diff.append(result) 184 return diff 185 186def genCompareReport (batchResults, title, ignoreCases): 187 class TableRow: 188 def __init__ (self, testCaseName, innerHTML): 189 self.testCaseName = testCaseName 190 self.innerHTML = innerHTML 191 192 unifiedResults = computeUnifiedResults(batchResults) 193 diffResults = computeDiffResults(unifiedResults) 194 rows = [] 195 196 # header 197 headerCol = '<td class="Header">Test case</td>\n' 198 for batchResult in batchResults: 199 headerCol += '<td class="Header">%s</td>\n' % batchResult.name 200 rows.append(TableRow(None, headerCol)) 201 202 # results 203 for result in diffResults: 204 col = '<td class="CasePath">%s</td>\n' % result.name 205 for statusCode in result.statusCodes: 206 col += '<td class="%s">%s</td>\n' % (statusCode, statusCode) 207 208 rows.append(TableRow(result.name, col)) 209 210 tableStr = "" 211 for row in rows: 212 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases): 213 tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML 214 else: 215 tableStr += '<tr>\n%s</tr>\n' % row.innerHTML 216 217 html = COMPARE_REPORT_TMPL 218 html = html.replace("${TITLE}", title) 219 html = html.replace("${RESULTS}", tableStr) 220 221 return html 222 223def matchesAnyPattern (name, patterns): 224 for pattern in patterns: 225 if fnmatch.fnmatch(name, pattern): 226 return True 227 return False 228 229def statusCodesMatch (refResult, resResult): 230 return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult 231 232def compareBatchResults (referenceBatch, resultBatch, ignoreCases): 233 unifiedResults = computeUnifiedResults([referenceBatch, resultBatch]) 234 failedCases = [] 235 236 for result in unifiedResults: 237 if not matchesAnyPattern(result.name, ignoreCases): 238 refResult = result.statusCodes[0] 239 resResult = result.statusCodes[1] 240 241 if not statusCodesMatch(refResult, resResult): 242 failedCases.append(result) 243 244 return failedCases 245 246def getUnusedPort (): 247 # \note Not 100%-proof method as other apps may grab this port before we launch execserver 248 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 249 s.bind(('localhost', 0)) 250 addr, port = s.getsockname() 251 s.close() 252 return port 253 254def runNightly (config): 255 build(config.buildConfig, config.generator) 256 257 # Run parameters 258 timestamp = getResultTimestamp() 259 logDir = os.path.join(BASE_LOGS_DIR, config.name) 260 testLogPath = os.path.join(logDir, timestamp + ".qpa") 261 infoLogPath = os.path.join(logDir, timestamp + ".txt") 262 csvLogPath = os.path.join(logDir, timestamp + ".csv") 263 compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv") 264 port = getUnusedPort() 265 266 if not os.path.exists(logDir): 267 os.makedirs(logDir) 268 269 if os.path.exists(testLogPath) or os.path.exists(infoLogPath): 270 raise Exception("Result '%s' already exists", timestamp) 271 272 # Paths, etc. 273 binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName)) 274 workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName)) 275 276 execArgs = [ 277 config.getBinaryPath(EXECUTOR_PATH), 278 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH), 279 '--port=%d' % port, 280 '--binaryname=%s' % binaryName, 281 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "), 282 '--workdir=%s' % workingDir, 283 '--caselistdir=%s' % CASELIST_PATH, 284 '--testset=%s' % string.join(config.testset, ","), 285 '--out=%s' % testLogPath, 286 '--info=%s' % infoLogPath, 287 '--summary=no' 288 ] 289 290 if len(config.exclude) > 0: 291 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")] 292 293 execute(execArgs) 294 295 # Translate to CSV for comparison purposes 296 lastResultCsv = parseAsCSV(testLogPath, config) 297 writeFile(csvLogPath, lastResultCsv) 298 299 if os.path.exists(compareLogPath): 300 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference") 301 else: 302 refBatchResult = None 303 304 # Generate comparison report 305 compareFilenames = getCompareFilenames(logDir) 306 batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames] 307 308 if refBatchResult != None: 309 batchResults = [refBatchResult] + batchResults 310 311 writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore)) 312 print "Comparison report written to %s" % COMPARE_REPORT_NAME 313 314 # Compare to reference 315 if refBatchResult != None: 316 curBatchResult = BatchResult("current") 317 curBatchResult.results = parseResultCsv(lastResultCsv) 318 failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore) 319 320 print "" 321 for result in failedCases: 322 print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1]) 323 324 print "" 325 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed") 326 327 if len(failedCases) > 0: 328 return False 329 330 return True 331 332# Configurations 333 334DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR 335DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR 336 337WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl']) 338ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu']) 339 340BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable'] 341 342CONFIGS = [ 343 NightlyRunConfig( 344 name = "wgl_x64_release_gles2", 345 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 346 generator = DEFAULT_WIN64_GENERATOR, 347 binaryName = "modules/gles2/deqp-gles2", 348 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 349 testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"], 350 exclude = [ 351 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*", 352 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*", 353 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*", 354 ], 355 ignore = [] 356 ), 357 NightlyRunConfig( 358 name = "wgl_x64_release_gles3", 359 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 360 generator = DEFAULT_WIN64_GENERATOR, 361 binaryName = "modules/gles3/deqp-gles3", 362 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 363 testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"], 364 exclude = [ 365 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*", 366 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*", 367 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*", 368 ], 369 ignore = [ 370 "dEQP-GLES3.functional.transform_feedback.*", 371 "dEQP-GLES3.functional.occlusion_query.*", 372 "dEQP-GLES3.functional.lifetime.*", 373 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops", 374 ] 375 ), 376 NightlyRunConfig( 377 name = "wgl_x64_release_gles31", 378 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 379 generator = DEFAULT_WIN64_GENERATOR, 380 binaryName = "modules/gles31/deqp-gles31", 381 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 382 testset = ["dEQP-GLES31.*"], 383 exclude = [], 384 ignore = [ 385 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3", 386 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer", 387 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset", 388 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint", 389 "dEQP-GLES31.functional.blend_equation_advanced.basic.*", 390 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*", 391 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*", 392 "dEQP-GLES31.functional.uniform_location.*", 393 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv", 394 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv", 395 "dEQP-GLES31.functional.debug.error_filters.case_0", 396 "dEQP-GLES31.functional.debug.error_filters.case_2", 397 ] 398 ), 399 NightlyRunConfig( 400 name = "wgl_x64_release_gl3", 401 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 402 generator = DEFAULT_WIN64_GENERATOR, 403 binaryName = "modules/gl3/deqp-gl3", 404 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 405 testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"], 406 exclude = [ 407 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*", 408 "dEQP-GL3.functional.shaders.loops.*while*only_continue*", 409 "dEQP-GL3.functional.shaders.loops.*while*double_continue*", 410 ], 411 ignore = [ 412 "dEQP-GL3.functional.transform_feedback.*" 413 ] 414 ), 415 NightlyRunConfig( 416 name = "arm_gles3_emu_x32_egl", 417 buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG, 418 generator = DEFAULT_WIN32_GENERATOR, 419 binaryName = "modules/egl/deqp-egl", 420 args = BASE_ARGS, 421 testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"], 422 exclude = [ 423 "dEQP-EGL.functional.sharing.gles2.multithread.*", 424 "dEQP-EGL.functional.multithread.*", 425 ], 426 ignore = [] 427 ), 428 NightlyRunConfig( 429 name = "opencl_x64_release", 430 buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']), 431 generator = DEFAULT_WIN64_GENERATOR, 432 binaryName = "modules/opencl/deqp-opencl", 433 args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS, 434 testset = ["dEQP-CL.*"], 435 exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"], 436 ignore = [ 437 "dEQP-CL.scheduler.random.*", 438 "dEQP-CL.language.set_kernel_arg.random_structs.*", 439 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset", 440 "dEQP-CL.language.call_function.arguments.random_structs.*", 441 "dEQP-CL.language.call_kernel.random_structs.*", 442 "dEQP-CL.language.inf_nan.nan.frexp.float", 443 "dEQP-CL.language.inf_nan.nan.lgamma_r.float", 444 "dEQP-CL.language.inf_nan.nan.modf.float", 445 "dEQP-CL.language.inf_nan.nan.sqrt.float", 446 "dEQP-CL.api.multithread.*", 447 "dEQP-CL.api.callback.random.nested.*", 448 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb", 449 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb", 450 "dEQP-CL.image.addressing_filtering12.1d_array.*", 451 "dEQP-CL.image.addressing_filtering12.2d_array.*" 452 ] 453 ) 454] 455 456if __name__ == "__main__": 457 config = None 458 459 if len(sys.argv) == 2: 460 cfgName = sys.argv[1] 461 for curCfg in CONFIGS: 462 if curCfg.name == cfgName: 463 config = curCfg 464 break 465 466 if config != None: 467 isOk = runNightly(config) 468 if not isOk: 469 sys.exit(-1) 470 else: 471 print "%s: [config]" % sys.argv[0] 472 print "" 473 print " Available configs:" 474 for config in CONFIGS: 475 print " %s" % config.name 476 sys.exit(-1) 477