1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# drawElements Quality Program utilities 5# -------------------------------------- 6# 7# Copyright 2015 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23from build.common import * 24from build.config import ANY_GENERATOR 25from build.build import build 26from build_caselists import Module, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 27from fnmatch import fnmatch 28from copy import copy 29 30import xml.etree.cElementTree as ElementTree 31import xml.dom.minidom as minidom 32 33CTS_DATA_DIR = os.path.join(DEQP_DIR, "android", "cts") 34 35class Configuration: 36 def __init__ (self, name, glconfig, rotation, surfacetype, filters): 37 self.name = name 38 self.glconfig = glconfig 39 self.rotation = rotation 40 self.surfacetype = surfacetype 41 self.filters = filters 42 43class Package: 44 def __init__ (self, module, configurations, splitFilters = {}): 45 self.module = module 46 self.configurations = configurations 47 # Map of name:[include filters]. Each will generate <api>.<name> package 48 # Test cases that didn't match any split filter will be in <api> package, 49 # i.e., the default value keeps everything in one package. 50 self.splitFilters = splitFilters 51 52class Mustpass: 53 def __init__ (self, version, packages): 54 self.version = version 55 self.packages = packages 56 57class Filter: 58 TYPE_INCLUDE = 0 59 TYPE_EXCLUDE = 1 60 61 def __init__ (self, type, filename): 62 self.type = type 63 self.filename = filename 64 65class TestRoot: 66 def __init__ (self): 67 self.children = [] 68 69class TestGroup: 70 def __init__ (self, name): 71 self.name = name 72 self.children = [] 73 74class TestCase: 75 def __init__ (self, name): 76 self.name = name 77 self.configurations = [] 78 79class GLESVersion: 80 def __init__(self, major, minor): 81 self.major = major 82 self.minor = minor 83 84 def encode (self): 85 return (self.major << 16) | (self.minor) 86 87def getModuleGLESVersion (module): 88 versions = { 89 'dEQP-EGL': GLESVersion(2,0), 90 'dEQP-GLES2': GLESVersion(2,0), 91 'dEQP-GLES3': GLESVersion(3,0), 92 'dEQP-GLES31': GLESVersion(3,1) 93 } 94 return versions[module.name] 95 96def getSrcDir (mustpass): 97 return os.path.join(CTS_DATA_DIR, mustpass.version, "src") 98 99def getTmpDir (mustpass): 100 return os.path.join(CTS_DATA_DIR, mustpass.version, "tmp") 101 102def getModuleShorthand (module): 103 assert module.name[:5] == "dEQP-" 104 return module.name[5:].lower() 105 106def getCaseListFileName (package, configuration): 107 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 108 109def getDstCaseListPath (mustpass, package, configuration): 110 return os.path.join(CTS_DATA_DIR, mustpass.version, getCaseListFileName(package, configuration)) 111 112def getCTSPackageName (package, splitName): 113 if splitName == None: 114 return "com.drawelements.deqp." + getModuleShorthand(package.module) 115 return "com.drawelements.deqp." + getModuleShorthand(package.module) + "." + splitName 116 117def getCommandLine (config): 118 return "--deqp-gl-config-name=%s --deqp-screen-rotation=%s --deqp-surface-type=%s --deqp-watchdog=enable" % (config.glconfig, config.rotation, config.surfacetype) 119 120def readCaseList (filename): 121 cases = [] 122 with open(filename, 'rb') as f: 123 for line in f: 124 if line[:6] == "TEST: ": 125 cases.append(line[6:].strip()) 126 return cases 127 128def getCaseList (mustpass, module): 129 generator = ANY_GENERATOR 130 buildCfg = getBuildConfig(DEFAULT_BUILD_DIR, DEFAULT_TARGET, "Debug") 131 132 #build(buildCfg, generator, [module.binName]) 133 genCaseList(buildCfg, generator, module, "txt") 134 135 return readCaseList(getCaseListPath(buildCfg, module, "txt")) 136 137def readPatternList (filename): 138 ptrns = [] 139 with open(filename, 'rb') as f: 140 for line in f: 141 line = line.strip() 142 if len(line) > 0 and line[0] != '#': 143 ptrns.append(line) 144 return ptrns 145 146def applyPatterns (caseList, patterns, op): 147 matched = set() 148 errors = [] 149 curList = copy(caseList) 150 trivialPtrns = [p for p in patterns if p.find('*') < 0] 151 regularPtrns = [p for p in patterns if p.find('*') >= 0] 152 153 # Apply trivial (just case paths) 154 allCasesSet = set(caseList) 155 for path in trivialPtrns: 156 if path in allCasesSet: 157 if path in matched: 158 errors.append((path, "Same case specified more than once")) 159 matched.add(path) 160 else: 161 errors.append((path, "Test case not found")) 162 163 curList = [c for c in curList if c not in matched] 164 165 for pattern in regularPtrns: 166 matchedThisPtrn = set() 167 168 for case in curList: 169 if fnmatch(case, pattern): 170 matchedThisPtrn.add(case) 171 172 if len(matchedThisPtrn) == 0: 173 errors.append((pattern, "Pattern didn't match any cases")) 174 175 matched = matched | matchedThisPtrn 176 curList = [c for c in curList if c not in matched] 177 178 for pattern, reason in errors: 179 print "ERROR: %s: %s" % (reason, pattern) 180 181 if len(errors) > 0: 182 die("Found %s invalid patterns" % len(errors)) 183 184 return [c for c in caseList if op(c in matched)] 185 186def applyInclude (caseList, patterns): 187 return applyPatterns(caseList, patterns, lambda b: b) 188 189def applyExclude (caseList, patterns): 190 return applyPatterns(caseList, patterns, lambda b: not b) 191 192def readPatternLists (mustpass): 193 lists = {} 194 for package in mustpass.packages: 195 for cfg in package.configurations: 196 for filter in cfg.filters: 197 if not filter.filename in lists: 198 lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) 199 return lists 200 201def applyFilters (caseList, patternLists, filters): 202 res = copy(caseList) 203 for filter in filters: 204 ptrnList = patternLists[filter.filename] 205 if filter.type == Filter.TYPE_INCLUDE: 206 res = applyInclude(res, ptrnList) 207 else: 208 assert filter.type == Filter.TYPE_EXCLUDE 209 res = applyExclude(res, ptrnList) 210 return res 211 212def appendToHierarchy (root, casePath): 213 def findChild (node, name): 214 for child in node.children: 215 if child.name == name: 216 return child 217 return None 218 219 curNode = root 220 components = casePath.split('.') 221 222 for component in components[:-1]: 223 nextNode = findChild(curNode, component) 224 if not nextNode: 225 nextNode = TestGroup(component) 226 curNode.children.append(nextNode) 227 curNode = nextNode 228 229 if not findChild(curNode, components[-1]): 230 curNode.children.append(TestCase(components[-1])) 231 232def buildTestHierachy (caseList): 233 root = TestRoot() 234 for case in caseList: 235 appendToHierarchy(root, case) 236 return root 237 238def buildTestCaseMap (root): 239 caseMap = {} 240 241 def recursiveBuild (curNode, prefix): 242 curPath = prefix + curNode.name 243 if isinstance(curNode, TestCase): 244 caseMap[curPath] = curNode 245 else: 246 for child in curNode.children: 247 recursiveBuild(child, curPath + '.') 248 249 for child in root.children: 250 recursiveBuild(child, '') 251 252 return caseMap 253 254def include (filename): 255 return Filter(Filter.TYPE_INCLUDE, filename) 256 257def exclude (filename): 258 return Filter(Filter.TYPE_EXCLUDE, filename) 259 260def prettifyXML (doc): 261 uglyString = ElementTree.tostring(doc, 'utf-8') 262 reparsed = minidom.parseString(uglyString) 263 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 264 265def genCTSPackageXML (package, root, name): 266 def isLeafGroup (testGroup): 267 numGroups = 0 268 numTests = 0 269 270 for child in testGroup.children: 271 if isinstance(child, TestCase): 272 numTests += 1 273 else: 274 numGroups += 1 275 276 assert numGroups + numTests > 0 277 278 if numGroups > 0 and numTests > 0: 279 die("Mixed groups and cases in %s" % testGroup.name) 280 281 return numGroups == 0 282 283 def makeConfiguration (parentElem, configuration): 284 return ElementTree.SubElement(parentElem, "TestInstance", glconfig=configuration.glconfig, rotation=configuration.rotation, surfacetype=configuration.surfacetype) 285 286 def makeTestCase (parentElem, testCase): 287 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name) 288 for config in testCase.configurations: 289 makeConfiguration(caseElem, config) 290 return caseElem 291 292 def makeTestGroup (parentElem, testGroup): 293 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name) 294 for child in testGroup.children: 295 if isinstance(child, TestCase): 296 makeTestCase(groupElem, child) 297 else: 298 makeTestGroup(groupElem, child) 299 return groupElem 300 301 pkgElem = ElementTree.Element("TestPackage", 302 name = package.module.name, 303 appPackageName = name, 304 testType = "deqpTest") 305 306 pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp") 307 pkgElem.set("deqp:glesVersion", str(getModuleGLESVersion(package.module).encode())) 308 309 for child in root.children: 310 makeTestGroup(pkgElem, child) 311 312 return pkgElem 313 314def genSpecXML (mustpass): 315 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 316 317 for package in mustpass.packages: 318 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 319 320 for config in package.configurations: 321 configElem = ElementTree.SubElement(packageElem, "Configuration", 322 name = config.name, 323 caseListFile = getCaseListFileName(package, config), 324 commandLine = getCommandLine(config)) 325 326 return mustpassElem 327 328def genCTSPackage (package, cases, matchingByConfig, packageName, xmlFilename): 329 root = buildTestHierachy(cases) 330 testCaseMap = buildTestCaseMap(root) 331 332 for config in package.configurations: 333 for case in matchingByConfig[config]: 334 if case in testCaseMap: 335 testCaseMap[case].configurations.append(config) 336 337 packageXml = genCTSPackageXML(package, root, packageName) 338 339 print " Writing CTS caselist: " + xmlFilename 340 writeFile(xmlFilename, prettifyXML(packageXml)) 341 342def genMustpass (mustpass, moduleCaseLists): 343 print "Generating mustpass '%s'" % mustpass.version 344 345 patternLists = readPatternLists(mustpass) 346 347 for package in mustpass.packages: 348 allCasesInPkg = moduleCaseLists[package.module] 349 matchingByConfig = {} 350 allMatchingSet = set() 351 352 for config in package.configurations: 353 filtered = applyFilters(allCasesInPkg, patternLists, config.filters) 354 dstFile = getDstCaseListPath(mustpass, package, config) 355 356 print " Writing deqp caselist: " + dstFile 357 writeFile(dstFile, "\n".join(filtered) + "\n") 358 359 matchingByConfig[config] = filtered 360 allMatchingSet = allMatchingSet | set(filtered) 361 362 allMatchingCases = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering 363 splitFilters = package.splitFilters 364 for splitName in splitFilters.keys(): 365 splitIncludeFilters = splitFilters[splitName] 366 splitCases = applyInclude(allMatchingCases, splitIncludeFilters) 367 packageName = getCTSPackageName(package, splitName) 368 xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, packageName + ".xml") 369 genCTSPackage(package, splitCases, matchingByConfig, packageName, xmlFilename) 370 371 # The cases not matching any of the includes 372 combinedSplitFilters = reduce(lambda x,y: x+y, splitFilters.values(), []) 373 restOfCases = applyExclude(allMatchingCases, combinedSplitFilters) 374 packageName = getCTSPackageName(package, None) 375 xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, packageName + ".xml") 376 genCTSPackage(package, restOfCases, matchingByConfig, packageName, xmlFilename) 377 378 specXML = genSpecXML(mustpass) 379 specFilename = os.path.join(CTS_DATA_DIR, mustpass.version, "mustpass.xml") 380 381 print " Writing spec: " + specFilename 382 writeFile(specFilename, prettifyXML(specXML)) 383 384 print "Done!" 385 386def genMustpassLists (mustpassLists): 387 moduleCaseLists = {} 388 389 # Getting case lists involves invoking build, so we want to cache the results 390 for mustpass in mustpassLists: 391 for package in mustpass.packages: 392 if not package.module in moduleCaseLists: 393 moduleCaseLists[package.module] = getCaseList(mustpass, package.module) 394 395 for mustpass in mustpassLists: 396 genMustpass(mustpass, moduleCaseLists) 397 398EGL_MODULE = Module(name = "dEQP-EGL", dirName = "egl", binName = "deqp-egl") 399GLES2_MODULE = Module(name = "dEQP-GLES2", dirName = "gles2", binName = "deqp-gles2") 400GLES3_MODULE = Module(name = "dEQP-GLES3", dirName = "gles3", binName = "deqp-gles3") 401GLES31_MODULE = Module(name = "dEQP-GLES31", dirName = "gles31", binName = "deqp-gles31") 402 403LMP_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ 404 Configuration(name = "master", 405 glconfig = "rgba8888d24s8ms0", 406 rotation = "unspecified", 407 surfacetype = "window", 408 filters = [include("es30-lmp.txt")]), 409 ]) 410LMP_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ 411 Configuration(name = "master", 412 glconfig = "rgba8888d24s8ms0", 413 rotation = "unspecified", 414 surfacetype = "window", 415 filters = [include("es31-lmp.txt")]), 416 ]) 417 418LMP_MR1_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ 419 Configuration(name = "master", 420 glconfig = "rgba8888d24s8ms0", 421 rotation = "unspecified", 422 surfacetype = "window", 423 filters = [include("es30-lmp-mr1.txt")]), 424 ]) 425LMP_MR1_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ 426 Configuration(name = "master", 427 glconfig = "rgba8888d24s8ms0", 428 rotation = "unspecified", 429 surfacetype = "window", 430 filters = [include("es31-lmp-mr1.txt")]), 431 ]) 432 433MASTER_EGL_COMMON_FILTERS = [include("egl-master.txt"), exclude("egl-failures.txt")] 434MASTER_EGL_PKG = Package(module = EGL_MODULE, configurations = [ 435 # Master 436 Configuration(name = "master", 437 glconfig = "rgba8888d24s8ms0", 438 rotation = "unspecified", 439 surfacetype = "window", 440 filters = MASTER_EGL_COMMON_FILTERS), 441 ]) 442 443MASTER_GLES2_COMMON_FILTERS = [ 444 include("gles2-master.txt"), 445 exclude("gles2-test-issues.txt"), 446 exclude("gles2-failures.txt") 447 ] 448MASTER_GLES2_PKG = Package(module = GLES2_MODULE, configurations = [ 449 # Master 450 Configuration(name = "master", 451 glconfig = "rgba8888d24s8ms0", 452 rotation = "unspecified", 453 surfacetype = "window", 454 filters = MASTER_GLES2_COMMON_FILTERS), 455 ]) 456 457MASTER_GLES3_COMMON_FILTERS = [ 458 include("gles3-master.txt"), 459 exclude("gles3-hw-issues.txt"), 460 exclude("gles3-driver-issues.txt"), 461 exclude("gles3-test-issues.txt"), 462 exclude("gles3-spec-issues.txt") 463 ] 464MASTER_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ 465 # Master 466 Configuration(name = "master", 467 glconfig = "rgba8888d24s8ms0", 468 rotation = "unspecified", 469 surfacetype = "window", 470 filters = MASTER_GLES3_COMMON_FILTERS), 471 # Rotations 472 Configuration(name = "rotate-portrait", 473 glconfig = "rgba8888d24s8ms0", 474 rotation = "0", 475 surfacetype = "window", 476 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 477 Configuration(name = "rotate-landscape", 478 glconfig = "rgba8888d24s8ms0", 479 rotation = "90", 480 surfacetype = "window", 481 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 482 Configuration(name = "rotate-reverse-portrait", 483 glconfig = "rgba8888d24s8ms0", 484 rotation = "180", 485 surfacetype = "window", 486 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 487 Configuration(name = "rotate-reverse-landscape", 488 glconfig = "rgba8888d24s8ms0", 489 rotation = "270", 490 surfacetype = "window", 491 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), 492 493 # MSAA 494 Configuration(name = "multisample", 495 glconfig = "rgba8888d24s8ms4", 496 rotation = "unspecified", 497 surfacetype = "window", 498 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-multisample.txt"), 499 exclude("gles3-multisample-issues.txt")]), 500 501 # Pixel format 502 Configuration(name = "565-no-depth-no-stencil", 503 glconfig = "rgb565d0s0ms0", 504 rotation = "unspecified", 505 surfacetype = "window", 506 filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-pixelformat.txt"), 507 exclude("gles3-pixelformat-issues.txt")]), 508 ]) 509 510MASTER_GLES31_COMMON_FILTERS = [ 511 include("gles31-master.txt"), 512 exclude("gles31-hw-issues.txt"), 513 exclude("gles31-driver-issues.txt"), 514 exclude("gles31-test-issues.txt"), 515 exclude("gles31-spec-issues.txt"), 516 ] 517MASTER_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ 518 # Master 519 Configuration(name = "master", 520 glconfig = "rgba8888d24s8ms0", 521 rotation = "unspecified", 522 surfacetype = "window", 523 filters = MASTER_GLES31_COMMON_FILTERS), 524 525 # Rotations 526 Configuration(name = "rotate-portrait", 527 glconfig = "rgba8888d24s8ms0", 528 rotation = "0", 529 surfacetype = "window", 530 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 531 Configuration(name = "rotate-landscape", 532 glconfig = "rgba8888d24s8ms0", 533 rotation = "90", 534 surfacetype = "window", 535 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 536 Configuration(name = "rotate-reverse-portrait", 537 glconfig = "rgba8888d24s8ms0", 538 rotation = "180", 539 surfacetype = "window", 540 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 541 Configuration(name = "rotate-reverse-landscape", 542 glconfig = "rgba8888d24s8ms0", 543 rotation = "270", 544 surfacetype = "window", 545 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), 546 547 # MSAA 548 Configuration(name = "multisample", 549 glconfig = "rgba8888d24s8ms4", 550 rotation = "unspecified", 551 surfacetype = "window", 552 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-multisample.txt")]), 553 554 # Pixel format 555 Configuration(name = "565-no-depth-no-stencil", 556 glconfig = "rgb565d0s0ms0", 557 rotation = "unspecified", 558 surfacetype = "window", 559 filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-pixelformat.txt")]), 560 ], 561 splitFilters = {"copy_image_compressed": ["dEQP-GLES31.functional.copy_image.compressed.*"], 562 "copy_image_non_compressed": ["dEQP-GLES31.functional.copy_image.non_compressed.*"], 563 "copy_image_mixed": ["dEQP-GLES31.functional.copy_image.mixed.*"], 564 } 565 ) 566 567MUSTPASS_LISTS = [ 568 Mustpass(version = "lmp", packages = [LMP_GLES3_PKG, LMP_GLES31_PKG]), 569 Mustpass(version = "lmp-mr1", packages = [LMP_MR1_GLES3_PKG, LMP_MR1_GLES31_PKG]), 570 Mustpass(version = "master", packages = [MASTER_EGL_PKG, MASTER_GLES2_PKG, MASTER_GLES3_PKG, MASTER_GLES31_PKG]) 571 ] 572 573if __name__ == "__main__": 574 genMustpassLists(MUSTPASS_LISTS) 575