1#!/usr/bin/env python3 2# Copyright (c) 2015-2017 The Khronos Group Inc. 3# Copyright (c) 2015-2017 Valve Corporation 4# Copyright (c) 2015-2017 LunarG, Inc. 5# Copyright (c) 2015-2017 Google Inc. 6# 7# Licensed under the Apache License, Version 2.0 (the "License"); 8# you may not use this file except in compliance with the License. 9# You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, software 14# distributed under the License is distributed on an "AS IS" BASIS, 15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16# See the License for the specific language governing permissions and 17# limitations under the License. 18# 19# Author: Tobin Ehlis <tobine@google.com> 20 21import argparse 22import os 23import sys 24import platform 25 26# vk_validation_stats.py overview 27# 28# usage: 29# python vk_validation_stats.py [verbose] 30# 31# Arguments: 32# verbose - enables verbose output, including VUID duplicates 33# 34# This script is intended to generate statistics on the state of validation code 35# based on information parsed from the source files and the database file 36# Here's what it currently does: 37# 1. Parse vk_validation_error_database.txt to store claimed state of validation checks 38# 2. Parse vk_validation_error_messages.h to verify the actual checks in header vs. the 39# claimed state of the checks 40# 3. Parse source files to identify which checks are implemented and verify that this 41# exactly matches the list of checks claimed to be implemented in the database 42# 4. Parse test file(s) and verify that reported tests exist 43# 5. Report out stats on number of checks, implemented checks, and duplicated checks 44# 45# If a mis-match is found during steps 2, 3, or 4, then the script exits w/ a non-zero error code 46# otherwise, the script will exit(0) 47# 48# TODO: 49# 1. Would also like to report out number of existing checks that don't yet use new, unique enum 50# 2. Could use notes to store custom fields (like TODO) and print those out here 51# 3. Update test code to check if tests use new, unique enums to check for errors instead of strings 52 53db_file = '../layers/vk_validation_error_database.txt' 54generated_layer_source_directories = [ 55'build', 56'dbuild', 57'release', 58] 59generated_layer_source_files = [ 60'parameter_validation.cpp', 61'object_tracker.cpp', 62] 63layer_source_files = [ 64'../layers/core_validation.cpp', 65'../layers/descriptor_sets.cpp', 66'../layers/parameter_validation_utils.cpp', 67'../layers/object_tracker_utils.cpp', 68'../layers/shader_validation.cpp', 69'../layers/buffer_validation.cpp', 70] 71header_file = '../layers/vk_validation_error_messages.h' 72# TODO : Don't hardcode linux path format if we want this to run on windows 73test_file = '../tests/layer_validation_tests.cpp' 74# List of enums that are allowed to be used more than once so don't warn on their duplicates 75duplicate_exceptions = [ 76'VALIDATION_ERROR_258004ea', # This covers the broad case that all child objects must be destroyed at DestroyInstance time 77'VALIDATION_ERROR_24a002f4', # This covers the broad case that all child objects must be destroyed at DestroyDevice time 78'VALIDATION_ERROR_0280006e', # Obj tracker check makes sure non-null framebuffer is valid & CV check makes sure it's compatible w/ renderpass framebuffer 79'VALIDATION_ERROR_12200682', # This is an aliasing error that we report twice, for each of the two allocations that are aliasing 80'VALIDATION_ERROR_1060d201', # Covers valid shader module handle for both Compute & Graphics pipelines 81'VALIDATION_ERROR_0c20c601', # This is a case for VkMappedMemoryRange struct that is used by both Flush & Invalidate MappedMemoryRange 82'VALIDATION_ERROR_0a400c01', # This is a blanket case for all invalid image aspect bit errors. The spec link has appropriate details for all separate cases. 83'VALIDATION_ERROR_0a8007fc', # This case covers two separate checks which are done independently 84'VALIDATION_ERROR_0a800800', # This case covers two separate checks which are done independently 85'VALIDATION_ERROR_15c0028a', # This is a descriptor set write update error that we use for a couple copy cases as well 86'VALIDATION_ERROR_1bc002de', # Single error for mis-matched stageFlags of vkCmdPushConstants() that is flagged for no stage flags & mis-matched flags 87'VALIDATION_ERROR_1880000e', # Handles both depth/stencil & compressed image errors for vkCmdClearColorImage() 88'VALIDATION_ERROR_0a600152', # Used for the mipLevel check of both dst & src images on vkCmdCopyImage call 89'VALIDATION_ERROR_0a600154', # Used for the arraySize check of both dst & src images on vkCmdCopyImage call 90'VALIDATION_ERROR_1500099e', # Used for both x & y bounds of viewport 91'VALIDATION_ERROR_1d8004a6', # Used for both x & y value of scissors to make sure they're not negative 92'VALIDATION_ERROR_1462ec01', # Surface of VkSwapchainCreateInfoKHR must be valid when creating both single or shared swapchains 93'VALIDATION_ERROR_1460de01', # oldSwapchain of VkSwapchainCreateInfoKHR must be valid when creating both single or shared swapchains 94'VALIDATION_ERROR_146009f2', # Single error for both imageFormat & imageColorSpace requirements when creating swapchain 95'VALIDATION_ERROR_15c00294', # Used twice for the same error codepath as both a param & to set a variable, so not really a duplicate 96] 97 98class ValidationDatabase: 99 def __init__(self, filename=db_file): 100 self.db_file = filename 101 self.delimiter = '~^~' 102 self.db_dict = {} # complete dict of all db values per error enum 103 # specialized data structs with slices of complete dict 104 self.db_implemented_enums = [] # list of all error enums claiming to be implemented in database file 105 self.db_unimplemented_implicit = [] # list of all implicit checks that aren't marked implemented 106 self.db_enum_to_tests = {} # dict where enum is key to lookup list of tests implementing the enum 107 self.db_invalid_implemented = [] # list of checks with invalid check_implemented flags 108 #self.src_implemented_enums 109 def read(self): 110 """Read a database file into internal data structures, format of each line is <enum><implemented Y|N?><testname><api><errormsg><notes>""" 111 #db_dict = {} # This is a simple db of just enum->errormsg, the same as is created from spec 112 #max_id = 0 113 with open(self.db_file, "r", encoding="utf8") as infile: 114 for line in infile: 115 line = line.strip() 116 if line.startswith('#') or '' == line: 117 continue 118 db_line = line.split(self.delimiter) 119 if len(db_line) != 8: 120 print("ERROR: Bad database line doesn't have 8 elements: %s" % (line)) 121 error_enum = db_line[0] 122 implemented = db_line[1] 123 testname = db_line[2] 124 api = db_line[3] 125 vuid_string = db_line[4] 126 core_ext = db_line[5] 127 error_str = db_line[6] 128 note = db_line[7] 129 # Read complete database contents into our class var for later use 130 self.db_dict[error_enum] = {} 131 self.db_dict[error_enum]['check_implemented'] = implemented 132 self.db_dict[error_enum]['testname'] = testname 133 self.db_dict[error_enum]['api'] = api 134 self.db_dict[error_enum]['vuid_string'] = vuid_string 135 self.db_dict[error_enum]['core_ext'] = core_ext 136 self.db_dict[error_enum]['error_string'] = error_str 137 self.db_dict[error_enum]['note'] = note 138 # Now build custom data structs 139 if 'Y' == implemented: 140 self.db_implemented_enums.append(error_enum) 141 elif 'implicit' in note: # only make note of non-implemented implicit checks 142 self.db_unimplemented_implicit.append(error_enum) 143 if implemented not in ['Y', 'N']: 144 self.db_invalid_implemented.append(error_enum) 145 if testname.lower() not in ['unknown', 'none', 'nottestable']: 146 self.db_enum_to_tests[error_enum] = testname.split(',') 147 #if len(self.db_enum_to_tests[error_enum]) > 1: 148 # print "Found check %s that has multiple tests: %s" % (error_enum, self.db_enum_to_tests[error_enum]) 149 #else: 150 # print "Check %s has single test: %s" % (error_enum, self.db_enum_to_tests[error_enum]) 151 #unique_id = int(db_line[0].split('_')[-1]) 152 #if unique_id > max_id: 153 # max_id = unique_id 154 #print "Found %d total enums in database" % (len(self.db_dict.keys())) 155 #print "Found %d enums claiming to be implemented in source" % (len(self.db_implemented_enums)) 156 #print "Found %d enums claiming to have tests implemented" % (len(self.db_enum_to_tests.keys())) 157 158class ValidationHeader: 159 def __init__(self, filename=header_file): 160 self.filename = header_file 161 self.enums = [] 162 def read(self): 163 """Read unique error enum header file into internal data structures""" 164 grab_enums = False 165 with open(self.filename, "r") as infile: 166 for line in infile: 167 line = line.strip() 168 if 'enum UNIQUE_VALIDATION_ERROR_CODE {' in line: 169 grab_enums = True 170 continue 171 if grab_enums: 172 if 'VALIDATION_ERROR_MAX_ENUM' in line: 173 grab_enums = False 174 break # done 175 elif 'VALIDATION_ERROR_UNDEFINED' in line: 176 continue 177 elif 'VALIDATION_ERROR_' in line: 178 enum = line.split(' = ')[0] 179 self.enums.append(enum) 180 #print "Found %d error enums. First is %s and last is %s." % (len(self.enums), self.enums[0], self.enums[-1]) 181 182class ValidationSource: 183 def __init__(self, source_file_list, generated_source_file_list, generated_source_directories): 184 self.source_files = source_file_list 185 self.generated_source_files = generated_source_file_list 186 self.generated_source_dirs = generated_source_directories 187 188 if len(self.generated_source_files) > 0: 189 qualified_paths = [] 190 for source in self.generated_source_files: 191 for build_dir in self.generated_source_dirs: 192 filepath = '../%s/layers/%s' % (build_dir, source) 193 if os.path.isfile(filepath): 194 qualified_paths.append(filepath) 195 break 196 if len(self.generated_source_files) != len(qualified_paths): 197 print("Error: Unable to locate one or more of the following source files in the %s directories" % (", ".join(generated_source_directories))) 198 print(self.generated_source_files) 199 print("Skipping documentation validation test") 200 exit(1) 201 else: 202 self.source_files.extend(qualified_paths) 203 204 self.enum_count_dict = {} # dict of enum values to the count of how much they're used, and location of where they're used 205 def parse(self): 206 duplicate_checks = 0 207 for sf in self.source_files: 208 line_num = 0 209 with open(sf) as f: 210 for line in f: 211 line_num = line_num + 1 212 if True in [line.strip().startswith(comment) for comment in ['//', '/*']]: 213 continue 214 # Find enums 215 #if 'VALIDATION_ERROR_' in line and True not in [ignore in line for ignore in ['[VALIDATION_ERROR_', 'UNIQUE_VALIDATION_ERROR_CODE']]: 216 if 'VALIDATION_ERROR_' in line: 217 # Need to isolate the validation error enum 218 #print("Line has check:%s" % (line)) 219 line_list = line.split() 220 enum_list = [] 221 for str in line_list: 222 if 'VALIDATION_ERROR_' in str and True not in [ignore_str in str for ignore_str in ['[VALIDATION_ERROR_', 'VALIDATION_ERROR_UNDEFINED', 'UNIQUE_VALIDATION_ERROR_CODE']]: 223 enum_list.append(str.strip(',);{}')) 224 #break 225 for enum in enum_list: 226 if enum != '': 227 if enum not in self.enum_count_dict: 228 self.enum_count_dict[enum] = {} 229 self.enum_count_dict[enum]['count'] = 1 230 self.enum_count_dict[enum]['file_line'] = [] 231 self.enum_count_dict[enum]['file_line'].append('%s,%d' % (sf, line_num)) 232 #print "Found enum %s implemented for first time in file %s" % (enum, sf) 233 else: 234 self.enum_count_dict[enum]['count'] = self.enum_count_dict[enum]['count'] + 1 235 self.enum_count_dict[enum]['file_line'].append('%s,%d' % (sf, line_num)) 236 #print "Found enum %s implemented for %d time in file %s" % (enum, self.enum_count_dict[enum], sf) 237 duplicate_checks = duplicate_checks + 1 238 #else: 239 #print("Didn't find actual check in line:%s" % (line)) 240 #print "Found %d unique implemented checks and %d are duplicated at least once" % (len(self.enum_count_dict.keys()), duplicate_checks) 241 242# Class to parse the validation layer test source and store testnames 243# TODO: Enhance class to detect use of unique error enums in the test 244class TestParser: 245 def __init__(self, test_file_list, test_group_name=['VkLayerTest', 'VkPositiveLayerTest', 'VkWsiEnabledLayerTest']): 246 self.test_files = test_file_list 247 self.test_to_errors = {} # Dict where testname maps to list of error enums found in that test 248 self.test_trigger_txt_list = [] 249 for tg in test_group_name: 250 self.test_trigger_txt_list.append('TEST_F(%s' % tg) 251 #print('Test trigger test list: %s' % (self.test_trigger_txt_list)) 252 253 # Parse test files into internal data struct 254 def parse(self): 255 # For each test file, parse test names into set 256 grab_next_line = False # handle testname on separate line than wildcard 257 testname = '' 258 for test_file in self.test_files: 259 with open(test_file) as tf: 260 for line in tf: 261 if True in [line.strip().startswith(comment) for comment in ['//', '/*']]: 262 continue 263 264 if True in [ttt in line for ttt in self.test_trigger_txt_list]: 265 #print('Test wildcard in line: %s' % (line)) 266 testname = line.split(',')[-1] 267 testname = testname.strip().strip(' {)') 268 #print('Inserting test: "%s"' % (testname)) 269 if ('' == testname): 270 grab_next_line = True 271 continue 272 self.test_to_errors[testname] = [] 273 if grab_next_line: # test name on its own line 274 grab_next_line = False 275 testname = testname.strip().strip(' {)') 276 self.test_to_errors[testname] = [] 277 if ' VALIDATION_ERROR_' in line: 278 line_list = line.split() 279 for sub_str in line_list: 280 if 'VALIDATION_ERROR_' in sub_str and True not in [ignore_str in sub_str for ignore_str in ['VALIDATION_ERROR_UNDEFINED', 'UNIQUE_VALIDATION_ERROR_CODE', 'VALIDATION_ERROR_MAX_ENUM']]: 281 #print("Trying to add enums for line: %s" % ()) 282 #print("Adding enum %s to test %s" % (sub_str.strip(',);'), testname)) 283 self.test_to_errors[testname].append(sub_str.strip(',);')) 284 285# Little helper class for coloring cmd line output 286class bcolors: 287 288 def __init__(self): 289 self.GREEN = '\033[0;32m' 290 self.RED = '\033[0;31m' 291 self.YELLOW = '\033[1;33m' 292 self.ENDC = '\033[0m' 293 if 'Linux' != platform.system(): 294 self.GREEN = '' 295 self.RED = '' 296 self.YELLOW = '' 297 self.ENDC = '' 298 299 def green(self): 300 return self.GREEN 301 302 def red(self): 303 return self.RED 304 305 def yellow(self): 306 return self.YELLOW 307 308 def endc(self): 309 return self.ENDC 310 311def main(argv): 312 result = 0 # Non-zero result indicates an error case 313 verbose_mode = 'verbose' in sys.argv 314 # parse db 315 val_db = ValidationDatabase() 316 val_db.read() 317 # parse header 318 val_header = ValidationHeader() 319 val_header.read() 320 # Create parser for layer files 321 val_source = ValidationSource(layer_source_files, generated_layer_source_files, generated_layer_source_directories) 322 val_source.parse() 323 # Parse test files 324 test_parser = TestParser([test_file, ]) 325 test_parser.parse() 326 327 # Process stats - Just doing this inline in main, could make a fancy class to handle 328 # all the processing of data and then get results from that 329 txt_color = bcolors() 330 if verbose_mode: 331 print("Validation Statistics") 332 else: 333 print("Validation/Documentation Consistency Test") 334 # First give number of checks in db & header and report any discrepancies 335 db_enums = len(val_db.db_dict.keys()) 336 hdr_enums = len(val_header.enums) 337 if verbose_mode: 338 print(" Database file includes %d unique checks" % (db_enums)) 339 print(" Header file declares %d unique checks" % (hdr_enums)) 340 341 # Report any checks that have an invalid check_implemented flag 342 if len(val_db.db_invalid_implemented) > 0: 343 result = 1 344 print(txt_color.red() + "The following checks have an invalid check_implemented flag (must be 'Y' or 'N'):" + txt_color.endc()) 345 for invalid_imp_enum in val_db.db_invalid_implemented: 346 check_implemented = val_db.db_dict[invalid_imp_enum]['check_implemented'] 347 print(txt_color.red() + " %s has check_implemented flag '%s'" % (invalid_imp_enum, check_implemented) + txt_color.endc()) 348 349 # Report details about how well the Database and Header are synchronized. 350 tmp_db_dict = val_db.db_dict 351 db_missing = [] 352 for enum in val_header.enums: 353 if not tmp_db_dict.pop(enum, False): 354 db_missing.append(enum) 355 if db_enums == hdr_enums and len(db_missing) == 0 and len(tmp_db_dict.keys()) == 0: 356 if verbose_mode: 357 print(txt_color.green() + " Database and Header match, GREAT!" + txt_color.endc()) 358 else: 359 print(txt_color.red() + " Uh oh, Database doesn't match Header :(" + txt_color.endc()) 360 result = 1 361 if len(db_missing) != 0: 362 print(txt_color.red() + " The following checks are in header but missing from database:" + txt_color.endc()) 363 for missing_enum in db_missing: 364 print(txt_color.red() + " %s" % (missing_enum) + txt_color.endc()) 365 if len(tmp_db_dict.keys()) != 0: 366 print(txt_color.red() + " The following checks are in database but haven't been declared in the header:" + txt_color.endc()) 367 for extra_enum in tmp_db_dict: 368 print(txt_color.red() + " %s" % (extra_enum) + txt_color.endc()) 369 370 # Report out claimed implemented checks vs. found actual implemented checks 371 imp_not_found = [] # Checks claimed to implemented in DB file but no source found 372 imp_not_claimed = [] # Checks found implemented but not claimed to be in DB 373 multiple_uses = False # Flag if any enums are used multiple times 374 for db_imp in val_db.db_implemented_enums: 375 if db_imp not in val_source.enum_count_dict: 376 imp_not_found.append(db_imp) 377 for src_enum in val_source.enum_count_dict: 378 if val_source.enum_count_dict[src_enum]['count'] > 1 and src_enum not in duplicate_exceptions: 379 multiple_uses = True 380 if src_enum not in val_db.db_implemented_enums: 381 imp_not_claimed.append(src_enum) 382 if verbose_mode: 383 print(" Database file claims that %d checks (%s) are implemented in source." % (len(val_db.db_implemented_enums), "{0:.0f}%".format(float(len(val_db.db_implemented_enums))/db_enums * 100))) 384 385 if len(val_db.db_unimplemented_implicit) > 0 and verbose_mode: 386 print(" Database file claims %d implicit checks (%s) that are not implemented." % (len(val_db.db_unimplemented_implicit), "{0:.0f}%".format(float(len(val_db.db_unimplemented_implicit))/db_enums * 100))) 387 total_checks = len(val_db.db_implemented_enums) + len(val_db.db_unimplemented_implicit) 388 print(" If all implicit checks are handled by parameter validation this is a total of %d (%s) checks covered." % (total_checks, "{0:.0f}%".format(float(total_checks)/db_enums * 100))) 389 if len(imp_not_found) == 0 and len(imp_not_claimed) == 0: 390 if verbose_mode: 391 print(txt_color.green() + " All claimed Database implemented checks have been found in source, and no source checks aren't claimed in Database, GREAT!" + txt_color.endc()) 392 else: 393 result = 1 394 print(txt_color.red() + " Uh oh, Database claimed implemented don't match Source :(" + txt_color.endc()) 395 if len(imp_not_found) != 0: 396 print(txt_color.red() + " The following %d checks are claimed to be implemented in Database, but weren't found in source:" % (len(imp_not_found)) + txt_color.endc()) 397 for not_imp_enum in imp_not_found: 398 print(txt_color.red() + " %s" % (not_imp_enum) + txt_color.endc()) 399 if len(imp_not_claimed) != 0: 400 print(txt_color.red() + " The following checks are implemented in source, but not claimed to be in Database:" + txt_color.endc()) 401 for imp_enum in imp_not_claimed: 402 print(txt_color.red() + " %s" % (imp_enum) + txt_color.endc()) 403 404 if multiple_uses and verbose_mode: 405 print(txt_color.yellow() + " Note that some checks are used multiple times. These may be good candidates for new valid usage spec language." + txt_color.endc()) 406 print(txt_color.yellow() + " Here is a list of each check used multiple times with its number of uses:" + txt_color.endc()) 407 for enum in val_source.enum_count_dict: 408 if val_source.enum_count_dict[enum]['count'] > 1 and enum not in duplicate_exceptions: 409 print(txt_color.yellow() + " %s: %d uses in file,line:" % (enum, val_source.enum_count_dict[enum]['count']) + txt_color.endc()) 410 for file_line in val_source.enum_count_dict[enum]['file_line']: 411 print(txt_color.yellow() + " \t%s" % (file_line) + txt_color.endc()) 412 413 # Now check that tests claimed to be implemented are actual test names 414 bad_testnames = [] 415 tests_missing_enum = {} # Report tests that don't use validation error enum to check for error case 416 for enum in val_db.db_enum_to_tests: 417 for testname in val_db.db_enum_to_tests[enum]: 418 if testname not in test_parser.test_to_errors: 419 bad_testnames.append(testname) 420 else: 421 enum_found = False 422 for test_enum in test_parser.test_to_errors[testname]: 423 if test_enum == enum: 424 #print("Found test that correctly checks for enum: %s" % (enum)) 425 enum_found = True 426 if not enum_found: 427 #print("Test %s is not using enum %s to check for error" % (testname, enum)) 428 if testname not in tests_missing_enum: 429 tests_missing_enum[testname] = [] 430 tests_missing_enum[testname].append(enum) 431 if tests_missing_enum and verbose_mode: 432 print(txt_color.yellow() + " \nThe following tests do not use their reported enums to check for the validation error. You may want to update these to pass the expected enum to SetDesiredFailureMsg:" + txt_color.endc()) 433 for testname in tests_missing_enum: 434 print(txt_color.yellow() + " Testname %s does not explicitly check for these ids:" % (testname) + txt_color.endc()) 435 for enum in tests_missing_enum[testname]: 436 print(txt_color.yellow() + " %s" % (enum) + txt_color.endc()) 437 438 # TODO : Go through all enums found in the test file and make sure they're correctly documented in the database file 439 if verbose_mode: 440 print(" Database file claims that %d checks have tests written." % len(val_db.db_enum_to_tests)) 441 if len(bad_testnames) == 0: 442 if verbose_mode: 443 print(txt_color.green() + " All claimed tests have valid names. That's good!" + txt_color.endc()) 444 else: 445 print(txt_color.red() + " The following testnames in Database appear to be invalid:") 446 result = 1 447 for bt in bad_testnames: 448 print(txt_color.red() + " %s" % (bt) + txt_color.endc()) 449 450 return result 451 452if __name__ == "__main__": 453 sys.exit(main(sys.argv[1:])) 454 455