test_importer.py revision 781d269bb961a7ea7bfb61a9f8c1d87320ed8ad0
1#!/usr/bin/python 2# 3# Copyright 2008 Google Inc. All Rights Reserved. 4""" 5This utility allows for easy updating, removing and importing 6of tests into the autotest_web afe_autotests table. 7 8Example of updating client side tests: 9./test_importer.py -t /usr/local/autotest/client/tests 10 11If, for example, not all of your control files adhere to the standard outlined 12at http://autotest.kernel.org/wiki/ControlRequirements, you can force options: 13 14./test_importer.py --test-type server -t /usr/local/autotest/server/tests 15 16You would need to pass --add-noncompliant to include such control files, 17however. An easy way to check for compliance is to run in dry mode: 18 19./test_importer.py --dry-run -t /usr/local/autotest/server/tests/mytest 20 21Or to check a single control file, you can use check_control_file_vars.py. 22 23Running with no options is equivalent to --add-all --db-clear-tests. 24 25Most options should be fairly self explanatory, use --help to display them. 26""" 27 28 29import common 30import logging, re, os, sys, optparse, compiler 31from autotest_lib.frontend import setup_django_environment 32from autotest_lib.frontend.afe import models 33from autotest_lib.client.common_lib import control_data, utils 34 35 36logging.basicConfig(level=logging.ERROR) 37# Global 38DRY_RUN = False 39DEPENDENCIES_NOT_FOUND = set() 40 41 42def update_all(autotest_dir, add_noncompliant, add_experimental): 43 """ 44 Function to scan through all tests and add them to the database. 45 46 This function invoked when no parameters supplied to the command line. 47 It 'synchronizes' the test database with the current contents of the 48 client and server test directories. When test code is discovered 49 in the file system new tests may be added to the db. Likewise, 50 if test code is not found in the filesystem, tests may be removed 51 from the db. The base test directories are hard-coded to client/tests, 52 client/site_tests, server/tests and server/site_tests. 53 54 @param autotest_dir: prepended to path strings (/usr/local/autotest). 55 @param add_noncompliant: attempt adding test with invalid control files. 56 @param add_experimental: add tests with experimental attribute set. 57 """ 58 for path in [ 'server/tests', 'server/site_tests', 'client/tests', 59 'client/site_tests']: 60 test_path = os.path.join(autotest_dir, path) 61 if not os.path.exists(test_path): 62 continue 63 logging.info("Scanning %s", test_path) 64 tests = [] 65 tests = get_tests_from_fs(test_path, "^control.*", 66 add_noncompliant=add_noncompliant) 67 update_tests_in_db(tests, add_experimental=add_experimental, 68 add_noncompliant=add_noncompliant, 69 autotest_dir=autotest_dir) 70 test_suite_path = os.path.join(autotest_dir, 'test_suites') 71 if os.path.exists(test_suite_path): 72 logging.info("Scanning %s", test_suite_path) 73 tests = get_tests_from_fs(test_suite_path, '.*', 74 add_noncompliant=add_noncompliant) 75 update_tests_in_db(tests, add_experimental=add_experimental, 76 add_noncompliant=add_noncompliant, 77 autotest_dir=autotest_dir) 78 79 profilers_path = os.path.join(autotest_dir, "client/profilers") 80 if os.path.exists(profilers_path): 81 logging.info("Scanning %s", profilers_path) 82 profilers = get_tests_from_fs(profilers_path, '.*py$') 83 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant, 84 description='NA') 85 # Clean bad db entries 86 db_clean_broken(autotest_dir) 87 88 89def update_samples(autotest_dir, add_noncompliant, add_experimental): 90 """ 91 Add only sample tests to the database from the filesystem. 92 93 This function invoked when -S supplied on command line. 94 Only adds tests to the database - does not delete any. 95 Samples tests are formatted slightly differently than other tests. 96 97 @param autotest_dir: prepended to path strings (/usr/local/autotest). 98 @param add_noncompliant: attempt adding test with invalid control files. 99 @param add_experimental: add tests with experimental attribute set. 100 """ 101 sample_path = os.path.join(autotest_dir, 'server/samples') 102 if os.path.exists(sample_path): 103 logging.info("Scanning %s", sample_path) 104 tests = get_tests_from_fs(sample_path, '.*srv$', 105 add_noncompliant=add_noncompliant) 106 update_tests_in_db(tests, add_experimental=add_experimental, 107 add_noncompliant=add_noncompliant, 108 autotest_dir=autotest_dir) 109 110 111def db_clean_broken(autotest_dir): 112 """ 113 Remove tests from autotest_web that do not have valid control files 114 115 This function invoked when -c supplied on the command line and when 116 running update_all(). Removes tests from database which are not 117 found in the filesystem. Also removes profilers which are just 118 a special case of tests. 119 120 @param autotest_dir: prepended to path strings (/usr/local/autotest). 121 """ 122 for test in models.Test.objects.all(): 123 full_path = os.path.join(autotest_dir, test.path) 124 if not os.path.isfile(full_path): 125 logging.info("Removing %s", test.path) 126 _log_or_execute(repr(test), test.delete) 127 128 # Find profilers that are no longer present 129 for profiler in models.Profiler.objects.all(): 130 full_path = os.path.join(autotest_dir, "client", "profilers", 131 profiler.name) 132 if not os.path.exists(full_path): 133 logging.info("Removing %s", profiler.name) 134 _log_or_execute(repr(profiler), profiler.delete) 135 136 137def db_clean_all(autotest_dir): 138 """ 139 Remove all tests from autotest_web - very destructive 140 141 This function invoked when -C supplied on the command line. 142 Removes ALL tests from the database. 143 144 @param autotest_dir: prepended to path strings (/usr/local/autotest). 145 """ 146 for test in models.Test.objects.all(): 147 full_path = os.path.join(autotest_dir, test.path) 148 logging.info("Removing %s", test.path) 149 _log_or_execute(repr(test), test.delete) 150 151 # Find profilers that are no longer present 152 for profiler in models.Profiler.objects.all(): 153 full_path = os.path.join(autotest_dir, "client", "profilers", 154 profiler.name) 155 logging.info("Removing %s", profiler.name) 156 _log_or_execute(repr(profiler), profiler.delete) 157 158 159def update_profilers_in_db(profilers, description='NA', 160 add_noncompliant=False): 161 """ 162 Add only profilers to the database from the filesystem. 163 164 This function invoked when -p supplied on command line. 165 Only adds profilers to the database - does not delete any. 166 Profilers are formatted slightly differently than tests. 167 168 @param profilers: list of profilers found in the file system. 169 @param description: simple text to satisfy docstring. 170 @param add_noncompliant: attempt adding test with invalid control files. 171 """ 172 for profiler in profilers: 173 name = os.path.basename(profiler).rstrip(".py") 174 if not profilers[profiler]: 175 if add_noncompliant: 176 doc = description 177 else: 178 logging.info("Skipping %s, missing docstring", profiler) 179 continue 180 else: 181 doc = profilers[profiler] 182 183 model = models.Profiler.objects.get_or_create(name=name)[0] 184 model.description = doc 185 _log_or_execute(repr(model), model.save) 186 187 188def update_tests_in_db(tests, dry_run=False, add_experimental=False, 189 add_noncompliant=False, autotest_dir=None): 190 """ 191 Scans through all tests and add them to the database. 192 193 This function invoked when -t supplied and for update_all. 194 When test code is discovered in the file system new tests may be added 195 196 @param tests: list of tests found in the filesystem. 197 @param dry_run: not used at this time. 198 @param add_experimental: add tests with experimental attribute set. 199 @param add_noncompliant: attempt adding test with invalid control files. 200 @param autotest_dir: prepended to path strings (/usr/local/autotest). 201 """ 202 site_set_attributes_module = utils.import_site_module( 203 __file__, 'autotest_lib.utils.site_test_importer_attributes') 204 205 for test in tests: 206 new_test = models.Test.objects.get_or_create( 207 path=test.replace(autotest_dir, '').lstrip('/'))[0] 208 logging.info("Processing %s", new_test.path) 209 210 # Set the test's attributes 211 data = tests[test] 212 _set_attributes_clean(new_test, data) 213 214 # Custom Attribute Update 215 if site_set_attributes_module: 216 site_set_attributes_module._set_attributes_custom(new_test, data) 217 218 # This only takes place if --add-noncompliant is provided on the CLI 219 if not new_test.name: 220 test_new_test = test.split('/') 221 if test_new_test[-1] == 'control': 222 new_test.name = test_new_test[-2] 223 else: 224 control_name = "%s:%s" 225 control_name %= (test_new_test[-2], 226 test_new_test[-1]) 227 new_test.name = control_name.replace('control.', '') 228 229 # Experimental Check 230 if not add_experimental and new_test.experimental: 231 continue 232 233 _log_or_execute(repr(new_test), new_test.save) 234 add_label_dependencies(new_test) 235 236 237def _set_attributes_clean(test, data): 238 """ 239 First pass sets the attributes of the Test object from file system. 240 241 @param test: a test object to be populated for the database. 242 @param data: object with test data from the file system. 243 """ 244 test_type = { 'client' : 1, 245 'server' : 2, } 246 test_time = { 'short' : 1, 247 'medium' : 2, 248 'long' : 3, } 249 250 251 string_attributes = ('name', 'author', 'test_class', 'test_category', 252 'test_category', 'sync_count') 253 for attribute in string_attributes: 254 setattr(test, attribute, getattr(data, attribute)) 255 256 test.description = data.doc 257 test.dependencies = ", ".join(data.dependencies) 258 259 int_attributes = ('experimental', 'run_verify') 260 for attribute in int_attributes: 261 setattr(test, attribute, int(getattr(data, attribute))) 262 263 try: 264 test.test_type = int(data.test_type) 265 if test.test_type != 1 and test.test_type != 2: 266 raise Exception('Incorrect number %d for test_type' % 267 test.test_type) 268 except ValueError: 269 pass 270 try: 271 test.test_time = int(data.time) 272 if test.test_time < 1 or test.time > 3: 273 raise Exception('Incorrect number %d for time' % test.time) 274 except ValueError: 275 pass 276 277 if not test.test_time and str == type(data.time): 278 test.test_time = test_time[data.time.lower()] 279 if not test.test_type and str == type(data.test_type): 280 test.test_type = test_type[data.test_type.lower()] 281 282 283def add_label_dependencies(test): 284 """ 285 Add proper many-to-many relationships from DEPENDENCIES field. 286 287 @param test: test object for the database. 288 """ 289 290 # clear out old relationships 291 _log_or_execute(repr(test), test.dependency_labels.clear, 292 subject='clear dependencies from') 293 294 for label_name in test.dependencies.split(','): 295 label_name = label_name.strip().lower() 296 if not label_name: 297 continue 298 299 try: 300 label = models.Label.objects.get(name=label_name) 301 except models.Label.DoesNotExist: 302 log_dependency_not_found(label_name) 303 continue 304 305 _log_or_execute(repr(label), test.dependency_labels.add, label, 306 subject='add dependency to %s' % test.name) 307 308 309def log_dependency_not_found(label_name): 310 """ 311 Exception processing when label not found in database. 312 313 @param label_name: from test dependencies. 314 """ 315 if label_name in DEPENDENCIES_NOT_FOUND: 316 return 317 logging.info("Dependency %s not found", label_name) 318 DEPENDENCIES_NOT_FOUND.add(label_name) 319 320 321def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False): 322 """ 323 Find control files in file system and load a list with their info. 324 325 @param parent_dir: directory to search recursively. 326 @param control_pattern: name format of control file. 327 @param add_noncompliant: ignore control file parse errors. 328 329 @return: dictionary of the form: tests[file_path] = parsed_object 330 """ 331 tests = {} 332 profilers = False 333 if 'client/profilers' in parent_dir: 334 profilers = True 335 for dir in [ parent_dir ]: 336 files = recursive_walk(dir, control_pattern) 337 for file in files: 338 if '__init__.py' in file or '.svn' in file: 339 continue 340 if not profilers: 341 if not add_noncompliant: 342 try: 343 found_test = control_data.parse_control(file, 344 raise_warnings=True) 345 tests[file] = found_test 346 except control_data.ControlVariableException, e: 347 logging.info("Skipping %s\n%s", file, e) 348 pass 349 else: 350 found_test = control_data.parse_control(file) 351 tests[file] = found_test 352 else: 353 script = file.rstrip(".py") 354 tests[file] = compiler.parseFile(file).doc 355 return tests 356 357 358def recursive_walk(path, wildcard): 359 """ 360 Recursively go through a directory. 361 362 This function invoked by get_tests_from_fs(). 363 364 @param path: base directory to start search. 365 @param wildcard: name format to match. 366 367 @return: A list of files that match wildcard 368 """ 369 files = [] 370 directories = [ path ] 371 while len(directories)>0: 372 directory = directories.pop() 373 for name in os.listdir(directory): 374 fullpath = os.path.join(directory, name) 375 if os.path.isfile(fullpath): 376 # if we are a control file 377 if re.search(wildcard, name): 378 files.append(fullpath) 379 elif os.path.isdir(fullpath): 380 directories.append(fullpath) 381 return files 382 383 384def _log_or_execute(content, func, *args, **kwargs): 385 """ 386 Log a message if dry_run is enabled, or execute the given function. 387 388 Relies on the DRY_RUN global variable. 389 390 @param content: the actual log message. 391 @param func: function to execute if dry_run is not enabled. 392 @param subject: (Optional) The type of log being written. Defaults to 393 the name of the provided function. 394 """ 395 subject = kwargs.get('subject', func.__name__) 396 397 if DRY_RUN: 398 logging.info("Would %s: %s", subject, content) 399 else: 400 func(*args) 401 402 403def _create_whitelist_set(whitelist_path): 404 """ 405 Create a set with contents from a whitelist file for membership testing. 406 407 @param whitelist_path: full path to the whitelist file. 408 409 @return: set with files listed one/line - newlines included. 410 """ 411 f = open(whitelist_path, 'r') 412 whitelist_set = set([line.strip() for line in f]) 413 f.close() 414 return whitelist_set 415 416 417def update_from_whitelist(whitelist_set, add_experimental, add_noncompliant, 418 autotest_dir): 419 """ 420 Scans through all tests in the whitelist and add them to the database. 421 422 This function invoked when -w supplied. 423 424 @param whitelist_set: set of tests in full-path form from a whitelist. 425 @param add_experimental: add tests with experimental attribute set. 426 @param add_noncompliant: attempt adding test with invalid control files. 427 @param autotest_dir: prepended to path strings (/usr/local/autotest). 428 """ 429 tests = {} 430 profilers = {} 431 for file_path in whitelist_set: 432 if file_path.find('client/profilers') == -1: 433 try: 434 found_test = control_data.parse_control(file_path, 435 raise_warnings=True) 436 tests[file_path] = found_test 437 except control_data.ControlVariableException, e: 438 logging.info("Skipping %s\n%s", file, e) 439 pass 440 else: 441 profilers[file_path] = compiler.parseFile(file_path).doc 442 443 if len(tests) > 0: 444 update_tests_in_db(tests, add_experimental=add_experimental, 445 add_noncompliant=add_noncompliant, 446 autotest_dir=autotest_dir) 447 if len(profilers) > 0: 448 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant, 449 description='NA') 450 451 452def main(argv): 453 """Main function""" 454 455 global DRY_RUN 456 parser = optparse.OptionParser() 457 parser.add_option('-c', '--db-clean-tests', 458 dest='clean_tests', action='store_true', 459 default=False, 460 help='Clean client and server tests with invalid control files') 461 parser.add_option('-C', '--db-clear-all-tests', 462 dest='clear_all_tests', action='store_true', 463 default=False, 464 help='Clear ALL client and server tests') 465 parser.add_option('-d', '--dry-run', 466 dest='dry_run', action='store_true', default=False, 467 help='Dry run for operation') 468 parser.add_option('-A', '--add-all', 469 dest='add_all', action='store_true', 470 default=False, 471 help='Add site_tests, tests, and test_suites') 472 parser.add_option('-S', '--add-samples', 473 dest='add_samples', action='store_true', 474 default=False, 475 help='Add samples.') 476 parser.add_option('-E', '--add-experimental', 477 dest='add_experimental', action='store_true', 478 default=True, 479 help='Add experimental tests to frontend') 480 parser.add_option('-N', '--add-noncompliant', 481 dest='add_noncompliant', action='store_true', 482 default=False, 483 help='Add non-compliant tests (i.e. tests that do not ' 484 'define all required control variables)') 485 parser.add_option('-p', '--profile-dir', dest='profile_dir', 486 help='Directory to recursively check for profiles') 487 parser.add_option('-t', '--tests-dir', dest='tests_dir', 488 help='Directory to recursively check for control.*') 489 parser.add_option('-r', '--control-pattern', dest='control_pattern', 490 default='^control.*', 491 help='The pattern to look for in directories for control files') 492 parser.add_option('-v', '--verbose', 493 dest='verbose', action='store_true', default=False, 494 help='Run in verbose mode') 495 parser.add_option('-w', '--whitelist-file', dest='whitelist_file', 496 help='Filename for list of test names that must match') 497 parser.add_option('-z', '--autotest-dir', dest='autotest_dir', 498 default=os.path.join(os.path.dirname(__file__), '..'), 499 help='Autotest directory root') 500 options, args = parser.parse_args() 501 DRY_RUN = options.dry_run 502 # Make sure autotest_dir is the absolute path 503 options.autotest_dir = os.path.abspath(options.autotest_dir) 504 505 if len(args) > 0: 506 logging.error("Invalid option(s) provided: %s", args) 507 parser.print_help() 508 return 1 509 510 if options.verbose: 511 logging.getLogger().setLevel(logging.DEBUG) 512 513 if len(argv) == 1 or (len(argv) == 2 and options.verbose): 514 update_all(options.autotest_dir, options.add_noncompliant, 515 options.add_experimental) 516 db_clean_broken(options.autotest_dir) 517 return 0 518 519 if options.clear_all_tests: 520 if (options.clean_tests or options.add_all or options.add_samples or 521 options.add_noncompliant): 522 logging.error( 523 "Can only pass --autotest-dir, --dry-run and --verbose with " 524 "--db-clear-all-tests") 525 return 1 526 db_clean_all(options.autotest_dir) 527 528 whitelist_set = None 529 if options.whitelist_file: 530 if options.add_all: 531 logging.error("Cannot pass both --add-all and --whitelist-file") 532 return 1 533 whitelist_path = os.path.abspath(options.whitelist_file) 534 if not os.path.isfile(whitelist_path): 535 logging.error("--whitelist-file (%s) not found", whitelist_path) 536 return 1 537 logging.info("Using whitelist file %s", whitelist_path) 538 whitelist_set = _create_whitelist_set(whitelist_path) 539 update_from_whitelist(whitelist_set, 540 add_experimental=options.add_experimental, 541 add_noncompliant=options.add_noncompliant, 542 autotest_dir=options.autotest_dir) 543 if options.add_all: 544 update_all(options.autotest_dir, options.add_noncompliant, 545 options.add_experimental) 546 if options.add_samples: 547 update_samples(options.autotest_dir, options.add_noncompliant, 548 options.add_experimental) 549 if options.tests_dir: 550 options.tests_dir = os.path.abspath(options.tests_dir) 551 tests = get_tests_from_fs(options.tests_dir, options.control_pattern, 552 add_noncompliant=options.add_noncompliant) 553 update_tests_in_db(tests, add_experimental=options.add_experimental, 554 add_noncompliant=options.add_noncompliant, 555 autotest_dir=options.autotest_dir) 556 if options.profile_dir: 557 profilers = get_tests_from_fs(options.profile_dir, '.*py$') 558 update_profilers_in_db(profilers, 559 add_noncompliant=options.add_noncompliant, 560 description='NA') 561 if options.clean_tests: 562 db_clean_broken(options.autotest_dir) 563 564 565if __name__ == "__main__": 566 main(sys.argv) 567