test_importer.py revision bdaab795cffa33f9a37995bb283047cb03699272
1#!/usr/bin/python 2# 3# Copyright 2008 Google Inc. All Rights Reserved. 4""" 5This utility allows for easy updating, removing and importing 6of tests into the autotest_web afe_autotests table. 7 8Example of updating client side tests: 9./test_importer.py -t /usr/local/autotest/client/tests 10 11If, for example, not all of your control files adhere to the standard outlined 12at http://autotest.kernel.org/wiki/ControlRequirements, you can force options: 13 14./test_importer.py --test-type server -t /usr/local/autotest/server/tests 15 16You would need to pass --add-noncompliant to include such control files, 17however. An easy way to check for compliance is to run in dry mode: 18 19./test_importer.py --dry-run -t /usr/local/autotest/server/tests/mytest 20 21Or to check a single control file, you can use check_control_file_vars.py. 22 23Running with no options is equivalent to --add-all --db-clear-tests. 24 25Most options should be fairly self explanatory, use --help to display them. 26""" 27 28 29import common 30import logging, re, os, sys, optparse, compiler 31from autotest_lib.frontend import setup_django_environment 32from autotest_lib.frontend.afe import models 33from autotest_lib.client.common_lib import control_data, utils 34from autotest_lib.client.common_lib import logging_config, logging_manager 35 36 37class TestImporterLoggingConfig(logging_config.LoggingConfig): 38 def configure_logging(self, results_dir=None, verbose=False): 39 super(TestImporterLoggingConfig, self).configure_logging( 40 use_console=True, 41 verbose=verbose) 42 43 44# Global 45DRY_RUN = False 46DEPENDENCIES_NOT_FOUND = set() 47 48 49def update_all(autotest_dir, add_noncompliant, add_experimental): 50 """ 51 Function to scan through all tests and add them to the database. 52 53 This function invoked when no parameters supplied to the command line. 54 It 'synchronizes' the test database with the current contents of the 55 client and server test directories. When test code is discovered 56 in the file system new tests may be added to the db. Likewise, 57 if test code is not found in the filesystem, tests may be removed 58 from the db. The base test directories are hard-coded to client/tests, 59 client/site_tests, server/tests and server/site_tests. 60 61 @param autotest_dir: prepended to path strings (/usr/local/autotest). 62 @param add_noncompliant: attempt adding test with invalid control files. 63 @param add_experimental: add tests with experimental attribute set. 64 """ 65 for path in [ 'server/tests', 'server/site_tests', 'client/tests', 66 'client/site_tests']: 67 test_path = os.path.join(autotest_dir, path) 68 if not os.path.exists(test_path): 69 continue 70 logging.info("Scanning %s", test_path) 71 tests = [] 72 tests = get_tests_from_fs(test_path, "^control.*", 73 add_noncompliant=add_noncompliant) 74 update_tests_in_db(tests, add_experimental=add_experimental, 75 add_noncompliant=add_noncompliant, 76 autotest_dir=autotest_dir) 77 test_suite_path = os.path.join(autotest_dir, 'test_suites') 78 if os.path.exists(test_suite_path): 79 logging.info("Scanning %s", test_suite_path) 80 tests = get_tests_from_fs(test_suite_path, '.*', 81 add_noncompliant=add_noncompliant) 82 update_tests_in_db(tests, add_experimental=add_experimental, 83 add_noncompliant=add_noncompliant, 84 autotest_dir=autotest_dir) 85 86 profilers_path = os.path.join(autotest_dir, "client/profilers") 87 if os.path.exists(profilers_path): 88 logging.info("Scanning %s", profilers_path) 89 profilers = get_tests_from_fs(profilers_path, '.*py$') 90 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant, 91 description='NA') 92 # Clean bad db entries 93 db_clean_broken(autotest_dir) 94 95 96def update_samples(autotest_dir, add_noncompliant, add_experimental): 97 """ 98 Add only sample tests to the database from the filesystem. 99 100 This function invoked when -S supplied on command line. 101 Only adds tests to the database - does not delete any. 102 Samples tests are formatted slightly differently than other tests. 103 104 @param autotest_dir: prepended to path strings (/usr/local/autotest). 105 @param add_noncompliant: attempt adding test with invalid control files. 106 @param add_experimental: add tests with experimental attribute set. 107 """ 108 sample_path = os.path.join(autotest_dir, 'server/samples') 109 if os.path.exists(sample_path): 110 logging.info("Scanning %s", sample_path) 111 tests = get_tests_from_fs(sample_path, '.*srv$', 112 add_noncompliant=add_noncompliant) 113 update_tests_in_db(tests, add_experimental=add_experimental, 114 add_noncompliant=add_noncompliant, 115 autotest_dir=autotest_dir) 116 117 118def db_clean_broken(autotest_dir): 119 """ 120 Remove tests from autotest_web that do not have valid control files 121 122 This function invoked when -c supplied on the command line and when 123 running update_all(). Removes tests from database which are not 124 found in the filesystem. Also removes profilers which are just 125 a special case of tests. 126 127 @param autotest_dir: prepended to path strings (/usr/local/autotest). 128 """ 129 for test in models.Test.objects.all(): 130 full_path = os.path.join(autotest_dir, test.path) 131 if not os.path.isfile(full_path): 132 logging.info("Removing %s", test.path) 133 _log_or_execute(repr(test), test.delete) 134 135 # Find profilers that are no longer present 136 for profiler in models.Profiler.objects.all(): 137 full_path = os.path.join(autotest_dir, "client", "profilers", 138 profiler.name) 139 if not os.path.exists(full_path): 140 logging.info("Removing %s", profiler.name) 141 _log_or_execute(repr(profiler), profiler.delete) 142 143 144def db_clean_all(autotest_dir): 145 """ 146 Remove all tests from autotest_web - very destructive 147 148 This function invoked when -C supplied on the command line. 149 Removes ALL tests from the database. 150 151 @param autotest_dir: prepended to path strings (/usr/local/autotest). 152 """ 153 for test in models.Test.objects.all(): 154 full_path = os.path.join(autotest_dir, test.path) 155 logging.info("Removing %s", test.path) 156 _log_or_execute(repr(test), test.delete) 157 158 # Find profilers that are no longer present 159 for profiler in models.Profiler.objects.all(): 160 full_path = os.path.join(autotest_dir, "client", "profilers", 161 profiler.name) 162 logging.info("Removing %s", profiler.name) 163 _log_or_execute(repr(profiler), profiler.delete) 164 165 166def update_profilers_in_db(profilers, description='NA', 167 add_noncompliant=False): 168 """ 169 Add only profilers to the database from the filesystem. 170 171 This function invoked when -p supplied on command line. 172 Only adds profilers to the database - does not delete any. 173 Profilers are formatted slightly differently than tests. 174 175 @param profilers: list of profilers found in the file system. 176 @param description: simple text to satisfy docstring. 177 @param add_noncompliant: attempt adding test with invalid control files. 178 """ 179 for profiler in profilers: 180 name = os.path.basename(profiler) 181 if name.endswith('.py'): 182 name = name[:-3] 183 if not profilers[profiler]: 184 if add_noncompliant: 185 doc = description 186 else: 187 logging.warn("Skipping %s, missing docstring", profiler) 188 continue 189 else: 190 doc = profilers[profiler] 191 192 model = models.Profiler.objects.get_or_create(name=name)[0] 193 model.description = doc 194 _log_or_execute(repr(model), model.save) 195 196 197def update_tests_in_db(tests, dry_run=False, add_experimental=False, 198 add_noncompliant=False, autotest_dir=None): 199 """ 200 Scans through all tests and add them to the database. 201 202 This function invoked when -t supplied and for update_all. 203 When test code is discovered in the file system new tests may be added 204 205 @param tests: list of tests found in the filesystem. 206 @param dry_run: not used at this time. 207 @param add_experimental: add tests with experimental attribute set. 208 @param add_noncompliant: attempt adding test with invalid control files. 209 @param autotest_dir: prepended to path strings (/usr/local/autotest). 210 """ 211 site_set_attributes_module = utils.import_site_module( 212 __file__, 'autotest_lib.utils.site_test_importer_attributes') 213 214 for test in tests: 215 new_test = models.Test.objects.get_or_create( 216 path=test.replace(autotest_dir, '').lstrip('/'))[0] 217 logging.info("Processing %s", new_test.path) 218 219 # Set the test's attributes 220 data = tests[test] 221 _set_attributes_clean(new_test, data) 222 223 # Custom Attribute Update 224 if site_set_attributes_module: 225 site_set_attributes_module._set_attributes_custom(new_test, data) 226 227 # This only takes place if --add-noncompliant is provided on the CLI 228 if not new_test.name: 229 test_new_test = test.split('/') 230 if test_new_test[-1] == 'control': 231 new_test.name = test_new_test[-2] 232 else: 233 control_name = "%s:%s" 234 control_name %= (test_new_test[-2], 235 test_new_test[-1]) 236 new_test.name = control_name.replace('control.', '') 237 238 # Experimental Check 239 if not add_experimental and new_test.experimental: 240 continue 241 242 _log_or_execute(repr(new_test), new_test.save) 243 add_label_dependencies(new_test) 244 245 246def _set_attributes_clean(test, data): 247 """ 248 First pass sets the attributes of the Test object from file system. 249 250 @param test: a test object to be populated for the database. 251 @param data: object with test data from the file system. 252 """ 253 test_type = { 'client' : 1, 254 'server' : 2, } 255 test_time = { 'short' : 1, 256 'medium' : 2, 257 'long' : 3, } 258 259 260 string_attributes = ('name', 'author', 'test_class', 'test_category', 261 'test_category', 'sync_count') 262 for attribute in string_attributes: 263 setattr(test, attribute, getattr(data, attribute)) 264 265 test.description = data.doc 266 test.dependencies = ", ".join(data.dependencies) 267 268 int_attributes = ('experimental', 'run_verify') 269 for attribute in int_attributes: 270 setattr(test, attribute, int(getattr(data, attribute))) 271 272 try: 273 test.test_type = int(data.test_type) 274 if test.test_type != 1 and test.test_type != 2: 275 raise Exception('Incorrect number %d for test_type' % 276 test.test_type) 277 except ValueError: 278 pass 279 try: 280 test.test_time = int(data.time) 281 if test.test_time < 1 or test.time > 3: 282 raise Exception('Incorrect number %d for time' % test.time) 283 except ValueError: 284 pass 285 286 if not test.test_time and str == type(data.time): 287 test.test_time = test_time[data.time.lower()] 288 if not test.test_type and str == type(data.test_type): 289 test.test_type = test_type[data.test_type.lower()] 290 291 292def add_label_dependencies(test): 293 """ 294 Add proper many-to-many relationships from DEPENDENCIES field. 295 296 @param test: test object for the database. 297 """ 298 299 # clear out old relationships 300 _log_or_execute(repr(test), test.dependency_labels.clear, 301 subject='clear dependencies from') 302 303 for label_name in test.dependencies.split(','): 304 label_name = label_name.strip().lower() 305 if not label_name: 306 continue 307 308 try: 309 label = models.Label.objects.get(name=label_name) 310 except models.Label.DoesNotExist: 311 log_dependency_not_found(label_name) 312 continue 313 314 _log_or_execute(repr(label), test.dependency_labels.add, label, 315 subject='add dependency to %s' % test.name) 316 317 318def log_dependency_not_found(label_name): 319 """ 320 Exception processing when label not found in database. 321 322 @param label_name: from test dependencies. 323 """ 324 if label_name in DEPENDENCIES_NOT_FOUND: 325 return 326 logging.info("Dependency %s not found", label_name) 327 DEPENDENCIES_NOT_FOUND.add(label_name) 328 329 330def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False): 331 """ 332 Find control files in file system and load a list with their info. 333 334 @param parent_dir: directory to search recursively. 335 @param control_pattern: name format of control file. 336 @param add_noncompliant: ignore control file parse errors. 337 338 @return dictionary of the form: tests[file_path] = parsed_object 339 """ 340 tests = {} 341 profilers = False 342 if 'client/profilers' in parent_dir: 343 profilers = True 344 for dir in [ parent_dir ]: 345 files = recursive_walk(dir, control_pattern) 346 for file in files: 347 if '__init__.py' in file or '.svn' in file: 348 continue 349 if not profilers: 350 if not add_noncompliant: 351 try: 352 found_test = control_data.parse_control(file, 353 raise_warnings=True) 354 tests[file] = found_test 355 except control_data.ControlVariableException, e: 356 logging.warn("Skipping %s\n%s", file, e) 357 except Exception, e: 358 logging.error("Bad %s\n%s", file, e) 359 else: 360 found_test = control_data.parse_control(file) 361 tests[file] = found_test 362 else: 363 tests[file] = compiler.parseFile(file).doc 364 return tests 365 366 367def recursive_walk(path, wildcard): 368 """ 369 Recursively go through a directory. 370 371 This function invoked by get_tests_from_fs(). 372 373 @param path: base directory to start search. 374 @param wildcard: name format to match. 375 376 @return A list of files that match wildcard 377 """ 378 files = [] 379 directories = [ path ] 380 while len(directories)>0: 381 directory = directories.pop() 382 for name in os.listdir(directory): 383 fullpath = os.path.join(directory, name) 384 if os.path.isfile(fullpath): 385 # if we are a control file 386 if re.search(wildcard, name): 387 files.append(fullpath) 388 elif os.path.isdir(fullpath): 389 directories.append(fullpath) 390 return files 391 392 393def _log_or_execute(content, func, *args, **kwargs): 394 """ 395 Log a message if dry_run is enabled, or execute the given function. 396 397 Relies on the DRY_RUN global variable. 398 399 @param content: the actual log message. 400 @param func: function to execute if dry_run is not enabled. 401 @param subject: (Optional) The type of log being written. Defaults to 402 the name of the provided function. 403 """ 404 subject = kwargs.get('subject', func.__name__) 405 406 if DRY_RUN: 407 logging.info("Would %s: %s", subject, content) 408 else: 409 func(*args) 410 411 412def _create_whitelist_set(whitelist_path): 413 """ 414 Create a set with contents from a whitelist file for membership testing. 415 416 @param whitelist_path: full path to the whitelist file. 417 418 @return set with files listed one/line - newlines included. 419 """ 420 f = open(whitelist_path, 'r') 421 whitelist_set = set([line.strip() for line in f]) 422 f.close() 423 return whitelist_set 424 425 426def update_from_whitelist(whitelist_set, add_experimental, add_noncompliant, 427 autotest_dir): 428 """ 429 Scans through all tests in the whitelist and add them to the database. 430 431 This function invoked when -w supplied. 432 433 @param whitelist_set: set of tests in full-path form from a whitelist. 434 @param add_experimental: add tests with experimental attribute set. 435 @param add_noncompliant: attempt adding test with invalid control files. 436 @param autotest_dir: prepended to path strings (/usr/local/autotest). 437 """ 438 tests = {} 439 profilers = {} 440 for file_path in whitelist_set: 441 if file_path.find('client/profilers') == -1: 442 try: 443 found_test = control_data.parse_control(file_path, 444 raise_warnings=True) 445 tests[file_path] = found_test 446 except control_data.ControlVariableException, e: 447 logging.warn("Skipping %s\n%s", file, e) 448 else: 449 profilers[file_path] = compiler.parseFile(file_path).doc 450 451 if len(tests) > 0: 452 update_tests_in_db(tests, add_experimental=add_experimental, 453 add_noncompliant=add_noncompliant, 454 autotest_dir=autotest_dir) 455 if len(profilers) > 0: 456 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant, 457 description='NA') 458 459 460def main(argv): 461 """Main function""" 462 463 global DRY_RUN 464 parser = optparse.OptionParser() 465 parser.add_option('-c', '--db-clean-tests', 466 dest='clean_tests', action='store_true', 467 default=False, 468 help='Clean client and server tests with invalid control files') 469 parser.add_option('-C', '--db-clear-all-tests', 470 dest='clear_all_tests', action='store_true', 471 default=False, 472 help='Clear ALL client and server tests') 473 parser.add_option('-d', '--dry-run', 474 dest='dry_run', action='store_true', default=False, 475 help='Dry run for operation') 476 parser.add_option('-A', '--add-all', 477 dest='add_all', action='store_true', 478 default=False, 479 help='Add site_tests, tests, and test_suites') 480 parser.add_option('-S', '--add-samples', 481 dest='add_samples', action='store_true', 482 default=False, 483 help='Add samples.') 484 parser.add_option('-E', '--add-experimental', 485 dest='add_experimental', action='store_true', 486 default=True, 487 help='Add experimental tests to frontend') 488 parser.add_option('-N', '--add-noncompliant', 489 dest='add_noncompliant', action='store_true', 490 default=False, 491 help='Add non-compliant tests (i.e. tests that do not ' 492 'define all required control variables)') 493 parser.add_option('-p', '--profile-dir', dest='profile_dir', 494 help='Directory to recursively check for profiles') 495 parser.add_option('-t', '--tests-dir', dest='tests_dir', 496 help='Directory to recursively check for control.*') 497 parser.add_option('-r', '--control-pattern', dest='control_pattern', 498 default='^control.*', 499 help='The pattern to look for in directories for control files') 500 parser.add_option('-v', '--verbose', 501 dest='verbose', action='store_true', default=False, 502 help='Run in verbose mode') 503 parser.add_option('-w', '--whitelist-file', dest='whitelist_file', 504 help='Filename for list of test names that must match') 505 parser.add_option('-z', '--autotest-dir', dest='autotest_dir', 506 default=os.path.join(os.path.dirname(__file__), '..'), 507 help='Autotest directory root') 508 options, args = parser.parse_args() 509 510 logging_manager.configure_logging(TestImporterLoggingConfig(), 511 verbose=options.verbose) 512 513 DRY_RUN = options.dry_run 514 if DRY_RUN: 515 logging.getLogger().setLevel(logging.WARN) 516 517 # Make sure autotest_dir is the absolute path 518 options.autotest_dir = os.path.abspath(options.autotest_dir) 519 520 if len(args) > 0: 521 logging.error("Invalid option(s) provided: %s", args) 522 parser.print_help() 523 return 1 524 525 if options.verbose: 526 logging.getLogger().setLevel(logging.DEBUG) 527 528 if len(argv) == 1 or (len(argv) == 2 and options.verbose): 529 update_all(options.autotest_dir, options.add_noncompliant, 530 options.add_experimental) 531 db_clean_broken(options.autotest_dir) 532 return 0 533 534 if options.clear_all_tests: 535 if (options.clean_tests or options.add_all or options.add_samples or 536 options.add_noncompliant): 537 logging.error( 538 "Can only pass --autotest-dir, --dry-run and --verbose with " 539 "--db-clear-all-tests") 540 return 1 541 db_clean_all(options.autotest_dir) 542 543 whitelist_set = None 544 if options.whitelist_file: 545 if options.add_all: 546 logging.error("Cannot pass both --add-all and --whitelist-file") 547 return 1 548 whitelist_path = os.path.abspath(options.whitelist_file) 549 if not os.path.isfile(whitelist_path): 550 logging.error("--whitelist-file (%s) not found", whitelist_path) 551 return 1 552 logging.info("Using whitelist file %s", whitelist_path) 553 whitelist_set = _create_whitelist_set(whitelist_path) 554 update_from_whitelist(whitelist_set, 555 add_experimental=options.add_experimental, 556 add_noncompliant=options.add_noncompliant, 557 autotest_dir=options.autotest_dir) 558 if options.add_all: 559 update_all(options.autotest_dir, options.add_noncompliant, 560 options.add_experimental) 561 if options.add_samples: 562 update_samples(options.autotest_dir, options.add_noncompliant, 563 options.add_experimental) 564 if options.tests_dir: 565 options.tests_dir = os.path.abspath(options.tests_dir) 566 tests = get_tests_from_fs(options.tests_dir, options.control_pattern, 567 add_noncompliant=options.add_noncompliant) 568 update_tests_in_db(tests, add_experimental=options.add_experimental, 569 add_noncompliant=options.add_noncompliant, 570 autotest_dir=options.autotest_dir) 571 if options.profile_dir: 572 profilers = get_tests_from_fs(options.profile_dir, '.*py$') 573 update_profilers_in_db(profilers, 574 add_noncompliant=options.add_noncompliant, 575 description='NA') 576 if options.clean_tests: 577 db_clean_broken(options.autotest_dir) 578 579 580if __name__ == "__main__": 581 main(sys.argv) 582