test_importer.py revision 68558223091c765d0cef92438f09c3f9918fa3fc 
1#!/usr/bin/python
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4"""
5This utility allows for easy updating, removing and importing
6of tests into the autotest_web autotests table.
7
8Example of updating client side tests:
9./tests.py -t /usr/local/autotest/client/tests
10
11If for example not all of your control files adhere to the standard outlined at
12http://test.kernel.org/autotest/ControlRequirements
13
14You can force options:
15./tests.py --test-type server -t /usr/local/autotest/server/tests
16
17
18Most options should be fairly self explanatory use --help to display them.
19"""
20
21
22import logging, time, re, os, MySQLdb, sys, optparse, compiler
23import common
24from autotest_lib.client.common_lib import control_data, test, global_config
25from autotest_lib.client.common_lib import utils
26
27
28logging.basicConfig(logging.DEBUG)
29# Global
30DRY_RUN = False
31DEPENDENCIES_NOT_FOUND = set()
32
33def main(argv):
34    """Main function"""
35    global DRY_RUN
36    parser = optparse.OptionParser()
37    parser.add_option('-c', '--db-clear-tests',
38                      dest='clear_tests', action='store_true',
39                      default=False,
40                help='Clear client and server tests with invalid control files')
41    parser.add_option('-d', '--dry-run',
42                      dest='dry_run', action='store_true', default=False,
43                      help='Dry run for operation')
44    parser.add_option('-A', '--add-all',
45                      dest='add_all', action='store_true',
46                      default=False,
47                      help='Add samples, site_tests, tests, and test_suites')
48    parser.add_option('-E', '--add-experimental',
49                      dest='add_experimental', action='store_true',
50                      default=True,
51                      help='Add experimental tests to frontend')
52    parser.add_option('-N', '--add-noncompliant',
53                      dest='add_noncompliant', action='store_true',
54                      default=False,
55                      help='Skip any tests that are not compliant')
56    parser.add_option('-p', '--profile-dir', dest='profile_dir',
57                      help='Directory to recursively check for profiles')
58    parser.add_option('-t', '--tests-dir', dest='tests_dir',
59                      help='Directory to recursively check for control.*')
60    parser.add_option('-r', '--control-pattern', dest='control_pattern',
61                      default='^control.*',
62               help='The pattern to look for in directories for control files')
63    parser.add_option('-v', '--verbose',
64                      dest='verbose', action='store_true', default=False,
65                      help='Run in verbose mode')
66    parser.add_option('-z', '--autotest_dir', dest='autotest_dir',
67                      default=os.path.join(os.path.dirname(__file__), '..'),
68                      help='Autotest directory root')
69    options, args = parser.parse_args()
70    DRY_RUN = options.dry_run
71    # Make sure autotest_dir is the absolute path
72    options.autotest_dir = os.path.abspath(options.autotest_dir)
73
74    if len(args) > 0:
75        print "Invalid option(s) provided: ", args
76        parser.print_help()
77        return 1
78
79    if len(argv) == 1:
80        update_all(options.autotest_dir, options.add_noncompliant,
81                   options.add_experimental, options.verbose)
82        db_clean_broken(options.autotest_dir, options.verbose)
83        return 0
84
85    if options.add_all:
86        update_all(options.autotest_dir, options.add_noncompliant,
87                   options.add_experimental, options.verbose)
88    if options.clear_tests:
89        db_clean_broken(options.autotest_dir, options.verbose)
90    if options.tests_dir:
91        if ".." in options.tests_dir:
92            path = os.path.join(os.getcwd(), options.tests_dir)
93            options.tests_dir = os.path.abspath(path)
94        tests = get_tests_from_fs(options.tests_dir, options.control_pattern,
95                                  add_noncompliant=options.add_noncompliant)
96        update_tests_in_db(tests, add_experimental=options.add_experimental,
97                           add_noncompliant=options.add_noncompliant,
98                           autotest_dir=options.autotest_dir,
99                           verbose=options.verbose)
100    if options.profile_dir:
101        profilers = get_tests_from_fs(options.profile_dir, '.*py$')
102        update_profilers_in_db(profilers, verbose=options.verbose,
103                               add_noncompliant=options.add_noncompliant,
104                               description='NA')
105
106
107def update_all(autotest_dir, add_noncompliant, add_experimental, verbose):
108    """Function to scan through all tests and add them to the database."""
109    for path in [ 'server/tests', 'server/site_tests', 'client/tests',
110                  'client/site_tests']:
111        test_path = os.path.join(autotest_dir, path)
112        if not os.path.exists(test_path):
113            continue
114        if verbose:
115            print "Scanning " + test_path
116        tests = []
117        tests = get_tests_from_fs(test_path, "^control.*",
118                                 add_noncompliant=add_noncompliant)
119        update_tests_in_db(tests, add_experimental=add_experimental,
120                           add_noncompliant=add_noncompliant,
121                           autotest_dir=autotest_dir,
122                           verbose=verbose)
123    test_suite_path = os.path.join(autotest_dir, 'test_suites')
124    if os.path.exists(test_suite_path):
125        if verbose:
126            print "Scanning " + test_suite_path
127        tests = get_tests_from_fs(test_suite_path, '.*',
128                                 add_noncompliant=add_noncompliant)
129        update_tests_in_db(tests, add_experimental=add_experimental,
130                           add_noncompliant=add_noncompliant,
131                           autotest_dir=autotest_dir,
132                           verbose=verbose)
133    sample_path = os.path.join(autotest_dir, 'server/samples')
134    if os.path.exists(sample_path):
135        if verbose:
136            print "Scanning " + sample_path
137        tests = get_tests_from_fs(sample_path, '.*srv$',
138                                 add_noncompliant=add_noncompliant)
139        update_tests_in_db(tests, add_experimental=add_experimental,
140                           add_noncompliant=add_noncompliant,
141                           autotest_dir=autotest_dir,
142                           verbose=verbose)
143
144    profilers_path = os.path.join(autotest_dir, "client/profilers")
145    if os.path.exists(profilers_path):
146        if verbose:
147            print "Scanning " + profilers_path
148        profilers = get_tests_from_fs(profilers_path, '.*py$')
149        update_profilers_in_db(profilers, verbose=verbose,
150                               add_noncompliant=add_noncompliant,
151                               description='NA')
152    # Clean bad db entries
153    db_clean_broken(autotest_dir, verbose)
154
155
156def db_clean_broken(autotest_dir, verbose):
157    """Remove tests from autotest_web that do not have valid control files
158
159       Arguments:
160        tests: a list of control file relative paths used as keys for deletion.
161    """
162    connection=db_connect()
163    cursor = connection.cursor()
164    # Get tests
165    sql = "SELECT id, path FROM autotests";
166    cursor.execute(sql)
167    results = cursor.fetchall()
168    for test_id, path in results:
169        full_path = os.path.join(autotest_dir, path)
170        if not os.path.isfile(full_path):
171            if verbose:
172                print "Removing " + path
173            db_execute(cursor, "DELETE FROM autotests WHERE id=%s" % test_id)
174            db_execute(cursor, "DELETE FROM autotests_dependency_labels WHERE "
175                               "test_id=%s" % test_id)
176
177    # Find profilers that are no longer present
178    profilers = []
179    sql = "SELECT name FROM profilers"
180    cursor.execute(sql)
181    results = cursor.fetchall()
182    for path in results:
183        full_path = os.path.join(autotest_dir, "client/profilers", path[0])
184        if not os.path.exists(full_path):
185            if verbose:
186                print "Removing " + path[0]
187            sql = "DELETE FROM profilers WHERE name='%s'" % path[0]
188            db_execute(cursor, sql)
189
190
191    connection.commit()
192    connection.close()
193
194
195def update_profilers_in_db(profilers, verbose=False, description='NA',
196                           add_noncompliant=False):
197    """Update profilers in autotest_web database"""
198    connection=db_connect()
199    cursor = connection.cursor()
200    for profiler in profilers:
201        name = os.path.basename(profiler).rstrip(".py")
202        if not profilers[profiler]:
203            if add_noncompliant:
204                doc = description
205            else:
206                print "Skipping %s, missing docstring" % profiler
207        else:
208            doc = profilers[profiler]
209        # check if test exists
210        sql = "SELECT name FROM profilers WHERE name='%s'" % name
211        cursor.execute(sql)
212        results = cursor.fetchall()
213        if results:
214            sql = "UPDATE profilers SET name='%s', description='%s' "\
215                  "WHERE name='%s'"
216            sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc),
217                    MySQLdb.escape_string(name))
218        else:
219            # Insert newly into DB
220            sql = "INSERT into profilers (name, description) VALUES('%s', '%s')"
221            sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc))
222
223        db_execute(cursor, sql)
224
225    connection.commit()
226    connection.close()
227
228
229def update_tests_in_db(tests, dry_run=False, add_experimental=False,
230                       add_noncompliant=False, verbose=False,
231                       autotest_dir=None):
232    """Update or add each test to the database"""
233    connection=db_connect()
234    cursor = connection.cursor()
235    new_test_dicts = []
236    for test in tests:
237        new_test = {}
238        new_test['path'] = test.replace(autotest_dir, '').lstrip('/')
239        if verbose:
240            print "Processing " + new_test['path']
241        # Create a name for the test
242        for key in dir(tests[test]):
243            if not key.startswith('__'):
244                value = getattr(tests[test], key)
245                if not callable(value):
246                    new_test[key] = value
247        # This only takes place if --add-noncompliant is provided on the CLI
248        if 'name' not in new_test:
249            test_new_test = test.split('/')
250            if test_new_test[-1] == 'control':
251                new_test['name'] = test_new_test[-2]
252            else:
253                control_name = "%s:%s"
254                control_name %= (test_new_test[-2],
255                                 test_new_test[-1])
256                new_test['name'] = control_name.replace('control.', '')
257        # Experimental Check
258        if not add_experimental:
259            if int(new_test['experimental']):
260                continue
261        # clean tests for insertion into db
262        new_test = dict_db_clean(new_test)
263        new_test_dicts.append(new_test)
264        sql = "SELECT name,path FROM autotests WHERE path='%s' LIMIT 1"
265        sql %= new_test['path']
266        cursor.execute(sql)
267        # check for entries already in existence
268        results = cursor.fetchall()
269        if results:
270            sql = ("UPDATE autotests SET name='%s', test_class='%s',"
271                  "description='%s', test_type=%d, path='%s',"
272                  "author='%s', dependencies='%s',"
273                  "experimental=%d, run_verify=%d, test_time=%d,"
274                  "test_category='%s', sync_count=%d"
275                  " WHERE path='%s'")
276            sql %= (new_test['name'], new_test['test_class'], new_test['doc'],
277                    int(new_test['test_type']), new_test['path'],
278                    new_test['author'],
279                    new_test['dependencies'], int(new_test['experimental']),
280                    int(new_test['run_verify']), new_test['time'],
281                    new_test['test_category'], new_test['sync_count'],
282                    new_test['path'])
283        else:
284            # Create a relative path
285            path = test.replace(autotest_dir, '')
286            sql = ("INSERT INTO autotests"
287                  "(name, test_class, description, test_type, path, "
288                  "author, dependencies, experimental, "
289                  "run_verify, test_time, test_category, sync_count) "
290                  "VALUES('%s','%s','%s',%d,'%s','%s','%s',%d,%d,%d,"
291                  "'%s',%d)")
292            sql %= (new_test['name'], new_test['test_class'], new_test['doc'],
293                    int(new_test['test_type']), new_test['path'],
294                    new_test['author'], new_test['dependencies'],
295                    int(new_test['experimental']), int(new_test['run_verify']),
296                    new_test['time'], new_test['test_category'],
297                    new_test['sync_count'])
298
299        db_execute(cursor, sql)
300
301    add_label_dependencies(new_test_dicts, cursor)
302
303    connection.commit()
304    connection.close()
305
306
307def dict_db_clean(test):
308    """Take a tests dictionary from update_db and make it pretty for SQL"""
309
310    test_type = { 'client' : 1,
311                  'server' : 2, }
312    test_time = { 'short' : 1,
313                  'medium' : 2,
314                  'long' : 3, }
315
316    test['name'] = MySQLdb.escape_string(test['name'])
317    test['author'] = MySQLdb.escape_string(test['author'])
318    test['test_class'] = MySQLdb.escape_string(test['test_class'])
319    test['test_category'] = MySQLdb.escape_string(test['test_category'])
320    test['doc'] = MySQLdb.escape_string(test['doc'])
321    test['dependencies'] = ", ".join(test['dependencies'])
322    # TODO Fix when we move from synch_type to sync_count
323    if test['sync_count'] == 1:
324        test['synch_type'] = 1
325    else:
326        test['synch_type'] = 2
327    try:
328        test['test_type'] = int(test['test_type'])
329        if test['test_type'] != 1 and test['test_type'] != 2:
330            raise Exception('Incorrect number %d for test_type' %
331                            test['test_type'])
332    except ValueError:
333        pass
334    try:
335        test['time'] = int(test['time'])
336        if test['time'] < 1 or test['time'] > 3:
337            raise Exception('Incorrect number %d for time' %
338                            test['time'])
339    except ValueError:
340        pass
341
342    if str == type(test['time']):
343        test['time'] = test_time[test['time'].lower()]
344    if str == type(test['test_type']):
345        test['test_type'] = test_type[test['test_type'].lower()]
346
347    return test
348
349
350def add_label_dependencies(tests, cursor):
351    """
352    Look at the DEPENDENCIES field for each test and add the proper many-to-many
353    relationships.
354    """
355    label_name_to_id = get_id_map(cursor, 'labels', 'name')
356    test_path_to_id = get_id_map(cursor, 'autotests', 'path')
357
358    # clear out old relationships
359    test_ids = ','.join(str(test_path_to_id[test['path']])
360                        for test in tests)
361    db_execute(cursor,
362               'DELETE FROM autotests_dependency_labels WHERE test_id IN (%s)' %
363               test_ids)
364
365    value_pairs = []
366    for test in tests:
367        test_id = test_path_to_id[test['path']]
368        for label_name in test['dependencies'].split(','):
369            label_name = label_name.strip().lower()
370            if not label_name:
371                continue
372            if label_name not in label_name_to_id:
373                log_dependency_not_found(label_name)
374                continue
375            label_id = label_name_to_id[label_name]
376            value_pairs.append('(%s, %s)' % (test_id, label_id))
377
378    if not value_pairs:
379        return
380
381    query = ('INSERT INTO autotests_dependency_labels (test_id, label_id) '
382             'VALUES ' + ','.join(value_pairs))
383    db_execute(cursor, query)
384
385
386def log_dependency_not_found(label_name):
387    if label_name in DEPENDENCIES_NOT_FOUND:
388        return
389    print 'Dependency %s not found' % label_name
390    DEPENDENCIES_NOT_FOUND.add(label_name)
391
392
393def get_id_map(cursor, table_name, name_field):
394    cursor.execute('SELECT id, %s FROM %s' % (name_field, table_name))
395    name_to_id = {}
396    for item_id, item_name in cursor.fetchall():
397        name_to_id[item_name] = item_id
398    return name_to_id
399
400
401def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False):
402    """Find control jobs in location and create one big job
403       Returns:
404        dictionary of the form:
405            tests[file_path] = parsed_object
406
407    """
408    tests = {}
409    profilers = False
410    if 'client/profilers' in parent_dir:
411        profilers = True
412    for dir in [ parent_dir ]:
413        files = recursive_walk(dir, control_pattern)
414        for file in files:
415            if '__init__.py' in file or '.svn' in file:
416                continue
417            if not profilers:
418                if not add_noncompliant:
419                    try:
420                        found_test = control_data.parse_control(file,
421                                                            raise_warnings=True)
422                        tests[file] = found_test
423                    except control_data.ControlVariableException, e:
424                        print "Skipping %s\n%s" % (file, e)
425                        pass
426                else:
427                    found_test = control_data.parse_control(file)
428                    tests[file] = found_test
429            else:
430                script = file.rstrip(".py")
431                tests[file] = compiler.parseFile(file).doc
432    return tests
433
434
435def recursive_walk(path, wildcard):
436    """Recurisvely go through a directory.
437        Returns:
438        A list of files that match wildcard
439    """
440    files = []
441    directories = [ path ]
442    while len(directories)>0:
443        directory = directories.pop()
444        for name in os.listdir(directory):
445            fullpath = os.path.join(directory, name)
446            if os.path.isfile(fullpath):
447                # if we are a control file
448                if re.search(wildcard, name):
449                    files.append(fullpath)
450            elif os.path.isdir(fullpath):
451                directories.append(fullpath)
452    return files
453
454
455def db_connect():
456    """Connect to the AUTOTEST_WEB database and return a connect object."""
457    c = global_config.global_config
458    db_host = c.get_config_value('AUTOTEST_WEB', 'host')
459    db_name = c.get_config_value('AUTOTEST_WEB', 'database')
460    username = c.get_config_value('AUTOTEST_WEB', 'user')
461    password = c.get_config_value('AUTOTEST_WEB', 'password')
462    connection = MySQLdb.connect(host=db_host, db=db_name,
463                                 user=username,
464                                 passwd=password)
465    return connection
466
467
468def db_execute(cursor, sql):
469    """Execute SQL or print out what would be executed if dry_run is defined"""
470
471    if DRY_RUN:
472        print "Would run: " + sql
473    else:
474        cursor.execute(sql)
475
476
477if __name__ == "__main__":
478    main(sys.argv)
479