test_importer.py revision 4b5c31e1821307b880b57a41f6d8ea21948fc9e8 
1#!/usr/bin/python
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4"""
5This utility allows for easy updating, removing and importing
6of tests into the autotest_web autotests table.
7
8Example of updating client side tests:
9./test_importer.py -t /usr/local/autotest/client/tests
10
11If, for example, not all of your control files adhere to the standard outlined
12at http://autotest.kernel.org/wiki/ControlRequirements, you can force options:
13
14./test_importer.py --test-type server -t /usr/local/autotest/server/tests
15
16You would need to pass --add-noncompliant to include such control files,
17however.  An easy way to check for compliance is to run in dry mode:
18
19./test_importer.py --dry-run -t /usr/local/autotest/server/tests/mytest
20
21Or to check a single control file, you can use check_control_file_vars.py.
22
23Running with no options is equivalent to --add-all --db-clear-tests.
24
25Most options should be fairly self explanatory, use --help to display them.
26"""
27
28
29import logging, time, re, os, MySQLdb, sys, optparse, compiler
30import common
31from autotest_lib.client.common_lib import control_data, test, global_config
32from autotest_lib.client.common_lib import utils
33
34
35logging.basicConfig(level=logging.DEBUG)
36# Global
37DRY_RUN = False
38DEPENDENCIES_NOT_FOUND = set()
39
40def main(argv):
41    """Main function"""
42    global DRY_RUN
43    parser = optparse.OptionParser()
44    parser.add_option('-c', '--db-clear-tests',
45                      dest='clear_tests', action='store_true',
46                      default=False,
47                help='Clear client and server tests with invalid control files')
48    parser.add_option('-d', '--dry-run',
49                      dest='dry_run', action='store_true', default=False,
50                      help='Dry run for operation')
51    parser.add_option('-A', '--add-all',
52                      dest='add_all', action='store_true',
53                      default=False,
54                      help='Add samples, site_tests, tests, and test_suites')
55    parser.add_option('-E', '--add-experimental',
56                      dest='add_experimental', action='store_true',
57                      default=True,
58                      help='Add experimental tests to frontend')
59    parser.add_option('-N', '--add-noncompliant',
60                      dest='add_noncompliant', action='store_true',
61                      default=False,
62                      help='Add non-compliant tests (i.e. tests that do not '
63                           'define all required control variables)')
64    parser.add_option('-p', '--profile-dir', dest='profile_dir',
65                      help='Directory to recursively check for profiles')
66    parser.add_option('-t', '--tests-dir', dest='tests_dir',
67                      help='Directory to recursively check for control.*')
68    parser.add_option('-r', '--control-pattern', dest='control_pattern',
69                      default='^control.*',
70               help='The pattern to look for in directories for control files')
71    parser.add_option('-v', '--verbose',
72                      dest='verbose', action='store_true', default=False,
73                      help='Run in verbose mode')
74    parser.add_option('-z', '--autotest_dir', dest='autotest_dir',
75                      default=os.path.join(os.path.dirname(__file__), '..'),
76                      help='Autotest directory root')
77    options, args = parser.parse_args()
78    DRY_RUN = options.dry_run
79    # Make sure autotest_dir is the absolute path
80    options.autotest_dir = os.path.abspath(options.autotest_dir)
81
82    if len(args) > 0:
83        print "Invalid option(s) provided: ", args
84        parser.print_help()
85        return 1
86
87    if len(argv) == 1:
88        update_all(options.autotest_dir, options.add_noncompliant,
89                   options.add_experimental, options.verbose)
90        db_clean_broken(options.autotest_dir, options.verbose)
91        return 0
92
93    if options.add_all:
94        update_all(options.autotest_dir, options.add_noncompliant,
95                   options.add_experimental, options.verbose)
96    if options.clear_tests:
97        db_clean_broken(options.autotest_dir, options.verbose)
98    if options.tests_dir:
99        options.tests_dir = os.path.abspath(options.tests_dir)
100        tests = get_tests_from_fs(options.tests_dir, options.control_pattern,
101                                  add_noncompliant=options.add_noncompliant)
102        update_tests_in_db(tests, add_experimental=options.add_experimental,
103                           add_noncompliant=options.add_noncompliant,
104                           autotest_dir=options.autotest_dir,
105                           verbose=options.verbose)
106    if options.profile_dir:
107        profilers = get_tests_from_fs(options.profile_dir, '.*py$')
108        update_profilers_in_db(profilers, verbose=options.verbose,
109                               add_noncompliant=options.add_noncompliant,
110                               description='NA')
111
112
113def update_all(autotest_dir, add_noncompliant, add_experimental, verbose):
114    """Function to scan through all tests and add them to the database."""
115    for path in [ 'server/tests', 'server/site_tests', 'client/tests',
116                  'client/site_tests']:
117        test_path = os.path.join(autotest_dir, path)
118        if not os.path.exists(test_path):
119            continue
120        if verbose:
121            print "Scanning " + test_path
122        tests = []
123        tests = get_tests_from_fs(test_path, "^control.*",
124                                 add_noncompliant=add_noncompliant)
125        update_tests_in_db(tests, add_experimental=add_experimental,
126                           add_noncompliant=add_noncompliant,
127                           autotest_dir=autotest_dir,
128                           verbose=verbose)
129    test_suite_path = os.path.join(autotest_dir, 'test_suites')
130    if os.path.exists(test_suite_path):
131        if verbose:
132            print "Scanning " + test_suite_path
133        tests = get_tests_from_fs(test_suite_path, '.*',
134                                 add_noncompliant=add_noncompliant)
135        update_tests_in_db(tests, add_experimental=add_experimental,
136                           add_noncompliant=add_noncompliant,
137                           autotest_dir=autotest_dir,
138                           verbose=verbose)
139    sample_path = os.path.join(autotest_dir, 'server/samples')
140    if os.path.exists(sample_path):
141        if verbose:
142            print "Scanning " + sample_path
143        tests = get_tests_from_fs(sample_path, '.*srv$',
144                                 add_noncompliant=add_noncompliant)
145        update_tests_in_db(tests, add_experimental=add_experimental,
146                           add_noncompliant=add_noncompliant,
147                           autotest_dir=autotest_dir,
148                           verbose=verbose)
149
150    profilers_path = os.path.join(autotest_dir, "client/profilers")
151    if os.path.exists(profilers_path):
152        if verbose:
153            print "Scanning " + profilers_path
154        profilers = get_tests_from_fs(profilers_path, '.*py$')
155        update_profilers_in_db(profilers, verbose=verbose,
156                               add_noncompliant=add_noncompliant,
157                               description='NA')
158    # Clean bad db entries
159    db_clean_broken(autotest_dir, verbose)
160
161
162def db_clean_broken(autotest_dir, verbose):
163    """Remove tests from autotest_web that do not have valid control files
164
165       Arguments:
166        tests: a list of control file relative paths used as keys for deletion.
167    """
168    connection=db_connect()
169    cursor = connection.cursor()
170    # Get tests
171    sql = "SELECT id, path FROM autotests";
172    cursor.execute(sql)
173    results = cursor.fetchall()
174    for test_id, path in results:
175        full_path = os.path.join(autotest_dir, path)
176        if not os.path.isfile(full_path):
177            if verbose:
178                print "Removing " + path
179            db_execute(cursor, "DELETE FROM autotests WHERE id=%s" % test_id)
180            db_execute(cursor, "DELETE FROM autotests_dependency_labels WHERE "
181                               "test_id=%s" % test_id)
182
183    # Find profilers that are no longer present
184    profilers = []
185    sql = "SELECT name FROM profilers"
186    cursor.execute(sql)
187    results = cursor.fetchall()
188    for path in results:
189        full_path = os.path.join(autotest_dir, "client/profilers", path[0])
190        if not os.path.exists(full_path):
191            if verbose:
192                print "Removing " + path[0]
193            sql = "DELETE FROM profilers WHERE name='%s'" % path[0]
194            db_execute(cursor, sql)
195
196
197    connection.commit()
198    connection.close()
199
200
201def update_profilers_in_db(profilers, verbose=False, description='NA',
202                           add_noncompliant=False):
203    """Update profilers in autotest_web database"""
204    connection=db_connect()
205    cursor = connection.cursor()
206    for profiler in profilers:
207        name = os.path.basename(profiler).rstrip(".py")
208        if not profilers[profiler]:
209            if add_noncompliant:
210                doc = description
211            else:
212                print "Skipping %s, missing docstring" % profiler
213        else:
214            doc = profilers[profiler]
215        # check if test exists
216        sql = "SELECT name FROM profilers WHERE name='%s'" % name
217        cursor.execute(sql)
218        results = cursor.fetchall()
219        if results:
220            sql = "UPDATE profilers SET name='%s', description='%s' "\
221                  "WHERE name='%s'"
222            sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc),
223                    MySQLdb.escape_string(name))
224        else:
225            # Insert newly into DB
226            sql = "INSERT into profilers (name, description) VALUES('%s', '%s')"
227            sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc))
228
229        db_execute(cursor, sql)
230
231    connection.commit()
232    connection.close()
233
234
235def update_tests_in_db(tests, dry_run=False, add_experimental=False,
236                       add_noncompliant=False, verbose=False,
237                       autotest_dir=None):
238    """Update or add each test to the database"""
239    connection=db_connect()
240    cursor = connection.cursor()
241    new_test_dicts = []
242    for test in tests:
243        new_test = {}
244        new_test['path'] = test.replace(autotest_dir, '').lstrip('/')
245        if verbose:
246            print "Processing " + new_test['path']
247        # Create a name for the test
248        for key in dir(tests[test]):
249            if not key.startswith('__'):
250                value = getattr(tests[test], key)
251                if not callable(value):
252                    new_test[key] = value
253        # This only takes place if --add-noncompliant is provided on the CLI
254        if 'name' not in new_test:
255            test_new_test = test.split('/')
256            if test_new_test[-1] == 'control':
257                new_test['name'] = test_new_test[-2]
258            else:
259                control_name = "%s:%s"
260                control_name %= (test_new_test[-2],
261                                 test_new_test[-1])
262                new_test['name'] = control_name.replace('control.', '')
263        # Experimental Check
264        if not add_experimental:
265            if int(new_test['experimental']):
266                continue
267        # clean tests for insertion into db
268        new_test = dict_db_clean(new_test)
269        new_test_dicts.append(new_test)
270        sql = "SELECT name,path FROM autotests WHERE path='%s' LIMIT 1"
271        sql %= new_test['path']
272        cursor.execute(sql)
273        # check for entries already in existence
274        results = cursor.fetchall()
275        if results:
276            sql = ("UPDATE autotests SET name='%s', test_class='%s',"
277                  "description='%s', test_type=%d, path='%s',"
278                  "author='%s', dependencies='%s',"
279                  "experimental=%d, run_verify=%d, test_time=%d,"
280                  "test_category='%s', sync_count=%d"
281                  " WHERE path='%s'")
282            sql %= (new_test['name'], new_test['test_class'], new_test['doc'],
283                    int(new_test['test_type']), new_test['path'],
284                    new_test['author'],
285                    new_test['dependencies'], int(new_test['experimental']),
286                    int(new_test['run_verify']), new_test['time'],
287                    new_test['test_category'], new_test['sync_count'],
288                    new_test['path'])
289        else:
290            # Create a relative path
291            path = test.replace(autotest_dir, '')
292            sql = ("INSERT INTO autotests"
293                  "(name, test_class, description, test_type, path, "
294                  "author, dependencies, experimental, "
295                  "run_verify, test_time, test_category, sync_count) "
296                  "VALUES('%s','%s','%s',%d,'%s','%s','%s',%d,%d,%d,"
297                  "'%s',%d)")
298            sql %= (new_test['name'], new_test['test_class'], new_test['doc'],
299                    int(new_test['test_type']), new_test['path'],
300                    new_test['author'], new_test['dependencies'],
301                    int(new_test['experimental']), int(new_test['run_verify']),
302                    new_test['time'], new_test['test_category'],
303                    new_test['sync_count'])
304
305        db_execute(cursor, sql)
306
307    add_label_dependencies(new_test_dicts, cursor)
308
309    connection.commit()
310    connection.close()
311
312
313def dict_db_clean(test):
314    """Take a tests dictionary from update_db and make it pretty for SQL"""
315
316    test_type = { 'client' : 1,
317                  'server' : 2, }
318    test_time = { 'short' : 1,
319                  'medium' : 2,
320                  'long' : 3, }
321
322    test['name'] = MySQLdb.escape_string(test['name'])
323    test['author'] = MySQLdb.escape_string(test['author'])
324    test['test_class'] = MySQLdb.escape_string(test['test_class'])
325    test['test_category'] = MySQLdb.escape_string(test['test_category'])
326    test['doc'] = MySQLdb.escape_string(test['doc'])
327    test['dependencies'] = ", ".join(test['dependencies'])
328    # TODO Fix when we move from synch_type to sync_count
329    if test['sync_count'] == 1:
330        test['synch_type'] = 1
331    else:
332        test['synch_type'] = 2
333    try:
334        test['test_type'] = int(test['test_type'])
335        if test['test_type'] != 1 and test['test_type'] != 2:
336            raise Exception('Incorrect number %d for test_type' %
337                            test['test_type'])
338    except ValueError:
339        pass
340    try:
341        test['time'] = int(test['time'])
342        if test['time'] < 1 or test['time'] > 3:
343            raise Exception('Incorrect number %d for time' %
344                            test['time'])
345    except ValueError:
346        pass
347
348    if str == type(test['time']):
349        test['time'] = test_time[test['time'].lower()]
350    if str == type(test['test_type']):
351        test['test_type'] = test_type[test['test_type'].lower()]
352
353    return test
354
355
356def add_label_dependencies(tests, cursor):
357    """
358    Look at the DEPENDENCIES field for each test and add the proper many-to-many
359    relationships.
360    """
361    label_name_to_id = get_id_map(cursor, 'labels', 'name')
362    test_path_to_id = get_id_map(cursor, 'autotests', 'path')
363
364    # clear out old relationships
365    test_ids = ','.join(str(test_path_to_id[test['path']])
366                        for test in tests)
367    db_execute(cursor,
368               'DELETE FROM autotests_dependency_labels WHERE test_id IN (%s)' %
369               test_ids)
370
371    value_pairs = []
372    for test in tests:
373        test_id = test_path_to_id[test['path']]
374        for label_name in test['dependencies'].split(','):
375            label_name = label_name.strip().lower()
376            if not label_name:
377                continue
378            if label_name not in label_name_to_id:
379                log_dependency_not_found(label_name)
380                continue
381            label_id = label_name_to_id[label_name]
382            value_pairs.append('(%s, %s)' % (test_id, label_id))
383
384    if not value_pairs:
385        return
386
387    query = ('INSERT INTO autotests_dependency_labels (test_id, label_id) '
388             'VALUES ' + ','.join(value_pairs))
389    db_execute(cursor, query)
390
391
392def log_dependency_not_found(label_name):
393    if label_name in DEPENDENCIES_NOT_FOUND:
394        return
395    print 'Dependency %s not found' % label_name
396    DEPENDENCIES_NOT_FOUND.add(label_name)
397
398
399def get_id_map(cursor, table_name, name_field):
400    cursor.execute('SELECT id, %s FROM %s' % (name_field, table_name))
401    name_to_id = {}
402    for item_id, item_name in cursor.fetchall():
403        name_to_id[item_name] = item_id
404    return name_to_id
405
406
407def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False):
408    """Find control jobs in location and create one big job
409       Returns:
410        dictionary of the form:
411            tests[file_path] = parsed_object
412
413    """
414    tests = {}
415    profilers = False
416    if 'client/profilers' in parent_dir:
417        profilers = True
418    for dir in [ parent_dir ]:
419        files = recursive_walk(dir, control_pattern)
420        for file in files:
421            if '__init__.py' in file or '.svn' in file:
422                continue
423            if not profilers:
424                if not add_noncompliant:
425                    try:
426                        found_test = control_data.parse_control(file,
427                                                            raise_warnings=True)
428                        tests[file] = found_test
429                    except control_data.ControlVariableException, e:
430                        print "Skipping %s\n%s" % (file, e)
431                        pass
432                else:
433                    found_test = control_data.parse_control(file)
434                    tests[file] = found_test
435            else:
436                script = file.rstrip(".py")
437                tests[file] = compiler.parseFile(file).doc
438    return tests
439
440
441def recursive_walk(path, wildcard):
442    """Recurisvely go through a directory.
443        Returns:
444        A list of files that match wildcard
445    """
446    files = []
447    directories = [ path ]
448    while len(directories)>0:
449        directory = directories.pop()
450        for name in os.listdir(directory):
451            fullpath = os.path.join(directory, name)
452            if os.path.isfile(fullpath):
453                # if we are a control file
454                if re.search(wildcard, name):
455                    files.append(fullpath)
456            elif os.path.isdir(fullpath):
457                directories.append(fullpath)
458    return files
459
460
461def db_connect():
462    """Connect to the AUTOTEST_WEB database and return a connect object."""
463    c = global_config.global_config
464    db_host = c.get_config_value('AUTOTEST_WEB', 'host')
465    db_name = c.get_config_value('AUTOTEST_WEB', 'database')
466    username = c.get_config_value('AUTOTEST_WEB', 'user')
467    password = c.get_config_value('AUTOTEST_WEB', 'password')
468    connection = MySQLdb.connect(host=db_host, db=db_name,
469                                 user=username,
470                                 passwd=password)
471    return connection
472
473
474def db_execute(cursor, sql):
475    """Execute SQL or print out what would be executed if dry_run is defined"""
476
477    if DRY_RUN:
478        print "Would run: " + sql
479    else:
480        cursor.execute(sql)
481
482
483if __name__ == "__main__":
484    main(sys.argv)
485