test_importer.py revision b6f95534ef8ef87144491d6ec14f4f3182b47bc8 
1#!/usr/bin/python2.4
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4"""
5This utility allows for easy updating, removing and importing
6of tests into the autotest_web autotests table.
7
8Example of updating client side tests:
9./tests.py -t /usr/local/autotest/client/tests
10
11If for example not all of your control files adhere to the standard outlined at
12http://test.kernel.org/autotest/ControlRequirements
13
14You can force options:
15./tests.py --test-type server -t /usr/local/autotest/server/tests
16
17
18Most options should be fairly self explanatory use --help to display them.
19"""
20
21
22import time, re, os, MySQLdb, sys, optparse, compiler
23import common
24from autotest_lib.client.common_lib import control_data, test, global_config
25from autotest_lib.client.common_lib import utils
26
27
28# Global
29DRY_RUN = False
30
31
32def main(argv):
33    """Main function"""
34    global DRY_RUN
35    parser = optparse.OptionParser()
36    parser.add_option('-c', '--db-clear-tests',
37                      dest='clear_tests', action='store_true',
38                      default=False,
39                help='Clear client and server tests with invalid control files')
40    parser.add_option('-d', '--dry-run',
41                      dest='dry_run', action='store_true', default=False,
42                      help='Dry run for operation')
43    parser.add_option('-A', '--add-all',
44                      dest='add_all', action='store_true',
45                      default=False,
46                      help='Add samples, site_tests, tests, and test_suites')
47    parser.add_option('-E', '--add-experimental',
48                      dest='add_experimental', action='store_true',
49                      default=False,
50                      help='Add experimental tests to frontend')
51    parser.add_option('-N', '--add-noncompliant',
52                      dest='add_noncompliant', action='store_true',
53                      default=False,
54                      help='Skip any tests that are not compliant')
55    parser.add_option('-p', '--profile-dir', dest='profile_dir',
56                      help='Directory to recursively check for profiles')
57    parser.add_option('-t', '--tests-dir', dest='tests_dir',
58                      help='Directory to recursively check for control.*')
59    parser.add_option('-r', '--control-pattern', dest='control_pattern',
60                      default='^control.*',
61               help='The pattern to look for in directories for control files')
62    parser.add_option('-v', '--verbose',
63                      dest='verbose', action='store_true', default=False,
64                      help='Run in verbose mode')
65    parser.add_option('-z', '--autotest_dir', dest='autotest_dir',
66                      default=os.path.join(os.path.dirname(__file__), '..'),
67                      help='Autotest directory root')
68    options, args = parser.parse_args()
69    DRY_RUN = options.dry_run
70    # Make sure autotest_dir is the absolute path
71    options.autotest_dir = os.path.abspath(options.autotest_dir)
72
73    if len(args) > 0:
74        print "Invalid option(s) provided: ", args
75        parser.print_help()
76        return 1
77
78    if len(argv) == 1:
79        update_all(options.autotest_dir, options.add_noncompliant,
80                   options.add_experimental, options.verbose)
81        db_clean_broken(options.autotest_dir, options.verbose)
82        return 0
83
84    if options.add_all:
85        update_all(options.autotest_dir, options.add_noncompliant,
86                   options.add_experimental, options.verbose)
87    if options.clear_tests:
88        db_clean_broken(options.autotest_dir, options.verbose)
89    if options.tests_dir:
90        if ".." in options.tests_dir:
91            path = os.path.join(os.getcwd(), options.tests_dir)
92            options.tests_dir = os.path.abspath(path)
93        tests = get_tests_from_fs(options.tests_dir, options.control_pattern,
94                                  add_noncompliant=options.add_noncompliant)
95        update_tests_in_db(tests, add_experimental=options.add_experimental,
96                           add_noncompliant=options.add_noncompliant,
97                           autotest_dir=options.autotest_dir,
98                           verbose=options.verbose)
99    if options.profile_dir:
100        profilers = get_tests_from_fs(options.profile_dir, '.*py$')
101        update_profilers_in_db(profilers, verbose=options.verbose,
102                               add_noncompliant=options.add_noncompliant,
103                               description='NA')
104
105
106def update_all(autotest_dir, add_noncompliant, add_experimental, verbose):
107    """Function to scan through all tests and add them to the database."""
108    for path in [ 'server/tests', 'server/site_tests', 'client/tests',
109                  'client/site_tests']:
110        test_path = os.path.join(autotest_dir, path)
111        if not os.path.exists(test_path):
112            continue
113        if verbose:
114            print "Scanning " + test_path
115        tests = []
116        tests = get_tests_from_fs(test_path, "^control.*",
117                                 add_noncompliant=add_noncompliant)
118        update_tests_in_db(tests, add_experimental=add_experimental,
119                           add_noncompliant=add_noncompliant,
120                           autotest_dir=autotest_dir,
121                           verbose=verbose)
122    test_suite_path = os.path.join(autotest_dir, 'test_suites')
123    if os.path.exists(test_suite_path):
124        if verbose:
125            print "Scanning " + test_suite_path
126        tests = get_tests_from_fs(test_suite_path, '.*',
127                                 add_noncompliant=add_noncompliant)
128        update_tests_in_db(tests, add_experimental=add_experimental,
129                           add_noncompliant=add_noncompliant,
130                           autotest_dir=autotest_dir,
131                           verbose=verbose)
132    sample_path = os.path.join(autotest_dir, 'server/samples')
133    if os.path.exists(sample_path):
134        if verbose:
135            print "Scanning " + sample_path
136        tests = get_tests_from_fs(sample_path, '.*srv$',
137                                 add_noncompliant=add_noncompliant)
138        update_tests_in_db(tests, add_experimental=add_experimental,
139                           add_noncompliant=add_noncompliant,
140                           autotest_dir=autotest_dir,
141                           verbose=verbose)
142
143    profilers_path = os.path.join(autotest_dir, "client/profilers")
144    if os.path.exists(profilers_path):
145        if verbose:
146            print "Scanning " + profilers_path
147        profilers = get_tests_from_fs(profilers_path, '.*py$')
148        update_profilers_in_db(profilers, verbose=verbose,
149                               add_noncompliant=add_noncompliant,
150                               description='NA')
151    # Clean bad db entries
152    db_clean_broken(autotest_dir, verbose)
153
154
155def db_clean_broken(autotest_dir, verbose):
156    """Remove tests from autotest_web that do not have valid control files
157
158       Arguments:
159        tests: a list of control file relative paths used as keys for deletion.
160    """
161    connection=db_connect()
162    cursor = connection.cursor()
163    # Get tests
164    sql = "SELECT path FROM autotests";
165    cursor.execute(sql)
166    results = cursor.fetchall()
167    for path in results:
168        full_path = os.path.join(autotest_dir, path[0])
169        if not os.path.isfile(full_path):
170            if verbose:
171                print "Removing " + path[0]
172            sql = "DELETE FROM autotests WHERE path='%s'" % path[0]
173            db_execute(cursor, sql)
174
175    # Find profilers that are no longer present
176    profilers = []
177    sql = "SELECT name FROM profilers"
178    cursor.execute(sql)
179    results = cursor.fetchall()
180    for path in results:
181        full_path = os.path.join(autotest_dir, "client/profilers", path[0])
182        if not os.path.exists(full_path):
183            if verbose:
184                print "Removing " + path[0]
185            sql = "DELETE FROM profilers WHERE name='%s'" % path[0]
186            db_execute(cursor, sql)
187
188
189    connection.commit()
190    connection.close()
191
192
193def update_profilers_in_db(profilers, verbose=False, description='NA',
194                           add_noncompliant=False):
195    """Update profilers in autotest_web database"""
196    connection=db_connect()
197    cursor = connection.cursor()
198    for profiler in profilers:
199        name = os.path.basename(profiler).rstrip(".py")
200        if not profilers[profiler]:
201            if add_noncompliant:
202                doc = description
203            else:
204                print "Skipping %s, missing docstring" % profiler
205        else:
206            doc = profilers[profiler]
207        # check if test exists
208        sql = "SELECT name FROM profilers WHERE name='%s'" % name
209        cursor.execute(sql)
210        results = cursor.fetchall()
211        if results:
212            sql = "UPDATE profilers SET name='%s', description='%s' "\
213                  "WHERE name='%s'"
214            sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc),
215                    MySQLdb.escape_string(name))
216        else:
217            # Insert newly into DB
218            sql = "INSERT into profilers (name, description) VALUES('%s', '%s')"
219            sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc))
220
221        db_execute(cursor, sql)
222
223    connection.commit()
224    connection.close()
225
226
227def update_tests_in_db(tests, dry_run=False, add_experimental=False,
228                       add_noncompliant=False, verbose=False,
229                       autotest_dir=None):
230    """Update or add each test to the database"""
231    connection=db_connect()
232    cursor = connection.cursor()
233    for test in tests:
234        new_test = {}
235        new_test['path'] = test.replace(autotest_dir, '').lstrip('/')
236        if verbose:
237            print "Processing " + new_test['path']
238        # Create a name for the test
239        for key in dir(tests[test]):
240            if not key.startswith('__'):
241                value = getattr(tests[test], key)
242                if not callable(value):
243                    new_test[key] = value
244        # This only takes place if --add-noncompliant is provided on the CLI
245        if 'name' not in new_test:
246            test_new_test = test.split('/')
247            if test_new_test[-1] == 'control':
248                new_test['name'] = test_new_test[-2]
249            else:
250                control_name = "%s:%s"
251                control_name %= (test_new_test[-2],
252                                 test_new_test[-1])
253                new_test['name'] = control_name.replace('control.', '')
254        # Experimental Check
255        if not add_experimental:
256            if int(new_test['experimental']):
257                continue
258        # clean tests for insertion into db
259        new_test = dict_db_clean(new_test)
260        sql = "SELECT name,path FROM autotests WHERE path='%s' LIMIT 1"
261        sql %= new_test['path']
262        cursor.execute(sql)
263        # check for entries already in existence
264        results = cursor.fetchall()
265        if results:
266            sql = "UPDATE autotests SET name='%s', test_class='%s',"\
267                  "description='%s', test_type=%d, path='%s',"\
268                  "synch_type=%d, author='%s', dependencies='%s',"\
269                  "experimental=%d, run_verify=%d, test_time=%d,"\
270                  "test_category='%s', sync_count=%d"\
271                  " WHERE path='%s'"
272            sql %= (new_test['name'], new_test['test_class'], new_test['doc'],
273                    int(new_test['test_type']), new_test['path'],
274                    int(new_test['synch_type']), new_test['author'],
275                    new_test['dependencies'], int(new_test['experimental']),
276                    int(new_test['run_verify']), new_test['time'],
277                    new_test['test_category'], new_test['sync_count'],
278                    new_test['path'])
279        else:
280            # Create a relative path
281            path = test.replace(autotest_dir, '')
282            sql = "INSERT INTO autotests"\
283                  "(name, test_class, description, test_type, path, "\
284                  "synch_type, author, dependencies, experimental, "\
285                  "run_verify, test_time, test_category, sync_count) "\
286                  "VALUES('%s','%s','%s',%d,'%s',%d,'%s','%s',%d,%d,%d,"\
287                  "'%s',%d)"
288            sql %= (new_test['name'], new_test['test_class'], new_test['doc'],
289                    int(new_test['test_type']), new_test['path'],
290                    int(new_test['synch_type']), new_test['author'],
291                    new_test['dependencies'], int(new_test['experimental']),
292                    int(new_test['run_verify']), new_test['time'],
293                    new_test['test_category'], new_test['sync_count'])
294
295        db_execute(cursor, sql)
296
297    connection.commit()
298    connection.close()
299
300
301def dict_db_clean(test):
302    """Take a tests dictionary from update_db and make it pretty for SQL"""
303
304    test_type = { 'client' : 1,
305                  'server' : 2, }
306    test_time = { 'short' : 1,
307                  'medium' : 2,
308                  'long' : 3, }
309
310    test['name'] = MySQLdb.escape_string(test['name'])
311    test['author'] = MySQLdb.escape_string(test['author'])
312    test['test_class'] = MySQLdb.escape_string(test['test_class'])
313    test['test_category'] = MySQLdb.escape_string(test['test_category'])
314    test['doc'] = MySQLdb.escape_string(test['doc'])
315    test['dependencies'] = ", ".join(test['dependencies'])
316    # TODO Fix when we move from synch_type to sync_count
317    if test['sync_count'] == 1:
318        test['synch_type'] = 1
319    else:
320        test['synch_type'] = 2
321    try:
322        test['test_type'] = int(test['test_type'])
323        if test['test_type'] != 1 and test['test_type'] != 2:
324            raise Exception('Incorrect number %d for test_type' %
325                            test['test_type'])
326    except ValueError:
327        pass
328    try:
329        test['time'] = int(test['time'])
330        if test['time'] < 1 or test['time'] > 3:
331            raise Exception('Incorrect number %d for time' %
332                            test['time'])
333    except ValueError:
334        pass
335
336    if str == type(test['time']):
337        test['time'] = test_time[test['time'].lower()]
338    if str == type(test['test_type']):
339        test['test_type'] = test_type[test['test_type'].lower()]
340    return test
341
342
343def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False):
344    """Find control jobs in location and create one big job
345       Returns:
346        dictionary of the form:
347            tests[file_path] = parsed_object
348
349    """
350    tests = {}
351    profilers = False
352    if 'client/profilers' in parent_dir:
353        profilers = True
354    for dir in [ parent_dir ]:
355        files = recursive_walk(dir, control_pattern)
356        for file in files:
357            if '__init__.py' in file:
358                continue
359            if not profilers:
360                if not add_noncompliant:
361                    try:
362                        found_test = control_data.parse_control(file,
363                                                            raise_warnings=True)
364                        tests[file] = found_test
365                    except control_data.ControlVariableException, e:
366                        print "Skipping %s\n%s" % (file, e)
367                        pass
368                else:
369                    found_test = control_data.parse_control(file)
370                    tests[file] = found_test
371            else:
372                script = file.rstrip(".py")
373                tests[file] = compiler.parseFile(file).doc
374    return tests
375
376
377def recursive_walk(path, wildcard):
378    """Recurisvely go through a directory.
379        Returns:
380        A list of files that match wildcard
381    """
382    files = []
383    directories = [ path ]
384    while len(directories)>0:
385        directory = directories.pop()
386        for name in os.listdir(directory):
387            fullpath = os.path.join(directory, name)
388            if os.path.isfile(fullpath):
389                # if we are a control file
390                if re.search(wildcard, name):
391                    files.append(fullpath)
392            elif os.path.isdir(fullpath):
393                directories.append(fullpath)
394    return files
395
396
397def db_connect():
398    """Connect to the AUTOTEST_WEB database and return a connect object."""
399    c = global_config.global_config
400    db_host = c.get_config_value('AUTOTEST_WEB', 'host')
401    db_name = c.get_config_value('AUTOTEST_WEB', 'database')
402    username = c.get_config_value('AUTOTEST_WEB', 'user')
403    password = c.get_config_value('AUTOTEST_WEB', 'password')
404    connection = MySQLdb.connect(host=db_host, db=db_name,
405                                 user=username,
406                                 passwd=password)
407    return connection
408
409
410def db_execute(cursor, sql):
411    """Execute SQL or print out what would be executed if dry_run is defined"""
412
413    if DRY_RUN:
414        print "Would run: " + sql
415    else:
416        cursor.execute(sql)
417
418
419if __name__ == "__main__":
420    main(sys.argv)
421