run_layout_tests.py revision 12077e1179b4035ae2b1a44ccf9cd540e14b182e 
1#!/usr/bin/python
2
3"""Run layout tests using Android emulator and instrumentation.
4
5  First, you need to get an SD card or sdcard image that has layout tests on it.
6  Layout tests are in following directory:
7    /sdcard/android/layout_tests
8  For example, /sdcard/android/layout_tests/fast
9
10  Usage:
11    Run all tests under fast/ directory:
12      run_layout_tests.py, or
13      run_layout_tests.py fast
14
15    Run all tests under a sub directory:
16      run_layout_tests.py fast/dom
17
18    Run a single test:
19      run_layout_tests.py fast/dom/
20
21  After a merge, if there are changes of layout tests in SD card, you need to
22  use --refresh-test-list option *once* to re-generate test list on the card.
23
24  Some other options are:
25    --rebaseline generates expected layout tests results under /sdcard/android/expected_result/
26    --time-out-ms (default is 8000 millis) for each test
27    --adb-options="-e" passes option string to adb
28    --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
29    --js-engine the JavaScript engine currently in use, determines which set of Android-specific expected results we should use, should be 'jsc' or 'v8'
30"""
31
32import logging
33import optparse
34import os
35import subprocess
36import sys
37import time
38
39def CountLineNumber(filename):
40  """Compute the number of lines in a given file.
41
42  Args:
43    filename: a file name related to the current directory.
44  """
45
46  fp = open(os.path.abspath(filename), "r");
47  lines = 0
48  for line in fp.readlines():
49    lines = lines + 1
50  fp.close()
51  return lines
52
53def DumpRenderTreeFinished(adb_cmd):
54  """ Check if DumpRenderTree finished running tests
55
56  Args:
57    output: adb_cmd string
58  """
59
60  # pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
61  shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
62  adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
63  return adb_output.strip() == "#DONE"
64
65def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
66                new_count_first=True):
67   """ Given two result files, generate diff and
68       write to diff_results file. All arguments are absolute paths
69       to files.
70   """
71   old_file = open(old_results, "r")
72   new_file = open(new_results, "r")
73   diff_file = open(diff_results, "a")
74
75   # Read lines from each file
76   ndict = new_file.readlines()
77   cdict = old_file.readlines()
78
79   # Write marker to diff file
80   diff_file.writelines(marker + "\n")
81   diff_file.writelines("###############\n")
82
83   # Strip reason from result lines
84   if strip_reason is True:
85     for i in range(0, len(ndict)):
86       ndict[i] = ndict[i].split(' ')[0] + "\n"
87     for i in range(0, len(cdict)):
88       cdict[i] = cdict[i].split(' ')[0] + "\n"
89
90   params = {
91       "new": [0, ndict, cdict, "+"],
92       "miss": [0, cdict, ndict, "-"]
93       }
94   if new_count_first:
95     order = ["new", "miss"]
96   else:
97     order = ["miss", "new"]
98
99   for key in order:
100     for line in params[key][1]:
101       if line not in params[key][2]:
102         if line[-1] != "\n":
103           line += "\n";
104         diff_file.writelines(params[key][3] + line)
105         params[key][0] += 1
106
107   logging.info(marker + "  >>> " + str(params["new"][0]) + " new, " +
108                str(params["miss"][0]) + " misses")
109
110   diff_file.writelines("\n\n")
111
112   old_file.close()
113   new_file.close()
114   diff_file.close()
115   return
116
117def CompareResults(ref_dir, results_dir):
118  """Compare results in two directories
119
120  Args:
121    ref_dir: the reference directory having layout results as references
122    results_dir: the results directory
123  """
124  logging.info("Comparing results to " + ref_dir)
125
126  diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
127  if os.path.exists(diff_result):
128    os.remove(diff_result)
129
130  files=["crashed", "failed", "passed", "nontext"]
131  for f in files:
132    result_file_name = "layout_tests_" + f + ".txt"
133    DiffResults(f, os.path.join(results_dir, result_file_name),
134                os.path.join(ref_dir, result_file_name), diff_result,
135                False, f != "passed")
136  logging.info("Detailed diffs are in " + diff_result)
137
138def main(options, args):
139  """Run the tests. Will call sys.exit when complete.
140
141  Args:
142    options: a dictionary of command line options
143    args: a list of sub directories or files to test
144  """
145
146  # Set up logging format.
147  log_level = logging.INFO
148  if options.verbose:
149    log_level = logging.DEBUG
150  logging.basicConfig(level=log_level,
151                      format='%(message)s')
152
153  # Include all tests if none are specified.
154  if not args:
155    path = '/';
156  else:
157    path = ' '.join(args);
158
159  adb_cmd = "adb ";
160  if options.adb_options:
161    adb_cmd += options.adb_options
162
163  # Re-generate the test list if --refresh-test-list is on
164  if options.refresh_test_list:
165    logging.info("Generating test list.");
166    generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
167    adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
168
169    if adb_output.find('Process crashed') != -1:
170       logging.info("Aborting because cannot generate test list.\n" + adb_output)
171       sys.exit(1)
172
173
174  logging.info("Running tests")
175
176  # Count crashed tests.
177  crashed_tests = []
178
179  timeout_ms = '30000'
180  if options.time_out_ms:
181    timeout_ms = options.time_out_ms
182
183  # Run test until it's done
184
185  run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
186
187  run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
188  if options.rebaseline:
189    run_layout_test_cmd_postfix += " -e rebaseline true"
190
191  # If the JS engine is not specified on the command line, try reading the
192  # JS_ENGINE environment  variable, which is used by the build system in
193  # external/webkit/Android.mk.
194  js_engine = options.js_engine
195  if not js_engine:
196    js_engine = os.environ['JS_ENGINE']
197  if js_engine:
198    run_layout_test_cmd_postfix += " -e jsengine " + js_engine
199
200  run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
201
202  # Call LayoutTestsAutoTest::startLayoutTests.
203  run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
204
205  adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
206  while not DumpRenderTreeFinished(adb_cmd):
207    # Get the running_test.txt
208    logging.error("DumpRenderTree crashed, output:\n" + adb_output)
209
210    shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
211    crashed_test = ""
212    while not crashed_test:
213      (crashed_test, err) = subprocess.Popen(
214          shell_cmd_str, shell=True, stdout=subprocess.PIPE,
215          stderr=subprocess.PIPE).communicate()
216      crashed_test = crashed_test.strip()
217      if not crashed_test:
218        logging.error('Cannot get crashed test name, device offline?')
219        logging.error('stderr: ' + err)
220        logging.error('retrying in 10s...')
221        time.sleep(10)
222
223    logging.info(crashed_test + " CRASHED");
224    crashed_tests.append(crashed_test);
225
226    logging.info("Resuming layout test runner...");
227    # Call LayoutTestsAutoTest::resumeLayoutTests
228    run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
229
230    adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
231
232  if adb_output.find('INSTRUMENTATION_FAILED') != -1:
233    logging.error("Error happened : " + adb_output)
234    sys.exit(1)
235
236  logging.debug(adb_output);
237  logging.info("Done\n");
238
239  # Pull results from /sdcard
240  results_dir = options.results_directory
241  if not os.path.exists(results_dir):
242    os.makedirs(results_dir)
243  if not os.path.isdir(results_dir):
244    logging.error("Cannot create results dir: " + results_dir);
245    sys.exit(1);
246
247  result_files = ["/sdcard/layout_tests_passed.txt",
248                  "/sdcard/layout_tests_failed.txt",
249                  "/sdcard/layout_tests_nontext.txt"]
250  for file in result_files:
251    shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
252    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
253    logging.debug(adb_output)
254
255  # Create the crash list.
256  fp = open(results_dir + "/layout_tests_crashed.txt", "w");
257  for crashed_test in crashed_tests:
258    fp.writelines(crashed_test + '\n')
259  fp.close()
260
261  # Count the number of tests in each category.
262  passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
263  logging.info(str(passed_tests) + " passed")
264  failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
265  logging.info(str(failed_tests) + " failed")
266  crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
267  logging.info(str(crashed_tests) + " crashed")
268  nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
269  logging.info(str(nontext_tests) + " no dumpAsText")
270
271  logging.info("Results are stored under: " + results_dir + "\n")
272
273  # Comparing results to references to find new fixes and regressions.
274  results_dir = os.path.abspath(options.results_directory)
275  ref_dir = options.ref_directory
276
277  # if ref_dir is null, cannonify ref_dir to the script dir.
278  if not ref_dir:
279    script_self = sys.argv[0]
280    script_dir = os.path.dirname(script_self)
281    ref_dir = os.path.join(script_dir, "results")
282
283  ref_dir = os.path.abspath(ref_dir)
284
285  CompareResults(ref_dir, results_dir)
286
287if '__main__' == __name__:
288  option_parser = optparse.OptionParser()
289  option_parser.add_option("", "--rebaseline", action="store_true",
290                           default=False,
291                           help="generate expected results for those tests not having one")
292  option_parser.add_option("", "--time-out-ms",
293                           default=None,
294                           help="set the timeout for each test")
295  option_parser.add_option("", "--verbose", action="store_true",
296                           default=False,
297                           help="include debug-level logging")
298  option_parser.add_option("", "--refresh-test-list", action="store_true",
299                           default=False,
300                           help="re-generate test list, it may take some time.")
301  option_parser.add_option("", "--adb-options",
302                           default=None,
303                           help="pass options to adb, such as -d -e, etc");
304  option_parser.add_option("", "--results-directory",
305                           default="layout-test-results",
306                           help="directory which results are stored.")
307  option_parser.add_option("", "--ref-directory",
308                           default=None,
309                           dest="ref_directory",
310                           help="directory where reference results are stored.")
311  option_parser.add_option("", "--js-engine",
312                           default=None,
313                           help="The JavaScript engine currently in use, which determines which set of Android-specific expected results we should use. Should be 'jsc' or 'v8'.");
314
315  options, args = option_parser.parse_args();
316  main(options, args)
317