run_layout_tests.py revision 9db3d07b9620b4269ab33f78604a36327e536ce1 
1#!/usr/bin/python
2
3"""Run layout tests using Android emulator and instrumentation.
4
5  First, you need to get an SD card or sdcard image that has layout tests on it.
6  Layout tests are in following directory:
7    /sdcard/android/layout_tests
8  For example, /sdcard/android/layout_tests/fast
9
10  Usage:
11    Run all tests under fast/ directory:
12      run_layout_tests.py, or
13      run_layout_tests.py fast
14
15    Run all tests under a sub directory:
16      run_layout_tests.py fast/dom
17
18    Run a single test:
19      run_layout_tests.py fast/dom/
20
21  After a merge, if there are changes of layout tests in SD card, you need to
22  use --refresh-test-list option *once* to re-generate test list on the card.
23
24  Some other options are:
25    --rebaseline generates expected layout tests results under /sdcard/android/expected_result/
26    --time-out-ms (default is 8000 millis) for each test
27    --adb-options="-e" passes option string to adb
28    --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
29"""
30
31import logging
32import optparse
33import os
34import subprocess
35import sys
36import time
37
38def CountLineNumber(filename):
39  """Compute the number of lines in a given file.
40
41  Args:
42    filename: a file name related to the current directory.
43  """
44
45  fp = open(os.path.abspath(filename), "r");
46  lines = 0
47  for line in fp.readlines():
48    lines = lines + 1
49  fp.close()
50  return lines
51
52def DumpRenderTreeFinished(adb_cmd):
53  """ Check if DumpRenderTree finished running tests
54
55  Args:
56    output: adb_cmd string
57  """
58
59  # pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
60  shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
61  adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
62  return adb_output.strip() == "#DONE"
63
64def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
65                new_count_first=True):
66   """ Given two result files, generate diff and
67       write to diff_results file. All arguments are absolute paths
68       to files.
69   """
70   old_file = open(old_results, "r")
71   new_file = open(new_results, "r")
72   diff_file = open(diff_results, "a")
73
74   # Read lines from each file
75   ndict = new_file.readlines()
76   cdict = old_file.readlines()
77
78   # Write marker to diff file
79   diff_file.writelines(marker + "\n")
80   diff_file.writelines("###############\n")
81
82   # Strip reason from result lines
83   if strip_reason is True:
84     for i in range(0, len(ndict)):
85       ndict[i] = ndict[i].split(' ')[0] + "\n"
86     for i in range(0, len(cdict)):
87       cdict[i] = cdict[i].split(' ')[0] + "\n"
88
89   params = {
90       "new": [0, ndict, cdict, "+"],
91       "miss": [0, cdict, ndict, "-"]
92       }
93   if new_count_first:
94     order = ["new", "miss"]
95   else:
96     order = ["miss", "new"]
97
98   for key in order:
99     for line in params[key][1]:
100       if line not in params[key][2]:
101         if line[-1] != "\n":
102           line += "\n";
103         diff_file.writelines(params[key][3] + line)
104         params[key][0] += 1
105
106   logging.info(marker + "  >>> " + str(params["new"][0]) + " new, " +
107                str(params["miss"][0]) + " misses")
108
109   diff_file.writelines("\n\n")
110
111   old_file.close()
112   new_file.close()
113   diff_file.close()
114   return
115
116def CompareResults(ref_dir, results_dir):
117  """Compare results in two directories
118
119  Args:
120    ref_dir: the reference directory having layout results as references
121    results_dir: the results directory
122  """
123  logging.info("Comparing results to " + ref_dir)
124
125  diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
126  if os.path.exists(diff_result):
127    os.remove(diff_result)
128
129  files=["crashed", "failed", "passed", "nontext"]
130  for f in files:
131    result_file_name = "layout_tests_" + f + ".txt"
132    DiffResults(f, os.path.join(results_dir, result_file_name),
133                os.path.join(ref_dir, result_file_name), diff_result,
134                False, f != "passed")
135  logging.info("Detailed diffs are in " + diff_result)
136
137def main(options, args):
138  """Run the tests. Will call sys.exit when complete.
139
140  Args:
141    options: a dictionary of command line options
142    args: a list of sub directories or files to test
143  """
144
145  # Set up logging format.
146  log_level = logging.INFO
147  if options.verbose:
148    log_level = logging.DEBUG
149  logging.basicConfig(level=log_level,
150                      format='%(message)s')
151
152  # Include all tests if none are specified.
153  if not args:
154    path = '/';
155  else:
156    path = ' '.join(args);
157
158  adb_cmd = "adb ";
159  if options.adb_options:
160    adb_cmd += options.adb_options
161
162  # Re-generate the test list if --refresh-test-list is on
163  if options.refresh_test_list:
164    logging.info("Generating test list.");
165    generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
166    adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
167
168    if adb_output.find('Process crashed') != -1:
169       logging.info("Aborting because cannot generate test list.\n" + adb_output)
170       sys.exit(1)
171
172
173  logging.info("Running tests")
174
175  # Count crashed tests.
176  crashed_tests = []
177
178  timeout_ms = '30000'
179  if options.time_out_ms:
180    timeout_ms = options.time_out_ms
181
182  # Run test until it's done
183
184  run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
185
186  run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
187  if options.rebaseline:
188    run_layout_test_cmd_postfix += " -e rebaseline true"
189  run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
190
191  # Call LayoutTestsAutoTest::startLayoutTests.
192  run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
193
194  adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
195  while not DumpRenderTreeFinished(adb_cmd):
196    # Get the running_test.txt
197    logging.error("DumpRenderTree crashed, output:\n" + adb_output)
198
199    shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
200    crashed_test = ""
201    while not crashed_test:
202      (crashed_test, err) = subprocess.Popen(
203          shell_cmd_str, shell=True, stdout=subprocess.PIPE,
204          stderr=subprocess.PIPE).communicate()
205      crashed_test = crashed_test.strip()
206      if not crashed_test:
207        logging.error('Cannot get crashed test name, device offline?')
208        logging.error('stderr: ' + err)
209        logging.error('retrying in 10s...')
210        time.sleep(10)
211
212    logging.info(crashed_test + " CRASHED");
213    crashed_tests.append(crashed_test);
214
215    logging.info("Resuming layout test runner...");
216    # Call LayoutTestsAutoTest::resumeLayoutTests
217    run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
218
219    adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
220
221  if adb_output.find('INSTRUMENTATION_FAILED') != -1:
222    logging.error("Error happened : " + adb_output)
223    sys.exit(1)
224
225  logging.debug(adb_output);
226  logging.info("Done\n");
227
228  # Pull results from /sdcard
229  results_dir = options.results_directory
230  if not os.path.exists(results_dir):
231    os.makedirs(results_dir)
232  if not os.path.isdir(results_dir):
233    logging.error("Cannot create results dir: " + results_dir);
234    sys.exit(1);
235
236  result_files = ["/sdcard/layout_tests_passed.txt",
237                  "/sdcard/layout_tests_failed.txt",
238                  "/sdcard/layout_tests_nontext.txt"]
239  for file in result_files:
240    shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
241    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
242    logging.debug(adb_output)
243
244  # Create the crash list.
245  fp = open(results_dir + "/layout_tests_crashed.txt", "w");
246  for crashed_test in crashed_tests:
247    fp.writelines(crashed_test + '\n')
248  fp.close()
249
250  # Count the number of tests in each category.
251  passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
252  logging.info(str(passed_tests) + " passed")
253  failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
254  logging.info(str(failed_tests) + " failed")
255  crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
256  logging.info(str(crashed_tests) + " crashed")
257  nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
258  logging.info(str(nontext_tests) + " no dumpAsText")
259
260  logging.info("Results are stored under: " + results_dir + "\n")
261
262  # Comparing results to references to find new fixes and regressions.
263  results_dir = os.path.abspath(options.results_directory)
264  ref_dir = options.ref_directory
265
266  # if ref_dir is null, cannonify ref_dir to the script dir.
267  if not ref_dir:
268    script_self = sys.argv[0]
269    script_dir = os.path.dirname(script_self)
270    ref_dir = os.path.join(script_dir, "results")
271
272  ref_dir = os.path.abspath(ref_dir)
273
274  CompareResults(ref_dir, results_dir)
275
276if '__main__' == __name__:
277  option_parser = optparse.OptionParser()
278  option_parser.add_option("", "--rebaseline", action="store_true",
279                           default=False,
280                           help="generate expected results for those tests not having one")
281  option_parser.add_option("", "--time-out-ms",
282                           default=None,
283                           help="set the timeout for each test")
284  option_parser.add_option("", "--verbose", action="store_true",
285                           default=False,
286                           help="include debug-level logging")
287  option_parser.add_option("", "--refresh-test-list", action="store_true",
288                           default=False,
289                           help="re-generate test list, it may take some time.")
290  option_parser.add_option("", "--adb-options",
291                           default=None,
292                           help="pass options to adb, such as -d -e, etc");
293  option_parser.add_option("", "--results-directory",
294                           default="layout-test-results",
295                           help="directory which results are stored.")
296  option_parser.add_option("", "--ref-directory",
297                           default=None,
298                           dest="ref_directory",
299                           help="directory where reference results are stored.")
300
301  options, args = option_parser.parse_args();
302  main(options, args)
303