run_layout_tests.py revision 36fd6d243ee5414618e8bc2cbdc6c0a6f311157f 
1#!/usr/bin/python
2
3"""Run layout tests using Android emulator and instrumentation.
4
5  First, you need to get an SD card or sdcard image that has layout tests on it.
6  Layout tests are in following directory:
7    /sdcard/android/layout_tests
8  For example, /sdcard/android/layout_tests/fast
9
10  Usage:
11    Run all tests under fast/ directory:
12      run_layout_tests.py, or
13      run_layout_tests.py fast
14
15    Run all tests under a sub directory:
16      run_layout_tests.py fast/dom
17
18    Run a single test:
19      run_layout_tests.py fast/dom/
20
21  After a merge, if there are changes of layout tests in SD card, you need to
22  use --refresh-test-list option *once* to re-generate test list on the card.
23
24  Some other options are:
25    --rebaseline generates expected layout tests results under /sdcard/android/expected_result/
26    --time-out-ms (default is 8000 millis) for each test
27    --adb-options="-e" passes option string to adb
28    --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
29"""
30
31import logging
32import optparse
33import os
34import subprocess
35import sys
36import time
37
38def CountLineNumber(filename):
39  """Compute the number of lines in a given file.
40
41  Args:
42    filename: a file name related to the current directory.
43  """
44
45  fp = open(os.path.abspath(filename), "r");
46  lines = 0
47  for line in fp.readlines():
48    lines = lines + 1
49  fp.close()
50  return lines
51
52def DumpRenderTreeFinished(adb_cmd):
53  """ Check if DumpRenderTree finished running tests
54
55  Args:
56    output: adb_cmd string
57  """
58
59  # pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
60  shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
61  adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
62  return adb_output.strip() == "#DONE"
63
64def DiffResults(marker, new_results, old_results, diff_results, strip_reason):
65   """ Given two result files, generate diff and
66       write to diff_results file. All arguments are absolute paths
67       to files.
68   """
69   old_file = open(old_results, "r")
70   new_file = open(new_results, "r")
71   diff_file = open(diff_results, "a")
72
73   # Read lines from each file
74   ndict = new_file.readlines()
75   cdict = old_file.readlines()
76
77   # Write marker to diff file
78   diff_file.writelines(marker + "\n")
79   diff_file.writelines("###############\n")
80
81   # Strip reason from result lines
82   if strip_reason is True:
83     for i in range(0, len(ndict)):
84       ndict[i] = ndict[i].split(' ')[0] + "\n"
85     for i in range(0, len(cdict)):
86       cdict[i] = cdict[i].split(' ')[0] + "\n"
87
88   # Find results in new_results missing in old_results
89   new_count=0
90   for line in ndict:
91     if line not in cdict:
92       diff_file.writelines("+ " + line)
93       new_count += 1
94
95   # Find results in old_results missing in new_results
96   missing_count=0
97   for line in cdict:
98     if line not in ndict:
99       diff_file.writelines("- " + line)
100       missing_count += 1
101
102   logging.info(marker + "  >>> " + str(new_count) + " new, " + str(missing_count) + " misses")
103
104   diff_file.writelines("\n\n")
105
106   old_file.close()
107   new_file.close()
108   diff_file.close()
109   return
110
111def CompareResults(ref_dir, results_dir):
112  """Compare results in two directories
113
114  Args:
115    ref_dir: the reference directory having layout results as references
116    results_dir: the results directory
117  """
118  logging.info("Comparing results to " + ref_dir)
119
120  diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
121  if os.path.exists(diff_result):
122    os.remove(diff_result)
123
124  files=["passed", "failed", "nontext", "crashed"]
125  for f in files:
126    result_file_name = "layout_tests_" + f + ".txt"
127    DiffResults(f, os.path.join(results_dir, result_file_name),
128                os.path.join(ref_dir, result_file_name), diff_result,
129                f == "failed")
130  logging.info("Detailed diffs are in " + diff_result)
131
132def main(options, args):
133  """Run the tests. Will call sys.exit when complete.
134
135  Args:
136    options: a dictionary of command line options
137    args: a list of sub directories or files to test
138  """
139
140  # Set up logging format.
141  log_level = logging.INFO
142  if options.verbose:
143    log_level = logging.DEBUG
144  logging.basicConfig(level=log_level,
145                      format='%(message)s')
146
147  # Include all tests if none are specified.
148  if not args:
149    path = '/';
150  else:
151    path = ' '.join(args);
152
153  adb_cmd = "adb ";
154  if options.adb_options:
155    adb_cmd += options.adb_options
156
157  # Re-generate the test list if --refresh-test-list is on
158  if options.refresh_test_list:
159    logging.info("Generating test list.");
160    generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
161    adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
162
163    if adb_output.find('Process crashed') != -1:
164       logging.info("Aborting because cannot generate test list.\n" + adb_output)
165       sys.exit(1)
166
167
168  logging.info("Running tests")
169
170  # Count crashed tests.
171  crashed_tests = []
172
173  timeout_ms = '5000'
174  if options.time_out_ms:
175    timeout_ms = options.time_out_ms
176
177  # Run test until it's done
178
179  run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
180
181  run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
182  if options.rebaseline:
183    run_layout_test_cmd_postfix += " -e rebaseline true"
184  run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
185
186  # Call LayoutTestsAutoTest::startLayoutTests.
187  run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
188
189  adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
190  while not DumpRenderTreeFinished(adb_cmd):
191    # Get the running_test.txt
192    logging.error("DumpRenderTree crashed, output:\n" + adb_output)
193
194    shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
195    crashed_test = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE).communicate()[0]
196
197    logging.info(crashed_test + " CRASHED");
198    crashed_tests.append(crashed_test);
199
200    logging.info("Resuming layout test runner...");
201    # Call LayoutTestsAutoTest::resumeLayoutTests
202    run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
203
204    adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
205
206  if adb_output.find('INSTRUMENTATION_FAILED') != -1:
207    logging.error("Error happened : " + adb_output)
208    sys.exit(1)
209
210  logging.debug(adb_output);
211  logging.info("Done\n");
212
213  # Pull results from /sdcard
214  results_dir = options.results_directory
215  if not os.path.exists(results_dir):
216    os.makedirs(results_dir)
217  if not os.path.isdir(results_dir):
218    logging.error("Cannot create results dir: " + results_dir);
219    sys.exit(1);
220
221  result_files = ["/sdcard/layout_tests_passed.txt",
222                  "/sdcard/layout_tests_failed.txt",
223                  "/sdcard/layout_tests_nontext.txt"]
224  for file in result_files:
225    shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
226    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
227    logging.debug(adb_output)
228
229  # Create the crash list.
230  fp = open(results_dir + "/layout_tests_crashed.txt", "w");
231  fp.writelines('\n'.join(crashed_tests))
232  fp.close()
233
234  # Count the number of tests in each category.
235  passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
236  logging.info(str(passed_tests) + " passed")
237  failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
238  logging.info(str(failed_tests) + " failed")
239  crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
240  logging.info(str(crashed_tests) + " crashed")
241  nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
242  logging.info(str(nontext_tests) + " no dumpAsText")
243
244  logging.info("Results are stored under: " + results_dir + "\n")
245
246  # Comparing results to references to find new fixes and regressions.
247  results_dir = os.path.abspath(options.results_directory)
248  ref_dir = options.ref_directory
249
250  # if ref_dir is null, cannonify ref_dir to the script dir.
251  if not ref_dir:
252    script_self = sys.argv[0]
253    script_dir = os.path.dirname(script_self)
254    ref_dir = os.path.join(script_dir, "results")
255
256  ref_dir = os.path.abspath(ref_dir)
257
258  CompareResults(ref_dir, results_dir)
259
260if '__main__' == __name__:
261  option_parser = optparse.OptionParser()
262  option_parser.add_option("", "--rebaseline", action="store_true",
263                           default=False,
264                           help="generate expected results for those tests not having one")
265  option_parser.add_option("", "--time-out-ms",
266                           default=None,
267                           help="set the timeout for each test")
268  option_parser.add_option("", "--verbose", action="store_true",
269                           default=False,
270                           help="include debug-level logging")
271  option_parser.add_option("", "--refresh-test-list", action="store_true",
272                           default=False,
273                           help="re-generate test list, it may take some time.")
274  option_parser.add_option("", "--adb-options",
275                           default=None,
276                           help="pass options to adb, such as -d -e, etc");
277  option_parser.add_option("", "--results-directory",
278                           default="layout-test-results",
279                           help="directory which results are stored.")
280  option_parser.add_option("", "--ref-directory",
281                           default=None,
282                           dest="ref_directory",
283                           help="directory where reference results are stored.")
284
285  options, args = option_parser.parse_args();
286  main(options, args)
287