run_layout_tests.py revision 595fbd6ea22708504dc9e24b44fa5eb357a576ec 
1#!/usr/bin/python
2
3"""Run layout tests using Android emulator and instrumentation.
4
5  First, you need to get an SD card or sdcard image that has layout tests on it.
6  Layout tests are in following directory:
7    /sdcard/android/layout_tests
8  For example, /sdcard/android/layout_tests/fast
9
10  Usage:
11    Run all tests under fast/ directory:
12      run_layout_tests.py, or
13      run_layout_tests.py fast
14
15    Run all tests under a sub directory:
16      run_layout_tests.py fast/dom
17
18    Run a single test:
19      run_layout_tests.py fast/dom/
20
21  After a merge, if there are changes of layout tests in SD card, you need to
22  use --refresh-test-list option *once* to re-generate test list on the card.
23
24  Some other options are:
25    --rebaseline generates expected layout tests results under /sdcard/android/expected_result/
26    --time-out-ms (default is 8000 millis) for each test
27    --adb-options="-e" passes option string to adb
28    --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
29"""
30
31import logging
32import optparse
33import os
34import subprocess
35import sys
36import time
37
38def CountLineNumber(filename):
39  """Compute the number of lines in a given file.
40
41  Args:
42    filename: a file name related to the current directory.
43  """
44
45  fp = open(os.path.abspath(filename), "r");
46  lines = 0
47  for line in fp.readlines():
48    lines = lines + 1
49  fp.close()
50  return lines
51
52def DumpRenderTreeFinished(adb_cmd):
53  """ Check if DumpRenderTree finished running tests
54
55  Args:
56    output: adb_cmd string
57  """
58
59  # pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
60  shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
61  adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
62  return adb_output.strip() == "#DONE"
63
64def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
65                new_count_first=True):
66   """ Given two result files, generate diff and
67       write to diff_results file. All arguments are absolute paths
68       to files.
69   """
70   old_file = open(old_results, "r")
71   new_file = open(new_results, "r")
72   diff_file = open(diff_results, "a")
73
74   # Read lines from each file
75   ndict = new_file.readlines()
76   cdict = old_file.readlines()
77
78   # Write marker to diff file
79   diff_file.writelines(marker + "\n")
80   diff_file.writelines("###############\n")
81
82   # Strip reason from result lines
83   if strip_reason is True:
84     for i in range(0, len(ndict)):
85       ndict[i] = ndict[i].split(' ')[0] + "\n"
86     for i in range(0, len(cdict)):
87       cdict[i] = cdict[i].split(' ')[0] + "\n"
88
89   params = {
90       "new": [0, ndict, cdict, "+"],
91       "miss": [0, cdict, ndict, "-"]
92       }
93   if new_count_first:
94     order = ["new", "miss"]
95   else:
96     order = ["miss", "new"]
97
98   for key in order:
99     for line in params[key][1]:
100       if line not in params[key][2]:
101         if line[-1] != "\n":
102           line += "\n";
103         diff_file.writelines(params[key][3] + line)
104         params[key][0] += 1
105
106   logging.info(marker + "  >>> " + str(params["new"][0]) + " new, " +
107                str(params["miss"][0]) + " misses")
108
109   diff_file.writelines("\n\n")
110
111   old_file.close()
112   new_file.close()
113   diff_file.close()
114   return
115
116def CompareResults(ref_dir, results_dir):
117  """Compare results in two directories
118
119  Args:
120    ref_dir: the reference directory having layout results as references
121    results_dir: the results directory
122  """
123  logging.info("Comparing results to " + ref_dir)
124
125  diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
126  if os.path.exists(diff_result):
127    os.remove(diff_result)
128
129  files=["crashed", "failed", "passed", "nontext"]
130  for f in files:
131    result_file_name = "layout_tests_" + f + ".txt"
132    DiffResults(f, os.path.join(results_dir, result_file_name),
133                os.path.join(ref_dir, result_file_name), diff_result,
134                False, f != "passed")
135  logging.info("Detailed diffs are in " + diff_result)
136
137def main(options, args):
138  """Run the tests. Will call sys.exit when complete.
139
140  Args:
141    options: a dictionary of command line options
142    args: a list of sub directories or files to test
143  """
144
145  # Set up logging format.
146  log_level = logging.INFO
147  if options.verbose:
148    log_level = logging.DEBUG
149  logging.basicConfig(level=log_level,
150                      format='%(message)s')
151
152  # Include all tests if none are specified.
153  if not args:
154    path = '/';
155  else:
156    path = ' '.join(args);
157
158  adb_cmd = "adb ";
159  if options.adb_options:
160    adb_cmd += options.adb_options
161
162  # Re-generate the test list if --refresh-test-list is on
163  if options.refresh_test_list:
164    logging.info("Generating test list.");
165    generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
166    adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
167
168    if adb_output.find('Process crashed') != -1:
169       logging.info("Aborting because cannot generate test list.\n" + adb_output)
170       sys.exit(1)
171
172
173  logging.info("Running tests")
174
175  # Count crashed tests.
176  crashed_tests = []
177
178  timeout_ms = '5000'
179  if options.time_out_ms:
180    timeout_ms = options.time_out_ms
181
182  # Run test until it's done
183
184  run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
185
186  run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
187  if options.rebaseline:
188    run_layout_test_cmd_postfix += " -e rebaseline true"
189  run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
190
191  # Call LayoutTestsAutoTest::startLayoutTests.
192  run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
193
194  adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
195  while not DumpRenderTreeFinished(adb_cmd):
196    # Get the running_test.txt
197    logging.error("DumpRenderTree crashed, output:\n" + adb_output)
198
199    shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
200    crashed_test = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE).communicate()[0]
201
202    logging.info(crashed_test + " CRASHED");
203    crashed_tests.append(crashed_test);
204
205    logging.info("Resuming layout test runner...");
206    # Call LayoutTestsAutoTest::resumeLayoutTests
207    run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
208
209    adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
210
211  if adb_output.find('INSTRUMENTATION_FAILED') != -1:
212    logging.error("Error happened : " + adb_output)
213    sys.exit(1)
214
215  logging.debug(adb_output);
216  logging.info("Done\n");
217
218  # Pull results from /sdcard
219  results_dir = options.results_directory
220  if not os.path.exists(results_dir):
221    os.makedirs(results_dir)
222  if not os.path.isdir(results_dir):
223    logging.error("Cannot create results dir: " + results_dir);
224    sys.exit(1);
225
226  result_files = ["/sdcard/layout_tests_passed.txt",
227                  "/sdcard/layout_tests_failed.txt",
228                  "/sdcard/layout_tests_nontext.txt"]
229  for file in result_files:
230    shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
231    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
232    logging.debug(adb_output)
233
234  # Create the crash list.
235  fp = open(results_dir + "/layout_tests_crashed.txt", "w");
236  fp.writelines('\n'.join(crashed_tests))
237  fp.close()
238
239  # Count the number of tests in each category.
240  passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
241  logging.info(str(passed_tests) + " passed")
242  failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
243  logging.info(str(failed_tests) + " failed")
244  crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
245  logging.info(str(crashed_tests) + " crashed")
246  nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
247  logging.info(str(nontext_tests) + " no dumpAsText")
248
249  logging.info("Results are stored under: " + results_dir + "\n")
250
251  # Comparing results to references to find new fixes and regressions.
252  results_dir = os.path.abspath(options.results_directory)
253  ref_dir = options.ref_directory
254
255  # if ref_dir is null, cannonify ref_dir to the script dir.
256  if not ref_dir:
257    script_self = sys.argv[0]
258    script_dir = os.path.dirname(script_self)
259    ref_dir = os.path.join(script_dir, "results")
260
261  ref_dir = os.path.abspath(ref_dir)
262
263  CompareResults(ref_dir, results_dir)
264
265if '__main__' == __name__:
266  option_parser = optparse.OptionParser()
267  option_parser.add_option("", "--rebaseline", action="store_true",
268                           default=False,
269                           help="generate expected results for those tests not having one")
270  option_parser.add_option("", "--time-out-ms",
271                           default=None,
272                           help="set the timeout for each test")
273  option_parser.add_option("", "--verbose", action="store_true",
274                           default=False,
275                           help="include debug-level logging")
276  option_parser.add_option("", "--refresh-test-list", action="store_true",
277                           default=False,
278                           help="re-generate test list, it may take some time.")
279  option_parser.add_option("", "--adb-options",
280                           default=None,
281                           help="pass options to adb, such as -d -e, etc");
282  option_parser.add_option("", "--results-directory",
283                           default="layout-test-results",
284                           help="directory which results are stored.")
285  option_parser.add_option("", "--ref-directory",
286                           default=None,
287                           dest="ref_directory",
288                           help="directory where reference results are stored.")
289
290  options, args = option_parser.parse_args();
291  main(options, args)
292