run_reliability_tests.py revision 17f8fa6b24da34724b4497622f374b27f3041e1c 
1#!/usr/bin/python2.4
2
3"""Run reliability tests using Android instrumentation.
4
5  A test file consists of list web sites to test is needed as a parameter
6
7  Usage:
8    run_reliability_tests.py path/to/url/list
9"""
10
11import logging
12import optparse
13import os
14import subprocess
15import sys
16import time
17
18TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
19TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
20TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
21HTTP_URL_FILE = "urllist_http"
22HTTPS_URL_FILE = "urllist_https"
23NUM_URLS = 25
24
25
26def DumpRenderTreeFinished(adb_cmd):
27  """Check if DumpRenderTree finished running.
28
29  Args:
30    adb_cmd: adb command string
31
32  Returns:
33    True if DumpRenderTree has finished, False otherwise
34  """
35
36  # pull test status file and look for "#DONE"
37  shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
38  adb_output = subprocess.Popen(shell_cmd_str,
39                                shell=True, stdout=subprocess.PIPE,
40                                stderr=subprocess.PIPE).communicate()[0]
41  return adb_output.strip() == "#DONE"
42
43
44def RemoveDeviceFile(adb_cmd, file_name):
45  shell_cmd_str = adb_cmd + " shell rm " + file_name
46  subprocess.Popen(shell_cmd_str,
47                   shell=True, stdout=subprocess.PIPE,
48                   stderr=subprocess.PIPE).communicate()
49
50
51def Bugreport(url, bugreport_dir, adb_cmd):
52  """Pull a bugreport from the device."""
53  bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
54                                                            int(time.time()))
55
56  # prepend the report with url
57  handle = open(bugreport_filename, "w")
58  handle.writelines("Bugreport for crash in url - %s\n\n" % url)
59  handle.close()
60
61  cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
62  os.system(cmd)
63
64
65def main(options, args):
66  """Send the url list to device and start testing, restart if crashed."""
67
68  # Set up logging format.
69  log_level = logging.INFO
70  if options.verbose:
71    log_level = logging.DEBUG
72  logging.basicConfig(level=log_level,
73                      format="%(message)s")
74
75  # Include all tests if none are specified.
76  if not args:
77    print "Missing URL list file"
78    sys.exit(1)
79  else:
80    path = args[0]
81
82  if not options.crash_file:
83    print "Missing crash file name, use --crash-file to specify"
84    sys.exit(1)
85  else:
86    crashed_file = options.crash_file
87
88  if not options.timeout_file:
89    print "Missing timeout file, use --timeout-file to specify"
90    sys.exit(1)
91  else:
92    timedout_file = options.timeout_file
93
94  if not options.delay:
95    manual_delay = 0
96  else:
97    manual_delay = options.delay
98
99  if not options.bugreport:
100    bugreport_dir = "."
101  else:
102    bugreport_dir = options.bugreport
103  if not os.path.exists(bugreport_dir):
104    os.makedirs(bugreport_dir)
105  if not os.path.isdir(bugreport_dir):
106    logging.error("Cannot create results dir: " + bugreport_dir)
107    sys.exit(1)
108
109  adb_cmd = "adb "
110  if options.adb_options:
111    adb_cmd += options.adb_options + " "
112
113  # push url list to device
114  test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
115  proc = subprocess.Popen(test_cmd, shell=True,
116                          stdout=subprocess.PIPE,
117                          stderr=subprocess.PIPE)
118  (adb_output, adb_error) = proc.communicate()
119  if proc.returncode != 0:
120    logging.error("failed to push url list to device.")
121    logging.error(adb_output)
122    logging.error(adb_error)
123    sys.exit(1)
124
125  # clean up previous results
126  RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
127  RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
128
129  logging.info("Running the test ...")
130
131  # Count crashed tests.
132  crashed_tests = []
133
134  if options.time_out_ms:
135    timeout_ms = options.time_out_ms
136
137  # Run test until it's done
138  test_cmd_prefix = adb_cmd + " shell am instrument"
139  test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
140
141  # Call ReliabilityTestsAutoTest#startReliabilityTests
142  test_cmd = (test_cmd_prefix + " -e class "
143              "com.android.dumprendertree.ReliabilityTest#"
144              "runReliabilityTest -e timeout %s -e delay %s %s" %
145              (str(timeout_ms), str(manual_delay), test_cmd_postfix))
146
147  adb_output = subprocess.Popen(test_cmd, shell=True,
148                                stdout=subprocess.PIPE,
149                                stderr=subprocess.PIPE).communicate()[0]
150  while not DumpRenderTreeFinished(adb_cmd):
151    logging.error("DumpRenderTree exited before all URLs are visited.")
152    shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
153    crashed_test = subprocess.Popen(shell_cmd_str, shell=True,
154                                    stdout=subprocess.PIPE).communicate()[0]
155    logging.info(crashed_test + " CRASHED")
156    crashed_tests.append(crashed_test)
157    Bugreport(crashed_test, bugreport_dir, adb_cmd)
158    logging.info("Resuming reliability test runner...")
159
160    adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
161                                  stderr=subprocess.PIPE).communicate()[0]
162
163  if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
164      adb_output.find("Process crashed.") != -1):
165    logging.error("Error happened : " + adb_output)
166    sys.exit(1)
167
168  logging.info(adb_output)
169  logging.info("Done\n")
170
171  if crashed_tests:
172    file_handle = open(crashed_file, "w")
173    file_handle.writelines("\n".join(crashed_tests))
174    logging.info("Crashed URL list stored in: " + crashed_file)
175    file_handle.close()
176  else:
177    logging.info("No crash found.")
178
179  test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
180              + timedout_file +  "\"")
181
182  subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
183                   stderr=subprocess.PIPE).communicate()
184
185
186if "__main__" == __name__:
187  option_parser = optparse.OptionParser()
188  option_parser.add_option("-t", "--time-out-ms",
189                           default=60000,
190                           help="set the timeout for each test")
191  option_parser.add_option("-v", "--verbose", action="store_true",
192                           default=False,
193                           help="include debug-level logging")
194  option_parser.add_option("-a", "--adb-options",
195                           default=None,
196                           help="pass options to adb, such as -d -e, etc")
197  option_parser.add_option("-c", "--crash-file",
198                           default="reliability_crashed_sites.txt",
199                           help="the list of sites that cause browser to crash")
200  option_parser.add_option("-f", "--timeout-file",
201                           default="reliability_timedout_sites.txt",
202                           help="the list of sites that timedout during test")
203  option_parser.add_option("-d", "--delay",
204                           default=0,
205                           help="add a manual delay between pages (in ms)")
206  option_parser.add_option("-b", "--bugreport",
207                           default=".",
208                           help="the directory to store bugreport for crashes")
209  opts, arguments = option_parser.parse_args()
210  main(opts, arguments)
211