1#!/usr/bin/python2.4
2
3"""Run reliability tests using Android instrumentation.
4
5  A test file consists of list web sites to test is needed as a parameter
6
7  Usage:
8    run_reliability_tests.py path/to/url/list
9"""
10
11import logging
12import optparse
13import os
14import subprocess
15import sys
16import time
17from Numeric import *
18
19TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
20TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
21TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
22TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt"
23HTTP_URL_FILE = "urllist_http"
24HTTPS_URL_FILE = "urllist_https"
25NUM_URLS = 25
26
27
28def DumpRenderTreeFinished(adb_cmd):
29  """Check if DumpRenderTree finished running.
30
31  Args:
32    adb_cmd: adb command string
33
34  Returns:
35    True if DumpRenderTree has finished, False otherwise
36  """
37
38  # pull test status file and look for "#DONE"
39  shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
40  adb_output = subprocess.Popen(shell_cmd_str,
41                                shell=True, stdout=subprocess.PIPE,
42                                stderr=subprocess.PIPE).communicate()[0]
43  return adb_output.strip() == "#DONE"
44
45
46def RemoveDeviceFile(adb_cmd, file_name):
47  shell_cmd_str = adb_cmd + " shell rm " + file_name
48  subprocess.Popen(shell_cmd_str,
49                   shell=True, stdout=subprocess.PIPE,
50                   stderr=subprocess.PIPE).communicate()
51
52
53def Bugreport(url, bugreport_dir, adb_cmd):
54  """Pull a bugreport from the device."""
55  bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
56                                                            int(time.time()))
57
58  # prepend the report with url
59  handle = open(bugreport_filename, "w")
60  handle.writelines("Bugreport for crash in url - %s\n\n" % url)
61  handle.close()
62
63  cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
64  os.system(cmd)
65
66
67def ProcessPageLoadTime(raw_log):
68  """Processes the raw page load time logged by test app."""
69  log_handle = open(raw_log, "r")
70  load_times = {}
71
72  for line in log_handle:
73    line = line.strip()
74    pair = line.split("|")
75    if len(pair) != 2:
76      logging.info("Line has more than one '|': " + line)
77      continue
78    if pair[0] not in load_times:
79      load_times[pair[0]] = []
80    try:
81      pair[1] = int(pair[1])
82    except ValueError:
83      logging.info("Lins has non-numeric load time: " + line)
84      continue
85    load_times[pair[0]].append(pair[1])
86
87  log_handle.close()
88
89  # rewrite the average time to file
90  log_handle = open(raw_log, "w")
91  for url, times in load_times.iteritems():
92    # calculate std
93    arr = array(times)
94    avg = average(arr)
95    d = arr - avg
96    std = sqrt(sum(d * d) / len(arr))
97    output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" %
98              (url, min(arr), max(arr), avg, std,
99               array2string(arr)))
100    log_handle.write(output)
101  log_handle.close()
102
103
104def main(options, args):
105  """Send the url list to device and start testing, restart if crashed."""
106
107  # Set up logging format.
108  log_level = logging.INFO
109  if options.verbose:
110    log_level = logging.DEBUG
111  logging.basicConfig(level=log_level,
112                      format="%(message)s")
113
114  # Include all tests if none are specified.
115  if not args:
116    print "Missing URL list file"
117    sys.exit(1)
118  else:
119    path = args[0]
120
121  if not options.crash_file:
122    print "Missing crash file name, use --crash-file to specify"
123    sys.exit(1)
124  else:
125    crashed_file = options.crash_file
126
127  if not options.timeout_file:
128    print "Missing timeout file, use --timeout-file to specify"
129    sys.exit(1)
130  else:
131    timedout_file = options.timeout_file
132
133  if not options.delay:
134    manual_delay = 0
135  else:
136    manual_delay = options.delay
137
138  if not options.bugreport:
139    bugreport_dir = "."
140  else:
141    bugreport_dir = options.bugreport
142  if not os.path.exists(bugreport_dir):
143    os.makedirs(bugreport_dir)
144  if not os.path.isdir(bugreport_dir):
145    logging.error("Cannot create results dir: " + bugreport_dir)
146    sys.exit(1)
147
148  adb_cmd = "adb "
149  if options.adb_options:
150    adb_cmd += options.adb_options + " "
151
152  # push url list to device
153  test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
154  proc = subprocess.Popen(test_cmd, shell=True,
155                          stdout=subprocess.PIPE,
156                          stderr=subprocess.PIPE)
157  (adb_output, adb_error) = proc.communicate()
158  if proc.returncode != 0:
159    logging.error("failed to push url list to device.")
160    logging.error(adb_output)
161    logging.error(adb_error)
162    sys.exit(1)
163
164  # clean up previous results
165  RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
166  RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
167  RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE)
168
169  logging.info("Running the test ...")
170
171  # Count crashed tests.
172  crashed_tests = []
173
174  if options.time_out_ms:
175    timeout_ms = options.time_out_ms
176
177  # Run test until it's done
178  test_cmd_prefix = adb_cmd + " shell am instrument"
179  test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
180
181  # Call ReliabilityTestsAutoTest#startReliabilityTests
182  test_cmd = (test_cmd_prefix + " -e class "
183              "com.android.dumprendertree.ReliabilityTest#"
184              "runReliabilityTest -e timeout %s -e delay %s" %
185              (str(timeout_ms), str(manual_delay)))
186
187  if options.logtime:
188    test_cmd += " -e logtime true"
189
190  test_cmd += test_cmd_postfix
191
192  adb_output = subprocess.Popen(test_cmd, shell=True,
193                                stdout=subprocess.PIPE,
194                                stderr=subprocess.PIPE).communicate()[0]
195  while not DumpRenderTreeFinished(adb_cmd):
196    logging.error("DumpRenderTree exited before all URLs are visited.")
197    shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
198    crashed_test = ""
199    while not crashed_test:
200      (crashed_test, err) = subprocess.Popen(
201          shell_cmd_str, shell=True, stdout=subprocess.PIPE,
202          stderr=subprocess.PIPE).communicate()
203      crashed_test = crashed_test.strip()
204      if not crashed_test:
205        logging.error('Cannot get crashed test name, device offline?')
206        logging.error('stderr: ' + err)
207        logging.error('retrying in 10s...')
208        time.sleep(10)
209
210    logging.info(crashed_test + " CRASHED")
211    crashed_tests.append(crashed_test)
212    Bugreport(crashed_test, bugreport_dir, adb_cmd)
213    logging.info("Resuming reliability test runner...")
214
215    adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
216                                  stderr=subprocess.PIPE).communicate()[0]
217
218  if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
219      adb_output.find("Process crashed.") != -1):
220    logging.error("Error happened : " + adb_output)
221    sys.exit(1)
222
223  logging.info(adb_output)
224  logging.info("Done\n")
225
226  if crashed_tests:
227    file_handle = open(crashed_file, "w")
228    file_handle.writelines("\n".join(crashed_tests))
229    logging.info("Crashed URL list stored in: " + crashed_file)
230    file_handle.close()
231  else:
232    logging.info("No crash found.")
233
234  # get timeout file from sdcard
235  test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
236              + timedout_file +  "\"")
237  subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
238                   stderr=subprocess.PIPE).communicate()
239
240  if options.logtime:
241    # get logged page load times from sdcard
242    test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \""
243                + options.logtime +  "\"")
244    subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
245                     stderr=subprocess.PIPE).communicate()
246    ProcessPageLoadTime(options.logtime)
247
248
249if "__main__" == __name__:
250  option_parser = optparse.OptionParser()
251  option_parser.add_option("-t", "--time-out-ms",
252                           default=60000,
253                           help="set the timeout for each test")
254  option_parser.add_option("-v", "--verbose", action="store_true",
255                           default=False,
256                           help="include debug-level logging")
257  option_parser.add_option("-a", "--adb-options",
258                           default=None,
259                           help="pass options to adb, such as -d -e, etc")
260  option_parser.add_option("-c", "--crash-file",
261                           default="reliability_crashed_sites.txt",
262                           help="the list of sites that cause browser to crash")
263  option_parser.add_option("-f", "--timeout-file",
264                           default="reliability_timedout_sites.txt",
265                           help="the list of sites that timedout during test")
266  option_parser.add_option("-d", "--delay",
267                           default=0,
268                           help="add a manual delay between pages (in ms)")
269  option_parser.add_option("-b", "--bugreport",
270                           default=".",
271                           help="the directory to store bugreport for crashes")
272  option_parser.add_option("-l", "--logtime",
273                           default=None,
274                           help="Logs page load time for each url to the file")
275  opts, arguments = option_parser.parse_args()
276  main(opts, arguments)
277