run_reliability_tests.py revision 2ab6f1fe0b17d281ea215f8ca412a5e1992011cc
1#!/usr/bin/python2.4 2 3"""Run reliability tests using Android instrumentation. 4 5 A test file consists of list web sites to test is needed as a parameter 6 7 Usage: 8 run_reliability_tests.py path/to/url/list 9""" 10 11import logging 12import optparse 13import os 14import subprocess 15import sys 16import time 17 18TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt" 19TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt" 20TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt" 21TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt" 22HTTP_URL_FILE = "urllist_http" 23HTTPS_URL_FILE = "urllist_https" 24NUM_URLS = 25 25 26 27def DumpRenderTreeFinished(adb_cmd): 28 """Check if DumpRenderTree finished running. 29 30 Args: 31 adb_cmd: adb command string 32 33 Returns: 34 True if DumpRenderTree has finished, False otherwise 35 """ 36 37 # pull test status file and look for "#DONE" 38 shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 39 adb_output = subprocess.Popen(shell_cmd_str, 40 shell=True, stdout=subprocess.PIPE, 41 stderr=subprocess.PIPE).communicate()[0] 42 return adb_output.strip() == "#DONE" 43 44 45def RemoveDeviceFile(adb_cmd, file_name): 46 shell_cmd_str = adb_cmd + " shell rm " + file_name 47 subprocess.Popen(shell_cmd_str, 48 shell=True, stdout=subprocess.PIPE, 49 stderr=subprocess.PIPE).communicate() 50 51 52def Bugreport(url, bugreport_dir, adb_cmd): 53 """Pull a bugreport from the device.""" 54 bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir, 55 int(time.time())) 56 57 # prepend the report with url 58 handle = open(bugreport_filename, "w") 59 handle.writelines("Bugreport for crash in url - %s\n\n" % url) 60 handle.close() 61 62 cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename) 63 os.system(cmd) 64 65 66def ProcessPageLoadTime(raw_log): 67 """Processes the raw page load time logged by test app.""" 68 log_handle = open(raw_log, "r") 69 load_times = {} 70 71 for line in log_handle: 72 line = line.strip() 73 pair = line.split("|") 74 if len(pair) != 2: 75 logging.info("Line has more than one '|': " + line) 76 continue 77 if pair[0] not in load_times: 78 load_times[pair[0]] = [0, 0] 79 try: 80 pair[1] = int(pair[1]) 81 except ValueError: 82 logging.info("Lins has non-numeric load time: " + line) 83 continue 84 load_times[pair[0]][0] += pair[1] 85 load_times[pair[0]][1] += 1 86 87 log_handle.close() 88 89 # rewrite the average time to file 90 log_handle = open(raw_log, "w") 91 for url, times in load_times.iteritems(): 92 log_handle.write("%s|%f\n" % (url, float(times[0]) / times[1])) 93 log_handle.close() 94 95 96def main(options, args): 97 """Send the url list to device and start testing, restart if crashed.""" 98 99 # Set up logging format. 100 log_level = logging.INFO 101 if options.verbose: 102 log_level = logging.DEBUG 103 logging.basicConfig(level=log_level, 104 format="%(message)s") 105 106 # Include all tests if none are specified. 107 if not args: 108 print "Missing URL list file" 109 sys.exit(1) 110 else: 111 path = args[0] 112 113 if not options.crash_file: 114 print "Missing crash file name, use --crash-file to specify" 115 sys.exit(1) 116 else: 117 crashed_file = options.crash_file 118 119 if not options.timeout_file: 120 print "Missing timeout file, use --timeout-file to specify" 121 sys.exit(1) 122 else: 123 timedout_file = options.timeout_file 124 125 if not options.delay: 126 manual_delay = 0 127 else: 128 manual_delay = options.delay 129 130 if not options.bugreport: 131 bugreport_dir = "." 132 else: 133 bugreport_dir = options.bugreport 134 if not os.path.exists(bugreport_dir): 135 os.makedirs(bugreport_dir) 136 if not os.path.isdir(bugreport_dir): 137 logging.error("Cannot create results dir: " + bugreport_dir) 138 sys.exit(1) 139 140 adb_cmd = "adb " 141 if options.adb_options: 142 adb_cmd += options.adb_options + " " 143 144 # push url list to device 145 test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\"" 146 proc = subprocess.Popen(test_cmd, shell=True, 147 stdout=subprocess.PIPE, 148 stderr=subprocess.PIPE) 149 (adb_output, adb_error) = proc.communicate() 150 if proc.returncode != 0: 151 logging.error("failed to push url list to device.") 152 logging.error(adb_output) 153 logging.error(adb_error) 154 sys.exit(1) 155 156 # clean up previous results 157 RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE) 158 RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE) 159 160 logging.info("Running the test ...") 161 162 # Count crashed tests. 163 crashed_tests = [] 164 165 if options.time_out_ms: 166 timeout_ms = options.time_out_ms 167 168 # Run test until it's done 169 test_cmd_prefix = adb_cmd + " shell am instrument" 170 test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner" 171 172 # Call ReliabilityTestsAutoTest#startReliabilityTests 173 test_cmd = (test_cmd_prefix + " -e class " 174 "com.android.dumprendertree.ReliabilityTest#" 175 "runReliabilityTest -e timeout %s -e delay %s" % 176 (str(timeout_ms), str(manual_delay))) 177 178 if options.logtime: 179 test_cmd += " -e logtime true" 180 181 test_cmd += test_cmd_postfix 182 183 adb_output = subprocess.Popen(test_cmd, shell=True, 184 stdout=subprocess.PIPE, 185 stderr=subprocess.PIPE).communicate()[0] 186 while not DumpRenderTreeFinished(adb_cmd): 187 logging.error("DumpRenderTree exited before all URLs are visited.") 188 shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 189 crashed_test = subprocess.Popen(shell_cmd_str, shell=True, 190 stdout=subprocess.PIPE).communicate()[0] 191 logging.info(crashed_test + " CRASHED") 192 crashed_tests.append(crashed_test) 193 Bugreport(crashed_test, bugreport_dir, adb_cmd) 194 logging.info("Resuming reliability test runner...") 195 196 adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 197 stderr=subprocess.PIPE).communicate()[0] 198 199 if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or 200 adb_output.find("Process crashed.") != -1): 201 logging.error("Error happened : " + adb_output) 202 sys.exit(1) 203 204 logging.info(adb_output) 205 logging.info("Done\n") 206 207 if crashed_tests: 208 file_handle = open(crashed_file, "w") 209 file_handle.writelines("\n".join(crashed_tests)) 210 logging.info("Crashed URL list stored in: " + crashed_file) 211 file_handle.close() 212 else: 213 logging.info("No crash found.") 214 215 # get timeout file from sdcard 216 test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \"" 217 + timedout_file + "\"") 218 subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 219 stderr=subprocess.PIPE).communicate() 220 221 if options.logtime: 222 # get logged page load times from sdcard 223 test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \"" 224 + options.logtime + "\"") 225 subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 226 stderr=subprocess.PIPE).communicate() 227 ProcessPageLoadTime(options.logtime) 228 229 230if "__main__" == __name__: 231 option_parser = optparse.OptionParser() 232 option_parser.add_option("-t", "--time-out-ms", 233 default=60000, 234 help="set the timeout for each test") 235 option_parser.add_option("-v", "--verbose", action="store_true", 236 default=False, 237 help="include debug-level logging") 238 option_parser.add_option("-a", "--adb-options", 239 default=None, 240 help="pass options to adb, such as -d -e, etc") 241 option_parser.add_option("-c", "--crash-file", 242 default="reliability_crashed_sites.txt", 243 help="the list of sites that cause browser to crash") 244 option_parser.add_option("-f", "--timeout-file", 245 default="reliability_timedout_sites.txt", 246 help="the list of sites that timedout during test") 247 option_parser.add_option("-d", "--delay", 248 default=0, 249 help="add a manual delay between pages (in ms)") 250 option_parser.add_option("-b", "--bugreport", 251 default=".", 252 help="the directory to store bugreport for crashes") 253 option_parser.add_option("-l", "--logtime", 254 default=None, 255 help="Logs page load time for each url to the file") 256 opts, arguments = option_parser.parse_args() 257 main(opts, arguments) 258