run_reliability_tests.py revision 65455a1a46e3bec020befd27ff5ad6bfdab61a7d
1#!/usr/bin/python2.4 2 3"""Run reliability tests using Android instrumentation. 4 5 A test file consists of list web sites to test is needed as a parameter 6 7 Usage: 8 run_reliability_tests.py path/to/url/list 9""" 10 11import logging 12import optparse 13import subprocess 14import sys 15 16TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt" 17TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt" 18TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt" 19HTTP_URL_FILE = "urllist_http" 20HTTPS_URL_FILE = "urllist_https" 21NUM_URLS = 25 22 23 24def DumpRenderTreeFinished(adb_cmd): 25 """Check if DumpRenderTree finished running. 26 27 Args: 28 adb_cmd: adb command string 29 30 Returns: 31 True if DumpRenderTree has finished, False otherwise 32 """ 33 34 # pull test status file and look for "#DONE" 35 shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 36 adb_output = subprocess.Popen(shell_cmd_str, 37 shell=True, stdout=subprocess.PIPE, 38 stderr=subprocess.PIPE).communicate()[0] 39 return adb_output.strip() == "#DONE" 40 41 42def RemoveDeviceFile(adb_cmd, file_name): 43 shell_cmd_str = adb_cmd + " shell rm " + file_name 44 subprocess.Popen(shell_cmd_str, 45 shell=True, stdout=subprocess.PIPE, 46 stderr=subprocess.PIPE).communicate() 47 48 49def main(options, args): 50 """Send the url list to device and start testing, restart if crashed.""" 51 52 # Set up logging format. 53 log_level = logging.INFO 54 if options.verbose: 55 log_level = logging.DEBUG 56 logging.basicConfig(level=log_level, 57 format="%(message)s") 58 59 # Include all tests if none are specified. 60 if not args: 61 print "Missing URL list file" 62 sys.exit(1) 63 else: 64 path = args[0] 65 66 if not options.crash_file: 67 print "Missing crash file name, use --crash-file to specify" 68 sys.exit(1) 69 else: 70 crashed_file = options.crash_file 71 72 if not options.timeout_file: 73 print "Missing timeout file, use --timeout-file to specify" 74 sys.exit(1) 75 else: 76 timedout_file = options.timeout_file 77 78 adb_cmd = "adb " 79 if options.adb_options: 80 adb_cmd += options.adb_options + " " 81 82 # push url list to device 83 test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\"" 84 proc = subprocess.Popen(test_cmd, shell=True, 85 stdout=subprocess.PIPE, 86 stderr=subprocess.PIPE) 87 (adb_output, adb_error) = proc.communicate() 88 if proc.returncode != 0: 89 logging.error("failed to push url list to device.") 90 logging.error(adb_output) 91 logging.error(adb_error) 92 sys.exit(1) 93 94 # clean up previous results 95 RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE) 96 RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE) 97 98 logging.info("Running the test ...") 99 100 # Count crashed tests. 101 crashed_tests = [] 102 103 if options.time_out_ms: 104 timeout_ms = options.time_out_ms 105 106 # Run test until it's done 107 test_cmd_prefix = adb_cmd + " shell am instrument" 108 test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner" 109 110 # Call ReliabilityTestsAutoTest#startReliabilityTests 111 test_cmd = (test_cmd_prefix + " -e class " 112 "com.android.dumprendertree.ReliabilityTest#" 113 "runTest -e timeout %d %s" % 114 (timeout_ms, test_cmd_postfix)) 115 116 adb_output = subprocess.Popen(test_cmd, shell=True, 117 stdout=subprocess.PIPE, 118 stderr=subprocess.PIPE).communicate()[0] 119 while not DumpRenderTreeFinished(adb_cmd): 120 logging.error("DumpRenderTree exited before all URLs are visited.") 121 shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 122 crashed_test = subprocess.Popen(shell_cmd_str, shell=True, 123 stdout=subprocess.PIPE).communicate()[0] 124 logging.info(crashed_test + " CRASHED") 125 crashed_tests.append(crashed_test) 126 logging.info("Resuming reliability test runner...") 127 128 test_cmd = (test_cmd_prefix + " -e class " 129 "com.android.dumprendertree.ReliabilityTest#" 130 "runTest -e timeout %d %s" % 131 (timeout_ms, test_cmd_postfix)) 132 adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 133 stderr=subprocess.PIPE).communicate()[0] 134 135 if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or 136 adb_output.find("Process crashed.") != -1): 137 logging.error("Error happened : " + adb_output) 138 sys.exit(1) 139 140 logging.info(adb_output) 141 logging.info("Done\n") 142 143 if crashed_tests: 144 file_handle = open(crashed_file, "w") 145 file_handle.writelines("\n".join(crashed_tests)) 146 logging.info("Crashed URL list stored in: " + crashed_file) 147 file_handle.close() 148 else: 149 logging.info("No crash found.") 150 151 test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \"" 152 + timedout_file + "\"") 153 154 subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 155 stderr=subprocess.PIPE).communicate() 156 157 158if "__main__" == __name__: 159 option_parser = optparse.OptionParser() 160 option_parser.add_option("", "--time-out-ms", 161 default=60000, 162 help="set the timeout for each test") 163 option_parser.add_option("", "--verbose", action="store_true", 164 default=False, 165 help="include debug-level logging") 166 option_parser.add_option("", "--adb-options", 167 default=None, 168 help="pass options to adb, such as -d -e, etc") 169 option_parser.add_option("", "--crash-file", 170 default="reliability_crashed_sites.txt", 171 help="the list of sites that cause browser to crash") 172 option_parser.add_option("", "--timeout-file", 173 default="reliability_timedout_sites.txt", 174 help="the list of sites that timedout during test.") 175 opts, arguments = option_parser.parse_args() 176 main(opts, arguments) 177