run_reliability_tests.py revision 3e8950c0c73f9c1574ce3388c754009edf6bc930
14010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu#!/usr/bin/python2.4 24010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 34010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu"""Run reliability tests using Android instrumentation. 44010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 54010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu A test file consists of list web sites to test is needed as a parameter 64010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 74010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu Usage: 84010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu run_reliability_tests.py path/to/url/list 94010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu""" 104010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 114010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhuimport logging 124010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhuimport optparse 134010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhuimport subprocess 144010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhuimport sys 154010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 164010ac35b1e49d659d7a32cc191302b8e2d8758aGuang ZhuTEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt" 174010ac35b1e49d659d7a32cc191302b8e2d8758aGuang ZhuTEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt" 184010ac35b1e49d659d7a32cc191302b8e2d8758aGuang ZhuTEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt" 194010ac35b1e49d659d7a32cc191302b8e2d8758aGuang ZhuHTTP_URL_FILE = "urllist_http" 204010ac35b1e49d659d7a32cc191302b8e2d8758aGuang ZhuHTTPS_URL_FILE = "urllist_https" 214010ac35b1e49d659d7a32cc191302b8e2d8758aGuang ZhuNUM_URLS = 25 224010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 234010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 244010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhudef DumpRenderTreeFinished(adb_cmd): 254010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu """Check if DumpRenderTree finished running. 264010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 274010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu Args: 284010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu adb_cmd: adb command string 294010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 304010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu Returns: 314010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu True if DumpRenderTree has finished, False otherwise 324010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu """ 334010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 344010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu # pull test status file and look for "#DONE" 354010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 364010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu adb_output = subprocess.Popen(shell_cmd_str, 374010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu shell=True, stdout=subprocess.PIPE, 384010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stderr=subprocess.PIPE).communicate()[0] 394010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu return adb_output.strip() == "#DONE" 404010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 414010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 4265455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhudef RemoveDeviceFile(adb_cmd, file_name): 4365455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu shell_cmd_str = adb_cmd + " shell rm " + file_name 4465455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu subprocess.Popen(shell_cmd_str, 4565455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu shell=True, stdout=subprocess.PIPE, 4665455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu stderr=subprocess.PIPE).communicate() 474010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 484010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 494010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhudef main(options, args): 504010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu """Send the url list to device and start testing, restart if crashed.""" 514010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 524010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu # Set up logging format. 534010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu log_level = logging.INFO 544010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if options.verbose: 554010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu log_level = logging.DEBUG 564010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.basicConfig(level=log_level, 574010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu format="%(message)s") 584010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 594010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu # Include all tests if none are specified. 604010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if not args: 6165455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu print "Missing URL list file" 6265455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu sys.exit(1) 634010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu else: 644010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu path = args[0] 654010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 664010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if not options.crash_file: 6765455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu print "Missing crash file name, use --crash-file to specify" 684010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu sys.exit(1) 694010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu else: 704010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu crashed_file = options.crash_file 714010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 724010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if not options.timeout_file: 7365455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu print "Missing timeout file, use --timeout-file to specify" 744010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu sys.exit(1) 754010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu else: 764010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu timedout_file = options.timeout_file 774010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 783e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu if not options.delay: 793e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu manual_delay = 0 803e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu else: 813e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu manual_delay = options.delay 823e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu 834010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu adb_cmd = "adb " 844010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if options.adb_options: 854010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu adb_cmd += options.adb_options + " " 864010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 874010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu # push url list to device 884010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\"" 894010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu proc = subprocess.Popen(test_cmd, shell=True, 904010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stdout=subprocess.PIPE, 914010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stderr=subprocess.PIPE) 924010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu (adb_output, adb_error) = proc.communicate() 934010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if proc.returncode != 0: 944010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.error("failed to push url list to device.") 954010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.error(adb_output) 964010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.error(adb_error) 974010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu sys.exit(1) 984010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 9965455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu # clean up previous results 10065455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE) 10165455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE) 10265455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu 1034010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.info("Running the test ...") 1044010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1054010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu # Count crashed tests. 1064010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu crashed_tests = [] 1074010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1084010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if options.time_out_ms: 1094010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu timeout_ms = options.time_out_ms 1104010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1114010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu # Run test until it's done 1124010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu test_cmd_prefix = adb_cmd + " shell am instrument" 1134010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner" 1144010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1154010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu # Call ReliabilityTestsAutoTest#startReliabilityTests 1164010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu test_cmd = (test_cmd_prefix + " -e class " 11765455a1a46e3bec020befd27ff5ad6bfdab61a7dGuang Zhu "com.android.dumprendertree.ReliabilityTest#" 1183e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu "runReliabilityTest -e timeout %s -e delay %s %s" % 1193e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu (str(timeout_ms), str(manual_delay), test_cmd_postfix)) 1204010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1214010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu adb_output = subprocess.Popen(test_cmd, shell=True, 1224010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stdout=subprocess.PIPE, 1234010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stderr=subprocess.PIPE).communicate()[0] 1244010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu while not DumpRenderTreeFinished(adb_cmd): 1254010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.error("DumpRenderTree exited before all URLs are visited.") 1264010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE 1274010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu crashed_test = subprocess.Popen(shell_cmd_str, shell=True, 1284010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stdout=subprocess.PIPE).communicate()[0] 1294010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.info(crashed_test + " CRASHED") 1304010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu crashed_tests.append(crashed_test) 1314010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.info("Resuming reliability test runner...") 1324010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1334010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 1344010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stderr=subprocess.PIPE).communicate()[0] 1354010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1364010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or 1374010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu adb_output.find("Process crashed.") != -1): 1384010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.error("Error happened : " + adb_output) 1394010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu sys.exit(1) 1404010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1414010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.info(adb_output) 1424010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.info("Done\n") 1434010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1444010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu if crashed_tests: 1454010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu file_handle = open(crashed_file, "w") 1464010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu file_handle.writelines("\n".join(crashed_tests)) 1474010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.info("Crashed URL list stored in: " + crashed_file) 1484010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu file_handle.close() 1494010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu else: 1504010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu logging.info("No crash found.") 1514010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1524010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \"" 1534010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu + timedout_file + "\"") 1544010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1554010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, 1564010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu stderr=subprocess.PIPE).communicate() 1574010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1584010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu 1594010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhuif "__main__" == __name__: 1604010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu option_parser = optparse.OptionParser() 1613e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu option_parser.add_option("-t", "--time-out-ms", 1624010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu default=60000, 1634010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu help="set the timeout for each test") 1643e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu option_parser.add_option("-v", "--verbose", action="store_true", 1654010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu default=False, 1664010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu help="include debug-level logging") 1673e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu option_parser.add_option("-a", "--adb-options", 1684010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu default=None, 1694010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu help="pass options to adb, such as -d -e, etc") 1703e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu option_parser.add_option("-c", "--crash-file", 1714010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu default="reliability_crashed_sites.txt", 1724010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu help="the list of sites that cause browser to crash") 1733e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu option_parser.add_option("-f", "--timeout-file", 1744010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu default="reliability_timedout_sites.txt", 1754010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu help="the list of sites that timedout during test.") 1763e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu option_parser.add_option("-d", "--delay", 1773e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu default=0, 1783e8950c0c73f9c1574ce3388c754009edf6bc930Guang Zhu help="add a manual delay between pages (in ms)") 1794010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu opts, arguments = option_parser.parse_args() 1804010ac35b1e49d659d7a32cc191302b8e2d8758aGuang Zhu main(opts, arguments) 181