17e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi#!/usr/bin/python 27e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi# 37e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 47e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi# Use of this source code is governed by a BSD-style license that can be 57e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi# found in the LICENSE file. 67e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 77e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi"""Tool to validate code in prod branch before pushing to lab. 87e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 97e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiThe script runs push_to_prod suite to verify code in prod branch is ready to be 107e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shipushed. Link to design document: 117e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shihttps://docs.google.com/a/google.com/document/d/1JMz0xS3fZRSHMpFkkKAL_rxsdbNZomhHbC3B8L71uuI/edit 127e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 137e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiTo verify if prod branch can be pushed to lab, run following command in 147e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shichromeos-autotest.cbf server: 1552d9f1ff5cadba2828e64288afb5491122e3078dMichael Liang/usr/local/autotest/site_utils/test_push.py -e someone@company.com 167e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 177e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiThe script uses latest stumpy canary build as test build by default. 187e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 197e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi""" 207e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 217e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport argparse 227e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport getpass 23ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shiimport multiprocessing 247e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport os 257e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport re 267e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport subprocess 277e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport sys 28ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shiimport time 29ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shiimport traceback 307e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport urllib2 317e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 327e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiimport common 33a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shitry: 34a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi from autotest_lib.frontend import setup_django_environment 35a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi from autotest_lib.frontend.afe import models 36a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shiexcept ImportError: 37a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi # Unittest may not have Django database configured and will fail to import. 38a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi pass 395fa602cbd7787ff604758f03b71da19b66263ca1Dan Shifrom autotest_lib.client.common_lib import global_config 407e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shifrom autotest_lib.server import site_utils 4147d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shifrom autotest_lib.server.cros import provision 427e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shifrom autotest_lib.server.cros.dynamic_suite import frontend_wrappers 437e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shifrom autotest_lib.server.cros.dynamic_suite import reporting 446e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Chengfrom autotest_lib.server.hosts import factory 455fa602cbd7787ff604758f03b71da19b66263ca1Dan Shifrom autotest_lib.site_utils import gmail_lib 4647d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shifrom autotest_lib.site_utils.suite_scheduler import constants 477e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 487e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiCONFIG = global_config.global_config 497e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 50efd403e23fe939e464364c880a9f054005b48545Dan ShiAFE = frontend_wrappers.RetryingAFE(timeout_min=0.5, delay_sec=2) 51efd403e23fe939e464364c880a9f054005b48545Dan Shi 527e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiMAIL_FROM = 'chromeos-test@google.com' 535ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan ShiDEVSERVERS = CONFIG.get_config_value('CROS', 'dev_server', type=list, 545ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi default=[]) 555ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan ShiBUILD_REGEX = '^R[\d]+-[\d]+\.[\d]+\.[\d]+$' 567e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiRUN_SUITE_COMMAND = 'run_suite.py' 577e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiPUSH_TO_PROD_SUITE = 'push_to_prod' 588f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob JuelichDUMMY_SUITE = 'dummy' 597e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiAU_SUITE = 'paygen_au_canary' 607e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 616dddf60ffaf30337e17d393eb797f67f2419f02bFang DengSUITE_JOB_START_INFO_REGEX = ('^.*Created suite job:.*' 626dddf60ffaf30337e17d393eb797f67f2419f02bFang Deng 'tab_id=view_job&object_id=(\d+)$') 637e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 647e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi# Dictionary of test results keyed by test name regular expression. 657e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiEXPECTED_TEST_RESULTS = {'^SERVER_JOB$': 'GOOD', 667e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # This is related to dummy_Fail/control.dependency. 677e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.dependency$': 'TEST_NA', 68dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi 'login_LoginSuccess.*': 'GOOD', 697e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'platform_InstallTestImage_SERVER_JOB$': 'GOOD', 7047d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shi 'provision_AutoUpdate.double': 'GOOD', 717e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Pass.*': 'GOOD', 727e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.Fail$': 'FAIL', 737e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.RetryFail$': 'FAIL', 747e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.RetrySuccess': 'GOOD', 757e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.Error$': 'ERROR', 767e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.Warn$': 'WARN', 777e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.NAError$': 'TEST_NA', 787e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'dummy_Fail.Crash$': 'GOOD', 797e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi } 807e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 818f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob JuelichEXPECTED_TEST_RESULTS_DUMMY = {'^SERVER_JOB$': 'GOOD', 828f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'dummy_Pass.*': 'GOOD', 838f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'dummy_Fail.Fail': 'FAIL', 848f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'dummy_Fail.Warn': 'WARN', 858f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'dummy_Fail.Crash': 'GOOD', 868f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'dummy_Fail.Error': 'ERROR', 878f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'dummy_Fail.NAError': 'TEST_NA',} 888f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 897e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiEXPECTED_TEST_RESULTS_AU = {'SERVER_JOB$': 'GOOD', 9015e6d724dff398a8a30788360c25c16b0f256ef8Dan Shi 'autoupdate_EndToEndTest.paygen_au_canary_delta.*': 'GOOD', 9115e6d724dff398a8a30788360c25c16b0f256ef8Dan Shi 'autoupdate_EndToEndTest.paygen_au_canary_full.*': 'GOOD', 927e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi } 937e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 947e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi# Anchor for the auto-filed bug for dummy_Fail tests. 957e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiBUG_ANCHOR = 'TestFailure(push_to_prod,dummy_Fail.Fail,always fail)' 967e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 977e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiURL_HOST = CONFIG.get_config_value('SERVER', 'hostname', type=str) 987e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan ShiURL_PATTERN = CONFIG.get_config_value('CROS', 'log_url_pattern', type=str) 997e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 100dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi# Some test could be missing from the test results for various reasons. Add 101dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi# such test in this list and explain the reason. 102dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan ShiIGNORE_MISSING_TESTS = [ 103dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi # For latest build, npo_test_delta does not exist. 104dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi 'autoupdate_EndToEndTest.npo_test_delta.*', 105dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi # For trybot build, nmo_test_delta does not exist. 106dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi 'autoupdate_EndToEndTest.nmo_test_delta.*', 107dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi # Older build does not have login_LoginSuccess test in push_to_prod suite. 108dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi # TODO(dshi): Remove following lines after R41 is stable. 109dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi 'login_LoginSuccess'] 110dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi 1117e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi# Save all run_suite command output. 1127e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shirun_suite_output = [] 1137e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 1147e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiclass TestPushException(Exception): 1157e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Exception to be raised when the test to push to prod failed.""" 1167e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi pass 1177e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 1185ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi 1196e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Chengdef powerwash_dut(hostname): 1206e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng """Powerwash the dut with the given hostname. 1216e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng 1226e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng @param hostname: hostname of the dut. 1236e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng """ 1246e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng host = factory.create_host(hostname) 1256e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng host.run('echo "fast safe" > ' 1266e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng '/mnt/stateful_partition/factory_install_reset') 1276e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng host.run('reboot') 1286e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng host.close() 1296e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng 1306e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng 1315ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shidef get_default_build(devserver=None, board='stumpy'): 1325ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi """Get the default build to be used for test. 1335ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi 1345ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi @param devserver: devserver used to look for latest staged build. If value 1355ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi is None, all devservers in config will be tried. 1365ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi @param board: Name of board to be tested, default is stumpy. 1375ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi @return: Build to be tested, e.g., stumpy-release/R36-5881.0.0 1385ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi """ 1395ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi LATEST_BUILD_URL_PATTERN = '%s/latestbuild?target=%s-release' 1405ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi build = None 1415ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi if not devserver: 1425ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi for server in DEVSERVERS: 143ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi url = LATEST_BUILD_URL_PATTERN % (server, board) 144ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi build = urllib2.urlopen(url).read() 145ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi if build and re.match(BUILD_REGEX, build): 146ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi return '%s-release/%s' % (board, build) 1475ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi 1485ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi # If no devserver has any build staged for the given board, use the stable 1495ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi # build in config. 1505ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi build = CONFIG.get_config_value('CROS', 'stable_cros_version') 1515ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi return '%s-release/%s' % (board, build) 1525ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi 1535ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi 1547e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shidef parse_arguments(): 1557e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Parse arguments for test_push tool. 1567e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 1577e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @return: Parsed arguments. 1587e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 1597e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 1607e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser = argparse.ArgumentParser() 1617e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser.add_argument('-b', '--board', dest='board', default='stumpy', 1627e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi help='Default is stumpy.') 1638f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich parser.add_argument('-sb', '--shard_board', dest='shard_board', 1648f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich default='quawks', 1658f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich help='Default is quawks.') 1667e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser.add_argument('-i', '--build', dest='build', default=None, 1677e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi help='Default is the latest canary build of given ' 1687e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'board. Must be a canary build, otherwise AU test ' 1697e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'will fail.') 1708f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich parser.add_argument('-si', '--shard_build', dest='shard_build', default=None, 1718f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich help='Default is the latest canary build of given ' 1728f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'board. Must be a canary build, otherwise AU test ' 1738f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 'will fail.') 1747e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser.add_argument('-p', '--pool', dest='pool', default='bvt') 1757e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser.add_argument('-u', '--num', dest='num', type=int, default=3, 1767e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi help='Run on at most NUM machines.') 1777e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser.add_argument('-f', '--file_bugs', dest='file_bugs', default='True', 1787e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi help='File bugs on test failures. Must pass "True" or ' 1797e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi '"False" if used.') 1807e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser.add_argument('-e', '--email', dest='email', default=None, 1817e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi help='Email address for the notification to be sent to ' 1827e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'after the script finished running.') 1837e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi parser.add_argument('-d', '--devserver', dest='devserver', 1845ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi default=None, 1857e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi help='devserver to find what\'s the latest build.') 186d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao parser.add_argument('-t', '--timeout_min', dest='timeout_min', type=int, 187d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao default=24, 188d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao help='Time in mins to wait before abort the jobs we ' 189d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao 'are waiting on. Only for the asynchronous suites ' 190d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao 'triggered by create_and_return flag.') 1917e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 1927e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi arguments = parser.parse_args(sys.argv[1:]) 1937e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 1947e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Get latest canary build as default build. 1957e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not arguments.build: 1965ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi arguments.build = get_default_build(arguments.devserver, 1975ba5d2e1433f180e2eaeac5721547e6f61ab30a7Dan Shi arguments.board) 1988f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich if not arguments.shard_build: 1998f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich arguments.shard_build = get_default_build(arguments.devserver, 2008f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich arguments.shard_board) 2017e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2027e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi return arguments 2037e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2047e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 205d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhaodef do_run_suite(suite_name, arguments, use_shard=False, 206d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao create_and_return=False): 2077e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Call run_suite to run a suite job, and return the suite job id. 2087e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2097e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi The script waits the suite job to finish before returning the suite job id. 2107e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi Also it will echo the run_suite output to stdout. 2117e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2127e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param suite_name: Name of a suite, e.g., dummy. 2137e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param arguments: Arguments for run_suite command. 2148f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich @param use_shard: If true, suite is scheduled for shard board. 215d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao @param create_and_return: If True, run_suite just creates the suite, print 216d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao the job id, then finish immediately. 2178f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 2187e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @return: Suite job ID. 2197e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2207e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 2218f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich if not use_shard: 2228f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich board = arguments.board 2238f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich build = arguments.build 2248f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich else: 2258f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich board = arguments.shard_board 2268f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich build = arguments.shard_build 2278f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 22847d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shi # Remove cros-version label to force provision. 229efd403e23fe939e464364c880a9f054005b48545Dan Shi hosts = AFE.get_hosts(label=constants.Labels.BOARD_PREFIX+board) 23047d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shi for host in hosts: 23147d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shi for label in [l for l in host.labels 23247d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shi if l.startswith(provision.CROS_VERSION_PREFIX)]: 233efd403e23fe939e464364c880a9f054005b48545Dan Shi AFE.run('host_remove_labels', id=host.id, labels=[label]) 23447d3288e3ad2feba6c866ec369f2cfa8abb9346aDan Shi 2356e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng if use_shard and not create_and_return: 2366e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng # Let's verify the repair flow and powerwash the duts. We can 2376e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng # assume they're all cros hosts (valid assumption?) so powerwash 2386e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng # will work. 2396e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng try: 2406e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng powerwash_dut(host.hostname) 2416e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng except Exception as e: 2426e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng raise TestPushException('Failed to powerwash dut %s. Make ' 2436e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng 'sure the dut is working first. ' 2446e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng 'Error: %s' % (host.hostname, e)) 245efd403e23fe939e464364c880a9f054005b48545Dan Shi AFE.reverify_hosts(hostnames=[host.hostname]) 2466e4c264cbc80979cf9e80478d9e99ed532e64790Kevin Cheng 247ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi current_dir = os.path.dirname(os.path.realpath(__file__)) 248ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi cmd = [os.path.join(current_dir, RUN_SUITE_COMMAND), 2497e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi '-s', suite_name, 2508f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich '-b', board, 2518f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich '-i', build, 2527e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi '-p', arguments.pool, 2537e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi '-u', str(arguments.num), 2547e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi '-f', arguments.file_bugs] 255d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao if create_and_return: 256d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao cmd += ['-c'] 2577e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2587e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi suite_job_id = None 2597e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2607e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, 2617e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi stderr=subprocess.STDOUT) 2627e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2637e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi while True: 2647e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi line = proc.stdout.readline() 2657e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2667e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Break when run_suite process completed. 2677e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not line and proc.poll() != None: 2687e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi break 2697e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi print line.rstrip() 2707e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi run_suite_output.append(line.rstrip()) 2717e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2727e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not suite_job_id: 2737e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi m = re.match(SUITE_JOB_START_INFO_REGEX, line) 2747e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if m and m.group(1): 2757e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi suite_job_id = int(m.group(1)) 2767e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2777e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not suite_job_id: 2787e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise TestPushException('Failed to retrieve suite job ID.') 279a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi 280d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao # If create_and_return specified, wait for the suite to finish. 281d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao if create_and_return: 282d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao end = time.time() + arguments.timeout_min * 60 283efd403e23fe939e464364c880a9f054005b48545Dan Shi while not AFE.get_jobs(id=suite_job_id, finished=True): 284d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao if time.time() < end: 285d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao time.sleep(10) 286d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao else: 287efd403e23fe939e464364c880a9f054005b48545Dan Shi AFE.run('abort_host_queue_entries', job=suite_job_id) 288d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao raise TestPushException( 289d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao 'Asynchronous suite triggered by create_and_return ' 290d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao 'flag has timed out after %d mins. Aborting it.' % 291d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao arguments.timeout_min) 292d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao 293a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi print 'Suite job %s is completed.' % suite_job_id 2947e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi return suite_job_id 2957e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2967e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 297a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shidef check_dut_image(build, suite_job_id): 298a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi """Confirm all DUTs used for the suite are imaged to expected build. 299a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi 300a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi @param build: Expected build to be imaged. 301a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi @param suite_job_id: job ID of the suite job. 302a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi @raise TestPushException: If a DUT does not have expected build imaged. 303a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi """ 304a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi print 'Checking image installed in DUTs...' 305a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi job_ids = [job.id for job in 306a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi models.Job.objects.filter(parent_job_id=suite_job_id)] 307a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi hqes = [models.HostQueueEntry.objects.filter(job_id=job_id)[0] 308a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi for job_id in job_ids] 309a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi hostnames = set([hqe.host.hostname for hqe in hqes]) 310a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi for hostname in hostnames: 311efd403e23fe939e464364c880a9f054005b48545Dan Shi found_build = site_utils.get_build_from_afe(hostname, AFE) 312a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi if found_build != build: 313a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi raise TestPushException('DUT is not imaged properly. Host %s has ' 314a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi 'build %s, while build %s is expected.' % 315a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi (hostname, found_build, build)) 316a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi 317a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi 318d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhaodef test_suite(suite_name, expected_results, arguments, use_shard=False, 319d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao create_and_return=False): 3207e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Call run_suite to start a suite job and verify results. 3217e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3227e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param suite_name: Name of a suite, e.g., dummy 3237e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param expected_results: A dictionary of test name to test result. 3247e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param arguments: Arguments for run_suite command. 3258f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich @param use_shard: If true, suite is scheduled for shard board. 326d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao @param create_and_return: If True, run_suite just creates the suite, print 327d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao the job id, then finish immediately. 3287e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 329d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao suite_job_id = do_run_suite(suite_name, arguments, use_shard, 330d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao create_and_return) 3317e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 332a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi # Confirm all DUTs used for the suite are imaged to expected build. 3338f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich # hqe.host_id for jobs running in shard is not synced back to master db, 3348f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich # therefore, skip verifying dut build for jobs running in shard. 3358f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich if suite_name != AU_SUITE and not use_shard: 336a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi check_dut_image(arguments.build, suite_job_id) 337a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi 3387e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Find all tests and their status 339a8da7602ee0113004ef1b7213eab5599e3236ae7Dan Shi print 'Comparing test results...' 3407e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi TKO = frontend_wrappers.RetryingTKO(timeout_min=0.1, delay_sec=10) 3417e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_views = site_utils.get_test_views_from_tko(suite_job_id, TKO) 3427e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3437e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi mismatch_errors = [] 3447e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi extra_test_errors = [] 3457e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3467e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi found_keys = set() 3477e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi for test_name,test_status in test_views.items(): 3487e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi print "%s%s" % (test_name.ljust(30), test_status) 3497e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_found = False 3507e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi for key,val in expected_results.items(): 3517e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if re.search(key, test_name): 3527e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_found = True 3537e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi found_keys.add(key) 3547e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # TODO(dshi): result for this test is ignored until servo is 3557e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # added to a host accessible by cbf server (crbug.com/277109). 3567e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if key == 'platform_InstallTestImage_SERVER_JOB$': 3577e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi continue 3587e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if val != test_status: 3597e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi error = ('%s Expected: [%s], Actual: [%s]' % 3607e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi (test_name, val, test_status)) 3617e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi mismatch_errors.append(error) 3627e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not test_found: 3637e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi extra_test_errors.append(test_name) 3647e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3657e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi missing_test_errors = set(expected_results.keys()) - found_keys 366dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi for exception in IGNORE_MISSING_TESTS: 367dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi try: 368dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi missing_test_errors.remove(exception) 369dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi except KeyError: 370dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi pass 371dc9eb17ac97d5407a2ebce0510fe0311b3f18bfcDan Shi 3727e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary = [] 3737e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if mismatch_errors: 3747e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.append(('Results of %d test(s) do not match expected ' 3757e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'values:') % len(mismatch_errors)) 3767e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.extend(mismatch_errors) 3777e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.append('\n') 3787e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3797e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if extra_test_errors: 3807e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.append('%d test(s) are not expected to be run:' % 3817e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi len(extra_test_errors)) 3827e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.extend(extra_test_errors) 3837e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.append('\n') 3847e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3857e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if missing_test_errors: 3867e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.append('%d test(s) are missing from the results:' % 3877e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi len(missing_test_errors)) 3887e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.extend(missing_test_errors) 3897e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.append('\n') 3907e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3917e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Test link to log can be loaded. 3927e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi job_name = '%s-%s' % (suite_job_id, getpass.getuser()) 3937e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi log_link = URL_PATTERN % (URL_HOST, job_name) 3947e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi try: 3957e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi urllib2.urlopen(log_link).read() 3967e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi except urllib2.URLError: 3977e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi summary.append('Failed to load page for link to log: %s.' % log_link) 3987e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3997e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if summary: 4007e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise TestPushException('\n'.join(summary)) 4017e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 4027e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 403ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shidef test_suite_wrapper(queue, suite_name, expected_results, arguments, 404d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao use_shard=False, create_and_return=False): 405ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi """Wrapper to call test_suite. Handle exception and pipe it to parent 406ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi process. 407ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 408ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi @param queue: Queue to save exception to be accessed by parent process. 409ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi @param suite_name: Name of a suite, e.g., dummy 410ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi @param expected_results: A dictionary of test name to test result. 411ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi @param arguments: Arguments for run_suite command. 412ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi @param use_shard: If true, suite is scheduled for shard board. 413d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao @param create_and_return: If True, run_suite just creates the suite, print 414d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao the job id, then finish immediately. 415ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi """ 416ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi try: 417d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao test_suite(suite_name, expected_results, arguments, use_shard, 418d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao create_and_return) 419ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi except: 420ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi # Store the whole exc_info leads to a PicklingError. 421ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi except_type, except_value, tb = sys.exc_info() 422ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi queue.put((except_type, except_value, traceback.extract_tb(tb))) 423ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 424ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 4257e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shidef close_bug(): 4267e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Close all existing bugs filed for dummy_Fail. 4277e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 4287e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @return: A list of issue ids to be used in check_bug_filed_and_deduped. 4297e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 4307e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi old_issue_ids = [] 4317e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi reporter = reporting.Reporter() 4327e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi while True: 4337e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi issue = reporter.find_issue_by_marker(BUG_ANCHOR) 4347e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not issue: 4357e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi return old_issue_ids 4367e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if issue.id in old_issue_ids: 4377e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise TestPushException('Failed to close issue %d' % issue.id) 4387e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi old_issue_ids.append(issue.id) 4397e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi reporter.modify_bug_report(issue.id, 4407e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi comment='Issue closed by test_push script.', 4417e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi label_update='', 4427e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi status='WontFix') 4437e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 4447e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 4457e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shidef check_bug_filed_and_deduped(old_issue_ids): 4467e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Confirm bug related to dummy_Fail was filed and deduped. 4477e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 4487e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param old_issue_ids: A list of issue ids that was closed earlier. id of the 4497e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi new issue must be not in this list. 4507e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @raise TestPushException: If auto bug file failed to create a new issue or 4517e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi dedupe multiple failures. 4527e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 4537e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi reporter = reporting.Reporter() 4547e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi issue = reporter.find_issue_by_marker(BUG_ANCHOR) 4557e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not issue: 4567e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise TestPushException('Auto bug file failed. Unable to locate bug ' 4577e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'with marker %s' % BUG_ANCHOR) 4587e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if old_issue_ids and issue.id in old_issue_ids: 4597e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise TestPushException('Auto bug file failed to create a new issue. ' 4607e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'id of the old issue found is %d.' % issue.id) 4617e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not ('%s2' % reporter.AUTOFILED_COUNT) in issue.labels: 4627e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise TestPushException(('Auto bug file failed to dedupe for issue %d ' 4637e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'with labels of %s.') % 4647e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi (issue.id, issue.labels)) 4657e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Close the bug, and do the search again, which should return None. 4667e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi reporter.modify_bug_report(issue.id, 4677e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi comment='Issue closed by test_push script.', 4687e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi label_update='', 4697e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi status='WontFix') 4707e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi second_issue = reporter.find_issue_by_marker(BUG_ANCHOR) 4717e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if second_issue: 4727e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi ids = '%d, %d' % (issue.id, second_issue.id) 4737e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise TestPushException(('Auto bug file failed. Multiple issues (%s) ' 4747e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 'filed with marker %s') % (ids, BUG_ANCHOR)) 4757e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi print 'Issue %d was filed and deduped successfully.' % issue.id 4767e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 4777e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 478ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shidef check_queue(queue): 479ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi """Check the queue for any exception being raised. 480ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 481ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi @param queue: Queue used to store exception for parent process to access. 482ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi @raise: Any exception found in the queue. 483ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi """ 484ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi if queue.empty(): 485ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi return 486ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi exc_info = queue.get() 487ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi # Raise the exception with original backtrace. 488ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi print 'Original stack trace of the exception:\n%s' % exc_info[2] 489ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi raise exc_info[0](exc_info[1]) 490ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 491ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 4927e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shidef main(): 4937e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Entry point for test_push script.""" 4947e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi arguments = parse_arguments() 4957e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 4967e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi try: 4977e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Close existing bugs. New bug should be filed in dummy_Fail test. 4987e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi old_issue_ids = close_bug() 4997e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 500ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi queue = multiprocessing.Queue() 501ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 502ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi push_to_prod_suite = multiprocessing.Process( 503ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi target=test_suite_wrapper, 504ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi args=(queue, PUSH_TO_PROD_SUITE, EXPECTED_TEST_RESULTS, 505ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi arguments)) 506ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi push_to_prod_suite.start() 5078f14391ed7dbbb0c8081d8e9a0ba60b7fa09ff4fJakob Juelich 5087e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # TODO(dshi): Remove following line after crbug.com/267644 is fixed. 5097e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Also, merge EXPECTED_TEST_RESULTS_AU to EXPECTED_TEST_RESULTS 510ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi au_suite = multiprocessing.Process( 511ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi target=test_suite_wrapper, 512ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi args=(queue, AU_SUITE, EXPECTED_TEST_RESULTS_AU, 513ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi arguments)) 514ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi au_suite.start() 515ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 516ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi shard_suite = multiprocessing.Process( 517ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi target=test_suite_wrapper, 518ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi args=(queue, DUMMY_SUITE, EXPECTED_TEST_RESULTS_DUMMY, 519ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi arguments, True)) 520ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi shard_suite.start() 521ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 522d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao # suite test with --create_and_return flag 523d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao asynchronous_suite = multiprocessing.Process( 524d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao target=test_suite_wrapper, 525d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao args=(queue, DUMMY_SUITE, EXPECTED_TEST_RESULTS_DUMMY, 526d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao arguments, True, True)) 527d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao asynchronous_suite.start() 528d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao 529ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi bug_filing_checked = False 530ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi while (push_to_prod_suite.is_alive() or au_suite.is_alive() or 531d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao shard_suite.is_alive() or asynchronous_suite.is_alive()): 532ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi check_queue(queue) 533ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi # Check bug filing results to fail early if bug filing failed. 534ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi if not bug_filing_checked and not push_to_prod_suite.is_alive(): 535ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi check_bug_filed_and_deduped(old_issue_ids) 536ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi bug_filing_checked = True 537ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi time.sleep(5) 538ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 539ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi check_queue(queue) 540ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi 541ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi push_to_prod_suite.join() 542ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi au_suite.join() 543ef1a5c0de1a2cfb039574f9d011d256114bda99aDan Shi shard_suite.join() 544d486477b09ae09e22dccf1a421055c847c439738Shuqian Zhao asynchronous_suite.join() 5457e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi except Exception as e: 5467e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi print 'Test for pushing to prod failed:\n' 5477e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi print str(e) 5487e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Send out email about the test failure. 5497e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if arguments.email: 5505fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi gmail_lib.send_email( 5515fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi arguments.email, 5525fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi 'Test for pushing to prod failed. Do NOT push!', 5535fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi ('Errors occurred during the test:\n\n%s\n\n' % str(e) + 5545fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi 'run_suite output:\n\n%s' % '\n'.join(run_suite_output))) 5557e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise 5567e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 5577e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi message = ('\nAll tests are completed successfully, prod branch is ready to' 5587e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi ' be pushed.') 5597e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi print message 5607e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi # Send out email about test completed successfully. 5617e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if arguments.email: 5625fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi gmail_lib.send_email( 5635fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi arguments.email, 5645fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi 'Test for pushing to prod completed successfully', 5655fa602cbd7787ff604758f03b71da19b66263ca1Dan Shi message) 5667e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 5677e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 5687e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shiif __name__ == '__main__': 5697e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi sys.exit(main()) 570