1cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish# Use of this source code is governed by a BSD-style license that can be 3506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish# found in the LICENSE file. 4506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 5ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambishimport contextlib 6506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambishimport datetime 7385213165849f3867702f3467ec803957ebe3506Kris Rambishimport logging 826d3d4d699070320ad45c28749518373422f759dbmahadevimport pprint 90ccfc15cd256dde8dfe55428a87b671fb1f49c78Kris Rambishimport time 10506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 11c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Piusimport common 12d2ae97483945e7f78e57e41cafcd800553e08598Tien Changfrom autotest_lib.client.common_lib import error 1322093e32603a2c850e8a15121f4718f15e435415Roshan Piusfrom autotest_lib.client.common_lib.cros.network import ap_constants 1406977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadevfrom autotest_lib.client.common_lib.cros.network import iw_runner 15506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambishfrom autotest_lib.server import hosts 16506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambishfrom autotest_lib.server import site_linux_system 17506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambishfrom autotest_lib.server.cros import host_lock_manager 1822093e32603a2c850e8a15121f4718f15e435415Roshan Piusfrom autotest_lib.server.cros.ap_configurators import ap_batch_locker 19c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Piusfrom autotest_lib.server.cros.network import chaos_clique_utils as utils 20ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambishfrom autotest_lib.server.cros.network import wifi_client 21ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 22d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang# Webdriver master hostname 23d2ae97483945e7f78e57e41cafcd800553e08598Tien ChangMASTERNAME = 'chromeos3-chaosvmmaster.cros.corp.google.com' 24d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang 25506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 26506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambishclass ChaosRunner(object): 27506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish """Object to run a network_WiFi_ChaosXXX test.""" 28506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 29506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 30093d651517da7a4c760aa3fe07d8466ebb15d6aebmahadev def __init__(self, test, host, spec, broken_pdus=list()): 31506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish """Initializes and runs test. 32506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 33506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish @param test: a string, test name. 34506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish @param host: an Autotest host object, device under test. 3526d3d4d699070320ad45c28749518373422f759dbmahadev @param spec: an APSpec object. 3626d3d4d699070320ad45c28749518373422f759dbmahadev @param broken_pdus: list of offline PDUs. 37506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 38506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish """ 39506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish self._test = test 40506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish self._host = host 41299dad5b2a0cd35e27719d730231ccafb0226385Kris Rambish self._ap_spec = spec 4226d3d4d699070320ad45c28749518373422f759dbmahadev self._broken_pdus = broken_pdus 43506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish # Log server and DUT times 44506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish dt = datetime.datetime.now() 45506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish logging.info('Server time: %s', dt.strftime('%a %b %d %H:%M:%S %Y')) 46506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish logging.info('DUT time: %s', self._host.run('date').stdout.strip()) 47506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 48506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 4965186bae7b1f7633b182e5f8af98a75831e7749cKris Rambish def run(self, job, batch_size=7, tries=10, capturer_hostname=None, 5069ceb35ae6df406df4bf1c6d44a4f4d3b87f8115Jason Abele conn_worker=None, work_client_hostname=None, 5169ceb35ae6df406df4bf1c6d44a4f4d3b87f8115Jason Abele disabled_sysinfo=False): 52506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish """Executes Chaos test. 53506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 54506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish @param job: an Autotest job object. 55506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish @param batch_size: an integer, max number of APs to lock in one batch. 56506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish @param tries: an integer, number of iterations to run per AP. 57618f21f85484caf173e42e9b7f0a3bd892482b2eJason Abele @param capturer_hostname: a string or None, hostname or IP of capturer. 587bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan @param conn_worker: ConnectionWorkerAbstract or None, to run extra 597bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan work after successful connection. 607bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan @param work_client_hostname: a string or None, hostname of work client 6169ceb35ae6df406df4bf1c6d44a4f4d3b87f8115Jason Abele @param disabled_sysinfo: a bool, disable collection of logs from DUT. 62506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 63d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang 64d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang @raises TestError: Issues locking VM webdriver instance 65506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish """ 66506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 67506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish lock_manager = host_lock_manager.HostLockManager() 68d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang webdriver_master = hosts.SSHHost(MASTERNAME, user='chaosvmmaster') 69506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish with host_lock_manager.HostsLockedBy(lock_manager): 70c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius capture_host = utils.allocate_packet_capturer( 71618f21f85484caf173e42e9b7f0a3bd892482b2eJason Abele lock_manager, hostname=capturer_hostname) 7206977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev # Cleanup and reboot packet capturer before the test. 7306977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev utils.sanitize_client(capture_host) 7405b03e057e1acff341ffcbd4d358e38331dfa9f8Bindu Mahadev capturer = site_linux_system.LinuxSystem(capture_host, {}, 7505b03e057e1acff341ffcbd4d358e38331dfa9f8Bindu Mahadev 'packet_capturer') 7606977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev 7706977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev # Run iw scan and abort if more than allowed number of APs are up. 7806977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev iw_command = iw_runner.IwRunner(capture_host) 7906977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev start_time = time.time() 8006977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev logging.info('Performing a scan with a max timeout of 30 seconds.') 8106977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev while time.time() - start_time <= ap_constants.MAX_SCAN_TIMEOUT: 8206977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev networks = iw_command.scan('wlan0') 8306977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev if networks is None: 8406977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev if (time.time() - start_time == 8506977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev ap_constants.MAX_SCAN_TIMEOUT): 8606977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev raise error.TestError( 8706977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev 'Packet capturer is not responding to scans. Check' 8806977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev 'device and re-run test') 8906977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev continue 9006977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev elif len(networks) < ap_constants.MAX_SSID_COUNT: 9106977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev break 9206977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev elif len(networks) >= ap_constants.MAX_SSID_COUNT: 9306977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev raise error.TestError( 9406977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev 'Probably someone is already running a' 9506977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev 'chaos test?!') 9606977bf3187e127ac369700ff4cb9fd23f61dbf1Bindu Mahadev 977bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan if conn_worker is not None: 98c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius work_client_machine = utils.allocate_packet_capturer( 997bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan lock_manager, hostname=work_client_hostname) 1007bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan conn_worker.prepare_work_client(work_client_machine) 101d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang 102d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang webdriver_instance = utils.allocate_webdriver_instance(lock_manager) 103d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang self._ap_spec._webdriver_hostname = webdriver_instance 104d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang 105e91dc9bdaf6d44b9543886dbcdabd9086822ef36Kris Rambish # If a test is cancelled or aborted the VM may be left on. Always 106e91dc9bdaf6d44b9543886dbcdabd9086822ef36Kris Rambish # turn of the VM to return it to a clean state. 107e91dc9bdaf6d44b9543886dbcdabd9086822ef36Kris Rambish try: 1089fcf81129c44c7d43edf7a1d6f269803d22233e0Tien Chang logging.info('Always power off VM %s', webdriver_instance) 109e91dc9bdaf6d44b9543886dbcdabd9086822ef36Kris Rambish utils.power_off_VM(webdriver_master, webdriver_instance) 110e91dc9bdaf6d44b9543886dbcdabd9086822ef36Kris Rambish except: 111e91dc9bdaf6d44b9543886dbcdabd9086822ef36Kris Rambish logging.debug('VM was already off, ignoring.') 112e91dc9bdaf6d44b9543886dbcdabd9086822ef36Kris Rambish 1139fcf81129c44c7d43edf7a1d6f269803d22233e0Tien Chang logging.info('Starting up VM %s', webdriver_instance) 1149fcf81129c44c7d43edf7a1d6f269803d22233e0Tien Chang utils.power_on_VM(webdriver_master, webdriver_instance) 115d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang 11622093e32603a2c850e8a15121f4718f15e435415Roshan Pius batch_locker = ap_batch_locker.ApBatchLocker( 11722093e32603a2c850e8a15121f4718f15e435415Roshan Pius lock_manager, self._ap_spec, 11822093e32603a2c850e8a15121f4718f15e435415Roshan Pius ap_test_type=ap_constants.AP_TEST_TYPE_CHAOS) 1197bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan 120506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish while batch_locker.has_more_aps(): 1212bd021fbe04113afc538afa4aa02da888d57df60Kris Rambish # Work around crbug.com/358716 122c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius utils.sanitize_client(self._host) 1231afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish healthy_dut = True 1249fcf81129c44c7d43edf7a1d6f269803d22233e0Tien Chang 125ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish with contextlib.closing(wifi_client.WiFiClient( 126ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish hosts.create_host(self._host.hostname), 127ce74f5fcaaebe2987d5b714edf23c9aea1fe16d7Roshan Pius './debug', 128ce74f5fcaaebe2987d5b714edf23c9aea1fe16d7Roshan Pius False)) as client: 129ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 130ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish aps = batch_locker.get_ap_batch(batch_size=batch_size) 131ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish if not aps: 132ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish logging.info('No more APs to test.') 133ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish break 134ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 135ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish # Power down all of the APs because some can get grumpy 136ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish # if they are configured several times and remain on. 137ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish # User the cartridge to down group power downs and 138ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish # configurations. 139c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius utils.power_down_aps(aps, self._broken_pdus) 140c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius utils.configure_aps(aps, self._ap_spec, self._broken_pdus) 141ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 142c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius aps = utils.filter_quarantined_and_config_failed_aps(aps, 143c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius batch_locker, job, self._broken_pdus) 1441afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish 145ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish for ap in aps: 146ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish # http://crbug.com/306687 147ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish if ap.ssid == None: 148ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish logging.error('The SSID was not set for the AP:%s', 149ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish ap) 150ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 151c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius healthy_dut = utils.is_dut_healthy(client, ap) 1521afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish 1531afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish if not healthy_dut: 15489c93289686b415118e0fa54064ceddefcc58128Kris Rambish logging.error('DUT is not healthy, rebooting.') 1551afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish batch_locker.unlock_and_reclaim_aps() 1561afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish break 157ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 158c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius networks = utils.return_available_networks( 159c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius ap, capturer, job, self._ap_spec) 160299dad5b2a0cd35e27719d730231ccafb0226385Kris Rambish 161491aed5b5f9b9bb167944e2360e5f046bc1ac18bbmahadev if networks is None: 162d24b69d964c851a0f36562f844a2acbd3bab124abmahadev # If scan returned no networks, iw scan failed. 1631afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish # Reboot the packet capturer device and 1641afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish # reconfigure the capturer. 1651afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish batch_locker.unlock_and_reclaim_ap(ap.host_name) 16689c93289686b415118e0fa54064ceddefcc58128Kris Rambish logging.error('Packet capture is not healthy, ' 16789c93289686b415118e0fa54064ceddefcc58128Kris Rambish 'rebooting.') 168d24b69d964c851a0f36562f844a2acbd3bab124abmahadev capturer.host.reboot() 169d24b69d964c851a0f36562f844a2acbd3bab124abmahadev capturer = site_linux_system.LinuxSystem( 170d24b69d964c851a0f36562f844a2acbd3bab124abmahadev capture_host, {},'packet_capturer') 171ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish continue 172d24b69d964c851a0f36562f844a2acbd3bab124abmahadev if networks == list(): 173491aed5b5f9b9bb167944e2360e5f046bc1ac18bbmahadev # Packet capturer did not find the SSID in scan or 174491aed5b5f9b9bb167944e2360e5f046bc1ac18bbmahadev # there was a security mismatch. 175c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius utils.release_ap(ap, batch_locker, self._broken_pdus) 176d24b69d964c851a0f36562f844a2acbd3bab124abmahadev continue 177ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 178ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish assoc_params = ap.get_association_parameters() 1791afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish 180c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius if not utils.is_conn_worker_healthy( 181c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius conn_worker, ap, assoc_params, job): 182c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius utils.release_ap( 183c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius ap, batch_locker, self._broken_pdus) 1841afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish continue 1857bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan 186cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang name = ap.name 187cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang kernel_ver = self._host.get_kernel_ver() 188c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius firmware_ver = utils.get_firmware_ver(self._host) 189cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang if not firmware_ver: 190cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang firmware_ver = "Unknown" 191cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang 192a5c9abf9c20e1e6065b0a3860b5d1297bfac11bfKris Rambish debug_dict = {'+++PARSE DATA+++': '+++PARSE DATA+++', 193a5c9abf9c20e1e6065b0a3860b5d1297bfac11bfKris Rambish 'SSID': ap._ssid, 194a5c9abf9c20e1e6065b0a3860b5d1297bfac11bfKris Rambish 'DUT': client.wifi_mac, 195a5c9abf9c20e1e6065b0a3860b5d1297bfac11bfKris Rambish 'AP Info': ap.name, 196cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang 'kernel_version': kernel_ver, 197cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang 'wifi_firmware_version': firmware_ver} 198cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang debug_string = pprint.pformat(debug_dict) 199cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang 2004a637b1d020d2eeb294e29102b1cefbed17f1234Kris Rambish logging.info('Waiting %d seconds for the AP dhcp ' 2014a637b1d020d2eeb294e29102b1cefbed17f1234Kris Rambish 'server', ap.dhcp_delay) 2024a637b1d020d2eeb294e29102b1cefbed17f1234Kris Rambish time.sleep(ap.dhcp_delay) 2034a637b1d020d2eeb294e29102b1cefbed17f1234Kris Rambish 204ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish result = job.run_test(self._test, 205ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish capturer=capturer, 206ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish capturer_frequency=networks[0].frequency, 207ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish capturer_ht_type=networks[0].ht, 208ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish host=self._host, 209ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish assoc_params=assoc_params, 210ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish client=client, 211ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish tries=tries, 212cf3da1540523d918aadec06b3a0e4bb74e85185cTien Chang debug_info=debug_string, 213ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish # Copy all logs from the system 21469ceb35ae6df406df4bf1c6d44a4f4d3b87f8115Jason Abele disabled_sysinfo=disabled_sysinfo, 2157bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan conn_worker=conn_worker, 2167bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan tag=ap.ssid if conn_worker is None else 2177bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan '%s.%s' % (conn_worker.name, ap.ssid)) 218ebaf2ad2dbd521dde28ac7d8100f3dc375f150dcKris Rambish 219c6db3d062f9166eeccd816427dfc2f3e8b2e0052Roshan Pius utils.release_ap(ap, batch_locker, self._broken_pdus) 2201afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish 2217bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan if conn_worker is not None: 2227bf5d460a10c0ac9c1c22e3e23b898d9792a1af4Ashok Nagarajan conn_worker.cleanup() 223506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish 2241afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish if not healthy_dut: 2251afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish continue 2261afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish 227506a99d582456eff2f6d5e534cbf00a62990e97dKris Rambish batch_locker.unlock_aps() 228b125b27b573ce72946caf7174cac8747099e67e6Kris Rambish capturer.close() 229d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang logging.info('Powering off VM %s', webdriver_instance) 230d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang utils.power_off_VM(webdriver_master, webdriver_instance) 231d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang lock_manager.unlock(webdriver_instance) 232d2ae97483945e7f78e57e41cafcd800553e08598Tien Chang 23326d3d4d699070320ad45c28749518373422f759dbmahadev if self._broken_pdus: 2341afc69706f835eee0cd324e2aaf3f59101f89a69Kris Rambish logging.info('PDU is down!!!\nThe following PDUs are down:\n') 23526d3d4d699070320ad45c28749518373422f759dbmahadev pprint.pprint(self._broken_pdus) 236