1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import ctypes 6import datetime 7import logging 8import multiprocessing 9import os 10import pexpect 11import Queue 12import re 13import threading 14import time 15 16from config import rpm_config 17import dli_urllib 18import rpm_logging_config 19 20import common 21from autotest_lib.client.common_lib import error 22from autotest_lib.client.common_lib.cros import retry 23 24RPM_CALL_TIMEOUT_MINS = rpm_config.getint('RPM_INFRASTRUCTURE', 25 'call_timeout_mins') 26SET_POWER_STATE_TIMEOUT_SECONDS = rpm_config.getint( 27 'RPM_INFRASTRUCTURE', 'set_power_state_timeout_seconds') 28PROCESS_TIMEOUT_BUFFER = 30 29 30 31class RPMController(object): 32 """ 33 This abstract class implements RPM request queueing and 34 processes queued requests. 35 36 The actual interaction with the RPM device will be implemented 37 by the RPM specific subclasses. 38 39 It assumes that you know the RPM hostname and that the device is on 40 the specified RPM. 41 42 This class also allows support for RPM devices that can be accessed 43 directly or through a hydra serial concentrator device. 44 45 Implementation details: 46 This is an abstract class, subclasses must implement the methods 47 listed here. You must not instantiate this class but should 48 instantiate one of those leaf subclasses. Subclasses should 49 also set TYPE class attribute to indicate device type. 50 51 @var behind_hydra: boolean value to represent whether or not this RPM is 52 behind a hydra device. 53 @var hostname: hostname for this rpm device. 54 @var is_running_lock: lock used to control access to _running. 55 @var request_queue: queue used to store requested outlet state changes. 56 @var queue_lock: lock used to control access to request_queue. 57 @var _running: boolean value to represent if this controller is currently 58 looping over queued requests. 59 """ 60 61 62 SSH_LOGIN_CMD = ('ssh -l %s -o StrictHostKeyChecking=no ' 63 '-o ConnectTimeout=90 -o UserKnownHostsFile=/dev/null %s') 64 USERNAME_PROMPT = 'Username:' 65 HYRDA_RETRY_SLEEP_SECS = 10 66 HYDRA_MAX_CONNECT_RETRIES = 3 67 LOGOUT_CMD = 'logout' 68 CLI_CMD = 'CLI' 69 CLI_HELD = 'The administrator \[root\] has an active .* session.' 70 CLI_KILL_PREVIOUS = 'cancel' 71 CLI_PROMPT = 'cli>' 72 HYDRA_PROMPT = '#' 73 PORT_STATUS_CMD = 'portStatus' 74 QUIT_CMD = 'quit' 75 SESSION_KILL_CMD_FORMAT = 'administration sessions kill %s' 76 HYDRA_CONN_HELD_MSG_FORMAT = 'is being used' 77 CYCLE_SLEEP_TIME = 5 78 79 # Global Variables that will likely be changed by subclasses. 80 DEVICE_PROMPT = '$' 81 PASSWORD_PROMPT = 'Password:' 82 # The state change command can be any string format but must accept 2 vars: 83 # state followed by device/Plug name. 84 SET_STATE_CMD = '%s %s' 85 SUCCESS_MSG = None # Some RPM's may not return a success msg. 86 87 NEW_STATE_ON = 'ON' 88 NEW_STATE_OFF = 'OFF' 89 NEW_STATE_CYCLE = 'CYCLE' 90 TYPE = 'Should set TYPE in subclass.' 91 92 93 def __init__(self, rpm_hostname, hydra_hostname=None): 94 """ 95 RPMController Constructor. 96 To be called by subclasses. 97 98 @param rpm_hostname: hostname of rpm device to be controlled. 99 """ 100 self._dns_zone = rpm_config.get('CROS', 'dns_zone') 101 self.hostname = rpm_hostname 102 self.request_queue = Queue.Queue() 103 self._running = False 104 self.is_running_lock = threading.Lock() 105 # If a hydra name is provided by the subclass then we know we are 106 # talking to an rpm behind a hydra device. 107 self.hydra_hostname = hydra_hostname if hydra_hostname else None 108 self.behind_hydra = hydra_hostname is not None 109 110 111 def _start_processing_requests(self): 112 """ 113 Check if there is a thread processing requests. 114 If not start one. 115 """ 116 with self.is_running_lock: 117 if not self._running: 118 self._running = True 119 self._running_thread = threading.Thread(target=self._run) 120 self._running_thread.start() 121 122 123 def _stop_processing_requests(self): 124 """ 125 Called if the request request_queue is empty. 126 Set running status to false. 127 """ 128 with self.is_running_lock: 129 logging.debug('Request queue is empty. RPM Controller for %s' 130 ' is terminating.', self.hostname) 131 self._running = False 132 if not self.request_queue.empty(): 133 # This can occur if an item was pushed into the queue after we 134 # exited the while-check and before the _stop_processing_requests 135 # call was made. Therefore we need to start processing again. 136 self._start_processing_requests() 137 138 139 def _run(self): 140 """ 141 Processes all queued up requests for this RPM Controller. 142 Callers should first request_queue up atleast one request and if this 143 RPM Controller is not running then call run. 144 145 Caller can either simply call run but then they will be blocked or 146 can instantiate a new thread to process all queued up requests. 147 For example: 148 threading.Thread(target=rpm_controller.run).start() 149 150 Requests are in the format of: 151 [powerunit_info, new_state, condition_var, result] 152 Run will set the result with the correct value. 153 """ 154 while not self.request_queue.empty(): 155 try: 156 result = multiprocessing.Value(ctypes.c_bool, False) 157 request = self.request_queue.get() 158 device_hostname = request['powerunit_info'].device_hostname 159 if (datetime.datetime.utcnow() > (request['start_time'] + 160 datetime.timedelta(minutes=RPM_CALL_TIMEOUT_MINS))): 161 logging.error('The request was waited for too long to be ' 162 "processed. It is timed out and won't be " 163 'processed.') 164 request['result_queue'].put(False) 165 continue 166 167 is_timeout = multiprocessing.Value(ctypes.c_bool, False) 168 process = multiprocessing.Process(target=self._process_request, 169 args=(request, result, 170 is_timeout)) 171 process.start() 172 process.join(SET_POWER_STATE_TIMEOUT_SECONDS + 173 PROCESS_TIMEOUT_BUFFER) 174 if process.is_alive(): 175 logging.debug('%s: process (%s) still running, will be ' 176 'terminated!', device_hostname, process.pid) 177 process.terminate() 178 is_timeout.value = True 179 180 if is_timeout.value: 181 raise error.TimeoutException( 182 'Attempt to set power state is timed out after %s ' 183 'seconds.' % SET_POWER_STATE_TIMEOUT_SECONDS) 184 if not result.value: 185 logging.error('Request to change %s to state %s failed.', 186 device_hostname, request['new_state']) 187 except Exception as e: 188 logging.error('Request to change %s to state %s failed: ' 189 'Raised exception: %s', device_hostname, 190 request['new_state'], e) 191 result.value = False 192 193 # Put result inside the result Queue to allow the caller to resume. 194 request['result_queue'].put(result.value) 195 self._stop_processing_requests() 196 197 198 def _process_request(self, request, result, is_timeout): 199 """Process the request to change a device's outlet state. 200 201 The call of set_power_state is made in a new running process. If it 202 takes longer than SET_POWER_STATE_TIMEOUT_SECONDS, the request will be 203 timed out. 204 205 @param request: A request to change a device's outlet state. 206 @param result: A Value object passed to the new process for the caller 207 thread to retrieve the result. 208 @param is_timeout: A Value object passed to the new process for the 209 caller thread to retrieve the information about if 210 the set_power_state call timed out. 211 """ 212 try: 213 logging.getLogger().handlers = [] 214 is_timeout_value, result_value = retry.timeout( 215 rpm_logging_config.set_up_logging_to_server, 216 timeout_sec=10) 217 if is_timeout_value: 218 raise Exception('Setup local log server handler timed out.') 219 except Exception as e: 220 # Fail over to log to a new file. 221 LOG_FILENAME_FORMAT = rpm_config.get('GENERAL', 222 'dispatcher_logname_format') 223 log_filename_format = LOG_FILENAME_FORMAT.replace( 224 'dispatcher', 'controller_%d' % os.getpid()) 225 logging.getLogger().handlers = [] 226 rpm_logging_config.set_up_logging_to_file( 227 log_dir='./logs', 228 log_filename_format=log_filename_format, 229 use_log_server=False) 230 logging.info('Failed to set up logging through log server: %s', e) 231 kwargs = {'powerunit_info':request['powerunit_info'], 232 'new_state':request['new_state']} 233 try: 234 is_timeout_value, result_value = retry.timeout( 235 self.set_power_state, 236 args=(), 237 kwargs=kwargs, 238 timeout_sec=SET_POWER_STATE_TIMEOUT_SECONDS) 239 result.value = result_value 240 is_timeout.value = is_timeout_value 241 except Exception as e: 242 # This method runs in a subprocess. Must log the exception, 243 # otherwise exceptions raised in set_power_state just get lost. 244 # Need to convert e to a str type, because our logging server 245 # code doesn't handle the conversion very well. 246 logging.error('Request to change %s to state %s failed: ' 247 'Raised exception: %s', 248 request['powerunit_info'].device_hostname, 249 request['new_state'], str(e)) 250 raise e 251 252 253 def queue_request(self, powerunit_info, new_state): 254 """ 255 Queues up a requested state change for a device's outlet. 256 257 Requests are in the format of: 258 [powerunit_info, new_state, condition_var, result] 259 Run will set the result with the correct value. 260 261 @param powerunit_info: And PowerUnitInfo instance. 262 @param new_state: ON/OFF/CYCLE - state or action we want to perform on 263 the outlet. 264 """ 265 request = {} 266 request['powerunit_info'] = powerunit_info 267 request['new_state'] = new_state 268 request['start_time'] = datetime.datetime.utcnow() 269 # Reserve a spot for the result to be stored. 270 request['result_queue'] = Queue.Queue() 271 # Place in request_queue 272 self.request_queue.put(request) 273 self._start_processing_requests() 274 # Block until the request is processed. 275 result = request['result_queue'].get(block=True) 276 return result 277 278 279 def _kill_previous_connection(self): 280 """ 281 In case the port to the RPM through the hydra serial concentrator is in 282 use, terminate the previous connection so we can log into the RPM. 283 284 It logs into the hydra serial concentrator over ssh, launches the CLI 285 command, gets the port number and then kills the current session. 286 """ 287 ssh = self._authenticate_with_hydra(admin_override=True) 288 if not ssh: 289 return 290 ssh.expect(RPMController.PASSWORD_PROMPT, timeout=60) 291 ssh.sendline(rpm_config.get('HYDRA', 'admin_password')) 292 ssh.expect(RPMController.HYDRA_PROMPT) 293 ssh.sendline(RPMController.CLI_CMD) 294 cli_prompt_re = re.compile(RPMController.CLI_PROMPT) 295 cli_held_re = re.compile(RPMController.CLI_HELD) 296 response = ssh.expect_list([cli_prompt_re, cli_held_re], timeout=60) 297 if response == 1: 298 # Need to kill the previous adminstator's session. 299 logging.error("Need to disconnect previous administrator's CLI " 300 "session to release the connection to RPM device %s.", 301 self.hostname) 302 ssh.sendline(RPMController.CLI_KILL_PREVIOUS) 303 ssh.expect(RPMController.CLI_PROMPT) 304 ssh.sendline(RPMController.PORT_STATUS_CMD) 305 ssh.expect(': %s' % self.hostname) 306 ports_status = ssh.before 307 port_number = ports_status.split(' ')[-1] 308 ssh.expect(RPMController.CLI_PROMPT) 309 ssh.sendline(RPMController.SESSION_KILL_CMD_FORMAT % port_number) 310 ssh.expect(RPMController.CLI_PROMPT) 311 self._logout(ssh, admin_logout=True) 312 313 314 def _hydra_login(self, ssh): 315 """ 316 Perform the extra steps required to log into a hydra serial 317 concentrator. 318 319 @param ssh: pexpect.spawn object used to communicate with the hydra 320 serial concentrator. 321 322 @return: True if the login procedure is successful. False if an error 323 occurred. The most common case would be if another user is 324 logged into the device. 325 """ 326 try: 327 response = ssh.expect_list( 328 [re.compile(RPMController.PASSWORD_PROMPT), 329 re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)], 330 timeout=15) 331 except pexpect.TIMEOUT: 332 # If there was a timeout, this ssh tunnel could be set up to 333 # not require the hydra password. 334 ssh.sendline('') 335 try: 336 ssh.expect(re.compile(RPMController.USERNAME_PROMPT)) 337 logging.debug('Connected to rpm through hydra. Logging in.') 338 return True 339 except pexpect.ExceptionPexpect: 340 return False 341 if response == 0: 342 try: 343 ssh.sendline(rpm_config.get('HYDRA','password')) 344 ssh.sendline('') 345 response = ssh.expect_list( 346 [re.compile(RPMController.USERNAME_PROMPT), 347 re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)], 348 timeout=60) 349 except pexpect.EOF: 350 # Did not receive any of the expect responses, retry. 351 return False 352 except pexpect.TIMEOUT: 353 logging.debug('Timeout occurred logging in to hydra.') 354 return False 355 # Send the username that the subclass will have set in its 356 # construction. 357 if response == 1: 358 logging.debug('SSH Terminal most likely serving another' 359 ' connection, retrying.') 360 # Kill the connection for the next connection attempt. 361 try: 362 self._kill_previous_connection() 363 except pexpect.ExceptionPexpect: 364 logging.error('Failed to disconnect previous connection, ' 365 'retrying.') 366 raise 367 return False 368 logging.debug('Connected to rpm through hydra. Logging in.') 369 return True 370 371 372 def _authenticate_with_hydra(self, admin_override=False): 373 """ 374 Some RPM's are behind a hydra serial concentrator and require their ssh 375 connection to be tunneled through this device. This can fail if another 376 user is logged in; therefore this will retry multiple times. 377 378 This function also allows us to authenticate directly to the 379 administrator interface of the hydra device. 380 381 @param admin_override: Set to True if we are trying to access the 382 administrator interface rather than tunnel 383 through to the RPM. 384 385 @return: The connected pexpect.spawn instance if the login procedure is 386 successful. None if an error occurred. The most common case 387 would be if another user is logged into the device. 388 """ 389 if admin_override: 390 username = rpm_config.get('HYDRA', 'admin_username') 391 else: 392 username = '%s:%s' % (rpm_config.get('HYDRA','username'), 393 self.hostname) 394 cmd = RPMController.SSH_LOGIN_CMD % (username, self.hydra_hostname) 395 num_attempts = 0 396 while num_attempts < RPMController.HYDRA_MAX_CONNECT_RETRIES: 397 try: 398 ssh = pexpect.spawn(cmd) 399 except pexpect.ExceptionPexpect: 400 return None 401 if admin_override: 402 return ssh 403 if self._hydra_login(ssh): 404 return ssh 405 # Authenticating with hydra failed. Sleep then retry. 406 time.sleep(RPMController.HYRDA_RETRY_SLEEP_SECS) 407 num_attempts += 1 408 logging.error('Failed to connect to the hydra serial concentrator after' 409 ' %d attempts.', RPMController.HYDRA_MAX_CONNECT_RETRIES) 410 return None 411 412 413 def _login(self): 414 """ 415 Log in into the RPM Device. 416 417 The login process should be able to connect to the device whether or not 418 it is behind a hydra serial concentrator. 419 420 @return: ssh - a pexpect.spawn instance if the connection was successful 421 or None if it was not. 422 """ 423 if self.behind_hydra: 424 # Tunnel the connection through the hydra. 425 ssh = self._authenticate_with_hydra() 426 if not ssh: 427 return None 428 ssh.sendline(self._username) 429 else: 430 # Connect directly to the RPM over SSH. 431 hostname = '%s.%s' % (self.hostname, self._dns_zone) 432 cmd = RPMController.SSH_LOGIN_CMD % (self._username, hostname) 433 try: 434 ssh = pexpect.spawn(cmd) 435 except pexpect.ExceptionPexpect: 436 return None 437 # Wait for the password prompt 438 try: 439 ssh.expect(self.PASSWORD_PROMPT, timeout=60) 440 ssh.sendline(self._password) 441 ssh.expect(self.DEVICE_PROMPT, timeout=60) 442 except pexpect.ExceptionPexpect: 443 return None 444 return ssh 445 446 447 def _logout(self, ssh, admin_logout=False): 448 """ 449 Log out of the RPM device. 450 451 Send the device specific logout command and if the connection is through 452 a hydra serial concentrator, kill the ssh connection. 453 454 @param admin_logout: Set to True if we are trying to logout of the 455 administrator interface of a hydra serial 456 concentrator, rather than an RPM. 457 @param ssh: pexpect.spawn instance to use to send the logout command. 458 """ 459 if admin_logout: 460 ssh.sendline(RPMController.QUIT_CMD) 461 ssh.expect(RPMController.HYDRA_PROMPT) 462 ssh.sendline(self.LOGOUT_CMD) 463 if self.behind_hydra and not admin_logout: 464 # Terminate the hydra session. 465 ssh.sendline('~.') 466 # Wait a bit so hydra disconnects completely. Launching another 467 # request immediately can cause a timeout. 468 time.sleep(5) 469 470 471 def set_power_state(self, powerunit_info, new_state): 472 """ 473 Set the state of the dut's outlet on this RPM. 474 475 For ssh based devices, this will create the connection either directly 476 or through a hydra tunnel and call the underlying _change_state function 477 to be implemented by the subclass device. 478 479 For non-ssh based devices, this method should be overloaded with the 480 proper connection and state change code. And the subclass will handle 481 accessing the RPM devices. 482 483 @param powerunit_info: An instance of PowerUnitInfo. 484 @param new_state: ON/OFF/CYCLE - state or action we want to perform on 485 the outlet. 486 487 @return: True if the attempt to change power state was successful, 488 False otherwise. 489 """ 490 ssh = self._login() 491 if not ssh: 492 return False 493 if new_state == self.NEW_STATE_CYCLE: 494 logging.debug('Beginning Power Cycle for device: %s', 495 powerunit_info.device_hostname) 496 result = self._change_state(powerunit_info, self.NEW_STATE_OFF, ssh) 497 if not result: 498 return result 499 time.sleep(RPMController.CYCLE_SLEEP_TIME) 500 result = self._change_state(powerunit_info, self.NEW_STATE_ON, ssh) 501 else: 502 # Try to change the state of the device's power outlet. 503 result = self._change_state(powerunit_info, new_state, ssh) 504 505 # Terminate hydra connection if necessary. 506 self._logout(ssh) 507 ssh.close(force=True) 508 return result 509 510 511 def _change_state(self, powerunit_info, new_state, ssh): 512 """ 513 Perform the actual state change operation. 514 515 Once we have established communication with the RPM this method is 516 responsible for changing the state of the RPM outlet. 517 518 @param powerunit_info: An instance of PowerUnitInfo. 519 @param new_state: ON/OFF - state or action we want to perform on 520 the outlet. 521 @param ssh: The ssh connection used to execute the state change commands 522 on the RPM device. 523 524 @return: True if the attempt to change power state was successful, 525 False otherwise. 526 """ 527 outlet = powerunit_info.outlet 528 device_hostname = powerunit_info.device_hostname 529 if not outlet: 530 logging.error('Request to change outlet for device: %s to new ' 531 'state %s failed: outlet is unknown, please ' 532 'make sure POWERUNIT_OUTLET exist in the host\'s ' 533 'attributes in afe.', device_hostname, new_state) 534 ssh.sendline(self.SET_STATE_CMD % (new_state, outlet)) 535 if self.SUCCESS_MSG: 536 # If this RPM device returns a success message check for it before 537 # continuing. 538 try: 539 ssh.expect(self.SUCCESS_MSG, timeout=60) 540 except pexpect.ExceptionPexpect: 541 logging.error('Request to change outlet for device: %s to new ' 542 'state %s failed.', device_hostname, new_state) 543 return False 544 logging.debug('Outlet for device: %s set to %s', device_hostname, 545 new_state) 546 return True 547 548 549 def type(self): 550 """ 551 Get the type of RPM device we are interacting with. 552 Class attribute TYPE should be set by the subclasses. 553 554 @return: string representation of RPM device type. 555 """ 556 return self.TYPE 557 558 559class SentryRPMController(RPMController): 560 """ 561 This class implements power control for Sentry Switched CDU 562 http://www.servertech.com/products/switched-pdus/ 563 564 Example usage: 565 rpm = SentrySwitchedCDU('chromeos-rack1-rpm1') 566 rpm.queue_request('chromeos-rack1-host1', 'ON') 567 568 @var _username: username used to access device. 569 @var _password: password used to access device. 570 """ 571 572 573 DEVICE_PROMPT = 'Switched CDU:' 574 SET_STATE_CMD = '%s %s' 575 SUCCESS_MSG = 'Command successful' 576 NUM_OF_OUTLETS = 17 577 TYPE = 'Sentry' 578 579 580 def __init__(self, hostname, hydra_hostname=None): 581 super(SentryRPMController, self).__init__(hostname, hydra_hostname) 582 self._username = rpm_config.get('SENTRY', 'username') 583 self._password = rpm_config.get('SENTRY', 'password') 584 585 586 def _setup_test_user(self, ssh): 587 """Configure the test user for the RPM 588 589 @param ssh: Pexpect object to use to configure the RPM. 590 """ 591 # Create and configure the testing user profile. 592 testing_user = rpm_config.get('SENTRY','testing_user') 593 testing_password = rpm_config.get('SENTRY','testing_password') 594 ssh.sendline('create user %s' % testing_user) 595 response = ssh.expect_list([re.compile('not unique'), 596 re.compile(self.PASSWORD_PROMPT)]) 597 if not response: 598 return 599 # Testing user is not set up yet. 600 ssh.sendline(testing_password) 601 ssh.expect('Verify Password:') 602 ssh.sendline(testing_password) 603 ssh.expect(self.SUCCESS_MSG) 604 ssh.expect(self.DEVICE_PROMPT) 605 ssh.sendline('add outlettouser all %s' % testing_user) 606 ssh.expect(self.SUCCESS_MSG) 607 ssh.expect(self.DEVICE_PROMPT) 608 609 610 def _clear_outlet_names(self, ssh): 611 """ 612 Before setting the outlet names, we need to clear out all the old 613 names so there are no conflicts. For example trying to assign outlet 614 2 a name already assigned to outlet 9. 615 """ 616 for outlet in range(1, self.NUM_OF_OUTLETS): 617 outlet_name = 'Outlet_%d' % outlet 618 ssh.sendline(self.SET_OUTLET_NAME_CMD % (outlet, outlet_name)) 619 ssh.expect(self.SUCCESS_MSG) 620 ssh.expect(self.DEVICE_PROMPT) 621 622 623 def setup(self, outlet_naming_map): 624 """ 625 Configure the RPM by adding the test user and setting up the outlet 626 names. 627 628 Note the rpm infrastructure does not rely on the outlet name to map a 629 device to its outlet any more. We keep this method in case there is 630 a need to label outlets for other reasons. We may deprecate 631 this method if it has been proved the outlet names will not be used 632 in any scenario. 633 634 @param outlet_naming_map: Dictionary used to map the outlet numbers to 635 host names. Keys must be ints. And names are 636 in the format of 'hostX'. 637 638 @return: True if setup completed successfully, False otherwise. 639 """ 640 ssh = self._login() 641 if not ssh: 642 logging.error('Could not connect to %s.', self.hostname) 643 return False 644 try: 645 self._setup_test_user(ssh) 646 # Set up the outlet names. 647 # Hosts have the same name format as the RPM hostname except they 648 # end in hostX instead of rpmX. 649 dut_name_format = re.sub('-rpm[0-9]*', '', self.hostname) 650 if self.behind_hydra: 651 # Remove "chromeosX" from DUTs behind the hydra due to a length 652 # constraint on the names we can store inside the RPM. 653 dut_name_format = re.sub('chromeos[0-9]*-', '', dut_name_format) 654 dut_name_format = dut_name_format + '-%s' 655 self._clear_outlet_names(ssh) 656 for outlet, name in outlet_naming_map.items(): 657 dut_name = dut_name_format % name 658 ssh.sendline(self.SET_OUTLET_NAME_CMD % (outlet, dut_name)) 659 ssh.expect(self.SUCCESS_MSG) 660 ssh.expect(self.DEVICE_PROMPT) 661 except pexpect.ExceptionPexpect as e: 662 logging.error('Setup failed. %s', e) 663 return False 664 finally: 665 self._logout(ssh) 666 return True 667 668 669class WebPoweredRPMController(RPMController): 670 """ 671 This class implements RPMController for the Web Powered units 672 produced by Digital Loggers Inc. 673 674 @var _rpm: dli_urllib.Powerswitch instance used to interact with RPM. 675 """ 676 677 678 TYPE = 'Webpowered' 679 680 681 def __init__(self, hostname, powerswitch=None): 682 username = rpm_config.get('WEBPOWERED', 'username') 683 password = rpm_config.get('WEBPOWERED', 'password') 684 # Call the constructor in RPMController. However since this is a web 685 # accessible device, there should not be a need to tunnel through a 686 # hydra serial concentrator. 687 super(WebPoweredRPMController, self).__init__(hostname) 688 self.hostname = '%s.%s' % (self.hostname, self._dns_zone) 689 if not powerswitch: 690 self._rpm = dli_urllib.Powerswitch(hostname=self.hostname, 691 userid=username, 692 password=password) 693 else: 694 # Should only be used in unit_testing 695 self._rpm = powerswitch 696 697 698 def _get_outlet_state(self, outlet): 699 """ 700 Look up the state for a given outlet on the RPM. 701 702 @param outlet: the outlet to look up. 703 704 @return state: the outlet's current state. 705 """ 706 status_list = self._rpm.statuslist() 707 for outlet_name, hostname, state in status_list: 708 if outlet_name == outlet: 709 return state 710 return None 711 712 713 def set_power_state(self, powerunit_info, new_state): 714 """ 715 Since this does not utilize SSH in any manner, this will overload the 716 set_power_state in RPMController and completes all steps of changing 717 the device's outlet state. 718 """ 719 device_hostname = powerunit_info.device_hostname 720 outlet = powerunit_info.outlet 721 if not outlet: 722 logging.error('Request to change outlet for device %s to ' 723 'new state %s failed: outlet is unknown. Make sure ' 724 'POWERUNIT_OUTLET exists in the host\'s ' 725 'attributes in afe' , device_hostname, new_state) 726 return False 727 state = self._get_outlet_state(outlet) 728 expected_state = new_state 729 if new_state == self.NEW_STATE_CYCLE: 730 logging.debug('Beginning Power Cycle for device: %s', 731 device_hostname) 732 self._rpm.off(outlet) 733 logging.debug('Outlet for device: %s set to OFF', device_hostname) 734 # Pause for 5 seconds before restoring power. 735 time.sleep(RPMController.CYCLE_SLEEP_TIME) 736 self._rpm.on(outlet) 737 logging.debug('Outlet for device: %s set to ON', device_hostname) 738 expected_state = self.NEW_STATE_ON 739 if new_state == self.NEW_STATE_OFF: 740 self._rpm.off(outlet) 741 logging.debug('Outlet for device: %s set to OFF', device_hostname) 742 if new_state == self.NEW_STATE_ON: 743 self._rpm.on(outlet) 744 logging.debug('Outlet for device: %s set to ON', device_hostname) 745 # Lookup the final state of the outlet 746 return self._is_plug_state(powerunit_info, expected_state) 747 748 749 def _is_plug_state(self, powerunit_info, expected_state): 750 state = self._get_outlet_state(powerunit_info.outlet) 751 if expected_state not in state: 752 logging.error('Outlet for device: %s did not change to new state' 753 ' %s', powerunit_info.device_hostname, expected_state) 754 return False 755 return True 756 757 758class CiscoPOEController(RPMController): 759 """ 760 This class implements power control for Cisco POE switch. 761 762 Example usage: 763 poe = CiscoPOEController('chromeos1-poe-switch1') 764 poe.queue_request('chromeos1-rack5-host12-servo', 'ON') 765 """ 766 767 768 SSH_LOGIN_CMD = ('ssh -o StrictHostKeyChecking=no ' 769 '-o UserKnownHostsFile=/dev/null %s') 770 POE_USERNAME_PROMPT = 'User Name:' 771 POE_PROMPT = '%s#' 772 EXIT_CMD = 'exit' 773 END_CMD = 'end' 774 CONFIG = 'configure terminal' 775 CONFIG_PROMPT = '%s\(config\)#' 776 CONFIG_IF = 'interface %s' 777 CONFIG_IF_PROMPT = '%s\(config-if\)#' 778 SET_STATE_ON = 'power inline auto' 779 SET_STATE_OFF = 'power inline never' 780 CHECK_INTERFACE_STATE = 'show interface status %s' 781 INTERFACE_STATE_MSG = 'Port\s+.*%s(\s+(\S+)){6,6}' 782 CHECK_STATE_TIMEOUT = 60 783 CMD_TIMEOUT = 30 784 LOGIN_TIMEOUT = 60 785 PORT_UP = 'Up' 786 PORT_DOWN = 'Down' 787 TYPE = 'CiscoPOE' 788 789 790 def __init__(self, hostname): 791 """ 792 Initialize controller class for a Cisco POE switch. 793 794 @param hostname: the Cisco POE switch host name. 795 """ 796 super(CiscoPOEController, self).__init__(hostname) 797 self._username = rpm_config.get('CiscoPOE', 'username') 798 self._password = rpm_config.get('CiscoPOE', 'password') 799 # For a switch, e.g. 'chromeos2-poe-switch8', 800 # the device prompt looks like 'chromeos2-poe-sw8#'. 801 short_hostname = self.hostname.replace('switch', 'sw') 802 self.poe_prompt = self.POE_PROMPT % short_hostname 803 self.config_prompt = self.CONFIG_PROMPT % short_hostname 804 self.config_if_prompt = self.CONFIG_IF_PROMPT % short_hostname 805 806 807 def _login(self): 808 """ 809 Log in into the Cisco POE switch. 810 811 Overload _login in RPMController, as it always prompts for a user name. 812 813 @return: ssh - a pexpect.spawn instance if the connection was successful 814 or None if it was not. 815 """ 816 hostname = '%s.%s' % (self.hostname, self._dns_zone) 817 cmd = self.SSH_LOGIN_CMD % (hostname) 818 try: 819 ssh = pexpect.spawn(cmd) 820 except pexpect.ExceptionPexpect: 821 logging.error('Could not connect to switch %s', hostname) 822 return None 823 # Wait for the username and password prompt. 824 try: 825 ssh.expect(self.POE_USERNAME_PROMPT, timeout=self.LOGIN_TIMEOUT) 826 ssh.sendline(self._username) 827 ssh.expect(self.PASSWORD_PROMPT, timeout=self.LOGIN_TIMEOUT) 828 ssh.sendline(self._password) 829 ssh.expect(self.poe_prompt, timeout=self.LOGIN_TIMEOUT) 830 except pexpect.ExceptionPexpect: 831 logging.error('Could not log into switch %s', hostname) 832 return None 833 return ssh 834 835 836 def _enter_configuration_terminal(self, interface, ssh): 837 """ 838 Enter configuration terminal of |interface|. 839 840 This function expects that we've already logged into the switch 841 and the ssh is prompting the switch name. The work flow is 842 chromeos1-poe-sw1# 843 chromeos1-poe-sw1#configure terminal 844 chromeos1-poe-sw1(config)#interface fa36 845 chromeos1-poe-sw1(config-if)# 846 On success, the function exits with 'config-if' prompt. 847 On failure, the function exits with device prompt, 848 e.g. 'chromeos1-poe-sw1#' in the above case. 849 850 @param interface: the name of the interface. 851 @param ssh: pexpect.spawn instance to use. 852 853 @return: True on success otherwise False. 854 """ 855 try: 856 # Enter configure terminal. 857 ssh.sendline(self.CONFIG) 858 ssh.expect(self.config_prompt, timeout=self.CMD_TIMEOUT) 859 # Enter configure terminal of the interface. 860 ssh.sendline(self.CONFIG_IF % interface) 861 ssh.expect(self.config_if_prompt, timeout=self.CMD_TIMEOUT) 862 return True 863 except pexpect.ExceptionPexpect, e: 864 ssh.sendline(self.END_CMD) 865 logging.exception(e) 866 return False 867 868 869 def _exit_configuration_terminal(self, ssh): 870 """ 871 Exit interface configuration terminal. 872 873 On success, the function exits with device prompt, 874 e.g. 'chromeos1-poe-sw1#' in the above case. 875 On failure, the function exists with 'config-if' prompt. 876 877 @param ssh: pexpect.spawn instance to use. 878 879 @return: True on success otherwise False. 880 """ 881 try: 882 ssh.sendline(self.END_CMD) 883 ssh.expect(self.poe_prompt, timeout=self.CMD_TIMEOUT) 884 return True 885 except pexpect.ExceptionPexpect, e: 886 logging.exception(e) 887 return False 888 889 890 def _verify_state(self, interface, expected_state, ssh): 891 """ 892 Check whehter the current state of |interface| matches expected state. 893 894 This function tries to check the state of |interface| multiple 895 times until its state matches the expected state or time is out. 896 897 After the command of changing state has been executed, 898 the state of an interface doesn't always change immediately to 899 the expected state but requires some time. As such, we need 900 a retry logic here. 901 902 @param interface: the name of the interface. 903 @param expect_state: the expected state, 'ON' or 'OFF' 904 @param ssh: pexpect.spawn instance to use. 905 906 @return: True if the state of |interface| swiches to |expected_state|, 907 otherwise False. 908 """ 909 expected_state = (self.PORT_UP if expected_state == self.NEW_STATE_ON 910 else self.PORT_DOWN) 911 try: 912 start = time.time() 913 while((time.time() - start) < self.CHECK_STATE_TIMEOUT): 914 ssh.sendline(self.CHECK_INTERFACE_STATE % interface) 915 state_matcher = '.*'.join([self.INTERFACE_STATE_MSG % interface, 916 self.poe_prompt]) 917 ssh.expect(state_matcher, timeout=self.CMD_TIMEOUT) 918 state = ssh.match.group(2) 919 if state == expected_state: 920 return True 921 except pexpect.ExceptionPexpect, e: 922 logging.exception(e) 923 return False 924 925 926 def _logout(self, ssh, admin_logout=False): 927 """ 928 Log out of the Cisco POE switch after changing state. 929 930 Overload _logout in RPMController. 931 932 @param admin_logout: ignored by this method. 933 @param ssh: pexpect.spawn instance to use to send the logout command. 934 """ 935 ssh.sendline(self.EXIT_CMD) 936 937 938 def _change_state(self, powerunit_info, new_state, ssh): 939 """ 940 Perform the actual state change operation. 941 942 Overload _change_state in RPMController. 943 944 @param powerunit_info: An PowerUnitInfo instance. 945 @param new_state: ON/OFF - state or action we want to perform on 946 the outlet. 947 @param ssh: The ssh connection used to execute the state change commands 948 on the POE switch. 949 950 @return: True if the attempt to change power state was successful, 951 False otherwise. 952 """ 953 interface = powerunit_info.outlet 954 device_hostname = powerunit_info.device_hostname 955 if not interface: 956 logging.error('Could not change state: the interface on %s for %s ' 957 'was not given.', self.hostname, device_hostname) 958 return False 959 960 # Enter configuration terminal. 961 if not self._enter_configuration_terminal(interface, ssh): 962 logging.error('Could not enter configuration terminal for %s', 963 interface) 964 return False 965 # Change the state. 966 if new_state == self.NEW_STATE_ON: 967 ssh.sendline(self.SET_STATE_ON) 968 elif new_state == self.NEW_STATE_OFF: 969 ssh.sendline(self.SET_STATE_OFF) 970 else: 971 logging.error('Unknown state request: %s', new_state) 972 return False 973 # Exit configuraiton terminal. 974 if not self._exit_configuration_terminal(ssh): 975 logging.error('Skipping verifying outlet state for device: %s, ' 976 'because could not exit configuration terminal.', 977 device_hostname) 978 return False 979 # Verify if the state has changed successfully. 980 if not self._verify_state(interface, new_state, ssh): 981 logging.error('Could not verify state on interface %s', interface) 982 return False 983 984 logging.debug('Outlet for device: %s set to %s', 985 device_hostname, new_state) 986 return True 987