1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import ctypes 6import datetime 7import logging 8import multiprocessing 9import os 10import pexpect 11import Queue 12import re 13import threading 14import time 15 16from config import rpm_config 17import dli_urllib 18import rpm_logging_config 19 20import common 21from autotest_lib.client.common_lib import error 22from autotest_lib.client.common_lib.cros import retry 23 24RPM_CALL_TIMEOUT_MINS = rpm_config.getint('RPM_INFRASTRUCTURE', 25 'call_timeout_mins') 26SET_POWER_STATE_TIMEOUT_SECONDS = rpm_config.getint( 27 'RPM_INFRASTRUCTURE', 'set_power_state_timeout_seconds') 28PROCESS_TIMEOUT_BUFFER = 30 29 30 31class RPMController(object): 32 """ 33 This abstract class implements RPM request queueing and 34 processes queued requests. 35 36 The actual interaction with the RPM device will be implemented 37 by the RPM specific subclasses. 38 39 It assumes that you know the RPM hostname and that the device is on 40 the specified RPM. 41 42 This class also allows support for RPM devices that can be accessed 43 directly or through a hydra serial concentrator device. 44 45 Implementation details: 46 This is an abstract class, subclasses must implement the methods 47 listed here. You must not instantiate this class but should 48 instantiate one of those leaf subclasses. Subclasses should 49 also set TYPE class attribute to indicate device type. 50 51 @var behind_hydra: boolean value to represent whether or not this RPM is 52 behind a hydra device. 53 @var hostname: hostname for this rpm device. 54 @var is_running_lock: lock used to control access to _running. 55 @var request_queue: queue used to store requested outlet state changes. 56 @var queue_lock: lock used to control access to request_queue. 57 @var _running: boolean value to represent if this controller is currently 58 looping over queued requests. 59 """ 60 61 62 SSH_LOGIN_CMD = ('ssh -l %s -o StrictHostKeyChecking=no ' 63 '-o ConnectTimeout=90 -o UserKnownHostsFile=/dev/null %s') 64 USERNAME_PROMPT = 'Username:' 65 HYRDA_RETRY_SLEEP_SECS = 10 66 HYDRA_MAX_CONNECT_RETRIES = 3 67 LOGOUT_CMD = 'logout' 68 CLI_CMD = 'CLI' 69 CLI_HELD = r'The administrator \[root\] has an active .* session.' 70 CLI_KILL_PREVIOUS = 'cancel' 71 CLI_PROMPT = 'cli>' 72 HYDRA_PROMPT = '#' 73 PORT_STATUS_CMD = 'portStatus' 74 QUIT_CMD = 'quit' 75 SESSION_KILL_CMD_FORMAT = 'administration sessions kill %s' 76 HYDRA_CONN_HELD_MSG_FORMAT = 'is being used' 77 CYCLE_SLEEP_TIME = 5 78 79 # Global Variables that will likely be changed by subclasses. 80 DEVICE_PROMPT = '$' 81 PASSWORD_PROMPT = 'Password:' 82 # The state change command can be any string format but must accept 2 vars: 83 # state followed by device/Plug name. 84 SET_STATE_CMD = '%s %s' 85 SUCCESS_MSG = None # Some RPM's may not return a success msg. 86 87 NEW_STATE_ON = 'ON' 88 NEW_STATE_OFF = 'OFF' 89 NEW_STATE_CYCLE = 'CYCLE' 90 TYPE = 'Should set TYPE in subclass.' 91 92 93 def __init__(self, rpm_hostname, hydra_hostname=None): 94 """ 95 RPMController Constructor. 96 To be called by subclasses. 97 98 @param rpm_hostname: hostname of rpm device to be controlled. 99 """ 100 self._dns_zone = rpm_config.get('CROS', 'dns_zone') 101 self.hostname = rpm_hostname 102 self.request_queue = Queue.Queue() 103 self._running = False 104 self.is_running_lock = threading.Lock() 105 # If a hydra name is provided by the subclass then we know we are 106 # talking to an rpm behind a hydra device. 107 self.hydra_hostname = hydra_hostname if hydra_hostname else None 108 self.behind_hydra = hydra_hostname is not None 109 110 111 def _start_processing_requests(self): 112 """ 113 Check if there is a thread processing requests. 114 If not start one. 115 """ 116 with self.is_running_lock: 117 if not self._running: 118 self._running = True 119 self._running_thread = threading.Thread(target=self._run) 120 self._running_thread.start() 121 122 123 def _stop_processing_requests(self): 124 """ 125 Called if the request request_queue is empty. 126 Set running status to false. 127 """ 128 with self.is_running_lock: 129 logging.debug('Request queue is empty. RPM Controller for %s' 130 ' is terminating.', self.hostname) 131 self._running = False 132 if not self.request_queue.empty(): 133 # This can occur if an item was pushed into the queue after we 134 # exited the while-check and before the _stop_processing_requests 135 # call was made. Therefore we need to start processing again. 136 self._start_processing_requests() 137 138 139 def _run(self): 140 """ 141 Processes all queued up requests for this RPM Controller. 142 Callers should first request_queue up atleast one request and if this 143 RPM Controller is not running then call run. 144 145 Caller can either simply call run but then they will be blocked or 146 can instantiate a new thread to process all queued up requests. 147 For example: 148 threading.Thread(target=rpm_controller.run).start() 149 150 Requests are in the format of: 151 [powerunit_info, new_state, condition_var, result] 152 Run will set the result with the correct value. 153 """ 154 while not self.request_queue.empty(): 155 try: 156 result = multiprocessing.Value(ctypes.c_bool, False) 157 request = self.request_queue.get() 158 device_hostname = request['powerunit_info'].device_hostname 159 if (datetime.datetime.utcnow() > (request['start_time'] + 160 datetime.timedelta(minutes=RPM_CALL_TIMEOUT_MINS))): 161 logging.error('The request was waited for too long to be ' 162 "processed. It is timed out and won't be " 163 'processed.') 164 request['result_queue'].put(False) 165 continue 166 167 is_timeout = multiprocessing.Value(ctypes.c_bool, False) 168 process = multiprocessing.Process(target=self._process_request, 169 args=(request, result, 170 is_timeout)) 171 process.start() 172 process.join(SET_POWER_STATE_TIMEOUT_SECONDS + 173 PROCESS_TIMEOUT_BUFFER) 174 if process.is_alive(): 175 logging.debug('%s: process (%s) still running, will be ' 176 'terminated!', device_hostname, process.pid) 177 process.terminate() 178 is_timeout.value = True 179 180 if is_timeout.value: 181 raise error.TimeoutException( 182 'Attempt to set power state is timed out after %s ' 183 'seconds.' % SET_POWER_STATE_TIMEOUT_SECONDS) 184 if not result.value: 185 logging.error('Request to change %s to state %s failed.', 186 device_hostname, request['new_state']) 187 except Exception as e: 188 logging.error('Request to change %s to state %s failed: ' 189 'Raised exception: %s', device_hostname, 190 request['new_state'], e) 191 result.value = False 192 193 # Put result inside the result Queue to allow the caller to resume. 194 request['result_queue'].put(result.value) 195 self._stop_processing_requests() 196 197 198 def _process_request(self, request, result, is_timeout): 199 """Process the request to change a device's outlet state. 200 201 The call of set_power_state is made in a new running process. If it 202 takes longer than SET_POWER_STATE_TIMEOUT_SECONDS, the request will be 203 timed out. 204 205 @param request: A request to change a device's outlet state. 206 @param result: A Value object passed to the new process for the caller 207 thread to retrieve the result. 208 @param is_timeout: A Value object passed to the new process for the 209 caller thread to retrieve the information about if 210 the set_power_state call timed out. 211 """ 212 try: 213 logging.getLogger().handlers = [] 214 is_timeout_value, result_value = retry.timeout( 215 rpm_logging_config.set_up_logging_to_server, 216 timeout_sec=10) 217 if is_timeout_value: 218 raise Exception('Setup local log server handler timed out.') 219 except Exception as e: 220 # Fail over to log to a new file. 221 LOG_FILENAME_FORMAT = rpm_config.get('GENERAL', 222 'dispatcher_logname_format') 223 log_filename_format = LOG_FILENAME_FORMAT.replace( 224 'dispatcher', 'controller_%d' % os.getpid()) 225 logging.getLogger().handlers = [] 226 rpm_logging_config.set_up_logging_to_file( 227 log_dir='./logs', 228 log_filename_format=log_filename_format) 229 logging.info('Failed to set up logging through log server: %s', e) 230 kwargs = {'powerunit_info':request['powerunit_info'], 231 'new_state':request['new_state']} 232 try: 233 is_timeout_value, result_value = retry.timeout( 234 self.set_power_state, 235 args=(), 236 kwargs=kwargs, 237 timeout_sec=SET_POWER_STATE_TIMEOUT_SECONDS) 238 result.value = result_value 239 is_timeout.value = is_timeout_value 240 except Exception as e: 241 # This method runs in a subprocess. Must log the exception, 242 # otherwise exceptions raised in set_power_state just get lost. 243 # Need to convert e to a str type, because our logging server 244 # code doesn't handle the conversion very well. 245 logging.error('Request to change %s to state %s failed: ' 246 'Raised exception: %s', 247 request['powerunit_info'].device_hostname, 248 request['new_state'], str(e)) 249 raise e 250 251 252 def queue_request(self, powerunit_info, new_state): 253 """ 254 Queues up a requested state change for a device's outlet. 255 256 Requests are in the format of: 257 [powerunit_info, new_state, condition_var, result] 258 Run will set the result with the correct value. 259 260 @param powerunit_info: And PowerUnitInfo instance. 261 @param new_state: ON/OFF/CYCLE - state or action we want to perform on 262 the outlet. 263 """ 264 request = {} 265 request['powerunit_info'] = powerunit_info 266 request['new_state'] = new_state 267 request['start_time'] = datetime.datetime.utcnow() 268 # Reserve a spot for the result to be stored. 269 request['result_queue'] = Queue.Queue() 270 # Place in request_queue 271 self.request_queue.put(request) 272 self._start_processing_requests() 273 # Block until the request is processed. 274 result = request['result_queue'].get(block=True) 275 return result 276 277 278 def _kill_previous_connection(self): 279 """ 280 In case the port to the RPM through the hydra serial concentrator is in 281 use, terminate the previous connection so we can log into the RPM. 282 283 It logs into the hydra serial concentrator over ssh, launches the CLI 284 command, gets the port number and then kills the current session. 285 """ 286 ssh = self._authenticate_with_hydra(admin_override=True) 287 if not ssh: 288 return 289 ssh.expect(RPMController.PASSWORD_PROMPT, timeout=60) 290 ssh.sendline(rpm_config.get('HYDRA', 'admin_password')) 291 ssh.expect(RPMController.HYDRA_PROMPT) 292 ssh.sendline(RPMController.CLI_CMD) 293 cli_prompt_re = re.compile(RPMController.CLI_PROMPT) 294 cli_held_re = re.compile(RPMController.CLI_HELD) 295 response = ssh.expect_list([cli_prompt_re, cli_held_re], timeout=60) 296 if response == 1: 297 # Need to kill the previous adminstator's session. 298 logging.error("Need to disconnect previous administrator's CLI " 299 "session to release the connection to RPM device %s.", 300 self.hostname) 301 ssh.sendline(RPMController.CLI_KILL_PREVIOUS) 302 ssh.expect(RPMController.CLI_PROMPT) 303 ssh.sendline(RPMController.PORT_STATUS_CMD) 304 ssh.expect(': %s' % self.hostname) 305 ports_status = ssh.before 306 port_number = ports_status.split(' ')[-1] 307 ssh.expect(RPMController.CLI_PROMPT) 308 ssh.sendline(RPMController.SESSION_KILL_CMD_FORMAT % port_number) 309 ssh.expect(RPMController.CLI_PROMPT) 310 self._logout(ssh, admin_logout=True) 311 312 313 def _hydra_login(self, ssh): 314 """ 315 Perform the extra steps required to log into a hydra serial 316 concentrator. 317 318 @param ssh: pexpect.spawn object used to communicate with the hydra 319 serial concentrator. 320 321 @return: True if the login procedure is successful. False if an error 322 occurred. The most common case would be if another user is 323 logged into the device. 324 """ 325 try: 326 response = ssh.expect_list( 327 [re.compile(RPMController.PASSWORD_PROMPT), 328 re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)], 329 timeout=15) 330 except pexpect.TIMEOUT: 331 # If there was a timeout, this ssh tunnel could be set up to 332 # not require the hydra password. 333 ssh.sendline('') 334 try: 335 ssh.expect(re.compile(RPMController.USERNAME_PROMPT)) 336 logging.debug('Connected to rpm through hydra. Logging in.') 337 return True 338 except pexpect.ExceptionPexpect: 339 return False 340 if response == 0: 341 try: 342 ssh.sendline(rpm_config.get('HYDRA','password')) 343 ssh.sendline('') 344 response = ssh.expect_list( 345 [re.compile(RPMController.USERNAME_PROMPT), 346 re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)], 347 timeout=60) 348 except pexpect.EOF: 349 # Did not receive any of the expect responses, retry. 350 return False 351 except pexpect.TIMEOUT: 352 logging.debug('Timeout occurred logging in to hydra.') 353 return False 354 # Send the username that the subclass will have set in its 355 # construction. 356 if response == 1: 357 logging.debug('SSH Terminal most likely serving another' 358 ' connection, retrying.') 359 # Kill the connection for the next connection attempt. 360 try: 361 self._kill_previous_connection() 362 except pexpect.ExceptionPexpect: 363 logging.error('Failed to disconnect previous connection, ' 364 'retrying.') 365 raise 366 return False 367 logging.debug('Connected to rpm through hydra. Logging in.') 368 return True 369 370 371 def _authenticate_with_hydra(self, admin_override=False): 372 """ 373 Some RPM's are behind a hydra serial concentrator and require their ssh 374 connection to be tunneled through this device. This can fail if another 375 user is logged in; therefore this will retry multiple times. 376 377 This function also allows us to authenticate directly to the 378 administrator interface of the hydra device. 379 380 @param admin_override: Set to True if we are trying to access the 381 administrator interface rather than tunnel 382 through to the RPM. 383 384 @return: The connected pexpect.spawn instance if the login procedure is 385 successful. None if an error occurred. The most common case 386 would be if another user is logged into the device. 387 """ 388 if admin_override: 389 username = rpm_config.get('HYDRA', 'admin_username') 390 else: 391 username = '%s:%s' % (rpm_config.get('HYDRA','username'), 392 self.hostname) 393 cmd = RPMController.SSH_LOGIN_CMD % (username, self.hydra_hostname) 394 num_attempts = 0 395 while num_attempts < RPMController.HYDRA_MAX_CONNECT_RETRIES: 396 try: 397 ssh = pexpect.spawn(cmd) 398 except pexpect.ExceptionPexpect: 399 return None 400 if admin_override: 401 return ssh 402 if self._hydra_login(ssh): 403 return ssh 404 # Authenticating with hydra failed. Sleep then retry. 405 time.sleep(RPMController.HYRDA_RETRY_SLEEP_SECS) 406 num_attempts += 1 407 logging.error('Failed to connect to the hydra serial concentrator after' 408 ' %d attempts.', RPMController.HYDRA_MAX_CONNECT_RETRIES) 409 return None 410 411 412 def _login(self): 413 """ 414 Log in into the RPM Device. 415 416 The login process should be able to connect to the device whether or not 417 it is behind a hydra serial concentrator. 418 419 @return: ssh - a pexpect.spawn instance if the connection was successful 420 or None if it was not. 421 """ 422 if self.behind_hydra: 423 # Tunnel the connection through the hydra. 424 ssh = self._authenticate_with_hydra() 425 if not ssh: 426 return None 427 ssh.sendline(self._username) 428 else: 429 # Connect directly to the RPM over SSH. 430 hostname = '%s.%s' % (self.hostname, self._dns_zone) 431 cmd = RPMController.SSH_LOGIN_CMD % (self._username, hostname) 432 try: 433 ssh = pexpect.spawn(cmd) 434 except pexpect.ExceptionPexpect: 435 return None 436 # Wait for the password prompt 437 try: 438 ssh.expect(self.PASSWORD_PROMPT, timeout=60) 439 ssh.sendline(self._password) 440 ssh.expect(self.DEVICE_PROMPT, timeout=60) 441 except pexpect.ExceptionPexpect: 442 return None 443 return ssh 444 445 446 def _logout(self, ssh, admin_logout=False): 447 """ 448 Log out of the RPM device. 449 450 Send the device specific logout command and if the connection is through 451 a hydra serial concentrator, kill the ssh connection. 452 453 @param admin_logout: Set to True if we are trying to logout of the 454 administrator interface of a hydra serial 455 concentrator, rather than an RPM. 456 @param ssh: pexpect.spawn instance to use to send the logout command. 457 """ 458 if admin_logout: 459 ssh.sendline(RPMController.QUIT_CMD) 460 ssh.expect(RPMController.HYDRA_PROMPT) 461 ssh.sendline(self.LOGOUT_CMD) 462 if self.behind_hydra and not admin_logout: 463 # Terminate the hydra session. 464 ssh.sendline('~.') 465 # Wait a bit so hydra disconnects completely. Launching another 466 # request immediately can cause a timeout. 467 time.sleep(5) 468 469 470 def set_power_state(self, powerunit_info, new_state): 471 """ 472 Set the state of the dut's outlet on this RPM. 473 474 For ssh based devices, this will create the connection either directly 475 or through a hydra tunnel and call the underlying _change_state function 476 to be implemented by the subclass device. 477 478 For non-ssh based devices, this method should be overloaded with the 479 proper connection and state change code. And the subclass will handle 480 accessing the RPM devices. 481 482 @param powerunit_info: An instance of PowerUnitInfo. 483 @param new_state: ON/OFF/CYCLE - state or action we want to perform on 484 the outlet. 485 486 @return: True if the attempt to change power state was successful, 487 False otherwise. 488 """ 489 ssh = self._login() 490 if not ssh: 491 return False 492 if new_state == self.NEW_STATE_CYCLE: 493 logging.debug('Beginning Power Cycle for device: %s', 494 powerunit_info.device_hostname) 495 result = self._change_state(powerunit_info, self.NEW_STATE_OFF, ssh) 496 if not result: 497 return result 498 time.sleep(RPMController.CYCLE_SLEEP_TIME) 499 result = self._change_state(powerunit_info, self.NEW_STATE_ON, ssh) 500 else: 501 # Try to change the state of the device's power outlet. 502 result = self._change_state(powerunit_info, new_state, ssh) 503 504 # Terminate hydra connection if necessary. 505 self._logout(ssh) 506 ssh.close(force=True) 507 return result 508 509 510 def _change_state(self, powerunit_info, new_state, ssh): 511 """ 512 Perform the actual state change operation. 513 514 Once we have established communication with the RPM this method is 515 responsible for changing the state of the RPM outlet. 516 517 @param powerunit_info: An instance of PowerUnitInfo. 518 @param new_state: ON/OFF - state or action we want to perform on 519 the outlet. 520 @param ssh: The ssh connection used to execute the state change commands 521 on the RPM device. 522 523 @return: True if the attempt to change power state was successful, 524 False otherwise. 525 """ 526 outlet = powerunit_info.outlet 527 device_hostname = powerunit_info.device_hostname 528 if not outlet: 529 logging.error('Request to change outlet for device: %s to new ' 530 'state %s failed: outlet is unknown, please ' 531 'make sure POWERUNIT_OUTLET exist in the host\'s ' 532 'attributes in afe.', device_hostname, new_state) 533 ssh.sendline(self.SET_STATE_CMD % (new_state, outlet)) 534 if self.SUCCESS_MSG: 535 # If this RPM device returns a success message check for it before 536 # continuing. 537 try: 538 ssh.expect(self.SUCCESS_MSG, timeout=60) 539 except pexpect.ExceptionPexpect: 540 logging.error('Request to change outlet for device: %s to new ' 541 'state %s failed.', device_hostname, new_state) 542 return False 543 logging.debug('Outlet for device: %s set to %s', device_hostname, 544 new_state) 545 return True 546 547 548 def type(self): 549 """ 550 Get the type of RPM device we are interacting with. 551 Class attribute TYPE should be set by the subclasses. 552 553 @return: string representation of RPM device type. 554 """ 555 return self.TYPE 556 557 558class SentryRPMController(RPMController): 559 """ 560 This class implements power control for Sentry Switched CDU 561 http://www.servertech.com/products/switched-pdus/ 562 563 Example usage: 564 rpm = SentrySwitchedCDU('chromeos-rack1-rpm1') 565 rpm.queue_request('chromeos-rack1-host1', 'ON') 566 567 @var _username: username used to access device. 568 @var _password: password used to access device. 569 """ 570 571 DEVICE_PROMPT = ['Switched CDU:', 'Switched PDU:'] 572 SET_STATE_CMD = '%s %s' 573 SUCCESS_MSG = 'Command successful' 574 NUM_OF_OUTLETS = 17 575 TYPE = 'Sentry' 576 577 578 def __init__(self, hostname, hydra_hostname=None): 579 super(SentryRPMController, self).__init__(hostname, hydra_hostname) 580 self._username = rpm_config.get('SENTRY', 'username') 581 self._password = rpm_config.get('SENTRY', 'password') 582 583 584 def _setup_test_user(self, ssh): 585 """Configure the test user for the RPM 586 587 @param ssh: Pexpect object to use to configure the RPM. 588 """ 589 # Create and configure the testing user profile. 590 testing_user = rpm_config.get('SENTRY','testing_user') 591 testing_password = rpm_config.get('SENTRY','testing_password') 592 ssh.sendline('create user %s' % testing_user) 593 response = ssh.expect_list([re.compile('not unique'), 594 re.compile(self.PASSWORD_PROMPT)]) 595 if not response: 596 return 597 # Testing user is not set up yet. 598 ssh.sendline(testing_password) 599 ssh.expect('Verify Password:') 600 ssh.sendline(testing_password) 601 ssh.expect(self.SUCCESS_MSG) 602 ssh.expect(self.DEVICE_PROMPT) 603 ssh.sendline('add outlettouser all %s' % testing_user) 604 ssh.expect(self.SUCCESS_MSG) 605 ssh.expect(self.DEVICE_PROMPT) 606 607 608 def _clear_outlet_names(self, ssh): 609 """ 610 Before setting the outlet names, we need to clear out all the old 611 names so there are no conflicts. For example trying to assign outlet 612 2 a name already assigned to outlet 9. 613 """ 614 for outlet in range(1, self.NUM_OF_OUTLETS): 615 outlet_name = 'Outlet_%d' % outlet 616 ssh.sendline(self.SET_OUTLET_NAME_CMD % (outlet, outlet_name)) 617 ssh.expect(self.SUCCESS_MSG) 618 ssh.expect(self.DEVICE_PROMPT) 619 620 621 def setup(self, outlet_naming_map): 622 """ 623 Configure the RPM by adding the test user and setting up the outlet 624 names. 625 626 Note the rpm infrastructure does not rely on the outlet name to map a 627 device to its outlet any more. We keep this method in case there is 628 a need to label outlets for other reasons. We may deprecate 629 this method if it has been proved the outlet names will not be used 630 in any scenario. 631 632 @param outlet_naming_map: Dictionary used to map the outlet numbers to 633 host names. Keys must be ints. And names are 634 in the format of 'hostX'. 635 636 @return: True if setup completed successfully, False otherwise. 637 """ 638 ssh = self._login() 639 if not ssh: 640 logging.error('Could not connect to %s.', self.hostname) 641 return False 642 try: 643 self._setup_test_user(ssh) 644 # Set up the outlet names. 645 # Hosts have the same name format as the RPM hostname except they 646 # end in hostX instead of rpmX. 647 dut_name_format = re.sub('-rpm[0-9]*', '', self.hostname) 648 if self.behind_hydra: 649 # Remove "chromeosX" from DUTs behind the hydra due to a length 650 # constraint on the names we can store inside the RPM. 651 dut_name_format = re.sub('chromeos[0-9]*-', '', dut_name_format) 652 dut_name_format = dut_name_format + '-%s' 653 self._clear_outlet_names(ssh) 654 for outlet, name in outlet_naming_map.items(): 655 dut_name = dut_name_format % name 656 ssh.sendline(self.SET_OUTLET_NAME_CMD % (outlet, dut_name)) 657 ssh.expect(self.SUCCESS_MSG) 658 ssh.expect(self.DEVICE_PROMPT) 659 except pexpect.ExceptionPexpect as e: 660 logging.error('Setup failed. %s', e) 661 return False 662 finally: 663 self._logout(ssh) 664 return True 665 666 667class WebPoweredRPMController(RPMController): 668 """ 669 This class implements RPMController for the Web Powered units 670 produced by Digital Loggers Inc. 671 672 @var _rpm: dli_urllib.Powerswitch instance used to interact with RPM. 673 """ 674 675 676 TYPE = 'Webpowered' 677 678 679 def __init__(self, hostname, powerswitch=None): 680 username = rpm_config.get('WEBPOWERED', 'username') 681 password = rpm_config.get('WEBPOWERED', 'password') 682 # Call the constructor in RPMController. However since this is a web 683 # accessible device, there should not be a need to tunnel through a 684 # hydra serial concentrator. 685 super(WebPoweredRPMController, self).__init__(hostname) 686 self.hostname = '%s.%s' % (self.hostname, self._dns_zone) 687 if not powerswitch: 688 self._rpm = dli_urllib.Powerswitch(hostname=self.hostname, 689 userid=username, 690 password=password) 691 else: 692 # Should only be used in unit_testing 693 self._rpm = powerswitch 694 695 696 def _get_outlet_state(self, outlet): 697 """ 698 Look up the state for a given outlet on the RPM. 699 700 @param outlet: the outlet to look up. 701 702 @return state: the outlet's current state. 703 """ 704 status_list = self._rpm.statuslist() 705 for outlet_name, _, state in status_list: 706 if outlet_name == outlet: 707 return state 708 return None 709 710 711 def set_power_state(self, powerunit_info, new_state): 712 """ 713 Since this does not utilize SSH in any manner, this will overload the 714 set_power_state in RPMController and completes all steps of changing 715 the device's outlet state. 716 """ 717 device_hostname = powerunit_info.device_hostname 718 outlet = powerunit_info.outlet 719 if not outlet: 720 logging.error('Request to change outlet for device %s to ' 721 'new state %s failed: outlet is unknown. Make sure ' 722 'POWERUNIT_OUTLET exists in the host\'s ' 723 'attributes in afe' , device_hostname, new_state) 724 return False 725 expected_state = new_state 726 if new_state == self.NEW_STATE_CYCLE: 727 logging.debug('Beginning Power Cycle for device: %s', 728 device_hostname) 729 self._rpm.off(outlet) 730 logging.debug('Outlet for device: %s set to OFF', device_hostname) 731 # Pause for 5 seconds before restoring power. 732 time.sleep(RPMController.CYCLE_SLEEP_TIME) 733 self._rpm.on(outlet) 734 logging.debug('Outlet for device: %s set to ON', device_hostname) 735 expected_state = self.NEW_STATE_ON 736 if new_state == self.NEW_STATE_OFF: 737 self._rpm.off(outlet) 738 logging.debug('Outlet for device: %s set to OFF', device_hostname) 739 if new_state == self.NEW_STATE_ON: 740 self._rpm.on(outlet) 741 logging.debug('Outlet for device: %s set to ON', device_hostname) 742 # Lookup the final state of the outlet 743 return self._is_plug_state(powerunit_info, expected_state) 744 745 746 def _is_plug_state(self, powerunit_info, expected_state): 747 state = self._get_outlet_state(powerunit_info.outlet) 748 if expected_state not in state: 749 logging.error('Outlet for device: %s did not change to new state' 750 ' %s', powerunit_info.device_hostname, expected_state) 751 return False 752 return True 753 754 755class CiscoPOEController(RPMController): 756 """ 757 This class implements power control for Cisco POE switch. 758 759 Example usage: 760 poe = CiscoPOEController('chromeos1-poe-switch1') 761 poe.queue_request('chromeos1-rack5-host12-servo', 'ON') 762 """ 763 764 765 SSH_LOGIN_CMD = ('ssh -o StrictHostKeyChecking=no ' 766 '-o UserKnownHostsFile=/dev/null %s') 767 POE_USERNAME_PROMPT = 'User Name:' 768 POE_PROMPT = '%s#' 769 EXIT_CMD = 'exit' 770 END_CMD = 'end' 771 CONFIG = 'configure terminal' 772 CONFIG_PROMPT = r'%s\(config\)#' 773 CONFIG_IF = 'interface %s' 774 CONFIG_IF_PROMPT = r'%s\(config-if\)#' 775 SET_STATE_ON = 'power inline auto' 776 SET_STATE_OFF = 'power inline never' 777 CHECK_INTERFACE_STATE = 'show interface status %s' 778 INTERFACE_STATE_MSG = r'Port\s+.*%s(\s+(\S+)){6,6}' 779 CHECK_STATE_TIMEOUT = 60 780 CMD_TIMEOUT = 30 781 LOGIN_TIMEOUT = 60 782 PORT_UP = 'Up' 783 PORT_DOWN = 'Down' 784 TYPE = 'CiscoPOE' 785 786 787 def __init__(self, hostname): 788 """ 789 Initialize controller class for a Cisco POE switch. 790 791 @param hostname: the Cisco POE switch host name. 792 """ 793 super(CiscoPOEController, self).__init__(hostname) 794 self._username = rpm_config.get('CiscoPOE', 'username') 795 self._password = rpm_config.get('CiscoPOE', 'password') 796 # For a switch, e.g. 'chromeos2-poe-switch8', 797 # the device prompt looks like 'chromeos2-poe-sw8#'. 798 short_hostname = self.hostname.replace('switch', 'sw') 799 self.poe_prompt = self.POE_PROMPT % short_hostname 800 self.config_prompt = self.CONFIG_PROMPT % short_hostname 801 self.config_if_prompt = self.CONFIG_IF_PROMPT % short_hostname 802 803 804 def _login(self): 805 """ 806 Log in into the Cisco POE switch. 807 808 Overload _login in RPMController, as it always prompts for a user name. 809 810 @return: ssh - a pexpect.spawn instance if the connection was successful 811 or None if it was not. 812 """ 813 hostname = '%s.%s' % (self.hostname, self._dns_zone) 814 cmd = self.SSH_LOGIN_CMD % (hostname) 815 try: 816 ssh = pexpect.spawn(cmd) 817 except pexpect.ExceptionPexpect: 818 logging.error('Could not connect to switch %s', hostname) 819 return None 820 # Wait for the username and password prompt. 821 try: 822 ssh.expect(self.POE_USERNAME_PROMPT, timeout=self.LOGIN_TIMEOUT) 823 ssh.sendline(self._username) 824 ssh.expect(self.PASSWORD_PROMPT, timeout=self.LOGIN_TIMEOUT) 825 ssh.sendline(self._password) 826 ssh.expect(self.poe_prompt, timeout=self.LOGIN_TIMEOUT) 827 except pexpect.ExceptionPexpect: 828 logging.error('Could not log into switch %s', hostname) 829 return None 830 return ssh 831 832 833 def _enter_configuration_terminal(self, interface, ssh): 834 """ 835 Enter configuration terminal of |interface|. 836 837 This function expects that we've already logged into the switch 838 and the ssh is prompting the switch name. The work flow is 839 chromeos1-poe-sw1# 840 chromeos1-poe-sw1#configure terminal 841 chromeos1-poe-sw1(config)#interface fa36 842 chromeos1-poe-sw1(config-if)# 843 On success, the function exits with 'config-if' prompt. 844 On failure, the function exits with device prompt, 845 e.g. 'chromeos1-poe-sw1#' in the above case. 846 847 @param interface: the name of the interface. 848 @param ssh: pexpect.spawn instance to use. 849 850 @return: True on success otherwise False. 851 """ 852 try: 853 # Enter configure terminal. 854 ssh.sendline(self.CONFIG) 855 ssh.expect(self.config_prompt, timeout=self.CMD_TIMEOUT) 856 # Enter configure terminal of the interface. 857 ssh.sendline(self.CONFIG_IF % interface) 858 ssh.expect(self.config_if_prompt, timeout=self.CMD_TIMEOUT) 859 return True 860 except pexpect.ExceptionPexpect, e: 861 ssh.sendline(self.END_CMD) 862 logging.exception(e) 863 return False 864 865 866 def _exit_configuration_terminal(self, ssh): 867 """ 868 Exit interface configuration terminal. 869 870 On success, the function exits with device prompt, 871 e.g. 'chromeos1-poe-sw1#' in the above case. 872 On failure, the function exists with 'config-if' prompt. 873 874 @param ssh: pexpect.spawn instance to use. 875 876 @return: True on success otherwise False. 877 """ 878 try: 879 ssh.sendline(self.END_CMD) 880 ssh.expect(self.poe_prompt, timeout=self.CMD_TIMEOUT) 881 return True 882 except pexpect.ExceptionPexpect, e: 883 logging.exception(e) 884 return False 885 886 887 def _verify_state(self, interface, expected_state, ssh): 888 """ 889 Check whehter the current state of |interface| matches expected state. 890 891 This function tries to check the state of |interface| multiple 892 times until its state matches the expected state or time is out. 893 894 After the command of changing state has been executed, 895 the state of an interface doesn't always change immediately to 896 the expected state but requires some time. As such, we need 897 a retry logic here. 898 899 @param interface: the name of the interface. 900 @param expect_state: the expected state, 'ON' or 'OFF' 901 @param ssh: pexpect.spawn instance to use. 902 903 @return: True if the state of |interface| swiches to |expected_state|, 904 otherwise False. 905 """ 906 expected_state = (self.PORT_UP if expected_state == self.NEW_STATE_ON 907 else self.PORT_DOWN) 908 try: 909 start = time.time() 910 while((time.time() - start) < self.CHECK_STATE_TIMEOUT): 911 ssh.sendline(self.CHECK_INTERFACE_STATE % interface) 912 state_matcher = '.*'.join([self.INTERFACE_STATE_MSG % interface, 913 self.poe_prompt]) 914 ssh.expect(state_matcher, timeout=self.CMD_TIMEOUT) 915 state = ssh.match.group(2) 916 if state == expected_state: 917 return True 918 except pexpect.ExceptionPexpect, e: 919 logging.exception(e) 920 return False 921 922 923 def _logout(self, ssh, admin_logout=False): 924 """ 925 Log out of the Cisco POE switch after changing state. 926 927 Overload _logout in RPMController. 928 929 @param admin_logout: ignored by this method. 930 @param ssh: pexpect.spawn instance to use to send the logout command. 931 """ 932 ssh.sendline(self.EXIT_CMD) 933 934 935 def _change_state(self, powerunit_info, new_state, ssh): 936 """ 937 Perform the actual state change operation. 938 939 Overload _change_state in RPMController. 940 941 @param powerunit_info: An PowerUnitInfo instance. 942 @param new_state: ON/OFF - state or action we want to perform on 943 the outlet. 944 @param ssh: The ssh connection used to execute the state change commands 945 on the POE switch. 946 947 @return: True if the attempt to change power state was successful, 948 False otherwise. 949 """ 950 interface = powerunit_info.outlet 951 device_hostname = powerunit_info.device_hostname 952 if not interface: 953 logging.error('Could not change state: the interface on %s for %s ' 954 'was not given.', self.hostname, device_hostname) 955 return False 956 957 # Enter configuration terminal. 958 if not self._enter_configuration_terminal(interface, ssh): 959 logging.error('Could not enter configuration terminal for %s', 960 interface) 961 return False 962 # Change the state. 963 if new_state == self.NEW_STATE_ON: 964 ssh.sendline(self.SET_STATE_ON) 965 elif new_state == self.NEW_STATE_OFF: 966 ssh.sendline(self.SET_STATE_OFF) 967 else: 968 logging.error('Unknown state request: %s', new_state) 969 return False 970 # Exit configuraiton terminal. 971 if not self._exit_configuration_terminal(ssh): 972 logging.error('Skipping verifying outlet state for device: %s, ' 973 'because could not exit configuration terminal.', 974 device_hostname) 975 return False 976 # Verify if the state has changed successfully. 977 if not self._verify_state(interface, new_state, ssh): 978 logging.error('Could not verify state on interface %s', interface) 979 return False 980 981 logging.debug('Outlet for device: %s set to %s', 982 device_hostname, new_state) 983 return True 984