1# Lint as: python2, python3 2# Copyright 2016 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import absolute_import 7from __future__ import division 8from __future__ import print_function 9 10import sys 11import functools 12import logging 13import math 14import time 15 16import common 17from autotest_lib.client.common_lib import error 18from autotest_lib.client.common_lib import hosts 19from autotest_lib.client.common_lib import utils 20from autotest_lib.server.cros.power import servo_charger 21from autotest_lib.server.cros.servo import servo 22from autotest_lib.server.hosts import cros_constants 23from autotest_lib.server.hosts import repair_utils 24from autotest_lib.server.hosts import servo_constants 25from autotest_lib.server.cros.servo.topology import servo_topology 26from autotest_lib.site_utils.admin_audit import servo_updater 27import six 28 29try: 30 from chromite.lib import metrics 31except ImportError: 32 metrics = utils.metrics_mock 33 34 35# TODO(gregorynisbet): will importing chromite always succeed in all contexts? 36from chromite.lib import timeout_util 37 38 39def ignore_exception_for_non_cros_host(func): 40 """ 41 Decorator to ignore ControlUnavailableError if servo host is not cros host. 42 When using test_that command on a workstation, this enables usage of 43 additional servo devices such as servo micro and Sweetberry. This shall not 44 change any lab behavior. 45 """ 46 @functools.wraps(func) 47 def wrapper(self, host): 48 """ 49 Wrapper around func. 50 """ 51 try: 52 func(self, host) 53 except servo.ControlUnavailableError as e: 54 if host.is_cros_host(): 55 raise 56 logging.warning("Servo host is not cros host, ignore %s: %s", 57 type(e).__name__, e) 58 return wrapper 59 60 61class _UpdateVerifier(hosts.Verifier): 62 """ 63 Verifier to trigger a servo host update, if necessary. 64 65 The operation doesn't wait for the update to complete and is 66 considered a success whether or not the servo is currently 67 up-to-date. 68 """ 69 70 @timeout_util.TimeoutDecorator(cros_constants.LONG_VERIFY_TIMEOUT_SEC) 71 def verify(self, host): 72 # First, only run this verifier if the host is in the physical lab. 73 # Secondly, skip if the test is being run by test_that, because subnet 74 # restrictions can cause the update to fail. 75 try: 76 if host.is_labstation(): 77 logging.info("Skip update check because the host is a" 78 " labstation and labstation update is handled" 79 " by labstation AdminRepair task.") 80 return 81 if host.is_in_lab() and host.job and host.job.in_lab: 82 if ( 83 not host.get_dut_host_info() or 84 not host.get_dut_host_info().servo_cros_stable_version 85 ): 86 logging.info('Servo stable version missed.' 87 ' Skip update check action.') 88 return 89 # We have seen cases that invalid GPT headers/entries block 90 # v3s from been update, so always try to repair here. 91 # See crbug.com/994396, crbug.com/1057302. 92 host.run('cgpt repair /dev/mmcblk0', ignore_status=True) 93 host.update_image() 94 # We don't want failure from update block DUT repair action. 95 # See crbug.com/1029950. 96 except Exception as e: 97 six.reraise(hosts.AutoservNonCriticalVerifyError, str(e), 98 sys.exc_info()[2]) 99 100 @property 101 def description(self): 102 return 'servo host software is up-to-date' 103 104 105class _ConfigVerifier(hosts.Verifier): 106 """ 107 Base verifier for the servo config file verifiers. 108 """ 109 110 CONFIG_FILE = '/var/lib/servod/config' 111 ATTR = '' 112 113 @staticmethod 114 def _get_config_val(host, config_file, attr): 115 """ 116 Get the `attr` for `host` from `config_file`. 117 118 @param host Host to be checked for `config_file`. 119 @param config_file Path to the config file to be tested. 120 @param attr Attribute to get from config file. 121 122 @return The attr val as set in the config file, or `None` if 123 the file was absent. 124 """ 125 getboard = ('CONFIG=%s ; [ -f $CONFIG ] && ' 126 '. $CONFIG && echo $%s' % (config_file, attr)) 127 attr_val = host.run(getboard, ignore_status=True).stdout 128 return attr_val.strip('\n') if attr_val else None 129 130 @staticmethod 131 def _validate_attr(host, val, expected_val, attr, config_file): 132 """ 133 Check that the attr setting is valid for the host. 134 135 This presupposes that a valid config file was found. Raise an 136 execption if: 137 * There was no attr setting from the file (i.e. the setting 138 is an empty string), or 139 * The attr setting is valid, the attr is known, 140 and the setting doesn't match the DUT. 141 142 @param host Host to be checked for `config_file`. 143 @param val Value to be tested. 144 @param expected_val Expected value. 145 @param attr Attribute we're validating. 146 @param config_file Path to the config file to be tested. 147 """ 148 if not val: 149 raise hosts.AutoservVerifyError( 150 'config file %s exists, but %s ' 151 'is not set' % (attr, config_file)) 152 if expected_val is not None and val != expected_val: 153 raise hosts.AutoservVerifyError( 154 '%s is %s; it should be %s' % (attr, val, expected_val)) 155 156 157 def _get_config(self, host): 158 """ 159 Return the config file to check. 160 161 @param host Host object. 162 163 @return The config file to check. 164 """ 165 return '%s_%d' % (self.CONFIG_FILE, host.servo_port) 166 167 @property 168 def description(self): 169 return 'servo %s setting is correct' % self.ATTR 170 171 172class _SerialConfigVerifier(_ConfigVerifier): 173 """ 174 Verifier for the servo SERIAL configuration. 175 """ 176 177 ATTR = 'SERIAL' 178 179 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 180 def verify(self, host): 181 """ 182 Test whether the `host` has a `SERIAL` setting configured. 183 184 This tests the config file names used by the `servod` upstart 185 job for a valid setting of the `SERIAL` variable. The following 186 conditions raise errors: 187 * The SERIAL setting doesn't match the DUT's entry in the AFE 188 database. 189 * There is no config file. 190 """ 191 if not host.is_cros_host(): 192 return 193 # Not all servo hosts will have a servo serial so don't verify if it's 194 # not set. 195 if host.servo_serial is None: 196 return 197 config = self._get_config(host) 198 serialval = self._get_config_val(host, config, self.ATTR) 199 if serialval is None: 200 raise hosts.AutoservVerifyError( 201 'Servo serial is unconfigured; should be %s' 202 % host.servo_serial 203 ) 204 205 self._validate_attr(host, serialval, host.servo_serial, self.ATTR, 206 config) 207 208 209 210class _BoardConfigVerifier(_ConfigVerifier): 211 """ 212 Verifier for the servo BOARD configuration. 213 """ 214 215 ATTR = 'BOARD' 216 217 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 218 def verify(self, host): 219 """ 220 Test whether the `host` has a `BOARD` setting configured. 221 222 This tests the config file names used by the `servod` upstart 223 job for a valid setting of the `BOARD` variable. The following 224 conditions raise errors: 225 * A config file exists, but the content contains no setting 226 for BOARD. 227 * The BOARD setting doesn't match the DUT's entry in the AFE 228 database. 229 * There is no config file. 230 """ 231 if not host.is_cros_host(): 232 return 233 config = self._get_config(host) 234 boardval = self._get_config_val(host, config, self.ATTR) 235 if boardval is None: 236 msg = 'Servo board is unconfigured' 237 if host.servo_board is not None: 238 msg += '; should be %s' % host.servo_board 239 raise hosts.AutoservVerifyError(msg) 240 241 self._validate_attr(host, boardval, host.servo_board, self.ATTR, 242 config) 243 244 245class _ServodJobVerifier(hosts.Verifier): 246 """ 247 Verifier to check that the `servod` upstart job is running. 248 """ 249 250 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 251 def verify(self, host): 252 if not host.is_cros_host(): 253 return 254 status_cmd = 'status servod PORT=%d' % host.servo_port 255 job_status = host.run(status_cmd, ignore_status=True).stdout 256 if 'start/running' not in job_status: 257 raise hosts.AutoservVerifyError( 258 'servod not running on %s port %d' % 259 (host.hostname, host.servo_port)) 260 261 @property 262 def description(self): 263 return 'servod upstart job is running' 264 265 266class _DiskSpaceVerifier(hosts.Verifier): 267 """ 268 Verifier to make sure there is enough disk space left on servohost. 269 """ 270 271 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 272 def verify(self, host): 273 # Check available space of stateful is greater than threshold, in Gib. 274 host.check_diskspace('/mnt/stateful_partition', 0.5) 275 276 @property 277 def description(self): 278 return 'servohost has enough disk space.' 279 280 281class _ServodConnectionVerifier(hosts.Verifier): 282 """ 283 Verifier to check that we can connect to servod server. 284 285 If this verifier failed, it most likely servod was crashed or in a 286 crashing loop. For servo_v4 it's usually caused by not able to detect 287 CCD or servo_micro. 288 """ 289 290 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 291 def verify(self, host): 292 host.initilize_servo() 293 294 @property 295 def description(self): 296 return 'servod service is taking calls' 297 298 299class _ServodControlVerifier(hosts.Verifier): 300 """ 301 Verifier to check basic servo control functionality. 302 303 This tests the connection to the target servod service with a simple 304 method call. As a side-effect, all servo signals are initialized to 305 default values. 306 307 N.B. Initializing servo signals is necessary because the power 308 button and lid switch verifiers both test against expected initial 309 values. 310 """ 311 312 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 313 def verify(self, host): 314 try: 315 host.initialize_dut_for_servo() 316 except Exception as e: 317 six.reraise(hosts.AutoservNonCriticalVerifyError, str(e), 318 sys.exc_info()[2]) 319 320 @property 321 def description(self): 322 return 'Basic servod control is working' 323 324 325class _Cr50ConsoleVerifier(hosts.Verifier): 326 """Verifier to check if cr50 console is present and working. 327 328 Validating based by running commands and expect they will not fail. 329 If any command fail then console is not working as expected. 330 """ 331 332 COMMAND_TO_CHECK_CONSOLE = ( 333 'cr50_ccd_level', 334 'cr50_testlab', 335 'cr50_ccd_state_flags', 336 ) 337 338 @ignore_exception_for_non_cros_host 339 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 340 def verify(self, host): 341 try: 342 for command in self.COMMAND_TO_CHECK_CONSOLE: 343 if host.get_servo().has_control(command): 344 # Response of command is not important. 345 host.get_servo().get(command) 346 except Exception as e: 347 six.reraise(hosts.AutoservNonCriticalVerifyError, str(e), 348 sys.exc_info()[2]) 349 350 def _is_applicable(self, host): 351 # Only when DUT is running through ccd. 352 # TODO(coconutruben): replace with ccd API when available in servo.py 353 return (host.get_servo() 354 and host.get_servo().get_main_servo_device() == 'ccd_cr50') 355 356 @property 357 def description(self): 358 return 'CR50 console is working' 359 360 361class _CCDTestlabVerifier(hosts.Verifier): 362 """ 363 Verifier to check that ccd testlab is enabled. 364 365 All DUT connected by ccd has to supported cr50 with enabled testlab 366 to allow manipulation by servo. The flag testlab is sticky and will 367 stay enabled if was set up. The testlab can be enabled when ccd is 368 open. (go/ccd-setup) 369 """ 370 @ignore_exception_for_non_cros_host 371 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 372 def verify(self, host): 373 if not host.get_servo().has_control('cr50_testlab'): 374 raise hosts.AutoservVerifyError( 375 'cr50 has to be supported when use servo with ' 376 'ccd_cr50/type-c connection') 377 378 status = host.get_servo().get('cr50_testlab') 379 # check by 'on' to fail when get unexpected value 380 if status == 'on': 381 # ccd testlab enabled 382 return 383 raise hosts.AutoservNonCriticalVerifyError( 384 'The ccd testlab is disabled; DUT requires manual work ' 385 'to enable it (go/ccd-setup).') 386 387 def _is_applicable(self, host): 388 # Only when DUT is running through ccd. 389 # TODO(coconutruben): replace with ccd API when available in servo.py 390 return (host.get_servo() 391 and host.get_servo().get_main_servo_device() == 'ccd_cr50') 392 393 @property 394 def description(self): 395 return 'ccd testlab enabled' 396 397class _CCDPowerDeliveryVerifier(hosts.Verifier): 398 """Verifier to check and reset servo_v4_role for servos that support 399 power delivery feature(a.k.a power pass through). 400 401 There are currently two position of servo_v4_role, src and snk: 402 src -- servo in power delivery mode and passes power to the DUT. 403 snk -- servo in normal mode and not passes power to DUT. 404 We want to ensure that servo_v4_role is set to src. 405 406 TODO(xianuowang@) Convert it to verifier/repair action pair or remove it 407 once we collected enough metrics. 408 """ 409 # Change to use the constant value in CrosHost if we move it to 410 # verifier/repair pair. 411 CHANGE_SERVO_ROLE_TIMEOUT = 180 412 413 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 414 def verify(self, host): 415 if host.get_servo().get('servo_v4_role') == 'snk': 416 logging.warning('The servo initlized with role snk while' 417 ' supporting power delivery, resetting role' 418 ' to src...') 419 420 try: 421 logging.info('setting power direction with retries') 422 # do not pass host since host does not inherit from CrosHost. 423 charge_manager = servo_charger.ServoV4ChargeManager( 424 host=None, 425 servo=host.get_servo(), 426 ) 427 attempts = charge_manager.start_charging() 428 logging.info('setting power direction took %d tries', attempts) 429 # if control makes it here, we successfully changed the host 430 # direction 431 result = 'src' 432 except Exception as e: 433 logging.error( 434 'setting power direction with retries failed %s', 435 str(e), 436 ) 437 finally: 438 time.sleep(self.CHANGE_SERVO_ROLE_TIMEOUT) 439 440 result = host.get_servo().get('servo_v4_role') 441 logging.debug('Servo_v4 role after reset: %s', result) 442 443 metrics_data = { 444 'hostname': host.get_dut_hostname() or 'unknown', 445 'status': 'success' if result == 'src' else 'failed', 446 'board': host.servo_board or 'unknown', 447 'model': host.servo_model or 'unknown' 448 } 449 metrics.Counter( 450 'chromeos/autotest/repair/verifier/power_delivery3' 451 ).increment(fields=metrics_data) 452 453 def _is_applicable(self, host): 454 return (host.is_in_lab() and 455 host.get_servo().supports_built_in_pd_control()) 456 457 @property 458 def description(self): 459 return 'ensure applicable servo is in "src" mode for power delivery' 460 461 462class _BaseDUTConnectionVerifier(hosts.Verifier): 463 """Verifier to check connection between DUT and servo.""" 464 465 # Bus voltage on ppdut5. Value can be: 466 # - less than 500 - DUT is likely not connected 467 # - between 500 and 4000 - unexpected value 468 # - more than 4000 - DUT is likely connected 469 MAX_PPDUT5_MV_WHEN_NOT_CONNECTED = 500 470 MIN_PPDUT5_MV_WHEN_CONNECTED = 4000 471 472 def _is_usb_hub_connected(self, host): 473 """Checking bus voltage on ppdut5. 474 475 Supported only on servo_v4 boards. 476 If voltage value is lower than 500 then device is not connected. 477 When value higher 4000 means the device is connected. If value 478 between 500 and 4000 is not expected and will be marked as connected 479 and collected information which DUT has this exception. 480 481 @returns: bool 482 """ 483 logging.debug('Started check by ppdut5_mv:on') 484 try: 485 val = host.get_servo().get('ppdut5_mv') 486 if val < self.MAX_PPDUT5_MV_WHEN_NOT_CONNECTED: 487 # servo is not connected to the DUT 488 return False 489 if val < self.MIN_PPDUT5_MV_WHEN_CONNECTED: 490 # is unexpected value. 491 # collecting metrics to look case by case 492 # TODO(otabek) for analysis b:163845694 493 data = host._get_host_metrics_data() 494 metrics.Counter('chromeos/autotest/repair/ppdut5_mv_case' 495 ).increment(fields=data) 496 # else: 497 # servo is physical connected to the DUT 498 except Exception as e: 499 logging.debug('(Not critical) %s', e) 500 return True 501 502 def _is_ribbon_cable_connected(self, host): 503 """Check if ribbon cable is connected to the DUT. 504 505 The servo_micro/flex - can be checked by `cold_reset` signal. 506 When `cold_reset` is `on` it commonly indicates that the DUT 507 is disconnected. To avoid mistake of real signal we try 508 switch it off and if is cannot then servo is not connected. 509 510 @returns: bool 511 """ 512 logging.debug('Started check by cold_reset:on') 513 try: 514 if host.get_servo().get('cold_reset') == 'on': 515 # If cold_reset has is on can be right signal 516 # or caused by missing connection between servo_micro and DUT. 517 # if we can switch it to the off then it was signal. 518 host.get_servo().set('cold_reset', 'off') 519 except error.TestFail: 520 logging.debug('Ribbon cable is not connected to the DUT.') 521 return False 522 except Exception as e: 523 logging.debug('(Not critical) %s', e) 524 return True 525 526 def _is_dut_power_on(self, host): 527 # DUT is running in normal state. 528 # if EC not supported by board then we expect error 529 try: 530 return host.get_servo().get('ec_system_powerstate') == 'S0' 531 except Exception as e: 532 logging.debug('(Not critical) %s', e) 533 return False 534 535 def _is_servo_v4_type_a(self, host): 536 return (host.is_labstation() 537 and host.get_servo().has_control('servo_v4_type') 538 and host.get_servo().get('servo_v4_type') == 'type-a') 539 540 def _is_servo_v4_type_c(self, host): 541 return (host.is_labstation() 542 and host.get_servo().has_control('servo_v4_type') 543 and host.get_servo().get('servo_v4_type') == 'type-c') 544 545 def _is_servo_v3(self, host): 546 return not host.is_labstation() 547 548 549class _DUTConnectionVerifier(_BaseDUTConnectionVerifier): 550 """Verifier to check connection Servo to the DUT. 551 552 Servo_v4 type-a connected to the DUT by: 553 1) servo_micro - checked by `cold_reset`. 554 Servo_v4 type-c connected to the DUT by: 555 1) ccd - checked by ppdut5_mv. 556 Servo_v3 connected to the DUT by: 557 1) legacy servo header - can be checked by `cold_reset`. 558 """ 559 560 @ignore_exception_for_non_cros_host 561 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 562 def verify(self, host): 563 if self._is_servo_v4_type_a(host): 564 if not self._is_ribbon_cable_connected(host): 565 raise hosts.AutoservVerifyError( 566 'Servo_micro is likely not connected to the DUT.') 567 elif self._is_servo_v4_type_c(host): 568 logging.info('Skip check for type-c till confirm it in the lab') 569 # TODO(otabek@) block check till verify on the lab 570 # if not self._is_usb_hub_connected(host): 571 # raise hosts.AutoservVerifyError( 572 # 'Servo_v4 is likely not connected to the DUT.') 573 elif self._is_servo_v3(host): 574 if not self._is_ribbon_cable_connected(host): 575 raise hosts.AutoservVerifyError( 576 'Servo_v3 is likely not connected to the DUT.') 577 else: 578 logging.warn('Unsupported servo type!') 579 580 def _is_applicable(self, host): 581 if host.is_ec_supported(): 582 return True 583 logging.info('DUT is not support EC.') 584 return False 585 586 @property 587 def description(self): 588 return 'Ensure the Servo connected to the DUT.' 589 590 591class _ServoHubConnectionVerifier(_BaseDUTConnectionVerifier): 592 """Verifier to check connection ServoHub to DUT. 593 594 Servo_v4 type-a connected to the DUT by: 595 1) USB hub - checked by ppdut5_mv. 596 """ 597 598 @ignore_exception_for_non_cros_host 599 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 600 def verify(self, host): 601 if self._is_servo_v4_type_a(host): 602 if (self._is_dut_power_on(host) 603 and not self._is_usb_hub_connected(host)): 604 raise hosts.AutoservVerifyError( 605 'Servo USB hub is likely not connected to the DUT.') 606 607 def _is_applicable(self, host): 608 if host.is_ec_supported(): 609 return True 610 logging.info('DUT is not support EC.') 611 return False 612 613 @property 614 def description(self): 615 return 'Ensure the Servo HUB connected to the DUT.' 616 617 618class _TopologyVerifier(hosts.Verifier): 619 """Verifier that all servo component is presented.""" 620 621 @ignore_exception_for_non_cros_host 622 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 623 def verify(self, host): 624 topology = servo_topology.ServoTopology(host) 625 topology.read(host.get_dut_host_info()) 626 try: 627 # Linux takes 1 second to detect and enumerate USB device since 628 # 2010 year. We take 10 seconds to be sure as old standard was 629 # 5 seconds. 630 time.sleep(10) 631 topology.validate(raise_error=True, 632 dual_set=host.is_dual_setup(), 633 compare=True) 634 except servo_topology.ServoTopologyError as e: 635 six.reraise(hosts.AutoservVerifyError, str(e), sys.exc_info()[2]) 636 637 def _is_applicable(self, host): 638 if host.is_localhost(): 639 logging.info('Target servo is not in a lab,' 640 ' action is not applicable.') 641 return False 642 if not host.is_servo_topology_supported(): 643 logging.info('Target servo-topology is not supported,' 644 ' action is not applicable.') 645 return False 646 return True 647 648 @property 649 def description(self): 650 return 'Ensure all Servo component present.' 651 652 653class _PowerButtonVerifier(hosts.Verifier): 654 """ 655 Verifier to check sanity of the `pwr_button` signal. 656 657 Tests that the `pwr_button` signal shows the power button has been 658 released. When `pwr_button` is stuck at `press`, it commonly 659 indicates that the ribbon cable is disconnected. 660 """ 661 # TODO (crbug.com/646593) - Remove list below once servo has been updated 662 # with a dummy pwr_button signal. 663 _BOARDS_WO_PWR_BUTTON = ['arkham', 'gale', 'mistral', 'storm', 'whirlwind'] 664 665 @ignore_exception_for_non_cros_host 666 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 667 def verify(self, host): 668 if host.servo_board in self._BOARDS_WO_PWR_BUTTON: 669 return 670 try: 671 button = host.get_servo().get('pwr_button') 672 except Exception as e: 673 six.reraise(hosts.AutoservNonCriticalVerifyError, str(e), 674 sys.exc_info()[2]) 675 676 if button != 'release': 677 raise hosts.AutoservNonCriticalVerifyError( 678 'Check ribbon cable: \'pwr_button\' is stuck') 679 680 def _is_applicable(self, host): 681 return (host.get_servo() and host.get_servo().main_device_is_flex()) 682 683 @property 684 def description(self): 685 return 'pwr_button control is normal' 686 687 688class _BatteryVerifier(hosts.Verifier): 689 """Collect battery info for analysis.""" 690 691 @ignore_exception_for_non_cros_host 692 @timeout_util.TimeoutDecorator(cros_constants.VERIFY_TIMEOUT_SEC) 693 def verify(self, host): 694 try: 695 servo = host.get_servo() 696 charging = False 697 if servo.has_control('battery_is_charging'): 698 charging = servo.get('battery_is_charging') 699 level = -1 700 if servo.has_control('battery_charge_percent'): 701 level = servo.get('battery_charge_percent') 702 design_mah = servo.get('battery_full_design_mah') 703 charge_mah = servo.get('battery_full_charge_mah') 704 logging.info('Charging: %s', charging) 705 logging.info('Percentage: %s', level) 706 logging.info('Full charge max: %s', charge_mah) 707 logging.info('Full design max: %s', design_mah) 708 # based on analysis of ratio we can find out what is 709 # the level when we can say that battery is dead 710 ratio = int(math.floor(charge_mah / design_mah * 100.0)) 711 logging.info('Ratio: %s', ratio) 712 data = { 713 'board': host.servo_board or 'unknown', 714 'model': host.servo_model or 'unknown', 715 'ratio': ratio 716 } 717 metrics.Counter('chromeos/autotest/battery/ratio').increment( 718 fields=data) 719 except Exception as e: 720 # Keeping it with info level because we do not expect it. 721 logging.info('(Not critical) %s', e) 722 723 def _is_applicable(self, host): 724 if not host.is_ec_supported(): 725 logging.info('The board not support EC') 726 return False 727 dut_info = host.get_dut_host_info() 728 if dut_info: 729 host_info = host.get_dut_host_info() 730 if host_info.get_label_value('power') != 'battery': 731 logging.info('The board does not have battery') 732 return False 733 servo = host.get_servo() 734 if (not servo.has_control('battery_full_design_mah') 735 or not servo.has_control('battery_full_charge_mah')): 736 logging.info('The board is not supported battery controls...') 737 return False 738 return True 739 740 @property 741 def description(self): 742 return 'Logs battery levels' 743 744 745class _LidVerifier(hosts.Verifier): 746 """ 747 Verifier to check sanity of the `lid_open` signal. 748 """ 749 750 @ignore_exception_for_non_cros_host 751 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 752 def verify(self, host): 753 try: 754 lid_open = host.get_servo().get('lid_open') 755 except Exception as e: 756 six.reraise(hosts.AutoservNonCriticalVerifyError, str(e), 757 sys.exc_info()[2]) 758 759 if lid_open != 'yes' and lid_open != 'not_applicable': 760 raise hosts.AutoservNonCriticalVerifyError( 761 'Check lid switch: lid_open is %s' % lid_open) 762 763 @property 764 def description(self): 765 return 'lid_open control is normal' 766 767 768class _EcBoardVerifier(hosts.Verifier): 769 """ 770 Verifier response from the 'ec_board' control. 771 """ 772 773 @ignore_exception_for_non_cros_host 774 @timeout_util.TimeoutDecorator(cros_constants.SHORT_VERIFY_TIMEOUT_SEC) 775 def verify(self, host): 776 if host.is_ec_supported(): 777 ec_board_name = '' 778 try: 779 ec_board_name = host.get_servo().get_ec_board() 780 logging.debug('EC board: %s', ec_board_name) 781 except Exception as e: 782 raise hosts.AutoservNonCriticalVerifyError( 783 '`ec_board` control is not responding; ' 784 'may be caused of broken EC firmware') 785 else: 786 logging.info('The board not support EC') 787 788 @property 789 def description(self): 790 return 'Check EC by get `ec_board` control' 791 792 793class _RestartServod(hosts.RepairAction): 794 """Restart `servod` with the proper BOARD setting.""" 795 796 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 797 def repair(self, host): 798 if not host.is_cros_host(): 799 raise hosts.AutoservRepairError( 800 'Can\'t restart servod: not running ' 801 'embedded Chrome OS.', 802 'servo_not_applicable_to_non_cros_host') 803 host.restart_servod() 804 805 @property 806 def description(self): 807 return 'Start servod with the proper config settings.' 808 809 810class _ServoRebootRepair(repair_utils.RebootRepair): 811 """Try repair servo by reboot servohost. 812 813 This is the same as the standard `RebootRepair`, for servo_v3 it will 814 reboot the beaglebone board immidiately while for labstation it will 815 request a reboot by touch a flag file on its labstation, then 816 labstation reboot will be handled by labstation AdminRepair task as 817 labstation host multiple servos and need do an synchronized reboot. 818 """ 819 820 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 821 def repair(self, host): 822 super(_ServoRebootRepair, self).repair(host) 823 # restart servod for v3 after reboot. 824 host.restart_servod() 825 826 def _is_applicable(self, host): 827 if host.is_localhost() or not host.is_cros_host(): 828 logging.info('Target servo is not in a lab, the reboot repair' 829 ' action is not applicable.') 830 return False 831 832 if host.is_labstation(): 833 host.request_reboot() 834 logging.info('Reboot labstation requested, it will be handled' 835 ' by labstation AdminRepair task.') 836 return False 837 return True 838 839 @property 840 def description(self): 841 return 'Reboot the servo host.' 842 843 844class _ToggleCCLineRepair(hosts.RepairAction): 845 """Try repair servod by toggle cc. 846 847 When cr50 is not enumerated we can try to recover it by toggle cc line. 848 Repair action running from servohost. 849 We using usb_console temporally witch required stop servod. 850 851 TODO(otabek@) review the logic when b/159755652 implemented 852 """ 853 854 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 855 def repair(self, host): 856 host.stop_servod() 857 self._reset_usbc_pigtail_connection(host) 858 host.restart_servod() 859 860 def _is_applicable(self, host): 861 if host.is_localhost() or not host.is_labstation(): 862 return False 863 if not host.servo_serial: 864 return False 865 return self._is_type_c(host) 866 867 def _is_type_c(self, host): 868 if host.get_dut_host_info(): 869 servo_type = host.get_dut_host_info().get_label_value( 870 servo_constants.SERVO_TYPE_LABEL_PREFIX) 871 return 'ccd_cr50' in servo_type 872 return False 873 874 def _reset_usbc_pigtail_connection(self, host): 875 """Reset USBC pigtail connection on servo board. 876 877 To reset need to run 'cc off' and then 'cc srcdts' in usb_console. 878 """ 879 logging.debug('Starting reset USBC pigtail connection.') 880 881 def _run_command(cc_command): 882 """Run configuration channel commands. 883 884 @returns: True if pas successful and False if fail. 885 """ 886 try: 887 cmd = (r"echo 'cc %s' | usb_console -d 18d1:501b -s %s" % 888 (cc_command, host.servo_serial)) 889 resp = host.run(cmd, timeout=host.DEFAULT_TERMINAL_TIMEOUT) 890 return True 891 except Exception as e: 892 logging.info('(Non-critical) %s.', e) 893 return False 894 895 logging.info('Turn off configuration channel. And wait 5 seconds.') 896 if _run_command('off'): 897 # wait till command will be effected 898 time.sleep(5) 899 logging.info('Turn on configuration channel. ' 900 'And wait 15 seconds.') 901 if _run_command('srcdts'): 902 # wait till command will be effected 903 time.sleep(15) 904 905 @property 906 def description(self): 907 return 'Toggle cc lines' 908 909 910class _ECRebootRepair(hosts.RepairAction): 911 """ 912 Reboot EC on DUT from servo. 913 """ 914 915 def _is_applicable(self, host): 916 return (not host.is_localhost()) and host.is_ec_supported() 917 918 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 919 def repair(self, host): 920 host.get_servo().ec_reboot() 921 922 @property 923 def description(self): 924 return 'Reboot EC' 925 926 927class _DutRebootRepair(hosts.RepairAction): 928 """ 929 Reboot DUT to recover some servo controls depending on EC console. 930 931 Some servo controls, like lid_open, requires communicating with DUT through 932 EC UART console. Failure of this kinds of controls can be recovered by 933 rebooting the DUT. 934 """ 935 936 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 937 def repair(self, host): 938 host.get_servo().get_power_state_controller().reset() 939 # Get the lid_open value which requires EC console. 940 lid_open = host.get_servo().get('lid_open') 941 if lid_open != 'yes' and lid_open != 'not_applicable': 942 raise hosts.AutoservVerifyError( 943 'Still fail to contact EC console after rebooting DUT') 944 945 @property 946 def description(self): 947 return 'Reset the DUT via servo' 948 949 950class _DiskCleanupRepair(hosts.RepairAction): 951 """ 952 Remove old logs/metrics/crash_dumps on servohost to free up disk space. 953 """ 954 KEEP_LOGS_MAX_DAYS = 5 955 956 FILE_TO_REMOVE = [ 957 '/var/lib/metrics/uma-events', '/var/spool/crash/*', 958 '/var/log/chrome/*', '/var/log/ui/*', 959 '/home/chronos/BrowserMetrics/*' 960 ] 961 962 @timeout_util.TimeoutDecorator(cros_constants.SHORT_REPAIR_TIMEOUT_SEC) 963 def repair(self, host): 964 if host.is_localhost(): 965 # we don't want to remove anything from local testing. 966 return 967 968 # Remove old servod logs. 969 host.run('/usr/bin/find /var/log/servod_* -mtime +%d -print -delete' 970 % self.KEEP_LOGS_MAX_DAYS, ignore_status=True) 971 972 # Remove pre-defined metrics and crash dumps. 973 for path in self.FILE_TO_REMOVE: 974 host.run('rm %s' % path, ignore_status=True) 975 976 @property 977 def description(self): 978 return 'Clean up old logs/metrics on servohost to free up disk space.' 979 980 981class _ServoMicroFlashRepair(hosts.RepairAction): 982 """ 983 Remove old logs/metrics/crash_dumps on servohost to free up disk space. 984 """ 985 _TARGET_SERVO = 'servo_micro' 986 987 @timeout_util.TimeoutDecorator(cros_constants.REPAIR_TIMEOUT_SEC) 988 def repair(self, host): 989 if not host.is_cros_host(): 990 raise hosts.AutoservRepairError( 991 'Can\'t restart servod: not running ' 992 'embedded Chrome OS.', 993 'servo_not_applicable_to_non_cros_host') 994 servo = host.get_servo() 995 if not servo or self._TARGET_SERVO not in servo.get_servo_type(): 996 logging.info("Servo-micro is not present on set-up") 997 return 998 999 try: 1000 servo_updater.update_servo_firmware(host, 1001 boards=(self._TARGET_SERVO, ), 1002 force_update=True, 1003 ignore_version=True) 1004 except Exception as e: 1005 logging.debug("(Not critical) Servo device update error: %s", e) 1006 raise hosts.AutoservVerifyError( 1007 'Still fail to contact EC console after rebooting DUT') 1008 # Update time when we reflashed the fw on the device 1009 dhp = host.get_dut_health_profile() 1010 dhp.refresh_servo_miro_fw_update_run_time() 1011 host.restart_servod() 1012 1013 def is_time_to_try(self, dhp): 1014 """Verify that it is time when we can try to re-flash fw on servo_micro. 1015 1016 Re-flashing limited to once per 2 weeks to avoid over-flashing 1017 the servo device. 1018 """ 1019 today_time = int(time.time()) 1020 last_check = dhp.get_servo_micro_fw_update_time_epoch() 1021 can_run = today_time > (last_check + (14 * 24 * 60 * 60)) 1022 if not can_run: 1023 logging.info("The servo_micro fw updated in las 2 weeks ago.") 1024 return can_run 1025 1026 def _is_applicable(self, host): 1027 return (not host.is_localhost() and host.get_dut_health_profile() 1028 and self.is_time_to_try(host.get_dut_health_profile())) 1029 1030 @property 1031 def description(self): 1032 return 'Re-flash servo_micro firmware.' 1033 1034 1035def create_servo_repair_strategy(): 1036 """ 1037 Return a `RepairStrategy` for a `ServoHost`. 1038 """ 1039 config = ['brd_config', 'ser_config'] 1040 verify_dag = [ 1041 (repair_utils.SshVerifier, 'servo_ssh', []), 1042 (_DiskSpaceVerifier, 'disk_space', ['servo_ssh']), 1043 (_UpdateVerifier, 'update', ['servo_ssh']), 1044 (_BoardConfigVerifier, 'brd_config', ['servo_ssh']), 1045 (_SerialConfigVerifier, 'ser_config', ['servo_ssh']), 1046 (_ServodJobVerifier, 'servod_job', config + ['disk_space']), 1047 (_TopologyVerifier, 'servo_topology', ['servod_job']), 1048 (_ServodConnectionVerifier, 'servod_connection', ['servod_job']), 1049 (_ServodControlVerifier, 'servod_control', ['servod_connection']), 1050 (_DUTConnectionVerifier, 'dut_connected', ['servod_connection']), 1051 (_ServoHubConnectionVerifier, 'hub_connected', ['dut_connected']), 1052 (_PowerButtonVerifier, 'pwr_button', ['hub_connected']), 1053 (_BatteryVerifier, 'battery', ['hub_connected']), 1054 (_LidVerifier, 'lid_open', ['hub_connected']), 1055 (_EcBoardVerifier, 'ec_board', ['dut_connected']), 1056 (_Cr50ConsoleVerifier, 'cr50_console', ['dut_connected']), 1057 (_CCDTestlabVerifier, 'ccd_testlab', ['cr50_console']), 1058 (_CCDPowerDeliveryVerifier, 'power_delivery', ['dut_connected']), 1059 ] 1060 1061 servod_deps = [ 1062 'servod_job', 'servo_topology', 'servod_connection', 1063 'servod_control', 'dut_connected', 'hub_connected', 'pwr_button', 1064 'cr50_console' 1065 ] 1066 repair_actions = [ 1067 (_DiskCleanupRepair, 'disk_cleanup', ['servo_ssh'], ['disk_space' 1068 ]), 1069 (_ServoMicroFlashRepair, 'servo_micro_flash', 1070 ['servo_ssh', 'servo_topology'], ['dut_connected']), 1071 (_RestartServod, 'restart', ['servo_ssh'], config + servod_deps), 1072 (_ServoRebootRepair, 'servo_reboot', ['servo_ssh'], servod_deps), 1073 (_ToggleCCLineRepair, 'servo_cc', ['servo_ssh'], servod_deps), 1074 (_DutRebootRepair, 'dut_reboot', ['servod_connection'], 1075 ['servod_control', 'lid_open', 'ec_board']), 1076 (_ECRebootRepair, 'ec_reboot', ['servod_connection'], 1077 ['servod_control', 'lid_open', 'ec_board']), 1078 ] 1079 return hosts.RepairStrategy(verify_dag, repair_actions, 'servo') 1080