1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import glob 6import logging 7import os 8import re 9import urlparse 10import urllib2 11 12from autotest_lib.client.bin import utils 13from autotest_lib.client.common_lib import error, global_config 14from autotest_lib.client.common_lib.cros import dev_server 15from autotest_lib.server import utils as server_utils 16from chromite.lib import retry_util 17 18try: 19 from chromite.lib import metrics 20except ImportError: 21 metrics = utils.metrics_mock 22 23try: 24 import devserver 25 STATEFUL_UPDATE_PATH = devserver.__path__[0] 26except ImportError: 27 STATEFUL_UPDATE_PATH = '/usr/bin' 28 29# Local stateful update path is relative to the CrOS source directory. 30STATEFUL_UPDATE_SCRIPT = 'stateful_update' 31UPDATER_IDLE = 'UPDATE_STATUS_IDLE' 32UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT' 33# A list of update engine client states that occur after an update is triggered. 34UPDATER_PROCESSING_UPDATE = ['UPDATE_STATUS_CHECKING_FORUPDATE', 35 'UPDATE_STATUS_UPDATE_AVAILABLE', 36 'UPDATE_STATUS_DOWNLOADING', 37 'UPDATE_STATUS_FINALIZING'] 38 39class ChromiumOSError(error.InstallError): 40 """Generic error for ChromiumOS-specific exceptions.""" 41 42 43class RootFSUpdateError(ChromiumOSError): 44 """Raised when the RootFS fails to update.""" 45 46 47class StatefulUpdateError(ChromiumOSError): 48 """Raised when the stateful partition fails to update.""" 49 50 51def url_to_version(update_url): 52 """Return the version based on update_url. 53 54 @param update_url: url to the image to update to. 55 56 """ 57 # The Chrome OS version is generally the last element in the URL. The only 58 # exception is delta update URLs, which are rooted under the version; e.g., 59 # http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to 60 # strip off the au section of the path before reading the version. 61 return re.sub('/au/.*', '', 62 urlparse.urlparse(update_url).path).split('/')[-1].strip() 63 64 65def url_to_image_name(update_url): 66 """Return the image name based on update_url. 67 68 From a URL like: 69 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0 70 return lumpy-release/R27-3837.0.0 71 72 @param update_url: url to the image to update to. 73 @returns a string representing the image name in the update_url. 74 75 """ 76 return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:]) 77 78 79def _get_devserver_build_from_update_url(update_url): 80 """Get the devserver and build from the update url. 81 82 @param update_url: The url for update. 83 Eg: http://devserver:port/update/build. 84 85 @return: A tuple of (devserver url, build) or None if the update_url 86 doesn't match the expected pattern. 87 88 @raises ValueError: If the update_url doesn't match the expected pattern. 89 @raises ValueError: If no global_config was found, or it doesn't contain an 90 image_url_pattern. 91 """ 92 pattern = global_config.global_config.get_config_value( 93 'CROS', 'image_url_pattern', type=str, default='') 94 if not pattern: 95 raise ValueError('Cannot parse update_url, the global config needs ' 96 'an image_url_pattern.') 97 re_pattern = pattern.replace('%s', '(\S+)') 98 parts = re.search(re_pattern, update_url) 99 if not parts or len(parts.groups()) < 2: 100 raise ValueError('%s is not an update url' % update_url) 101 return parts.groups() 102 103 104def list_image_dir_contents(update_url): 105 """Lists the contents of the devserver for a given build/update_url. 106 107 @param update_url: An update url. Eg: http://devserver:port/update/build. 108 """ 109 if not update_url: 110 logging.warning('Need update_url to list contents of the devserver.') 111 return 112 error_msg = 'Cannot check contents of devserver, update url %s' % update_url 113 try: 114 devserver_url, build = _get_devserver_build_from_update_url(update_url) 115 except ValueError as e: 116 logging.warning('%s: %s', error_msg, e) 117 return 118 devserver = dev_server.ImageServer(devserver_url) 119 try: 120 devserver.list_image_dir(build) 121 # The devserver will retry on URLError to avoid flaky connections, but will 122 # eventually raise the URLError if it persists. All HTTPErrors get 123 # converted to DevServerExceptions. 124 except (dev_server.DevServerException, urllib2.URLError) as e: 125 logging.warning('%s: %s', error_msg, e) 126 127 128# TODO(garnold) This implements shared updater functionality needed for 129# supporting the autoupdate_EndToEnd server-side test. We should probably 130# migrate more of the existing ChromiumOSUpdater functionality to it as we 131# expand non-CrOS support in other tests. 132class BaseUpdater(object): 133 """Platform-agnostic DUT update functionality.""" 134 135 def __init__(self, updater_ctrl_bin, update_url, host, interactive=True): 136 """Initializes the object. 137 138 @param updater_ctrl_bin: Path to update_engine_client. 139 @param update_url: The URL we want the update to use. 140 @param host: A client.common_lib.hosts.Host implementation. 141 @param interactive: Bool whether we are doing an interactive update. 142 """ 143 self.updater_ctrl_bin = updater_ctrl_bin 144 self.update_url = update_url 145 self.host = host 146 self.interactive = interactive 147 148 149 def check_update_status(self): 150 """Returns the current update engine state. 151 152 We use the `update_engine_client -status' command and parse the line 153 indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE". 154 """ 155 update_status = self.host.run(command='%s -status | grep CURRENT_OP' % 156 self.updater_ctrl_bin) 157 return update_status.stdout.strip().split('=')[-1] 158 159 160 def get_last_update_error(self): 161 """Get the last autoupdate error code.""" 162 error_msg = self.host.run( 163 '%s --last_attempt_error' % self.updater_ctrl_bin) 164 error_msg = (error_msg.stdout.strip()).replace('\n', ', ') 165 return error_msg 166 167 168 def _base_update_handler_no_retry(self, run_args): 169 """Base function to handle a remote update ssh call. 170 171 @param run_args: Dictionary of args passed to ssh_host.run function. 172 173 @throws: intercepts and re-throws all exceptions 174 """ 175 try: 176 self.host.run(**run_args) 177 except Exception as e: 178 logging.debug('exception in update handler: %s', e) 179 raise e 180 181 182 def _base_update_handler(self, run_args, err_msg_prefix=None): 183 """Handle a remote update ssh call, possibly with retries. 184 185 @param run_args: Dictionary of args passed to ssh_host.run function. 186 @param err_msg_prefix: Prefix of the exception error message. 187 """ 188 def exception_handler(e): 189 """Examines exceptions and returns True if the update handler 190 should be retried. 191 192 @param e: the exception intercepted by the retry util. 193 """ 194 return (isinstance(e, error.AutoservSSHTimeout) or 195 (isinstance(e, error.GenericHostRunError) and 196 hasattr(e, 'description') and 197 (re.search('ERROR_CODE=37', e.description) or 198 re.search('generic error .255.', e.description)))) 199 200 try: 201 # Try the update twice (arg 2 is max_retry, not including the first 202 # call). Some exceptions may be caught by the retry handler. 203 retry_util.GenericRetry(exception_handler, 1, 204 self._base_update_handler_no_retry, 205 run_args) 206 except Exception as e: 207 message = err_msg_prefix + ': ' + str(e) 208 raise RootFSUpdateError(message) 209 210 211 def _wait_for_update_service(self): 212 """Ensure that the update engine daemon is running, possibly 213 by waiting for it a bit in case the DUT just rebooted and the 214 service hasn't started yet. 215 """ 216 def handler(e): 217 """Retry exception handler. 218 219 Assumes that the error is due to the update service not having 220 started yet. 221 222 @param e: the exception intercepted by the retry util. 223 """ 224 if isinstance(e, error.AutoservRunError): 225 logging.debug('update service check exception: %s\n' 226 'retrying...', e) 227 return True 228 else: 229 return False 230 231 # Retry at most three times, every 5s. 232 status = retry_util.GenericRetry(handler, 3, 233 self.check_update_status, 234 sleep=5) 235 236 # Expect the update engine to be idle. 237 if status != UPDATER_IDLE: 238 raise ChromiumOSError('%s is not in an installable state' % 239 self.host.hostname) 240 241 242 def trigger_update(self): 243 """Triggers a background update. 244 245 @raise RootFSUpdateError or unknown Exception if anything went wrong. 246 """ 247 # If this function is called immediately after reboot (which it is at 248 # this time), there is no guarantee that the update service is up and 249 # running yet, so wait for it. 250 self._wait_for_update_service() 251 252 autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' % 253 (self.updater_ctrl_bin, self.update_url)) 254 run_args = {'command': autoupdate_cmd} 255 err_prefix = 'Failed to trigger an update on %s. ' % self.host.hostname 256 logging.info('Triggering update via: %s', autoupdate_cmd) 257 metric_fields = {'success': False} 258 try: 259 self._base_update_handler(run_args, err_prefix) 260 metric_fields['success'] = True 261 finally: 262 c = metrics.Counter('chromeos/autotest/autoupdater/trigger') 263 metric_fields.update(self._get_metric_fields()) 264 c.increment(fields=metric_fields) 265 266 267 def _get_metric_fields(self): 268 """Return a dict of metric fields. 269 270 This is used for sending autoupdate metrics for this instance. 271 """ 272 build_name = url_to_image_name(self.update_url) 273 try: 274 board, build_type, milestone, _ = server_utils.ParseBuildName( 275 build_name) 276 except server_utils.ParseBuildNameException: 277 logging.warning('Unable to parse build name %s for metrics. ' 278 'Continuing anyway.', build_name) 279 board, build_type, milestone = ('', '', '') 280 return { 281 'dev_server': dev_server.get_hostname(self.update_url), 282 'board': board, 283 'build_type': build_type, 284 'milestone': milestone, 285 } 286 287 288 def _verify_update_completed(self): 289 """Verifies that an update has completed. 290 291 @raise RootFSUpdateError: if verification fails. 292 """ 293 status = self.check_update_status() 294 if status != UPDATER_NEED_REBOOT: 295 error_msg = '' 296 if status == UPDATER_IDLE: 297 error_msg = 'Update error: %s' % self.get_last_update_error() 298 raise RootFSUpdateError('Update did not complete with correct ' 299 'status. Expecting %s, actual %s. %s' % 300 (UPDATER_NEED_REBOOT, status, error_msg)) 301 302 303 def update_image(self): 304 """Updates the device image and verifies success.""" 305 autoupdate_cmd = ('%s --update --omaha_url=%s' % 306 (self.updater_ctrl_bin, self.update_url)) 307 if not self.interactive: 308 autoupdate_cmd = '%s --interactive=false' % autoupdate_cmd 309 run_args = {'command': autoupdate_cmd, 'timeout': 3600} 310 err_prefix = ('Failed to install device image using payload at %s ' 311 'on %s. ' % (self.update_url, self.host.hostname)) 312 logging.info('Updating image via: %s', autoupdate_cmd) 313 metric_fields = {'success': False} 314 try: 315 self._base_update_handler(run_args, err_prefix) 316 metric_fields['success'] = True 317 finally: 318 c = metrics.Counter('chromeos/autotest/autoupdater/update') 319 metric_fields.update(self._get_metric_fields()) 320 c.increment(fields=metric_fields) 321 322 self._verify_update_completed() 323 324 325class ChromiumOSUpdater(BaseUpdater): 326 """Helper class used to update DUT with image of desired version.""" 327 REMOTE_STATEFUL_UPDATE_PATH = os.path.join( 328 '/usr/local/bin', STATEFUL_UPDATE_SCRIPT) 329 REMOTE_TMP_STATEFUL_UPDATE = os.path.join( 330 '/tmp', STATEFUL_UPDATE_SCRIPT) 331 UPDATER_BIN = '/usr/bin/update_engine_client' 332 UPDATED_MARKER = '/run/update_engine_autoupdate_completed' 333 UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine'] 334 335 KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3} 336 KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5} 337 # Time to wait for new kernel to be marked successful after 338 # auto update. 339 KERNEL_UPDATE_TIMEOUT = 120 340 341 def __init__(self, update_url, host=None, local_devserver=False, 342 interactive=True): 343 super(ChromiumOSUpdater, self).__init__(self.UPDATER_BIN, update_url, 344 host, interactive=interactive) 345 self.local_devserver = local_devserver 346 if not local_devserver: 347 self.update_version = url_to_version(update_url) 348 else: 349 self.update_version = None 350 351 352 def reset_update_engine(self): 353 """Resets the host to prepare for a clean update regardless of state.""" 354 self._run('rm -f %s' % self.UPDATED_MARKER) 355 self._run('stop ui || true') 356 self._run('stop update-engine || true') 357 self._run('start update-engine') 358 359 # Wait for update engine to be ready. 360 self._wait_for_update_service() 361 362 363 def _run(self, cmd, *args, **kwargs): 364 """Abbreviated form of self.host.run(...)""" 365 return self.host.run(cmd, *args, **kwargs) 366 367 368 def rootdev(self, options=''): 369 """Returns the stripped output of rootdev <options>. 370 371 @param options: options to run rootdev. 372 373 """ 374 return self._run('rootdev %s' % options).stdout.strip() 375 376 377 def get_kernel_state(self): 378 """Returns the (<active>, <inactive>) kernel state as a pair.""" 379 active_root = int(re.findall('\d+\Z', self.rootdev('-s'))[0]) 380 if active_root == self.KERNEL_A['root']: 381 return self.KERNEL_A, self.KERNEL_B 382 elif active_root == self.KERNEL_B['root']: 383 return self.KERNEL_B, self.KERNEL_A 384 else: 385 raise ChromiumOSError('Encountered unknown root partition: %s' % 386 active_root) 387 388 389 def _cgpt(self, flag, kernel, dev='$(rootdev -s -d)'): 390 """Return numeric cgpt value for the specified flag, kernel, device. """ 391 return int(self._run('cgpt show -n -i %d %s %s' % ( 392 kernel['kernel'], flag, dev)).stdout.strip()) 393 394 395 def get_kernel_priority(self, kernel): 396 """Return numeric priority for the specified kernel. 397 398 @param kernel: information of the given kernel, KERNEL_A or KERNEL_B. 399 400 """ 401 return self._cgpt('-P', kernel) 402 403 404 def get_kernel_success(self, kernel): 405 """Return boolean success flag for the specified kernel. 406 407 @param kernel: information of the given kernel, KERNEL_A or KERNEL_B. 408 409 """ 410 return self._cgpt('-S', kernel) != 0 411 412 413 def get_kernel_tries(self, kernel): 414 """Return tries count for the specified kernel. 415 416 @param kernel: information of the given kernel, KERNEL_A or KERNEL_B. 417 418 """ 419 return self._cgpt('-T', kernel) 420 421 422 def get_stateful_update_script(self): 423 """Returns the path to the stateful update script on the target. 424 425 When runnning test_that, stateful_update is in chroot /usr/sbin, 426 as installed by chromeos-base/devserver packages. 427 In the lab, it is installed with the python module devserver, by 428 build_externals.py command. 429 430 If we can find it, we hope it exists already on the DUT, we assert 431 otherwise. 432 """ 433 stateful_update_file = os.path.join(STATEFUL_UPDATE_PATH, 434 STATEFUL_UPDATE_SCRIPT) 435 if os.path.exists(stateful_update_file): 436 self.host.send_file( 437 stateful_update_file, self.REMOTE_TMP_STATEFUL_UPDATE, 438 delete_dest=True) 439 return self.REMOTE_TMP_STATEFUL_UPDATE 440 441 if self.host.path_exists(self.REMOTE_STATEFUL_UPDATE_PATH): 442 logging.warning('Could not chroot %s script, falling back on %s', 443 STATEFUL_UPDATE_SCRIPT, self.REMOTE_STATEFUL_UPDATE_PATH) 444 return self.REMOTE_STATEFUL_UPDATE_PATH 445 else: 446 raise ChromiumOSError('Could not locate %s', 447 STATEFUL_UPDATE_SCRIPT) 448 449 450 def reset_stateful_partition(self): 451 """Clear any pending stateful update request.""" 452 statefuldev_cmd = [self.get_stateful_update_script()] 453 statefuldev_cmd += ['--stateful_change=reset', '2>&1'] 454 self._run(' '.join(statefuldev_cmd)) 455 456 457 def revert_boot_partition(self): 458 """Revert the boot partition.""" 459 part = self.rootdev('-s') 460 logging.warning('Reverting update; Boot partition will be %s', part) 461 return self._run('/postinst %s 2>&1' % part) 462 463 464 def rollback_rootfs(self, powerwash): 465 """Triggers rollback and waits for it to complete. 466 467 @param powerwash: If true, powerwash as part of rollback. 468 469 @raise RootFSUpdateError if anything went wrong. 470 471 """ 472 version = self.host.get_release_version() 473 # Introduced can_rollback in M36 (build 5772). # etc/lsb-release matches 474 # X.Y.Z. This version split just pulls the first part out. 475 try: 476 build_number = int(version.split('.')[0]) 477 except ValueError: 478 logging.error('Could not parse build number.') 479 build_number = 0 480 481 if build_number >= 5772: 482 can_rollback_cmd = '%s --can_rollback' % self.UPDATER_BIN 483 logging.info('Checking for rollback.') 484 try: 485 self._run(can_rollback_cmd) 486 except error.AutoservRunError as e: 487 raise RootFSUpdateError("Rollback isn't possible on %s: %s" % 488 (self.host.hostname, str(e))) 489 490 rollback_cmd = '%s --rollback --follow' % self.UPDATER_BIN 491 if not powerwash: 492 rollback_cmd += ' --nopowerwash' 493 494 logging.info('Performing rollback.') 495 try: 496 self._run(rollback_cmd) 497 except error.AutoservRunError as e: 498 raise RootFSUpdateError('Rollback failed on %s: %s' % 499 (self.host.hostname, str(e))) 500 501 self._verify_update_completed() 502 503 504 # TODO(garnold) This is here for backward compatibility and should be 505 # deprecated once we shift to using update_image() everywhere. 506 def update_rootfs(self): 507 """Run the standard command to force an update.""" 508 return self.update_image() 509 510 511 def update_stateful(self, clobber=True): 512 """Updates the stateful partition. 513 514 @param clobber: If True, a clean stateful installation. 515 """ 516 logging.info('Updating stateful partition...') 517 statefuldev_url = self.update_url.replace('update', 518 'static') 519 520 # Attempt stateful partition update; this must succeed so that the newly 521 # installed host is testable after update. 522 statefuldev_cmd = [self.get_stateful_update_script(), statefuldev_url] 523 if clobber: 524 statefuldev_cmd.append('--stateful_change=clean') 525 526 statefuldev_cmd.append('2>&1') 527 try: 528 self._run(' '.join(statefuldev_cmd), timeout=1200) 529 except error.AutoservRunError: 530 update_error = StatefulUpdateError( 531 'Failed to perform stateful update on %s' % 532 self.host.hostname) 533 raise update_error 534 535 def run_update(self, update_root=True): 536 """Update the DUT with image of specific version. 537 538 @param update_root: True to force a rootfs update. 539 """ 540 booted_version = self.host.get_release_version() 541 if self.update_version: 542 logging.info('Updating from version %s to %s.', 543 booted_version, self.update_version) 544 545 # Check that Dev Server is accepting connections (from autoserv's host). 546 # If we can't talk to it, the machine host probably can't either. 547 auserver_host = 'http://%s' % urlparse.urlparse(self.update_url)[1] 548 try: 549 if not dev_server.ImageServer.devserver_healthy(auserver_host): 550 raise ChromiumOSError( 551 'Update server at %s not healthy' % auserver_host) 552 except Exception as e: 553 logging.debug('Error happens in connection to devserver: %r', e) 554 raise ChromiumOSError( 555 'Update server at %s not available' % auserver_host) 556 557 logging.info('Installing from %s to %s', self.update_url, 558 self.host.hostname) 559 560 # Reset update state. 561 self.reset_update_engine() 562 self.reset_stateful_partition() 563 564 try: 565 try: 566 if not update_root: 567 logging.info('Root update is skipped.') 568 else: 569 self.update_rootfs() 570 571 self.update_stateful() 572 except: 573 self.revert_boot_partition() 574 self.reset_stateful_partition() 575 raise 576 577 logging.info('Update complete.') 578 except: 579 # Collect update engine logs in the event of failure. 580 if self.host.job: 581 logging.info('Collecting update engine logs due to failure...') 582 self.host.get_file( 583 self.UPDATER_LOGS, self.host.job.sysinfo.sysinfodir, 584 preserve_perm=False) 585 list_image_dir_contents(self.update_url) 586 raise 587 finally: 588 logging.info('Update engine log has downloaded in ' 589 'sysinfo/update_engine dir. Check the lastest.') 590 591 592 def check_version(self): 593 """Check the image running in DUT has the desired version. 594 595 @returns: True if the DUT's image version matches the version that 596 the autoupdater tries to update to. 597 598 """ 599 booted_version = self.host.get_release_version() 600 return (self.update_version and 601 self.update_version.endswith(booted_version)) 602 603 604 def check_version_to_confirm_install(self): 605 """Check image running in DUT has the desired version to be installed. 606 607 The method should not be used to check if DUT needs to have a full 608 reimage. Only use it to confirm a image is installed. 609 610 The method is designed to verify version for following 6 scenarios with 611 samples of version to update to and expected booted version: 612 1. trybot paladin build. 613 update version: trybot-lumpy-paladin/R27-3837.0.0-b123 614 booted version: 3837.0.2013_03_21_1340 615 616 2. trybot release build. 617 update version: trybot-lumpy-release/R27-3837.0.0-b456 618 booted version: 3837.0.0 619 620 3. buildbot official release build. 621 update version: lumpy-release/R27-3837.0.0 622 booted version: 3837.0.0 623 624 4. non-official paladin rc build. 625 update version: lumpy-paladin/R27-3878.0.0-rc7 626 booted version: 3837.0.0-rc7 627 628 5. chrome-perf build. 629 update version: lumpy-chrome-perf/R28-3837.0.0-b2996 630 booted version: 3837.0.0 631 632 6. pgo-generate build. 633 update version: lumpy-release-pgo-generate/R28-3837.0.0-b2996 634 booted version: 3837.0.0-pgo-generate 635 636 When we are checking if a DUT needs to do a full install, we should NOT 637 use this method to check if the DUT is running the same version, since 638 it may return false positive for a DUT running trybot paladin build to 639 be updated to another trybot paladin build. 640 641 TODO: This logic has a bug if a trybot paladin build failed to be 642 installed in a DUT running an older trybot paladin build with same 643 platform number, but different build number (-b###). So to conclusively 644 determine if a tryjob paladin build is imaged successfully, we may need 645 to find out the date string from update url. 646 647 @returns: True if the DUT's image version (without the date string if 648 the image is a trybot build), matches the version that the 649 autoupdater is trying to update to. 650 651 """ 652 # In the local_devserver case, we can't know the expected 653 # build, so just pass. 654 if not self.update_version: 655 return True 656 657 # Always try the default check_version method first, this prevents 658 # any backward compatibility issue. 659 if self.check_version(): 660 return True 661 662 return utils.version_match(self.update_version, 663 self.host.get_release_version(), 664 self.update_url) 665 666 667 def verify_boot_expectations(self, expected_kernel_state, rollback_message): 668 """Verifies that we fully booted given expected kernel state. 669 670 This method both verifies that we booted using the correct kernel 671 state and that the OS has marked the kernel as good. 672 673 @param expected_kernel_state: kernel state that we are verifying with 674 i.e. I expect to be booted onto partition 4 etc. See output of 675 get_kernel_state. 676 @param rollback_message: string to raise as a ChromiumOSError 677 if we booted with the wrong partition. 678 679 @raises ChromiumOSError: If we didn't. 680 """ 681 # Figure out the newly active kernel. 682 active_kernel_state = self.get_kernel_state()[0] 683 684 # Check for rollback due to a bad build. 685 if (expected_kernel_state and 686 active_kernel_state != expected_kernel_state): 687 688 # Kernel crash reports should be wiped between test runs, but 689 # may persist from earlier parts of the test, or from problems 690 # with provisioning. 691 # 692 # Kernel crash reports will NOT be present if the crash happened 693 # before encrypted stateful is mounted. 694 # 695 # TODO(dgarrett): Integrate with server/crashcollect.py at some 696 # point. 697 kernel_crashes = glob.glob('/var/spool/crash/kernel.*.kcrash') 698 if kernel_crashes: 699 rollback_message += ': kernel_crash' 700 logging.debug('Found %d kernel crash reports:', 701 len(kernel_crashes)) 702 # The crash names contain timestamps that may be useful: 703 # kernel.20131207.005945.0.kcrash 704 for crash in kernel_crashes: 705 logging.debug(' %s', os.path.basename(crash)) 706 707 # Print out some information to make it easier to debug 708 # the rollback. 709 logging.debug('Dumping partition table.') 710 self._run('cgpt show $(rootdev -s -d)') 711 logging.debug('Dumping crossystem for firmware debugging.') 712 self._run('crossystem --all') 713 raise ChromiumOSError(rollback_message) 714 715 # Make sure chromeos-setgoodkernel runs. 716 try: 717 utils.poll_for_condition( 718 lambda: (self.get_kernel_tries(active_kernel_state) == 0 719 and self.get_kernel_success(active_kernel_state)), 720 exception=ChromiumOSError(), 721 timeout=self.KERNEL_UPDATE_TIMEOUT, sleep_interval=5) 722 except ChromiumOSError: 723 services_status = self._run('status system-services').stdout 724 if services_status != 'system-services start/running\n': 725 event = ('Chrome failed to reach login screen') 726 else: 727 event = ('update-engine failed to call ' 728 'chromeos-setgoodkernel') 729 raise ChromiumOSError( 730 'After update and reboot, %s ' 731 'within %d seconds' % (event, 732 self.KERNEL_UPDATE_TIMEOUT)) 733 734