1#!/usr/bin/python 2# Copyright 2016 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" 7Automatically update the afe_stable_versions table. 8 9This command updates the stable repair version for selected boards 10in the lab. For each board, if the version that Omaha is serving 11on the Beta channel for the board is more recent than the current 12stable version in the AFE database, then the AFE is updated to use 13the version on Omaha. 14 15The upgrade process is applied to every "managed board" in the test 16lab. Generally, a managed board is a board with both spare and 17critical scheduling pools. 18 19See `autotest_lib.site_utils.lab_inventory` for the full definition 20of "managed board". 21 22The command supports a `--dry-run` option that reports changes that 23would be made, without making the actual RPC calls to change the 24database. 25 26""" 27 28import argparse 29import json 30import subprocess 31import sys 32 33import common 34from autotest_lib.client.common_lib import utils 35from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 36from autotest_lib.site_utils import lab_inventory 37 38 39# _OMAHA_STATUS - URI of a file in GoogleStorage with a JSON object 40# summarizing all versions currently being served by Omaha. 41# 42# The principle data is in an array named 'omaha_data'. Each entry 43# in the array contains information relevant to one image being 44# served by Omaha, including the following information: 45# * The board name of the product, as known to Omaha. 46# * The channel associated with the image. 47# * The Chrome and Chrome OS version strings for the image 48# being served. 49# 50_OMAHA_STATUS = 'gs://chromeos-build-release-console/omaha_status.json' 51 52 53# _BUILD_METADATA_PATTERN - Format string for the URI of a file in 54# GoogleStorage with a JSON object that contains metadata about 55# a given build. The metadata includes the version of firmware 56# bundled with the build. 57# 58_BUILD_METADATA_PATTERN = 'gs://chromeos-image-archive/%s/metadata.json' 59 60 61# _DEFAULT_BOARD - The distinguished board name used to identify a 62# stable version mapping that is used for any board without an explicit 63# mapping of its own. 64# 65# _DEFAULT_VERSION_TAG - A string used to signify that there is no 66# mapping for a board, in other words, the board is mapped to the 67# default version. 68# 69_DEFAULT_BOARD = 'DEFAULT' 70_DEFAULT_VERSION_TAG = '(default)' 71 72 73# _FIRMWARE_UPGRADE_BLACKLIST - a set of boards that are exempt from 74# automatic stable firmware version assignment. This blacklist is 75# here out of an abundance of caution, on the general principle of "if 76# it ain't broke, don't fix it." Specifically, these are old, legacy 77# boards and: 78# * They're working fine with whatever firmware they have in the lab 79# right now. Moreover, because of their age, we can expect that 80# they will never get any new firmware updates in future. 81# * Servo support is spotty or missing, so there's no certainty 82# that DUTs bricked by a firmware update can be repaired. 83# * Because of their age, they are somewhere between hard and 84# impossible to replace. In some cases, they are also already 85# in short supply. 86# 87# N.B. HARDCODED BOARD NAMES ARE EVIL!!! This blacklist uses hardcoded 88# names because it's meant to define a list of legacies that will shrivel 89# and die over time. 90# 91# DO NOT ADD TO THIS LIST. If there's a new use case that requires 92# extending the blacklist concept, you should find a maintainable 93# solution that deletes this code. 94# 95# TODO(jrbarnette): When any board is past EOL, and removed from the 96# lab, it can be removed from the blacklist. When all the boards are 97# past EOL, the blacklist should be removed. 98 99_FIRMWARE_UPGRADE_BLACKLIST = set([ 100 'butterfly', 101 'daisy', 102 'daisy_skate', 103 'daisy_spring', 104 'lumpy', 105 'parrot', 106 'parrot_ivb', 107 'peach_pi', 108 'peach_pit', 109 'stout', 110 'stumpy', 111 'x86-alex', 112 'x86-mario', 113 'x86-zgb', 114 ]) 115 116 117class _VersionUpdater(object): 118 """ 119 Class to report and apply version changes. 120 121 This class is responsible for the low-level logic of applying 122 version upgrades and reporting them as command output. 123 124 This class exists to solve two problems: 125 1. To distinguish "normal" vs. "dry-run" modes. Each mode has a 126 subclass; methods that perform actual AFE updates are 127 implemented for the normal mode subclass only. 128 2. To provide hooks for unit tests. The unit tests override both 129 the reporting and modification behaviors, in order to test the 130 higher level logic that decides what changes are needed. 131 132 Methods meant merely to report changes to command output have names 133 starting with "report" or "_report". Methods that are meant to 134 change the AFE in normal mode have names starting with "_do" 135 """ 136 137 def __init__(self, afe): 138 image_types = [afe.CROS_IMAGE_TYPE, afe.FIRMWARE_IMAGE_TYPE] 139 self._version_maps = { 140 image_type: afe.get_stable_version_map(image_type) 141 for image_type in image_types 142 } 143 self._cros_map = self._version_maps[afe.CROS_IMAGE_TYPE] 144 self._selected_map = None 145 146 def select_version_map(self, image_type): 147 """ 148 Select an AFE version map object based on `image_type`. 149 150 This creates and remembers an AFE version mapper object to be 151 used for making changes in normal mode. 152 153 @param image_type Image type parameter for the version mapper 154 object. 155 @returns The full set of mappings for the image type. 156 """ 157 self._selected_map = self._version_maps[image_type] 158 return self._selected_map.get_all_versions() 159 160 def get_cros_image_name(self, board, version): 161 """ 162 Get the CrOS image name of a given board and version. 163 164 Returns the string naming a complete Chrome OS image 165 for the `board` and `version`. The name is part of a URI 166 in storage naming artifacts associated with the build 167 for the given board. 168 169 The returned string is generally in a form like 170 "celes-release/R56-9000.29.3". 171 172 @returns A CrOS image name. 173 """ 174 return self._cros_map.format_image_name(board, version) 175 176 def announce(self): 177 """Announce the start of processing to the user.""" 178 pass 179 180 def report(self, message): 181 """ 182 Report a pre-formatted message for the user. 183 184 The message is printed to stdout, followed by a newline. 185 186 @param message The message to be provided to the user. 187 """ 188 print message 189 190 def report_default_changed(self, old_default, new_default): 191 """ 192 Report that the default version mapping is changing. 193 194 This merely reports a text description of the pending change 195 without executing it. 196 197 @param old_default The original default version. 198 @param new_default The new default version to be applied. 199 """ 200 self.report('Default %s -> %s' % (old_default, new_default)) 201 202 def _report_board_changed(self, board, old_version, new_version): 203 """ 204 Report a change in one board's assigned version mapping. 205 206 This merely reports a text description of the pending change 207 without executing it. 208 209 @param board The board with the changing version. 210 @param old_version The original version mapped to the board. 211 @param new_version The new version to be applied to the board. 212 """ 213 template = ' %-22s %s -> %s' 214 self.report(template % (board, old_version, new_version)) 215 216 def report_board_unchanged(self, board, old_version): 217 """ 218 Report that a board's version mapping is unchanged. 219 220 This reports that a board has a non-default mapping that will be 221 unchanged. 222 223 @param board The board that is not changing. 224 @param old_version The board's version mapping. 225 """ 226 self._report_board_changed(board, '(no change)', old_version) 227 228 def _do_set_mapping(self, board, new_version): 229 """ 230 Change one board's assigned version mapping. 231 232 @param board The board with the changing version. 233 @param new_version The new version to be applied to the board. 234 """ 235 pass 236 237 def _do_delete_mapping(self, board): 238 """ 239 Delete one board's assigned version mapping. 240 241 @param board The board with the version to be deleted. 242 """ 243 pass 244 245 def set_mapping(self, board, old_version, new_version): 246 """ 247 Change and report a board version mapping. 248 249 @param board The board with the changing version. 250 @param old_version The original version mapped to the board. 251 @param new_version The new version to be applied to the board. 252 """ 253 self._report_board_changed(board, old_version, new_version) 254 self._do_set_mapping(board, new_version) 255 256 def upgrade_default(self, new_default): 257 """ 258 Apply a default version change. 259 260 @param new_default The new default version to be applied. 261 """ 262 self._do_set_mapping(_DEFAULT_BOARD, new_default) 263 264 def delete_mapping(self, board, old_version): 265 """ 266 Delete a board version mapping, and report the change. 267 268 @param board The board with the version to be deleted. 269 @param old_version The board's verson prior to deletion. 270 """ 271 assert board != _DEFAULT_BOARD 272 self._report_board_changed(board, 273 old_version, 274 _DEFAULT_VERSION_TAG) 275 self._do_delete_mapping(board) 276 277 278class _DryRunUpdater(_VersionUpdater): 279 """Code for handling --dry-run execution.""" 280 281 def announce(self): 282 self.report('Dry run: no changes will be made.') 283 284 285class _NormalModeUpdater(_VersionUpdater): 286 """Code for handling normal execution.""" 287 288 def _do_set_mapping(self, board, new_version): 289 self._selected_map.set_version(board, new_version) 290 291 def _do_delete_mapping(self, board): 292 self._selected_map.delete_version(board) 293 294 295def _read_gs_json_data(gs_uri): 296 """ 297 Read and parse a JSON file from googlestorage. 298 299 This is a wrapper around `gsutil cat` for the specified URI. 300 The standard output of the command is parsed as JSON, and the 301 resulting object returned. 302 303 @return A JSON object parsed from `gs_uri`. 304 """ 305 with open('/dev/null', 'w') as ignore_errors: 306 sp = subprocess.Popen(['gsutil', 'cat', gs_uri], 307 stdout=subprocess.PIPE, 308 stderr=ignore_errors) 309 try: 310 json_object = json.load(sp.stdout) 311 finally: 312 sp.stdout.close() 313 sp.wait() 314 return json_object 315 316 317def _make_omaha_versions(omaha_status): 318 """ 319 Convert parsed omaha versions data to a versions mapping. 320 321 Returns a dictionary mapping board names to the currently preferred 322 version for the Beta channel as served by Omaha. The mappings are 323 provided by settings in the JSON object `omaha_status`. 324 325 The board names are the names as known to Omaha: If the board name 326 in the AFE contains '_', the corresponding Omaha name uses '-' 327 instead. The boards mapped may include boards not in the list of 328 managed boards in the lab. 329 330 @return A dictionary mapping Omaha boards to Beta versions. 331 """ 332 def _entry_valid(json_entry): 333 return json_entry['channel'] == 'beta' 334 335 def _get_omaha_data(json_entry): 336 board = json_entry['board']['public_codename'] 337 milestone = json_entry['milestone'] 338 build = json_entry['chrome_os_version'] 339 version = 'R%d-%s' % (milestone, build) 340 return (board, version) 341 342 return dict(_get_omaha_data(e) for e in omaha_status['omaha_data'] 343 if _entry_valid(e)) 344 345 346def _get_upgrade_versions(afe_versions, omaha_versions, boards): 347 """ 348 Get the new stable versions to which we should update. 349 350 The new versions are returned as a tuple of a dictionary mapping 351 board names to versions, plus a new default board setting. The 352 new default is determined as the most commonly used version 353 across the given boards. 354 355 The new dictionary will have a mapping for every board in `boards`. 356 That mapping will be taken from `afe_versions`, unless the board has 357 a mapping in `omaha_versions` _and_ the omaha version is more recent 358 than the AFE version. 359 360 @param afe_versions The current board->version mappings in the 361 AFE. 362 @param omaha_versions The current board->version mappings from 363 Omaha for the Beta channel. 364 @param boards Set of boards to be upgraded. 365 @return Tuple of (mapping, default) where mapping is a dictionary 366 mapping boards to versions, and default is a version string. 367 """ 368 upgrade_versions = {} 369 version_counts = {} 370 afe_default = afe_versions[_DEFAULT_BOARD] 371 for board in boards: 372 version = afe_versions.get(board, afe_default) 373 omaha_version = omaha_versions.get(board.replace('_', '-')) 374 if (omaha_version is not None and 375 utils.compare_versions(version, omaha_version) < 0): 376 version = omaha_version 377 upgrade_versions[board] = version 378 version_counts.setdefault(version, 0) 379 version_counts[version] += 1 380 return (upgrade_versions, 381 max(version_counts.items(), key=lambda x: x[1])[0]) 382 383 384def _get_by_key_path(dictdict, key_path): 385 """ 386 Traverse a sequence of keys in a dict of dicts. 387 388 The `dictdict` parameter is a dict of nested dict values, and 389 `key_path` a list of keys. 390 391 A single-element key path returns `dictdict[key_path[0]]`, a 392 two-element path returns `dictdict[key_path[0]][key_path[1]]`, and 393 so forth. If any key in the path is not found, return `None`. 394 395 @param dictdict A dictionary of nested dictionaries. 396 @param key_path The sequence of keys to look up in `dictdict`. 397 @return The value found by successive dictionary lookups, or `None`. 398 """ 399 value = dictdict 400 for key in key_path: 401 value = value.get(key) 402 if value is None: 403 break 404 return value 405 406 407def _get_firmware_version(updater, board, cros_version): 408 """ 409 Get the firmware version for a given board and CrOS version. 410 411 @param updater A `_VersionUpdater` to use for extracting the 412 image name. 413 @param board The board for the firmware version to be 414 determined. 415 @param cros_version The CrOS version bundling the firmware. 416 @return The version string of the firmware for `board` bundled with 417 `cros_version`. 418 """ 419 try: 420 uri = (_BUILD_METADATA_PATTERN 421 % updater.get_cros_image_name(board, cros_version)) 422 key_path = ['board-metadata', board, 'main-firmware-version'] 423 return _get_by_key_path(_read_gs_json_data(uri), key_path) 424 except: 425 # TODO(jrbarnette): If we get here, it likely means that 426 # the repair build for our board doesn't exist. That can 427 # happen if a board doesn't release on the Beta channel for 428 # at least 6 months. 429 # 430 # We can't allow this error to propogate up the call chain 431 # because that will kill assigning versions to all the other 432 # boards that are still OK, so for now we ignore it. We 433 # really should do better. 434 return None 435 436 437def _get_firmware_upgrades(afe_versions, cros_versions): 438 """ 439 Get the new firmware versions to which we should update. 440 441 The new versions are returned in a dictionary mapping board names to 442 firmware versions. The new dictionary will have a mapping for every 443 board in `cros_versions`, excluding boards named in 444 `_FIRMWARE_UPGRADE_BLACKLIST`. 445 446 The firmware for each board is determined from the JSON metadata for 447 the CrOS build for that board, as specified in `cros_versions`. 448 449 @param afe_versions The current board->version mappings in the 450 AFE. 451 @param cros_versions Current board->cros version mappings in the 452 AFE. 453 @return A dictionary mapping boards to firmware versions. 454 """ 455 return { 456 board: _get_firmware_version(afe_versions, board, version) 457 for board, version in cros_versions.iteritems() 458 if board not in _FIRMWARE_UPGRADE_BLACKLIST 459 } 460 461 462def _apply_cros_upgrades(updater, old_versions, new_versions, 463 new_default): 464 """ 465 Change CrOS stable version mappings in the AFE. 466 467 The input `old_versions` dictionary represents the content of the 468 `afe_stable_versions` database table; it contains mappings for a 469 default version, plus exceptions for boards with non-default 470 mappings. 471 472 The `new_versions` dictionary contains a mapping for every board, 473 including boards that will be mapped to the new default version. 474 475 This function applies the AFE changes necessary to produce the new 476 AFE mappings indicated by `new_versions` and `new_default`. The 477 changes are ordered so that at any moment, every board is mapped 478 either according to the old or the new mapping. 479 480 @param updater Instance of _VersionUpdater responsible for 481 making the actual database changes. 482 @param old_versions The current board->version mappings in the 483 AFE. 484 @param new_versions New board->version mappings obtained by 485 applying Beta channel upgrades from Omaha. 486 @param new_default The new default build for the AFE. 487 """ 488 old_default = old_versions[_DEFAULT_BOARD] 489 if old_default != new_default: 490 updater.report_default_changed(old_default, new_default) 491 updater.report('Applying stable version changes:') 492 default_count = 0 493 for board, new_build in new_versions.items(): 494 if new_build == new_default: 495 default_count += 1 496 elif board in old_versions and new_build == old_versions[board]: 497 updater.report_board_unchanged(board, new_build) 498 else: 499 old_build = old_versions.get(board) 500 if old_build is None: 501 old_build = _DEFAULT_VERSION_TAG 502 updater.set_mapping(board, old_build, new_build) 503 if old_default != new_default: 504 updater.upgrade_default(new_default) 505 for board, new_build in new_versions.items(): 506 if new_build == new_default and board in old_versions: 507 updater.delete_mapping(board, old_versions[board]) 508 updater.report('%d boards now use the default mapping' % 509 default_count) 510 511 512def _apply_firmware_upgrades(updater, old_versions, new_versions): 513 """ 514 Change firmware version mappings in the AFE. 515 516 The input `old_versions` dictionary represents the content of the 517 firmware mappings in the `afe_stable_versions` database table. 518 There is no default version; missing boards simply have no current 519 version. 520 521 This function applies the AFE changes necessary to produce the new 522 AFE mappings indicated by `new_versions`. 523 524 TODO(jrbarnette) This function ought to remove any mapping not found 525 in `new_versions`. However, in theory, that's only needed to 526 account for boards that are removed from the lab, and that hasn't 527 happened yet. 528 529 @param updater Instance of _VersionUpdater responsible for 530 making the actual database changes. 531 @param old_versions The current board->version mappings in the 532 AFE. 533 @param new_versions New board->version mappings obtained by 534 applying Beta channel upgrades from Omaha. 535 """ 536 unchanged = 0 537 no_version = 0 538 for board, new_firmware in new_versions.items(): 539 if new_firmware is None: 540 no_version += 1 541 elif board not in old_versions: 542 updater.set_mapping(board, '(nothing)', new_firmware) 543 else: 544 old_firmware = old_versions[board] 545 if new_firmware != old_firmware: 546 updater.set_mapping(board, old_firmware, new_firmware) 547 else: 548 unchanged += 1 549 updater.report('%d boards have no firmware mapping' % no_version) 550 updater.report('%d boards are unchanged' % unchanged) 551 552 553def _parse_command_line(argv): 554 """ 555 Parse the command line arguments. 556 557 Create an argument parser for this command's syntax, parse the 558 command line, and return the result of the ArgumentParser 559 parse_args() method. 560 561 @param argv Standard command line argument vector; argv[0] is 562 assumed to be the command name. 563 @return Result returned by ArgumentParser.parse_args(). 564 565 """ 566 parser = argparse.ArgumentParser( 567 prog=argv[0], 568 description='Update the stable repair version for all ' 569 'boards') 570 parser.add_argument('-n', '--dry-run', dest='updater_mode', 571 action='store_const', const=_DryRunUpdater, 572 help='print changes without executing them') 573 parser.add_argument('extra_boards', nargs='*', metavar='BOARD', 574 help='Names of additional boards to be updated.') 575 arguments = parser.parse_args(argv[1:]) 576 if not arguments.updater_mode: 577 arguments.updater_mode = _NormalModeUpdater 578 return arguments 579 580 581def main(argv): 582 """ 583 Standard main routine. 584 585 @param argv Command line arguments including `sys.argv[0]`. 586 """ 587 arguments = _parse_command_line(argv) 588 afe = frontend_wrappers.RetryingAFE(server=None) 589 updater = arguments.updater_mode(afe) 590 updater.announce() 591 boards = (set(arguments.extra_boards) | 592 lab_inventory.get_managed_boards(afe)) 593 594 afe_versions = updater.select_version_map(afe.CROS_IMAGE_TYPE) 595 omaha_versions = _make_omaha_versions( 596 _read_gs_json_data(_OMAHA_STATUS)) 597 upgrade_versions, new_default = ( 598 _get_upgrade_versions(afe_versions, omaha_versions, boards)) 599 _apply_cros_upgrades(updater, afe_versions, 600 upgrade_versions, new_default) 601 602 updater.report('\nApplying firmware updates:') 603 fw_versions = updater.select_version_map( 604 afe.FIRMWARE_IMAGE_TYPE) 605 firmware_upgrades = _get_firmware_upgrades(updater, upgrade_versions) 606 _apply_firmware_upgrades(updater, fw_versions, firmware_upgrades) 607 608 609if __name__ == '__main__': 610 main(sys.argv) 611