1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Script that reads omahaproxy and gsutil to determine version of SDK to put 7in manifest. 8""" 9 10# pylint is convinced the email module is missing attributes 11# pylint: disable=E1101 12 13import buildbot_common 14import csv 15import cStringIO 16import difflib 17import email 18import logging 19import logging.handlers 20import manifest_util 21import optparse 22import os 23import posixpath 24import re 25import smtplib 26import subprocess 27import sys 28import time 29import traceback 30import urllib2 31 32MANIFEST_BASENAME = 'naclsdk_manifest2.json' 33SCRIPT_DIR = os.path.dirname(__file__) 34REPO_MANIFEST = os.path.join(SCRIPT_DIR, 'json', MANIFEST_BASENAME) 35GS_BUCKET_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/' 36GS_SDK_MANIFEST = GS_BUCKET_PATH + MANIFEST_BASENAME 37GS_SDK_MANIFEST_LOG = GS_BUCKET_PATH + MANIFEST_BASENAME + '.log' 38GS_MANIFEST_BACKUP_DIR = GS_BUCKET_PATH + 'manifest_backups/' 39 40CANARY_BUNDLE_NAME = 'pepper_canary' 41BIONIC_CANARY_BUNDLE_NAME = 'bionic_canary' 42CANARY = 'canary' 43NACLPORTS_ARCHIVE_NAME = 'naclports.tar.bz2' 44 45 46logger = logging.getLogger(__name__) 47 48 49def SplitVersion(version_string): 50 """Split a version string (e.g. "18.0.1025.163") into its components. 51 52 e.g. 53 SplitVersion("trunk.123456") => ("trunk", "123456") 54 SplitVersion("18.0.1025.163") => (18, 0, 1025, 163) 55 """ 56 parts = version_string.split('.') 57 if parts[0] == 'trunk': 58 return (parts[0], int(parts[1])) 59 return tuple([int(p) for p in parts]) 60 61 62def GetMajorVersion(version_string): 63 """Get the major version number from a version string (e.g. "18.0.1025.163"). 64 65 e.g. 66 GetMajorVersion("trunk.123456") => "trunk" 67 GetMajorVersion("18.0.1025.163") => 18 68 """ 69 return SplitVersion(version_string)[0] 70 71 72def CompareVersions(version1, version2): 73 """Compare two version strings and return -1, 0, 1 (similar to cmp). 74 75 Versions can only be compared if they are both trunk versions, or neither is. 76 77 e.g. 78 CompareVersions("trunk.123", "trunk.456") => -1 79 CompareVersions("18.0.1025.163", "37.0.2054.3") => -1 80 CompareVersions("trunk.123", "18.0.1025.163") => Error 81 82 """ 83 split1 = SplitVersion(version1) 84 split2 = SplitVersion(version2) 85 if split1[0] == split2[0]: 86 return cmp(split1[1:], split2[1:]) 87 88 if split1 == 'trunk' or split2 == 'trunk': 89 raise Exception("Unable to compare versions %s and %s" % ( 90 version1, version2)) 91 92 return cmp(split1, split2) 93 94 95def JoinVersion(version_tuple): 96 """Create a string from a version tuple. 97 98 The tuple should be of the form (18, 0, 1025, 163). 99 """ 100 assert len(version_tuple) == 4 101 assert version_tuple[0] != 'trunk' 102 return '.'.join(map(str, version_tuple)) 103 104 105def GetTimestampManifestName(): 106 """Create a manifest name with a timestamp. 107 108 Returns: 109 A manifest name with an embedded date. This should make it easier to roll 110 back if necessary. 111 """ 112 return time.strftime('naclsdk_manifest2.%Y_%m_%d_%H_%M_%S.json', 113 time.gmtime()) 114 115 116def GetPlatformArchiveName(platform): 117 """Get the basename of an archive given a platform string. 118 119 Args: 120 platform: One of ('win', 'mac', 'linux'). 121 122 Returns: 123 The basename of the sdk archive for that platform. 124 """ 125 return 'naclsdk_%s.tar.bz2' % platform 126 127 128def GetBionicArchiveName(): 129 """Get the basename of an archive. Currently this is linux-only""" 130 return 'naclsdk_bionic.tar.bz2' 131 132 133def GetCanonicalArchiveName(url): 134 """Get the canonical name of an archive given its URL. 135 136 This will convert "naclsdk_linux.bz2" -> "naclsdk_linux.tar.bz2", and also 137 remove everything but the filename of the URL. 138 139 This is used below to determine if an expected bundle is found in an version 140 directory; the archives all have the same name, but may not exist for a given 141 version. 142 143 Args: 144 url: The url to parse. 145 146 Returns: 147 The canonical name as described above. 148 """ 149 name = posixpath.basename(url) 150 match = re.match(r'naclsdk_(.*?)(?:\.tar)?\.bz2', name) 151 if match: 152 return 'naclsdk_%s.tar.bz2' % match.group(1) 153 154 return name 155 156 157class Delegate(object): 158 """Delegate all external access; reading/writing to filesystem, gsutil etc.""" 159 160 def GetRepoManifest(self): 161 """Read the manifest file from the NaCl SDK repository. 162 163 This manifest is used as a template for the auto updater; only pepper 164 bundles with no archives are considered for auto updating. 165 166 Returns: 167 A manifest_util.SDKManifest object read from the NaCl SDK repo.""" 168 raise NotImplementedError() 169 170 def GetHistory(self): 171 """Read Chrome release history from omahaproxy.appspot.com 172 173 Here is an example of data from this URL: 174 cros,stable,18.0.1025.168,2012-05-01 17:04:05.962578\n 175 win,canary,20.0.1123.0,2012-05-01 13:59:31.703020\n 176 mac,canary,20.0.1123.0,2012-05-01 11:54:13.041875\n 177 win,stable,18.0.1025.168,2012-04-30 20:34:56.078490\n 178 mac,stable,18.0.1025.168,2012-04-30 20:34:55.231141\n 179 ... 180 Where each line has comma separated values in the following format: 181 platform, channel, version, date/time\n 182 183 Returns: 184 A list where each element is a line from the document, represented as a 185 tuple.""" 186 raise NotImplementedError() 187 188 def GsUtil_ls(self, url): 189 """Runs gsutil ls |url| 190 191 Args: 192 url: The cloud storage url to list. 193 Returns: 194 A list of URLs, all with the gs:// schema.""" 195 raise NotImplementedError() 196 197 def GsUtil_cat(self, url): 198 """Runs gsutil cat |url| 199 200 Args: 201 url: The cloud storage url to read from. 202 Returns: 203 A string with the contents of the file at |url|.""" 204 raise NotImplementedError() 205 206 def GsUtil_cp(self, src, dest, stdin=None): 207 """Runs gsutil cp |src| |dest| 208 209 Args: 210 src: The file path or url to copy from. 211 dest: The file path or url to copy to. 212 stdin: If src is '-', this is used as the stdin to give to gsutil. The 213 effect is that text in stdin is copied to |dest|.""" 214 raise NotImplementedError() 215 216 def SendMail(self, subject, text): 217 """Send an email. 218 219 Args: 220 subject: The subject of the email. 221 text: The text of the email. 222 """ 223 raise NotImplementedError() 224 225 226class RealDelegate(Delegate): 227 def __init__(self, dryrun=False, gsutil=None, mailfrom=None, mailto=None): 228 super(RealDelegate, self).__init__() 229 self.dryrun = dryrun 230 self.mailfrom = mailfrom 231 self.mailto = mailto 232 if gsutil: 233 self.gsutil = gsutil 234 else: 235 self.gsutil = buildbot_common.GetGsutil() 236 237 def GetRepoManifest(self): 238 """See Delegate.GetRepoManifest""" 239 with open(REPO_MANIFEST, 'r') as sdk_stream: 240 sdk_json_string = sdk_stream.read() 241 242 manifest = manifest_util.SDKManifest() 243 manifest.LoadDataFromString(sdk_json_string, add_missing_info=True) 244 return manifest 245 246 def GetHistory(self): 247 """See Delegate.GetHistory""" 248 url_stream = urllib2.urlopen('https://omahaproxy.appspot.com/history') 249 history = [(platform, channel, version, date) 250 for platform, channel, version, date in csv.reader(url_stream)] 251 252 # The first line of this URL is the header: 253 # os,channel,version,timestamp 254 return history[1:] 255 256 def GsUtil_ls(self, url): 257 """See Delegate.GsUtil_ls""" 258 try: 259 stdout = self._RunGsUtil(None, False, 'ls', url) 260 except subprocess.CalledProcessError: 261 return [] 262 263 # filter out empty lines 264 return filter(None, stdout.split('\n')) 265 266 def GsUtil_cat(self, url): 267 """See Delegate.GsUtil_cat""" 268 return self._RunGsUtil(None, True, 'cat', url) 269 270 def GsUtil_cp(self, src, dest, stdin=None): 271 """See Delegate.GsUtil_cp""" 272 if self.dryrun: 273 logger.info("Skipping upload: %s -> %s" % (src, dest)) 274 if src == '-': 275 logger.info(' contents = """%s"""' % stdin) 276 return 277 278 return self._RunGsUtil(stdin, True, 'cp', '-a', 'public-read', src, dest) 279 280 def SendMail(self, subject, text): 281 """See Delegate.SendMail""" 282 if self.mailfrom and self.mailto: 283 msg = email.MIMEMultipart.MIMEMultipart() 284 msg['From'] = self.mailfrom 285 msg['To'] = ', '.join(self.mailto) 286 msg['Date'] = email.Utils.formatdate(localtime=True) 287 msg['Subject'] = subject 288 msg.attach(email.MIMEText.MIMEText(text)) 289 smtp_obj = smtplib.SMTP('localhost') 290 smtp_obj.sendmail(self.mailfrom, self.mailto, msg.as_string()) 291 smtp_obj.close() 292 293 def _RunGsUtil(self, stdin, log_errors, *args): 294 """Run gsutil as a subprocess. 295 296 Args: 297 stdin: If non-None, used as input to the process. 298 log_errors: If True, write errors to stderr. 299 *args: Arguments to pass to gsutil. The first argument should be an 300 operation such as ls, cp or cat. 301 Returns: 302 The stdout from the process.""" 303 cmd = [self.gsutil] + list(args) 304 logger.debug("Running: %s" % str(cmd)) 305 if stdin: 306 stdin_pipe = subprocess.PIPE 307 else: 308 stdin_pipe = None 309 310 try: 311 process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE, 312 stderr=subprocess.PIPE) 313 stdout, stderr = process.communicate(stdin) 314 except OSError as e: 315 raise manifest_util.Error("Unable to run '%s': %s" % (cmd[0], str(e))) 316 317 if process.returncode: 318 if log_errors: 319 logger.error(stderr) 320 raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd)) 321 return stdout 322 323 324class GsutilLoggingHandler(logging.handlers.BufferingHandler): 325 def __init__(self, delegate): 326 logging.handlers.BufferingHandler.__init__(self, capacity=0) 327 self.delegate = delegate 328 329 def shouldFlush(self, record): 330 # BufferingHandler.shouldFlush automatically flushes if the length of the 331 # buffer is greater than self.capacity. We don't want that behavior, so 332 # return False here. 333 return False 334 335 def flush(self): 336 # Do nothing here. We want to be explicit about uploading the log. 337 pass 338 339 def upload(self): 340 output_list = [] 341 for record in self.buffer: 342 output_list.append(self.format(record)) 343 output = '\n'.join(output_list) 344 self.delegate.GsUtil_cp('-', GS_SDK_MANIFEST_LOG, stdin=output) 345 346 logging.handlers.BufferingHandler.flush(self) 347 348 349class NoSharedVersionException(Exception): 350 pass 351 352 353class VersionFinder(object): 354 """Finds a version of a pepper bundle that all desired platforms share. 355 356 Args: 357 delegate: See Delegate class above. 358 platforms: A sequence of platforms to consider, e.g. 359 ('mac', 'linux', 'win') 360 extra_archives: A sequence of tuples: (archive_basename, minimum_version), 361 e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')] 362 These archives must exist to consider a version for inclusion, as 363 long as that version is greater than the archive's minimum version. 364 is_bionic: True if we are searching for bionic archives. 365 """ 366 def __init__(self, delegate, platforms, extra_archives=None, is_bionic=False): 367 self.delegate = delegate 368 self.history = delegate.GetHistory() 369 self.platforms = platforms 370 self.extra_archives = extra_archives 371 self.is_bionic = is_bionic 372 373 def GetMostRecentSharedVersion(self, major_version): 374 """Returns the most recent version of a pepper bundle that exists on all 375 given platforms. 376 377 Specifically, the resulting version should be the most recently released 378 (meaning closest to the top of the listing on 379 omahaproxy.appspot.com/history) version that has a Chrome release on all 380 given platforms, and has a pepper bundle archive for each platform as well. 381 382 Args: 383 major_version: The major version of the pepper bundle, e.g. 19. 384 Returns: 385 A tuple (version, channel, archives). The version is a string such as 386 "19.0.1084.41". The channel is one of ('stable', 'beta', or 'dev'). 387 |archives| is a list of archive URLs.""" 388 def GetPlatformHistory(platform): 389 return self._GetPlatformMajorVersionHistory(major_version, platform) 390 391 shared_version_generator = self._FindNextSharedVersion(self.platforms, 392 GetPlatformHistory) 393 return self._DoGetMostRecentSharedVersion(shared_version_generator) 394 395 def GetMostRecentSharedCanary(self): 396 """Returns the most recent version of a canary pepper bundle that exists on 397 all given platforms. 398 399 Canary is special-cased because we don't care about its major version; we 400 always use the most recent canary, regardless of major version. 401 402 Returns: 403 A tuple (version, channel, archives). The version is a string such as 404 "trunk.123456". The channel is always 'canary'. |archives| is a list of 405 archive URLs.""" 406 version_generator = self._FindNextTrunkVersion() 407 return self._DoGetMostRecentSharedVersion(version_generator) 408 409 def GetAvailablePlatformArchivesFor(self, version): 410 """Returns a sequence of archives that exist for a given version, on the 411 given platforms. 412 413 The second element of the returned tuple is a list of all platforms that do 414 not have an archive for the given version. 415 416 Args: 417 version: The version to find archives for. (e.g. "18.0.1025.164") 418 Returns: 419 A tuple (archives, missing_archives). |archives| is a list of archive 420 URLs, |missing_archives| is a list of archive names. 421 """ 422 archive_urls = self._GetAvailableArchivesFor(version) 423 424 if self.is_bionic: 425 # Bionic currently is Linux-only. 426 expected_archives = set([GetBionicArchiveName()]) 427 else: 428 expected_archives = set(GetPlatformArchiveName(p) for p in self.platforms) 429 430 if self.extra_archives: 431 for extra_archive, extra_archive_min_version in self.extra_archives: 432 if CompareVersions(version, extra_archive_min_version) >= 0: 433 expected_archives.add(extra_archive) 434 found_archives = set(GetCanonicalArchiveName(a) for a in archive_urls) 435 missing_archives = expected_archives - found_archives 436 437 # Only return archives that are "expected". 438 def IsExpected(url): 439 return GetCanonicalArchiveName(url) in expected_archives 440 441 expected_archive_urls = [u for u in archive_urls if IsExpected(u)] 442 return expected_archive_urls, missing_archives 443 444 def _DoGetMostRecentSharedVersion(self, shared_version_generator): 445 """Returns the most recent version of a pepper bundle that exists on all 446 given platforms. 447 448 This function does the real work for the public GetMostRecentShared* above. 449 450 Args: 451 shared_version_generator: A generator that will yield (version, channel) 452 tuples in order of most recent to least recent. 453 Returns: 454 A tuple (version, channel, archives). The version is a string such as 455 "19.0.1084.41". The channel is one of ('stable', 'beta', 'dev', 456 'canary'). |archives| is a list of archive URLs.""" 457 version = None 458 skipped_versions = [] 459 channel = '' 460 while True: 461 try: 462 version, channel = shared_version_generator.next() 463 except StopIteration: 464 msg = 'No shared version for platforms: %s\n' % ( 465 ', '.join(self.platforms)) 466 msg += 'Last version checked = %s.\n' % (version,) 467 if skipped_versions: 468 msg += 'Versions skipped due to missing archives:\n' 469 for version, channel, missing_archives in skipped_versions: 470 archive_msg = '(missing %s)' % (', '.join(missing_archives)) 471 msg += ' %s (%s) %s\n' % (version, channel, archive_msg) 472 raise NoSharedVersionException(msg) 473 474 logger.info('Found shared version: %s, channel: %s' % ( 475 version, channel)) 476 477 archives, missing_archives = self.GetAvailablePlatformArchivesFor(version) 478 479 if not missing_archives: 480 return version, channel, archives 481 482 logger.info(' skipping. Missing archives: %s' % ( 483 ', '.join(missing_archives))) 484 485 skipped_versions.append((version, channel, missing_archives)) 486 487 def _GetPlatformMajorVersionHistory(self, with_major_version, with_platform): 488 """Yields Chrome history for a given platform and major version. 489 490 Args: 491 with_major_version: The major version to filter for. If 0, match all 492 versions. 493 with_platform: The name of the platform to filter for. 494 Returns: 495 A generator that yields a tuple (channel, version) for each version that 496 matches the platform and major version. The version returned is a tuple as 497 returned from SplitVersion. 498 """ 499 for platform, channel, version, _ in self.history: 500 version = SplitVersion(version) 501 if (with_platform == platform and 502 (with_major_version == 0 or with_major_version == version[0])): 503 yield channel, version 504 505 def _FindNextSharedVersion(self, platforms, generator_func): 506 """Yields versions of Chrome that exist on all given platforms, in order of 507 newest to oldest. 508 509 Versions are compared in reverse order of release. That is, the most 510 recently updated version will be tested first. 511 512 Args: 513 platforms: A sequence of platforms to consider, e.g. 514 ('mac', 'linux', 'win') 515 generator_func: A function which takes a platform and returns a 516 generator that yields (channel, version) tuples. 517 Returns: 518 A generator that yields a tuple (version, channel) for each version that 519 matches all platforms and the major version. The version returned is a 520 string (e.g. "18.0.1025.164"). 521 """ 522 platform_generators = [] 523 for platform in platforms: 524 platform_generators.append(generator_func(platform)) 525 526 shared_version = None 527 platform_versions = [] 528 for platform_gen in platform_generators: 529 platform_versions.append(platform_gen.next()) 530 531 while True: 532 if logger.isEnabledFor(logging.INFO): 533 msg_info = [] 534 for i, platform in enumerate(platforms): 535 msg_info.append('%s: %s' % ( 536 platform, JoinVersion(platform_versions[i][1]))) 537 logger.info('Checking versions: %s' % ', '.join(msg_info)) 538 539 shared_version = min((v for c, v in platform_versions)) 540 541 if all(v == shared_version for c, v in platform_versions): 542 # grab the channel from an arbitrary platform 543 first_platform = platform_versions[0] 544 channel = first_platform[0] 545 yield JoinVersion(shared_version), channel 546 547 # force increment to next version for all platforms 548 shared_version = None 549 550 # Find the next version for any platform that isn't at the shared version. 551 try: 552 for i, platform_gen in enumerate(platform_generators): 553 if platform_versions[i][1] != shared_version: 554 platform_versions[i] = platform_gen.next() 555 except StopIteration: 556 return 557 558 559 def _FindNextTrunkVersion(self): 560 """Yields all trunk versions that exist in the cloud storage bucket, newest 561 to oldest. 562 563 Returns: 564 A generator that yields a tuple (version, channel) for each version that 565 matches all platforms and the major version. The version returned is a 566 string (e.g. "trunk.123456"). 567 """ 568 files = self.delegate.GsUtil_ls(GS_BUCKET_PATH) 569 assert all(f.startswith('gs://') for f in files) 570 571 trunk_versions = [] 572 for f in files: 573 match = re.search(r'(trunk\.\d+)', f) 574 if match: 575 trunk_versions.append(match.group(1)) 576 577 trunk_versions.sort(reverse=True) 578 579 for version in trunk_versions: 580 yield version, 'canary' 581 582 583 def _GetAvailableArchivesFor(self, version_string): 584 """Downloads a list of all available archives for a given version. 585 586 Args: 587 version_string: The version to find archives for. (e.g. "18.0.1025.164") 588 Returns: 589 A list of strings, each of which is a platform-specific archive URL. (e.g. 590 "gs://nativeclient_mirror/nacl/nacl_sdk/18.0.1025.164/" 591 "naclsdk_linux.tar.bz2"). 592 593 All returned URLs will use the gs:// schema.""" 594 files = self.delegate.GsUtil_ls(GS_BUCKET_PATH + version_string) 595 596 assert all(f.startswith('gs://') for f in files) 597 598 archives = [f for f in files if not f.endswith('.json')] 599 manifests = [f for f in files if f.endswith('.json')] 600 601 # don't include any archives that don't have an associated manifest. 602 return filter(lambda a: a + '.json' in manifests, archives) 603 604 605class UnknownLockedBundleException(Exception): 606 pass 607 608 609class Updater(object): 610 def __init__(self, delegate): 611 self.delegate = delegate 612 self.versions_to_update = [] 613 self.locked_bundles = [] 614 self.online_manifest = manifest_util.SDKManifest() 615 self._FetchOnlineManifest() 616 617 def AddVersionToUpdate(self, bundle_name, version, channel, archives): 618 """Add a pepper version to update in the uploaded manifest. 619 620 Args: 621 bundle_name: The name of the pepper bundle, e.g. 'pepper_18' 622 version: The version of the pepper bundle, e.g. '18.0.1025.64' 623 channel: The stability of the pepper bundle, e.g. 'beta' 624 archives: A sequence of archive URLs for this bundle.""" 625 self.versions_to_update.append((bundle_name, version, channel, archives)) 626 627 def AddLockedBundle(self, bundle_name): 628 """Add a "locked" bundle to the updater. 629 630 A locked bundle is a bundle that wasn't found in the history. When this 631 happens, the bundle is now "locked" to whatever was last found. We want to 632 ensure that the online manifest has this bundle. 633 634 Args: 635 bundle_name: The name of the locked bundle. 636 """ 637 self.locked_bundles.append(bundle_name) 638 639 def Update(self, manifest): 640 """Update a manifest and upload it. 641 642 Note that bundles will not be updated if the current version is newer. 643 That is, the updater will never automatically update to an older version of 644 a bundle. 645 646 Args: 647 manifest: The manifest used as a template for updating. Only pepper 648 bundles that contain no archives will be considered for auto-updating.""" 649 # Make sure there is only one stable branch: the one with the max version. 650 # All others are post-stable. 651 stable_major_versions = [GetMajorVersion(version) for _, version, channel, _ 652 in self.versions_to_update if channel == 'stable'] 653 # Add 0 in case there are no stable versions. 654 max_stable_version = max([0] + stable_major_versions) 655 656 # Ensure that all locked bundles exist in the online manifest. 657 for bundle_name in self.locked_bundles: 658 online_bundle = self.online_manifest.GetBundle(bundle_name) 659 if online_bundle: 660 manifest.SetBundle(online_bundle) 661 else: 662 msg = ('Attempted to update bundle "%s", but no shared versions were ' 663 'found, and there is no online bundle with that name.') 664 raise UnknownLockedBundleException(msg % bundle_name) 665 666 if self.locked_bundles: 667 # Send a nagging email that we shouldn't be wasting time looking for 668 # bundles that are no longer in the history. 669 scriptname = os.path.basename(sys.argv[0]) 670 subject = '[%s] Reminder: remove bundles from %s' % (scriptname, 671 MANIFEST_BASENAME) 672 text = 'These bundles are not in the omahaproxy history anymore: ' + \ 673 ', '.join(self.locked_bundles) 674 self.delegate.SendMail(subject, text) 675 676 677 # Update all versions. 678 logger.info('>>> Updating bundles...') 679 for bundle_name, version, channel, archives in self.versions_to_update: 680 logger.info('Updating %s to %s...' % (bundle_name, version)) 681 bundle = manifest.GetBundle(bundle_name) 682 for archive in archives: 683 platform_bundle = self._GetPlatformArchiveBundle(archive) 684 # Normally the manifest snippet's bundle name matches our bundle name. 685 # pepper_canary, however is called "pepper_###" in the manifest 686 # snippet. 687 platform_bundle.name = bundle_name 688 bundle.MergeWithBundle(platform_bundle) 689 690 # Fix the stability and recommended values 691 major_version = GetMajorVersion(version) 692 if major_version < max_stable_version: 693 bundle.stability = 'post_stable' 694 else: 695 bundle.stability = channel 696 # We always recommend the stable version. 697 if bundle.stability == 'stable': 698 bundle.recommended = 'yes' 699 else: 700 bundle.recommended = 'no' 701 702 # Check to ensure this bundle is newer than the online bundle. 703 online_bundle = self.online_manifest.GetBundle(bundle_name) 704 if online_bundle: 705 # This test used to be online_bundle.revision >= bundle.revision. 706 # That doesn't do quite what we want: sometimes the metadata changes 707 # but the revision stays the same -- we still want to push those 708 # changes. 709 if online_bundle.revision > bundle.revision or online_bundle == bundle: 710 logger.info( 711 ' Revision %s is not newer than than online revision %s. ' 712 'Skipping.' % (bundle.revision, online_bundle.revision)) 713 714 manifest.SetBundle(online_bundle) 715 continue 716 self._UploadManifest(manifest) 717 logger.info('Done.') 718 719 def _GetPlatformArchiveBundle(self, archive): 720 """Downloads the manifest "snippet" for an archive, and reads it as a 721 Bundle. 722 723 Args: 724 archive: A full URL of a platform-specific archive, using the gs schema. 725 Returns: 726 An object of type manifest_util.Bundle, read from a JSON file storing 727 metadata for this archive. 728 """ 729 stdout = self.delegate.GsUtil_cat(archive + '.json') 730 bundle = manifest_util.Bundle('') 731 bundle.LoadDataFromString(stdout) 732 # Some snippets were uploaded with revisions and versions as strings. Fix 733 # those here. 734 bundle.revision = int(bundle.revision) 735 bundle.version = int(bundle.version) 736 737 # HACK. The naclports archive specifies host_os as linux. Change it to all. 738 for archive in bundle.GetArchives(): 739 if NACLPORTS_ARCHIVE_NAME in archive.url: 740 archive.host_os = 'all' 741 return bundle 742 743 def _UploadManifest(self, manifest): 744 """Upload a serialized manifest_util.SDKManifest object. 745 746 Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to 747 gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json. 748 749 Args: 750 manifest: The new manifest to upload. 751 """ 752 new_manifest_string = manifest.GetDataAsString() 753 online_manifest_string = self.online_manifest.GetDataAsString() 754 755 if self.delegate.dryrun: 756 logger.info(''.join(list(difflib.unified_diff( 757 online_manifest_string.splitlines(1), 758 new_manifest_string.splitlines(1))))) 759 return 760 else: 761 online_manifest = manifest_util.SDKManifest() 762 online_manifest.LoadDataFromString(online_manifest_string) 763 764 if online_manifest == manifest: 765 logger.info('New manifest doesn\'t differ from online manifest.' 766 'Skipping upload.') 767 return 768 769 timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \ 770 GetTimestampManifestName() 771 self.delegate.GsUtil_cp('-', timestamp_manifest_path, 772 stdin=manifest.GetDataAsString()) 773 774 # copy from timestampped copy over the official manifest. 775 self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST) 776 777 def _FetchOnlineManifest(self): 778 try: 779 online_manifest_string = self.delegate.GsUtil_cat(GS_SDK_MANIFEST) 780 except subprocess.CalledProcessError: 781 # It is not a failure if the online manifest doesn't exist. 782 online_manifest_string = '' 783 784 if online_manifest_string: 785 self.online_manifest.LoadDataFromString(online_manifest_string) 786 787 788def Run(delegate, platforms, extra_archives, fixed_bundle_versions=None): 789 """Entry point for the auto-updater. 790 791 Args: 792 delegate: The Delegate object to use for reading Urls, files, etc. 793 platforms: A sequence of platforms to consider, e.g. 794 ('mac', 'linux', 'win') 795 extra_archives: A sequence of tuples: (archive_basename, minimum_version), 796 e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')] 797 These archives must exist to consider a version for inclusion, as 798 long as that version is greater than the archive's minimum version. 799 fixed_bundle_versions: A sequence of tuples (bundle_name, version_string). 800 e.g. ('pepper_21', '21.0.1145.0') 801 """ 802 if fixed_bundle_versions: 803 fixed_bundle_versions = dict(fixed_bundle_versions) 804 else: 805 fixed_bundle_versions = {} 806 807 manifest = delegate.GetRepoManifest() 808 auto_update_bundles = [] 809 for bundle in manifest.GetBundles(): 810 if not bundle.name.startswith(('pepper_', 'bionic_')): 811 continue 812 archives = bundle.GetArchives() 813 if not archives: 814 auto_update_bundles.append(bundle) 815 816 if not auto_update_bundles: 817 logger.info('No versions need auto-updating.') 818 return 819 820 updater = Updater(delegate) 821 822 for bundle in auto_update_bundles: 823 try: 824 if bundle.name == BIONIC_CANARY_BUNDLE_NAME: 825 logger.info('>>> Looking for most recent bionic_canary...') 826 # Ignore extra_archives on bionic; There is no naclports bundle yet. 827 version_finder = VersionFinder(delegate, platforms, None, 828 is_bionic=True) 829 version, channel, archives = version_finder.GetMostRecentSharedCanary() 830 elif bundle.name == CANARY_BUNDLE_NAME: 831 logger.info('>>> Looking for most recent pepper_canary...') 832 version_finder = VersionFinder(delegate, platforms, extra_archives) 833 version, channel, archives = version_finder.GetMostRecentSharedCanary() 834 else: 835 logger.info('>>> Looking for most recent pepper_%s...' % 836 bundle.version) 837 version_finder = VersionFinder(delegate, platforms, extra_archives) 838 version, channel, archives = version_finder.GetMostRecentSharedVersion( 839 bundle.version) 840 except NoSharedVersionException: 841 # If we can't find a shared version, make sure that there is an uploaded 842 # bundle with that name already. 843 updater.AddLockedBundle(bundle.name) 844 continue 845 846 if bundle.name in fixed_bundle_versions: 847 # Ensure this version is valid for all platforms. 848 # If it is, use the channel found above (because the channel for this 849 # version may not be in the history.) 850 version = fixed_bundle_versions[bundle.name] 851 logger.info('Fixed bundle version: %s, %s' % (bundle.name, version)) 852 archives, missing = \ 853 version_finder.GetAvailablePlatformArchivesFor(version) 854 if missing: 855 logger.warn( 856 'Some archives for version %s of bundle %s don\'t exist: ' 857 'Missing %s' % (version, bundle.name, ', '.join(missing))) 858 return 859 860 updater.AddVersionToUpdate(bundle.name, version, channel, archives) 861 862 updater.Update(manifest) 863 864 865class CapturedFile(object): 866 """A file-like object that captures text written to it, but also passes it 867 through to an underlying file-like object.""" 868 def __init__(self, passthrough): 869 self.passthrough = passthrough 870 self.written = cStringIO.StringIO() 871 872 def write(self, s): 873 self.written.write(s) 874 if self.passthrough: 875 self.passthrough.write(s) 876 877 def getvalue(self): 878 return self.written.getvalue() 879 880 881def main(args): 882 parser = optparse.OptionParser() 883 parser.add_option('--gsutil', help='path to gsutil.') 884 parser.add_option('-d', '--debug', help='run in debug mode.', 885 action='store_true') 886 parser.add_option('--mailfrom', help='email address of sender.') 887 parser.add_option('--mailto', help='send error mails to...', action='append') 888 parser.add_option('-n', '--dryrun', help="don't upload the manifest.", 889 action='store_true') 890 parser.add_option('-v', '--verbose', help='print more diagnotic messages. ' 891 'Use more than once for more info.', 892 action='count') 893 parser.add_option('--log-file', metavar='FILE', help='log to FILE') 894 parser.add_option('--upload-log', help='Upload log alongside the manifest.', 895 action='store_true') 896 parser.add_option('--bundle-version', 897 help='Manually set a bundle version. This can be passed more than once. ' 898 'format: --bundle-version pepper_24=24.0.1312.25', action='append') 899 options, args = parser.parse_args(args[1:]) 900 901 if (options.mailfrom is None) != (not options.mailto): 902 options.mailfrom = None 903 options.mailto = None 904 logger.warning('Disabling email, one of --mailto or --mailfrom ' 905 'was missing.\n') 906 907 if options.verbose >= 2: 908 logging.basicConfig(level=logging.DEBUG, filename=options.log_file) 909 elif options.verbose: 910 logging.basicConfig(level=logging.INFO, filename=options.log_file) 911 else: 912 logging.basicConfig(level=logging.WARNING, filename=options.log_file) 913 914 # Parse bundle versions. 915 fixed_bundle_versions = {} 916 if options.bundle_version: 917 for bundle_version_string in options.bundle_version: 918 bundle_name, version = bundle_version_string.split('=') 919 fixed_bundle_versions[bundle_name] = version 920 921 if options.mailfrom and options.mailto: 922 # Capture stderr so it can be emailed, if necessary. 923 sys.stderr = CapturedFile(sys.stderr) 924 925 try: 926 try: 927 delegate = RealDelegate(options.dryrun, options.gsutil, 928 options.mailfrom, options.mailto) 929 930 if options.upload_log: 931 gsutil_logging_handler = GsutilLoggingHandler(delegate) 932 logger.addHandler(gsutil_logging_handler) 933 934 # Only look for naclports archives >= 27. The old ports bundles don't 935 # include license information. 936 extra_archives = [('naclports.tar.bz2', '27.0.0.0')] 937 Run(delegate, ('mac', 'win', 'linux'), extra_archives, 938 fixed_bundle_versions) 939 return 0 940 except Exception: 941 if options.mailfrom and options.mailto: 942 traceback.print_exc() 943 scriptname = os.path.basename(sys.argv[0]) 944 subject = '[%s] Failed to update manifest' % (scriptname,) 945 text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname, 946 sys.stderr.getvalue()) 947 delegate.SendMail(subject, text) 948 return 1 949 else: 950 raise 951 finally: 952 if options.upload_log: 953 gsutil_logging_handler.upload() 954 except manifest_util.Error as e: 955 if options.debug: 956 raise 957 sys.stderr.write(str(e) + '\n') 958 return 1 959 960 961if __name__ == '__main__': 962 sys.exit(main(sys.argv)) 963