1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Script that reads omahaproxy and gsutil to determine version of SDK to put
7in manifest.
8"""
9
10# pylint is convinced the email module is missing attributes
11# pylint: disable=E1101
12
13import buildbot_common
14import csv
15import cStringIO
16import difflib
17import email
18import logging
19import logging.handlers
20import manifest_util
21import optparse
22import os
23import posixpath
24import re
25import smtplib
26import subprocess
27import sys
28import time
29import traceback
30import urllib2
31
32MANIFEST_BASENAME = 'naclsdk_manifest2.json'
33SCRIPT_DIR = os.path.dirname(__file__)
34REPO_MANIFEST = os.path.join(SCRIPT_DIR, 'json', MANIFEST_BASENAME)
35GS_BUCKET_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
36GS_SDK_MANIFEST = GS_BUCKET_PATH + MANIFEST_BASENAME
37GS_SDK_MANIFEST_LOG = GS_BUCKET_PATH + MANIFEST_BASENAME + '.log'
38GS_MANIFEST_BACKUP_DIR = GS_BUCKET_PATH + 'manifest_backups/'
39
40CANARY_BUNDLE_NAME = 'pepper_canary'
41BIONIC_CANARY_BUNDLE_NAME = 'bionic_canary'
42CANARY = 'canary'
43NACLPORTS_ARCHIVE_NAME = 'naclports.tar.bz2'
44
45
46logger = logging.getLogger(__name__)
47
48
49def SplitVersion(version_string):
50  """Split a version string (e.g. "18.0.1025.163") into its components.
51
52  e.g.
53  SplitVersion("trunk.123456") => ("trunk", "123456")
54  SplitVersion("18.0.1025.163") => (18, 0, 1025, 163)
55  """
56  parts = version_string.split('.')
57  if parts[0] == 'trunk':
58    return (parts[0], int(parts[1]))
59  return tuple([int(p) for p in parts])
60
61
62def GetMajorVersion(version_string):
63  """Get the major version number from a version string (e.g. "18.0.1025.163").
64
65  e.g.
66  GetMajorVersion("trunk.123456") => "trunk"
67  GetMajorVersion("18.0.1025.163") => 18
68  """
69  return SplitVersion(version_string)[0]
70
71
72def CompareVersions(version1, version2):
73  """Compare two version strings and return -1, 0, 1 (similar to cmp).
74
75  Versions can only be compared if they are both trunk versions, or neither is.
76
77  e.g.
78  CompareVersions("trunk.123", "trunk.456") => -1
79  CompareVersions("18.0.1025.163", "37.0.2054.3") => -1
80  CompareVersions("trunk.123", "18.0.1025.163") => Error
81
82  """
83  split1 = SplitVersion(version1)
84  split2 = SplitVersion(version2)
85  if split1[0] == split2[0]:
86    return cmp(split1[1:], split2[1:])
87
88  if split1 == 'trunk' or split2 == 'trunk':
89    raise Exception("Unable to compare versions %s and %s" % (
90      version1, version2))
91
92  return cmp(split1, split2)
93
94
95def JoinVersion(version_tuple):
96  """Create a string from a version tuple.
97
98  The tuple should be of the form (18, 0, 1025, 163).
99  """
100  assert len(version_tuple) == 4
101  assert version_tuple[0] != 'trunk'
102  return '.'.join(map(str, version_tuple))
103
104
105def GetTimestampManifestName():
106  """Create a manifest name with a timestamp.
107
108  Returns:
109    A manifest name with an embedded date. This should make it easier to roll
110    back if necessary.
111  """
112  return time.strftime('naclsdk_manifest2.%Y_%m_%d_%H_%M_%S.json',
113      time.gmtime())
114
115
116def GetPlatformArchiveName(platform):
117  """Get the basename of an archive given a platform string.
118
119  Args:
120    platform: One of ('win', 'mac', 'linux').
121
122  Returns:
123    The basename of the sdk archive for that platform.
124  """
125  return 'naclsdk_%s.tar.bz2' % platform
126
127
128def GetBionicArchiveName():
129  """Get the basename of an archive. Currently this is linux-only"""
130  return 'naclsdk_bionic.tar.bz2'
131
132
133def GetCanonicalArchiveName(url):
134  """Get the canonical name of an archive given its URL.
135
136  This will convert "naclsdk_linux.bz2" -> "naclsdk_linux.tar.bz2", and also
137  remove everything but the filename of the URL.
138
139  This is used below to determine if an expected bundle is found in an version
140  directory; the archives all have the same name, but may not exist for a given
141  version.
142
143  Args:
144    url: The url to parse.
145
146  Returns:
147    The canonical name as described above.
148  """
149  name = posixpath.basename(url)
150  match = re.match(r'naclsdk_(.*?)(?:\.tar)?\.bz2', name)
151  if match:
152    return 'naclsdk_%s.tar.bz2' % match.group(1)
153
154  return name
155
156
157class Delegate(object):
158  """Delegate all external access; reading/writing to filesystem, gsutil etc."""
159
160  def GetRepoManifest(self):
161    """Read the manifest file from the NaCl SDK repository.
162
163    This manifest is used as a template for the auto updater; only pepper
164    bundles with no archives are considered for auto updating.
165
166    Returns:
167      A manifest_util.SDKManifest object read from the NaCl SDK repo."""
168    raise NotImplementedError()
169
170  def GetHistory(self):
171    """Read Chrome release history from omahaproxy.appspot.com
172
173    Here is an example of data from this URL:
174      cros,stable,18.0.1025.168,2012-05-01 17:04:05.962578\n
175      win,canary,20.0.1123.0,2012-05-01 13:59:31.703020\n
176      mac,canary,20.0.1123.0,2012-05-01 11:54:13.041875\n
177      win,stable,18.0.1025.168,2012-04-30 20:34:56.078490\n
178      mac,stable,18.0.1025.168,2012-04-30 20:34:55.231141\n
179      ...
180    Where each line has comma separated values in the following format:
181    platform, channel, version, date/time\n
182
183    Returns:
184      A list where each element is a line from the document, represented as a
185      tuple."""
186    raise NotImplementedError()
187
188  def GsUtil_ls(self, url):
189    """Runs gsutil ls |url|
190
191    Args:
192      url: The cloud storage url to list.
193    Returns:
194      A list of URLs, all with the gs:// schema."""
195    raise NotImplementedError()
196
197  def GsUtil_cat(self, url):
198    """Runs gsutil cat |url|
199
200    Args:
201      url: The cloud storage url to read from.
202    Returns:
203      A string with the contents of the file at |url|."""
204    raise NotImplementedError()
205
206  def GsUtil_cp(self, src, dest, stdin=None):
207    """Runs gsutil cp |src| |dest|
208
209    Args:
210      src: The file path or url to copy from.
211      dest: The file path or url to copy to.
212      stdin: If src is '-', this is used as the stdin to give to gsutil. The
213          effect is that text in stdin is copied to |dest|."""
214    raise NotImplementedError()
215
216  def SendMail(self, subject, text):
217    """Send an email.
218
219    Args:
220      subject: The subject of the email.
221      text: The text of the email.
222    """
223    raise NotImplementedError()
224
225
226class RealDelegate(Delegate):
227  def __init__(self, dryrun=False, gsutil=None, mailfrom=None, mailto=None):
228    super(RealDelegate, self).__init__()
229    self.dryrun = dryrun
230    self.mailfrom = mailfrom
231    self.mailto = mailto
232    if gsutil:
233      self.gsutil = gsutil
234    else:
235      self.gsutil = buildbot_common.GetGsutil()
236
237  def GetRepoManifest(self):
238    """See Delegate.GetRepoManifest"""
239    with open(REPO_MANIFEST, 'r') as sdk_stream:
240      sdk_json_string = sdk_stream.read()
241
242    manifest = manifest_util.SDKManifest()
243    manifest.LoadDataFromString(sdk_json_string, add_missing_info=True)
244    return manifest
245
246  def GetHistory(self):
247    """See Delegate.GetHistory"""
248    url_stream = urllib2.urlopen('https://omahaproxy.appspot.com/history')
249    history = [(platform, channel, version, date)
250        for platform, channel, version, date in csv.reader(url_stream)]
251
252    # The first line of this URL is the header:
253    #   os,channel,version,timestamp
254    return history[1:]
255
256  def GsUtil_ls(self, url):
257    """See Delegate.GsUtil_ls"""
258    try:
259      stdout = self._RunGsUtil(None, False, 'ls', url)
260    except subprocess.CalledProcessError:
261      return []
262
263    # filter out empty lines
264    return filter(None, stdout.split('\n'))
265
266  def GsUtil_cat(self, url):
267    """See Delegate.GsUtil_cat"""
268    return self._RunGsUtil(None, True, 'cat', url)
269
270  def GsUtil_cp(self, src, dest, stdin=None):
271    """See Delegate.GsUtil_cp"""
272    if self.dryrun:
273      logger.info("Skipping upload: %s -> %s" % (src, dest))
274      if src == '-':
275        logger.info('  contents = """%s"""' % stdin)
276      return
277
278    return self._RunGsUtil(stdin, True, 'cp', '-a', 'public-read', src, dest)
279
280  def SendMail(self, subject, text):
281    """See Delegate.SendMail"""
282    if self.mailfrom and self.mailto:
283      msg = email.MIMEMultipart.MIMEMultipart()
284      msg['From'] = self.mailfrom
285      msg['To'] = ', '.join(self.mailto)
286      msg['Date'] = email.Utils.formatdate(localtime=True)
287      msg['Subject'] = subject
288      msg.attach(email.MIMEText.MIMEText(text))
289      smtp_obj = smtplib.SMTP('localhost')
290      smtp_obj.sendmail(self.mailfrom, self.mailto, msg.as_string())
291      smtp_obj.close()
292
293  def _RunGsUtil(self, stdin, log_errors, *args):
294    """Run gsutil as a subprocess.
295
296    Args:
297      stdin: If non-None, used as input to the process.
298      log_errors: If True, write errors to stderr.
299      *args: Arguments to pass to gsutil. The first argument should be an
300          operation such as ls, cp or cat.
301    Returns:
302      The stdout from the process."""
303    cmd = [self.gsutil] + list(args)
304    logger.debug("Running: %s" % str(cmd))
305    if stdin:
306      stdin_pipe = subprocess.PIPE
307    else:
308      stdin_pipe = None
309
310    try:
311      process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE,
312          stderr=subprocess.PIPE)
313      stdout, stderr = process.communicate(stdin)
314    except OSError as e:
315      raise manifest_util.Error("Unable to run '%s': %s" % (cmd[0], str(e)))
316
317    if process.returncode:
318      if log_errors:
319        logger.error(stderr)
320      raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd))
321    return stdout
322
323
324class GsutilLoggingHandler(logging.handlers.BufferingHandler):
325  def __init__(self, delegate):
326    logging.handlers.BufferingHandler.__init__(self, capacity=0)
327    self.delegate = delegate
328
329  def shouldFlush(self, record):
330    # BufferingHandler.shouldFlush automatically flushes if the length of the
331    # buffer is greater than self.capacity. We don't want that behavior, so
332    # return False here.
333    return False
334
335  def flush(self):
336    # Do nothing here. We want to be explicit about uploading the log.
337    pass
338
339  def upload(self):
340    output_list = []
341    for record in self.buffer:
342      output_list.append(self.format(record))
343    output = '\n'.join(output_list)
344    self.delegate.GsUtil_cp('-', GS_SDK_MANIFEST_LOG, stdin=output)
345
346    logging.handlers.BufferingHandler.flush(self)
347
348
349class NoSharedVersionException(Exception):
350  pass
351
352
353class VersionFinder(object):
354  """Finds a version of a pepper bundle that all desired platforms share.
355
356  Args:
357    delegate: See Delegate class above.
358    platforms: A sequence of platforms to consider, e.g.
359        ('mac', 'linux', 'win')
360    extra_archives: A sequence of tuples: (archive_basename, minimum_version),
361        e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')]
362        These archives must exist to consider a version for inclusion, as
363        long as that version is greater than the archive's minimum version.
364    is_bionic: True if we are searching for bionic archives.
365  """
366  def __init__(self, delegate, platforms, extra_archives=None, is_bionic=False):
367    self.delegate = delegate
368    self.history = delegate.GetHistory()
369    self.platforms = platforms
370    self.extra_archives = extra_archives
371    self.is_bionic = is_bionic
372
373  def GetMostRecentSharedVersion(self, major_version):
374    """Returns the most recent version of a pepper bundle that exists on all
375    given platforms.
376
377    Specifically, the resulting version should be the most recently released
378    (meaning closest to the top of the listing on
379    omahaproxy.appspot.com/history) version that has a Chrome release on all
380    given platforms, and has a pepper bundle archive for each platform as well.
381
382    Args:
383      major_version: The major version of the pepper bundle, e.g. 19.
384    Returns:
385      A tuple (version, channel, archives). The version is a string such as
386      "19.0.1084.41". The channel is one of ('stable', 'beta', or 'dev').
387      |archives| is a list of archive URLs."""
388    def GetPlatformHistory(platform):
389      return self._GetPlatformMajorVersionHistory(major_version, platform)
390
391    shared_version_generator = self._FindNextSharedVersion(self.platforms,
392                                                           GetPlatformHistory)
393    return self._DoGetMostRecentSharedVersion(shared_version_generator)
394
395  def GetMostRecentSharedCanary(self):
396    """Returns the most recent version of a canary pepper bundle that exists on
397    all given platforms.
398
399    Canary is special-cased because we don't care about its major version; we
400    always use the most recent canary, regardless of major version.
401
402    Returns:
403      A tuple (version, channel, archives). The version is a string such as
404      "trunk.123456". The channel is always 'canary'. |archives| is a list of
405      archive URLs."""
406    version_generator = self._FindNextTrunkVersion()
407    return self._DoGetMostRecentSharedVersion(version_generator)
408
409  def GetAvailablePlatformArchivesFor(self, version):
410    """Returns a sequence of archives that exist for a given version, on the
411    given platforms.
412
413    The second element of the returned tuple is a list of all platforms that do
414    not have an archive for the given version.
415
416    Args:
417      version: The version to find archives for. (e.g. "18.0.1025.164")
418    Returns:
419      A tuple (archives, missing_archives). |archives| is a list of archive
420      URLs, |missing_archives| is a list of archive names.
421    """
422    archive_urls = self._GetAvailableArchivesFor(version)
423
424    if self.is_bionic:
425      # Bionic currently is Linux-only.
426      expected_archives = set([GetBionicArchiveName()])
427    else:
428      expected_archives = set(GetPlatformArchiveName(p) for p in self.platforms)
429
430    if self.extra_archives:
431      for extra_archive, extra_archive_min_version in self.extra_archives:
432        if CompareVersions(version, extra_archive_min_version) >= 0:
433          expected_archives.add(extra_archive)
434    found_archives = set(GetCanonicalArchiveName(a) for a in archive_urls)
435    missing_archives = expected_archives - found_archives
436
437    # Only return archives that are "expected".
438    def IsExpected(url):
439      return GetCanonicalArchiveName(url) in expected_archives
440
441    expected_archive_urls = [u for u in archive_urls if IsExpected(u)]
442    return expected_archive_urls, missing_archives
443
444  def _DoGetMostRecentSharedVersion(self, shared_version_generator):
445    """Returns the most recent version of a pepper bundle that exists on all
446    given platforms.
447
448    This function does the real work for the public GetMostRecentShared* above.
449
450    Args:
451      shared_version_generator: A generator that will yield (version, channel)
452          tuples in order of most recent to least recent.
453    Returns:
454      A tuple (version, channel, archives). The version is a string such as
455      "19.0.1084.41". The channel is one of ('stable', 'beta', 'dev',
456      'canary'). |archives| is a list of archive URLs."""
457    version = None
458    skipped_versions = []
459    channel = ''
460    while True:
461      try:
462        version, channel = shared_version_generator.next()
463      except StopIteration:
464        msg = 'No shared version for platforms: %s\n' % (
465            ', '.join(self.platforms))
466        msg += 'Last version checked = %s.\n' % (version,)
467        if skipped_versions:
468          msg += 'Versions skipped due to missing archives:\n'
469          for version, channel, missing_archives in skipped_versions:
470            archive_msg = '(missing %s)' % (', '.join(missing_archives))
471          msg += '  %s (%s) %s\n' % (version, channel, archive_msg)
472        raise NoSharedVersionException(msg)
473
474      logger.info('Found shared version: %s, channel: %s' % (
475          version, channel))
476
477      archives, missing_archives = self.GetAvailablePlatformArchivesFor(version)
478
479      if not missing_archives:
480        return version, channel, archives
481
482      logger.info('  skipping. Missing archives: %s' % (
483          ', '.join(missing_archives)))
484
485      skipped_versions.append((version, channel, missing_archives))
486
487  def _GetPlatformMajorVersionHistory(self, with_major_version, with_platform):
488    """Yields Chrome history for a given platform and major version.
489
490    Args:
491      with_major_version: The major version to filter for. If 0, match all
492          versions.
493      with_platform: The name of the platform to filter for.
494    Returns:
495      A generator that yields a tuple (channel, version) for each version that
496      matches the platform and major version. The version returned is a tuple as
497      returned from SplitVersion.
498    """
499    for platform, channel, version, _ in self.history:
500      version = SplitVersion(version)
501      if (with_platform == platform and
502          (with_major_version == 0 or with_major_version == version[0])):
503        yield channel, version
504
505  def _FindNextSharedVersion(self, platforms, generator_func):
506    """Yields versions of Chrome that exist on all given platforms, in order of
507       newest to oldest.
508
509    Versions are compared in reverse order of release. That is, the most
510    recently updated version will be tested first.
511
512    Args:
513      platforms: A sequence of platforms to consider, e.g.
514          ('mac', 'linux', 'win')
515      generator_func: A function which takes a platform and returns a
516          generator that yields (channel, version) tuples.
517    Returns:
518      A generator that yields a tuple (version, channel) for each version that
519      matches all platforms and the major version. The version returned is a
520      string (e.g. "18.0.1025.164").
521    """
522    platform_generators = []
523    for platform in platforms:
524      platform_generators.append(generator_func(platform))
525
526    shared_version = None
527    platform_versions = []
528    for platform_gen in platform_generators:
529      platform_versions.append(platform_gen.next())
530
531    while True:
532      if logger.isEnabledFor(logging.INFO):
533        msg_info = []
534        for i, platform in enumerate(platforms):
535          msg_info.append('%s: %s' % (
536              platform, JoinVersion(platform_versions[i][1])))
537        logger.info('Checking versions: %s' % ', '.join(msg_info))
538
539      shared_version = min((v for c, v in platform_versions))
540
541      if all(v == shared_version for c, v in platform_versions):
542        # grab the channel from an arbitrary platform
543        first_platform = platform_versions[0]
544        channel = first_platform[0]
545        yield JoinVersion(shared_version), channel
546
547        # force increment to next version for all platforms
548        shared_version = None
549
550      # Find the next version for any platform that isn't at the shared version.
551      try:
552        for i, platform_gen in enumerate(platform_generators):
553          if platform_versions[i][1] != shared_version:
554            platform_versions[i] = platform_gen.next()
555      except StopIteration:
556        return
557
558
559  def _FindNextTrunkVersion(self):
560    """Yields all trunk versions that exist in the cloud storage bucket, newest
561    to oldest.
562
563    Returns:
564      A generator that yields a tuple (version, channel) for each version that
565      matches all platforms and the major version. The version returned is a
566      string (e.g. "trunk.123456").
567    """
568    files = self.delegate.GsUtil_ls(GS_BUCKET_PATH)
569    assert all(f.startswith('gs://') for f in files)
570
571    trunk_versions = []
572    for f in files:
573      match = re.search(r'(trunk\.\d+)', f)
574      if match:
575        trunk_versions.append(match.group(1))
576
577    trunk_versions.sort(reverse=True)
578
579    for version in trunk_versions:
580      yield version, 'canary'
581
582
583  def _GetAvailableArchivesFor(self, version_string):
584    """Downloads a list of all available archives for a given version.
585
586    Args:
587      version_string: The version to find archives for. (e.g. "18.0.1025.164")
588    Returns:
589      A list of strings, each of which is a platform-specific archive URL. (e.g.
590      "gs://nativeclient_mirror/nacl/nacl_sdk/18.0.1025.164/"
591      "naclsdk_linux.tar.bz2").
592
593      All returned URLs will use the gs:// schema."""
594    files = self.delegate.GsUtil_ls(GS_BUCKET_PATH + version_string)
595
596    assert all(f.startswith('gs://') for f in files)
597
598    archives = [f for f in files if not f.endswith('.json')]
599    manifests = [f for f in files if f.endswith('.json')]
600
601    # don't include any archives that don't have an associated manifest.
602    return filter(lambda a: a + '.json' in manifests, archives)
603
604
605class UnknownLockedBundleException(Exception):
606  pass
607
608
609class Updater(object):
610  def __init__(self, delegate):
611    self.delegate = delegate
612    self.versions_to_update = []
613    self.locked_bundles = []
614    self.online_manifest = manifest_util.SDKManifest()
615    self._FetchOnlineManifest()
616
617  def AddVersionToUpdate(self, bundle_name, version, channel, archives):
618    """Add a pepper version to update in the uploaded manifest.
619
620    Args:
621      bundle_name: The name of the pepper bundle, e.g. 'pepper_18'
622      version: The version of the pepper bundle, e.g. '18.0.1025.64'
623      channel: The stability of the pepper bundle, e.g. 'beta'
624      archives: A sequence of archive URLs for this bundle."""
625    self.versions_to_update.append((bundle_name, version, channel, archives))
626
627  def AddLockedBundle(self, bundle_name):
628    """Add a "locked" bundle to the updater.
629
630    A locked bundle is a bundle that wasn't found in the history. When this
631    happens, the bundle is now "locked" to whatever was last found. We want to
632    ensure that the online manifest has this bundle.
633
634    Args:
635      bundle_name: The name of the locked bundle.
636    """
637    self.locked_bundles.append(bundle_name)
638
639  def Update(self, manifest):
640    """Update a manifest and upload it.
641
642    Note that bundles will not be updated if the current version is newer.
643    That is, the updater will never automatically update to an older version of
644    a bundle.
645
646    Args:
647      manifest: The manifest used as a template for updating. Only pepper
648      bundles that contain no archives will be considered for auto-updating."""
649    # Make sure there is only one stable branch: the one with the max version.
650    # All others are post-stable.
651    stable_major_versions = [GetMajorVersion(version) for _, version, channel, _
652                             in self.versions_to_update if channel == 'stable']
653    # Add 0 in case there are no stable versions.
654    max_stable_version = max([0] + stable_major_versions)
655
656    # Ensure that all locked bundles exist in the online manifest.
657    for bundle_name in self.locked_bundles:
658      online_bundle = self.online_manifest.GetBundle(bundle_name)
659      if online_bundle:
660        manifest.SetBundle(online_bundle)
661      else:
662        msg = ('Attempted to update bundle "%s", but no shared versions were '
663            'found, and there is no online bundle with that name.')
664        raise UnknownLockedBundleException(msg % bundle_name)
665
666    if self.locked_bundles:
667      # Send a nagging email that we shouldn't be wasting time looking for
668      # bundles that are no longer in the history.
669      scriptname = os.path.basename(sys.argv[0])
670      subject = '[%s] Reminder: remove bundles from %s' % (scriptname,
671                                                           MANIFEST_BASENAME)
672      text = 'These bundles are not in the omahaproxy history anymore: ' + \
673              ', '.join(self.locked_bundles)
674      self.delegate.SendMail(subject, text)
675
676
677    # Update all versions.
678    logger.info('>>> Updating bundles...')
679    for bundle_name, version, channel, archives in self.versions_to_update:
680      logger.info('Updating %s to %s...' % (bundle_name, version))
681      bundle = manifest.GetBundle(bundle_name)
682      for archive in archives:
683        platform_bundle = self._GetPlatformArchiveBundle(archive)
684        # Normally the manifest snippet's bundle name matches our bundle name.
685        # pepper_canary, however is called "pepper_###" in the manifest
686        # snippet.
687        platform_bundle.name = bundle_name
688        bundle.MergeWithBundle(platform_bundle)
689
690      # Fix the stability and recommended values
691      major_version = GetMajorVersion(version)
692      if major_version < max_stable_version:
693        bundle.stability = 'post_stable'
694      else:
695        bundle.stability = channel
696      # We always recommend the stable version.
697      if bundle.stability == 'stable':
698        bundle.recommended = 'yes'
699      else:
700        bundle.recommended = 'no'
701
702      # Check to ensure this bundle is newer than the online bundle.
703      online_bundle = self.online_manifest.GetBundle(bundle_name)
704      if online_bundle:
705        # This test used to be online_bundle.revision >= bundle.revision.
706        # That doesn't do quite what we want: sometimes the metadata changes
707        # but the revision stays the same -- we still want to push those
708        # changes.
709        if online_bundle.revision > bundle.revision or online_bundle == bundle:
710          logger.info(
711              '  Revision %s is not newer than than online revision %s. '
712              'Skipping.' % (bundle.revision, online_bundle.revision))
713
714          manifest.SetBundle(online_bundle)
715          continue
716    self._UploadManifest(manifest)
717    logger.info('Done.')
718
719  def _GetPlatformArchiveBundle(self, archive):
720    """Downloads the manifest "snippet" for an archive, and reads it as a
721       Bundle.
722
723    Args:
724      archive: A full URL of a platform-specific archive, using the gs schema.
725    Returns:
726      An object of type manifest_util.Bundle, read from a JSON file storing
727      metadata for this archive.
728    """
729    stdout = self.delegate.GsUtil_cat(archive + '.json')
730    bundle = manifest_util.Bundle('')
731    bundle.LoadDataFromString(stdout)
732    # Some snippets were uploaded with revisions and versions as strings. Fix
733    # those here.
734    bundle.revision = int(bundle.revision)
735    bundle.version = int(bundle.version)
736
737    # HACK. The naclports archive specifies host_os as linux. Change it to all.
738    for archive in bundle.GetArchives():
739      if NACLPORTS_ARCHIVE_NAME in archive.url:
740        archive.host_os = 'all'
741    return bundle
742
743  def _UploadManifest(self, manifest):
744    """Upload a serialized manifest_util.SDKManifest object.
745
746    Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to
747    gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json.
748
749    Args:
750      manifest: The new manifest to upload.
751    """
752    new_manifest_string = manifest.GetDataAsString()
753    online_manifest_string = self.online_manifest.GetDataAsString()
754
755    if self.delegate.dryrun:
756      logger.info(''.join(list(difflib.unified_diff(
757          online_manifest_string.splitlines(1),
758          new_manifest_string.splitlines(1)))))
759      return
760    else:
761      online_manifest = manifest_util.SDKManifest()
762      online_manifest.LoadDataFromString(online_manifest_string)
763
764      if online_manifest == manifest:
765        logger.info('New manifest doesn\'t differ from online manifest.'
766            'Skipping upload.')
767        return
768
769    timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \
770        GetTimestampManifestName()
771    self.delegate.GsUtil_cp('-', timestamp_manifest_path,
772        stdin=manifest.GetDataAsString())
773
774    # copy from timestampped copy over the official manifest.
775    self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST)
776
777  def _FetchOnlineManifest(self):
778    try:
779      online_manifest_string = self.delegate.GsUtil_cat(GS_SDK_MANIFEST)
780    except subprocess.CalledProcessError:
781      # It is not a failure if the online manifest doesn't exist.
782      online_manifest_string = ''
783
784    if online_manifest_string:
785      self.online_manifest.LoadDataFromString(online_manifest_string)
786
787
788def Run(delegate, platforms, extra_archives, fixed_bundle_versions=None):
789  """Entry point for the auto-updater.
790
791  Args:
792    delegate: The Delegate object to use for reading Urls, files, etc.
793    platforms: A sequence of platforms to consider, e.g.
794        ('mac', 'linux', 'win')
795      extra_archives: A sequence of tuples: (archive_basename, minimum_version),
796          e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')]
797          These archives must exist to consider a version for inclusion, as
798          long as that version is greater than the archive's minimum version.
799    fixed_bundle_versions: A sequence of tuples (bundle_name, version_string).
800        e.g. ('pepper_21', '21.0.1145.0')
801  """
802  if fixed_bundle_versions:
803    fixed_bundle_versions = dict(fixed_bundle_versions)
804  else:
805    fixed_bundle_versions = {}
806
807  manifest = delegate.GetRepoManifest()
808  auto_update_bundles = []
809  for bundle in manifest.GetBundles():
810    if not bundle.name.startswith(('pepper_', 'bionic_')):
811      continue
812    archives = bundle.GetArchives()
813    if not archives:
814      auto_update_bundles.append(bundle)
815
816  if not auto_update_bundles:
817    logger.info('No versions need auto-updating.')
818    return
819
820  updater = Updater(delegate)
821
822  for bundle in auto_update_bundles:
823    try:
824      if bundle.name == BIONIC_CANARY_BUNDLE_NAME:
825        logger.info('>>> Looking for most recent bionic_canary...')
826        # Ignore extra_archives on bionic; There is no naclports bundle yet.
827        version_finder = VersionFinder(delegate, platforms, None,
828                                       is_bionic=True)
829        version, channel, archives = version_finder.GetMostRecentSharedCanary()
830      elif bundle.name == CANARY_BUNDLE_NAME:
831        logger.info('>>> Looking for most recent pepper_canary...')
832        version_finder = VersionFinder(delegate, platforms, extra_archives)
833        version, channel, archives = version_finder.GetMostRecentSharedCanary()
834      else:
835        logger.info('>>> Looking for most recent pepper_%s...' %
836            bundle.version)
837        version_finder = VersionFinder(delegate, platforms, extra_archives)
838        version, channel, archives = version_finder.GetMostRecentSharedVersion(
839            bundle.version)
840    except NoSharedVersionException:
841      # If we can't find a shared version, make sure that there is an uploaded
842      # bundle with that name already.
843      updater.AddLockedBundle(bundle.name)
844      continue
845
846    if bundle.name in fixed_bundle_versions:
847      # Ensure this version is valid for all platforms.
848      # If it is, use the channel found above (because the channel for this
849      # version may not be in the history.)
850      version = fixed_bundle_versions[bundle.name]
851      logger.info('Fixed bundle version: %s, %s' % (bundle.name, version))
852      archives, missing = \
853          version_finder.GetAvailablePlatformArchivesFor(version)
854      if missing:
855        logger.warn(
856            'Some archives for version %s of bundle %s don\'t exist: '
857            'Missing %s' % (version, bundle.name, ', '.join(missing)))
858        return
859
860    updater.AddVersionToUpdate(bundle.name, version, channel, archives)
861
862  updater.Update(manifest)
863
864
865class CapturedFile(object):
866  """A file-like object that captures text written to it, but also passes it
867  through to an underlying file-like object."""
868  def __init__(self, passthrough):
869    self.passthrough = passthrough
870    self.written = cStringIO.StringIO()
871
872  def write(self, s):
873    self.written.write(s)
874    if self.passthrough:
875      self.passthrough.write(s)
876
877  def getvalue(self):
878    return self.written.getvalue()
879
880
881def main(args):
882  parser = optparse.OptionParser()
883  parser.add_option('--gsutil', help='path to gsutil.')
884  parser.add_option('-d', '--debug', help='run in debug mode.',
885      action='store_true')
886  parser.add_option('--mailfrom', help='email address of sender.')
887  parser.add_option('--mailto', help='send error mails to...', action='append')
888  parser.add_option('-n', '--dryrun', help="don't upload the manifest.",
889      action='store_true')
890  parser.add_option('-v', '--verbose', help='print more diagnotic messages. '
891      'Use more than once for more info.',
892      action='count')
893  parser.add_option('--log-file', metavar='FILE', help='log to FILE')
894  parser.add_option('--upload-log', help='Upload log alongside the manifest.',
895                    action='store_true')
896  parser.add_option('--bundle-version',
897      help='Manually set a bundle version. This can be passed more than once. '
898      'format: --bundle-version pepper_24=24.0.1312.25', action='append')
899  options, args = parser.parse_args(args[1:])
900
901  if (options.mailfrom is None) != (not options.mailto):
902    options.mailfrom = None
903    options.mailto = None
904    logger.warning('Disabling email, one of --mailto or --mailfrom '
905        'was missing.\n')
906
907  if options.verbose >= 2:
908    logging.basicConfig(level=logging.DEBUG, filename=options.log_file)
909  elif options.verbose:
910    logging.basicConfig(level=logging.INFO, filename=options.log_file)
911  else:
912    logging.basicConfig(level=logging.WARNING, filename=options.log_file)
913
914  # Parse bundle versions.
915  fixed_bundle_versions = {}
916  if options.bundle_version:
917    for bundle_version_string in options.bundle_version:
918      bundle_name, version = bundle_version_string.split('=')
919      fixed_bundle_versions[bundle_name] = version
920
921  if options.mailfrom and options.mailto:
922    # Capture stderr so it can be emailed, if necessary.
923    sys.stderr = CapturedFile(sys.stderr)
924
925  try:
926    try:
927      delegate = RealDelegate(options.dryrun, options.gsutil,
928                              options.mailfrom, options.mailto)
929
930      if options.upload_log:
931        gsutil_logging_handler = GsutilLoggingHandler(delegate)
932        logger.addHandler(gsutil_logging_handler)
933
934      # Only look for naclports archives >= 27. The old ports bundles don't
935      # include license information.
936      extra_archives = [('naclports.tar.bz2', '27.0.0.0')]
937      Run(delegate, ('mac', 'win', 'linux'), extra_archives,
938          fixed_bundle_versions)
939      return 0
940    except Exception:
941      if options.mailfrom and options.mailto:
942        traceback.print_exc()
943        scriptname = os.path.basename(sys.argv[0])
944        subject = '[%s] Failed to update manifest' % (scriptname,)
945        text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname,
946                                                sys.stderr.getvalue())
947        delegate.SendMail(subject, text)
948        return 1
949      else:
950        raise
951    finally:
952      if options.upload_log:
953        gsutil_logging_handler.upload()
954  except manifest_util.Error as e:
955    if options.debug:
956      raise
957    sys.stderr.write(str(e) + '\n')
958    return 1
959
960
961if __name__ == '__main__':
962  sys.exit(main(sys.argv))
963