12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#!/usr/bin/env python 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)"""Script that reads omahaproxy and gsutil to determine a version of the 72a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)sdk_tools bundle to use. 82a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 92a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)Please note the differences between this script and update_nacl_manifest.py: 102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)update_sdktools.py is run by a SDK-team developer to assist in updating to a 122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)new sdk_tools bundle. A file on the developer's hard drive is modified, and 132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)must be checked in for the new sdk_tools bundle to be used. 142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)update_nacl_manifest.py is customarily run by a cron job, and does not check in 165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)any changes. Instead it modifies the manifest file in cloud storage.""" 172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import collections 202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import difflib 212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import json 222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import optparse 232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import re 242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import sys 252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import urllib2 262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from manifest_util import DownloadAndComputeHash, DictToJSON 282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from update_nacl_manifest import RealDelegate 292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)SDK_TOOLS_DESCRIPTION_FORMAT = 'Native Client SDK Tools, revision %d' 322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)BUCKET_PATH = 'nativeclient-mirror/nacl/nacl_sdk/' 332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)GS_BUCKET_PATH = 'gs://' + BUCKET_PATH 345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)HTTPS_BUCKET_PATH = 'https://storage.googleapis.com/' + BUCKET_PATH 352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def GetSdkToolsUrl(revision): 382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return HTTPS_BUCKET_PATH + 'trunk.%d/sdk_tools.tgz' % revision 392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def GetTrunkRevisions(delegate): 422a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) urls = delegate.GsUtil_ls(GS_BUCKET_PATH) 432a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) revisions = [] 442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for url in urls: 452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) m = re.match(GS_BUCKET_PATH + 'trunk\.(\d+)', url) 462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if m: 472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) revisions.append((int(m.group(1)), url)) 482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return sorted(revisions) 492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def FindMostRecentSdkTools(delegate): 522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for revision, url in reversed(GetTrunkRevisions(delegate)): 532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sdktools_url = url + 'sdk_tools.tgz' 542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if delegate.GsUtil_ls(sdktools_url): 552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return revision, sdktools_url 562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return None 572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def JsonLoadFromString(json_string): 602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if sys.version_info > (2, 7): 612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return json.loads(json_string, object_pairs_hook=collections.OrderedDict) 622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) else: 632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return json.loads(json_string) 642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def GetBundleByName(bundles, name): 672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for bundle in bundles: 682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if bundle['name'] == name: 692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return bundle 702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return None 712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def UpdateSdkToolsBundle(sdk_tools_bundle, revision, url, sha1, size): 742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sdk_tools_bundle['description'] = SDK_TOOLS_DESCRIPTION_FORMAT % revision 752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sdk_tools_bundle['revision'] = revision 762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Update archive for each OS 772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for archive in sdk_tools_bundle['archives']: 782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) archive['url'] = url 792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) archive['checksum']['sha1'] = sha1 802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) archive['size'] = size 812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def UpdateManifest(manifest, revision): 842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sdk_tools_bundle = GetBundleByName(manifest['bundles'], 'sdk_tools') 852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) url = GetSdkToolsUrl(revision) 862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sha1, size = DownloadAndComputeHash(urllib2.urlopen(url)) 872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) UpdateSdkToolsBundle(sdk_tools_bundle, revision, url, sha1, size) 882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def UpdateManifestFileToRevision(filename, revision): 912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) with open(filename) as stream: 922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) manifest_string = stream.read() 932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) manifest = JsonLoadFromString(manifest_string) 952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) UpdateManifest(manifest, revision) 962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) new_manifest_string = DictToJSON(manifest) 972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) diff_string = ''.join(difflib.unified_diff(manifest_string.splitlines(1), 992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) new_manifest_string.splitlines(1))) 1002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) print 'diff %s' % filename 1022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) print diff_string 1032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) print 1042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) with open(filename, 'w') as stream: 1062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) stream.write(new_manifest_string) 1072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1082a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1092a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def main(args): 1104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) parser = optparse.OptionParser(description=__doc__) 1112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.add_option('-r', '--revision', 1122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) help='set revision manually, rather than using the latest version') 1132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) options, args = parser.parse_args(args[1:]) 1142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if len(args) != 0: 1152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.error('Unexpected args: %s' % ', '.join(args)) 1162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # TODO(binji): http://crbug.com/169047. Rename RealDelegate to something else. 1182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) delegate = RealDelegate() 1192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if not options.revision: 1202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) revision, _ = FindMostRecentSdkTools(delegate) 1212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) else: 1222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) revision = int(options.revision) 1232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) UpdateManifestFileToRevision('json/naclsdk_manifest0.json', revision) 1252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) UpdateManifestFileToRevision('json/naclsdk_manifest2.json', revision) 1262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)if __name__ == '__main__': 1292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sys.exit(main(sys.argv)) 130