1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import hashlib
7import json
8import os
9import sys
10
11import buildbot_common
12import build_version
13from build_paths import SCRIPT_DIR
14
15GS_MANIFEST_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
16SDK_MANIFEST = 'naclsdk_manifest2.json'
17MONO_MANIFEST = 'naclmono_manifest.json'
18
19def build_and_upload_mono(sdk_revision, pepper_revision, sdk_url,
20                          upload_path, args):
21  install_dir = 'naclmono'
22  buildbot_common.RemoveDir(install_dir)
23
24  revision_opt = ['--sdk-revision', sdk_revision] if sdk_revision else []
25  url_opt = ['--sdk-url', sdk_url] if sdk_url else []
26
27  buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
28                      '--arch', 'x86-32', '--install-dir', install_dir] +
29                      revision_opt + url_opt + args)
30  buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
31                      '--arch', 'x86-64', '--install-dir', install_dir] +
32                      revision_opt + url_opt + args)
33  buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
34                      '--arch', 'arm', '--install-dir', install_dir] +
35                      revision_opt + url_opt + args)
36  buildbot_common.Run([sys.executable, 'nacl-mono-archive.py',
37                      '--upload-path', upload_path,
38                      '--pepper-revision', pepper_revision,
39                      '--install-dir', install_dir] + args)
40
41def get_sdk_build_info():
42  '''Returns a list of dictionaries for versions of NaCl Mono to build which are
43     out of date compared to the SDKs available to naclsdk'''
44
45  # Get a copy of the naclsdk manifest file
46  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
47      GS_MANIFEST_PATH + SDK_MANIFEST, '.'])
48  manifest_file = open(SDK_MANIFEST, 'r')
49  sdk_manifest = json.loads(manifest_file.read())
50  manifest_file.close()
51
52  pepper_infos = []
53  for key, value in sdk_manifest.items():
54    if key == 'bundles':
55      stabilities = ['stable', 'beta', 'dev', 'post_stable']
56      # Pick pepper_* bundles, need pepper_19 or greater to build Mono
57      bundles = filter(lambda b: (b['stability'] in stabilities
58                                  and 'pepper_' in b['name'])
59                                  and b['version'] >= 19, value)
60      for b in bundles:
61        newdict = {}
62        newdict['pepper_revision'] = str(b['version'])
63        linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives'])
64        newdict['sdk_url'] = linux_arch[0]['url']
65        newdict['sdk_revision'] = b['revision']
66        newdict['stability'] = b['stability']
67        newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision']
68        pepper_infos.append(newdict)
69
70  # Get a copy of the naclmono manifest file
71  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
72      GS_MANIFEST_PATH + MONO_MANIFEST, '.'])
73  manifest_file = open(MONO_MANIFEST, 'r')
74  mono_manifest = json.loads(manifest_file.read())
75  manifest_file.close()
76
77  ret = []
78  mono_manifest_dirty = False
79  # Check to see if we need to rebuild mono based on sdk revision
80  for key, value in mono_manifest.items():
81    if key == 'bundles':
82      for info in pepper_infos:
83        bundle = filter(lambda b: b['name'] == info['naclmono_name'], value)
84        if len(bundle) == 0:
85          info['naclmono_rev'] = '1'
86          ret.append(info)
87        else:
88          if info['sdk_revision'] != bundle[0]['sdk_revision']:
89            # This bundle exists in the mono manifest, bump the revision
90            # for the new build we're about to make.
91            info['naclmono_rev'] = str(bundle[0]['revision'] + 1)
92            ret.append(info)
93          elif info['stability'] != bundle[0]['stability']:
94            # If all that happened was the SDK bundle was promoted in stability,
95            # change only that and re-write the manifest
96            mono_manifest_dirty = True
97            bundle[0]['stability'] = info['stability']
98
99  # re-write the manifest here because there are no bundles to build but
100  # the manifest has changed
101  if mono_manifest_dirty and len(ret) == 0:
102    manifest_file = open(MONO_MANIFEST, 'w')
103    manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
104    manifest_file.close()
105    buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
106        MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
107
108  return ret
109
110def update_mono_sdk_json(infos):
111  '''Update the naclmono manifest with the newly built packages'''
112  if len(infos) == 0:
113    return
114
115  manifest_file = open(MONO_MANIFEST, 'r')
116  mono_manifest = json.loads(manifest_file.read())
117  manifest_file.close()
118
119  for info in infos:
120    bundle = {}
121    bundle['name'] = info['naclmono_name']
122    bundle['description'] = 'Mono for Native Client'
123    bundle['stability'] = info['stability']
124    bundle['recommended'] = 'no'
125    bundle['version'] = 'experimental'
126    archive = {}
127    sha1_hash = hashlib.sha1()
128    f = open(info['naclmono_name'] + '.bz2', 'rb')
129    sha1_hash.update(f.read())
130    archive['size'] = f.tell()
131    f.close()
132    archive['checksum'] = { 'sha1': sha1_hash.hexdigest() }
133    archive['host_os'] = 'all'
134    archive['url'] = ('https://storage.googleapis.com/'
135                      'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2'
136                      % (info['naclmono_name'], info['naclmono_rev'],
137                        info['naclmono_name']))
138    bundle['archives'] = [archive]
139    bundle['revision'] = int(info['naclmono_rev'])
140    bundle['sdk_revision'] = int(info['sdk_revision'])
141
142    # Insert this new bundle into the manifest,
143    # probably overwriting an existing bundle.
144    for key, value in mono_manifest.items():
145      if key == 'bundles':
146        existing = filter(lambda b: b['name'] == info['naclmono_name'], value)
147        if len(existing) > 0:
148          loc = value.index(existing[0])
149          value[loc] = bundle
150        else:
151          value.append(bundle)
152
153  # Write out the file locally, then upload to its known location.
154  manifest_file = open(MONO_MANIFEST, 'w')
155  manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
156  manifest_file.close()
157  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
158      MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
159
160
161def main(args):
162  args = args[1:]
163
164  # Delete global configs that would override the mono builders' configuration.
165  if 'AWS_CREDENTIAL_FILE' in os.environ:
166    del os.environ['AWS_CREDENTIAL_FILE']
167  if 'BOTO_CONFIG' in os.environ:
168    del os.environ['BOTO_CONFIG']
169
170  buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
171  buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
172
173  os.chdir(SCRIPT_DIR)
174
175  if buildername == 'linux-sdk-mono32':
176    assert buildbot_revision
177    sdk_revision = buildbot_revision.split(':')[0]
178    pepper_revision = build_version.ChromeMajorVersion()
179    build_and_upload_mono(sdk_revision, pepper_revision, None,
180                          'trunk.' + sdk_revision, args)
181  elif buildername == 'linux-sdk-mono64':
182    infos = get_sdk_build_info()
183    for info in infos:
184      # This will put the file in naclmono_19/1/naclmono_19.bz2 for example.
185      upload_path = info['naclmono_name'] + '/' + info['naclmono_rev']
186      build_and_upload_mono(None, info['pepper_revision'], info['sdk_url'],
187          upload_path, args)
188    update_mono_sdk_json(infos)
189
190
191
192if __name__ == '__main__':
193  sys.exit(main(sys.argv))
194