1#!/usr/bin/env python
2# Copyright (c) 2012 Google Inc. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6
7"""Argument-less script to select what to run on the buildbots."""
8
9
10import filecmp
11import os
12import shutil
13import subprocess
14import sys
15
16
17if sys.platform in ['win32', 'cygwin']:
18  EXE_SUFFIX = '.exe'
19else:
20  EXE_SUFFIX = ''
21
22
23BUILDBOT_DIR = os.path.dirname(os.path.abspath(__file__))
24TRUNK_DIR = os.path.dirname(BUILDBOT_DIR)
25ROOT_DIR = os.path.dirname(TRUNK_DIR)
26ANDROID_DIR = os.path.join(ROOT_DIR, 'android')
27CMAKE_DIR = os.path.join(ROOT_DIR, 'cmake')
28CMAKE_BIN_DIR = os.path.join(CMAKE_DIR, 'bin')
29OUT_DIR = os.path.join(TRUNK_DIR, 'out')
30
31
32def CallSubProcess(*args, **kwargs):
33  """Wrapper around subprocess.call which treats errors as build exceptions."""
34  with open(os.devnull) as devnull_fd:
35    retcode = subprocess.call(stdin=devnull_fd, *args, **kwargs)
36  if retcode != 0:
37    print '@@@STEP_EXCEPTION@@@'
38    sys.exit(1)
39
40
41def PrepareCmake():
42  """Build CMake 2.8.8 since the version in Precise is 2.8.7."""
43  if os.environ['BUILDBOT_CLOBBER'] == '1':
44    print '@@@BUILD_STEP Clobber CMake checkout@@@'
45    shutil.rmtree(CMAKE_DIR)
46
47  # We always build CMake 2.8.8, so no need to do anything
48  # if the directory already exists.
49  if os.path.isdir(CMAKE_DIR):
50    return
51
52  print '@@@BUILD_STEP Initialize CMake checkout@@@'
53  os.mkdir(CMAKE_DIR)
54
55  print '@@@BUILD_STEP Sync CMake@@@'
56  CallSubProcess(
57      ['git', 'clone',
58       '--depth', '1',
59       '--single-branch',
60       '--branch', 'v2.8.8',
61       '--',
62       'git://cmake.org/cmake.git',
63       CMAKE_DIR],
64      cwd=CMAKE_DIR)
65
66  print '@@@BUILD_STEP Build CMake@@@'
67  CallSubProcess(
68      ['/bin/bash', 'bootstrap', '--prefix=%s' % CMAKE_DIR],
69      cwd=CMAKE_DIR)
70
71  CallSubProcess( ['make', 'cmake'], cwd=CMAKE_DIR)
72
73
74_ANDROID_SETUP = 'source build/envsetup.sh && lunch full-eng'
75
76
77def PrepareAndroidTree():
78  """Prepare an Android tree to run 'android' format tests."""
79  if os.environ['BUILDBOT_CLOBBER'] == '1':
80    print '@@@BUILD_STEP Clobber Android checkout@@@'
81    shutil.rmtree(ANDROID_DIR)
82
83  # (Re)create the directory so that the following steps will succeed.
84  if not os.path.isdir(ANDROID_DIR):
85    os.mkdir(ANDROID_DIR)
86
87  # We use a manifest from the gyp project listing pinned revisions of AOSP to
88  # use, to ensure that we test against a stable target. This needs to be
89  # updated to pick up new build system changes sometimes, so we must test if
90  # it has changed.
91  manifest_filename = 'aosp_manifest.xml'
92  gyp_manifest = os.path.join(BUILDBOT_DIR, manifest_filename)
93  android_manifest = os.path.join(ANDROID_DIR, '.repo', 'manifests',
94                                  manifest_filename)
95  manifest_is_current = (os.path.isfile(android_manifest) and
96                         filecmp.cmp(gyp_manifest, android_manifest))
97  if not manifest_is_current:
98    # It's safe to repeat these steps, so just do them again to make sure we are
99    # in a good state.
100    print '@@@BUILD_STEP Initialize Android checkout@@@'
101    CallSubProcess(
102        ['repo', 'init',
103         '-u', 'https://android.googlesource.com/platform/manifest',
104         '-b', 'master',
105         '-g', 'all,-notdefault,-device,-darwin,-mips,-x86'],
106        cwd=ANDROID_DIR)
107    shutil.copy(gyp_manifest, android_manifest)
108
109    print '@@@BUILD_STEP Sync Android@@@'
110    CallSubProcess(['repo', 'sync', '-j4', '-m', manifest_filename],
111                   cwd=ANDROID_DIR)
112
113  # If we already built the system image successfully and didn't sync to a new
114  # version of the source, skip running the build again as it's expensive even
115  # when there's nothing to do.
116  system_img = os.path.join(ANDROID_DIR, 'out', 'target', 'product', 'generic',
117                            'system.img')
118  if manifest_is_current and os.path.isfile(system_img):
119    return
120
121  print '@@@BUILD_STEP Build Android@@@'
122  CallSubProcess(
123      ['/bin/bash',
124       '-c', '%s && make -j4' % _ANDROID_SETUP],
125      cwd=ANDROID_DIR)
126
127
128def StartAndroidEmulator():
129  """Start an android emulator from the built android tree."""
130  print '@@@BUILD_STEP Start Android emulator@@@'
131
132  CallSubProcess(['/bin/bash', '-c',
133      '%s && adb kill-server ' % _ANDROID_SETUP],
134      cwd=ANDROID_DIR)
135
136  # If taskset is available, use it to force adbd to run only on one core, as,
137  # sadly, it improves its reliability (see crbug.com/268450).
138  adbd_wrapper = ''
139  with open(os.devnull, 'w') as devnull_fd:
140    if subprocess.call(['which', 'taskset'], stdout=devnull_fd) == 0:
141      adbd_wrapper = 'taskset -c 0'
142  CallSubProcess(['/bin/bash', '-c',
143      '%s && %s adb start-server ' % (_ANDROID_SETUP, adbd_wrapper)],
144      cwd=ANDROID_DIR)
145
146  subprocess.Popen(
147      ['/bin/bash', '-c',
148       '%s && emulator -no-window' % _ANDROID_SETUP],
149      cwd=ANDROID_DIR)
150  CallSubProcess(
151      ['/bin/bash', '-c',
152       '%s && adb wait-for-device' % _ANDROID_SETUP],
153      cwd=ANDROID_DIR)
154
155
156def StopAndroidEmulator():
157  """Stop all android emulators."""
158  print '@@@BUILD_STEP Stop Android emulator@@@'
159  # If this fails, it's because there is no emulator running.
160  subprocess.call(['pkill', 'emulator.*'])
161
162
163def GypTestFormat(title, format=None, msvs_version=None, tests=[]):
164  """Run the gyp tests for a given format, emitting annotator tags.
165
166  See annotator docs at:
167    https://sites.google.com/a/chromium.org/dev/developers/testing/chromium-build-infrastructure/buildbot-annotations
168  Args:
169    format: gyp format to test.
170  Returns:
171    0 for sucesss, 1 for failure.
172  """
173  if not format:
174    format = title
175
176  print '@@@BUILD_STEP ' + title + '@@@'
177  sys.stdout.flush()
178  env = os.environ.copy()
179  if msvs_version:
180    env['GYP_MSVS_VERSION'] = msvs_version
181  command = ' '.join(
182      [sys.executable, 'trunk/gyptest.py',
183       '--all',
184       '--passed',
185       '--format', format,
186       '--path', CMAKE_BIN_DIR,
187       '--chdir', 'trunk'] + tests)
188  if format == 'android':
189    # gyptest needs the environment setup from envsetup/lunch in order to build
190    # using the 'android' backend, so this is done in a single shell.
191    retcode = subprocess.call(
192        ['/bin/bash',
193         '-c', '%s && cd %s && %s' % (_ANDROID_SETUP, ROOT_DIR, command)],
194        cwd=ANDROID_DIR, env=env)
195  else:
196    retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True)
197  if retcode:
198    # Emit failure tag, and keep going.
199    print '@@@STEP_FAILURE@@@'
200    return 1
201  return 0
202
203
204def GypBuild():
205  # Dump out/ directory.
206  print '@@@BUILD_STEP cleanup@@@'
207  print 'Removing %s...' % OUT_DIR
208  shutil.rmtree(OUT_DIR, ignore_errors=True)
209  print 'Done.'
210
211  retcode = 0
212  # The Android gyp bot runs on linux so this must be tested first.
213  if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-android':
214    PrepareAndroidTree()
215    StartAndroidEmulator()
216    try:
217      retcode += GypTestFormat('android')
218    finally:
219      StopAndroidEmulator()
220  elif sys.platform.startswith('linux'):
221    retcode += GypTestFormat('ninja')
222    retcode += GypTestFormat('make')
223    PrepareCmake()
224    retcode += GypTestFormat('cmake')
225  elif sys.platform == 'darwin':
226    retcode += GypTestFormat('ninja')
227    retcode += GypTestFormat('xcode')
228    retcode += GypTestFormat('make')
229  elif sys.platform == 'win32':
230    retcode += GypTestFormat('ninja')
231    if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-win64':
232      retcode += GypTestFormat('msvs-ninja-2012', format='msvs-ninja',
233                               msvs_version='2012',
234                               tests=[
235                                   'test\generator-output\gyptest-actions.py',
236                                   'test\generator-output\gyptest-relocate.py',
237                                   'test\generator-output\gyptest-rules.py'])
238      retcode += GypTestFormat('msvs-2010', format='msvs', msvs_version='2010')
239      retcode += GypTestFormat('msvs-2012', format='msvs', msvs_version='2012')
240  else:
241    raise Exception('Unknown platform')
242  if retcode:
243    # TODO(bradnelson): once the annotator supports a postscript (section for
244    #     after the build proper that could be used for cumulative failures),
245    #     use that instead of this. This isolates the final return value so
246    #     that it isn't misattributed to the last stage.
247    print '@@@BUILD_STEP failures@@@'
248    sys.exit(retcode)
249
250
251if __name__ == '__main__':
252  GypBuild()
253