1#!/usr/bin/env python 2# Copyright (c) 2012 Google Inc. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6 7"""Argument-less script to select what to run on the buildbots.""" 8 9 10import filecmp 11import os 12import shutil 13import subprocess 14import sys 15 16 17if sys.platform in ['win32', 'cygwin']: 18 EXE_SUFFIX = '.exe' 19else: 20 EXE_SUFFIX = '' 21 22 23BUILDBOT_DIR = os.path.dirname(os.path.abspath(__file__)) 24TRUNK_DIR = os.path.dirname(BUILDBOT_DIR) 25ROOT_DIR = os.path.dirname(TRUNK_DIR) 26ANDROID_DIR = os.path.join(ROOT_DIR, 'android') 27CMAKE_DIR = os.path.join(ROOT_DIR, 'cmake') 28CMAKE_BIN_DIR = os.path.join(CMAKE_DIR, 'bin') 29OUT_DIR = os.path.join(TRUNK_DIR, 'out') 30 31 32def CallSubProcess(*args, **kwargs): 33 """Wrapper around subprocess.call which treats errors as build exceptions.""" 34 with open(os.devnull) as devnull_fd: 35 retcode = subprocess.call(stdin=devnull_fd, *args, **kwargs) 36 if retcode != 0: 37 print '@@@STEP_EXCEPTION@@@' 38 sys.exit(1) 39 40 41def PrepareCmake(): 42 """Build CMake 2.8.8 since the version in Precise is 2.8.7.""" 43 if os.environ['BUILDBOT_CLOBBER'] == '1': 44 print '@@@BUILD_STEP Clobber CMake checkout@@@' 45 shutil.rmtree(CMAKE_DIR) 46 47 # We always build CMake 2.8.8, so no need to do anything 48 # if the directory already exists. 49 if os.path.isdir(CMAKE_DIR): 50 return 51 52 print '@@@BUILD_STEP Initialize CMake checkout@@@' 53 os.mkdir(CMAKE_DIR) 54 55 print '@@@BUILD_STEP Sync CMake@@@' 56 CallSubProcess( 57 ['git', 'clone', 58 '--depth', '1', 59 '--single-branch', 60 '--branch', 'v2.8.8', 61 '--', 62 'git://cmake.org/cmake.git', 63 CMAKE_DIR], 64 cwd=CMAKE_DIR) 65 66 print '@@@BUILD_STEP Build CMake@@@' 67 CallSubProcess( 68 ['/bin/bash', 'bootstrap', '--prefix=%s' % CMAKE_DIR], 69 cwd=CMAKE_DIR) 70 71 CallSubProcess( ['make', 'cmake'], cwd=CMAKE_DIR) 72 73 74_ANDROID_SETUP = 'source build/envsetup.sh && lunch full-eng' 75 76 77def PrepareAndroidTree(): 78 """Prepare an Android tree to run 'android' format tests.""" 79 if os.environ['BUILDBOT_CLOBBER'] == '1': 80 print '@@@BUILD_STEP Clobber Android checkout@@@' 81 shutil.rmtree(ANDROID_DIR) 82 83 # (Re)create the directory so that the following steps will succeed. 84 if not os.path.isdir(ANDROID_DIR): 85 os.mkdir(ANDROID_DIR) 86 87 # We use a manifest from the gyp project listing pinned revisions of AOSP to 88 # use, to ensure that we test against a stable target. This needs to be 89 # updated to pick up new build system changes sometimes, so we must test if 90 # it has changed. 91 manifest_filename = 'aosp_manifest.xml' 92 gyp_manifest = os.path.join(BUILDBOT_DIR, manifest_filename) 93 android_manifest = os.path.join(ANDROID_DIR, '.repo', 'manifests', 94 manifest_filename) 95 manifest_is_current = (os.path.isfile(android_manifest) and 96 filecmp.cmp(gyp_manifest, android_manifest)) 97 if not manifest_is_current: 98 # It's safe to repeat these steps, so just do them again to make sure we are 99 # in a good state. 100 print '@@@BUILD_STEP Initialize Android checkout@@@' 101 CallSubProcess( 102 ['repo', 'init', 103 '-u', 'https://android.googlesource.com/platform/manifest', 104 '-b', 'master', 105 '-g', 'all,-notdefault,-device,-darwin,-mips,-x86'], 106 cwd=ANDROID_DIR) 107 shutil.copy(gyp_manifest, android_manifest) 108 109 print '@@@BUILD_STEP Sync Android@@@' 110 CallSubProcess(['repo', 'sync', '-j4', '-m', manifest_filename], 111 cwd=ANDROID_DIR) 112 113 # If we already built the system image successfully and didn't sync to a new 114 # version of the source, skip running the build again as it's expensive even 115 # when there's nothing to do. 116 system_img = os.path.join(ANDROID_DIR, 'out', 'target', 'product', 'generic', 117 'system.img') 118 if manifest_is_current and os.path.isfile(system_img): 119 return 120 121 print '@@@BUILD_STEP Build Android@@@' 122 CallSubProcess( 123 ['/bin/bash', 124 '-c', '%s && make -j4' % _ANDROID_SETUP], 125 cwd=ANDROID_DIR) 126 127 128def StartAndroidEmulator(): 129 """Start an android emulator from the built android tree.""" 130 print '@@@BUILD_STEP Start Android emulator@@@' 131 132 CallSubProcess(['/bin/bash', '-c', 133 '%s && adb kill-server ' % _ANDROID_SETUP], 134 cwd=ANDROID_DIR) 135 136 # If taskset is available, use it to force adbd to run only on one core, as, 137 # sadly, it improves its reliability (see crbug.com/268450). 138 adbd_wrapper = '' 139 with open(os.devnull, 'w') as devnull_fd: 140 if subprocess.call(['which', 'taskset'], stdout=devnull_fd) == 0: 141 adbd_wrapper = 'taskset -c 0' 142 CallSubProcess(['/bin/bash', '-c', 143 '%s && %s adb start-server ' % (_ANDROID_SETUP, adbd_wrapper)], 144 cwd=ANDROID_DIR) 145 146 subprocess.Popen( 147 ['/bin/bash', '-c', 148 '%s && emulator -no-window' % _ANDROID_SETUP], 149 cwd=ANDROID_DIR) 150 CallSubProcess( 151 ['/bin/bash', '-c', 152 '%s && adb wait-for-device' % _ANDROID_SETUP], 153 cwd=ANDROID_DIR) 154 155 156def StopAndroidEmulator(): 157 """Stop all android emulators.""" 158 print '@@@BUILD_STEP Stop Android emulator@@@' 159 # If this fails, it's because there is no emulator running. 160 subprocess.call(['pkill', 'emulator.*']) 161 162 163def GypTestFormat(title, format=None, msvs_version=None, tests=[]): 164 """Run the gyp tests for a given format, emitting annotator tags. 165 166 See annotator docs at: 167 https://sites.google.com/a/chromium.org/dev/developers/testing/chromium-build-infrastructure/buildbot-annotations 168 Args: 169 format: gyp format to test. 170 Returns: 171 0 for sucesss, 1 for failure. 172 """ 173 if not format: 174 format = title 175 176 print '@@@BUILD_STEP ' + title + '@@@' 177 sys.stdout.flush() 178 env = os.environ.copy() 179 if msvs_version: 180 env['GYP_MSVS_VERSION'] = msvs_version 181 command = ' '.join( 182 [sys.executable, 'trunk/gyptest.py', 183 '--all', 184 '--passed', 185 '--format', format, 186 '--path', CMAKE_BIN_DIR, 187 '--chdir', 'trunk'] + tests) 188 if format == 'android': 189 # gyptest needs the environment setup from envsetup/lunch in order to build 190 # using the 'android' backend, so this is done in a single shell. 191 retcode = subprocess.call( 192 ['/bin/bash', 193 '-c', '%s && cd %s && %s' % (_ANDROID_SETUP, ROOT_DIR, command)], 194 cwd=ANDROID_DIR, env=env) 195 else: 196 retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True) 197 if retcode: 198 # Emit failure tag, and keep going. 199 print '@@@STEP_FAILURE@@@' 200 return 1 201 return 0 202 203 204def GypBuild(): 205 # Dump out/ directory. 206 print '@@@BUILD_STEP cleanup@@@' 207 print 'Removing %s...' % OUT_DIR 208 shutil.rmtree(OUT_DIR, ignore_errors=True) 209 print 'Done.' 210 211 retcode = 0 212 # The Android gyp bot runs on linux so this must be tested first. 213 if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-android': 214 PrepareAndroidTree() 215 StartAndroidEmulator() 216 try: 217 retcode += GypTestFormat('android') 218 finally: 219 StopAndroidEmulator() 220 elif sys.platform.startswith('linux'): 221 retcode += GypTestFormat('ninja') 222 retcode += GypTestFormat('make') 223 PrepareCmake() 224 retcode += GypTestFormat('cmake') 225 elif sys.platform == 'darwin': 226 retcode += GypTestFormat('ninja') 227 retcode += GypTestFormat('xcode') 228 retcode += GypTestFormat('make') 229 elif sys.platform == 'win32': 230 retcode += GypTestFormat('ninja') 231 if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-win64': 232 retcode += GypTestFormat('msvs-ninja-2012', format='msvs-ninja', 233 msvs_version='2012', 234 tests=[ 235 'test\generator-output\gyptest-actions.py', 236 'test\generator-output\gyptest-relocate.py', 237 'test\generator-output\gyptest-rules.py']) 238 retcode += GypTestFormat('msvs-2010', format='msvs', msvs_version='2010') 239 retcode += GypTestFormat('msvs-2012', format='msvs', msvs_version='2012') 240 else: 241 raise Exception('Unknown platform') 242 if retcode: 243 # TODO(bradnelson): once the annotator supports a postscript (section for 244 # after the build proper that could be used for cumulative failures), 245 # use that instead of this. This isolates the final return value so 246 # that it isn't misattributed to the last stage. 247 print '@@@BUILD_STEP failures@@@' 248 sys.exit(retcode) 249 250 251if __name__ == '__main__': 252 GypBuild() 253