android.py revision c7f1593f9af3ea1b9264b37628c36f3a70e1749a 
1#!/usr/bin/python
2#
3# Copyright 2011 Google Inc. All Rights Reserved.
4
5"""Helper modules for Android toolchain test infrastructure.
6
7Provides following Android toolchain test jobs and commands.
8. Checkout Android toolchain source code
9. Build Android toolchain
10. Checkout and build Android tree
11. Checkout/build/run Android benchmarks, generate size dashboard
12. Transform size dashboard to report, send perflab jobid to
13  perflab dashboard server.(TODO)
14"""
15
16__author__ = 'jingyu@google.com (Jing Yu)'
17
18import os.path
19
20from automation.clients.helper import jobs
21from automation.clients.helper import perforce
22from automation.common import command as cmd
23from automation.common import job
24
25
26class JobsFactory(object):
27  def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'):
28    assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk']
29    assert build_type in ['DEVELOPMENT', 'RELEASE']
30
31    self.gcc_version = gcc_version
32    self.commands = CommandsFactory(gcc_version, build_type)
33    self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type)
34
35  def CheckoutAndroidToolchain(self):
36    """Check out Android toolchain sources by release and gcc version."""
37    command = self.commands.CheckoutAndroidToolchain()
38    new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag,
39                                  command)
40    checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR)
41    return new_job, checkout_dir_dep
42
43  def BuildAndroidToolchain(self, checkout_dir_dep):
44    """Build Android Toolchain."""
45    command = self.commands.BuildAndroidToolchain()
46    new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag,
47                                  command)
48    new_job.DependsOnFolder(checkout_dir_dep)
49    tc_prefix_dep = job.FolderDependency(
50        new_job, self.commands.toolchain_prefix_dir)
51    return new_job, tc_prefix_dep
52
53  def BuildAndroidImage(self, tc_prefix_dep, product='stingray',
54                        branch='ics-release'):
55    assert product in ['stingray', 'passion', 'trygon', 'soju']
56    assert branch in ['honeycomb-release', 'ics-release']
57    command = self.commands.BuildAndroidImage(product, branch)
58    new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag,
59                                  command)
60    new_job.DependsOnFolder(tc_prefix_dep)
61    return new_job
62
63  def Benchmark(self, tc_prefix_dep, arch='soju'):
64    assert arch in ['soju', 'stingray']
65    script_cmd = self.commands.CheckoutScripts()
66    experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch)
67    build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch,
68                                                              experiment_tag)
69    command = cmd.Chain(script_cmd, build_run_benchmark_cmd)
70    new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag,
71                                  command)
72    new_job.DependsOnFolder(tc_prefix_dep)
73    return new_job
74
75
76class CommandsFactory(object):
77  CHECKOUT_DIR = 'androidtc-checkout-dir'
78  TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src')
79  TOOLCHAIN_BUILD_DIR = 'obj'
80  ANDROID_TREES_DIR = 'android_trees'
81  TOOLS_DIR = 'android-tools'
82  BENCHMARK_OUT_DIR = 'results'
83
84  def __init__(self, gcc_version, build_type):
85    assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk']
86    assert build_type in ['DEVELOPMENT', 'RELEASE']
87
88    self.build_type = build_type
89    self.gcc_version = gcc_version
90    self.binutils_version = '2.21'
91    self.gold_version = '2.21'
92    self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (
93        gcc_version, build_type)
94    self.p4client = self._CreatePerforceClient()
95    self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version,
96                                  self.gold_version)
97
98  def _CreatePerforceClient(self):
99    p4_dev_path = 'gcctools/google_vendor_src_branch'
100    mobile_rel_branch = ('branches/'
101                         'mobile_toolchain_v15_release_branch/gcctools/'
102                         'google_vendor_src_branch')
103    gcc_443_rel_branch = ('branches/'
104                          'android_compiler_v14_release_branch/gcctools/'
105                          'google_vendor_src_branch')
106
107    # Common views for tools
108    p4view = perforce.View('depot2',
109                           perforce.PathMapping.ListFromPathTuples(
110                               [('gcctools/android/build/...', 'src/build/...'),
111                                ('gcctools/android/Tarballs/...',
112                                 'src/tarballs/...')]))
113    for mapping in perforce.PathMapping.ListFromPathDict(
114        {'gcctools/android': ['tools/scripts/...', 'master/...']}):
115      p4view.add(mapping)
116
117    # Add views for gdb
118    p4view.add(perforce.PathMapping(p4_dev_path, 'src',
119                                    'gdb/gdb-7.1.x-android/...'))
120
121    # Add view for binutils for ld and gold
122    if self.build_type is 'RELEASE':
123      binutils_branch = mobile_rel_branch
124    else:
125      binutils_branch = p4_dev_path
126    p4view.add(perforce.PathMapping(binutils_branch, 'src',
127                                    ('binutils/binutils-%s/...' %
128                                     self.binutils_version)))
129    if self.binutils_version != self.gold_version:
130      p4view.add(perforce.PathMapping(binutils_branch, 'src',
131                                      ('binutils/binutils-%s/...' %
132                                       self.gold_version)))
133
134    # Add view for gcc if gcc_version is '4.4.3'.
135    if self.gcc_version == '4.4.3':
136      gcc443_path = 'gcc/gcc-4.4.3/...'
137      if self.build_type is 'RELEASE':
138        p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path))
139      else:
140        p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path))
141
142    return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view)
143
144  def _CheckoutGCCFromSVN(self):
145    """Check out gcc from fsf svn.
146
147       Return the command that check out gcc from svn
148       to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx).
149
150       TODO:
151         Create a svn class that does these jobs.
152         Parallelize p4 checkout and svn checkout.
153    """
154    if self.gcc_version == '4.4.3':
155      return ''
156    assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk']
157
158    gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6',
159                        'google_main': 'branches/google/main',
160                        'fsf_trunk': 'trunk'}
161
162    # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC
163    svn_get_revision = cmd.Pipe(
164        cmd.Shell('svn', 'info'),
165        cmd.Shell('grep', '"Revision:"'),
166        cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'),
167        output='../../../CLNUM_GCC')
168
169    svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % (
170        gcc_branches_dir[self.gcc_version])
171
172    gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc',
173                                    'gcc-%s' % self.gcc_version)
174
175    return cmd.Chain(cmd.MakeDir(gcc_required_dir),
176                     cmd.Wrapper(cmd.Chain(svn_co_command, svn_get_revision),
177                                 cwd=gcc_required_dir))
178
179  def CheckoutAndroidToolchain(self):
180    p4client = self.p4client
181    command = p4client.SetupAndDo(p4client.Sync(),
182                                  p4client.SaveCurrentCLNumber('CLNUM'),
183                                  p4client.Remove())
184    if self.gcc_version != '4.4.3':
185      command.append(self._CheckoutGCCFromSVN())
186
187    return command
188
189  def BuildAndroidToolchain(self):
190    script_cmd = self.scripts.BuildAndroidToolchain(self.toolchain_prefix_dir,
191                                                    self.CHECKOUT_DIR,
192                                                    self.TOOLCHAIN_BUILD_DIR,
193                                                    self.TOOLCHAIN_SRC_DIR)
194
195    # Record toolchain and gcc CL number
196    record_cl_cmd = cmd.Copy(os.path.join(self.CHECKOUT_DIR, 'CLNUM*'),
197                             to_dir=self.toolchain_prefix_dir)
198    save_cmd = cmd.Tar(os.path.join('$JOB_TMP', 'results',
199                                    '%s.tar.bz2' % self.toolchain_prefix_dir),
200                       self.toolchain_prefix_dir)
201    return cmd.Chain(script_cmd, record_cl_cmd, save_cmd)
202
203  def _BuildAndroidTree(self, local_android_branch_dir, product):
204    target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir,
205                                       'bin', 'arm-linux-androideabi-')
206    java_path = '/usr/lib/jvm/java-6-sun/bin'
207    build_cmd = cmd.Shell('make', '-j8',
208                          'PRODUCT-%s-userdebug' % product,
209                          'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix,
210                          'PATH=%s:$PATH' % java_path)
211    return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir)
212
213  def BuildAndroidImage(self, product, branch):
214    assert product in ['stingray', 'passion', 'trygon', 'soju']
215
216    # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch
217    androidtrees_host = 'atree.mtv.corp.google.com'
218    androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/'
219                         'android_trees')
220    remote_android_branch_path = os.path.join(androidtrees_path, branch)
221    local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch)
222    gettree_cmd = cmd.RemoteCopyFrom(androidtrees_host,
223                                     remote_android_branch_path,
224                                     local_android_branch_dir)
225
226    # Configure and build the tree
227    buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product)
228
229    # Compress and copy system.img to result
230    result_system_img = os.path.join(local_android_branch_dir, 'out', 'target',
231                                     'product', product, 'system.img')
232    copy_img = cmd.Copy(result_system_img, to_dir='results')
233    compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img'))
234
235    return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img)
236
237  def CheckoutScripts(self):
238    p4view = perforce.View('depot2', [perforce.PathMapping(
239        'gcctools/android/tools/...', 'tools/...')])
240    p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view)
241    return p4client.SetupAndDo(p4client.Sync(), p4client.Remove())
242
243  def BuildRunBenchmark(self, arch, run_experiment):
244    # Copy base benchmark binaries from atree.mtv.corp
245    base_benchbin_host = 'atree.mtv.corp.google.com'
246    base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/'
247                          'archive/v3binaries/2011-10-18')
248    local_basebenchbin_dir = 'base_benchmark_bin'
249    getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host,
250                                     base_benchbin_path,
251                                     local_basebenchbin_dir)
252
253    # Build and run benchmark.
254    android_arch = 'android_%s' % arch
255    run_label = 'normal'
256    benchmark_cmd = self.scripts.RunBenchmark(self.toolchain_prefix_dir,
257                                              self.TOOLS_DIR,
258                                              self.BENCHMARK_OUT_DIR,
259                                              run_label, run_experiment,
260                                              android_arch,
261                                              local_basebenchbin_dir)
262
263    # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file.
264    # Copy jobid to www server to generate performance dashboard.
265    # TODO(jingyu)
266
267    return cmd.Chain(getbase_cmd, benchmark_cmd)
268
269
270class ScriptsFactory(object):
271  def __init__(self, gcc_version, binutils_version, gold_version):
272    self._gcc_version = gcc_version
273    self._binutils_version = binutils_version
274    self._gold_version = gold_version
275
276  def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir,
277                            toolchain_build_dir, androidtc_src_dir):
278    if self._gcc_version == '4.4.3':
279      gold_option = 'both/gold'
280    else:
281      gold_option = 'default'
282
283    return cmd.Shell(
284        'build_androidtoolchain.sh',
285        '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir),
286        '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir),
287        '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir),
288        '--target=arm-linux-androideabi',
289        '--enable-gold=%s' % gold_option,
290        '--with-gcc-version=%s' % self._gcc_version,
291        '--with-binutils-version=%s' % self._binutils_version,
292        '--with-gold-version=%s' % self._gold_version,
293        '--with-gdb-version=7.1.x-android',
294        '--log-path=%s/logs' % '$JOB_HOME',
295        '--android-sysroot=%s' %
296        os.path.join('$JOB_TMP', checkout_dir, 'gcctools', 'android',
297                     'master', 'honeycomb_generic_sysroot'),
298        path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools',
299                          'scripts'))
300
301  def RunBenchmark(self, toolchain_prefix_dir, checkout_dir, output_dir,
302                   run_label, run_experiment, arch, base_bench_bin=None):
303    if base_bench_bin:
304      base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin
305    else:
306      base_bench_opt = ''
307
308    return cmd.Shell(
309        'benchmark.sh',
310        '--android_toolchain=%s' % os.path.join('$JOB_TMP',
311                                                toolchain_prefix_dir),
312        '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'),
313        '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir,
314                                            'bench_bin'),
315        base_bench_opt,
316        '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'),
317        '--arch=%s' % arch,
318        '--run_label=%s' % run_label,
319        '--run_experiment=%s' % run_experiment,
320        path=os.path.join(checkout_dir, 'tools', 'scripts'))
321