android.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1# Copyright 2011 Google Inc. All Rights Reserved.
2"""Helper modules for Android toolchain test infrastructure.
3
4Provides following Android toolchain test jobs and commands.
5. Checkout Android toolchain source code
6. Build Android toolchain
7. Checkout and build Android tree
8. Checkout/build/run Android benchmarks, generate size dashboard
9. Transform size dashboard to report, send perflab jobid to
10  perflab dashboard server.(TODO)
11"""
12
13__author__ = 'jingyu@google.com (Jing Yu)'
14
15import os.path
16
17from automation.clients.helper import jobs
18from automation.clients.helper import perforce
19from automation.common import command as cmd
20from automation.common import job
21
22
23class JobsFactory(object):
24
25  def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'):
26    assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk']
27    assert build_type in ['DEVELOPMENT', 'RELEASE']
28
29    self.gcc_version = gcc_version
30    self.commands = CommandsFactory(gcc_version, build_type)
31    self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type)
32
33  def CheckoutAndroidToolchain(self):
34    """Check out Android toolchain sources by release and gcc version."""
35    command = self.commands.CheckoutAndroidToolchain()
36    new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag,
37                                  command)
38    checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR)
39    return new_job, checkout_dir_dep
40
41  def BuildAndroidToolchain(self, checkout_dir_dep):
42    """Build Android Toolchain."""
43    command = self.commands.BuildAndroidToolchain()
44    new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag,
45                                  command)
46    new_job.DependsOnFolder(checkout_dir_dep)
47    tc_prefix_dep = job.FolderDependency(new_job,
48                                         self.commands.toolchain_prefix_dir)
49    return new_job, tc_prefix_dep
50
51  def BuildAndroidImage(self,
52                        tc_prefix_dep,
53                        product='stingray',
54                        branch='ics-release'):
55    assert product in ['stingray', 'passion', 'trygon', 'soju']
56    assert branch in ['honeycomb-release', 'ics-release']
57    command = self.commands.BuildAndroidImage(product, branch)
58    new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag,
59                                  command)
60    new_job.DependsOnFolder(tc_prefix_dep)
61    return new_job
62
63  def Benchmark(self, tc_prefix_dep, arch='soju'):
64    assert arch in ['soju', 'stingray']
65    script_cmd = self.commands.CheckoutScripts()
66    experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch)
67    build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch,
68                                                              experiment_tag)
69    command = cmd.Chain(script_cmd, build_run_benchmark_cmd)
70    new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag,
71                                  command)
72    new_job.DependsOnFolder(tc_prefix_dep)
73    return new_job
74
75
76class CommandsFactory(object):
77  CHECKOUT_DIR = 'androidtc-checkout-dir'
78  TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src')
79  TOOLCHAIN_BUILD_DIR = 'obj'
80  ANDROID_TREES_DIR = 'android_trees'
81  TOOLS_DIR = 'android-tools'
82  BENCHMARK_OUT_DIR = 'results'
83
84  def __init__(self, gcc_version, build_type):
85    assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk']
86    assert build_type in ['DEVELOPMENT', 'RELEASE']
87
88    self.build_type = build_type
89    self.gcc_version = gcc_version
90    self.binutils_version = '2.21'
91    self.gold_version = '2.21'
92    self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type)
93    self.p4client = self._CreatePerforceClient()
94    self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version,
95                                  self.gold_version)
96
97  def _CreatePerforceClient(self):
98    p4_dev_path = 'gcctools/google_vendor_src_branch'
99    mobile_rel_branch = ('branches/'
100                         'mobile_toolchain_v15_release_branch/gcctools/'
101                         'google_vendor_src_branch')
102    gcc_443_rel_branch = ('branches/'
103                          'android_compiler_v14_release_branch/gcctools/'
104                          'google_vendor_src_branch')
105
106    # Common views for tools
107    p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([(
108        'gcctools/android/build/...', 'src/build/...'), (
109            'gcctools/android/Tarballs/...', 'src/tarballs/...')]))
110    for mapping in perforce.PathMapping.ListFromPathDict(
111        {'gcctools/android': ['tools/scripts/...', 'master/...']}):
112      p4view.add(mapping)
113
114    # Add views for gdb
115    p4view.add(perforce.PathMapping(p4_dev_path, 'src',
116                                    'gdb/gdb-7.1.x-android/...'))
117
118    # Add view for binutils for ld and gold
119    if self.build_type is 'RELEASE':
120      binutils_branch = mobile_rel_branch
121    else:
122      binutils_branch = p4_dev_path
123    p4view.add(perforce.PathMapping(binutils_branch, 'src', (
124        'binutils/binutils-%s/...' % self.binutils_version)))
125    if self.binutils_version != self.gold_version:
126      p4view.add(perforce.PathMapping(binutils_branch, 'src', (
127          'binutils/binutils-%s/...' % self.gold_version)))
128
129    # Add view for gcc if gcc_version is '4.4.3'.
130    if self.gcc_version == '4.4.3':
131      gcc443_path = 'gcc/gcc-4.4.3/...'
132      if self.build_type is 'RELEASE':
133        p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path))
134      else:
135        p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path))
136
137    return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view)
138
139  def _CheckoutGCCFromSVN(self):
140    """Check out gcc from fsf svn.
141
142       Return the command that check out gcc from svn
143       to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx).
144
145       TODO:
146         Create a svn class that does these jobs.
147         Parallelize p4 checkout and svn checkout.
148    """
149    if self.gcc_version == '4.4.3':
150      return ''
151    assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk']
152
153    gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6',
154                        'google_main': 'branches/google/main',
155                        'fsf_trunk': 'trunk'}
156
157    # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC
158    svn_get_revision = cmd.Pipe(
159        cmd.Shell('svn', 'info'),
160        cmd.Shell('grep', '"Revision:"'),
161        cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'),
162        output='../../../CLNUM_GCC')
163
164    svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % (
165        gcc_branches_dir[self.gcc_version])
166
167    gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc',
168                                    'gcc-%s' % self.gcc_version)
169
170    return cmd.Chain(
171        cmd.MakeDir(gcc_required_dir),
172        cmd.Wrapper(
173            cmd.Chain(svn_co_command, svn_get_revision),
174            cwd=gcc_required_dir))
175
176  def CheckoutAndroidToolchain(self):
177    p4client = self.p4client
178    command = p4client.SetupAndDo(p4client.Sync(),
179                                  p4client.SaveCurrentCLNumber('CLNUM'),
180                                  p4client.Remove())
181    if self.gcc_version != '4.4.3':
182      command.append(self._CheckoutGCCFromSVN())
183
184    return command
185
186  def BuildAndroidToolchain(self):
187    script_cmd = self.scripts.BuildAndroidToolchain(
188        self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR,
189        self.TOOLCHAIN_SRC_DIR)
190
191    # Record toolchain and gcc CL number
192    record_cl_cmd = cmd.Copy(
193        os.path.join(self.CHECKOUT_DIR, 'CLNUM*'),
194        to_dir=self.toolchain_prefix_dir)
195    save_cmd = cmd.Tar(
196        os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' %
197                     self.toolchain_prefix_dir), self.toolchain_prefix_dir)
198    return cmd.Chain(script_cmd, record_cl_cmd, save_cmd)
199
200  def _BuildAndroidTree(self, local_android_branch_dir, product):
201    target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir,
202                                       'bin', 'arm-linux-androideabi-')
203    java_path = '/usr/lib/jvm/java-6-sun/bin'
204    build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product,
205                          'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix,
206                          'PATH=%s:$PATH' % java_path)
207    return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir)
208
209  def BuildAndroidImage(self, product, branch):
210    assert product in ['stingray', 'passion', 'trygon', 'soju']
211
212    # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch
213    androidtrees_host = 'atree.mtv.corp.google.com'
214    androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/'
215                         'android_trees')
216    remote_android_branch_path = os.path.join(androidtrees_path, branch)
217    local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch)
218    gettree_cmd = cmd.RemoteCopyFrom(
219        androidtrees_host, remote_android_branch_path, local_android_branch_dir)
220
221    # Configure and build the tree
222    buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product)
223
224    # Compress and copy system.img to result
225    result_system_img = os.path.join(local_android_branch_dir, 'out', 'target',
226                                     'product', product, 'system.img')
227    copy_img = cmd.Copy(result_system_img, to_dir='results')
228    compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img'))
229
230    return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img)
231
232  def CheckoutScripts(self):
233    p4view = perforce.View('depot2',
234                           [perforce.PathMapping('gcctools/android/tools/...',
235                                                 'tools/...')])
236    p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view)
237    return p4client.SetupAndDo(p4client.Sync(), p4client.Remove())
238
239  def BuildRunBenchmark(self, arch, run_experiment):
240    # Copy base benchmark binaries from atree.mtv.corp
241    base_benchbin_host = 'atree.mtv.corp.google.com'
242    base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/'
243                          'archive/v3binaries/2011-10-18')
244    local_basebenchbin_dir = 'base_benchmark_bin'
245    getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path,
246                                     local_basebenchbin_dir)
247
248    # Build and run benchmark.
249    android_arch = 'android_%s' % arch
250    run_label = 'normal'
251    benchmark_cmd = self.scripts.RunBenchmark(
252        self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR,
253        run_label, run_experiment, android_arch, local_basebenchbin_dir)
254
255    # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file.
256    # Copy jobid to www server to generate performance dashboard.
257    # TODO(jingyu)
258
259    return cmd.Chain(getbase_cmd, benchmark_cmd)
260
261
262class ScriptsFactory(object):
263
264  def __init__(self, gcc_version, binutils_version, gold_version):
265    self._gcc_version = gcc_version
266    self._binutils_version = binutils_version
267    self._gold_version = gold_version
268
269  def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir,
270                            toolchain_build_dir, androidtc_src_dir):
271    if self._gcc_version == '4.4.3':
272      gold_option = 'both/gold'
273    else:
274      gold_option = 'default'
275
276    return cmd.Shell(
277        'build_androidtoolchain.sh',
278        '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir),
279        '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir),
280        '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir),
281        '--target=arm-linux-androideabi',
282        '--enable-gold=%s' % gold_option,
283        '--with-gcc-version=%s' % self._gcc_version,
284        '--with-binutils-version=%s' % self._binutils_version,
285        '--with-gold-version=%s' % self._gold_version,
286        '--with-gdb-version=7.1.x-android',
287        '--log-path=%s/logs' % '$JOB_HOME',
288        '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir,
289                                              'gcctools', 'android', 'master',
290                                              'honeycomb_generic_sysroot'),
291        path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools',
292                          'scripts'))
293
294  def RunBenchmark(self,
295                   toolchain_prefix_dir,
296                   checkout_dir,
297                   output_dir,
298                   run_label,
299                   run_experiment,
300                   arch,
301                   base_bench_bin=None):
302    if base_bench_bin:
303      base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin
304    else:
305      base_bench_opt = ''
306
307    return cmd.Shell(
308        'benchmark.sh',
309        '--android_toolchain=%s' % os.path.join('$JOB_TMP',
310                                                toolchain_prefix_dir),
311        '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'),
312        '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir,
313                                            'bench_bin'),
314        base_bench_opt,
315        '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'),
316        '--arch=%s' % arch,
317        '--run_label=%s' % run_label,
318        '--run_experiment=%s' % run_experiment,
319        path=os.path.join(checkout_dir, 'tools', 'scripts'))
320