crosstool.py revision f81680c018729fd4499e1e200d04b48c4b90127c 
1#!/usr/bin/python2.6
2#
3# Copyright 2011 Google Inc. All Rights Reserved.
4
5__author__ = 'kbaclawski@google.com (Krystian Baclawski)'
6
7import os.path
8import time
9
10from automation.clients.helper import jobs
11from automation.clients.helper import perforce
12from automation.common import command as cmd
13from automation.common import job
14
15
16class JobsFactory(object):
17  def __init__(self):
18    self.commands = CommandsFactory()
19
20  def CheckoutCrosstool(self, target):
21    command = self.commands.CheckoutCrosstool()
22    new_job = jobs.CreateLinuxJob('CheckoutCrosstool(%s)' % target, command)
23    checkout_dir_dep = job.FolderDependency(
24        new_job, CommandsFactory.CHECKOUT_DIR)
25    manifests_dir_dep = job.FolderDependency(
26        new_job, os.path.join(self.commands.buildit_path, target), 'manifests')
27    return new_job, checkout_dir_dep, manifests_dir_dep
28
29  def BuildRelease(self, checkout_dir, target):
30    command = self.commands.BuildRelease(target)
31    new_job = jobs.CreateLinuxJob('BuildRelease(%s)' % target, command)
32    new_job.DependsOnFolder(checkout_dir)
33    build_tree_dep = job.FolderDependency(
34        new_job, self.commands.buildit_work_dir_path)
35    return new_job, build_tree_dep
36
37  def RunTests(self, checkout_dir, build_tree_dir, target, board, component):
38    command = self.commands.RunTests(target, board, component)
39    new_job = jobs.CreateLinuxJob(
40        'RunTests(%s, %s, %s)' % (target, component, board), command)
41    new_job.DependsOnFolder(checkout_dir)
42    new_job.DependsOnFolder(build_tree_dir)
43    testrun_dir_dep = job.FolderDependency(
44        new_job, self.commands.dejagnu_output_path, board)
45    return new_job, testrun_dir_dep
46
47  def GenerateReport(self, testrun_dirs, manifests_dir, target, boards):
48    command = self.commands.GenerateReport(boards)
49    new_job = jobs.CreateLinuxJob('GenerateReport(%s)' % target, command)
50    new_job.DependsOnFolder(manifests_dir)
51    for testrun_dir in testrun_dirs:
52      new_job.DependsOnFolder(testrun_dir)
53    return new_job
54
55
56class CommandsFactory(object):
57  CHECKOUT_DIR = 'crosstool-checkout-dir'
58
59  def __init__(self):
60    self.buildit_path = os.path.join(
61        self.CHECKOUT_DIR, 'gcctools', 'crosstool', 'v15')
62
63    self.buildit_work_dir = 'buildit-tmp'
64    self.buildit_work_dir_path = os.path.join('$JOB_TMP', self.buildit_work_dir)
65    self.dejagnu_output_path = os.path.join(
66        self.buildit_work_dir_path, 'dejagnu-output')
67
68    paths = {
69        'gcctools': [
70            'crosstool/v15/...',
71            'scripts/...'],
72        'gcctools/google_vendor_src_branch': [
73            'binutils/binutils-2.21/...',
74            'gdb/gdb-7.2.x/...',
75            'zlib/zlib-1.2.3/...'],
76        'gcctools/vendor_src': [
77            'gcc/google/gcc-4_6/...']}
78
79    p4view = perforce.View('depot2',
80                           perforce.PathMapping.ListFromPathDict(paths))
81
82    self.p4client = perforce.CommandsFactory(self.CHECKOUT_DIR, p4view)
83
84  def CheckoutCrosstool(self):
85    p4client = self.p4client
86
87    return p4client.SetupAndDo(p4client.Sync(),
88                               p4client.SaveCurrentCLNumber('CLNUM'),
89                               p4client.Remove())
90
91  def BuildRelease(self, target):
92    clnum_path = os.path.join('$JOB_TMP', self.CHECKOUT_DIR, 'CLNUM')
93
94    toolchain_root = os.path.join(
95        '/google/data/rw/projects/toolchains', target, 'unstable')
96    toolchain_path = os.path.join(toolchain_root, '${CLNUM}')
97
98    build_toolchain = cmd.Wrapper(
99        cmd.Chain(
100          cmd.MakeDir(toolchain_path),
101          cmd.Shell(
102              'buildit',
103              '--keep-work-dir',
104              '--build-type=release',
105              '--work-dir=%s' % self.buildit_work_dir_path,
106              '--results-dir=%s' % toolchain_path,
107              '--force-release=%s' % '${CLNUM}',
108              target,
109              path='.')),
110        cwd=self.buildit_path,
111        umask='0022',
112        env={'CLNUM': '$(< %s)' % clnum_path})
113
114    # remove all but 10 most recent directories
115    remove_old_toolchains_from_x20 = cmd.Wrapper(
116        cmd.Pipe(
117            cmd.Shell('ls', '-1', '-r'),
118            cmd.Shell('sed', '-e', '1,10d'),
119            cmd.Shell('xargs', 'rm', '-r', '-f')),
120        cwd=toolchain_root)
121
122    return cmd.Chain(build_toolchain, remove_old_toolchains_from_x20)
123
124  def RunTests(self, target, board, component='gcc'):
125    dejagnu_flags = ['--outdir=%s' % self.dejagnu_output_path,
126                     '--target_board=%s' % board]
127
128    # Look for {pandaboard,qemu}.exp files in
129    # //depot/google3/experimental/users/kbaclawski/dejagnu/boards
130
131    site_exp_file = os.path.join('/google/src/head/depot/google3',
132                                 'experimental/users/kbaclawski',
133                                 'dejagnu/site.exp')
134
135    build_dir_path = os.path.join(
136        target, 'rpmbuild/BUILD/crosstool*-0.0', 'build-%s' % component)
137
138    run_dejagnu = cmd.Wrapper(
139        cmd.Chain(
140            cmd.MakeDir(self.dejagnu_output_path),
141            cmd.Shell('make', 'check', '-k',
142                      '-j $(grep -c processor /proc/cpuinfo)',
143                      'RUNTESTFLAGS="%s"' % ' '.join(dejagnu_flags),
144                      'DEJAGNU="%s"' % site_exp_file,
145                      ignore_error=True)),
146        cwd=os.path.join(self.buildit_work_dir_path, build_dir_path),
147        env={'REMOTE_TMPDIR': 'job-$JOB_ID'})
148
149    save_results = cmd.Copy(
150        self.dejagnu_output_path, to_dir='$JOB_TMP/results', recursive=True)
151
152    return cmd.Chain(run_dejagnu, save_results)
153
154  def GenerateReport(self, boards):
155    sumfiles = [os.path.join('$JOB_TMP', board, '*.sum') for board in boards]
156
157    return cmd.Wrapper(
158        cmd.Shell('dejagnu.sh', 'report',
159                  '-m', '$JOB_TMP/manifests/*.xfail',
160                  '-o', '$JOB_TMP/results/report.html',
161                  *sumfiles,
162                  path='.'),
163        cwd='$HOME/automation/clients/report')
164