chromeos.py revision f81680c018729fd4499e1e200d04b48c4b90127c 
1#!/usr/bin/python2.6
2#
3# Copyright 2011 Google Inc. All Rights Reserved.
4
5__author__ = "asharif@google.com (Ahmad Sharif)"
6
7import os.path
8import re
9
10from automation.clients.helper import jobs
11from automation.clients.helper import perforce
12from automation.common import command as cmd
13from automation.common import machine
14
15
16class ScriptsFactory(object):
17  def __init__(self, chromeos_root, scripts_path):
18    self._chromeos_root = chromeos_root
19    self._scripts_path = scripts_path
20
21  def SummarizeResults(self, logs_path):
22    return cmd.Shell("summarize_results.py",
23                     logs_path,
24                     path=self._scripts_path)
25
26  def Buildbot(self, config_name):
27    buildbot = os.path.join(self._chromeos_root,
28                            "chromite/buildbot/cbuildbot.py")
29
30    return cmd.Shell(buildbot,
31                     "--buildroot=%s" % self._chromeos_root,
32                     "--resume",
33                     "--noarchive",
34                     "--noprebuilts",
35                     "--nosync",
36                     "--nouprev",
37                     "--notests",
38                     "--noclean",
39                     config_name)
40
41  def RunBenchmarks(self, board, tests):
42    image_path = os.path.join(self._chromeos_root,
43                              "src/build/images",
44                              board,
45                              "latest/chromiumos_image.bin")
46
47    return cmd.Shell("cros_run_benchmarks.py",
48                     "--remote=$SECONDARY_MACHINES[0]",
49                     "--board=%s" % board,
50                     "--tests=%s" % tests,
51                     "--full_table",
52                     image_path,
53                     path="/home/mobiletc-prebuild")
54
55  def SetupChromeOS(self, version="latest", use_minilayout=False):
56    setup_chromeos = cmd.Shell("setup_chromeos.py",
57                               "--public",
58                               "--dir=%s" % self._chromeos_root,
59                               "--version=%s" % version,
60                               path=self._scripts_path)
61
62    if use_minilayout:
63      setup_chromeos.AddOption("--minilayout")
64    return setup_chromeos
65
66
67class CommandsFactory(object):
68  DEPOT2_DIR = "//depot2/"
69  P4_CHECKOUT_DIR = "perforce2/"
70  P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, "gcctools/chromeos/v14")
71
72  CHROMEOS_ROOT = "chromeos"
73  CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, "src/scripts")
74  CHROMEOS_BUILDS_DIR = "/home/mobiletc-prebuild/www/chromeos_builds"
75
76  def __init__(self, chromeos_version, board, toolchain, p4_snapshot):
77    self.chromeos_version = chromeos_version
78    self.board = board
79    self.toolchain = toolchain
80    self.p4_snapshot = p4_snapshot
81
82    self.scripts = ScriptsFactory(self.CHROMEOS_ROOT, self.P4_VERSION_DIR)
83
84  def AddBuildbotConfig(self, config_name, config_list):
85    config_header = "add_config(%r, [%s])" % (config_name,
86                                              ", ".join(config_list))
87    config_file = os.path.join(self.CHROMEOS_ROOT,
88                               "chromite/buildbot/cbuildbot_config.py")
89    quoted_config_header = "%r" % config_header
90    quoted_config_header = re.sub("'", "\\\"", quoted_config_header)
91
92    return cmd.Pipe(cmd.Shell("echo", quoted_config_header),
93                    cmd.Shell("tee", "--append", config_file))
94
95  def RunBuildbot(self):
96    config_dict = {"board": self.board,
97                   "build_tests": True,
98                   "chrome_tests": True,
99                   "unittests": False,
100                   "vm_tests": False,
101                   "prebuilts": False,
102                   "latest_toolchain": True,
103                   "useflags": ["chrome_internal"],
104                   "usepkg_chroot": True,
105                   self.toolchain: True}
106    config_name = "%s-toolchain-test" % self.board
107    if "arm" in self.board:
108      config_list = ["arm"]
109    else:
110      config_list = []
111    config_list.extend(["internal", "full", "official", str(config_dict)])
112
113    add_config_shell = self.AddBuildbotConfig(config_name, config_list)
114    return cmd.Chain(add_config_shell, self.scripts.Buildbot(config_name))
115
116  def BuildAndBenchmark(self):
117    return cmd.Chain(
118        self.CheckoutV14Dir(),
119        self.SetupChromeOSCheckout(self.chromeos_version, True),
120        self.RunBuildbot(),
121        self.scripts.RunBenchmarks(self.board, "BootPerfServer,10:Page,3"))
122
123  def GetP4Snapshot(self, p4view):
124    p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view)
125
126    if self.p4_snapshot:
127      return p4client.CheckoutFromSnapshot(self.p4_snapshot)
128    else:
129      return p4client.SetupAndDo(p4client.Sync(), p4client.Remove())
130
131  def CheckoutV14Dir(self):
132    p4view = perforce.View(self.DEPOT2_DIR, [
133        perforce.PathMapping("gcctools/chromeos/v14/...")])
134    return self.GetP4Snapshot(p4view)
135
136  def SetupChromeOSCheckout(self, version, use_minilayout=False):
137    version_re = "^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$"
138
139    location = os.path.join(self.CHROMEOS_BUILDS_DIR, version)
140
141    if version in ["weekly", "quarterly"]:
142      assert os.path.islink(location), "Symlink %s does not exist." % location
143
144      location_expanded = os.path.abspath(os.path.realpath(location))
145      version = os.path.basename(location_expanded)
146
147    if version in ["top", "latest"] or re.match(version_re, version):
148      return self.scripts.SetupChromeOS(version, use_minilayout)
149
150    elif version.endswith("bz2") or version.endswith("gz"):
151      return cmd.UnTar(location_expanded, self.CHROMEOS_ROOT)
152
153    else:
154      signature_file_location = os.path.join(location,
155                                             "src/scripts/enter_chroot.sh")
156      assert os.path.exists(signature_file_location), (
157          "Signature file %s does not exist." % signature_file_location)
158
159      return cmd.Copy(location, to_dir=self.CHROMEOS_ROOT, recursive=True)
160
161
162class JobsFactory(object):
163  def __init__(self, chromeos_version="top", board="x86-mario",
164               toolchain="trunk", p4_snapshot=""):
165    self.chromeos_version = chromeos_version
166    self.board = board
167    self.toolchain = toolchain
168
169    self.commands = CommandsFactory(chromeos_version, board, toolchain,
170                                    p4_snapshot)
171
172  def BuildAndBenchmark(self):
173    command = self.commands.BuildAndBenchmark()
174
175    label = "BuildAndBenchmark(%s,%s,%s)" % (
176        self.toolchain, self.board, self.chromeos_version)
177
178    machine_label = "chromeos-%s" % self.board
179
180    job = jobs.CreateLinuxJob(label, command)
181    job.DependsOnMachine(
182        machine.MachineSpecification(label=machine_label, lock_required=True),
183        False)
184
185    return job
186