buildbot_test_toolchains.py revision f456b2035aec47012d7d111b0c5e21a3319823c0 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18import urllib2
19
20from utils import command_executer
21from utils import logger
22
23from utils import buildbot_utils
24
25# CL that updated GCC ebuilds to use 'next_gcc'.
26USE_NEXT_GCC_PATCH ="230260"
27
28WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
29ROLE_ACCOUNT = "mobiletc-prebuild"
30TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
31MAIL_PROGRAM = "~/var/bin/mail-sheriff"
32
33class ToolchainComparator():
34  """
35  Class for doing the nightly tests work.
36  """
37
38  def __init__(self, board, remotes, chromeos_root, weekday):
39    self._board = board
40    self._remotes = remotes
41    self._chromeos_root = chromeos_root
42    self._base_dir = os.getcwd()
43    self._ce = command_executer.GetCommandExecuter()
44    self._l = logger.GetLogger()
45    self._build = "%s-release" % board
46    if not weekday:
47      self._weekday = time.strftime("%a")
48    else:
49      self._weekday = weekday
50    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
51                                           "%Y-%m-%d_%H:%M:%S")
52    self._reports_dir = os.path.join(
53        os.path.expanduser("~/nightly_test_reports"),
54        "%s.%s" % (timestamp, board),
55        )
56
57  def _ParseVanillaImage(self, trybot_image):
58    """
59    Parse a trybot artifact name to get corresponding vanilla image.
60
61    This function takes an artifact name, such as
62    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
63    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
64    """
65    start_pos = trybot_image.find(self._build)
66    end_pos = trybot_image.rfind("-b")
67    vanilla_image = trybot_image[start_pos:end_pos]
68    return vanilla_image
69
70  def _FinishSetup(self):
71    """
72    Make sure testing_rsa file is properly set up.
73    """
74    # Fix protections on ssh key
75    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
76               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
77               "/testing_rsa")
78    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
79    if ret_val != 0:
80      raise RuntimeError("chmod for testing_rsa failed")
81
82  def _TestImages(self, trybot_image, vanilla_image):
83    """
84    Create crosperf experiment file.
85
86    Given the names of the trybot and vanilla images, create the
87    appropriate crosperf experiment file and launch crosperf on it.
88    """
89    experiment_file_dir = os.path.join (self._chromeos_root, "..",
90                                        self._weekday)
91    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
92    experiment_file = os.path.join (experiment_file_dir,
93                                    experiment_file_name)
94    experiment_header = """
95    board: %s
96    remote: %s
97    retries: 1
98    """ % (self._board, self._remotes)
99    experiment_tests = """
100    benchmark: all_toolchain_perf {
101      suite: telemetry_Crosperf
102      iterations: 3
103    }
104    """
105    with open(experiment_file, "w") as f:
106      print >> f, experiment_header
107      print >> f, experiment_tests
108
109      # Now add vanilla to test file.
110      official_image = """
111          vanilla_image {
112            chromeos_root: %s
113            chrome_src: /usr/local/google/crostc/new-chrome-src
114            build: %s
115          }
116          """ % (self._chromeos_root, vanilla_image)
117      print >> f, official_image
118
119      experiment_image = """
120          test_image {
121            chromeos_root: %s
122            chrome_src: /usr/local/google/crostc/new-chrome-src
123            build: %s
124          }
125          """ % (self._chromeos_root, trybot_image)
126      print >> f, experiment_image
127
128    crosperf = os.path.join(TOOLCHAIN_DIR,
129                            "crosperf",
130                            "crosperf")
131    command = ("%s --no_email=True --results_dir=%s %s" %
132               (crosperf, self._reports_dir, experiment_file))
133
134    ret = self._ce.RunCommand(command)
135    if ret != 0:
136      raise RuntimeError("Couldn't run crosperf!")
137    return
138
139  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
140    """
141    Put files in place for running seven-day reports.
142
143    Create tar files of the custom and official images and copy them
144    to the weekly reports directory, so they exist when the weekly report
145    gets generated.  IMPORTANT NOTE: This function must run *after*
146    crosperf has been run; otherwise the vanilla images will not be there.
147    """
148
149    dry_run = False
150    if (os.getlogin() != ROLE_ACCOUNT):
151      self._l.LogOutput("Running this from non-role account; not copying "
152                        "tar files for weekly reports.")
153      dry_run = True
154
155    images_path = os.path.join(os.path.realpath(self._chromeos_root),
156                               "chroot/tmp")
157
158    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
159    dest_dir = os.path.join (data_dir, self._weekday)
160    if not os.path.exists(dest_dir):
161      os.makedirs(dest_dir)
162
163    # Make sure dest_dir is empty (clean out last week's data).
164    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
165    if dry_run:
166      print "CMD: %s" % cmd
167    else:
168      self._ce.RunCommand(cmd)
169
170    # Now create new tar files and copy them over.
171    labels = [ "test", "vanilla" ]
172    for label_name in labels:
173      if label_name == "test":
174        test_path = trybot_image
175      else:
176        test_path = vanilla_image
177      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
178      cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
179             "cp %s %s/.") % (images_path,
180                              tar_file_name,
181                              test_path,
182                              tar_file_name,
183                              dest_dir)
184      if dry_run:
185        print "CMD: %s" % cmd
186        tar_ret = 0
187      else:
188        tar_ret = self._ce.RunCommand(cmd)
189      if tar_ret:
190        self._l.LogOutput("Error while creating/copying test tar file(%s)."
191                          % tar_file_name)
192
193  def _SendEmail(self):
194    """Find email message generated by crosperf and send it."""
195    filename = os.path.join(self._reports_dir,
196                            "msg_body.html")
197    if (os.path.exists(filename) and
198        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
199      command = ('cat %s | %s -s "buildbot test results, %s" -team -html'
200                 % (filename, MAIL_PROGRAM, self._board))
201      self._ce.RunCommand(command)
202
203  def DoAll(self):
204    """
205    Main function inside ToolchainComparator class.
206
207    Launch trybot, get image names, create crosperf experiment file, run
208    crosperf, and copy images into seven-day report directories.
209    """
210    date_str = datetime.date.today()
211    description = "master_%s_%s_%s" % (USE_NEXT_GCC_PATCH,
212                                       self._build,
213                                       date_str)
214    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
215                                                 self._build,
216                                                 [ USE_NEXT_GCC_PATCH ],
217                                                 description,
218                                                 build_toolchain=True)
219
220    vanilla_image = self._ParseVanillaImage(trybot_image)
221
222    print ("trybot_image: %s" % trybot_image)
223    print ("vanilla_image: %s" % vanilla_image)
224    if len(trybot_image) == 0:
225        self._l.LogError("Unable to find trybot_image for %s!" % description)
226        return 1
227    if len(vanilla_image) == 0:
228        self._l.LogError("Unable to find vanilla image for %s!" % description)
229        return 1
230    if os.getlogin() == ROLE_ACCOUNT:
231      self._FinishSetup()
232
233    self._TestImages(trybot_image, vanilla_image)
234    self._SendEmail()
235    # Only try to copy the image files if the test runs ran successfully.
236    self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
237    return 0
238
239
240def Main(argv):
241  """The main function."""
242
243  # Common initializations
244  command_executer.InitCommandExecuter()
245  parser = optparse.OptionParser()
246  parser.add_option("--remote",
247                    dest="remote",
248                    help="Remote machines to run tests on.")
249  parser.add_option("--board",
250                    dest="board",
251                    default="x86-zgb",
252                    help="The target board.")
253  parser.add_option("--chromeos_root",
254                    dest="chromeos_root",
255                    help="The chromeos root from which to run tests.")
256  parser.add_option("--weekday", default="",
257                    dest="weekday",
258                    help="The day of the week for which to run tests.")
259  options, _ = parser.parse_args(argv)
260  if not options.board:
261    print "Please give a board."
262    return 1
263  if not options.remote:
264    print "Please give at least one remote machine."
265    return 1
266  if not options.chromeos_root:
267    print "Please specify the ChromeOS root directory."
268    return 1
269
270  fc = ToolchainComparator(options.board, options.remote,
271                           options.chromeos_root, options.weekday)
272  return fc.DoAll()
273
274
275if __name__ == "__main__":
276  retval = Main(sys.argv)
277  sys.exit(retval)
278