buildbot_test_toolchains.py revision e1efeb8c08e51aac3ca80859a015758d682e86e6 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18import urllib2
19
20from utils import command_executer
21from utils import logger
22
23from utils import buildbot_utils
24
25# CL that updated GCC ebuilds to use 'next_gcc'.
26USE_NEXT_GCC_PATCH ="230260"
27
28WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
29ROLE_ACCOUNT = "mobiletc-prebuild"
30TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
31MAIL_PROGRAM = "~/var/bin/mail-sheriff"
32
33class ToolchainComparator():
34  """
35  Class for doing the nightly tests work.
36  """
37
38  def __init__(self, board, remotes, chromeos_root, weekday):
39    self._board = board
40    self._remotes = remotes
41    self._chromeos_root = chromeos_root
42    self._base_dir = os.getcwd()
43    self._ce = command_executer.GetCommandExecuter()
44    self._l = logger.GetLogger()
45    self._build = "%s-release" % board
46    if not weekday:
47      self._weekday = time.strftime("%a")
48    else:
49      self._weekday = weekday
50    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
51                                           "%Y-%m-%d_%H:%M:%S")
52    self._reports_dir = os.path.join(
53        os.path.expanduser("~/nightly_test_reports"),
54        "%s.%s" % (timestamp, board),
55        )
56
57  def _ParseVanillaImage(self, trybot_image):
58    """
59    Parse a trybot artifact name to get corresponding vanilla image.
60
61    This function takes an artifact name, such as
62    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
63    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
64    """
65    start_pos = trybot_image.find(self._build)
66    end_pos = trybot_image.rfind("-b")
67    vanilla_image = trybot_image[start_pos:end_pos]
68    return vanilla_image
69
70  def _FinishSetup(self):
71    """
72    Make sure testing_rsa file is properly set up.
73    """
74    # Fix protections on ssh key
75    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
76               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
77               "/testing_rsa")
78    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
79    if ret_val != 0:
80      raise RuntimeError("chmod for testing_rsa failed")
81
82  def _TestImages(self, trybot_image, vanilla_image):
83    """
84    Create crosperf experiment file.
85
86    Given the names of the trybot and vanilla images, create the
87    appropriate crosperf experiment file and launch crosperf on it.
88    """
89    experiment_file_dir = os.path.join (self._chromeos_root, "..",
90                                        self._weekday)
91    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
92    experiment_file = os.path.join (experiment_file_dir,
93                                    experiment_file_name)
94    experiment_header = """
95    board: %s
96    remote: %s
97    retries: 1
98    """ % (self._board, self._remotes)
99    experiment_tests = """
100    benchmark: all_perfv2 {
101      suite: telemetry_Crosperf
102      iterations: 3
103    }
104    """
105    with open(experiment_file, "w") as f:
106      print >> f, experiment_header
107      print >> f, experiment_tests
108
109      # Now add vanilla to test file.
110      official_image = """
111          vanilla_image {
112            chromeos_root: %s
113            build: %s
114          }
115          """ % (self._chromeos_root, vanilla_image)
116      print >> f, official_image
117
118      experiment_image = """
119          test_image {
120            chromeos_root: %s
121            build: %s
122          }
123          """ % (self._chromeos_root, trybot_image)
124      print >> f, experiment_image
125
126    crosperf = os.path.join(TOOLCHAIN_DIR,
127                            "crosperf",
128                            "crosperf")
129    command = ("%s --no_email=True --use_file_locks=True --results_dir=%s %s" %
130               (crosperf, self._reports_dir, experiment_file))
131
132    ret = self._ce.RunCommand(command)
133    if ret != 0:
134      raise RuntimeError("Couldn't run crosperf!")
135    return
136
137  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
138    """
139    Put files in place for running seven-day reports.
140
141    Create tar files of the custom and official images and copy them
142    to the weekly reports directory, so they exist when the weekly report
143    gets generated.  IMPORTANT NOTE: This function must run *after*
144    crosperf has been run; otherwise the vanilla images will not be there.
145    """
146
147    dry_run = False
148    if (os.getlogin() != ROLE_ACCOUNT):
149      self._l.LogOutput("Running this from non-role account; not copying "
150                        "tar files for weekly reports.")
151      dry_run = True
152
153    images_path = os.path.join(os.path.realpath(self._chromeos_root),
154                               "chroot/tmp")
155
156    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
157    dest_dir = os.path.join (data_dir, self._weekday)
158    if not os.path.exists(dest_dir):
159      os.makedirs(dest_dir)
160
161    # Make sure dest_dir is empty (clean out last week's data).
162    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
163    if dry_run:
164      print "CMD: %s" % cmd
165    else:
166      self._ce.RunCommand(cmd)
167
168    # Now create new tar files and copy them over.
169    labels = [ "test", "vanilla" ]
170    for label_name in labels:
171      if label_name == "test":
172        test_path = trybot_image
173      else:
174        test_path = vanilla_image
175      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
176      cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
177             "cp %s %s/.") % (images_path,
178                              tar_file_name,
179                              test_path,
180                              tar_file_name,
181                              dest_dir)
182      if dry_run:
183        print "CMD: %s" % cmd
184        tar_ret = 0
185      else:
186        tar_ret = self._ce.RunCommand(cmd)
187      if tar_ret:
188        self._l.LogOutput("Error while creating/copying test tar file(%s)."
189                          % tar_file_name)
190
191  def _SendEmail(self):
192    """Find email message generated by crosperf and send it."""
193    filename = os.path.join(self._reports_dir,
194                            "msg_body.html")
195    if (os.path.exists(filename) and
196        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
197      command = ('cat %s | %s -s "buildbot test results, %s" -team -html'
198                 % (filename, MAIL_PROGRAM, self._board))
199      self._ce.RunCommand(command)
200
201  def DoAll(self):
202    """
203    Main function inside ToolchainComparator class.
204
205    Launch trybot, get image names, create crosperf experiment file, run
206    crosperf, and copy images into seven-day report directories.
207    """
208    date_str = datetime.date.today()
209    description = "master_%s_%s_%s" % (USE_NEXT_GCC_PATCH,
210                                       self._build,
211                                       date_str)
212    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
213                                                 self._build,
214                                                 [ USE_NEXT_GCC_PATCH ],
215                                                 description,
216                                                 build_toolchain=True)
217
218    vanilla_image = self._ParseVanillaImage(trybot_image)
219
220    print ("trybot_image: %s" % trybot_image)
221    print ("vanilla_image: %s" % vanilla_image)
222    if len(trybot_image) == 0:
223        self._l.LogError("Unable to find trybot_image for %s!" % description)
224        return 1
225    if len(vanilla_image) == 0:
226        self._l.LogError("Unable to find vanilla image for %s!" % description)
227        return 1
228    if os.getlogin() == ROLE_ACCOUNT:
229      self._FinishSetup()
230
231    self._TestImages(trybot_image, vanilla_image)
232    self._SendEmail()
233    # Only try to copy the image files if the test runs ran successfully.
234    self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
235    return 0
236
237
238def Main(argv):
239  """The main function."""
240
241  # Common initializations
242  command_executer.InitCommandExecuter()
243  parser = optparse.OptionParser()
244  parser.add_option("--remote",
245                    dest="remote",
246                    help="Remote machines to run tests on.")
247  parser.add_option("--board",
248                    dest="board",
249                    default="x86-zgb",
250                    help="The target board.")
251  parser.add_option("--chromeos_root",
252                    dest="chromeos_root",
253                    help="The chromeos root from which to run tests.")
254  parser.add_option("--weekday", default="",
255                    dest="weekday",
256                    help="The day of the week for which to run tests.")
257  options, _ = parser.parse_args(argv)
258  if not options.board:
259    print "Please give a board."
260    return 1
261  if not options.remote:
262    print "Please give at least one remote machine."
263    return 1
264  if not options.chromeos_root:
265    print "Please specify the ChromeOS root directory."
266    return 1
267
268  fc = ToolchainComparator(options.board, options.remote,
269                           options.chromeos_root, options.weekday)
270  return fc.DoAll()
271
272
273if __name__ == "__main__":
274  retval = Main(sys.argv)
275  sys.exit(retval)
276