buildbot_test_toolchains.py revision 7f3190b3fb89522ea85b87200241962634369e9a 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18import urllib2
19
20from utils import command_executer
21from utils import logger
22
23from utils import buildbot_utils
24
25# CL that updated GCC ebuilds to use 'next_gcc'.
26USE_NEXT_GCC_PATCH ="230260"
27
28WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
29ROLE_ACCOUNT = "mobiletc-prebuild"
30TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
31MAIL_PROGRAM = "~/var/bin/mail-sheriff"
32
33class ToolchainComparator():
34  """
35  Class for doing the nightly tests work.
36  """
37
38  def __init__(self, board, remotes, chromeos_root, weekday):
39    self._board = board
40    self._remotes = remotes
41    self._chromeos_root = chromeos_root
42    self._base_dir = os.getcwd()
43    self._ce = command_executer.GetCommandExecuter()
44    self._l = logger.GetLogger()
45    self._build = "%s-release" % board
46    if not weekday:
47      self._weekday = time.strftime("%a")
48    else:
49      self._weekday = weekday
50    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
51                                           "%Y-%m-%d_%H:%M:%S")
52    self._reports_dir = os.path.join(
53        os.path.expanduser("~/nightly_test_reports"),
54        "%s.%s" % (timestamp, board),
55        )
56
57  def _ParseVanillaImage(self, trybot_image):
58    """
59    Parse a trybot artifact name to get corresponding vanilla image.
60
61    This function takes an artifact name, such as
62    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
63    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
64    """
65    start_pos = trybot_image.find(self._build)
66    end_pos = trybot_image.rfind("-b")
67    vanilla_image = trybot_image[start_pos:end_pos]
68    return vanilla_image
69
70  def _FinishSetup(self):
71    """
72    Make sure testing_rsa file is properly set up.
73    """
74    # Fix protections on ssh key
75    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
76               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
77               "/testing_rsa")
78    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
79    if ret_val != 0:
80      raise RuntimeError("chmod for testing_rsa failed")
81
82  def _TestImages(self, trybot_image, vanilla_image):
83    """
84    Create crosperf experiment file.
85
86    Given the names of the trybot and vanilla images, create the
87    appropriate crosperf experiment file and launch crosperf on it.
88    """
89    experiment_file_dir = os.path.join (self._chromeos_root, "..",
90                                        self._weekday)
91    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
92    experiment_file = os.path.join (experiment_file_dir,
93                                    experiment_file_name)
94    experiment_header = """
95    board: %s
96    remote: %s
97    """ % (self._board, self._remotes)
98    experiment_tests = """
99    benchmark: all_perfv2 {
100      suite: telemetry_Crosperf
101      iterations: 3
102    }
103    """
104    with open(experiment_file, "w") as f:
105      print >> f, experiment_header
106      print >> f, experiment_tests
107
108      # Now add vanilla to test file.
109      official_image = """
110          vanilla_image {
111            chromeos_root: %s
112            build: %s
113          }
114          """ % (self._chromeos_root, vanilla_image)
115      print >> f, official_image
116
117      experiment_image = """
118          test_image {
119            chromeos_root: %s
120            build: %s
121          }
122          """ % (self._chromeos_root, trybot_image)
123      print >> f, experiment_image
124
125    crosperf = os.path.join(TOOLCHAIN_DIR,
126                            "crosperf",
127                            "crosperf")
128    command = ("%s --no_email=True --results_dir=%s %s" % (crosperf,
129                                                           self._reports_dir,
130                                                           experiment_file))
131    ret = self._ce.RunCommand(command)
132    if ret != 0:
133      raise RuntimeError("Couldn't run crosperf!")
134    return
135
136  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
137    """
138    Put files in place for running seven-day reports.
139
140    Create tar files of the custom and official images and copy them
141    to the weekly reports directory, so they exist when the weekly report
142    gets generated.  IMPORTANT NOTE: This function must run *after*
143    crosperf has been run; otherwise the vanilla images will not be there.
144    """
145
146    dry_run = False
147    if (os.getlogin() != ROLE_ACCOUNT):
148      self._l.LogOutput("Running this from non-role account; not copying "
149                        "tar files for weekly reports.")
150      dry_run = True
151
152    images_path = os.path.join(os.path.realpath(self._chromeos_root),
153                               "chroot/tmp")
154
155    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
156    dest_dir = os.path.join (data_dir, self._weekday)
157    if not os.path.exists(dest_dir):
158      os.makedirs(dest_dir)
159
160    # Make sure dest_dir is empty (clean out last week's data).
161    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
162    if dry_run:
163      print "CMD: %s" % cmd
164    else:
165      self._ce.RunCommand(cmd)
166
167    # Now create new tar files and copy them over.
168    labels = [ "test", "vanilla" ]
169    for label_name in labels:
170      if label_name == "test":
171        test_path = trybot_image
172      else:
173        test_path = vanilla_image
174      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
175      cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
176             "cp %s %s/.") % (images_path,
177                              tar_file_name,
178                              test_path,
179                              tar_file_name,
180                              dest_dir)
181      if dry_run:
182        print "CMD: %s" % cmd
183        tar_ret = 0
184      else:
185        tar_ret = self._ce.RunCommand(cmd)
186      if tar_ret:
187        self._l.LogOutput("Error while creating/copying test tar file(%s)."
188                          % tar_file_name)
189
190  def _SendEmail(self):
191    """Find email message generated by crosperf and send it."""
192    filename = os.path.join(self._reports_dir,
193                            "msg_body.html")
194    if (os.path.exists(filename) and
195        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
196      command = ('cat %s | %s -s "buildbot test results, %s" -team -html'
197                 % (filename, MAIL_PROGRAM, self._board))
198      self._ce.RunCommand(command)
199
200  def DoAll(self):
201    """
202    Main function inside ToolchainComparator class.
203
204    Launch trybot, get image names, create crosperf experiment file, run
205    crosperf, and copy images into seven-day report directories.
206    """
207    date_str = datetime.date.today()
208    description = "master_%s_%s_%s" % (USE_NEXT_GCC_PATCH,
209                                       self._build,
210                                       date_str)
211    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
212                                                 self._build,
213                                                 [ USE_NEXT_GCC_PATCH ],
214                                                 description,
215                                                 build_toolchain=True)
216
217    vanilla_image = self._ParseVanillaImage(trybot_image)
218
219    print ("trybot_image: %s" % trybot_image)
220    print ("vanilla_image: %s" % vanilla_image)
221    if len(trybot_image) == 0:
222        self._l.LogError("Unable to find trybot_image for %s!" % description)
223        return 1
224    if len(vanilla_image) == 0:
225        self._l.LogError("Unable to find vanilla image for %s!" % description)
226        return 1
227    if os.getlogin() == ROLE_ACCOUNT:
228      self._FinishSetup()
229
230    self._TestImages(trybot_image, vanilla_image)
231    self._SendEmail()
232    # Only try to copy the image files if the test runs ran successfully.
233    self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
234    return 0
235
236
237def Main(argv):
238  """The main function."""
239
240  # Common initializations
241  command_executer.InitCommandExecuter()
242  parser = optparse.OptionParser()
243  parser.add_option("--remote",
244                    dest="remote",
245                    help="Remote machines to run tests on.")
246  parser.add_option("--board",
247                    dest="board",
248                    default="x86-zgb",
249                    help="The target board.")
250  parser.add_option("--chromeos_root",
251                    dest="chromeos_root",
252                    help="The chromeos root from which to run tests.")
253  parser.add_option("--weekday", default="",
254                    dest="weekday",
255                    help="The day of the week for which to run tests.")
256  options, _ = parser.parse_args(argv)
257  if not options.board:
258    print "Please give a board."
259    return 1
260  if not options.remote:
261    print "Please give at least one remote machine."
262    return 1
263  if not options.chromeos_root:
264    print "Please specify the ChromeOS root directory."
265    return 1
266
267  fc = ToolchainComparator(options.board, options.remote,
268                           options.chromeos_root, options.weekday)
269  return fc.DoAll()
270
271
272if __name__ == "__main__":
273  retval = Main(sys.argv)
274  sys.exit(retval)
275