buildbot_test_toolchains.py revision fe054f1eac0d0d739b74dee6bd45805af86568a2 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18import urllib2
19
20from utils import command_executer
21from utils import logger
22
23from utils import buildbot_utils
24
25# CL that updated GCC ebuilds to use 'next_gcc'.
26USE_NEXT_GCC_PATCH ="230260"
27
28WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
29ROLE_ACCOUNT = "mobiletc-prebuild"
30TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
31
32
33class ToolchainComparator():
34  """
35  Class for doing the nightly tests work.
36  """
37
38  def __init__(self, board, remotes, chromeos_root, weekday):
39    self._board = board
40    self._remotes = remotes
41    self._chromeos_root = chromeos_root
42    self._base_dir = os.getcwd()
43    self._ce = command_executer.GetCommandExecuter()
44    self._l = logger.GetLogger()
45    self._build = "%s-release" % board
46    if not weekday:
47      self._weekday = time.strftime("%a")
48    else:
49      self._weekday = weekday
50
51  def _ParseVanillaImage(self, trybot_image):
52    """
53    Parse a trybot artifact name to get corresponding vanilla image.
54
55    This function takes an artifact name, such as
56    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
57    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
58    """
59    start_pos = trybot_image.find(self._build)
60    end_pos = trybot_image.rfind("-b")
61    vanilla_image = trybot_image[start_pos:end_pos]
62    return vanilla_image
63
64
65  def _FinishSetup(self):
66    """
67    Make sure testing_rsa file is properly set up.
68    """
69    # Fix protections on ssh key
70    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
71               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
72               "/testing_rsa")
73    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
74    if ret_val:
75      raise Exception("chmod for testing_rsa failed")
76
77  def _TestImages(self, trybot_image, vanilla_image):
78    """
79    Create crosperf experiment file.
80
81    Given the names of the trybot and vanilla images, create the
82    appropriate crosperf experiment file and launch crosperf on it.
83    """
84    experiment_file_dir = os.path.join (self._chromeos_root, "..",
85                                        self._weekday)
86    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
87    experiment_file = os.path.join (experiment_file_dir,
88                                    experiment_file_name)
89    experiment_header = """
90    board: %s
91    remote: %s
92    """ % (self._board, self._remotes)
93    experiment_tests = """
94    benchmark: all_perfv2 {
95      suite: telemetry_Crosperf
96      iterations: 3
97    }
98    """
99    with open(experiment_file, "w") as f:
100      print >> f, experiment_header
101      print >> f, experiment_tests
102
103      # Now add vanilla to test file.
104      official_image = """
105          vanilla_image {
106            chromeos_root: %s
107            build: %s
108          }
109          """ % (self._chromeos_root, vanilla_image)
110      print >> f, official_image
111
112      experiment_image = """
113          test_image {
114            chromeos_root: %s
115            build: %s
116          }
117          """ % (self._chromeos_root, trybot_image)
118      print >> f, experiment_image
119
120    crosperf = os.path.join(TOOLCHAIN_DIR,
121                            "crosperf",
122                            "crosperf")
123    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
124    ret = self._ce.RunCommand(command)
125    if ret:
126      raise Exception("Couldn't run crosperf!")
127    return ret
128
129
130  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
131    """
132    Put files in place for running seven-day reports.
133
134    Create tar files of the custom and official images and copy them
135    to the weekly reports directory, so they exist when the weekly report
136    gets generated.  IMPORTANT NOTE: This function must run *after*
137    crosperf has been run; otherwise the vanilla images will not be there.
138    """
139
140    dry_run = False
141    if (os.getlogin() != ROLE_ACCOUNT):
142      self._l.LogOutput("Running this from non-role account; not copying "
143                        "tar files for weekly reports.")
144      dry_run = True
145
146    images_path = os.path.join(os.path.realpath(self._chromeos_root),
147                               "chroot/tmp")
148
149    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
150    dest_dir = os.path.join (data_dir, self._weekday)
151    if not os.path.exists(dest_dir):
152      os.makedirs(dest_dir)
153
154    # Make sure dest_dir is empty (clean out last week's data).
155    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
156    if dry_run:
157      print "CMD: %s" % cmd
158    else:
159      self._ce.RunCommand(cmd)
160
161    # Now create new tar files and copy them over.
162    labels = [ "test", "vanilla" ]
163    for label_name in labels:
164      if label_name == "test":
165        test_path = trybot_image
166      else:
167        test_path = vanilla_image
168      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
169      cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
170             "cp %s %s/.") % (images_path,
171                              tar_file_name,
172                              test_path,
173                              tar_file_name,
174                              dest_dir)
175      if dry_run:
176        print "CMD: %s" % cmd
177        tar_ret = 0
178      else:
179        tar_ret = self._ce.RunCommand(cmd)
180      if tar_ret:
181        self._l.LogOutput("Error while creating/copying test tar file(%s)."
182                          % tar_file_name)
183
184
185  def DoAll(self):
186    """
187    Main function inside ToolchainComparator class.
188
189    Launch trybot, get image names, create crosperf experiment file, run
190    crosperf, and copy images into seven-day report directories.
191    """
192    date_str = datetime.date.today()
193    description = "master_%s_%s_%s" % (USE_NEXT_GCC_PATCH,
194                                       self._build,
195                                       date_str)
196    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
197                                                 self._build,
198                                                 [ USE_NEXT_GCC_PATCH ],
199                                                 description)
200
201    vanilla_image = self._ParseVanillaImage(trybot_image)
202
203    print ("trybot_image: %s" % trybot_image)
204    print ("vanilla_image: %s" % vanilla_image)
205    if len(trybot_image) == 0:
206        self._l.LogError("Unable to find trybot_image for %s!" % description)
207        return 1
208    if len(vanilla_image) == 0:
209        self._l.LogError("Unable to find vanilla image for %s!" % description)
210        return 1
211    if os.getlogin() == ROLE_ACCOUNT:
212      self._FinishSetup()
213
214    if not self._TestImages(trybot_image, vanilla_image):
215      # Only try to copy the image files if the test runs ran successfully.
216      self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
217    return 0
218
219
220def Main(argv):
221  """The main function."""
222
223  # Common initializations
224  command_executer.InitCommandExecuter()
225  parser = optparse.OptionParser()
226  parser.add_option("--remote",
227                    dest="remote",
228                    help="Remote machines to run tests on.")
229  parser.add_option("--board",
230                    dest="board",
231                    default="x86-zgb",
232                    help="The target board.")
233  parser.add_option("--chromeos_root",
234                    dest="chromeos_root",
235                    help="The chromeos root from which to run tests.")
236  parser.add_option("--weekday", default="",
237                    dest="weekday",
238                    help="The day of the week for which to run tests.")
239  options, _ = parser.parse_args(argv)
240  if not options.board:
241    print "Please give a board."
242    return 1
243  if not options.remote:
244    print "Please give at least one remote machine."
245    return 1
246  if not options.chromeos_root:
247    print "Please specify the ChromeOS root directory."
248    return 1
249
250  fc = ToolchainComparator(options.board, options.remote,
251                           options.chromeos_root, options.weekday)
252  return fc.DoAll()
253
254
255if __name__ == "__main__":
256  retval = Main(sys.argv)
257  sys.exit(retval)
258