buildbot_test_toolchains.py revision 776d528ff8fa7e66d6abf96a735050c70f950a2b 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18import urllib2
19
20from utils import command_executer
21from utils import logger
22
23from utils import buildbot_utils
24
25# CL that updated GCC ebuilds to use 'next_gcc'.
26USE_NEXT_GCC_PATCH ="230260"
27
28WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
29ROLE_ACCOUNT = "mobiletc-prebuild"
30TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
31
32
33class ToolchainComparator():
34  """
35  Class for doing the nightly tests work.
36  """
37
38  def __init__(self, board, remotes, chromeos_root, weekday):
39    self._board = board
40    self._remotes = remotes
41    self._chromeos_root = chromeos_root
42    self._base_dir = os.getcwd()
43    self._ce = command_executer.GetCommandExecuter()
44    self._l = logger.GetLogger()
45    self._build = "%s-release" % board
46    if not weekday:
47      self._weekday = time.strftime("%a")
48    else:
49      self._weekday = weekday
50
51  def _ParseVanillaImage(self, trybot_image):
52    """
53    Parse a trybot artifact name to get corresponding vanilla image.
54
55    This function takes an artifact name, such as
56    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
57    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
58    """
59    start_pos = trybot_image.find(self._build)
60    end_pos = trybot_image.rfind("-b")
61    vanilla_image = trybot_image[start_pos:end_pos]
62    return vanilla_image
63
64
65  def _FinishSetup(self):
66    """
67    Make sure testing_rsa file is properly set up.
68    """
69    # Fix protections on ssh key
70    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
71               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
72               "/testing_rsa")
73    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
74    if ret_val:
75      raise Exception("chmod for testing_rsa failed")
76
77  def _TestImages(self, trybot_image, vanilla_image):
78    """
79    Create crosperf experiment file.
80
81    Given the names of the trybot and vanilla images, create the
82    appropriate crosperf experiment file and launch crosperf on it.
83    """
84    experiment_file_dir = os.path.join (self._chromeos_root, "..",
85                                        self._weekday)
86    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
87    experiment_file = os.path.join (experiment_file_dir,
88                                    experiment_file_name)
89    experiment_header = """
90    board: %s
91    remote: %s
92    """ % (self._board, self._remotes)
93    experiment_tests = """
94    benchmark: all_perfv2 {
95      suite: telemetry_Crosperf
96      iterations: 3
97    }
98    """
99    with open(experiment_file, "w") as f:
100      print >> f, experiment_header
101      print >> f, experiment_tests
102
103      # Now add vanilla to test file.
104      official_image = """
105          vanilla_image {
106            chromeos_root: %s
107            build: %s
108          }
109          """ % (self._chromeos_root, vanilla_image)
110      print >> f, official_image
111
112      experiment_image = """
113          test_image {
114            chromeos_root: %s
115            build: %s
116          }
117          """ % (self._chromeos_root, trybot_image)
118      print >> f, experiment_image
119
120    crosperf = os.path.join(TOOLCHAIN_DIR,
121                            "crosperf",
122                            "crosperf")
123    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
124    ret = self._ce.RunCommand(command)
125    if ret:
126      raise Exception("Couldn't run crosperf!")
127    return ret
128
129
130  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
131    """
132    Put files in place for running seven-day reports.
133
134    Create tar files of the custom and official images and copy them
135    to the weekly reports directory, so they exist when the weekly report
136    gets generated.  IMPORTANT NOTE: This function must run *after*
137    crosperf has been run; otherwise the vanilla images will not be there.
138    """
139
140    dry_run = False
141    if (os.getlogin() != ROLE_ACCOUNT):
142      self._l.LogOutput("Running this from non-role account; not copying "
143                        "tar files for weekly reports.")
144      dry_run = True
145
146    images_path = os.path.join(os.path.realpath(self._chromeos_root),
147                               "chroot/tmp")
148
149    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
150    dest_dir = os.path.join (data_dir, self._weekday)
151    if not os.path.exists(dest_dir):
152      os.makedirs(dest_dir)
153
154    # Make sure dest_dir is empty (clean out last week's data).
155    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
156    if dry_run:
157      print "CMD: %s" % cmd
158    else:
159      self._ce.RunCommand(cmd)
160
161    # Now create new tar files and copy them over.
162    labels = [ "test", "vanilla" ]
163    for label_name in labels:
164      if label_name == "test":
165        test_path = trybot_image
166      else:
167        test_path = vanilla_image
168      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
169      cmd = "cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,
170                                                     tar_file_name,
171                                                     test_path,
172                                                     tar_file_name,
173                                                     dest_dir)
174      if dry_run:
175        print "CMD: %s" % cmd
176        tar_ret = 0
177      else:
178        tar_ret = self._ce.RunCommand(cmd)
179      if tar_ret:
180        self._l.LogOutput("Error while creating/copying test tar file(%s)."
181                          % tar_file_name)
182
183
184  def DoAll(self):
185    """
186    Main function inside ToolchainComparator class.
187
188    Launch trybot, get image names, create crosperf experiment file, run
189    crosperf, and copy images into seven-day report directories.
190    """
191    date_str = datetime.date.today()
192    description = "master_%s_%s_%s" % (USE_NEXT_GCC_PATCH, self._build, date_str)
193    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
194                                                 self._build,
195                                                 [ USE_NEXT_GCC_PATCH ],
196                                                 description)
197
198    vanilla_image = self._ParseVanillaImage(trybot_image)
199
200    print ("trybot_image: %s" % trybot_image)
201    print ("vanilla_image: %s" % vanilla_image)
202    if len(trybot_image) == 0:
203        self._l.LogError("Unable to find trybot_image for %s!" % description)
204        return 1
205    if len(vanilla_image) == 0:
206        self._l.LogError("Unable to find vanilla image for %s!" % description)
207        return 1
208    if os.getlogin() == ROLE_ACCOUNT:
209      self._FinishSetup()
210
211    if not self._TestImages(trybot_image, vanilla_image):
212      # Only try to copy the image files if the test runs ran successfully.
213      self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
214    return 0
215
216
217def Main(argv):
218  """The main function."""
219
220  # Common initializations
221  command_executer.InitCommandExecuter()
222  parser = optparse.OptionParser()
223  parser.add_option("--remote",
224                    dest="remote",
225                    help="Remote machines to run tests on.")
226  parser.add_option("--board",
227                    dest="board",
228                    default="x86-zgb",
229                    help="The target board.")
230  parser.add_option("--chromeos_root",
231                    dest="chromeos_root",
232                    help="The chromeos root from which to run tests.")
233  parser.add_option("--weekday", default="",
234                    dest="weekday",
235                    help="The day of the week for which to run tests.")
236  options, _ = parser.parse_args(argv)
237  if not options.board:
238    print "Please give a board."
239    return 1
240  if not options.remote:
241    print "Please give at least one remote machine."
242    return 1
243  if not options.chromeos_root:
244    print "Please specify the ChromeOS root directory."
245    return 1
246
247  fc = ToolchainComparator(options.board, options.remote,
248                           options.chromeos_root, options.weekday)
249  return fc.DoAll()
250
251
252if __name__ == "__main__":
253  retval = Main(sys.argv)
254  sys.exit(retval)
255