buildbot_test_toolchains.py revision 46093e58cfe1bf29cc76e1ab87540239ddd7cb8b 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import optparse
14import os
15import sys
16import time
17import urllib2
18
19from utils import command_executer
20from utils import logger
21
22from utils import buildbot_utils
23
24# CL that updated GCC ebuilds to use 'next_gcc'.
25USE_NEXT_GCC_PATCH ="230260"
26
27WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
28ROLE_ACCOUNT = "mobiletc-prebuild"
29TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
30
31
32class ToolchainComparator():
33  """
34  Class for doing the nightly tests work.
35  """
36
37  def __init__(self, board, remotes, chromeos_root, weekday):
38    self._board = board
39    self._remotes = remotes
40    self._chromeos_root = chromeos_root
41    self._base_dir = os.getcwd()
42    self._ce = command_executer.GetCommandExecuter()
43    self._l = logger.GetLogger()
44    self._build = "%s-release" % board
45    if not weekday:
46      self._weekday = time.strftime("%a")
47    else:
48      self._weekday = weekday
49
50  def _ParseVanillaImage(self, trybot_image):
51    """
52    Parse a trybot artifact name to get corresponding vanilla image.
53
54    This function takes an artifact name, such as
55    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
56    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
57    """
58    start_pos = trybot_image.find(self._build)
59    end_pos = trybot_image.rfind("-b")
60    vanilla_image = trybot_image[start_pos:end_pos]
61    return vanilla_image
62
63
64  def _FinishSetup(self):
65    """
66    Make sure testing_rsa file is properly set up.
67    """
68    # Fix protections on ssh key
69    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
70               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
71               "/testing_rsa")
72    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
73    if ret_val:
74      raise Exception("chmod for testing_rsa failed")
75
76  def _TestImages(self, trybot_image, vanilla_image):
77    """
78    Create crosperf experiment file.
79
80    Given the names of the trybot and vanilla images, create the
81    appropriate crosperf experiment file and launch crosperf on it.
82    """
83    experiment_file_dir = os.path.join (self._chromeos_root, "..",
84                                        self._weekday)
85    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
86    experiment_file = os.path.join (experiment_file_dir,
87                                    experiment_file_name)
88    experiment_header = """
89    board: %s
90    remote: %s
91    """ % (self._board, self._remotes)
92    experiment_tests = """
93    benchmark: all_perfv2 {
94      suite: telemetry_Crosperf
95      iterations: 3
96    }
97    """
98    with open(experiment_file, "w") as f:
99      print >> f, experiment_header
100      print >> f, experiment_tests
101
102      # Now add vanilla to test file.
103      official_image = """
104          vanilla_image {
105            chromeos_root: %s
106            build: %s
107          }
108          """ % (self._chromeos_root, vanilla_image)
109      print >> f, official_image
110
111      experiment_image = """
112          test_image {
113            chromeos_root: %s
114            build: %s
115          }
116          """ % (self._chromeos_root, trybot_image)
117      print >> f, experiment_image
118
119    crosperf = os.path.join(TOOLCHAIN_DIR,
120                            "crosperf",
121                            "crosperf")
122    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
123    ret = self._ce.RunCommand(command)
124    if ret:
125      raise Exception("Couldn't run crosperf!")
126    return ret
127
128
129  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
130    """
131    Put files in place for running seven-day reports.
132
133    Create tar files of the custom and official images and copy them
134    to the weekly reports directory, so they exist when the weekly report
135    gets generated.  IMPORTANT NOTE: This function must run *after*
136    crosperf has been run; otherwise the vanilla images will not be there.
137    """
138
139    dry_run = False
140    if (os.getlogin() != ROLE_ACCOUNT):
141      self._l.LogOutput("Running this from non-role account; not copying "
142                        "tar files for weekly reports.")
143      dry_run = True
144
145    images_path = os.path.join(os.path.realpath(self._chromeos_root),
146                               "chroot/tmp")
147
148    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
149    dest_dir = os.path.join (data_dir, self._weekday)
150    if not os.path.exists(dest_dir):
151      os.makedirs(dest_dir)
152
153    # Make sure dest_dir is empty (clean out last week's data).
154    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
155    if dry_run:
156      print "CMD: %s" % cmd
157    else:
158      self._ce.RunCommand(cmd)
159
160    # Now create new tar files and copy them over.
161    labels = [ "test", "vanilla" ]
162    for label_name in labels:
163      if label_name == "test":
164        test_path = trybot_image
165      else:
166        test_path = vanilla_image
167      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
168      cmd = "cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,
169                                                     tar_file_name,
170                                                     test_path,
171                                                     tar_file_name,
172                                                     dest_dir)
173      if dry_run:
174        print "CMD: %s" % cmd
175        tar_ret = 0
176      else:
177        tar_ret = self._ce.RunCommand(cmd)
178      if tar_ret:
179        self._l.LogOutput("Error while creating/copying test tar file(%s)."
180                          % tar_file_name)
181
182
183  def DoAll(self):
184    """
185    Main function inside ToolchainComparator class.
186
187    Launch trybot, get image names, create crosperf experiment file, run
188    crosperf, and copy images into seven-day report directories.
189    """
190
191    description = "master_%s_%s" % (USE_NEXT_GCC_PATCH, self._build)
192    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
193                                                 self._build,
194                                                 [ USE_NEXT_GCC_PATCH ],
195                                                 description)
196
197    vanilla_image = self._ParseVanillaImage(trybot_image)
198
199    print ("trybot_image: %s" % trybot_image)
200    print ("vanilla_image: %s" % vanilla_image)
201    if os.getlogin() == ROLE_ACCOUNT:
202      self._FinishSetup()
203
204    if not self._TestImages(trybot_image, vanilla_image):
205      # Only try to copy the image files if the test runs ran successfully.
206      self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
207    return 0
208
209
210def Main(argv):
211  """The main function."""
212
213  # Common initializations
214  command_executer.InitCommandExecuter()
215  parser = optparse.OptionParser()
216  parser.add_option("--remote",
217                    dest="remote",
218                    help="Remote machines to run tests on.")
219  parser.add_option("--board",
220                    dest="board",
221                    default="x86-zgb",
222                    help="The target board.")
223  parser.add_option("--chromeos_root",
224                    dest="chromeos_root",
225                    help="The chromeos root from which to run tests.")
226  parser.add_option("--weekday", default="",
227                    dest="weekday",
228                    help="The day of the week for which to run tests.")
229  options, _ = parser.parse_args(argv)
230  if not options.board:
231    print "Please give a board."
232    return 1
233  if not options.remote:
234    print "Please give at least one remote machine."
235    return 1
236  if not options.chromeos_root:
237    print "Please specify the ChromeOS root directory."
238    return 1
239
240  fc = ToolchainComparator(options.board, options.remote,
241                           options.chromeos_root, options.weekday)
242  return fc.DoAll()
243
244
245if __name__ == "__main__":
246  retval = Main(sys.argv)
247  sys.exit(retval)
248