test_toolchains.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import datetime
5import optparse
6import os
7import string
8import sys
9import build_chromeos
10import setup_chromeos
11import time
12from utils import command_executer
13from utils import misc
14from utils import logger
15
16CROSTC_ROOT = '/usr/local/google/crostc'
17MAIL_PROGRAM = '~/var/bin/mail-sheriff'
18WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
19PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
20NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
21
22
23class GCCConfig(object):
24
25  def __init__(self, githash):
26    self.githash = githash
27
28
29class ToolchainConfig:
30
31  def __init__(self, gcc_config=None, binutils_config=None):
32    self.gcc_config = gcc_config
33
34
35class ChromeOSCheckout(object):
36
37  def __init__(self, board, chromeos_root):
38    self._board = board
39    self._chromeos_root = chromeos_root
40    self._ce = command_executer.GetCommandExecuter()
41    self._l = logger.GetLogger()
42    self._build_num = None
43
44  def _DeleteChroot(self):
45    command = 'cd %s; cros_sdk --delete' % self._chromeos_root
46    return self._ce.RunCommand(command)
47
48  def _DeleteCcahe(self):
49    # crosbug.com/34956
50    command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache')
51    return self._ce.RunCommand(command)
52
53  def _GetBuildNumber(self):
54    """ This function assumes a ChromeOS image has been built in the chroot.
55    It translates the 'latest' symlink in the
56    <chroot>/src/build/images/<board> directory, to find the actual
57    ChromeOS build number for the image that was built.  For example, if
58    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
59    This function would parse it out and assign 'R37-5982' to self._build_num.
60    This is used to determine the official, vanilla build to use for
61    comparison tests.
62    """
63    # Get the path to 'latest'
64    sym_path = os.path.join(
65        misc.GetImageDir(self._chromeos_root, self._board), 'latest')
66    # Translate the symlink to its 'real' path.
67    real_path = os.path.realpath(sym_path)
68    # Break up the path and get the last piece
69    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
70    path_pieces = real_path.split('/')
71    last_piece = path_pieces[-1]
72    # Break this piece into the image number + other pieces, and get the
73    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
74    image_parts = last_piece.split('.')
75    self._build_num = image_parts[0]
76
77  def _BuildLabelName(self, config, board):
78    pieces = config.split('/')
79    compiler_version = pieces[-1]
80    label = compiler_version + '_tot_afdo'
81    return label
82
83  def _BuildAndImage(self, label=''):
84    if (not label or
85        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
86      build_chromeos_args = [build_chromeos.__file__,
87                             '--chromeos_root=%s' % self._chromeos_root,
88                             '--board=%s' % self._board, '--rebuild']
89      if self._public:
90        build_chromeos_args.append('--env=USE=-chrome_internal')
91
92      ret = build_chromeos.Main(build_chromeos_args)
93      if ret != 0:
94        raise RuntimeError("Couldn't build ChromeOS!")
95
96      if not self._build_num:
97        self._GetBuildNumber()
98      # Check to see if we need to create the symbolic link for the vanilla
99      # image, and do so if appropriate.
100      if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'):
101        build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
102        full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root,
103                                         'chroot/tmp', build_name)
104        misc.LabelLatestImage(self._chromeos_root, self._board, label,
105                              full_vanilla_path)
106      else:
107        misc.LabelLatestImage(self._chromeos_root, self._board, label)
108    return label
109
110  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
111    env_string = misc.GetEnvStringFromDict(env_dict)
112    command = ('%s %s' % (env_string,
113                          misc.GetSetupBoardCommand(self._board,
114                                                    usepkg=usepkg_flag,
115                                                    force=clobber_flag)))
116    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
117    error_str = "Could not setup board: '%s'" % command
118    assert ret == 0, error_str
119
120  def _UnInstallToolchain(self):
121    command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' %
122               misc.GetCtargetFromBoard(self._board, self._chromeos_root))
123    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
124    if ret != 0:
125      raise RuntimeError("Couldn't uninstall the toolchain!")
126
127  def _CheckoutChromeOS(self):
128    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
129    if not os.path.exists(self._chromeos_root):
130      setup_chromeos_args = [setup_chromeos.__file__,
131                             '--dir=%s' % self._chromeos_root]
132      if self._public:
133        setup_chromeos_args.append('--public')
134      ret = setup_chromeos.Main(setup_chromeos_args)
135      if ret != 0:
136        raise RuntimeError("Couldn't run setup_chromeos!")
137
138  def _BuildToolchain(self, config):
139    # Call setup_board for basic, vanilla setup.
140    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
141    # Now uninstall the vanilla compiler and setup/build our custom
142    # compiler.
143    self._UnInstallToolchain()
144    envdict = {'USE': 'git_gcc',
145               'GCC_GITHASH': config.gcc_config.githash,
146               'EMERGE_DEFAULT_OPTS': '--exclude=gcc'}
147    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
148
149
150class ToolchainComparator(ChromeOSCheckout):
151
152  def __init__(self,
153               board,
154               remotes,
155               configs,
156               clean,
157               public,
158               force_mismatch,
159               noschedv2=False):
160    self._board = board
161    self._remotes = remotes
162    self._chromeos_root = 'chromeos'
163    self._configs = configs
164    self._clean = clean
165    self._public = public
166    self._force_mismatch = force_mismatch
167    self._ce = command_executer.GetCommandExecuter()
168    self._l = logger.GetLogger()
169    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
170                                           '%Y-%m-%d_%H:%M:%S')
171    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
172                                     '%s.%s' % (timestamp, board),)
173    self._noschedv2 = noschedv2
174    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
175
176  def _FinishSetup(self):
177    # Get correct .boto file
178    current_dir = os.getcwd()
179    src = '/usr/local/google/home/mobiletc-prebuild/.boto'
180    dest = os.path.join(current_dir, self._chromeos_root,
181                        'src/private-overlays/chromeos-overlay/'
182                        'googlestorage_account.boto')
183    # Copy the file to the correct place
184    copy_cmd = 'cp %s %s' % (src, dest)
185    retval = self._ce.RunCommand(copy_cmd)
186    if retval != 0:
187      raise RuntimeError("Couldn't copy .boto file for google storage.")
188
189    # Fix protections on ssh key
190    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
191               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
192               '/testing_rsa')
193    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
194    if retval != 0:
195      raise RuntimeError('chmod for testing_rsa failed')
196
197  def _TestLabels(self, labels):
198    experiment_file = 'toolchain_experiment.txt'
199    image_args = ''
200    if self._force_mismatch:
201      image_args = '--force-mismatch'
202    experiment_header = """
203    board: %s
204    remote: %s
205    retries: 1
206    """ % (self._board, self._remotes)
207    experiment_tests = """
208    benchmark: all_toolchain_perf {
209      suite: telemetry_Crosperf
210      iterations: 3
211    }
212    """
213
214    with open(experiment_file, 'w') as f:
215      print >> f, experiment_header
216      print >> f, experiment_tests
217      for label in labels:
218        # TODO(asharif): Fix crosperf so it accepts labels with symbols
219        crosperf_label = label
220        crosperf_label = crosperf_label.replace('-', '_')
221        crosperf_label = crosperf_label.replace('+', '_')
222        crosperf_label = crosperf_label.replace('.', '')
223
224        # Use the official build instead of building vanilla ourselves.
225        if label == 'vanilla':
226          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
227
228          # Now add 'official build' to test file.
229          official_image = """
230          official_image {
231            chromeos_root: %s
232            build: %s
233          }
234          """ % (self._chromeos_root, build_name)
235          print >> f, official_image
236
237        else:
238          experiment_image = """
239          %s {
240            chromeos_image: %s
241            image_args: %s
242          }
243          """ % (crosperf_label, os.path.join(
244              misc.GetImageDir(self._chromeos_root, self._board), label,
245              'chromiumos_test_image.bin'), image_args)
246          print >> f, experiment_image
247
248    crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf')
249    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
250    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
251               '--json_report=True {noschedv2_opts} {exp_file}').format(
252                   crosperf=crosperf,
253                   r_dir=self._reports_dir,
254                   noschedv2_opts=noschedv2_opts,
255                   exp_file=experiment_file)
256
257    ret = self._ce.RunCommand(command)
258    if ret != 0:
259      raise RuntimeError("Couldn't run crosperf!")
260    else:
261      # Copy json report to pending archives directory.
262      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
263      ret = self._ce.RunCommand(command)
264    return
265
266  def _CopyWeeklyReportFiles(self, labels):
267    """Create tar files of the custom and official images and copy them
268    to the weekly reports directory, so they exist when the weekly report
269    gets generated.  IMPORTANT NOTE: This function must run *after*
270    crosperf has been run; otherwise the vanilla images will not be there.
271    """
272    images_path = os.path.join(
273        os.path.realpath(self._chromeos_root), 'src/build/images', self._board)
274    weekday = time.strftime('%a')
275    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
276    dest_dir = os.path.join(data_dir, weekday)
277    if not os.path.exists(dest_dir):
278      os.makedirs(dest_dir)
279    # Make sure dest_dir is empty (clean out last week's data).
280    cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, weekday)
281    self._ce.RunCommand(cmd)
282    # Now create new tar files and copy them over.
283    for l in labels:
284      test_path = os.path.join(images_path, l)
285      if os.path.exists(test_path):
286        if l != 'vanilla':
287          label_name = 'test'
288        else:
289          label_name = 'vanilla'
290        tar_file_name = '%s_%s_image.tar' % (weekday, label_name)
291        cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
292               'cp %s %s/.') % (images_path, tar_file_name, l, tar_file_name,
293                                dest_dir)
294        tar_ret = self._ce.RunCommand(cmd)
295        if tar_ret != 0:
296          self._l.LogOutput('Error while creating/copying test tar file(%s).' %
297                            tar_file_name)
298
299  def _SendEmail(self):
300    """Find email msesage generated by crosperf and send it."""
301    filename = os.path.join(self._reports_dir, 'msg_body.html')
302    if (os.path.exists(filename) and
303        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
304      command = ('cat %s | %s -s "Nightly test results, %s" -team -html' %
305                 (filename, MAIL_PROGRAM, self._board))
306      self._ce.RunCommand(command)
307
308  def DoAll(self):
309    self._CheckoutChromeOS()
310    labels = []
311    labels.append('vanilla')
312    for config in self._configs:
313      label = self._BuildLabelName(config.gcc_config.githash, self._board)
314      if (not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
315        self._BuildToolchain(config)
316        label = self._BuildAndImage(label)
317      labels.append(label)
318    self._FinishSetup()
319    self._TestLabels(labels)
320    self._SendEmail()
321    # Only try to copy the image files if the test runs ran successfully.
322    self._CopyWeeklyReportFiles(labels)
323    if self._clean:
324      ret = self._DeleteChroot()
325      if ret != 0:
326        return ret
327      ret = self._DeleteCcahe()
328      if ret != 0:
329        return ret
330    return 0
331
332
333def Main(argv):
334  """The main function."""
335  # Common initializations
336  ###  command_executer.InitCommandExecuter(True)
337  command_executer.InitCommandExecuter()
338  parser = optparse.OptionParser()
339  parser.add_option('--remote',
340                    dest='remote',
341                    help='Remote machines to run tests on.')
342  parser.add_option('--board',
343                    dest='board',
344                    default='x86-zgb',
345                    help='The target board.')
346  parser.add_option('--githashes',
347                    dest='githashes',
348                    default='master',
349                    help='The gcc githashes to test.')
350  parser.add_option('--clean',
351                    dest='clean',
352                    default=False,
353                    action='store_true',
354                    help='Clean the chroot after testing.')
355  parser.add_option('--public',
356                    dest='public',
357                    default=False,
358                    action='store_true',
359                    help='Use the public checkout/build.')
360  parser.add_option('--force-mismatch',
361                    dest='force_mismatch',
362                    default='',
363                    help='Force the image regardless of board mismatch')
364  parser.add_option('--noschedv2',
365                    dest='noschedv2',
366                    action='store_true',
367                    default=False,
368                    help='Pass --noschedv2 to crosperf.')
369  options, _ = parser.parse_args(argv)
370  if not options.board:
371    print 'Please give a board.'
372    return 1
373  if not options.remote:
374    print 'Please give at least one remote machine.'
375    return 1
376  toolchain_configs = []
377  for githash in options.githashes.split(','):
378    gcc_config = GCCConfig(githash=githash)
379    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
380    toolchain_configs.append(toolchain_config)
381  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
382                           options.clean, options.public,
383                           options.force_mismatch, options.noschedv2)
384  return fc.DoAll()
385
386
387if __name__ == '__main__':
388  retval = Main(sys.argv)
389  sys.exit(retval)
390