test_toolchains.py revision 748254e9a5dd7227806ca618d1f9a32ab49f4056 
1#!/usr/bin/python2
2
3# Script to test different toolchains against ChromeOS benchmarks.
4"""Toolchain team nightly performance test script (local builds)."""
5
6from __future__ import print_function
7
8import argparse
9import datetime
10import os
11import sys
12import build_chromeos
13import setup_chromeos
14import time
15from cros_utils import command_executer
16from cros_utils import misc
17from cros_utils import logger
18
19CROSTC_ROOT = '/usr/local/google/crostc'
20MAIL_PROGRAM = '~/var/bin/mail-sheriff'
21WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
22PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
23NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
24
25
26class GCCConfig(object):
27  """GCC configuration class."""
28
29  def __init__(self, githash):
30    self.githash = githash
31
32
33class ToolchainConfig(object):
34  """Toolchain configuration class."""
35
36  def __init__(self, gcc_config=None):
37    self.gcc_config = gcc_config
38
39
40class ChromeOSCheckout(object):
41  """Main class for checking out, building and testing ChromeOS."""
42
43  def __init__(self, board, chromeos_root):
44    self._board = board
45    self._chromeos_root = chromeos_root
46    self._ce = command_executer.GetCommandExecuter()
47    self._l = logger.GetLogger()
48    self._build_num = None
49
50  def _DeleteChroot(self):
51    command = 'cd %s; cros_sdk --delete' % self._chromeos_root
52    return self._ce.RunCommand(command)
53
54  def _DeleteCcahe(self):
55    # crosbug.com/34956
56    command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache')
57    return self._ce.RunCommand(command)
58
59  def _GetBuildNumber(self):
60    """Get the build number of the ChromeOS image from the chroot.
61
62    This function assumes a ChromeOS image has been built in the chroot.
63    It translates the 'latest' symlink in the
64    <chroot>/src/build/images/<board> directory, to find the actual
65    ChromeOS build number for the image that was built.  For example, if
66    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
67    This function would parse it out and assign 'R37-5982' to self._build_num.
68    This is used to determine the official, vanilla build to use for
69    comparison tests.
70    """
71    # Get the path to 'latest'
72    sym_path = os.path.join(
73        misc.GetImageDir(self._chromeos_root, self._board), 'latest')
74    # Translate the symlink to its 'real' path.
75    real_path = os.path.realpath(sym_path)
76    # Break up the path and get the last piece
77    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
78    path_pieces = real_path.split('/')
79    last_piece = path_pieces[-1]
80    # Break this piece into the image number + other pieces, and get the
81    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
82    image_parts = last_piece.split('.')
83    self._build_num = image_parts[0]
84
85  def _BuildLabelName(self, config):
86    pieces = config.split('/')
87    compiler_version = pieces[-1]
88    label = compiler_version + '_tot_afdo'
89    return label
90
91  def _BuildAndImage(self, label=''):
92    if (not label or
93        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
94      build_chromeos_args = [build_chromeos.__file__,
95                             '--chromeos_root=%s' % self._chromeos_root,
96                             '--board=%s' % self._board, '--rebuild']
97      if self._public:
98        build_chromeos_args.append('--env=USE=-chrome_internal')
99
100      ret = build_chromeos.Main(build_chromeos_args)
101      if ret != 0:
102        raise RuntimeError("Couldn't build ChromeOS!")
103
104      if not self._build_num:
105        self._GetBuildNumber()
106      # Check to see if we need to create the symbolic link for the vanilla
107      # image, and do so if appropriate.
108      if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'):
109        build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
110        full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root,
111                                         'chroot/tmp', build_name)
112        misc.LabelLatestImage(self._chromeos_root, self._board, label,
113                              full_vanilla_path)
114      else:
115        misc.LabelLatestImage(self._chromeos_root, self._board, label)
116    return label
117
118  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
119    env_string = misc.GetEnvStringFromDict(env_dict)
120    command = ('%s %s' % (env_string,
121                          misc.GetSetupBoardCommand(self._board,
122                                                    usepkg=usepkg_flag,
123                                                    force=clobber_flag)))
124    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
125    error_str = "Could not setup board: '%s'" % command
126    assert ret == 0, error_str
127
128  def _UnInstallToolchain(self):
129    command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' %
130               misc.GetCtargetFromBoard(self._board, self._chromeos_root))
131    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
132    if ret != 0:
133      raise RuntimeError("Couldn't uninstall the toolchain!")
134
135  def _CheckoutChromeOS(self):
136    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
137    if not os.path.exists(self._chromeos_root):
138      setup_chromeos_args = [setup_chromeos.__file__,
139                             '--dir=%s' % self._chromeos_root]
140      if self._public:
141        setup_chromeos_args.append('--public')
142      ret = setup_chromeos.Main(setup_chromeos_args)
143      if ret != 0:
144        raise RuntimeError("Couldn't run setup_chromeos!")
145
146  def _BuildToolchain(self, config):
147    # Call setup_board for basic, vanilla setup.
148    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
149    # Now uninstall the vanilla compiler and setup/build our custom
150    # compiler.
151    self._UnInstallToolchain()
152    envdict = {'USE': 'git_gcc',
153               'GCC_GITHASH': config.gcc_config.githash,
154               'EMERGE_DEFAULT_OPTS': '--exclude=gcc'}
155    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
156
157
158class ToolchainComparator(ChromeOSCheckout):
159  """Main class for running tests and generating reports."""
160
161  def __init__(self,
162               board,
163               remotes,
164               configs,
165               clean,
166               public,
167               force_mismatch,
168               noschedv2=False):
169    self._board = board
170    self._remotes = remotes
171    self._chromeos_root = 'chromeos'
172    self._configs = configs
173    self._clean = clean
174    self._public = public
175    self._force_mismatch = force_mismatch
176    self._ce = command_executer.GetCommandExecuter()
177    self._l = logger.GetLogger()
178    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
179                                           '%Y-%m-%d_%H:%M:%S')
180    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
181                                     '%s.%s' % (timestamp, board),)
182    self._noschedv2 = noschedv2
183    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
184
185  def _FinishSetup(self):
186    # Get correct .boto file
187    current_dir = os.getcwd()
188    src = '/usr/local/google/home/mobiletc-prebuild/.boto'
189    dest = os.path.join(current_dir, self._chromeos_root,
190                        'src/private-overlays/chromeos-overlay/'
191                        'googlestorage_account.boto')
192    # Copy the file to the correct place
193    copy_cmd = 'cp %s %s' % (src, dest)
194    retv = self._ce.RunCommand(copy_cmd)
195    if retv != 0:
196      raise RuntimeError("Couldn't copy .boto file for google storage.")
197
198    # Fix protections on ssh key
199    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
200               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
201               '/testing_rsa')
202    retv = self._ce.ChrootRunCommand(self._chromeos_root, command)
203    if retv != 0:
204      raise RuntimeError('chmod for testing_rsa failed')
205
206  def _TestLabels(self, labels):
207    experiment_file = 'toolchain_experiment.txt'
208    image_args = ''
209    if self._force_mismatch:
210      image_args = '--force-mismatch'
211    experiment_header = """
212    board: %s
213    remote: %s
214    retries: 1
215    """ % (self._board, self._remotes)
216    experiment_tests = """
217    benchmark: all_toolchain_perf {
218      suite: telemetry_Crosperf
219      iterations: 3
220    }
221    """
222
223    with open(experiment_file, 'w') as f:
224      f.write(experiment_header)
225      f.write(experiment_tests)
226      for label in labels:
227        # TODO(asharif): Fix crosperf so it accepts labels with symbols
228        crosperf_label = label
229        crosperf_label = crosperf_label.replace('-', '_')
230        crosperf_label = crosperf_label.replace('+', '_')
231        crosperf_label = crosperf_label.replace('.', '')
232
233        # Use the official build instead of building vanilla ourselves.
234        if label == 'vanilla':
235          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
236
237          # Now add 'official build' to test file.
238          official_image = """
239          official_image {
240            chromeos_root: %s
241            build: %s
242          }
243          """ % (self._chromeos_root, build_name)
244          f.write(official_image)
245
246        else:
247          experiment_image = """
248          %s {
249            chromeos_image: %s
250            image_args: %s
251          }
252          """ % (crosperf_label, os.path.join(
253              misc.GetImageDir(self._chromeos_root, self._board), label,
254              'chromiumos_test_image.bin'), image_args)
255          f.write(experiment_image)
256
257    crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf')
258    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
259    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
260               '--json_report=True {noschedv2_opts} {exp_file}').format(
261                   crosperf=crosperf,
262                   r_dir=self._reports_dir,
263                   noschedv2_opts=noschedv2_opts,
264                   exp_file=experiment_file)
265
266    ret = self._ce.RunCommand(command)
267    if ret != 0:
268      raise RuntimeError("Couldn't run crosperf!")
269    else:
270      # Copy json report to pending archives directory.
271      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
272      ret = self._ce.RunCommand(command)
273    return
274
275  def _CopyWeeklyReportFiles(self, labels):
276    """Move files into place for creating 7-day reports.
277
278    Create tar files of the custom and official images and copy them
279    to the weekly reports directory, so they exist when the weekly report
280    gets generated.  IMPORTANT NOTE: This function must run *after*
281    crosperf has been run; otherwise the vanilla images will not be there.
282    """
283    images_path = os.path.join(
284        os.path.realpath(self._chromeos_root), 'src/build/images', self._board)
285    weekday = time.strftime('%a')
286    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
287    dest_dir = os.path.join(data_dir, weekday)
288    if not os.path.exists(dest_dir):
289      os.makedirs(dest_dir)
290    # Make sure dest_dir is empty (clean out last week's data).
291    cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, weekday)
292    self._ce.RunCommand(cmd)
293    # Now create new tar files and copy them over.
294    for l in labels:
295      test_path = os.path.join(images_path, l)
296      if os.path.exists(test_path):
297        if l != 'vanilla':
298          label_name = 'test'
299        else:
300          label_name = 'vanilla'
301        tar_file_name = '%s_%s_image.tar' % (weekday, label_name)
302        cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
303               'cp %s %s/.') % (images_path, tar_file_name, l, tar_file_name,
304                                dest_dir)
305        tar_ret = self._ce.RunCommand(cmd)
306        if tar_ret != 0:
307          self._l.LogOutput('Error while creating/copying test tar file(%s).' %
308                            tar_file_name)
309
310  def _SendEmail(self):
311    """Find email msesage generated by crosperf and send it."""
312    filename = os.path.join(self._reports_dir, 'msg_body.html')
313    if (os.path.exists(filename) and
314        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
315      command = ('cat %s | %s -s "Nightly test results, %s" -team -html' %
316                 (filename, MAIL_PROGRAM, self._board))
317      self._ce.RunCommand(command)
318
319  def DoAll(self):
320    self._CheckoutChromeOS()
321    labels = []
322    labels.append('vanilla')
323    for config in self._configs:
324      label = self._BuildLabelName(config.gcc_config.githash)
325      if not misc.DoesLabelExist(self._chromeos_root, self._board, label):
326        self._BuildToolchain(config)
327        label = self._BuildAndImage(label)
328      labels.append(label)
329    self._FinishSetup()
330    self._TestLabels(labels)
331    self._SendEmail()
332    # Only try to copy the image files if the test runs ran successfully.
333    self._CopyWeeklyReportFiles(labels)
334    if self._clean:
335      ret = self._DeleteChroot()
336      if ret != 0:
337        return ret
338      ret = self._DeleteCcahe()
339      if ret != 0:
340        return ret
341    return 0
342
343
344def Main(argv):
345  """The main function."""
346  # Common initializations
347  ###  command_executer.InitCommandExecuter(True)
348  command_executer.InitCommandExecuter()
349  parser = argparse.ArgumentParser()
350  parser.add_argument('--remote',
351                      dest='remote',
352                      help='Remote machines to run tests on.')
353  parser.add_argument('--board',
354                      dest='board',
355                      default='x86-alex',
356                      help='The target board.')
357  parser.add_argument('--githashes',
358                      dest='githashes',
359                      default='master',
360                      help='The gcc githashes to test.')
361  parser.add_argument('--clean',
362                      dest='clean',
363                      default=False,
364                      action='store_true',
365                      help='Clean the chroot after testing.')
366  parser.add_argument('--public',
367                      dest='public',
368                      default=False,
369                      action='store_true',
370                      help='Use the public checkout/build.')
371  parser.add_argument('--force-mismatch',
372                      dest='force_mismatch',
373                      default='',
374                      help='Force the image regardless of board mismatch')
375  parser.add_argument('--noschedv2',
376                      dest='noschedv2',
377                      action='store_true',
378                      default=False,
379                      help='Pass --noschedv2 to crosperf.')
380  options = parser.parse_args(argv)
381  if not options.board:
382    print('Please give a board.')
383    return 1
384  if not options.remote:
385    print('Please give at least one remote machine.')
386    return 1
387  toolchain_configs = []
388  for githash in options.githashes.split(','):
389    gcc_config = GCCConfig(githash=githash)
390    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
391    toolchain_configs.append(toolchain_config)
392  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
393                           options.clean, options.public,
394                           options.force_mismatch, options.noschedv2)
395  return fc.DoAll()
396
397
398if __name__ == '__main__':
399  retval = Main(sys.argv[1:])
400  sys.exit(retval)
401