misc.py revision 9099a788cd7124024559c064e425ed9caef6e0ac 
1# Copyright 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Utilities for toolchain build."""
5
6from __future__ import print_function
7
8__author__ = 'asharif@google.com (Ahmad Sharif)'
9
10from contextlib import contextmanager
11import os
12import re
13import shutil
14import sys
15import traceback
16
17import command_executer
18import logger
19
20CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts'
21TOOLCHAIN_UTILS_PATH = '~/trunk/src/platform/dev/toolchain_utils.sh'
22
23
24def GetChromeOSVersionFromLSBVersion(lsb_version):
25  """Get Chromeos version from Lsb version."""
26  ce = command_executer.GetCommandExecuter()
27  command = ('git ls-remote '
28             'https://chromium.googlesource.com/chromiumos/manifest.git')
29  ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
30  assert ret == 0, 'Command %s failed' % command
31  lower = []
32  for line in out.splitlines():
33    mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line)
34    if mo:
35      revision = int(mo.group(1))
36      build = int(mo.group(2))
37      lsb_build = int(lsb_version.split('.')[0])
38      if lsb_build > build:
39        lower.append(revision)
40  lower = sorted(lower)
41  if lower:
42    return 'R%d-%s' % (lower[-1] + 1, lsb_version)
43  else:
44    return 'Unknown'
45
46
47def ApplySubs(string, *substitutions):
48  for pattern, replacement in substitutions:
49    string = re.sub(pattern, replacement, string)
50  return string
51
52
53def UnitToNumber(unit_num, base=1000):
54  """Convert a number with unit to float."""
55  unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3}
56  unit_num = unit_num.lower()
57  mo = re.search(r'(\d*)(.+)?', unit_num)
58  number = mo.group(1)
59  unit = mo.group(2)
60  if not unit:
61    return float(number)
62  for k, v in unit_dict.items():
63    if k.startswith(unit):
64      return float(number) * v
65  raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num))
66
67
68def GetFilenameFromString(string):
69  return ApplySubs(string, (r'/', '__'), (r'\s', '_'), (r'[\\$="?^]', ''),)
70
71
72def GetRoot(scr_name):
73  """Break up pathname into (dir+name)."""
74  abs_path = os.path.abspath(scr_name)
75  return (os.path.dirname(abs_path), os.path.basename(abs_path))
76
77
78def GetChromeOSKeyFile(chromeos_root):
79  return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts',
80                      'ssh_keys', 'testing_rsa')
81
82
83def GetChrootPath(chromeos_root):
84  return os.path.join(chromeos_root, 'chroot')
85
86
87def GetInsideChrootPath(chromeos_root, file_path):
88  if not file_path.startswith(GetChrootPath(chromeos_root)):
89    raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" %
90                       (file_path, chromeos_root))
91  return file_path[len(GetChrootPath(chromeos_root)):]
92
93
94def GetOutsideChrootPath(chromeos_root, file_path):
95  return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/'))
96
97
98def FormatQuotedCommand(command):
99  return ApplySubs(command, ('"', r'\"'))
100
101
102def FormatCommands(commands):
103  return ApplySubs(
104      str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n'))
105
106
107def GetImageDir(chromeos_root, board):
108  return os.path.join(chromeos_root, 'src', 'build', 'images', board)
109
110
111def LabelLatestImage(chromeos_root, board, label, vanilla_path=None):
112  image_dir = GetImageDir(chromeos_root, board)
113  latest_image_dir = os.path.join(image_dir, 'latest')
114  latest_image_dir = os.path.realpath(latest_image_dir)
115  latest_image_dir = os.path.basename(latest_image_dir)
116  retval = 0
117  with WorkingDirectory(image_dir):
118    command = 'ln -sf -T %s %s' % (latest_image_dir, label)
119    ce = command_executer.GetCommandExecuter()
120    retval = ce.RunCommand(command)
121    if retval:
122      return retval
123    if vanilla_path:
124      command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla')
125      retval2 = ce.RunCommand(command)
126      return retval2
127  return retval
128
129
130def DoesLabelExist(chromeos_root, board, label):
131  image_label = os.path.join(GetImageDir(chromeos_root, board), label)
132  return os.path.exists(image_label)
133
134
135def GetBuildPackagesCommand(board, usepkg=False, debug=False):
136  if usepkg:
137    usepkg_flag = '--usepkg'
138  else:
139    usepkg_flag = '--nousepkg'
140  if debug:
141    withdebug_flag = '--withdebug'
142  else:
143    withdebug_flag = '--nowithdebug'
144  return ('%s/build_packages %s --withdev --withtest --withautotest '
145          '--skip_toolchain_update %s --board=%s '
146          '--accept_licenses=@CHROMEOS' %
147          (CHROMEOS_SCRIPTS_DIR, usepkg_flag, withdebug_flag, board))
148
149
150def GetBuildImageCommand(board, dev=False):
151  dev_args = ''
152  if dev:
153    dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs'
154  return ('%s/build_image --board=%s %s test' %
155          (CHROMEOS_SCRIPTS_DIR, board, dev_args))
156
157
158def GetSetupBoardCommand(board,
159                         gcc_version=None,
160                         binutils_version=None,
161                         usepkg=None,
162                         force=None):
163  """Get setup_board command."""
164  options = []
165
166  if gcc_version:
167    options.append('--gcc_version=%s' % gcc_version)
168
169  if binutils_version:
170    options.append('--binutils_version=%s' % binutils_version)
171
172  if usepkg:
173    options.append('--usepkg')
174  else:
175    options.append('--nousepkg')
176
177  if force:
178    options.append('--force')
179
180  options.append('--accept_licenses=@CHROMEOS')
181
182  return ('%s/setup_board --board=%s %s' %
183          (CHROMEOS_SCRIPTS_DIR, board, ' '.join(options)))
184
185
186def CanonicalizePath(path):
187  path = os.path.expanduser(path)
188  path = os.path.realpath(path)
189  return path
190
191
192def GetCtargetFromBoard(board, chromeos_root):
193  """Get Ctarget from board."""
194  base_board = board.split('_')[0]
195  command = ('source %s; get_ctarget_from_board %s' %
196             (TOOLCHAIN_UTILS_PATH, base_board))
197  ce = command_executer.GetCommandExecuter()
198  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
199  if ret != 0:
200    raise ValueError('Board %s is invalid!' % board)
201  # Remove ANSI escape sequences.
202  out = StripANSIEscapeSequences(out)
203  return out.strip()
204
205
206def GetArchFromBoard(board, chromeos_root):
207  """Get Arch from board."""
208  base_board = board.split('_')[0]
209  command = ('source %s; get_board_arch %s' %
210             (TOOLCHAIN_UTILS_PATH, base_board))
211  ce = command_executer.GetCommandExecuter()
212  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
213  if ret != 0:
214    raise ValueError('Board %s is invalid!' % board)
215  # Remove ANSI escape sequences.
216  out = StripANSIEscapeSequences(out)
217  return out.strip()
218
219
220def GetGccLibsDestForBoard(board, chromeos_root):
221  """Get gcc libs destination from board."""
222  arch = GetArchFromBoard(board, chromeos_root)
223  if arch == 'x86':
224    return '/build/%s/usr/lib/gcc/' % board
225  if arch == 'amd64':
226    return '/build/%s/usr/lib64/gcc/' % board
227  if arch == 'arm':
228    return '/build/%s/usr/lib/gcc/' % board
229  if arch == 'arm64':
230    return '/build/%s/usr/lib/gcc/' % board
231  raise ValueError('Arch %s is invalid!' % arch)
232
233
234def StripANSIEscapeSequences(string):
235  string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string)
236  return string
237
238
239def GetChromeSrcDir():
240  return 'var/cache/distfiles/target/chrome-src/src'
241
242
243def GetEnvStringFromDict(env_dict):
244  return ' '.join(["%s=\"%s\"" % var for var in env_dict.items()])
245
246
247def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
248  """Merge env string with dict."""
249  if not env_string.strip():
250    return GetEnvStringFromDict(env_dict)
251  override_env_list = []
252  ce = command_executer.GetCommandExecuter()
253  for k, v in env_dict.items():
254    v = v.strip("\"'")
255    if prepend:
256      new_env = "%s=\"%s $%s\"" % (k, v, k)
257    else:
258      new_env = "%s=\"$%s %s\"" % (k, k, v)
259    command = '; '.join([env_string, new_env, 'echo $%s' % k])
260    ret, out, _ = ce.RunCommandWOutput(command)
261    override_env_list.append('%s=%r' % (k, out.strip()))
262  ret = env_string + ' ' + ' '.join(override_env_list)
263  return ret.strip()
264
265
266def GetAllImages(chromeos_root, board):
267  ce = command_executer.GetCommandExecuter()
268  command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' %
269             (chromeos_root, board))
270  ret, out, _ = ce.RunCommandWOutput(command)
271  assert ret == 0, 'Could not run command: %s' % command
272  return out.splitlines()
273
274
275def IsFloat(text):
276  if text is None:
277    return False
278  try:
279    float(text)
280    return True
281  except ValueError:
282    return False
283
284
285def RemoveChromeBrowserObjectFiles(chromeos_root, board):
286  """Remove any object files from all the posible locations."""
287  out_dir = os.path.join(
288      GetChrootPath(chromeos_root),
289      'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board)
290  if os.path.exists(out_dir):
291    shutil.rmtree(out_dir)
292    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
293  out_dir = os.path.join(
294      GetChrootPath(chromeos_root),
295      'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board)
296  if os.path.exists(out_dir):
297    shutil.rmtree(out_dir)
298    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
299
300
301@contextmanager
302def WorkingDirectory(new_dir):
303  """Get the working directory."""
304  old_dir = os.getcwd()
305  if old_dir != new_dir:
306    msg = 'cd %s' % new_dir
307    logger.GetLogger().LogCmd(msg)
308  os.chdir(new_dir)
309  yield new_dir
310  if old_dir != new_dir:
311    msg = 'cd %s' % old_dir
312    logger.GetLogger().LogCmd(msg)
313  os.chdir(old_dir)
314
315
316def HasGitStagedChanges(git_dir):
317  """Return True if git repository has staged changes."""
318  command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
319      git_dir)
320  return command_executer.GetCommandExecuter().RunCommand(
321      command,
322      print_to_console=False)
323
324
325def HasGitUnstagedChanges(git_dir):
326  """Return True if git repository has un-staged changes."""
327  command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
328  return command_executer.GetCommandExecuter().RunCommand(
329      command,
330      print_to_console=False)
331
332
333def HasGitUntrackedChanges(git_dir):
334  """Return True if git repository has un-tracked changes."""
335  command = ('cd {0} && test -z '
336             '$(git ls-files --exclude-standard --others)').format(git_dir)
337  return command_executer.GetCommandExecuter().RunCommand(
338      command,
339      print_to_console=False)
340
341
342def GitGetCommitHash(git_dir, commit_symbolic_name):
343  """Return githash for the symbolic git commit.
344
345  For example, commit_symbolic_name could be
346  "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git
347  hash for this symbolic name.
348
349  Args:
350    git_dir: a git working tree.
351    commit_symbolic_name: a symbolic name for a particular git commit.
352
353  Returns:
354    The git hash for the symbolic name or None if fails.
355  """
356
357  command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format(
358      git_dir, commit_symbolic_name)
359  rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
360      command,
361      print_to_console=False)
362  if rv == 0:
363    return out.strip()
364  return None
365
366
367def IsGitTreeClean(git_dir):
368  """Test if git tree has no local changes.
369
370  Args:
371    git_dir: git tree directory.
372
373  Returns:
374    True if git dir is clean.
375  """
376  if HasGitStagedChanges(git_dir):
377    logger.GetLogger().LogWarning('Git tree has staged changes.')
378    return False
379  if HasGitUnstagedChanges(git_dir):
380    logger.GetLogger().LogWarning('Git tree has unstaged changes.')
381    return False
382  if HasGitUntrackedChanges(git_dir):
383    logger.GetLogger().LogWarning('Git tree has un-tracked changes.')
384    return False
385  return True
386
387
388def GetGitChangesAsList(git_dir, path=None, staged=False):
389  """Get changed files as a list.
390
391  Args:
392    git_dir: git tree directory.
393    path: a relative path that is part of the tree directory, could be null.
394    staged: whether to include staged files as well.
395
396  Returns:
397    A list containing all the changed files.
398  """
399  command = 'cd {0} && git diff --name-only'.format(git_dir)
400  if staged:
401    command += ' --cached'
402  if path:
403    command += ' -- ' + path
404  _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
405      command,
406      print_to_console=False)
407  rv = []
408  for line in out.splitlines():
409    rv.append(line)
410  return rv
411
412
413def IsChromeOsTree(chromeos_root):
414  return (os.path.isdir(os.path.join(chromeos_root,
415                                     'src/third_party/chromiumos-overlay')) and
416          os.path.isdir(os.path.join(chromeos_root, 'manifest')))
417
418
419def DeleteChromeOsTree(chromeos_root, dry_run=False):
420  """Delete a ChromeOs tree *safely*.
421
422  Args:
423    chromeos_root: dir of the tree, could be a relative one (but be careful)
424    dry_run: only prints out the command if True
425
426  Returns:
427    True if everything is ok.
428  """
429  if not IsChromeOsTree(chromeos_root):
430    logger.GetLogger().LogWarning(
431        '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
432            chromeos_root))
433    return False
434  cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root)
435  if dry_run:
436    print(cmd0)
437  else:
438    if command_executer.GetCommandExecuter().RunCommand(
439        cmd0,
440        print_to_console=True) != 0:
441      return False
442
443  cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
444          'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
445          'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME').format(
446              chromeos_root)
447  if dry_run:
448    print(cmd1)
449    return True
450
451  return command_executer.GetCommandExecuter().RunCommand(
452      cmd1,
453      print_to_console=True) == 0
454
455
456def ApplyGerritPatches(chromeos_root,
457                       gerrit_patch_string,
458                       branch='cros/master'):
459  """Apply gerrit patches on a chromeos tree.
460
461  Args:
462    chromeos_root: chromeos tree path
463    gerrit_patch_string: a patch string just like the one gives to cbuildbot,
464    'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch.
465    branch: the tree based on which to apply the patches.
466
467  Returns:
468    True if success.
469  """
470
471  ### First of all, we need chromite libs
472  sys.path.append(os.path.join(chromeos_root, 'chromite'))
473  # Imports below are ok after modifying path to add chromite.
474  # Pylint cannot detect that and complains.
475  # pylint: disable=import-error
476  from lib import git
477  from lib import gerrit
478  manifest = git.ManifestCheckout(chromeos_root)
479  patch_list = gerrit_patch_string.split(' ')
480  ### This takes time, print log information.
481  logger.GetLogger().LogOutput('Retrieving patch information from server ...')
482  patch_info_list = gerrit.GetGerritPatchInfo(patch_list)
483  for pi in patch_info_list:
484    project_checkout = manifest.FindCheckout(pi.project, strict=False)
485    if not project_checkout:
486      logger.GetLogger().LogError(
487          'Failed to find patch project "{project}" in manifest.'.format(
488              project=pi.project))
489      return False
490
491    pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref)
492    try:
493      project_git_path = project_checkout.GetPath(absolute=True)
494      logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format(
495          pi_str, project_git_path))
496      pi.Apply(project_git_path, branch, trivial=False)
497    except Exception:
498      traceback.print_exc(file=sys.stdout)
499      logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str))
500      return False
501  return True
502
503
504def BooleanPrompt(prompt='Do you want to continue?',
505                  default=True,
506                  true_value='yes',
507                  false_value='no',
508                  prolog=None):
509  """Helper function for processing boolean choice prompts.
510
511  Args:
512    prompt: The question to present to the user.
513    default: Boolean to return if the user just presses enter.
514    true_value: The text to display that represents a True returned.
515    false_value: The text to display that represents a False returned.
516    prolog: The text to display before prompt.
517
518  Returns:
519    True or False.
520  """
521  true_value, false_value = true_value.lower(), false_value.lower()
522  true_text, false_text = true_value, false_value
523  if true_value == false_value:
524    raise ValueError('true_value and false_value must differ: got %r' %
525                     true_value)
526
527  if default:
528    true_text = true_text[0].upper() + true_text[1:]
529  else:
530    false_text = false_text[0].upper() + false_text[1:]
531
532  prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text))
533
534  if prolog:
535    prompt = ('\n%s\n%s' % (prolog, prompt))
536
537  while True:
538    try:
539      response = raw_input(prompt).lower()
540    except EOFError:
541      # If the user hits CTRL+D, or stdin is disabled, use the default.
542      print()
543      response = None
544    except KeyboardInterrupt:
545      # If the user hits CTRL+C, just exit the process.
546      print()
547      print('CTRL+C detected; exiting')
548      sys.exit()
549
550    if not response:
551      return default
552    if true_value.startswith(response):
553      if not false_value.startswith(response):
554        return True
555      # common prefix between the two...
556    elif false_value.startswith(response):
557      return False
558