1#!/usr/bin/env python
2# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Setup links to a Chromium checkout for WebRTC.
11
12WebRTC standalone shares a lot of dependencies and build tools with Chromium.
13To do this, many of the paths of a Chromium checkout is emulated by creating
14symlinks to files and directories. This script handles the setup of symlinks to
15achieve this.
16
17It also handles cleanup of the legacy Subversion-based approach that was used
18before Chrome switched over their master repo from Subversion to Git.
19"""
20
21
22import ctypes
23import errno
24import logging
25import optparse
26import os
27import shelve
28import shutil
29import subprocess
30import sys
31import textwrap
32
33
34DIRECTORIES = [
35  'build',
36  'buildtools',
37  'testing',
38  'third_party/binutils',
39  'third_party/boringssl',
40  'third_party/colorama',
41  'third_party/drmemory',
42  'third_party/expat',
43  'third_party/ffmpeg',
44  'third_party/instrumented_libraries',
45  'third_party/jsoncpp',
46  'third_party/libc++-static',
47  'third_party/libjpeg',
48  'third_party/libjpeg_turbo',
49  'third_party/libsrtp',
50  'third_party/libudev',
51  'third_party/libvpx_new',
52  'third_party/libyuv',
53  'third_party/llvm-build',
54  'third_party/lss',
55  'third_party/nss',
56  'third_party/ocmock',
57  'third_party/openh264',
58  'third_party/openmax_dl',
59  'third_party/opus',
60  'third_party/proguard',
61  'third_party/protobuf',
62  'third_party/sqlite',
63  'third_party/syzygy',
64  'third_party/usrsctp',
65  'third_party/yasm',
66  'third_party/zlib',
67  'tools/clang',
68  'tools/generate_library_loader',
69  'tools/gn',
70  'tools/gyp',
71  'tools/memory',
72  'tools/protoc_wrapper',
73  'tools/python',
74  'tools/swarming_client',
75  'tools/valgrind',
76  'tools/vim',
77  'tools/win',
78  'tools/xdisplaycheck',
79]
80
81from sync_chromium import get_target_os_list
82target_os = get_target_os_list()
83if 'android' in target_os:
84  DIRECTORIES += [
85    'base',
86    'third_party/android_platform',
87    'third_party/android_tools',
88    'third_party/appurify-python',
89    'third_party/ashmem',
90    'third_party/catapult',
91    'third_party/icu',
92    'third_party/ijar',
93    'third_party/jsr-305',
94    'third_party/junit',
95    'third_party/libevent',
96    'third_party/libxml',
97    'third_party/mockito',
98    'third_party/modp_b64',
99    'third_party/requests',
100    'third_party/robolectric',
101    'tools/android',
102    'tools/grit',
103    'tools/telemetry',
104  ]
105if 'ios' in target_os:
106  DIRECTORIES.append('third_party/class-dump')
107
108FILES = {
109  'tools/isolate_driver.py': None,
110  'third_party/BUILD.gn': None,
111}
112
113ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
114CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
115LINKS_DB = 'links'
116
117# Version management to make future upgrades/downgrades easier to support.
118SCHEMA_VERSION = 1
119
120
121def query_yes_no(question, default=False):
122  """Ask a yes/no question via raw_input() and return their answer.
123
124  Modified from http://stackoverflow.com/a/3041990.
125  """
126  prompt = " [%s/%%s]: "
127  prompt = prompt % ('Y' if default is True  else 'y')
128  prompt = prompt % ('N' if default is False else 'n')
129
130  if default is None:
131    default = 'INVALID'
132
133  while True:
134    sys.stdout.write(question + prompt)
135    choice = raw_input().lower()
136    if choice == '' and default != 'INVALID':
137      return default
138
139    if 'yes'.startswith(choice):
140      return True
141    elif 'no'.startswith(choice):
142      return False
143
144    print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
145
146
147# Actions
148class Action(object):
149  def __init__(self, dangerous):
150    self.dangerous = dangerous
151
152  def announce(self, planning):
153    """Log a description of this action.
154
155    Args:
156      planning - True iff we're in the planning stage, False if we're in the
157                 doit stage.
158    """
159    pass
160
161  def doit(self, links_db):
162    """Execute the action, recording what we did to links_db, if necessary."""
163    pass
164
165
166class Remove(Action):
167  def __init__(self, path, dangerous):
168    super(Remove, self).__init__(dangerous)
169    self._priority = 0
170    self._path = path
171
172  def announce(self, planning):
173    log = logging.warn
174    filesystem_type = 'file'
175    if not self.dangerous:
176      log = logging.info
177      filesystem_type = 'link'
178    if planning:
179      log('Planning to remove %s: %s', filesystem_type, self._path)
180    else:
181      log('Removing %s: %s', filesystem_type, self._path)
182
183  def doit(self, _):
184    os.remove(self._path)
185
186
187class Rmtree(Action):
188  def __init__(self, path):
189    super(Rmtree, self).__init__(dangerous=True)
190    self._priority = 0
191    self._path = path
192
193  def announce(self, planning):
194    if planning:
195      logging.warn('Planning to remove directory: %s', self._path)
196    else:
197      logging.warn('Removing directory: %s', self._path)
198
199  def doit(self, _):
200    if sys.platform.startswith('win'):
201      # shutil.rmtree() doesn't work on Windows if any of the directories are
202      # read-only, which svn repositories are.
203      subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
204    else:
205      shutil.rmtree(self._path)
206
207
208class Makedirs(Action):
209  def __init__(self, path):
210    super(Makedirs, self).__init__(dangerous=False)
211    self._priority = 1
212    self._path = path
213
214  def doit(self, _):
215    try:
216      os.makedirs(self._path)
217    except OSError as e:
218      if e.errno != errno.EEXIST:
219        raise
220
221
222class Symlink(Action):
223  def __init__(self, source_path, link_path):
224    super(Symlink, self).__init__(dangerous=False)
225    self._priority = 2
226    self._source_path = source_path
227    self._link_path = link_path
228
229  def announce(self, planning):
230    if planning:
231      logging.info(
232          'Planning to create link from %s to %s', self._link_path,
233          self._source_path)
234    else:
235      logging.debug(
236          'Linking from %s to %s', self._link_path, self._source_path)
237
238  def doit(self, links_db):
239    # Files not in the root directory need relative path calculation.
240    # On Windows, use absolute paths instead since NTFS doesn't seem to support
241    # relative paths for symlinks.
242    if sys.platform.startswith('win'):
243      source_path = os.path.abspath(self._source_path)
244    else:
245      if os.path.dirname(self._link_path) != self._link_path:
246        source_path = os.path.relpath(self._source_path,
247                                      os.path.dirname(self._link_path))
248
249    os.symlink(source_path, os.path.abspath(self._link_path))
250    links_db[self._source_path] = self._link_path
251
252
253class LinkError(IOError):
254  """Failed to create a link."""
255  pass
256
257
258# Handles symlink creation on the different platforms.
259if sys.platform.startswith('win'):
260  def symlink(source_path, link_path):
261    flag = 1 if os.path.isdir(source_path) else 0
262    if not ctypes.windll.kernel32.CreateSymbolicLinkW(
263        unicode(link_path), unicode(source_path), flag):
264      raise OSError('Failed to create symlink to %s. Notice that only NTFS '
265                    'version 5.0 and up has all the needed APIs for '
266                    'creating symlinks.' % source_path)
267  os.symlink = symlink
268
269
270class WebRTCLinkSetup(object):
271  def __init__(self, links_db, force=False, dry_run=False, prompt=False):
272    self._force = force
273    self._dry_run = dry_run
274    self._prompt = prompt
275    self._links_db = links_db
276
277  def CreateLinks(self, on_bot):
278    logging.debug('CreateLinks')
279    # First, make a plan of action
280    actions = []
281
282    for source_path, link_path in FILES.iteritems():
283      actions += self._ActionForPath(
284          source_path, link_path, check_fn=os.path.isfile, check_msg='files')
285    for source_dir in DIRECTORIES:
286      actions += self._ActionForPath(
287          source_dir, None, check_fn=os.path.isdir,
288          check_msg='directories')
289
290    if not on_bot and self._force:
291      # When making the manual switch from legacy SVN checkouts to the new
292      # Git-based Chromium DEPS, the .gclient_entries file that contains cached
293      # URLs for all DEPS entries must be removed to avoid future sync problems.
294      entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
295      if os.path.exists(entries_file):
296        actions.append(Remove(entries_file, dangerous=True))
297
298    actions.sort()
299
300    if self._dry_run:
301      for action in actions:
302        action.announce(planning=True)
303      logging.info('Not doing anything because dry-run was specified.')
304      sys.exit(0)
305
306    if any(a.dangerous for a in actions):
307      logging.warn('Dangerous actions:')
308      for action in (a for a in actions if a.dangerous):
309        action.announce(planning=True)
310      print
311
312      if not self._force:
313        logging.error(textwrap.dedent("""\
314        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
315                              A C T I O N     R E Q I R E D
316        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
317
318        Because chromium/src is transitioning to Git (from SVN), we needed to
319        change the way that the WebRTC standalone checkout works. Instead of
320        individually syncing subdirectories of Chromium in SVN, we're now
321        syncing Chromium (and all of its DEPS, as defined by its own DEPS file),
322        into the `chromium/src` directory.
323
324        As such, all Chromium directories which are currently pulled by DEPS are
325        now replaced with a symlink into the full Chromium checkout.
326
327        To avoid disrupting developers, we've chosen to not delete your
328        directories forcibly, in case you have some work in progress in one of
329        them :).
330
331        ACTION REQUIRED:
332        Before running `gclient sync|runhooks` again, you must run:
333        %s%s --force
334
335        Which will replace all directories which now must be symlinks, after
336        prompting with a summary of the work-to-be-done.
337        """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0])
338        sys.exit(1)
339      elif self._prompt:
340        if not query_yes_no('Would you like to perform the above plan?'):
341          sys.exit(1)
342
343    for action in actions:
344      action.announce(planning=False)
345      action.doit(self._links_db)
346
347    if not on_bot and self._force:
348      logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
349                   'let the remaining hooks (that probably were interrupted) '
350                   'execute.')
351
352  def CleanupLinks(self):
353    logging.debug('CleanupLinks')
354    for source, link_path  in self._links_db.iteritems():
355      if source == 'SCHEMA_VERSION':
356        continue
357      if os.path.islink(link_path) or sys.platform.startswith('win'):
358        # os.path.islink() always returns false on Windows
359        # See http://bugs.python.org/issue13143.
360        logging.debug('Removing link to %s at %s', source, link_path)
361        if not self._dry_run:
362          if os.path.exists(link_path):
363            if sys.platform.startswith('win') and os.path.isdir(link_path):
364              subprocess.check_call(['rmdir', '/q', '/s', link_path],
365                                    shell=True)
366            else:
367              os.remove(link_path)
368          del self._links_db[source]
369
370  @staticmethod
371  def _ActionForPath(source_path, link_path=None, check_fn=None,
372                     check_msg=None):
373    """Create zero or more Actions to link to a file or directory.
374
375    This will be a symlink on POSIX platforms. On Windows this requires
376    that NTFS is version 5.0 or higher (Vista or newer).
377
378    Args:
379      source_path: Path relative to the Chromium checkout root.
380        For readability, the path may contain slashes, which will
381        automatically be converted to the right path delimiter on Windows.
382      link_path: The location for the link to create. If omitted it will be the
383        same path as source_path.
384      check_fn: A function returning true if the type of filesystem object is
385        correct for the attempted call. Otherwise an error message with
386        check_msg will be printed.
387      check_msg: String used to inform the user of an invalid attempt to create
388        a file.
389    Returns:
390      A list of Action objects.
391    """
392    def fix_separators(path):
393      if sys.platform.startswith('win'):
394        return path.replace(os.altsep, os.sep)
395      else:
396        return path
397
398    assert check_fn
399    assert check_msg
400    link_path = link_path or source_path
401    link_path = fix_separators(link_path)
402
403    source_path = fix_separators(source_path)
404    source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
405    if os.path.exists(source_path) and not check_fn:
406      raise LinkError('_LinkChromiumPath can only be used to link to %s: '
407                      'Tried to link to: %s' % (check_msg, source_path))
408
409    if not os.path.exists(source_path):
410      logging.debug('Silently ignoring missing source: %s. This is to avoid '
411                    'errors on platform-specific dependencies.', source_path)
412      return []
413
414    actions = []
415
416    if os.path.exists(link_path) or os.path.islink(link_path):
417      if os.path.islink(link_path):
418        actions.append(Remove(link_path, dangerous=False))
419      elif os.path.isfile(link_path):
420        actions.append(Remove(link_path, dangerous=True))
421      elif os.path.isdir(link_path):
422        actions.append(Rmtree(link_path))
423      else:
424        raise LinkError('Don\'t know how to plan: %s' % link_path)
425
426    # Create parent directories to the target link if needed.
427    target_parent_dirs = os.path.dirname(link_path)
428    if (target_parent_dirs and
429        target_parent_dirs != link_path and
430        not os.path.exists(target_parent_dirs)):
431      actions.append(Makedirs(target_parent_dirs))
432
433    actions.append(Symlink(source_path, link_path))
434
435    return actions
436
437def _initialize_database(filename):
438  links_database = shelve.open(filename)
439
440  # Wipe the database if this version of the script ends up looking at a
441  # newer (future) version of the links db, just to be sure.
442  version = links_database.get('SCHEMA_VERSION')
443  if version and version != SCHEMA_VERSION:
444    logging.info('Found database with schema version %s while this script only '
445                 'supports %s. Wiping previous database contents.', version,
446                 SCHEMA_VERSION)
447    links_database.clear()
448  links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
449  return links_database
450
451
452def main():
453  on_bot = os.environ.get('CHROME_HEADLESS') == '1'
454
455  parser = optparse.OptionParser()
456  parser.add_option('-d', '--dry-run', action='store_true', default=False,
457                    help='Print what would be done, but don\'t perform any '
458                         'operations. This will automatically set logging to '
459                         'verbose.')
460  parser.add_option('-c', '--clean-only', action='store_true', default=False,
461                    help='Only clean previously created links, don\'t create '
462                         'new ones. This will automatically set logging to '
463                         'verbose.')
464  parser.add_option('-f', '--force', action='store_true', default=on_bot,
465                    help='Force link creation. CAUTION: This deletes existing '
466                         'folders and files in the locations where links are '
467                         'about to be created.')
468  parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
469                    default=(not on_bot),
470                    help='Prompt if we\'re planning to do a dangerous action')
471  parser.add_option('-v', '--verbose', action='store_const',
472                    const=logging.DEBUG, default=logging.INFO,
473                    help='Print verbose output for debugging.')
474  options, _ = parser.parse_args()
475
476  if options.dry_run or options.force or options.clean_only:
477    options.verbose = logging.DEBUG
478  logging.basicConfig(format='%(message)s', level=options.verbose)
479
480  # Work from the root directory of the checkout.
481  script_dir = os.path.dirname(os.path.abspath(__file__))
482  os.chdir(script_dir)
483
484  if sys.platform.startswith('win'):
485    def is_admin():
486      try:
487        return os.getuid() == 0
488      except AttributeError:
489        return ctypes.windll.shell32.IsUserAnAdmin() != 0
490    if not is_admin():
491      logging.error('On Windows, you now need to have administrator '
492                    'privileges for the shell running %s (or '
493                    '`gclient sync|runhooks`).\nPlease start another command '
494                    'prompt as Administrator and try again.', sys.argv[0])
495      return 1
496
497  if not os.path.exists(CHROMIUM_CHECKOUT):
498    logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient '
499                  'sync" before running this script?', CHROMIUM_CHECKOUT)
500    return 2
501
502  links_database = _initialize_database(LINKS_DB)
503  try:
504    symlink_creator = WebRTCLinkSetup(links_database, options.force,
505                                      options.dry_run, options.prompt)
506    symlink_creator.CleanupLinks()
507    if not options.clean_only:
508      symlink_creator.CreateLinks(on_bot)
509  except LinkError as e:
510    print >> sys.stderr, e.message
511    return 3
512  finally:
513    links_database.close()
514  return 0
515
516
517if __name__ == '__main__':
518  sys.exit(main())
519