1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import fnmatch
6import imp
7import logging
8import modulefinder
9import optparse
10import os
11import sys
12import zipfile
13
14from telemetry import benchmark
15from telemetry.core import command_line
16from telemetry.core import discover
17from telemetry.core import util
18from telemetry.util import bootstrap
19from telemetry.util import cloud_storage
20from telemetry.util import path_set
21
22DEPS_FILE = 'bootstrap_deps'
23
24
25def _InDirectory(subdirectory, directory):
26  subdirectory = os.path.realpath(subdirectory)
27  directory = os.path.realpath(directory)
28  common_prefix = os.path.commonprefix([subdirectory, directory])
29  return common_prefix == directory
30
31
32def FindBootstrapDependencies(base_dir):
33  deps_file = os.path.join(base_dir, DEPS_FILE)
34  if not os.path.exists(deps_file):
35    return []
36  deps_paths = bootstrap.ListAllDepsPaths(deps_file)
37  return set(
38      os.path.realpath(os.path.join(util.GetChromiumSrcDir(), os.pardir, path))
39      for path in deps_paths)
40
41
42def FindPythonDependencies(module_path):
43  logging.info('Finding Python dependencies of %s' % module_path)
44
45  # Load the module to inherit its sys.path modifications.
46  imp.load_source(
47      os.path.splitext(os.path.basename(module_path))[0], module_path)
48
49  # Analyze the module for its imports.
50  finder = modulefinder.ModuleFinder()
51  finder.run_script(module_path)
52
53  # Filter for only imports in Chromium.
54  for module in finder.modules.itervalues():
55    # If it's an __init__.py, module.__path__ gives the package's folder.
56    module_path = module.__path__[0] if module.__path__ else module.__file__
57    if not module_path:
58      continue
59
60    module_path = os.path.realpath(module_path)
61    if not _InDirectory(module_path, util.GetChromiumSrcDir()):
62      continue
63
64    yield module_path
65
66
67def FindPageSetDependencies(base_dir):
68  logging.info('Finding page sets in %s' % base_dir)
69
70  # Add base_dir to path so our imports relative to base_dir will work.
71  sys.path.append(base_dir)
72  tests = discover.DiscoverClasses(base_dir, base_dir, benchmark.Benchmark,
73                                   index_by_class_name=True)
74
75  for test_class in tests.itervalues():
76    test_obj = test_class()
77
78    # Ensure the test's default options are set if needed.
79    parser = optparse.OptionParser()
80    test_obj.AddCommandLineArgs(parser)
81    options = optparse.Values()
82    for k, v in parser.get_default_values().__dict__.iteritems():
83      options.ensure_value(k, v)
84
85    # Page set paths are relative to their runner script, not relative to us.
86    util.GetBaseDir = lambda: base_dir
87    # TODO: Loading the page set will automatically download its Cloud Storage
88    # deps. This is really expensive, and we don't want to do this by default.
89    page_set = test_obj.CreatePageSet(options)
90
91    # Add all of its serving_dirs as dependencies.
92    for serving_dir in page_set.serving_dirs:
93      yield serving_dir
94    for page in page_set:
95      if page.is_file:
96        yield page.serving_dir
97
98
99def FindExcludedFiles(files, options):
100  def MatchesConditions(path, conditions):
101    for condition in conditions:
102      if condition(path):
103        return True
104    return False
105
106  # Define some filters for files.
107  def IsHidden(path):
108    for pathname_component in path.split(os.sep):
109      if pathname_component.startswith('.'):
110        return True
111    return False
112  def IsPyc(path):
113    return os.path.splitext(path)[1] == '.pyc'
114  def IsInCloudStorage(path):
115    return os.path.exists(path + '.sha1')
116  def MatchesExcludeOptions(path):
117    for pattern in options.exclude:
118      if (fnmatch.fnmatch(path, pattern) or
119          fnmatch.fnmatch(os.path.basename(path), pattern)):
120        return True
121    return False
122
123  # Collect filters we're going to use to exclude files.
124  exclude_conditions = [
125      IsHidden,
126      IsPyc,
127      IsInCloudStorage,
128      MatchesExcludeOptions,
129  ]
130
131  # Check all the files against the filters.
132  for path in files:
133    if MatchesConditions(path, exclude_conditions):
134      yield path
135
136
137def FindDependencies(paths, options):
138  # Verify arguments.
139  for path in paths:
140    if not os.path.exists(path):
141      raise ValueError('Path does not exist: %s' % path)
142
143  dependencies = path_set.PathSet()
144
145  # Including __init__.py will include Telemetry and its dependencies.
146  # If the user doesn't pass any arguments, we just have Telemetry.
147  dependencies |= FindPythonDependencies(os.path.realpath(
148    os.path.join(util.GetTelemetryDir(), 'telemetry', '__init__.py')))
149  dependencies |= FindBootstrapDependencies(util.GetTelemetryDir())
150
151  # Add dependencies.
152  for path in paths:
153    base_dir = os.path.dirname(os.path.realpath(path))
154
155    dependencies.add(base_dir)
156    dependencies |= FindBootstrapDependencies(base_dir)
157    dependencies |= FindPythonDependencies(path)
158    if options.include_page_set_data:
159      dependencies |= FindPageSetDependencies(base_dir)
160
161  # Remove excluded files.
162  dependencies -= FindExcludedFiles(set(dependencies), options)
163
164  return dependencies
165
166
167def ZipDependencies(paths, dependencies, options):
168  base_dir = os.path.dirname(os.path.realpath(util.GetChromiumSrcDir()))
169
170  with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file:
171    # Add dependencies to archive.
172    for path in dependencies:
173      path_in_archive = os.path.join(
174          'telemetry', os.path.relpath(path, base_dir))
175      zip_file.write(path, path_in_archive)
176
177    # Add symlinks to executable paths, for ease of use.
178    for path in paths:
179      link_info = zipfile.ZipInfo(
180          os.path.join('telemetry', os.path.basename(path)))
181      link_info.create_system = 3  # Unix attributes.
182      # 010 is regular file, 0111 is the permission bits rwxrwxrwx.
183      link_info.external_attr = 0100777 << 16  # Octal.
184
185      relative_path = os.path.relpath(path, base_dir)
186      link_script = (
187          '#!/usr/bin/env python\n\n'
188          'import os\n'
189          'import sys\n\n\n'
190          'script = os.path.join(os.path.dirname(__file__), \'%s\')\n'
191          'os.execv(sys.executable, [sys.executable, script] + sys.argv[1:])'
192        % relative_path)
193
194      zip_file.writestr(link_info, link_script)
195
196    # Add gsutil to the archive, if it's available. The gsutil in
197    # depot_tools is modified to allow authentication using prodaccess.
198    # TODO: If there's a gsutil in telemetry/third_party/, bootstrap_deps
199    # will include it. Then there will be two copies of gsutil at the same
200    # location in the archive. This can be confusing for users.
201    gsutil_path = os.path.realpath(cloud_storage.FindGsutil())
202    if cloud_storage.SupportsProdaccess(gsutil_path):
203      gsutil_base_dir = os.path.join(os.path.dirname(gsutil_path), os.pardir)
204      gsutil_dependencies = path_set.PathSet()
205      gsutil_dependencies.add(os.path.dirname(gsutil_path))
206      # Also add modules from depot_tools that are needed by gsutil.
207      gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'boto'))
208      gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'fancy_urllib'))
209      gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'retry_decorator'))
210      gsutil_dependencies -= FindExcludedFiles(
211          set(gsutil_dependencies), options)
212
213      # Also add upload.py to the archive from depot_tools, if it is available.
214      # This allows us to post patches without requiring a full depot_tools
215      # install. There's no real point in including upload.py if we do not
216      # also have gsutil, which is why this is inside the gsutil block.
217      gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'upload.py'))
218
219      for path in gsutil_dependencies:
220        path_in_archive = os.path.join(
221            'telemetry', os.path.relpath(util.GetTelemetryDir(), base_dir),
222            'third_party', os.path.relpath(path, gsutil_base_dir))
223        zip_file.write(path, path_in_archive)
224
225
226class FindDependenciesCommand(command_line.OptparseCommand):
227  """Prints all dependencies"""
228
229  @classmethod
230  def AddCommandLineArgs(cls, parser):
231    parser.add_option(
232        '-v', '--verbose', action='count', dest='verbosity',
233        help='Increase verbosity level (repeat as needed).')
234
235    parser.add_option(
236        '-p', '--include-page-set-data', action='store_true', default=False,
237        help='Scan tests for page set data and include them.')
238
239    parser.add_option(
240        '-e', '--exclude', action='append', default=[],
241        help='Exclude paths matching EXCLUDE. Can be used multiple times.')
242
243    parser.add_option(
244        '-z', '--zip',
245        help='Store files in a zip archive at ZIP.')
246
247  @classmethod
248  def ProcessCommandLineArgs(cls, parser, args):
249    if args.verbosity >= 2:
250      logging.getLogger().setLevel(logging.DEBUG)
251    elif args.verbosity:
252      logging.getLogger().setLevel(logging.INFO)
253    else:
254      logging.getLogger().setLevel(logging.WARNING)
255
256  def Run(self, args):
257    paths = args.positional_args
258    dependencies = FindDependencies(paths, args)
259    if args.zip:
260      ZipDependencies(paths, dependencies, args)
261      print 'Zip archive written to %s.' % args.zip
262    else:
263      print '\n'.join(sorted(dependencies))
264    return 0
265