find_dependencies.py revision 23730a6e56a168d1879203e4b3819bb36e3d8f1f 
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import fnmatch
6import imp
7import logging
8import modulefinder
9import optparse
10import os
11import sys
12import zipfile
13
14from telemetry import test
15from telemetry.core import command_line
16from telemetry.core import discover
17from telemetry.core import util
18from telemetry.page import cloud_storage
19from telemetry.util import bootstrap
20from telemetry.util import path_set
21
22
23DEPS_FILE = 'bootstrap_deps'
24
25
26def _InDirectory(subdirectory, directory):
27  subdirectory = os.path.realpath(subdirectory)
28  directory = os.path.realpath(directory)
29  common_prefix = os.path.commonprefix([subdirectory, directory])
30  return common_prefix == directory
31
32
33def FindBootstrapDependencies(base_dir):
34  deps_file = os.path.join(base_dir, DEPS_FILE)
35  if not os.path.exists(deps_file):
36    return []
37  deps_paths = bootstrap.ListAllDepsPaths(deps_file)
38  return set(
39      os.path.realpath(os.path.join(util.GetChromiumSrcDir(), os.pardir, path))
40      for path in deps_paths)
41
42
43def FindPythonDependencies(module_path):
44  logging.info('Finding Python dependencies of %s' % module_path)
45
46  # Load the module to inherit its sys.path modifications.
47  imp.load_source(
48      os.path.splitext(os.path.basename(module_path))[0], module_path)
49
50  # Analyze the module for its imports.
51  finder = modulefinder.ModuleFinder()
52  finder.run_script(module_path)
53
54  # Filter for only imports in Chromium.
55  for module in finder.modules.itervalues():
56    # If it's an __init__.py, module.__path__ gives the package's folder.
57    module_path = module.__path__[0] if module.__path__ else module.__file__
58    if not module_path:
59      continue
60
61    module_path = os.path.realpath(module_path)
62    if not _InDirectory(module_path, util.GetChromiumSrcDir()):
63      continue
64
65    yield module_path
66
67
68def FindPageSetDependencies(base_dir):
69  logging.info('Finding page sets in %s' % base_dir)
70
71  # Add base_dir to path so our imports relative to base_dir will work.
72  sys.path.append(base_dir)
73  tests = discover.DiscoverClasses(base_dir, base_dir, test.Test,
74                                   index_by_class_name=True)
75
76  for test_class in tests.itervalues():
77    test_obj = test_class()
78
79    # Ensure the test's default options are set if needed.
80    parser = optparse.OptionParser()
81    test_obj.AddCommandLineArgs(parser)
82    options = optparse.Values()
83    for k, v in parser.get_default_values().__dict__.iteritems():
84      options.ensure_value(k, v)
85
86    # Page set paths are relative to their runner script, not relative to us.
87    util.GetBaseDir = lambda: base_dir
88    # TODO: Loading the page set will automatically download its Cloud Storage
89    # deps. This is really expensive, and we don't want to do this by default.
90    page_set = test_obj.CreatePageSet(options)
91
92    # Add all of its serving_dirs as dependencies.
93    for serving_dir in page_set.serving_dirs:
94      yield serving_dir
95    for page in page_set:
96      if page.is_file:
97        yield page.serving_dir
98
99
100def FindExcludedFiles(files, options):
101  def MatchesConditions(path, conditions):
102    for condition in conditions:
103      if condition(path):
104        return True
105    return False
106
107  # Define some filters for files.
108  def IsHidden(path):
109    for pathname_component in path.split(os.sep):
110      if pathname_component.startswith('.'):
111        return True
112    return False
113  def IsPyc(path):
114    return os.path.splitext(path)[1] == '.pyc'
115  def IsInCloudStorage(path):
116    return os.path.exists(path + '.sha1')
117  def MatchesExcludeOptions(path):
118    for pattern in options.exclude:
119      if (fnmatch.fnmatch(path, pattern) or
120          fnmatch.fnmatch(os.path.basename(path), pattern)):
121        return True
122    return False
123
124  # Collect filters we're going to use to exclude files.
125  exclude_conditions = [
126      IsHidden,
127      IsPyc,
128      IsInCloudStorage,
129      MatchesExcludeOptions,
130  ]
131
132  # Check all the files against the filters.
133  for path in files:
134    if MatchesConditions(path, exclude_conditions):
135      yield path
136
137
138def FindDependencies(paths, options):
139  # Verify arguments.
140  for path in paths:
141    if not os.path.exists(path):
142      raise ValueError('Path does not exist: %s' % path)
143
144  dependencies = path_set.PathSet()
145
146  # Including __init__.py will include Telemetry and its dependencies.
147  # If the user doesn't pass any arguments, we just have Telemetry.
148  dependencies |= FindPythonDependencies(os.path.realpath(
149    os.path.join(util.GetTelemetryDir(), 'telemetry', '__init__.py')))
150  dependencies |= FindBootstrapDependencies(util.GetTelemetryDir())
151
152  # Add dependencies.
153  for path in paths:
154    base_dir = os.path.dirname(os.path.realpath(path))
155
156    dependencies.add(base_dir)
157    dependencies |= FindBootstrapDependencies(base_dir)
158    dependencies |= FindPythonDependencies(path)
159    if options.include_page_set_data:
160      dependencies |= FindPageSetDependencies(base_dir)
161
162  # Remove excluded files.
163  dependencies -= FindExcludedFiles(set(dependencies), options)
164
165  return dependencies
166
167
168def ZipDependencies(paths, dependencies, options):
169  base_dir = os.path.dirname(os.path.realpath(util.GetChromiumSrcDir()))
170
171  with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file:
172    # Add dependencies to archive.
173    for path in dependencies:
174      path_in_archive = os.path.join(
175          'telemetry', os.path.relpath(path, base_dir))
176      zip_file.write(path, path_in_archive)
177
178    # Add symlinks to executable paths, for ease of use.
179    for path in paths:
180      link_info = zipfile.ZipInfo(
181          os.path.join('telemetry', os.path.basename(path)))
182      link_info.create_system = 3  # Unix attributes.
183      # 010 is regular file, 0111 is the permission bits rwxrwxrwx.
184      link_info.external_attr = 0100777 << 16  # Octal.
185
186      relative_path = os.path.relpath(path, base_dir)
187      link_script = (
188          '#!/usr/bin/env python\n\n'
189          'import os\n'
190          'import sys\n\n\n'
191          'script = os.path.join(os.path.dirname(__file__), \'%s\')\n'
192          'os.execv(sys.executable, [sys.executable, script] + sys.argv[1:])'
193        % relative_path)
194
195      zip_file.writestr(link_info, link_script)
196
197    # Add gsutil to the archive, if it's available. The gsutil in
198    # depot_tools is modified to allow authentication using prodaccess.
199    # TODO: If there's a gsutil in telemetry/third_party/, bootstrap_deps
200    # will include it. Then there will be two copies of gsutil at the same
201    # location in the archive. This can be confusing for users.
202    gsutil_path = cloud_storage.FindGsutil()
203    if cloud_storage.SupportsProdaccess(gsutil_path):
204      gsutil_base_dir = os.path.join(os.path.dirname(gsutil_path), os.pardir)
205      gsutil_dependencies = path_set.PathSet()
206      gsutil_dependencies.add(os.path.dirname(gsutil_path))
207      # Also add modules from depot_tools that are needed by gsutil.
208      gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'boto'))
209      gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'retry_decorator'))
210      gsutil_dependencies -= FindExcludedFiles(
211          set(gsutil_dependencies), options)
212
213      for path in gsutil_dependencies:
214        path_in_archive = os.path.join(
215            'telemetry', os.path.relpath(util.GetTelemetryDir(), base_dir),
216            'third_party', os.path.relpath(path, gsutil_base_dir))
217        zip_file.write(path, path_in_archive)
218
219
220class FindDependenciesCommand(command_line.OptparseCommand):
221  """Prints all dependencies"""
222
223  @classmethod
224  def AddCommandLineArgs(cls, parser):
225    parser.add_option(
226        '-v', '--verbose', action='count', dest='verbosity',
227        help='Increase verbosity level (repeat as needed).')
228
229    parser.add_option(
230        '-p', '--include-page-set-data', action='store_true', default=False,
231        help='Scan tests for page set data and include them.')
232
233    parser.add_option(
234        '-e', '--exclude', action='append', default=[],
235        help='Exclude paths matching EXCLUDE. Can be used multiple times.')
236
237    parser.add_option(
238        '-z', '--zip',
239        help='Store files in a zip archive at ZIP.')
240
241  @classmethod
242  def ProcessCommandLineArgs(cls, parser, args):
243    if args.verbosity >= 2:
244      logging.getLogger().setLevel(logging.DEBUG)
245    elif args.verbosity:
246      logging.getLogger().setLevel(logging.INFO)
247    else:
248      logging.getLogger().setLevel(logging.WARNING)
249
250  def Run(self, args):
251    paths = args.positional_args
252    dependencies = FindDependencies(paths, args)
253    if args.zip:
254      ZipDependencies(paths, dependencies, args)
255      print 'Zip archive written to %s.' % args.zip
256    else:
257      print '\n'.join(sorted(dependencies))
258    return 0
259