1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import fnmatch 6import imp 7import logging 8import modulefinder 9import optparse 10import os 11import sys 12import zipfile 13 14from telemetry import benchmark 15from telemetry.core import command_line 16from telemetry.core import discover 17from telemetry.core import util 18from telemetry.util import bootstrap 19from telemetry.util import cloud_storage 20from telemetry.util import path_set 21 22DEPS_FILE = 'bootstrap_deps' 23 24 25def _InDirectory(subdirectory, directory): 26 subdirectory = os.path.realpath(subdirectory) 27 directory = os.path.realpath(directory) 28 common_prefix = os.path.commonprefix([subdirectory, directory]) 29 return common_prefix == directory 30 31 32def FindBootstrapDependencies(base_dir): 33 deps_file = os.path.join(base_dir, DEPS_FILE) 34 if not os.path.exists(deps_file): 35 return [] 36 deps_paths = bootstrap.ListAllDepsPaths(deps_file) 37 return set( 38 os.path.realpath(os.path.join(util.GetChromiumSrcDir(), os.pardir, path)) 39 for path in deps_paths) 40 41 42def FindPythonDependencies(module_path): 43 logging.info('Finding Python dependencies of %s' % module_path) 44 45 # Load the module to inherit its sys.path modifications. 46 imp.load_source( 47 os.path.splitext(os.path.basename(module_path))[0], module_path) 48 49 # Analyze the module for its imports. 50 finder = modulefinder.ModuleFinder() 51 finder.run_script(module_path) 52 53 # Filter for only imports in Chromium. 54 for module in finder.modules.itervalues(): 55 # If it's an __init__.py, module.__path__ gives the package's folder. 56 module_path = module.__path__[0] if module.__path__ else module.__file__ 57 if not module_path: 58 continue 59 60 module_path = os.path.realpath(module_path) 61 if not _InDirectory(module_path, util.GetChromiumSrcDir()): 62 continue 63 64 yield module_path 65 66 67def FindPageSetDependencies(base_dir): 68 logging.info('Finding page sets in %s' % base_dir) 69 70 # Add base_dir to path so our imports relative to base_dir will work. 71 sys.path.append(base_dir) 72 tests = discover.DiscoverClasses(base_dir, base_dir, benchmark.Benchmark, 73 index_by_class_name=True) 74 75 for test_class in tests.itervalues(): 76 test_obj = test_class() 77 78 # Ensure the test's default options are set if needed. 79 parser = optparse.OptionParser() 80 test_obj.AddCommandLineArgs(parser) 81 options = optparse.Values() 82 for k, v in parser.get_default_values().__dict__.iteritems(): 83 options.ensure_value(k, v) 84 85 # Page set paths are relative to their runner script, not relative to us. 86 util.GetBaseDir = lambda: base_dir 87 # TODO: Loading the page set will automatically download its Cloud Storage 88 # deps. This is really expensive, and we don't want to do this by default. 89 page_set = test_obj.CreatePageSet(options) 90 91 # Add all of its serving_dirs as dependencies. 92 for serving_dir in page_set.serving_dirs: 93 yield serving_dir 94 for page in page_set: 95 if page.is_file: 96 yield page.serving_dir 97 98 99def FindExcludedFiles(files, options): 100 def MatchesConditions(path, conditions): 101 for condition in conditions: 102 if condition(path): 103 return True 104 return False 105 106 # Define some filters for files. 107 def IsHidden(path): 108 for pathname_component in path.split(os.sep): 109 if pathname_component.startswith('.'): 110 return True 111 return False 112 def IsPyc(path): 113 return os.path.splitext(path)[1] == '.pyc' 114 def IsInCloudStorage(path): 115 return os.path.exists(path + '.sha1') 116 def MatchesExcludeOptions(path): 117 for pattern in options.exclude: 118 if (fnmatch.fnmatch(path, pattern) or 119 fnmatch.fnmatch(os.path.basename(path), pattern)): 120 return True 121 return False 122 123 # Collect filters we're going to use to exclude files. 124 exclude_conditions = [ 125 IsHidden, 126 IsPyc, 127 IsInCloudStorage, 128 MatchesExcludeOptions, 129 ] 130 131 # Check all the files against the filters. 132 for path in files: 133 if MatchesConditions(path, exclude_conditions): 134 yield path 135 136 137def FindDependencies(paths, options): 138 # Verify arguments. 139 for path in paths: 140 if not os.path.exists(path): 141 raise ValueError('Path does not exist: %s' % path) 142 143 dependencies = path_set.PathSet() 144 145 # Including __init__.py will include Telemetry and its dependencies. 146 # If the user doesn't pass any arguments, we just have Telemetry. 147 dependencies |= FindPythonDependencies(os.path.realpath( 148 os.path.join(util.GetTelemetryDir(), 'telemetry', '__init__.py'))) 149 dependencies |= FindBootstrapDependencies(util.GetTelemetryDir()) 150 151 # Add dependencies. 152 for path in paths: 153 base_dir = os.path.dirname(os.path.realpath(path)) 154 155 dependencies.add(base_dir) 156 dependencies |= FindBootstrapDependencies(base_dir) 157 dependencies |= FindPythonDependencies(path) 158 if options.include_page_set_data: 159 dependencies |= FindPageSetDependencies(base_dir) 160 161 # Remove excluded files. 162 dependencies -= FindExcludedFiles(set(dependencies), options) 163 164 return dependencies 165 166 167def ZipDependencies(paths, dependencies, options): 168 base_dir = os.path.dirname(os.path.realpath(util.GetChromiumSrcDir())) 169 170 with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: 171 # Add dependencies to archive. 172 for path in dependencies: 173 path_in_archive = os.path.join( 174 'telemetry', os.path.relpath(path, base_dir)) 175 zip_file.write(path, path_in_archive) 176 177 # Add symlinks to executable paths, for ease of use. 178 for path in paths: 179 link_info = zipfile.ZipInfo( 180 os.path.join('telemetry', os.path.basename(path))) 181 link_info.create_system = 3 # Unix attributes. 182 # 010 is regular file, 0111 is the permission bits rwxrwxrwx. 183 link_info.external_attr = 0100777 << 16 # Octal. 184 185 relative_path = os.path.relpath(path, base_dir) 186 link_script = ( 187 '#!/usr/bin/env python\n\n' 188 'import os\n' 189 'import sys\n\n\n' 190 'script = os.path.join(os.path.dirname(__file__), \'%s\')\n' 191 'os.execv(sys.executable, [sys.executable, script] + sys.argv[1:])' 192 % relative_path) 193 194 zip_file.writestr(link_info, link_script) 195 196 # Add gsutil to the archive, if it's available. The gsutil in 197 # depot_tools is modified to allow authentication using prodaccess. 198 # TODO: If there's a gsutil in telemetry/third_party/, bootstrap_deps 199 # will include it. Then there will be two copies of gsutil at the same 200 # location in the archive. This can be confusing for users. 201 gsutil_path = os.path.realpath(cloud_storage.FindGsutil()) 202 if cloud_storage.SupportsProdaccess(gsutil_path): 203 gsutil_base_dir = os.path.join(os.path.dirname(gsutil_path), os.pardir) 204 gsutil_dependencies = path_set.PathSet() 205 gsutil_dependencies.add(os.path.dirname(gsutil_path)) 206 # Also add modules from depot_tools that are needed by gsutil. 207 gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'boto')) 208 gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'fancy_urllib')) 209 gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'retry_decorator')) 210 gsutil_dependencies -= FindExcludedFiles( 211 set(gsutil_dependencies), options) 212 213 # Also add upload.py to the archive from depot_tools, if it is available. 214 # This allows us to post patches without requiring a full depot_tools 215 # install. There's no real point in including upload.py if we do not 216 # also have gsutil, which is why this is inside the gsutil block. 217 gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'upload.py')) 218 219 for path in gsutil_dependencies: 220 path_in_archive = os.path.join( 221 'telemetry', os.path.relpath(util.GetTelemetryDir(), base_dir), 222 'third_party', os.path.relpath(path, gsutil_base_dir)) 223 zip_file.write(path, path_in_archive) 224 225 226class FindDependenciesCommand(command_line.OptparseCommand): 227 """Prints all dependencies""" 228 229 @classmethod 230 def AddCommandLineArgs(cls, parser): 231 parser.add_option( 232 '-v', '--verbose', action='count', dest='verbosity', 233 help='Increase verbosity level (repeat as needed).') 234 235 parser.add_option( 236 '-p', '--include-page-set-data', action='store_true', default=False, 237 help='Scan tests for page set data and include them.') 238 239 parser.add_option( 240 '-e', '--exclude', action='append', default=[], 241 help='Exclude paths matching EXCLUDE. Can be used multiple times.') 242 243 parser.add_option( 244 '-z', '--zip', 245 help='Store files in a zip archive at ZIP.') 246 247 @classmethod 248 def ProcessCommandLineArgs(cls, parser, args): 249 if args.verbosity >= 2: 250 logging.getLogger().setLevel(logging.DEBUG) 251 elif args.verbosity: 252 logging.getLogger().setLevel(logging.INFO) 253 else: 254 logging.getLogger().setLevel(logging.WARNING) 255 256 def Run(self, args): 257 paths = args.positional_args 258 dependencies = FindDependencies(paths, args) 259 if args.zip: 260 ZipDependencies(paths, dependencies, args) 261 print 'Zip archive written to %s.' % args.zip 262 else: 263 print '\n'.join(sorted(dependencies)) 264 return 0 265