find_dependencies.py revision cedac228d2dd51db4b79ea1e72c7f249408ee061
1868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)# Copyright 2014 The Chromium Authors. All rights reserved. 2868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 3868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)# found in the LICENSE file. 4868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 5868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import fnmatch 6868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import imp 7868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import logging 8868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import modulefinder 9868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import optparse 10868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import os 11868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import sys 12868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import zipfile 13868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 14868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)from telemetry import test 15868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)from telemetry.core import command_line 16868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)from telemetry.core import discover 17868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)from telemetry.core import util 18868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)from telemetry.page import cloud_storage 19868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)from telemetry.util import bootstrap 20868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)from telemetry.util import path_set 21868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 22868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 23868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)DEPS_FILE = 'bootstrap_deps' 24868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 25868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 26868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)def _InDirectory(subdirectory, directory): 27868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) subdirectory = os.path.realpath(subdirectory) 28868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) directory = os.path.realpath(directory) 29868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) common_prefix = os.path.commonprefix([subdirectory, directory]) 30868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return common_prefix == directory 31868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 32868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 33868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)def FindBootstrapDependencies(base_dir): 34868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) deps_file = os.path.join(base_dir, DEPS_FILE) 35868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if not os.path.exists(deps_file): 36868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return [] 37868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) deps_paths = bootstrap.ListAllDepsPaths(deps_file) 38868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return set( 39868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) os.path.realpath(os.path.join(util.GetChromiumSrcDir(), os.pardir, path)) 40868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for path in deps_paths) 41868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 42868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 43868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)def FindPythonDependencies(module_path): 44868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) logging.info('Finding Python dependencies of %s' % module_path) 45868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 46868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Load the module to inherit its sys.path modifications. 47868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) imp.load_source( 48868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) os.path.splitext(os.path.basename(module_path))[0], module_path) 49868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 50868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Analyze the module for its imports. 51868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) finder = modulefinder.ModuleFinder() 52868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) finder.run_script(module_path) 53868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 54868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Filter for only imports in Chromium. 55868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for module in finder.modules.itervalues(): 56868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # If it's an __init__.py, module.__path__ gives the package's folder. 57868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) module_path = module.__path__[0] if module.__path__ else module.__file__ 58868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if not module_path: 59868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) continue 60868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 61868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) module_path = os.path.realpath(module_path) 62868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if not _InDirectory(module_path, util.GetChromiumSrcDir()): 63868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) continue 64868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 65868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) yield module_path 66868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 67868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 68868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)def FindPageSetDependencies(base_dir): 69868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) logging.info('Finding page sets in %s' % base_dir) 70868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 71868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Add base_dir to path so our imports relative to base_dir will work. 72868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) sys.path.append(base_dir) 73868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) tests = discover.DiscoverClasses(base_dir, base_dir, test.Test, 74868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) index_by_class_name=True) 75868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 76868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for test_class in tests.itervalues(): 77868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) test_obj = test_class() 78868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 79868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Ensure the test's default options are set if needed. 80868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) parser = optparse.OptionParser() 81868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) test_obj.AddCommandLineArgs(parser) 82868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) options = optparse.Values() 83868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for k, v in parser.get_default_values().__dict__.iteritems(): 84868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) options.ensure_value(k, v) 85868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 86868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Page set paths are relative to their runner script, not relative to us. 87868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) util.GetBaseDir = lambda: base_dir 88868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # TODO: Loading the page set will automatically download its Cloud Storage 89868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # deps. This is really expensive, and we don't want to do this by default. 90868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) page_set = test_obj.CreatePageSet(options) 91868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 92868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Add all of its serving_dirs as dependencies. 93868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for serving_dir in page_set.serving_dirs: 94868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) yield serving_dir 95868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for page in page_set: 96868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if page.is_file: 97868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) yield page.serving_dir 98868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 99868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 100868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)def FindExcludedFiles(files, options): 101868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def MatchesConditions(path, conditions): 102868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for condition in conditions: 103868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if condition(path): 104868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return True 105868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return False 106868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 107868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Define some filters for files. 108868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def IsHidden(path): 109868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for pathname_component in path.split(os.sep): 110868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if pathname_component.startswith('.'): 111868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return True 112868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return False 113868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def IsPyc(path): 114868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return os.path.splitext(path)[1] == '.pyc' 115868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def IsInCloudStorage(path): 116868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return os.path.exists(path + '.sha1') 117868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def MatchesExcludeOptions(path): 118868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for pattern in options.exclude: 119868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if (fnmatch.fnmatch(path, pattern) or 120868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) fnmatch.fnmatch(os.path.basename(path), pattern)): 121868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return True 122868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return False 123868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 124868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Collect filters we're going to use to exclude files. 125868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) exclude_conditions = [ 126868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) IsHidden, 127a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) IsPyc, 128a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) IsInCloudStorage, 129a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) MatchesExcludeOptions, 130a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) ] 131a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 132a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) # Check all the files against the filters. 133a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) for path in files: 134a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) if MatchesConditions(path, exclude_conditions): 135a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) yield path 136a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 137868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 138868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)def FindDependencies(paths, options): 139868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Verify arguments. 140a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) for path in paths: 141a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) if not os.path.exists(path): 142868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) raise ValueError('Path does not exist: %s' % path) 143868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 144868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies = path_set.PathSet() 145868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 146a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) # Including __init__.py will include Telemetry and its dependencies. 147868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # If the user doesn't pass any arguments, we just have Telemetry. 148868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies |= FindPythonDependencies(os.path.realpath( 149868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) os.path.join(util.GetTelemetryDir(), 'telemetry', '__init__.py'))) 150868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies |= FindBootstrapDependencies(util.GetTelemetryDir()) 151868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 152868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Add dependencies. 153cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for path in paths: 154868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) base_dir = os.path.dirname(os.path.realpath(path)) 155868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 156cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) dependencies.add(base_dir) 157868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies |= FindBootstrapDependencies(base_dir) 158868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies |= FindPythonDependencies(path) 159868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if options.include_page_set_data: 160868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies |= FindPageSetDependencies(base_dir) 161868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 162868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Remove excluded files. 163868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies -= FindExcludedFiles(set(dependencies), options) 164868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 165cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return dependencies 166868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 167868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 168868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)def ZipDependencies(paths, dependencies, options): 169cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) base_dir = os.path.dirname(os.path.realpath(util.GetChromiumSrcDir())) 170868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 171868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: 172868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Add dependencies to archive. 173868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for path in dependencies: 174868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) path_in_archive = os.path.join( 175868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 'telemetry', os.path.relpath(path, base_dir)) 176868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) zip_file.write(path, path_in_archive) 177cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 178868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Add symlinks to executable paths, for ease of use. 179868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for path in paths: 180cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) link_info = zipfile.ZipInfo( 181868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) os.path.join('telemetry', os.path.basename(path))) 182868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) link_info.create_system = 3 # Unix attributes. 183868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # 010 is regular file, 0111 is the permission bits rwxrwxrwx. 184868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) link_info.external_attr = 0100777 << 16 # Octal. 185868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 186868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) relative_path = os.path.relpath(path, base_dir) 187868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) link_script = ( 188868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) '#!/usr/bin/env python\n\n' 189868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 'import os\n' 190868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 'import sys\n\n\n' 191868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 'script = os.path.join(os.path.dirname(__file__), \'%s\')\n' 192868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 'os.execv(sys.executable, [sys.executable, script] + sys.argv[1:])' 193868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) % relative_path) 194868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 195868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) zip_file.writestr(link_info, link_script) 196868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 197868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Add gsutil to the archive, if it's available. The gsutil in 198868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # depot_tools is modified to allow authentication using prodaccess. 199868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # TODO: If there's a gsutil in telemetry/third_party/, bootstrap_deps 200868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # will include it. Then there will be two copies of gsutil at the same 201868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # location in the archive. This can be confusing for users. 202a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) gsutil_path = os.path.realpath(cloud_storage.FindGsutil()) 203868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if cloud_storage.SupportsProdaccess(gsutil_path): 204a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) gsutil_base_dir = os.path.join(os.path.dirname(gsutil_path), os.pardir) 205868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) gsutil_dependencies = path_set.PathSet() 206868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) gsutil_dependencies.add(os.path.dirname(gsutil_path)) 207868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Also add modules from depot_tools that are needed by gsutil. 208868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'boto')) 209868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'retry_decorator')) 210868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) gsutil_dependencies -= FindExcludedFiles( 211868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) set(gsutil_dependencies), options) 212868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 213868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # Also add upload.py to the archive from depot_tools, if it is available. 214868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # This allows us to post patches without requiring a full depot_tools 215868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # install. There's no real point in including upload.py if we do not 216868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) # also have gsutil, which is why this is inside the gsutil block. 217868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) gsutil_dependencies.add(os.path.join(gsutil_base_dir, 'upload.py')) 218868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 219868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for path in gsutil_dependencies: 220868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) path_in_archive = os.path.join( 221868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 'telemetry', os.path.relpath(util.GetTelemetryDir(), base_dir), 222868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 'third_party', os.path.relpath(path, gsutil_base_dir)) 223a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) zip_file.write(path, path_in_archive) 224a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 225868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 226868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)class FindDependenciesCommand(command_line.OptparseCommand): 227868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) """Prints all dependencies""" 228868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 229868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) @classmethod 230868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def AddCommandLineArgs(cls, parser): 231868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) parser.add_option( 232a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) '-v', '--verbose', action='count', dest='verbosity', 233a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) help='Increase verbosity level (repeat as needed).') 234a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 235a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) parser.add_option( 236a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) '-p', '--include-page-set-data', action='store_true', default=False, 237868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) help='Scan tests for page set data and include them.') 238868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 239868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) parser.add_option( 240868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) '-e', '--exclude', action='append', default=[], 241868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) help='Exclude paths matching EXCLUDE. Can be used multiple times.') 242868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 243868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) parser.add_option( 244868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) '-z', '--zip', 245868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) help='Store files in a zip archive at ZIP.') 246868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 247868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) @classmethod 248868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def ProcessCommandLineArgs(cls, parser, args): 249010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) if args.verbosity >= 2: 250010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) logging.getLogger().setLevel(logging.DEBUG) 251010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) elif args.verbosity: 252868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) logging.getLogger().setLevel(logging.INFO) 253868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) else: 254868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) logging.getLogger().setLevel(logging.WARNING) 255868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 2566e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) def Run(self, args): 257868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) paths = args.positional_args 258868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) dependencies = FindDependencies(paths, args) 259868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if args.zip: 260868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) ZipDependencies(paths, dependencies, args) 261868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) print 'Zip archive written to %s.' % args.zip 262868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) else: 263868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) print '\n'.join(sorted(dependencies)) 264868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return 0 265868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)