133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck# Copyright 2015 The Chromium Authors. All rights reserved. 233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck# Use of this source code is governed by a BSD-style license that can be 333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck# found in the LICENSE file. 433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckimport logging 633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckimport os 733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckfrom py_utils import binary_manager 933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckfrom py_utils import dependency_util 1033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckimport dependency_manager 1133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckfrom devil import devil_env 1233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 1333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckfrom telemetry.core import exceptions 1433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckfrom telemetry.core import util 1533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckfrom telemetry.core import platform as platform_module 1633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 1733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 1833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John ReckTELEMETRY_PROJECT_CONFIG = os.path.join( 1933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck util.GetTelemetryDir(), 'telemetry', 'internal', 'binary_dependencies.json') 2033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 2133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 2233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John ReckCHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'py_utils', 2333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'py_utils', 'chrome_binaries.json') 2433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 2533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 2633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John ReckBATTOR_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'battor', 2733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'battor', 'battor_binary_dependencies.json') 2833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 2933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 3033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John ReckNoPathFoundError = dependency_manager.NoPathFoundError 3133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John ReckCloudStorageError = dependency_manager.CloudStorageError 3233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 3333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 3433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck_binary_manager = None 3533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 3633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 3733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckdef NeedsInit(): 3833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck return not _binary_manager 3933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 4033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 4133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckdef InitDependencyManager(client_configs): 4233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck global _binary_manager 4333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if _binary_manager: 4433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck raise exceptions.InitializationError( 4533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'Trying to re-initialize the binary manager with config %s' 4633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck % client_configs) 4733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck configs = [] 4833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if client_configs: 4933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck configs += client_configs 5033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck configs += [TELEMETRY_PROJECT_CONFIG, CHROME_BINARY_CONFIG] 5133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck _binary_manager = binary_manager.BinaryManager(configs) 5233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 5333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck devil_env.config.Initialize() 5433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 5533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 5633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckdef FetchPath(binary_name, arch, os_name, os_version=None): 5733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck """ Return a path to the appropriate executable for <binary_name>, downloading 5833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck from cloud storage if needed, or None if it cannot be found. 5933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck """ 6033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if _binary_manager is None: 6133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck raise exceptions.InitializationError( 6233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'Called FetchPath with uninitialized binary manager.') 6333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck return _binary_manager.FetchPath(binary_name, os_name, arch, os_version) 6433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 6533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 6633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckdef LocalPath(binary_name, arch, os_name, os_version=None): 6733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck """ Return a local path to the given binary name, or None if an executable 6833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck cannot be found. Will not download the executable. 6933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck """ 7033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if _binary_manager is None: 7133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck raise exceptions.InitializationError( 7233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'Called LocalPath with uninitialized binary manager.') 7333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck return _binary_manager.LocalPath(binary_name, os_name, arch, os_version) 7433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 7533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 7633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckdef FetchBinaryDependencies(platform, client_configs, 7733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck fetch_reference_chrome_binary): 7833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck """ Fetch all binary dependenencies for the given |platform|. 7933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 8033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck Note: we don't fetch browser binaries by default because the size of the 8133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck binary is about 2Gb, and it requires cloud storage permission to 8233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck chrome-telemetry bucket. 8333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 8433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck Args: 8533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck platform: an instance of telemetry.core.platform 8633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck client_configs: A list of paths (string) to dependencies json files. 8733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck fetch_reference_chrome_binary: whether to fetch reference chrome binary for 8833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck the given platform. 8933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck """ 9033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck configs = [ 9133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck dependency_manager.BaseConfig(TELEMETRY_PROJECT_CONFIG), 9233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck dependency_manager.BaseConfig(BATTOR_BINARY_CONFIG) 9333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck ] 9433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck dep_manager = dependency_manager.DependencyManager(configs) 9533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck target_platform = '%s_%s' % (platform.GetOSName(), platform.GetArchName()) 9633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck dep_manager.PrefetchPaths(target_platform) 9733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 9833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck host_platform = None 9933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck fetch_devil_deps = False 10033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if platform.GetOSName() == 'android': 10133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck host_platform = '%s_%s' % ( 10233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck platform_module.GetHostPlatform().GetOSName(), 10333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck platform_module.GetHostPlatform().GetArchName()) 10433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck dep_manager.PrefetchPaths(host_platform) 10533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # TODO(aiolos): this is a hack to prefetch the devil deps. 10633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if host_platform == 'linux_x86_64': 10733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck fetch_devil_deps = True 10833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck else: 10933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck logging.error('Devil only supports 64 bit linux as a host platform. ' 11033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'Android tests may fail.') 11133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 11233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if fetch_reference_chrome_binary: 11333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck _FetchReferenceBrowserBinary(platform) 11433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 11533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # For now, handle client config separately because the BUILD.gn & .isolate of 11633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # telemetry tests in chromium src failed to include the files specified in its 11733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # client config. 11833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # (https://github.com/catapult-project/catapult/issues/2192) 11933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # For now this is ok because the client configs usually don't include cloud 12033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # storage infos. 12133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # TODO(nednguyen): remove the logic of swallowing exception once the issue is 12233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck # fixed on Chromium side. 12333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if client_configs: 12433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck manager = dependency_manager.DependencyManager( 12533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck list(dependency_manager.BaseConfig(c) for c in client_configs)) 12633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck try: 12733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck manager.PrefetchPaths(target_platform) 12833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if host_platform is not None: 12933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck manager.PrefetchPaths(host_platform) 13033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 13133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck except dependency_manager.NoPathFoundError as e: 13233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck logging.error('Error when trying to prefetch paths for %s: %s', 13333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck target_platform, e.message) 13433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 13533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if fetch_devil_deps: 13633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck devil_env.config.Initialize() 13733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck devil_env.config.PrefetchPaths(arch=platform.GetArchName()) 13833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck devil_env.config.PrefetchPaths() 13933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 14033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 14133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reckdef _FetchReferenceBrowserBinary(platform): 14233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck os_name = platform.GetOSName() 14333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck arch_name = platform.GetArchName() 14433259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck manager = binary_manager.BinaryManager( 14533259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck [CHROME_BINARY_CONFIG]) 14633259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck if os_name == 'android': 14733259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck os_version = dependency_util.GetChromeApkOsVersion( 14833259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck platform.GetOSVersionName()) 14933259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck manager.FetchPath( 15033259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'chrome_stable', os_name, arch_name, os_version) 15133259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck else: 15233259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck manager.FetchPath( 15333259e44c8229f70ffe0cf3bb5ca9375c4feb2f9John Reck 'chrome_stable', os_name, arch_name) 154