1dc89ca56e6f33790cda2cd93ad772ca61497346bborenet#!/usr/bin/env python 2dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# Use of this source code is governed by a BSD-style license that can be 4dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# found in the LICENSE file. 5dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 6dc89ca56e6f33790cda2cd93ad772ca61497346bborenet"""Archives or replays webpages and creates SKPs in a Google Storage location. 7dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 8dc89ca56e6f33790cda2cd93ad772ca61497346bborenetTo archive webpages and store SKP files (archives should be rarely updated): 9dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 10fcf35c533b15b8c1d7396bd8f5c1e02bf436aec2kkinnunencd skia 11b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenpython tools/skp/webpages_playback.py --data_store=gs://rmistry --record \ 12dc89ca56e6f33790cda2cd93ad772ca61497346bborenet--page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \ 13dc89ca56e6f33790cda2cd93ad772ca61497346bborenet--browser_executable=/tmp/chromium/out/Release/chrome 14dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 15b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenThe above command uses Google Storage bucket 'rmistry' to download needed files. 16dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 17dc89ca56e6f33790cda2cd93ad772ca61497346bborenetTo replay archived webpages and re-generate SKP files (should be run whenever 18dc89ca56e6f33790cda2cd93ad772ca61497346bborenetSkPicture.PICTURE_VERSION changes): 19dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 20fcf35c533b15b8c1d7396bd8f5c1e02bf436aec2kkinnunencd skia 21b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenpython tools/skp/webpages_playback.py --data_store=gs://rmistry \ 22dc89ca56e6f33790cda2cd93ad772ca61497346bborenet--page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \ 23dc89ca56e6f33790cda2cd93ad772ca61497346bborenet--browser_executable=/tmp/chromium/out/Release/chrome 24dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 25dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 26dc89ca56e6f33790cda2cd93ad772ca61497346bborenetSpecify the --page_sets flag (default value is 'all') to pick a list of which 27dc89ca56e6f33790cda2cd93ad772ca61497346bborenetwebpages should be archived and/or replayed. Eg: 28dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 29fcf35c533b15b8c1d7396bd8f5c1e02bf436aec2kkinnunen--page_sets=tools/skp/page_sets/skia_yahooanswers_desktop.py,\ 30fcf35c533b15b8c1d7396bd8f5c1e02bf436aec2kkinnunentools/skp/page_sets/skia_googlecalendar_nexus10.py 31dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 32dc89ca56e6f33790cda2cd93ad772ca61497346bborenetThe --browser_executable flag should point to the browser binary you want to use 33dc89ca56e6f33790cda2cd93ad772ca61497346bborenetto capture archives and/or capture SKP files. Majority of the time it should be 34dc89ca56e6f33790cda2cd93ad772ca61497346bboreneta newly built chrome binary. 35dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 36b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenThe --data_store flag controls where the needed artifacts, such as 37b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunencredential files, are downloaded from. It also controls where the 38b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunengenerated artifacts, such as recorded webpages and resulting skp renderings, 39b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenare uploaded to. URLs with scheme 'gs://' use Google Storage. Otherwise 40b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenuse local filesystem. 41b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 42b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenThe --upload=True flag means generated artifacts will be 43b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenuploaded or copied to the location specified by --data_store. (default value is 44b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenFalse if not specified). 45dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 46dc89ca56e6f33790cda2cd93ad772ca61497346bborenetThe --non-interactive flag controls whether the script will prompt the user 47dc89ca56e6f33790cda2cd93ad772ca61497346bborenet(default value is False if not specified). 48dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 49dc89ca56e6f33790cda2cd93ad772ca61497346bborenetThe --skia_tools flag if specified will allow this script to run 50dc89ca56e6f33790cda2cd93ad772ca61497346bborenetdebugger, render_pictures, and render_pdfs on the captured 51dc89ca56e6f33790cda2cd93ad772ca61497346bborenetSKP(s). The tools are run after all SKPs are succesfully captured to make sure 52dc89ca56e6f33790cda2cd93ad772ca61497346bborenetthey can be added to the buildbots with no breakages. 53dc89ca56e6f33790cda2cd93ad772ca61497346bborenet""" 54dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 55dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport glob 56dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport optparse 57dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport os 58dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport posixpath 59dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport shutil 60dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport subprocess 61dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport sys 62dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport tempfile 63dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport time 64dc89ca56e6f33790cda2cd93ad772ca61497346bborenetimport traceback 65dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 66dc89ca56e6f33790cda2cd93ad772ca61497346bborenetsys.path.insert(0, os.getcwd()) 67dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 68dc89ca56e6f33790cda2cd93ad772ca61497346bborenetfrom common.py.utils import gs_utils 69dc89ca56e6f33790cda2cd93ad772ca61497346bborenetfrom common.py.utils import shell_utils 70dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 71dc89ca56e6f33790cda2cd93ad772ca61497346bborenetROOT_PLAYBACK_DIR_NAME = 'playback' 72dc89ca56e6f33790cda2cd93ad772ca61497346bborenetSKPICTURES_DIR_NAME = 'skps' 73dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 74dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 75dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# Local archive and SKP directories. 76dc89ca56e6f33790cda2cd93ad772ca61497346bborenetLOCAL_PLAYBACK_ROOT_DIR = os.path.join( 77dc89ca56e6f33790cda2cd93ad772ca61497346bborenet tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME) 78dc89ca56e6f33790cda2cd93ad772ca61497346bborenetLOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join( 79dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data') 80dc89ca56e6f33790cda2cd93ad772ca61497346bborenetTMP_SKP_DIR = tempfile.mkdtemp() 81dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 82f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry# Location of the credentials.json file and the string that represents missing 83f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry# passwords. 84f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistryCREDENTIALS_FILE_PATH = os.path.join( 85f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data', 86f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry 'credentials.json' 87f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry) 88f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry 89dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# Name of the SKP benchmark 90dc89ca56e6f33790cda2cd93ad772ca61497346bborenetSKP_BENCHMARK = 'skpicture_printer' 91dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 92dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# The max base name length of Skp files. 93dc89ca56e6f33790cda2cd93ad772ca61497346bborenetMAX_SKP_BASE_NAME_LEN = 31 94dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 95dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# Dictionary of device to platform prefixes for SKP files. 96dc89ca56e6f33790cda2cd93ad772ca61497346bborenetDEVICE_TO_PLATFORM_PREFIX = { 97dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'desktop': 'desk', 98dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'galaxynexus': 'mobi', 99dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'nexus10': 'tabl' 100dc89ca56e6f33790cda2cd93ad772ca61497346bborenet} 101dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 102dc89ca56e6f33790cda2cd93ad772ca61497346bborenet# How many times the record_wpr binary should be retried. 103dc89ca56e6f33790cda2cd93ad772ca61497346bborenetRETRY_RECORD_WPR_COUNT = 5 104783991585be67da3286680d850a99c5410885965borenet# How many times the run_benchmark binary should be retried. 105dc89ca56e6f33790cda2cd93ad772ca61497346bborenetRETRY_RUN_MEASUREMENT_COUNT = 5 106dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 107f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry# Location of the credentials.json file in Google Storage. 108f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistryCREDENTIALS_GS_PATH = '/playback/credentials/credentials.json' 109f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry 110dc89ca56e6f33790cda2cd93ad772ca61497346bborenetX11_DISPLAY = os.getenv('DISPLAY', ':0') 111dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 112dc89ca56e6f33790cda2cd93ad772ca61497346bborenetGS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE 113dc89ca56e6f33790cda2cd93ad772ca61497346bborenetGS_FINE_GRAINED_ACL_LIST = [ 114dc89ca56e6f33790cda2cd93ad772ca61497346bborenet (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com', 115dc89ca56e6f33790cda2cd93ad772ca61497346bborenet gs_utils.GSUtils.Permission.READ), 116dc89ca56e6f33790cda2cd93ad772ca61497346bborenet] 117dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 11849d093cef9d571877a07173e879a765ef83a90b9rmistry# Path to Chromium's page sets. 11949d093cef9d571877a07173e879a765ef83a90b9rmistryCHROMIUM_PAGE_SETS_PATH = os.path.join('tools', 'perf', 'page_sets') 12049d093cef9d571877a07173e879a765ef83a90b9rmistry 12149d093cef9d571877a07173e879a765ef83a90b9rmistry# Dictionary of supported Chromium page sets to their file prefixes. 12249d093cef9d571877a07173e879a765ef83a90b9rmistryCHROMIUM_PAGE_SETS_TO_PREFIX = { 1235af9a3792f3b5a1b2153391ddf1064098a177d4drmistry 'key_mobile_sites_smooth.py': 'keymobi', 1242a3c8493af6efe2256025b6cfbec882845925bd0rmistry 'top_25_smooth.py': 'top25desk', 12549d093cef9d571877a07173e879a765ef83a90b9rmistry} 12649d093cef9d571877a07173e879a765ef83a90b9rmistry 12749d093cef9d571877a07173e879a765ef83a90b9rmistry 128e75d2d2d995dad4eeed1bd306ef987adabac2bfckkinnunendef remove_prefix(s, prefix): 129e75d2d2d995dad4eeed1bd306ef987adabac2bfckkinnunen if s.startswith(prefix): 130e75d2d2d995dad4eeed1bd306ef987adabac2bfckkinnunen return s[len(prefix):] 131e75d2d2d995dad4eeed1bd306ef987adabac2bfckkinnunen return s 132dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 13349d093cef9d571877a07173e879a765ef83a90b9rmistry 134dc89ca56e6f33790cda2cd93ad772ca61497346bborenetclass SkPicturePlayback(object): 135dc89ca56e6f33790cda2cd93ad772ca61497346bborenet """Class that archives or replays webpages and creates SKPs.""" 136dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 137dc89ca56e6f33790cda2cd93ad772ca61497346bborenet def __init__(self, parse_options): 138dc89ca56e6f33790cda2cd93ad772ca61497346bborenet """Constructs a SkPicturePlayback BuildStep instance.""" 139dc89ca56e6f33790cda2cd93ad772ca61497346bborenet assert parse_options.browser_executable, 'Must specify --browser_executable' 140dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._browser_executable = parse_options.browser_executable 141aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry self._browser_args = '--disable-setuid-sandbox' 142aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry if parse_options.browser_extra_args: 143aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry self._browser_args = '%s %s' % ( 144aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry self._browser_args, parse_options.browser_extra_args) 145dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 14649d093cef9d571877a07173e879a765ef83a90b9rmistry self._chrome_page_sets_path = os.path.join(parse_options.chrome_src_path, 14749d093cef9d571877a07173e879a765ef83a90b9rmistry CHROMIUM_PAGE_SETS_PATH) 148dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._all_page_sets_specified = parse_options.page_sets == 'all' 149dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._page_sets = self._ParsePageSets(parse_options.page_sets) 150dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 151dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._record = parse_options.record 152dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._skia_tools = parse_options.skia_tools 153dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._non_interactive = parse_options.non_interactive 154b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self._upload = parse_options.upload 155aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry self._skp_prefix = parse_options.skp_prefix 156b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen data_store_location = parse_options.data_store 157b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen if data_store_location.startswith(gs_utils.GS_PREFIX): 158b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self.gs = GoogleStorageDataStore(data_store_location) 159b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen else: 160b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self.gs = LocalFileSystemDataStore(data_store_location) 161dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._alternate_upload_dir = parse_options.alternate_upload_dir 162dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._telemetry_binaries_dir = os.path.join(parse_options.chrome_src_path, 163dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'tools', 'perf') 164dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 165dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._local_skp_dir = os.path.join( 166dc89ca56e6f33790cda2cd93ad772ca61497346bborenet parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, SKPICTURES_DIR_NAME) 167dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._local_record_webpages_archive_dir = os.path.join( 168dc89ca56e6f33790cda2cd93ad772ca61497346bborenet parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive') 169dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 170dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # List of SKP files generated by this script. 171dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._skp_files = [] 172dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 173dc89ca56e6f33790cda2cd93ad772ca61497346bborenet def _ParsePageSets(self, page_sets): 174dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if not page_sets: 175dc89ca56e6f33790cda2cd93ad772ca61497346bborenet raise ValueError('Must specify at least one page_set!') 176dc89ca56e6f33790cda2cd93ad772ca61497346bborenet elif self._all_page_sets_specified: 177dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Get everything from the page_sets directory. 178dc89ca56e6f33790cda2cd93ad772ca61497346bborenet page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 179dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'page_sets') 180dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ps = [os.path.join(page_sets_dir, page_set) 181dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for page_set in os.listdir(page_sets_dir) 182dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and 183dc89ca56e6f33790cda2cd93ad772ca61497346bborenet page_set.endswith('.py')] 18449d093cef9d571877a07173e879a765ef83a90b9rmistry chromium_ps = [ 18549d093cef9d571877a07173e879a765ef83a90b9rmistry os.path.join(self._chrome_page_sets_path, cr_page_set) 18649d093cef9d571877a07173e879a765ef83a90b9rmistry for cr_page_set in CHROMIUM_PAGE_SETS_TO_PREFIX] 18749d093cef9d571877a07173e879a765ef83a90b9rmistry ps.extend(chromium_ps) 188dc89ca56e6f33790cda2cd93ad772ca61497346bborenet elif '*' in page_sets: 189dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Explode and return the glob. 190dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ps = glob.glob(page_sets) 191dc89ca56e6f33790cda2cd93ad772ca61497346bborenet else: 192dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ps = page_sets.split(',') 193dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ps.sort() 194dc89ca56e6f33790cda2cd93ad772ca61497346bborenet return ps 195dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 19649d093cef9d571877a07173e879a765ef83a90b9rmistry def _IsChromiumPageSet(self, page_set): 19749d093cef9d571877a07173e879a765ef83a90b9rmistry """Returns true if the specified page set is a Chromium page set.""" 19849d093cef9d571877a07173e879a765ef83a90b9rmistry return page_set.startswith(self._chrome_page_sets_path) 19949d093cef9d571877a07173e879a765ef83a90b9rmistry 200dc89ca56e6f33790cda2cd93ad772ca61497346bborenet def Run(self): 201dc89ca56e6f33790cda2cd93ad772ca61497346bborenet """Run the SkPicturePlayback BuildStep.""" 202dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 203f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry # Download the credentials file if it was not previously downloaded. 204b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen if not os.path.isfile(CREDENTIALS_FILE_PATH): 205b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen # Download the credentials.json file from Google Storage. 206b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH) 207b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 208b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen if not os.path.isfile(CREDENTIALS_FILE_PATH): 209b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen print """\n\nCould not locate credentials file in the storage. 210b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen Please create a %s file that contains: 211f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry { 212f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry "google": { 213f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry "username": "google_testing_account_username", 214f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry "password": "google_testing_account_password" 215f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry }, 216f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry "facebook": { 217f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry "username": "facebook_testing_account_username", 218f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry "password": "facebook_testing_account_password" 219f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry } 220f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry }\n\n""" % CREDENTIALS_FILE_PATH 221f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry raw_input("Please press a key when you are ready to proceed...") 222f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry 223dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Delete any left over data files in the data directory. 224dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for archive_file in glob.glob( 225dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')): 226dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.remove(archive_file) 227dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 228dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Delete the local root directory if it already exists. 229dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR): 230dc89ca56e6f33790cda2cd93ad772ca61497346bborenet shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR) 231dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 232dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Create the required local storage directories. 233dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._CreateLocalStorageDirs() 234dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 235dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Start the timer. 236dc89ca56e6f33790cda2cd93ad772ca61497346bborenet start_time = time.time() 237dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 238dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Loop through all page_sets. 239dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for page_set in self._page_sets: 240dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 2417620bf06581aa36d8c478624eff240ff150f3bb7rmistry page_set_basename = os.path.basename(page_set).split('.')[0] 2427620bf06581aa36d8c478624eff240ff150f3bb7rmistry page_set_json_name = page_set_basename + '.json' 243dc89ca56e6f33790cda2cd93ad772ca61497346bborenet wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' 2447620bf06581aa36d8c478624eff240ff150f3bb7rmistry page_set_dir = os.path.dirname(page_set) 245dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 24649d093cef9d571877a07173e879a765ef83a90b9rmistry if self._IsChromiumPageSet(page_set): 24749d093cef9d571877a07173e879a765ef83a90b9rmistry print 'Using Chromium\'s captured archives for Chromium\'s page sets.' 24849d093cef9d571877a07173e879a765ef83a90b9rmistry elif self._record: 249dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Create an archive of the specified webpages if '--record=True' is 250dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # specified. 251dc89ca56e6f33790cda2cd93ad772ca61497346bborenet record_wpr_cmd = ( 2527620bf06581aa36d8c478624eff240ff150f3bb7rmistry 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 253dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'DISPLAY=%s' % X11_DISPLAY, 254dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.path.join(self._telemetry_binaries_dir, 'record_wpr'), 255aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry '--extra-browser-args="%s"' % self._browser_args, 256dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '--browser=exact', 257dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '--browser-executable=%s' % self._browser_executable, 2587620bf06581aa36d8c478624eff240ff150f3bb7rmistry '%s_page_set' % page_set_basename, 2597620bf06581aa36d8c478624eff240ff150f3bb7rmistry '--page-set-base-dir=%s' % page_set_dir 260dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ) 261dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for _ in range(RETRY_RECORD_WPR_COUNT): 2620ec28af1a7259ef0deec44bd3cba74aafc3c2ad1rmistry try: 2630ec28af1a7259ef0deec44bd3cba74aafc3c2ad1rmistry shell_utils.run(' '.join(record_wpr_cmd), shell=True) 264f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen 265f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen # Move over the created archive into the local webpages archive 266f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen # directory. 267f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen shutil.move( 268f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file), 269f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen self._local_record_webpages_archive_dir) 270f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen shutil.move( 271f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 272f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen page_set_json_name), 273f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen self._local_record_webpages_archive_dir) 274f9310fe2be460e2cefd844cd0f53f3e2b9f3488bkkinnunen 275dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Break out of the retry loop since there were no errors. 276dc89ca56e6f33790cda2cd93ad772ca61497346bborenet break 2770ec28af1a7259ef0deec44bd3cba74aafc3c2ad1rmistry except Exception: 2780ec28af1a7259ef0deec44bd3cba74aafc3c2ad1rmistry # There was a failure continue with the loop. 2790ec28af1a7259ef0deec44bd3cba74aafc3c2ad1rmistry traceback.print_exc() 280dc89ca56e6f33790cda2cd93ad772ca61497346bborenet else: 281dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # If we get here then record_wpr did not succeed and thus did not 282dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # break out of the loop. 283dc89ca56e6f33790cda2cd93ad772ca61497346bborenet raise Exception('record_wpr failed for page_set: %s' % page_set) 284dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 285dc89ca56e6f33790cda2cd93ad772ca61497346bborenet else: 286b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen # Get the webpages archive so that it can be replayed. 287b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self._DownloadWebpagesArchive(wpr_data_file, page_set_json_name) 288dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 289783991585be67da3286680d850a99c5410885965borenet run_benchmark_cmd = ( 290f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 291dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'DISPLAY=%s' % X11_DISPLAY, 292dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'timeout', '300', 293783991585be67da3286680d850a99c5410885965borenet os.path.join(self._telemetry_binaries_dir, 'run_benchmark'), 294aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry '--extra-browser-args="%s"' % self._browser_args, 295dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '--browser=exact', 296dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '--browser-executable=%s' % self._browser_executable, 297dc89ca56e6f33790cda2cd93ad772ca61497346bborenet SKP_BENCHMARK, 2987620bf06581aa36d8c478624eff240ff150f3bb7rmistry '--page-set-name=%s' % page_set_basename, 299f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry '--page-set-base-dir=%s' % page_set_dir, 300f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry '--skp-outdir=%s' % TMP_SKP_DIR, 301f802f326f46aebd9fbad5ffe63450f0dcad948bbrmistry '--also-run-disabled-tests' 302dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ) 303dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 304dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for _ in range(RETRY_RUN_MEASUREMENT_COUNT): 305dc89ca56e6f33790cda2cd93ad772ca61497346bborenet try: 306dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print '\n\n=======Capturing SKP of %s=======\n\n' % page_set 307783991585be67da3286680d850a99c5410885965borenet shell_utils.run(' '.join(run_benchmark_cmd), shell=True) 308dc89ca56e6f33790cda2cd93ad772ca61497346bborenet except shell_utils.CommandFailedException: 309dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # skpicture_printer sometimes fails with AssertionError but the 310dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # captured SKP is still valid. This is a known issue. 311dc89ca56e6f33790cda2cd93ad772ca61497346bborenet pass 312dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 313dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Rename generated SKP files into more descriptive names. 314dc89ca56e6f33790cda2cd93ad772ca61497346bborenet try: 315dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._RenameSkpFiles(page_set) 316dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Break out of the retry loop since there were no errors. 317dc89ca56e6f33790cda2cd93ad772ca61497346bborenet break 318dc89ca56e6f33790cda2cd93ad772ca61497346bborenet except Exception: 319dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # There was a failure continue with the loop. 320dc89ca56e6f33790cda2cd93ad772ca61497346bborenet traceback.print_exc() 321dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print '\n\n=======Retrying %s=======\n\n' % page_set 322dc89ca56e6f33790cda2cd93ad772ca61497346bborenet time.sleep(10) 323dc89ca56e6f33790cda2cd93ad772ca61497346bborenet else: 324783991585be67da3286680d850a99c5410885965borenet # If we get here then run_benchmark did not succeed and thus did not 325dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # break out of the loop. 326783991585be67da3286680d850a99c5410885965borenet raise Exception('run_benchmark failed for page_set: %s' % page_set) 327dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 328dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % ( 329dc89ca56e6f33790cda2cd93ad772ca61497346bborenet time.time() - start_time) 330dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 331dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if self._skia_tools: 332dc89ca56e6f33790cda2cd93ad772ca61497346bborenet render_pictures_cmd = [ 333dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.path.join(self._skia_tools, 'render_pictures'), 334dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '-r', self._local_skp_dir 335dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ] 336dc89ca56e6f33790cda2cd93ad772ca61497346bborenet render_pdfs_cmd = [ 337dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.path.join(self._skia_tools, 'render_pdfs'), 3383a6aa8697831e1957f7a47160c984f28f6e83182kkinnunen '-r', self._local_skp_dir 339dc89ca56e6f33790cda2cd93ad772ca61497346bborenet ] 340dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 341dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for tools_cmd in (render_pictures_cmd, render_pdfs_cmd): 342dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print '\n\n=======Running %s=======' % ' '.join(tools_cmd) 343dc89ca56e6f33790cda2cd93ad772ca61497346bborenet proc = subprocess.Popen(tools_cmd) 3440ec28af1a7259ef0deec44bd3cba74aafc3c2ad1rmistry (code, _) = shell_utils.log_process_after_completion(proc, echo=False) 345dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if code != 0: 346dc89ca56e6f33790cda2cd93ad772ca61497346bborenet raise Exception('%s failed!' % ' '.join(tools_cmd)) 347dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 348dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if not self._non_interactive: 349dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print '\n\n=======Running debugger=======' 350dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'), 351960fb50a1a7ac76fd51e22983812f26bfffa6d1ekkinnunen self._local_skp_dir)) 352dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 353dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print '\n\n' 354dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 355b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen if self._upload: 356b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen print '\n\n=======Uploading to %s=======\n\n' % self.gs.target_type() 357dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # Copy the directory structure in the root directory into Google Storage. 358dc89ca56e6f33790cda2cd93ad772ca61497346bborenet dest_dir_name = ROOT_PLAYBACK_DIR_NAME 359dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if self._alternate_upload_dir: 360dc89ca56e6f33790cda2cd93ad772ca61497346bborenet dest_dir_name = self._alternate_upload_dir 361dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 362b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self.gs.upload_dir_contents( 363009b069f5253f7e903b2f3913d56ded990969abdrmistry LOCAL_PLAYBACK_ROOT_DIR, dest_dir=dest_dir_name, 364dc89ca56e6f33790cda2cd93ad772ca61497346bborenet upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED, 365dc89ca56e6f33790cda2cd93ad772ca61497346bborenet predefined_acl=GS_PREDEFINED_ACL, 366dc89ca56e6f33790cda2cd93ad772ca61497346bborenet fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST) 367dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 368dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( 369b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen posixpath.join(self.gs.target_name(), dest_dir_name, 370b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen SKPICTURES_DIR_NAME)) 371dc89ca56e6f33790cda2cd93ad772ca61497346bborenet else: 372b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen print '\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type() 373dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print 'Generated resources are available in %s\n\n' % ( 374dc89ca56e6f33790cda2cd93ad772ca61497346bborenet LOCAL_PLAYBACK_ROOT_DIR) 375dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 376dc89ca56e6f33790cda2cd93ad772ca61497346bborenet return 0 377dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 37849d093cef9d571877a07173e879a765ef83a90b9rmistry def _GetSkiaSkpFileName(self, page_set): 37949d093cef9d571877a07173e879a765ef83a90b9rmistry """Returns the SKP file name for Skia page sets.""" 38049d093cef9d571877a07173e879a765ef83a90b9rmistry # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py 38149d093cef9d571877a07173e879a765ef83a90b9rmistry ps_filename = os.path.basename(page_set) 38249d093cef9d571877a07173e879a765ef83a90b9rmistry # skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop 38349d093cef9d571877a07173e879a765ef83a90b9rmistry ps_basename, _ = os.path.splitext(ps_filename) 38449d093cef9d571877a07173e879a765ef83a90b9rmistry # skia_yahooanswers_desktop -> skia, yahooanswers, desktop 38549d093cef9d571877a07173e879a765ef83a90b9rmistry _, page_name, device = ps_basename.split('_') 38649d093cef9d571877a07173e879a765ef83a90b9rmistry basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name) 38749d093cef9d571877a07173e879a765ef83a90b9rmistry return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp' 38849d093cef9d571877a07173e879a765ef83a90b9rmistry 38949d093cef9d571877a07173e879a765ef83a90b9rmistry def _GetChromiumSkpFileName(self, page_set, site): 39049d093cef9d571877a07173e879a765ef83a90b9rmistry """Returns the SKP file name for Chromium page sets.""" 39149d093cef9d571877a07173e879a765ef83a90b9rmistry # /path/to/http___mobile_news_sandbox_pt0 -> http___mobile_news_sandbox_pt0 39249d093cef9d571877a07173e879a765ef83a90b9rmistry _, webpage = os.path.split(site) 39349d093cef9d571877a07173e879a765ef83a90b9rmistry # http___mobile_news_sandbox_pt0 -> mobile_news_sandbox_pt0 39480bd3ae9d76bcb9bdbf28c978c64134cdd503385rmistry for prefix in ('http___', 'https___', 'www_'): 3952a3c8493af6efe2256025b6cfbec882845925bd0rmistry if webpage.startswith(prefix): 3962a3c8493af6efe2256025b6cfbec882845925bd0rmistry webpage = webpage[len(prefix):] 39749d093cef9d571877a07173e879a765ef83a90b9rmistry # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py 39849d093cef9d571877a07173e879a765ef83a90b9rmistry ps_filename = os.path.basename(page_set) 39949d093cef9d571877a07173e879a765ef83a90b9rmistry # http___mobile_news_sandbox -> pagesetprefix_http___mobile_news_sandbox 40049d093cef9d571877a07173e879a765ef83a90b9rmistry basename = '%s_%s' % (CHROMIUM_PAGE_SETS_TO_PREFIX[ps_filename], webpage) 40149d093cef9d571877a07173e879a765ef83a90b9rmistry return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp' 40249d093cef9d571877a07173e879a765ef83a90b9rmistry 403dc89ca56e6f33790cda2cd93ad772ca61497346bborenet def _RenameSkpFiles(self, page_set): 404dc89ca56e6f33790cda2cd93ad772ca61497346bborenet """Rename generated SKP files into more descriptive names. 405dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 406dc89ca56e6f33790cda2cd93ad772ca61497346bborenet Look into the subdirectory of TMP_SKP_DIR and find the most interesting 407dc89ca56e6f33790cda2cd93ad772ca61497346bborenet .skp in there to be this page_set's representative .skp. 408dc89ca56e6f33790cda2cd93ad772ca61497346bborenet """ 409dc89ca56e6f33790cda2cd93ad772ca61497346bborenet subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*')) 410dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for site in subdirs: 41149d093cef9d571877a07173e879a765ef83a90b9rmistry if self._IsChromiumPageSet(page_set): 41249d093cef9d571877a07173e879a765ef83a90b9rmistry filename = self._GetChromiumSkpFileName(page_set, site) 41349d093cef9d571877a07173e879a765ef83a90b9rmistry else: 41449d093cef9d571877a07173e879a765ef83a90b9rmistry filename = self._GetSkiaSkpFileName(page_set) 41580bd3ae9d76bcb9bdbf28c978c64134cdd503385rmistry filename = filename.lower() 41649d093cef9d571877a07173e879a765ef83a90b9rmistry 417aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry if self._skp_prefix: 418aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry filename = '%s%s' % (self._skp_prefix, filename) 419aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry 420dc89ca56e6f33790cda2cd93ad772ca61497346bborenet # We choose the largest .skp as the most likely to be interesting. 421dc89ca56e6f33790cda2cd93ad772ca61497346bborenet largest_skp = max(glob.glob(os.path.join(site, '*.skp')), 422dc89ca56e6f33790cda2cd93ad772ca61497346bborenet key=lambda path: os.stat(path).st_size) 423dc89ca56e6f33790cda2cd93ad772ca61497346bborenet dest = os.path.join(self._local_skp_dir, filename) 424dc89ca56e6f33790cda2cd93ad772ca61497346bborenet print 'Moving', largest_skp, 'to', dest 425dc89ca56e6f33790cda2cd93ad772ca61497346bborenet shutil.move(largest_skp, dest) 426dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._skp_files.append(filename) 427dc89ca56e6f33790cda2cd93ad772ca61497346bborenet shutil.rmtree(site) 428dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 429dc89ca56e6f33790cda2cd93ad772ca61497346bborenet def _CreateLocalStorageDirs(self): 430dc89ca56e6f33790cda2cd93ad772ca61497346bborenet """Creates required local storage directories for this script.""" 431dc89ca56e6f33790cda2cd93ad772ca61497346bborenet for d in (self._local_record_webpages_archive_dir, 432dc89ca56e6f33790cda2cd93ad772ca61497346bborenet self._local_skp_dir): 433dc89ca56e6f33790cda2cd93ad772ca61497346bborenet if os.path.exists(d): 434dc89ca56e6f33790cda2cd93ad772ca61497346bborenet shutil.rmtree(d) 435dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.makedirs(d) 436dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 4377620bf06581aa36d8c478624eff240ff150f3bb7rmistry def _DownloadWebpagesArchive(self, wpr_data_file, page_set_json_name): 438dc89ca56e6f33790cda2cd93ad772ca61497346bborenet """Downloads the webpages archive and its required page set from GS.""" 439dc89ca56e6f33790cda2cd93ad772ca61497346bborenet wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', 440dc89ca56e6f33790cda2cd93ad772ca61497346bborenet wpr_data_file) 441dc89ca56e6f33790cda2cd93ad772ca61497346bborenet page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 442dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'webpages_archive', 4437620bf06581aa36d8c478624eff240ff150f3bb7rmistry page_set_json_name) 444b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen gs = self.gs 445b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen if (gs.does_storage_object_exist(wpr_source) and 446b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen gs.does_storage_object_exist(page_set_source)): 447b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen gs.download_file(wpr_source, 448dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 449dc89ca56e6f33790cda2cd93ad772ca61497346bborenet wpr_data_file)) 450b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen gs.download_file(page_set_source, 451dc89ca56e6f33790cda2cd93ad772ca61497346bborenet os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 4527620bf06581aa36d8c478624eff240ff150f3bb7rmistry page_set_json_name)) 453dc89ca56e6f33790cda2cd93ad772ca61497346bborenet else: 454b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen raise Exception('%s and %s do not exist in %s!' % (gs.target_type(), 455b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen wpr_source, page_set_source)) 456b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 457b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenclass DataStore: 458b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen """An abstract base class for uploading recordings to a data storage. 459b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen The interface emulates the google storage api.""" 460b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def target_name(self): 461b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen raise NotImplementedError() 462b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def target_type(self): 463b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen raise NotImplementedError() 464b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def does_storage_object_exist(self, *args): 465b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen raise NotImplementedError() 466b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def download_file(self, *args): 467b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen raise NotImplementedError() 468b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def upload_dir_contents(self, source_dir, **kwargs): 469b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen raise NotImplementedError() 470b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 471b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenclass GoogleStorageDataStore(DataStore): 472b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def __init__(self, data_store_url): 473b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self._data_store_url = data_store_url 47449d093cef9d571877a07173e879a765ef83a90b9rmistry self._bucket = remove_prefix(self._data_store_url.lstrip(), 475b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen gs_utils.GS_PREFIX) 476b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self.gs = gs_utils.GSUtils() 477b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def target_name(self): 478b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen return self._data_store_url 479b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def target_type(self): 480b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen return 'Google Storage' 481b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def does_storage_object_exist(self, *args): 482b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen return self.gs.does_storage_object_exist(self._bucket, *args) 483b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def download_file(self, *args): 484b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self.gs.download_file(self._bucket, *args) 485b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def upload_dir_contents(self, source_dir, **kwargs): 486b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self.gs.upload_dir_contents(source_dir, self._bucket, **kwargs) 487b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 488b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunenclass LocalFileSystemDataStore(DataStore): 489b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def __init__(self, data_store_location): 490b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen self._base_dir = data_store_location 491b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def target_name(self): 492b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen return self._base_dir 493b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def target_type(self): 494b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen return self._base_dir 495b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def does_storage_object_exist(self, name, *args): 496b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen return os.path.isfile(os.path.join(self._base_dir, name)) 497b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def download_file(self, name, local_path, *args): 498b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen shutil.copyfile(os.path.join(self._base_dir, name), local_path) 499b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def upload_dir_contents(self, source_dir, dest_dir, **kwargs): 500b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen def copytree(source_dir, dest_dir): 501b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen if not os.path.exists(dest_dir): 502b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen os.makedirs(dest_dir) 503b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen for item in os.listdir(source_dir): 504b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen source = os.path.join(source_dir, item) 505b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen dest = os.path.join(dest_dir, item) 506b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen if os.path.isdir(source): 507b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen copytree(source, dest) 508b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen else: 509b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen shutil.copy2(source, dest) 510b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen copytree(source_dir, os.path.join(self._base_dir, dest_dir)) 511dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 512dc89ca56e6f33790cda2cd93ad772ca61497346bborenetif '__main__' == __name__: 513dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser = optparse.OptionParser() 514dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 515dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--page_sets', 516dc89ca56e6f33790cda2cd93ad772ca61497346bborenet help='Specifies the page sets to use to archive. Supports globs.', 517dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default='all') 518dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 519dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--record', action='store_true', 520dc89ca56e6f33790cda2cd93ad772ca61497346bborenet help='Specifies whether a new website archive should be created.', 521dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=False) 522dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 523dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--skia_tools', 524dc89ca56e6f33790cda2cd93ad772ca61497346bborenet help=('Path to compiled Skia executable tools. ' 525dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'render_pictures/render_pdfs is run on the set ' 526dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'after all SKPs are captured. If the script is run without ' 527dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '--non-interactive then the debugger is also run at the end. Debug ' 528dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'builds are recommended because they seem to catch more failures ' 529dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 'than Release builds.'), 530dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=None) 531dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 532b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen '', '--upload', action='store_true', 533b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen help=('Uploads to Google Storage or copies to local filesystem storage ' 534b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen ' if this is True.'), 535dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=False) 536dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 537b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen '', '--data_store', 538b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen help=('The location of the file storage to use to download and upload ' 539b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 'files. Can be \'gs://<bucket>\' for Google Storage, or ' 540b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 'a directory for local filesystem storage'), 541b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen default='gs://chromium-skia-gm') 542b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen option_parser.add_option( 543dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--alternate_upload_dir', 544b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen help= ('Uploads to a different directory in Google Storage or local ' 545b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 'storage if this flag is specified'), 546dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=None) 547dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 548dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--output_dir', 549b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen help=('Temporary directory where SKPs and webpage archives will be ' 550b4ee7ea834db121bebcff5dc265fee776d18b773kkinnunen 'outputted to.'), 551dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=tempfile.gettempdir()) 552dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 553dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--browser_executable', 554dc89ca56e6f33790cda2cd93ad772ca61497346bborenet help='The exact browser executable to run.', 555dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=None) 556dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 557aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry '', '--browser_extra_args', 558aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry help='Additional arguments to pass to the browser.', 559aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry default=None) 560aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry option_parser.add_option( 561dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--chrome_src_path', 562dc89ca56e6f33790cda2cd93ad772ca61497346bborenet help='Path to the chromium src directory.', 563dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=None) 564dc89ca56e6f33790cda2cd93ad772ca61497346bborenet option_parser.add_option( 565dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '', '--non-interactive', action='store_true', 566dc89ca56e6f33790cda2cd93ad772ca61497346bborenet help='Runs the script without any prompts. If this flag is specified and ' 567dc89ca56e6f33790cda2cd93ad772ca61497346bborenet '--skia_tools is specified then the debugger is not run.', 568dc89ca56e6f33790cda2cd93ad772ca61497346bborenet default=False) 569aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry option_parser.add_option( 570aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry '', '--skp_prefix', 571aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry help='Prefix to add to the names of generated SKPs.', 572aa31ee78eecef34128aa1eaf0e1bb9398a7caec5rmistry default=None) 573dc89ca56e6f33790cda2cd93ad772ca61497346bborenet options, unused_args = option_parser.parse_args() 574dc89ca56e6f33790cda2cd93ad772ca61497346bborenet 575dc89ca56e6f33790cda2cd93ad772ca61497346bborenet playback = SkPicturePlayback(options) 576dc89ca56e6f33790cda2cd93ad772ca61497346bborenet sys.exit(playback.Run()) 577