120a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski#!/usr/bin/python 220a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski# 3cb2e2b789b9377262b08c185b8b96e7c758ef9e5Scott Zawalski# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 420a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski# Use of this source code is governed by a BSD-style license that can be 520a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski# found in the LICENSE file. 620a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 7ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette"""Script to archive old Autotest results to Google Storage. 820a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 9ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard BarnetteUses gsutil to archive files to the configured Google Storage bucket. 10ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard BarnetteUpon successful copy, the local results directory is deleted. 1120a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski""" 1220a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 13b41527d6f865688299dc5250f8d58baca9432777Allen Liimport abc 14bd9ded0df72fbdd62f9823b21a87400e8bee1d25Simran Basiimport datetime 15faf50dbf4ba1fa5ac0609e998ef6c6977337e0d0Dan Shiimport errno 164211124209fdb4469462b6e1b39433cc52b76d02Ningning Xiaimport glob 174211124209fdb4469462b6e1b39433cc52b76d02Ningning Xiaimport gzip 189523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basiimport logging 19a253228ee6a824110069fa68072a0e4d4fd7a0d5Simran Basiimport logging.handlers 2020a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalskiimport os 21affb922d30a3f5e528103c0314d5f0586b7a90dcDan Shiimport re 2220a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalskiimport shutil 23b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shiimport socket 24ca7726dfcd4b9b70a75147bad1d1c2ec66b79816Laurence Goodbyimport stat 2520a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalskiimport subprocess 2620a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalskiimport sys 27b41527d6f865688299dc5250f8d58baca9432777Allen Liimport tarfile 289523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basiimport tempfile 299523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basiimport time 30cb2e2b789b9377262b08c185b8b96e7c758ef9e5Scott Zawalski 317d9a14925e2bf5fbba92ffecffdfa6f1ac2100bbSimran Basifrom optparse import OptionParser 327d9a14925e2bf5fbba92ffecffdfa6f1ac2100bbSimran Basi 33981a9274613d85833d5ada1463d794bc9f7cc37fSimran Basiimport common 3496c3bdc2e5de17857fabf4c40a0f371b0d657258Dan Shifrom autotest_lib.client.common_lib import file_utils 35beb9e01b8aebdd33b7c866fbdfef64bc77f139e5Prathmesh Prabhufrom autotest_lib.client.common_lib import global_config 36dd129979e8227c960f4c7a3d2a1665007f8a424eSimran Basifrom autotest_lib.client.common_lib import utils 37beb9e01b8aebdd33b7c866fbdfef64bc77f139e5Prathmesh Prabhufrom autotest_lib.site_utils import job_directories 380f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tangfrom autotest_lib.site_utils import cloud_console_client 39beb9e01b8aebdd33b7c866fbdfef64bc77f139e5Prathmesh Prabhufrom autotest_lib.tko import models 40b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shifrom autotest_lib.utils import labellib 41b41527d6f865688299dc5250f8d58baca9432777Allen Lifrom autotest_lib.utils import gslib 42b41527d6f865688299dc5250f8d58baca9432777Allen Lifrom chromite.lib import timeout_util 43beb9e01b8aebdd33b7c866fbdfef64bc77f139e5Prathmesh Prabhu 445e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shi# Autotest requires the psutil module from site-packages, so it must be imported 4573cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel# after "import common". 46627a75ebc4cc2f755ccad17d1c9f92c91cc2e776Simran Basitry: 47627a75ebc4cc2f755ccad17d1c9f92c91cc2e776Simran Basi # Does not exist, nor is needed, on moblab. 48627a75ebc4cc2f755ccad17d1c9f92c91cc2e776Simran Basi import psutil 49627a75ebc4cc2f755ccad17d1c9f92c91cc2e776Simran Basiexcept ImportError: 50627a75ebc4cc2f755ccad17d1c9f92c91cc2e776Simran Basi psutil = None 5120a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 525e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shifrom chromite.lib import parallel 535e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shitry: 545e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shi from chromite.lib import metrics 555e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shi from chromite.lib import ts_mon_config 565e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shiexcept ImportError: 57592bcce305c9a83d8b2635b2e1ddcd5cea18b5a3Allen Li from autotest_lib import site_utils 585e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shi metrics = site_utils.metrics_mock 595e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shi ts_mon_config = site_utils.metrics_mock 605e2efb71ffebead22aa4f0744ad843ee79814b43Dan Shi 6120a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 62f3e305f2c1342d8dd271256c2594ae54072c514bSimran BasiGS_OFFLOADING_ENABLED = global_config.global_config.get_config_value( 632c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'CROS', 'gs_offloading_enabled', type=bool, default=True) 64f3e305f2c1342d8dd271256c2594ae54072c514bSimran Basi 6520a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski# Nice setting for process, the higher the number the lower the priority. 6620a9b58ae53932cad6a536e23bb020bfb6e84a49Scott ZawalskiNICENESS = 10 6720a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 68ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette# Maximum number of seconds to allow for offloading a single 69ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette# directory. 707e0f859d06bad8acf56b539f48e60bcdd9bfdd92J. Richard BarnetteOFFLOAD_TIMEOUT_SECS = 60 * 60 719523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basi 72392d4a58952b5006b66bc45e575ca32a78052254Simran Basi# Sleep time per loop. 73392d4a58952b5006b66bc45e575ca32a78052254Simran BasiSLEEP_TIME_SECS = 5 74392d4a58952b5006b66bc45e575ca32a78052254Simran Basi 75ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette# Minimum number of seconds between e-mail reports. 76ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard BarnetteREPORT_INTERVAL_SECS = 60 * 60 77ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette 7820a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski# Location of Autotest results on disk. 7920a9b58ae53932cad6a536e23bb020bfb6e84a49Scott ZawalskiRESULTS_DIR = '/usr/local/autotest/results' 8016f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh PrabhuFAILED_OFFLOADS_FILE = os.path.join(RESULTS_DIR, 'FAILED_OFFLOADS') 8120a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 8231d561d34612b994c654fcd9ecfb6a6aaacf8162Simran Basi# Hosts sub-directory that contains cleanup, verify and repair jobs. 8331d561d34612b994c654fcd9ecfb6a6aaacf8162Simran BasiHOSTS_SUB_DIR = 'hosts' 8431d561d34612b994c654fcd9ecfb6a6aaacf8162Simran Basi 8516f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh PrabhuFAILED_OFFLOADS_FILE_HEADER = ''' 8616f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh PrabhuThis is the list of gs_offloader failed jobs. 8716f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh PrabhuLast offloader attempt at %s failed to offload %d files. 8816f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh PrabhuCheck http://go/cros-triage-gsoffloader to triage the issue 8916f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 9016f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 9116f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh PrabhuFirst failure Count Directory name 9216f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu=================== ====== ============================== 9316f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu''' 9416f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu# --+----1----+---- ----+ ----+----1----+----2----+----3 9516f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 9680dfb1e9366b74f6c5a928e33793012e5a8ebc5aPrathmesh PrabhuFAILED_OFFLOADS_LINE_FORMAT = '%19s %5d %-1s\n' 9780dfb1e9366b74f6c5a928e33793012e5a8ebc5aPrathmesh PrabhuFAILED_OFFLOADS_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' 989523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basi 9924f22c2fda0fac684286b168f642c7326a20fac7Jakob JuelichUSE_RSYNC_ENABLED = global_config.global_config.get_config_value( 1002c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'CROS', 'gs_offloader_use_rsync', type=bool, default=False) 10124f22c2fda0fac684286b168f642c7326a20fac7Jakob Juelich 1021b4c7c396d5043b24d6bf65c069a85371bb00e5cDan ShiLIMIT_FILE_COUNT = global_config.global_config.get_config_value( 1032c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'CROS', 'gs_offloader_limit_file_count', type=bool, default=False) 1042c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 1050df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang# Use multiprocessing for gsutil uploading. 1060df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael TangGS_OFFLOADER_MULTIPROCESSING = global_config.global_config.get_config_value( 1070df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang 'CROS', 'gs_offloader_multiprocessing', type=bool, default=False) 1080df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang 1094211124209fdb4469462b6e1b39433cc52b76d02Ningning XiaD = '[0-9][0-9]' 11097d188c2d78728c69a2d71f08de210fe683d3d1eMichael TangTIMESTAMP_PATTERN = '%s%s.%s.%s_%s.%s.%s' % (D, D, D, D, D, D, D) 1114211124209fdb4469462b6e1b39433cc52b76d02Ningning XiaCTS_RESULT_PATTERN = 'testResult.xml' 11273cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. FriedelCTS_V2_RESULT_PATTERN = 'test_result.xml' 1134211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia# Google Storage bucket URI to store results in. 1144211124209fdb4469462b6e1b39433cc52b76d02Ningning XiaDEFAULT_CTS_RESULTS_GSURI = global_config.global_config.get_config_value( 1154211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia 'CROS', 'cts_results_server', default='') 116205a1d4ee14e4f2d0dccc40e29522525f201da5aNingning XiaDEFAULT_CTS_APFE_GSURI = global_config.global_config.get_config_value( 117205a1d4ee14e4f2d0dccc40e29522525f201da5aNingning Xia 'CROS', 'cts_apfe_server', default='') 1181b4c7c396d5043b24d6bf65c069a85371bb00e5cDan Shi 119b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi# metadata type 120b2751fcd14518af3e6c5aba50c345abeded576b9Dan ShiGS_OFFLOADER_SUCCESS_TYPE = 'gs_offloader_success' 121b2751fcd14518af3e6c5aba50c345abeded576b9Dan ShiGS_OFFLOADER_FAILURE_TYPE = 'gs_offloader_failure' 12297d188c2d78728c69a2d71f08de210fe683d3d1eMichael Tang 1239523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basi 124b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shidef _get_metrics_fields(dir_entry): 125b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi """Get metrics fields for the given test result directory, including board 126b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi and milestone. 127b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 128b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi @param dir_entry: Directory entry to offload. 129b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi @return A dictionary for the metrics data to be uploaded. 130b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi """ 131b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi fields = {'board': 'unknown', 132b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 'milestone': 'unknown'} 133b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi if dir_entry: 134b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi # There could be multiple hosts in the job directory, use the first one 135b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi # available. 136b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi for host in glob.glob(os.path.join(dir_entry, '*')): 1372310901eb01272aa0301d57b4d99771bdce8eb5eDan Shi try: 1382310901eb01272aa0301d57b4d99771bdce8eb5eDan Shi keyval = models.test.parse_job_keyval(host) 1392310901eb01272aa0301d57b4d99771bdce8eb5eDan Shi except ValueError: 1402310901eb01272aa0301d57b4d99771bdce8eb5eDan Shi continue 141b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi build = keyval.get('build') 142b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi if build: 14302dd066dc11876736eaf1841e6893a805febd899Dan Shi try: 14402dd066dc11876736eaf1841e6893a805febd899Dan Shi cros_version = labellib.parse_cros_version(build) 14502dd066dc11876736eaf1841e6893a805febd899Dan Shi fields['board'] = cros_version.board 14602dd066dc11876736eaf1841e6893a805febd899Dan Shi fields['milestone'] = cros_version.milestone 14702dd066dc11876736eaf1841e6893a805febd899Dan Shi break 14802dd066dc11876736eaf1841e6893a805febd899Dan Shi except ValueError: 14902dd066dc11876736eaf1841e6893a805febd899Dan Shi # Ignore version parsing error so it won't crash 15002dd066dc11876736eaf1841e6893a805febd899Dan Shi # gs_offloader. 15102dd066dc11876736eaf1841e6893a805febd899Dan Shi pass 152b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 153b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi return fields; 154b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 155b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 156b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shidef _get_es_metadata(dir_entry): 157b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi """Get ES metadata for the given test result directory. 158b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 159b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi @param dir_entry: Directory entry to offload. 160b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi @return A dictionary for the metadata to be uploaded. 161b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi """ 162b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi fields = _get_metrics_fields(dir_entry) 163b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi fields['hostname'] = socket.gethostname() 164b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi # Include more data about the test job in metadata. 165b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi if dir_entry: 166b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi fields['dir_entry'] = dir_entry 167b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi fields['job_id'] = job_directories.get_job_id_or_task_id(dir_entry) 168b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 169b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi return fields 170b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 171b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 172b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _get_cmd_list(multiprocessing, dir_entry, gs_path): 1732c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Return the command to offload a specified directory. 1742c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 1752c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @param multiprocessing: True to turn on -m option for gsutil. 1762c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @param dir_entry: Directory entry/path that which we need a cmd_list 1772c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette to offload. 1782c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @param gs_path: Location in google storage where we will 1792c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette offload the directory. 1802c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 1812c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @return A command list to be executed by Popen. 1822c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 183365049f691c80802030f62cf2fce345bb670e00dDan Shi cmd = ['gsutil'] 1842c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if multiprocessing: 1852c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette cmd.append('-m') 1862c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if USE_RSYNC_ENABLED: 1872c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette cmd.append('rsync') 1882c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette target = os.path.join(gs_path, os.path.basename(dir_entry)) 1892c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette else: 1902c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette cmd.append('cp') 1912c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette target = gs_path 1922c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette cmd += ['-eR', dir_entry, target] 1932c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette return cmd 194bd9ded0df72fbdd62f9823b21a87400e8bee1d25Simran Basi 19524f22c2fda0fac684286b168f642c7326a20fac7Jakob Juelich 196b41527d6f865688299dc5250f8d58baca9432777Allen Lidef sanitize_dir(dirpath): 197b41527d6f865688299dc5250f8d58baca9432777Allen Li """Sanitize directory for gs upload. 198b41527d6f865688299dc5250f8d58baca9432777Allen Li 199b41527d6f865688299dc5250f8d58baca9432777Allen Li Symlinks and FIFOS are converted to regular files to fix bugs. 200b41527d6f865688299dc5250f8d58baca9432777Allen Li 201b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dirpath: Directory entry to be sanitized. 202b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 203b41527d6f865688299dc5250f8d58baca9432777Allen Li if not os.path.exists(dirpath): 204b41527d6f865688299dc5250f8d58baca9432777Allen Li return 205b41527d6f865688299dc5250f8d58baca9432777Allen Li _escape_rename(dirpath) 206b41527d6f865688299dc5250f8d58baca9432777Allen Li _escape_rename_dir_contents(dirpath) 207b41527d6f865688299dc5250f8d58baca9432777Allen Li _sanitize_fifos(dirpath) 208b41527d6f865688299dc5250f8d58baca9432777Allen Li _sanitize_symlinks(dirpath) 209affb922d30a3f5e528103c0314d5f0586b7a90dcDan Shi 210affb922d30a3f5e528103c0314d5f0586b7a90dcDan Shi 211b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _escape_rename_dir_contents(dirpath): 212b41527d6f865688299dc5250f8d58baca9432777Allen Li """Recursively rename directory to escape filenames for gs upload. 213b41527d6f865688299dc5250f8d58baca9432777Allen Li 214b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dirpath: Directory path string. 2152c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 216b41527d6f865688299dc5250f8d58baca9432777Allen Li for filename in os.listdir(dirpath): 217b41527d6f865688299dc5250f8d58baca9432777Allen Li path = os.path.join(dirpath, filename) 218b41527d6f865688299dc5250f8d58baca9432777Allen Li _escape_rename(path) 219b41527d6f865688299dc5250f8d58baca9432777Allen Li for filename in os.listdir(dirpath): 220b41527d6f865688299dc5250f8d58baca9432777Allen Li path = os.path.join(dirpath, filename) 221b41527d6f865688299dc5250f8d58baca9432777Allen Li if os.path.isdir(path): 222b41527d6f865688299dc5250f8d58baca9432777Allen Li _escape_rename_dir_contents(path) 223b41527d6f865688299dc5250f8d58baca9432777Allen Li 224affb922d30a3f5e528103c0314d5f0586b7a90dcDan Shi 225b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _escape_rename(path): 226b41527d6f865688299dc5250f8d58baca9432777Allen Li """Rename file to escape filenames for gs upload. 227b41527d6f865688299dc5250f8d58baca9432777Allen Li 228b41527d6f865688299dc5250f8d58baca9432777Allen Li @param path: File path string. 229b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 230b41527d6f865688299dc5250f8d58baca9432777Allen Li dirpath, filename = os.path.split(path) 231b41527d6f865688299dc5250f8d58baca9432777Allen Li sanitized_filename = gslib.escape(filename) 232b41527d6f865688299dc5250f8d58baca9432777Allen Li sanitized_path = os.path.join(dirpath, sanitized_filename) 233b41527d6f865688299dc5250f8d58baca9432777Allen Li os.rename(path, sanitized_path) 234affb922d30a3f5e528103c0314d5f0586b7a90dcDan Shi 2352c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 236b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _sanitize_fifos(dirpath): 237b41527d6f865688299dc5250f8d58baca9432777Allen Li """Convert fifos to regular files (fixes crbug.com/684122). 238ca7726dfcd4b9b70a75147bad1d1c2ec66b79816Laurence Goodby 239b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dirpath: Directory path string. 2402c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 241b41527d6f865688299dc5250f8d58baca9432777Allen Li for root, _, files in os.walk(dirpath): 242b41527d6f865688299dc5250f8d58baca9432777Allen Li for filename in files: 243b41527d6f865688299dc5250f8d58baca9432777Allen Li path = os.path.join(root, filename) 244b41527d6f865688299dc5250f8d58baca9432777Allen Li file_stat = os.lstat(path) 245ca7726dfcd4b9b70a75147bad1d1c2ec66b79816Laurence Goodby if stat.S_ISFIFO(file_stat.st_mode): 246b41527d6f865688299dc5250f8d58baca9432777Allen Li _replace_fifo_with_file(path) 247b41527d6f865688299dc5250f8d58baca9432777Allen Li 248b41527d6f865688299dc5250f8d58baca9432777Allen Li 249b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _replace_fifo_with_file(path): 250b41527d6f865688299dc5250f8d58baca9432777Allen Li """Replace a fifo with a normal file. 251b41527d6f865688299dc5250f8d58baca9432777Allen Li 252b41527d6f865688299dc5250f8d58baca9432777Allen Li @param path: Fifo path string. 253b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 254b41527d6f865688299dc5250f8d58baca9432777Allen Li logging.debug('Removing fifo %s', path) 255b41527d6f865688299dc5250f8d58baca9432777Allen Li os.remove(path) 256b41527d6f865688299dc5250f8d58baca9432777Allen Li logging.debug('Creating marker %s', path) 257b41527d6f865688299dc5250f8d58baca9432777Allen Li with open(path, 'w') as f: 258b41527d6f865688299dc5250f8d58baca9432777Allen Li f.write('<FIFO>') 259b41527d6f865688299dc5250f8d58baca9432777Allen Li 260b41527d6f865688299dc5250f8d58baca9432777Allen Li 261b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _sanitize_symlinks(dirpath): 262b41527d6f865688299dc5250f8d58baca9432777Allen Li """Convert Symlinks to regular files (fixes crbug.com/692788). 263b41527d6f865688299dc5250f8d58baca9432777Allen Li 264b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dirpath: Directory path string. 265b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 266b41527d6f865688299dc5250f8d58baca9432777Allen Li for root, _, files in os.walk(dirpath): 267b41527d6f865688299dc5250f8d58baca9432777Allen Li for filename in files: 268b41527d6f865688299dc5250f8d58baca9432777Allen Li path = os.path.join(root, filename) 269b41527d6f865688299dc5250f8d58baca9432777Allen Li file_stat = os.lstat(path) 270b41527d6f865688299dc5250f8d58baca9432777Allen Li if stat.S_ISLNK(file_stat.st_mode): 271b41527d6f865688299dc5250f8d58baca9432777Allen Li _replace_symlink_with_file(path) 272b41527d6f865688299dc5250f8d58baca9432777Allen Li 273b41527d6f865688299dc5250f8d58baca9432777Allen Li 274b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _replace_symlink_with_file(path): 275b41527d6f865688299dc5250f8d58baca9432777Allen Li """Replace a symlink with a normal file. 276b41527d6f865688299dc5250f8d58baca9432777Allen Li 277b41527d6f865688299dc5250f8d58baca9432777Allen Li @param path: Symlink path string. 278b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 279b41527d6f865688299dc5250f8d58baca9432777Allen Li target = os.readlink(path) 280b41527d6f865688299dc5250f8d58baca9432777Allen Li logging.debug('Removing symlink %s', path) 281b41527d6f865688299dc5250f8d58baca9432777Allen Li os.remove(path) 282b41527d6f865688299dc5250f8d58baca9432777Allen Li logging.debug('Creating marker %s', path) 283b41527d6f865688299dc5250f8d58baca9432777Allen Li with open(path, 'w') as f: 284b41527d6f865688299dc5250f8d58baca9432777Allen Li f.write('<symlink to %s>' % target) 285b41527d6f865688299dc5250f8d58baca9432777Allen Li 286b41527d6f865688299dc5250f8d58baca9432777Allen Li 287b41527d6f865688299dc5250f8d58baca9432777Allen Li# Maximum number of files in the folder. 288b41527d6f865688299dc5250f8d58baca9432777Allen Li_MAX_FILE_COUNT = 500 289b41527d6f865688299dc5250f8d58baca9432777Allen Li_FOLDERS_NEVER_ZIP = ['debug', 'ssp_logs', 'autoupdate_logs'] 2902c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 2912c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 2922c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnettedef _get_zippable_folders(dir_entry): 2932c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette folders_list = [] 2942c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette for folder in os.listdir(dir_entry): 2952c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette folder_path = os.path.join(dir_entry, folder) 2962c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if (not os.path.isfile(folder_path) and 297b41527d6f865688299dc5250f8d58baca9432777Allen Li not folder in _FOLDERS_NEVER_ZIP): 2982c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette folders_list.append(folder_path) 2992c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette return folders_list 300affb922d30a3f5e528103c0314d5f0586b7a90dcDan Shi 301affb922d30a3f5e528103c0314d5f0586b7a90dcDan Shi 3021b4c7c396d5043b24d6bf65c069a85371bb00e5cDan Shidef limit_file_count(dir_entry): 3032c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Limit the number of files in given directory. 3042c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 3052c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette The method checks the total number of files in the given directory. 306b41527d6f865688299dc5250f8d58baca9432777Allen Li If the number is greater than _MAX_FILE_COUNT, the method will 3072c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette compress each folder in the given directory, except folders in 308b41527d6f865688299dc5250f8d58baca9432777Allen Li _FOLDERS_NEVER_ZIP. 3091b4c7c396d5043b24d6bf65c069a85371bb00e5cDan Shi 3102c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @param dir_entry: Directory entry to be checked. 3112c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 3121b4c7c396d5043b24d6bf65c069a85371bb00e5cDan Shi try: 313b41527d6f865688299dc5250f8d58baca9432777Allen Li count = _count_files(dir_entry) 314b41527d6f865688299dc5250f8d58baca9432777Allen Li except ValueError: 3158f85cd2c7660da3f9498b9495e99f2eb39fe45f8Prathmesh Prabhu logging.warning('Fail to get the file count in folder %s.', dir_entry) 3162c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette return 317b41527d6f865688299dc5250f8d58baca9432777Allen Li if count < _MAX_FILE_COUNT: 3182c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette return 3192c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 3202c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # For test job, zip folders in a second level, e.g. 123-debug/host1. 3212c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # This is to allow autoserv debug folder still be accessible. 3222c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # For special task, it does not need to dig one level deeper. 3232c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette is_special_task = re.match(job_directories.SPECIAL_TASK_PATTERN, 3242c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette dir_entry) 3252c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 3262c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette folders = _get_zippable_folders(dir_entry) 3272c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if not is_special_task: 3282c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette subfolders = [] 3292c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette for folder in folders: 3302c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette subfolders.extend(_get_zippable_folders(folder)) 3312c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette folders = subfolders 3322c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 3332c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette for folder in folders: 334b41527d6f865688299dc5250f8d58baca9432777Allen Li _make_into_tarball(folder) 335b41527d6f865688299dc5250f8d58baca9432777Allen Li 336b41527d6f865688299dc5250f8d58baca9432777Allen Li 337b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _count_files(dirpath): 338b41527d6f865688299dc5250f8d58baca9432777Allen Li """Count the number of files in a directory recursively. 339b41527d6f865688299dc5250f8d58baca9432777Allen Li 340b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dirpath: Directory path string. 341b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 342b41527d6f865688299dc5250f8d58baca9432777Allen Li return sum(len(files) for _path, _dirs, files in os.walk(dirpath)) 343b41527d6f865688299dc5250f8d58baca9432777Allen Li 344b41527d6f865688299dc5250f8d58baca9432777Allen Li 345b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _make_into_tarball(dirpath): 346b41527d6f865688299dc5250f8d58baca9432777Allen Li """Make directory into tarball. 347b41527d6f865688299dc5250f8d58baca9432777Allen Li 348b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dirpath: Directory path string. 349b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 350b41527d6f865688299dc5250f8d58baca9432777Allen Li tarpath = '%s.tgz' % dirpath 351b41527d6f865688299dc5250f8d58baca9432777Allen Li with tarfile.open(tarpath, 'w:gz') as tar: 352b41527d6f865688299dc5250f8d58baca9432777Allen Li tar.add(dirpath, arcname=os.path.basename(dirpath)) 353b41527d6f865688299dc5250f8d58baca9432777Allen Li shutil.rmtree(dirpath) 3541b4c7c396d5043b24d6bf65c069a85371bb00e5cDan Shi 3551b4c7c396d5043b24d6bf65c069a85371bb00e5cDan Shi 356e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shidef correct_results_folder_permission(dir_entry): 357e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi """Make sure the results folder has the right permission settings. 358e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi 3592c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette For tests running with server-side packaging, the results folder has 3602c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette the owner of root. This must be changed to the user running the 3612c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette autoserv process, so parsing job can access the results folder. 362e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi 363e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi @param dir_entry: Path to the results folder. 364e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi """ 365e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi if not dir_entry: 3662c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette return 3676c4ed33bea57d412d2a9891910a30cd1f15138e3Prathmesh Prabhu 3686c4ed33bea57d412d2a9891910a30cd1f15138e3Prathmesh Prabhu logging.info('Trying to correct file permission of %s.', dir_entry) 369e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi try: 3702c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette subprocess.check_call( 3712c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette ['sudo', '-n', 'chown', '-R', str(os.getuid()), dir_entry]) 3722c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette subprocess.check_call( 3732c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette ['sudo', '-n', 'chgrp', '-R', str(os.getgid()), dir_entry]) 374e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi except subprocess.CalledProcessError as e: 3752c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.error('Failed to modify permission for %s: %s', 3762c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette dir_entry, e) 377e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi 378e4a4f9fd2ad9984882e50e199e24a73d871cb2a1Dan Shi 379b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _upload_cts_testresult(dir_entry, multiprocessing): 3802d88eec8f87da75d3ae5046ee34573562add3da8Ningning Xia """Upload test results to separate gs buckets. 3814211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia 382bfa63148ef62dc36e2d4b3456a4b64ae69b21172Ilja H. Friedel Upload testResult.xml.gz/test_result.xml.gz file to cts_results_bucket. 383205a1d4ee14e4f2d0dccc40e29522525f201da5aNingning Xia Upload timestamp.zip to cts_apfe_bucket. 3848db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia 3854211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia @param dir_entry: Path to the results folder. 3864211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia @param multiprocessing: True to turn on -m option for gsutil. 3874211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia """ 3882d981eee42e4f9fb4f6d726b97aa8122322543beNingning Xia for host in glob.glob(os.path.join(dir_entry, '*')): 3892d88eec8f87da75d3ae5046ee34573562add3da8Ningning Xia cts_path = os.path.join(host, 'cheets_CTS.*', 'results', '*', 3902d88eec8f87da75d3ae5046ee34573562add3da8Ningning Xia TIMESTAMP_PATTERN) 39173cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel cts_v2_path = os.path.join(host, 'cheets_CTS_*', 'results', '*', 39273cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel TIMESTAMP_PATTERN) 39373cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel gts_v2_path = os.path.join(host, 'cheets_GTS.*', 'results', '*', 39473cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel TIMESTAMP_PATTERN) 3952d88eec8f87da75d3ae5046ee34573562add3da8Ningning Xia for result_path, result_pattern in [(cts_path, CTS_RESULT_PATTERN), 39673cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel (cts_v2_path, CTS_V2_RESULT_PATTERN), 39773cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel (gts_v2_path, CTS_V2_RESULT_PATTERN)]: 3982d88eec8f87da75d3ae5046ee34573562add3da8Ningning Xia for path in glob.glob(result_path): 3990c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia try: 4000c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia _upload_files(host, path, result_pattern, multiprocessing) 4010c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia except Exception as e: 4020c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia logging.error('ERROR uploading test results %s to GS: %s', 4030c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia path, e) 4044211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia 4054211124209fdb4469462b6e1b39433cc52b76d02Ningning Xia 4068db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xiadef _is_valid_result(build, result_pattern, suite): 4078db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia """Check if the result should be uploaded to CTS/GTS buckets. 4088db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia 4098db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia @param build: Builder name. 4108db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia @param result_pattern: XML result file pattern. 4118db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia @param suite: Test suite name. 4128db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia 4138db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia @returns: Bool flag indicating whether a valid result. 4148db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia """ 4158db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia if build is None or suite is None: 4168db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia return False 4178db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia 4188db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia # Not valid if it's not a release build. 4198db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia if not re.match(r'(?!trybot-).*-release/.*', build): 4208db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia return False 4218db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia 422ad6d879832021e4d83960adc074a36a948a72171Ilja H. Friedel # Not valid if it's cts result but not 'arc-cts*' or 'test_that_wrapper' 423ad6d879832021e4d83960adc074a36a948a72171Ilja H. Friedel # suite. 42473cf6cd0049a0eddd1cfdc3f73cbaf2d1f114577Ilja H. Friedel result_patterns = [CTS_RESULT_PATTERN, CTS_V2_RESULT_PATTERN] 42561a70d321754db9f8267f794e8db563b850e2e9bIlja H. Friedel if result_pattern in result_patterns and not ( 42661a70d321754db9f8267f794e8db563b850e2e9bIlja H. Friedel suite.startswith('arc-cts') or suite.startswith('arc-gts') or 42761a70d321754db9f8267f794e8db563b850e2e9bIlja H. Friedel suite.startswith('test_that_wrapper')): 4288db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia return False 4298db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia 4308db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia return True 43121922c807733cab1a86096500304aa8ae68bd897Ningning Xia 43221922c807733cab1a86096500304aa8ae68bd897Ningning Xia 4332d88eec8f87da75d3ae5046ee34573562add3da8Ningning Xiadef _upload_files(host, path, result_pattern, multiprocessing): 4340c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia keyval = models.test.parse_job_keyval(host) 4358db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia build = keyval.get('build') 4368db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia suite = keyval.get('suite') 4370c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4388db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia if not _is_valid_result(build, result_pattern, suite): 4398db632fba7f5cbbc4b6693a678582b13039f01c4Ningning Xia # No need to upload current folder, return. 4400c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia return 4410c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4420c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia parent_job_id = str(keyval['parent_job_id']) 4430c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4440c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia folders = path.split(os.sep) 4450c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia job_id = folders[-6] 4460c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia package = folders[-4] 4470c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia timestamp = folders[-1] 4480c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4490c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia # Path: bucket/build/parent_job_id/cheets_CTS.*/job_id_timestamp/ 4500c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia # or bucket/build/parent_job_id/cheets_GTS.*/job_id_timestamp/ 4510c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia cts_apfe_gs_path = os.path.join( 4520c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia DEFAULT_CTS_APFE_GSURI, build, parent_job_id, 4530c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia package, job_id + '_' + timestamp) + '/' 4540c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4550c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia # Path: bucket/cheets_CTS.*/job_id_timestamp/ 4560c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia # or bucket/cheets_GTS.*/job_id_timestamp/ 4570c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia test_result_gs_path = os.path.join( 4580c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia DEFAULT_CTS_RESULTS_GSURI, package, 4590c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia job_id + '_' + timestamp) + '/' 4600c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4610c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia for zip_file in glob.glob(os.path.join('%s.zip' % path)): 462b41527d6f865688299dc5250f8d58baca9432777Allen Li utils.run(' '.join(_get_cmd_list( 4630c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia multiprocessing, zip_file, cts_apfe_gs_path))) 4640c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia logging.debug('Upload %s to %s ', zip_file, cts_apfe_gs_path) 4650c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4660c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia for test_result_file in glob.glob(os.path.join(path, result_pattern)): 467bfa63148ef62dc36e2d4b3456a4b64ae69b21172Ilja H. Friedel # gzip test_result_file(testResult.xml/test_result.xml) 468bfa63148ef62dc36e2d4b3456a4b64ae69b21172Ilja H. Friedel 4690c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia test_result_file_gz = '%s.gz' % test_result_file 4700c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia with open(test_result_file, 'r') as f_in, ( 4710c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia gzip.open(test_result_file_gz, 'w')) as f_out: 4720c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia shutil.copyfileobj(f_in, f_out) 473b41527d6f865688299dc5250f8d58baca9432777Allen Li utils.run(' '.join(_get_cmd_list( 4740c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia multiprocessing, test_result_file_gz, test_result_gs_path))) 4750c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia logging.debug('Zip and upload %s to %s', 4760c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia test_result_file_gz, test_result_gs_path) 477bfa63148ef62dc36e2d4b3456a4b64ae69b21172Ilja H. Friedel # Remove test_result_file_gz(testResult.xml.gz/test_result.xml.gz) 4780c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia os.remove(test_result_file_gz) 4790c27d9befe9523e70b2d69ee353caa7b43270dbdNingning Xia 4802d88eec8f87da75d3ae5046ee34573562add3da8Ningning Xia 4811d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshetdef _emit_gs_returncode_metric(returncode): 4821d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet """Increment the gs_returncode counter based on |returncode|.""" 4831d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet m_gs_returncode = 'chromeos/autotest/gs_offloader/gs_returncode' 4841d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet rcode = int(returncode) 4851d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet if rcode < 0 or rcode > 255: 4861d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet rcode = -1 4871d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet metrics.Counter(m_gs_returncode).increment(fields={'return_code': rcode}) 4881d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet 4891d8df7d72e1fcb24dc6e91cafe6f8314d976a1d3Aviv Keshet 490b41527d6f865688299dc5250f8d58baca9432777Allen Liclass BaseGSOffloader(object): 491ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette 492b41527d6f865688299dc5250f8d58baca9432777Allen Li """Google Storage offloader interface.""" 493dd129979e8227c960f4c7a3d2a1665007f8a424eSimran Basi 494b41527d6f865688299dc5250f8d58baca9432777Allen Li __metaclass__ = abc.ABCMeta 495b41527d6f865688299dc5250f8d58baca9432777Allen Li 496b41527d6f865688299dc5250f8d58baca9432777Allen Li @abc.abstractmethod 497b41527d6f865688299dc5250f8d58baca9432777Allen Li def offload(self, dir_entry, dest_path, job_complete_time): 498b41527d6f865688299dc5250f8d58baca9432777Allen Li """Offload a directory entry to Google Storage. 499b41527d6f865688299dc5250f8d58baca9432777Allen Li 500b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dir_entry: Directory entry to offload. 501b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dest_path: Location in google storage where we will 502b41527d6f865688299dc5250f8d58baca9432777Allen Li offload the directory. 503b41527d6f865688299dc5250f8d58baca9432777Allen Li @param job_complete_time: The complete time of the job from the AFE 504b41527d6f865688299dc5250f8d58baca9432777Allen Li database. 505b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 506b41527d6f865688299dc5250f8d58baca9432777Allen Li 507b41527d6f865688299dc5250f8d58baca9432777Allen Li 508b41527d6f865688299dc5250f8d58baca9432777Allen Liclass GSOffloader(BaseGSOffloader): 509b41527d6f865688299dc5250f8d58baca9432777Allen Li """Google Storage Offloader.""" 510b41527d6f865688299dc5250f8d58baca9432777Allen Li 5110f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang def __init__(self, gs_uri, multiprocessing, delete_age, 5120f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang console_client=None): 513b41527d6f865688299dc5250f8d58baca9432777Allen Li """Returns the offload directory function for the given gs_uri 514b41527d6f865688299dc5250f8d58baca9432777Allen Li 515b41527d6f865688299dc5250f8d58baca9432777Allen Li @param gs_uri: Google storage bucket uri to offload to. 516b41527d6f865688299dc5250f8d58baca9432777Allen Li @param multiprocessing: True to turn on -m option for gsutil. 5170f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang @param console_client: The cloud console client. If None, 5180f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang cloud console APIs are not called. 519b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 520b41527d6f865688299dc5250f8d58baca9432777Allen Li self._gs_uri = gs_uri 521b41527d6f865688299dc5250f8d58baca9432777Allen Li self._multiprocessing = multiprocessing 522b41527d6f865688299dc5250f8d58baca9432777Allen Li self._delete_age = delete_age 5230f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang self._console_client = console_client 524b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 525eeaa7ef307202c9494ec72fffd102f970842f659Prathmesh Prabhu @metrics.SecondsTimerDecorator( 526eeaa7ef307202c9494ec72fffd102f970842f659Prathmesh Prabhu 'chromeos/autotest/gs_offloader/job_offload_duration') 527b41527d6f865688299dc5250f8d58baca9432777Allen Li def offload(self, dir_entry, dest_path, job_complete_time): 5282c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Offload the specified directory entry to Google storage. 5292c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 5302c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @param dir_entry: Directory entry to offload. 5312c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @param dest_path: Location in google storage where we will 5322c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette offload the directory. 5335ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow @param job_complete_time: The complete time of the job from the AFE 5345ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow database. 535b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 536b41527d6f865688299dc5250f8d58baca9432777Allen Li with tempfile.TemporaryFile('w+') as stdout_file, \ 537b41527d6f865688299dc5250f8d58baca9432777Allen Li tempfile.TemporaryFile('w+') as stderr_file: 538b41527d6f865688299dc5250f8d58baca9432777Allen Li try: 539b41527d6f865688299dc5250f8d58baca9432777Allen Li self._offload(dir_entry, dest_path, stdout_file, stderr_file) 540b41527d6f865688299dc5250f8d58baca9432777Allen Li except _OffloadError as e: 541b41527d6f865688299dc5250f8d58baca9432777Allen Li metrics_fields = _get_metrics_fields(dir_entry) 542b41527d6f865688299dc5250f8d58baca9432777Allen Li m_any_error = 'chromeos/autotest/errors/gs_offloader/any_error' 543b41527d6f865688299dc5250f8d58baca9432777Allen Li metrics.Counter(m_any_error).increment(fields=metrics_fields) 544b41527d6f865688299dc5250f8d58baca9432777Allen Li 545b41527d6f865688299dc5250f8d58baca9432777Allen Li # Rewind the log files for stdout and stderr and log 546b41527d6f865688299dc5250f8d58baca9432777Allen Li # their contents. 547b41527d6f865688299dc5250f8d58baca9432777Allen Li stdout_file.seek(0) 548b41527d6f865688299dc5250f8d58baca9432777Allen Li stderr_file.seek(0) 549b41527d6f865688299dc5250f8d58baca9432777Allen Li stderr_content = stderr_file.read() 550b41527d6f865688299dc5250f8d58baca9432777Allen Li logging.warning('Error occurred when offloading %s:', dir_entry) 551b41527d6f865688299dc5250f8d58baca9432777Allen Li logging.warning('Stdout:\n%s \nStderr:\n%s', stdout_file.read(), 552b41527d6f865688299dc5250f8d58baca9432777Allen Li stderr_content) 553b41527d6f865688299dc5250f8d58baca9432777Allen Li 554b41527d6f865688299dc5250f8d58baca9432777Allen Li # Some result files may have wrong file permission. Try 555b41527d6f865688299dc5250f8d58baca9432777Allen Li # to correct such error so later try can success. 556b41527d6f865688299dc5250f8d58baca9432777Allen Li # TODO(dshi): The code is added to correct result files 557b41527d6f865688299dc5250f8d58baca9432777Allen Li # with wrong file permission caused by bug 511778. After 558b41527d6f865688299dc5250f8d58baca9432777Allen Li # this code is pushed to lab and run for a while to 559b41527d6f865688299dc5250f8d58baca9432777Allen Li # clean up these files, following code and function 560b41527d6f865688299dc5250f8d58baca9432777Allen Li # correct_results_folder_permission can be deleted. 561b41527d6f865688299dc5250f8d58baca9432777Allen Li if 'CommandException: Error opening file' in stderr_content: 562b41527d6f865688299dc5250f8d58baca9432777Allen Li correct_results_folder_permission(dir_entry) 563b41527d6f865688299dc5250f8d58baca9432777Allen Li else: 564b41527d6f865688299dc5250f8d58baca9432777Allen Li self._prune(dir_entry, job_complete_time) 5652c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 566b41527d6f865688299dc5250f8d58baca9432777Allen Li def _offload(self, dir_entry, dest_path, 567b41527d6f865688299dc5250f8d58baca9432777Allen Li stdout_file, stderr_file): 568b41527d6f865688299dc5250f8d58baca9432777Allen Li """Offload the specified directory entry to Google storage. 569b41527d6f865688299dc5250f8d58baca9432777Allen Li 570b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dir_entry: Directory entry to offload. 571b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dest_path: Location in google storage where we will 572b41527d6f865688299dc5250f8d58baca9432777Allen Li offload the directory. 573b41527d6f865688299dc5250f8d58baca9432777Allen Li @param job_complete_time: The complete time of the job from the AFE 574b41527d6f865688299dc5250f8d58baca9432777Allen Li database. 575b41527d6f865688299dc5250f8d58baca9432777Allen Li @param stdout_file: Log file. 576b41527d6f865688299dc5250f8d58baca9432777Allen Li @param stderr_file: Log file. 5772c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 578b41527d6f865688299dc5250f8d58baca9432777Allen Li if _is_uploaded(dir_entry): 579b41527d6f865688299dc5250f8d58baca9432777Allen Li return 580b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi start_time = time.time() 581b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi metrics_fields = _get_metrics_fields(dir_entry) 582b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi es_metadata = _get_es_metadata(dir_entry) 583b41527d6f865688299dc5250f8d58baca9432777Allen Li error_obj = _OffloadError(start_time, es_metadata) 5842c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette try: 585b41527d6f865688299dc5250f8d58baca9432777Allen Li sanitize_dir(dir_entry) 586b41527d6f865688299dc5250f8d58baca9432777Allen Li if DEFAULT_CTS_RESULTS_GSURI: 587b41527d6f865688299dc5250f8d58baca9432777Allen Li _upload_cts_testresult(dir_entry, self._multiprocessing) 588b41527d6f865688299dc5250f8d58baca9432777Allen Li 589b41527d6f865688299dc5250f8d58baca9432777Allen Li if LIMIT_FILE_COUNT: 590b41527d6f865688299dc5250f8d58baca9432777Allen Li limit_file_count(dir_entry) 591b41527d6f865688299dc5250f8d58baca9432777Allen Li es_metadata['size_kb'] = file_utils.get_directory_size_kibibytes(dir_entry) 592b41527d6f865688299dc5250f8d58baca9432777Allen Li 593b41527d6f865688299dc5250f8d58baca9432777Allen Li process = None 594b41527d6f865688299dc5250f8d58baca9432777Allen Li with timeout_util.Timeout(OFFLOAD_TIMEOUT_SECS): 595b41527d6f865688299dc5250f8d58baca9432777Allen Li gs_path = '%s%s' % (self._gs_uri, dest_path) 5965ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow process = subprocess.Popen( 597b41527d6f865688299dc5250f8d58baca9432777Allen Li _get_cmd_list(self._multiprocessing, dir_entry, gs_path), 5985ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow stdout=stdout_file, stderr=stderr_file) 5995ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow process.wait() 600b41527d6f865688299dc5250f8d58baca9432777Allen Li 601b41527d6f865688299dc5250f8d58baca9432777Allen Li _emit_gs_returncode_metric(process.returncode) 602b41527d6f865688299dc5250f8d58baca9432777Allen Li if process.returncode != 0: 603b41527d6f865688299dc5250f8d58baca9432777Allen Li raise error_obj 604b41527d6f865688299dc5250f8d58baca9432777Allen Li _emit_offload_metrics(dir_entry) 605b41527d6f865688299dc5250f8d58baca9432777Allen Li 6060f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang if self._console_client: 6070f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang gcs_uri = os.path.join(gs_path, 6080f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang os.path.basename(dir_entry)) 6090f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang if not self._console_client.send_test_job_offloaded_message( 6100f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang gcs_uri): 611b41527d6f865688299dc5250f8d58baca9432777Allen Li raise error_obj 6120f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang 613b41527d6f865688299dc5250f8d58baca9432777Allen Li _mark_uploaded(dir_entry) 614b41527d6f865688299dc5250f8d58baca9432777Allen Li except timeout_util.TimeoutError: 615268d3d06f45bcfe453126e68976ab017d8fe50a4Prathmesh Prabhu m_timeout = 'chromeos/autotest/errors/gs_offloader/timed_out_count' 616b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi metrics.Counter(m_timeout).increment(fields=metrics_fields) 6172c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # If we finished the call to Popen(), we may need to 6182c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # terminate the child process. We don't bother calling 6192c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # process.poll(); that inherently races because the child 6202c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # can die any time it wants. 6212c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if process: 6222c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette try: 6232c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette process.terminate() 6242c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette except OSError: 6252c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # We don't expect any error other than "No such 6262c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # process". 6272c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette pass 6282c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.error('Offloading %s timed out after waiting %d ' 6292c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'seconds.', dir_entry, OFFLOAD_TIMEOUT_SECS) 630b41527d6f865688299dc5250f8d58baca9432777Allen Li raise error_obj 631b41527d6f865688299dc5250f8d58baca9432777Allen Li 632b41527d6f865688299dc5250f8d58baca9432777Allen Li def _prune(self, dir_entry, job_complete_time): 633b41527d6f865688299dc5250f8d58baca9432777Allen Li """Prune directory if it is uploaded and expired. 634b41527d6f865688299dc5250f8d58baca9432777Allen Li 635b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dir_entry: Directory entry to offload. 636b41527d6f865688299dc5250f8d58baca9432777Allen Li @param job_complete_time: The complete time of the job from the AFE 637b41527d6f865688299dc5250f8d58baca9432777Allen Li database. 638b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 639b41527d6f865688299dc5250f8d58baca9432777Allen Li if not (_is_uploaded(dir_entry) 640b41527d6f865688299dc5250f8d58baca9432777Allen Li and job_directories.is_job_expired(self._delete_age, 641b41527d6f865688299dc5250f8d58baca9432777Allen Li job_complete_time)): 642b41527d6f865688299dc5250f8d58baca9432777Allen Li return 643b41527d6f865688299dc5250f8d58baca9432777Allen Li try: 644b41527d6f865688299dc5250f8d58baca9432777Allen Li shutil.rmtree(dir_entry) 6452c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette except OSError as e: 6462c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # The wrong file permission can lead call 6472c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # `shutil.rmtree(dir_entry)` to raise OSError with message 6482c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # 'Permission denied'. Details can be found in 6492c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # crbug.com/536151 6502c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if e.errno == errno.EACCES: 6512c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette correct_results_folder_permission(dir_entry) 652268d3d06f45bcfe453126e68976ab017d8fe50a4Prathmesh Prabhu m_permission_error = ('chromeos/autotest/errors/gs_offloader/' 653268d3d06f45bcfe453126e68976ab017d8fe50a4Prathmesh Prabhu 'wrong_permissions_count') 654b41527d6f865688299dc5250f8d58baca9432777Allen Li metrics_fields = _get_metrics_fields(dir_entry) 655b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi metrics.Counter(m_permission_error).increment(fields=metrics_fields) 656b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 657b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 658b41527d6f865688299dc5250f8d58baca9432777Allen Liclass _OffloadError(Exception): 659b41527d6f865688299dc5250f8d58baca9432777Allen Li """Google Storage offload failed.""" 660b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 661b41527d6f865688299dc5250f8d58baca9432777Allen Li def __init__(self, start_time, es_metadata): 662b41527d6f865688299dc5250f8d58baca9432777Allen Li super(_OffloadError, self).__init__(start_time, es_metadata) 663b41527d6f865688299dc5250f8d58baca9432777Allen Li self.start_time = start_time 664b41527d6f865688299dc5250f8d58baca9432777Allen Li self.es_metadata = es_metadata 665b2751fcd14518af3e6c5aba50c345abeded576b9Dan Shi 6669523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basi 66720a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalski 668b41527d6f865688299dc5250f8d58baca9432777Allen Liclass FakeGSOffloader(BaseGSOffloader): 669bd9ded0df72fbdd62f9823b21a87400e8bee1d25Simran Basi 670b41527d6f865688299dc5250f8d58baca9432777Allen Li """Fake Google Storage Offloader that only deletes directories.""" 671bd9ded0df72fbdd62f9823b21a87400e8bee1d25Simran Basi 672b41527d6f865688299dc5250f8d58baca9432777Allen Li def offload(self, dir_entry, dest_path, job_complete_time): 673b41527d6f865688299dc5250f8d58baca9432777Allen Li """Pretend to offload a directory and delete it. 674b41527d6f865688299dc5250f8d58baca9432777Allen Li 675b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dir_entry: Directory entry to offload. 676b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dest_path: Location in google storage where we will 677b41527d6f865688299dc5250f8d58baca9432777Allen Li offload the directory. 678b41527d6f865688299dc5250f8d58baca9432777Allen Li @param job_complete_time: The complete time of the job from the AFE 679b41527d6f865688299dc5250f8d58baca9432777Allen Li database. 680b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 681b41527d6f865688299dc5250f8d58baca9432777Allen Li shutil.rmtree(dir_entry) 682b41527d6f865688299dc5250f8d58baca9432777Allen Li 683b41527d6f865688299dc5250f8d58baca9432777Allen Li 684b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _is_expired(job, age_limit): 685b41527d6f865688299dc5250f8d58baca9432777Allen Li """Return whether job directory is expired for uploading 686b41527d6f865688299dc5250f8d58baca9432777Allen Li 687b41527d6f865688299dc5250f8d58baca9432777Allen Li @param job: _JobDirectory instance. 688b41527d6f865688299dc5250f8d58baca9432777Allen Li @param age_limit: Minimum age in days at which a job may be offloaded. 689b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 690b41527d6f865688299dc5250f8d58baca9432777Allen Li job_timestamp = job.get_timestamp_if_finished() 691b41527d6f865688299dc5250f8d58baca9432777Allen Li if not job_timestamp: 692b41527d6f865688299dc5250f8d58baca9432777Allen Li return False 693b41527d6f865688299dc5250f8d58baca9432777Allen Li return job_directories.is_job_expired(age_limit, job_timestamp) 694b41527d6f865688299dc5250f8d58baca9432777Allen Li 695b41527d6f865688299dc5250f8d58baca9432777Allen Li 696b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _emit_offload_metrics(dirpath): 697b41527d6f865688299dc5250f8d58baca9432777Allen Li """Emit gs offload metrics. 698b41527d6f865688299dc5250f8d58baca9432777Allen Li 699b41527d6f865688299dc5250f8d58baca9432777Allen Li @param dirpath: Offloaded directory path. 7002c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 701b41527d6f865688299dc5250f8d58baca9432777Allen Li dir_size = file_utils.get_directory_size_kibibytes(dirpath) 702b41527d6f865688299dc5250f8d58baca9432777Allen Li metrics_fields = _get_metrics_fields(dirpath) 703b41527d6f865688299dc5250f8d58baca9432777Allen Li 704b41527d6f865688299dc5250f8d58baca9432777Allen Li m_offload_count = ( 705b41527d6f865688299dc5250f8d58baca9432777Allen Li 'chromeos/autotest/gs_offloader/jobs_offloaded') 706b41527d6f865688299dc5250f8d58baca9432777Allen Li metrics.Counter(m_offload_count).increment( 707b41527d6f865688299dc5250f8d58baca9432777Allen Li fields=metrics_fields) 708b41527d6f865688299dc5250f8d58baca9432777Allen Li m_offload_size = ('chromeos/autotest/gs_offloader/' 709b41527d6f865688299dc5250f8d58baca9432777Allen Li 'kilobytes_transferred') 710b41527d6f865688299dc5250f8d58baca9432777Allen Li metrics.Counter(m_offload_size).increment_by( 711b41527d6f865688299dc5250f8d58baca9432777Allen Li dir_size, fields=metrics_fields) 712bd9ded0df72fbdd62f9823b21a87400e8bee1d25Simran Basi 713bd9ded0df72fbdd62f9823b21a87400e8bee1d25Simran Basi 7149579b38072f8008b92623d7c30e3fc55c05c7561Allen Lidef _is_uploaded(dirpath): 7159579b38072f8008b92623d7c30e3fc55c05c7561Allen Li """Return whether directory has been uploaded. 7169579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7179579b38072f8008b92623d7c30e3fc55c05c7561Allen Li @param dirpath: Directory path string. 7189579b38072f8008b92623d7c30e3fc55c05c7561Allen Li """ 7199579b38072f8008b92623d7c30e3fc55c05c7561Allen Li return os.path.isfile(_get_uploaded_marker_file(dirpath)) 7209579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7219579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7229579b38072f8008b92623d7c30e3fc55c05c7561Allen Lidef _mark_uploaded(dirpath): 7239579b38072f8008b92623d7c30e3fc55c05c7561Allen Li """Mark directory as uploaded. 7249579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7259579b38072f8008b92623d7c30e3fc55c05c7561Allen Li @param dirpath: Directory path string. 7269579b38072f8008b92623d7c30e3fc55c05c7561Allen Li """ 7279579b38072f8008b92623d7c30e3fc55c05c7561Allen Li with open(_get_uploaded_marker_file(dirpath), 'a'): 7289579b38072f8008b92623d7c30e3fc55c05c7561Allen Li pass 7299579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7309579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7319579b38072f8008b92623d7c30e3fc55c05c7561Allen Lidef _get_uploaded_marker_file(dirpath): 7329579b38072f8008b92623d7c30e3fc55c05c7561Allen Li """Return path to upload marker file for directory. 7339579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7349579b38072f8008b92623d7c30e3fc55c05c7561Allen Li @param dirpath: Directory path string. 7359579b38072f8008b92623d7c30e3fc55c05c7561Allen Li """ 7369579b38072f8008b92623d7c30e3fc55c05c7561Allen Li return '%s/.GS_UPLOADED' % (dirpath,) 7379579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 7389579b38072f8008b92623d7c30e3fc55c05c7561Allen Li 739fda271aecfd1402eaee58d0ff56e2b9bde192865Prathmesh Prabhudef _format_job_for_failure_reporting(job): 740fda271aecfd1402eaee58d0ff56e2b9bde192865Prathmesh Prabhu """Formats a _JobDirectory for reporting / logging. 741fda271aecfd1402eaee58d0ff56e2b9bde192865Prathmesh Prabhu 742fda271aecfd1402eaee58d0ff56e2b9bde192865Prathmesh Prabhu @param job: The _JobDirectory to format. 743fda271aecfd1402eaee58d0ff56e2b9bde192865Prathmesh Prabhu """ 744b41527d6f865688299dc5250f8d58baca9432777Allen Li d = datetime.datetime.fromtimestamp(job.first_offload_start) 74580dfb1e9366b74f6c5a928e33793012e5a8ebc5aPrathmesh Prabhu data = (d.strftime(FAILED_OFFLOADS_TIME_FORMAT), 746b41527d6f865688299dc5250f8d58baca9432777Allen Li job.offload_count, 747b41527d6f865688299dc5250f8d58baca9432777Allen Li job.dirname) 74880dfb1e9366b74f6c5a928e33793012e5a8ebc5aPrathmesh Prabhu return FAILED_OFFLOADS_LINE_FORMAT % data 749fda271aecfd1402eaee58d0ff56e2b9bde192865Prathmesh Prabhu 750fda271aecfd1402eaee58d0ff56e2b9bde192865Prathmesh Prabhu 751ac0edb27da1aaac3600b7aedc1568878fe61645eSimran Basidef wait_for_gs_write_access(gs_uri): 7522c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Verify and wait until we have write access to Google Storage. 753ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette 7542c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette @param gs_uri: The Google Storage URI we are trying to offload to. 755ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette """ 7562c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # TODO (sbasi) Try to use the gsutil command to check write access. 7572c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # Ensure we have write access to gs_uri. 7582c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette dummy_file = tempfile.NamedTemporaryFile() 759b41527d6f865688299dc5250f8d58baca9432777Allen Li test_cmd = _get_cmd_list(False, dummy_file.name, gs_uri) 7602c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette while True: 7612c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette try: 7622c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette subprocess.check_call(test_cmd) 7632c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette subprocess.check_call( 764365049f691c80802030f62cf2fce345bb670e00dDan Shi ['gsutil', 'rm', 7652c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette os.path.join(gs_uri, 7662c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette os.path.basename(dummy_file.name))]) 7672c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette break 7682c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette except subprocess.CalledProcessError: 7692c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.debug('Unable to offload to %s, sleeping.', gs_uri) 7702c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette time.sleep(120) 771ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette 772ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette 7732c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnetteclass Offloader(object): 7742c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """State of the offload process. 7752c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 7762c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette Contains the following member fields: 777b41527d6f865688299dc5250f8d58baca9432777Allen Li * _gs_offloader: _BaseGSOffloader to use to offload a job directory. 7782c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette * _jobdir_classes: List of classes of job directory to be 7792c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette offloaded. 7802c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette * _processes: Maximum number of outstanding offload processes 7812c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette to allow during an offload cycle. 7822c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette * _age_limit: Minimum age in days at which a job may be 7832c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette offloaded. 7842c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette * _open_jobs: a dictionary mapping directory paths to Job 7852c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette objects. 786ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette """ 787ea785366f46ec4ed7a75d382e152a77aa51069d7J. Richard Barnette 7882c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette def __init__(self, options): 7895ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow self._upload_age_limit = options.age_to_upload 7905ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow self._delete_age_limit = options.age_to_delete 7912c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if options.delete_only: 792b41527d6f865688299dc5250f8d58baca9432777Allen Li self._gs_offloader = FakeGSOffloader() 7932c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette else: 7942c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette self.gs_uri = utils.get_offload_gsuri() 7952c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.debug('Offloading to: %s', self.gs_uri) 7960df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang multiprocessing = False 7970df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang if options.multiprocessing: 7980df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang multiprocessing = True 7990df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang elif options.multiprocessing is None: 8000df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang multiprocessing = GS_OFFLOADER_MULTIPROCESSING 8010df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang logging.info( 8020df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang 'Offloader multiprocessing is set to:%r', multiprocessing) 8030f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang console_client = None 8040f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang if cloud_console_client.is_cloud_notification_enabled(): 8050f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang console_client = cloud_console_client.PubSubBasedClient() 806b41527d6f865688299dc5250f8d58baca9432777Allen Li self._gs_offloader = GSOffloader( 8075ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow self.gs_uri, multiprocessing, self._delete_age_limit, 8080f553bd773170c8ddb80d6061cdc0f133ba239b6Michael Tang console_client) 8092c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette classlist = [] 8102c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if options.process_hosts_only or options.process_all: 8112c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette classlist.append(job_directories.SpecialJobDirectory) 8122c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if not options.process_hosts_only: 8132c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette classlist.append(job_directories.RegularJobDirectory) 8142c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette self._jobdir_classes = classlist 8152c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette assert self._jobdir_classes 8162c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette self._processes = options.parallelism 8172c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette self._open_jobs = {} 81897d188c2d78728c69a2d71f08de210fe683d3d1eMichael Tang self._pusub_topic = None 8190be2f2da6a4179ccce3e220a48838e91ee617934Allen Li self._offload_count_limit = 3 8202c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8212c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8222c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette def _add_new_jobs(self): 8232c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Find new job directories that need offloading. 8242c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8252c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette Go through the file system looking for valid job directories 8262c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette that are currently not in `self._open_jobs`, and add them in. 8272c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8282c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 8292c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette new_job_count = 0 8302c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette for cls in self._jobdir_classes: 8312c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette for resultsdir in cls.get_job_directories(): 832b41527d6f865688299dc5250f8d58baca9432777Allen Li if ( 833b41527d6f865688299dc5250f8d58baca9432777Allen Li resultsdir in self._open_jobs 834b41527d6f865688299dc5250f8d58baca9432777Allen Li or _is_uploaded(resultsdir)): 8352c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette continue 8362c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette self._open_jobs[resultsdir] = cls(resultsdir) 8372c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette new_job_count += 1 8382c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.debug('Start of offload cycle - found %d new jobs', 8392c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette new_job_count) 8402c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8412c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8422c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette def _remove_offloaded_jobs(self): 8432c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Removed offloaded jobs from `self._open_jobs`.""" 8442c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette removed_job_count = 0 8452c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette for jobkey, job in self._open_jobs.items(): 846b41527d6f865688299dc5250f8d58baca9432777Allen Li if ( 847b41527d6f865688299dc5250f8d58baca9432777Allen Li not os.path.exists(job.dirname) 848b41527d6f865688299dc5250f8d58baca9432777Allen Li or _is_uploaded(job.dirname)): 8492c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette del self._open_jobs[jobkey] 8502c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette removed_job_count += 1 8512c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.debug('End of offload cycle - cleared %d new jobs, ' 8522c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'carrying %d open jobs', 8532c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette removed_job_count, len(self._open_jobs)) 8542c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8552c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 856b41527d6f865688299dc5250f8d58baca9432777Allen Li def _report_failed_jobs(self): 857b41527d6f865688299dc5250f8d58baca9432777Allen Li """Report status after attempting offload. 8582c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8592c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette This function processes all jobs in `self._open_jobs`, assuming 8602c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette an attempt has just been made to offload all of them. 8612c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8622c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette If any jobs have reportable errors, and we haven't generated 8632c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette an e-mail report in the last `REPORT_INTERVAL_SECS` seconds, 8642c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette send new e-mail describing the failures. 8652c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8662c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 867343d171b1ac05dbf89aab7439a6eb8be3a9c002fPrathmesh Prabhu failed_jobs = [j for j in self._open_jobs.values() if 868b41527d6f865688299dc5250f8d58baca9432777Allen Li j.first_offload_start] 869ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu self._report_failed_jobs_count(failed_jobs) 87016f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu self._log_failed_jobs_locally(failed_jobs) 871343d171b1ac05dbf89aab7439a6eb8be3a9c002fPrathmesh Prabhu 872343d171b1ac05dbf89aab7439a6eb8be3a9c002fPrathmesh Prabhu 8732c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette def offload_once(self): 8742c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Perform one offload cycle. 8752c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8762c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette Find all job directories for new jobs that we haven't seen 8772c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette before. Then, attempt to offload the directories for any 8782c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette jobs that have finished running. Offload of multiple jobs 8792c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette is done in parallel, up to `self._processes` at a time. 8802c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8812c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette After we've tried uploading all directories, go through the list 8822c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette checking the status of all uploaded directories. If necessary, 8832c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette report failures via e-mail. 8842c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 8852c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """ 8862c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette self._add_new_jobs() 887c9856853da581f3ab7c2d30c2f7d2b374aef4169Prathmesh Prabhu self._report_current_jobs_count() 8882c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette with parallel.BackgroundTaskRunner( 889b41527d6f865688299dc5250f8d58baca9432777Allen Li self._gs_offloader.offload, processes=self._processes) as queue: 8902c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette for job in self._open_jobs.values(): 891b41527d6f865688299dc5250f8d58baca9432777Allen Li _enqueue_offload(job, queue, self._upload_age_limit) 8920be2f2da6a4179ccce3e220a48838e91ee617934Allen Li self._give_up_on_jobs_over_limit() 893b41527d6f865688299dc5250f8d58baca9432777Allen Li self._remove_offloaded_jobs() 894b41527d6f865688299dc5250f8d58baca9432777Allen Li self._report_failed_jobs() 8959523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basi 8969523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basi 8970be2f2da6a4179ccce3e220a48838e91ee617934Allen Li def _give_up_on_jobs_over_limit(self): 8980be2f2da6a4179ccce3e220a48838e91ee617934Allen Li """Give up on jobs that have gone over the offload limit. 8990be2f2da6a4179ccce3e220a48838e91ee617934Allen Li 9000be2f2da6a4179ccce3e220a48838e91ee617934Allen Li We mark them as uploaded as we won't try to offload them any more. 9010be2f2da6a4179ccce3e220a48838e91ee617934Allen Li """ 9020be2f2da6a4179ccce3e220a48838e91ee617934Allen Li for job in self._open_jobs.values(): 903808828bdc884391b638525fa8ee8b22a400985cbAllen Li if job.offload_count >= self._offload_count_limit: 9040be2f2da6a4179ccce3e220a48838e91ee617934Allen Li _mark_uploaded(job.dirname) 9050be2f2da6a4179ccce3e220a48838e91ee617934Allen Li 9060be2f2da6a4179ccce3e220a48838e91ee617934Allen Li 90716f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu def _log_failed_jobs_locally(self, failed_jobs, 90816f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu log_file=FAILED_OFFLOADS_FILE): 90916f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu """Updates a local file listing all the failed jobs. 91016f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 91116f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu The dropped file can be used by the developers to list jobs that we have 91216f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu failed to upload. 91316f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 91416f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu @param failed_jobs: A list of failed _JobDirectory objects. 91516f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu @param log_file: The file to log the failed jobs to. 91616f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu """ 91716f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu now = datetime.datetime.now() 91880dfb1e9366b74f6c5a928e33793012e5a8ebc5aPrathmesh Prabhu now_str = now.strftime(FAILED_OFFLOADS_TIME_FORMAT) 91916f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu formatted_jobs = [_format_job_for_failure_reporting(job) 92016f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu for job in failed_jobs] 92116f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu formatted_jobs.sort() 92216f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 92316f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu with open(log_file, 'w') as logfile: 92416f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu logfile.write(FAILED_OFFLOADS_FILE_HEADER % 92516f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu (now_str, len(failed_jobs))) 92616f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu logfile.writelines(formatted_jobs) 92716f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 92816f9e5c9cbeadbfb030d2b2cf90c0816712d43d9Prathmesh Prabhu 929c9856853da581f3ab7c2d30c2f7d2b374aef4169Prathmesh Prabhu def _report_current_jobs_count(self): 930c9856853da581f3ab7c2d30c2f7d2b374aef4169Prathmesh Prabhu """Report the number of outstanding jobs to monarch.""" 931c9856853da581f3ab7c2d30c2f7d2b374aef4169Prathmesh Prabhu metrics.Gauge('chromeos/autotest/gs_offloader/current_jobs_count').set( 932c9856853da581f3ab7c2d30c2f7d2b374aef4169Prathmesh Prabhu len(self._open_jobs)) 933c9856853da581f3ab7c2d30c2f7d2b374aef4169Prathmesh Prabhu 934c9856853da581f3ab7c2d30c2f7d2b374aef4169Prathmesh Prabhu 935ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu def _report_failed_jobs_count(self, failed_jobs): 936ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu """Report the number of outstanding failed offload jobs to monarch. 937ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu 938ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu @param: List of failed jobs. 939ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu """ 940ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu metrics.Gauge('chromeos/autotest/gs_offloader/failed_jobs_count').set( 941ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu len(failed_jobs)) 942ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu 943ea869738e7219ff6ad91ce958aeb60430e690610Prathmesh Prabhu 944b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _enqueue_offload(job, queue, age_limit): 945b41527d6f865688299dc5250f8d58baca9432777Allen Li """Enqueue the job for offload, if it's eligible. 946b41527d6f865688299dc5250f8d58baca9432777Allen Li 947b41527d6f865688299dc5250f8d58baca9432777Allen Li The job is eligible for offloading if the database has marked 948b41527d6f865688299dc5250f8d58baca9432777Allen Li it finished, and the job is older than the `age_limit` 949b41527d6f865688299dc5250f8d58baca9432777Allen Li parameter. 950b41527d6f865688299dc5250f8d58baca9432777Allen Li 951b41527d6f865688299dc5250f8d58baca9432777Allen Li If the job is eligible, offload processing is requested by 952b41527d6f865688299dc5250f8d58baca9432777Allen Li passing the `queue` parameter's `put()` method a sequence with 953b41527d6f865688299dc5250f8d58baca9432777Allen Li the job's `dirname` attribute and its directory name. 954b41527d6f865688299dc5250f8d58baca9432777Allen Li 955b41527d6f865688299dc5250f8d58baca9432777Allen Li @param job _JobDirectory instance to offload. 956b41527d6f865688299dc5250f8d58baca9432777Allen Li @param queue If the job should be offloaded, put the offload 957b41527d6f865688299dc5250f8d58baca9432777Allen Li parameters into this queue for processing. 958b41527d6f865688299dc5250f8d58baca9432777Allen Li @param age_limit Minimum age for a job to be offloaded. A value 959b41527d6f865688299dc5250f8d58baca9432777Allen Li of 0 means that the job will be offloaded as 960b41527d6f865688299dc5250f8d58baca9432777Allen Li soon as it is finished. 961b41527d6f865688299dc5250f8d58baca9432777Allen Li 962b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 963b41527d6f865688299dc5250f8d58baca9432777Allen Li if not job.offload_count: 964b41527d6f865688299dc5250f8d58baca9432777Allen Li if not _is_expired(job, age_limit): 965b41527d6f865688299dc5250f8d58baca9432777Allen Li return 966b41527d6f865688299dc5250f8d58baca9432777Allen Li job.first_offload_start = time.time() 967b41527d6f865688299dc5250f8d58baca9432777Allen Li job.offload_count += 1 968b41527d6f865688299dc5250f8d58baca9432777Allen Li if job.process_gs_instructions(): 969b41527d6f865688299dc5250f8d58baca9432777Allen Li timestamp = job.get_timestamp_if_finished() 970b41527d6f865688299dc5250f8d58baca9432777Allen Li queue.put([job.dirname, os.path.dirname(job.dirname), timestamp]) 971b41527d6f865688299dc5250f8d58baca9432777Allen Li 972b41527d6f865688299dc5250f8d58baca9432777Allen Li 9737d9a14925e2bf5fbba92ffecffdfa6f1ac2100bbSimran Basidef parse_options(): 9742c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Parse the args passed into gs_offloader.""" 9752c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette defaults = 'Defaults:\n Destination: %s\n Results Path: %s' % ( 9762c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette utils.DEFAULT_OFFLOAD_GSURI, RESULTS_DIR) 9772c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette usage = 'usage: %prog [options]\n' + defaults 9782c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser = OptionParser(usage) 9792c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.add_option('-a', '--all', dest='process_all', 9802c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette action='store_true', 9812c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette help='Offload all files in the results directory.') 9822c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.add_option('-s', '--hosts', dest='process_hosts_only', 9832c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette action='store_true', 9842c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette help='Offload only the special tasks result files ' 9852c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'located in the results/hosts subdirectory') 9862c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.add_option('-p', '--parallelism', dest='parallelism', 9872c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette type='int', default=1, 9882c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette help='Number of parallel workers to use.') 9892c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.add_option('-o', '--delete_only', dest='delete_only', 9902c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette action='store_true', 9912c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette help='GS Offloader will only the delete the ' 9922c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'directories and will not offload them to google ' 9932c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'storage. NOTE: If global_config variable ' 9942c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'CROS.gs_offloading_enabled is False, --delete_only ' 9952c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'is automatically True.', 9962c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette default=not GS_OFFLOADING_ENABLED) 9972c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.add_option('-d', '--days_old', dest='days_old', 9982c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette help='Minimum job age in days before a result can be ' 9992c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'offloaded.', type='int', default=0) 10002c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.add_option('-l', '--log_size', dest='log_size', 10012c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette help='Limit the offloader logs to a specified ' 10022c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'number of Mega Bytes.', type='int', default=0) 10032c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.add_option('-m', dest='multiprocessing', action='store_true', 10040df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang help='Turn on -m option for gsutil. If not set, the ' 10050df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang 'global config setting gs_offloader_multiprocessing ' 10060df2eb4e1e8638a1a165e8f2dc3b9230f85acf06Michael Tang 'under CROS section is applied.') 100744b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow parser.add_option('-i', '--offload_once', dest='offload_once', 100844b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow action='store_true', 100944b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow help='Upload all available results and then exit.') 101044b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow parser.add_option('-y', '--normal_priority', dest='normal_priority', 101144b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow action='store_true', 101244b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow help='Upload using normal process priority.') 10135ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow parser.add_option('-u', '--age_to_upload', dest='age_to_upload', 10145ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow help='Minimum job age in days before a result can be ' 10155ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow 'offloaded, but not removed from local storage', 10165ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow type='int', default=None) 10175ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow parser.add_option('-n', '--age_to_delete', dest='age_to_delete', 10185ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow help='Minimum job age in days before a result can be ' 10195ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow 'removed from local storage', 10205ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow type='int', default=None) 10215ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow 10222c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette options = parser.parse_args()[0] 10232c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if options.process_all and options.process_hosts_only: 10242c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette parser.print_help() 10252c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette print ('Cannot process all files and only the hosts ' 10262c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 'subdirectory. Please remove an argument.') 10272c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette sys.exit(1) 10285ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow 10295ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow if options.days_old and (options.age_to_upload or options.age_to_delete): 10305ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow parser.print_help() 10315ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow print('Use the days_old option or the age_to_* options but not both') 10325ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow sys.exit(1) 10335ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow 10345ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow if options.age_to_upload == None: 10355ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow options.age_to_upload = options.days_old 10365ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow if options.age_to_delete == None: 10375ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow options.age_to_delete = options.days_old 10385ba5fb86d92a242691def5fc87ff35e51497b2aeKeith Haddow 10392c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette return options 1040cb2e2b789b9377262b08c185b8b96e7c758ef9e5Scott Zawalski 10419523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basi 10429523eaa3c3fea15563a3d32dbc0cdda2453ee492Simran Basidef main(): 10432c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette """Main method of gs_offloader.""" 10442c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette options = parse_options() 10452c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 10462c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if options.process_all: 10472c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette offloader_type = 'all' 10482c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette elif options.process_hosts_only: 10492c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette offloader_type = 'hosts' 10502c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette else: 10512c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette offloader_type = 'jobs' 10522c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 1053b41527d6f865688299dc5250f8d58baca9432777Allen Li _setup_logging(options, offloader_type) 10542c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 10552c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # Nice our process (carried to subprocesses) so we don't overload 10562c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # the system. 105744b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow if not options.normal_priority: 105844b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow logging.debug('Set process to nice value: %d', NICENESS) 105944b5e4b7c3ea255e7f700af45249861b3b8b2336Keith Haddow os.nice(NICENESS) 10602c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette if psutil: 10612c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette proc = psutil.Process() 10622c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.debug('Set process to ionice IDLE') 10632c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette proc.ionice(psutil.IOPRIO_CLASS_IDLE) 10642c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 10652c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # os.listdir returns relative paths, so change to where we need to 10662c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette # be to avoid an os.path.join on each loop. 10672c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette logging.debug('Offloading Autotest results in %s', RESULTS_DIR) 10682c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette os.chdir(RESULTS_DIR) 10692c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette 10706fe79f0d99fc08d39a9c602d1b442b783d920ad0Aviv Keshet service_name = 'gs_offloader(%s)' % offloader_type 10716fe79f0d99fc08d39a9c602d1b442b783d920ad0Aviv Keshet with ts_mon_config.SetupTsMonGlobalState(service_name, indirect=True, 1072c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu short_lived=False): 1073c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu offloader = Offloader(options) 1074c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu if not options.delete_only: 1075c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu wait_for_gs_write_access(offloader.gs_uri) 1076c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu while True: 1077c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu offloader.offload_once() 1078c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu if options.offload_once: 1079c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu break 1080c163e21afb1bb0b72d1cd629c506645ff4425fe6Prathmesh Prabhu time.sleep(SLEEP_TIME_SECS) 1081cb2e2b789b9377262b08c185b8b96e7c758ef9e5Scott Zawalski 1082cb2e2b789b9377262b08c185b8b96e7c758ef9e5Scott Zawalski 1083b41527d6f865688299dc5250f8d58baca9432777Allen Li_LOG_LOCATION = '/usr/local/autotest/logs/' 1084b41527d6f865688299dc5250f8d58baca9432777Allen Li_LOG_FILENAME_FORMAT = 'gs_offloader_%s_log_%s.txt' 1085b41527d6f865688299dc5250f8d58baca9432777Allen Li_LOG_TIMESTAMP_FORMAT = '%Y%m%d_%H%M%S' 1086b41527d6f865688299dc5250f8d58baca9432777Allen Li_LOGGING_FORMAT = '%(asctime)s - %(levelname)s - %(message)s' 1087b41527d6f865688299dc5250f8d58baca9432777Allen Li 1088b41527d6f865688299dc5250f8d58baca9432777Allen Li 1089b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _setup_logging(options, offloader_type): 1090b41527d6f865688299dc5250f8d58baca9432777Allen Li """Set up logging. 1091b41527d6f865688299dc5250f8d58baca9432777Allen Li 1092b41527d6f865688299dc5250f8d58baca9432777Allen Li @param options: Parsed options. 1093b41527d6f865688299dc5250f8d58baca9432777Allen Li @param offloader_type: Type of offloader action as string. 1094b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 1095b41527d6f865688299dc5250f8d58baca9432777Allen Li log_filename = _get_log_filename(options, offloader_type) 1096b41527d6f865688299dc5250f8d58baca9432777Allen Li log_formatter = logging.Formatter(_LOGGING_FORMAT) 1097b41527d6f865688299dc5250f8d58baca9432777Allen Li # Replace the default logging handler with a RotatingFileHandler. If 1098b41527d6f865688299dc5250f8d58baca9432777Allen Li # options.log_size is 0, the file size will not be limited. Keeps 1099b41527d6f865688299dc5250f8d58baca9432777Allen Li # one backup just in case. 1100b41527d6f865688299dc5250f8d58baca9432777Allen Li handler = logging.handlers.RotatingFileHandler( 1101b41527d6f865688299dc5250f8d58baca9432777Allen Li log_filename, maxBytes=1024 * options.log_size, backupCount=1) 1102b41527d6f865688299dc5250f8d58baca9432777Allen Li handler.setFormatter(log_formatter) 1103b41527d6f865688299dc5250f8d58baca9432777Allen Li logger = logging.getLogger() 1104b41527d6f865688299dc5250f8d58baca9432777Allen Li logger.setLevel(logging.DEBUG) 1105b41527d6f865688299dc5250f8d58baca9432777Allen Li logger.addHandler(handler) 1106b41527d6f865688299dc5250f8d58baca9432777Allen Li 1107b41527d6f865688299dc5250f8d58baca9432777Allen Li 1108b41527d6f865688299dc5250f8d58baca9432777Allen Lidef _get_log_filename(options, offloader_type): 1109b41527d6f865688299dc5250f8d58baca9432777Allen Li """Get log filename. 1110b41527d6f865688299dc5250f8d58baca9432777Allen Li 1111b41527d6f865688299dc5250f8d58baca9432777Allen Li @param options: Parsed options. 1112b41527d6f865688299dc5250f8d58baca9432777Allen Li @param offloader_type: Type of offloader action as string. 1113b41527d6f865688299dc5250f8d58baca9432777Allen Li """ 1114b41527d6f865688299dc5250f8d58baca9432777Allen Li if options.log_size > 0: 1115b41527d6f865688299dc5250f8d58baca9432777Allen Li log_timestamp = '' 1116b41527d6f865688299dc5250f8d58baca9432777Allen Li else: 1117b41527d6f865688299dc5250f8d58baca9432777Allen Li log_timestamp = time.strftime(_LOG_TIMESTAMP_FORMAT) 1118b41527d6f865688299dc5250f8d58baca9432777Allen Li log_basename = _LOG_FILENAME_FORMAT % (offloader_type, log_timestamp) 1119b41527d6f865688299dc5250f8d58baca9432777Allen Li return os.path.join(_LOG_LOCATION, log_basename) 1120b41527d6f865688299dc5250f8d58baca9432777Allen Li 1121b41527d6f865688299dc5250f8d58baca9432777Allen Li 112220a9b58ae53932cad6a536e23bb020bfb6e84a49Scott Zawalskiif __name__ == '__main__': 11232c41e1e049dc7761bc0143172d3364bfcf848006J. Richard Barnette main() 1124