1f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi#!/usr/bin/env python 2f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 3f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 4f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Use of this source code is governed by a BSD-style license that can be 5f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# found in the LICENSE file. 6f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 7f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Script to check the history of stage calls made to devserver. 8f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Following are some sample use cases: 9f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 10f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 1. Find all stage request for autotest and image nyan_big-release/R38-6055.0.0 11f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# in the last 10 days across all devservers. 12f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ./devserver_history.py --image_filters nyan_big 38 6055.0.0 -l 240 \ 13f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# --artifact_filters autotest -v 14f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# output: 15f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ============================================================================== 16f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 170.21.64.22 17f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ============================================================================== 18f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Number of calls: 1 19f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Number of unique images: 1 20f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 2014-08-23 12:45:00: nyan_big-release/R38-6055.0.0 autotest 21f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ============================================================================== 22f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 170.21.64.23 23f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ============================================================================== 24f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Number of calls: 2 25f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Number of unique images: 1 26f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 2014-08-23 12:45:00: nyan_big-release/R38-6055.0.0 autotest, test_suites 27f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 2014-08-23 12:55:00: nyan_big-release/R38-6055.0.0 autotest, test_suites 28f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 29f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 2. Find all duplicated stage request for the last 10 days. 30f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ./devserver_history.py -d -l 240 31f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# output: 32f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Detecting artifacts staged in multiple devservers. 33f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ============================================================================== 34f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# nyan_big-release/R38-6055.0.0 35f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# ============================================================================== 36f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 170.21.64.22: 23 requests 2014-09-04 22:44:28 -- 2014-09-05 00:03:23 37f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 170.21.64.23: 6 requests 2014-09-04 22:48:58 -- 2014-09-04 22:49:42 38f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 39f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# Count of images with duplicated stages on each devserver: 40f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 170.21.64.22 : 22 41f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi# 170.21.64.23 : 11 42f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 43f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 44f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiimport argparse 45f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiimport datetime 46f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiimport logging 47f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiimport operator 48f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiimport re 49f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiimport time 50f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shifrom itertools import groupby 51f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 52f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiimport common 53f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shifrom autotest_lib.client.common_lib import global_config 54f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shifrom autotest_lib.client.common_lib import time_utils 55b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Blackfrom autotest_lib.client.common_lib.cros.graphite import autotest_es 56f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 57f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 58f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiclass devserver_call(object): 59f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """A container to store the information of devserver stage call. 60f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """ 61f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 62f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi def __init__(self, hit): 63f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """Retrieve information from a ES query hit. 64f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """ 65b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black self.devserver = hit['devserver'] 66b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black self.subname = hit['subname'] 67b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black self.artifacts = hit['artifacts'].split(' ') 68b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black self.image = hit['image'] 69b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black self.value = hit['value'] 70f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi self.time_recorded = time_utils.epoch_time_to_date_string( 71b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black hit['time_recorded']) 72f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 73f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 74f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi def __str__(self): 75f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi pairs = ['%-20s: %s' % (attr, getattr(self, attr)) for attr in dir(self) 76f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if not attr.startswith('__') and 77f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi not callable(getattr(self, attr))] 78f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi return '\n'.join(pairs) 79f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 80f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 81f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shidef get_calls(time_start, time_end, artifact_filters=None, 82f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi regex_constraints=None, devserver=None, size=1e7): 83f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """Gets all devserver calls from es db with the given constraints. 84f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 85f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param time_start: Earliest time entry was recorded. 86f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param time_end: Latest time entry was recorded. 87f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param artifact_filters: A list of names to match artifacts. 88f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param regex_constraints: A list of regex constraints for ES query. 89f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param devserver: name of devserver to query for. If it's set to None, 90f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi return calls for all devservers. Default is set to None. 91f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param size: Max number of entries to return, default to 1 million. 92f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 93f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @returns: Entries from esdb. 94f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """ 95f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi eqs = [('_type', 'devserver')] 96f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if devserver: 97f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi eqs.append(('devserver', devserver)) 98f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if artifact_filters: 99f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for artifact in artifact_filters: 100f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi eqs.append(('artifacts', artifact)) 10155bfe14a6c2cc2710593ecf4d461af64181915c0Gabe Black time_start_epoch = time_utils.to_epoch_time(time_start) 10255bfe14a6c2cc2710593ecf4d461af64181915c0Gabe Black time_end_epoch = time_utils.to_epoch_time(time_end) 103b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black results = autotest_es.query( 104f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi fields_returned=None, 105f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi equality_constraints=eqs, 10655bfe14a6c2cc2710593ecf4d461af64181915c0Gabe Black range_constraints=[('time_recorded', time_start_epoch, 10755bfe14a6c2cc2710593ecf4d461af64181915c0Gabe Black time_end_epoch)], 108f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi size=size, 109f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi sort_specs=[{'time_recorded': 'desc'}], 110f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi regex_constraints=regex_constraints) 111f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi devserver_calls = [] 112b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Black for hit in results.hits: 113f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi devserver_calls.append(devserver_call(hit)) 114f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi logging.info('Found %d calls.', len(devserver_calls)) 115f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi return devserver_calls 116f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 117f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 118f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shidef print_call_details(calls, verbose): 119f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """Print details of each call to devserver to stage artifacts. 120f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 121f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param calls: A list of devserver stage requests. 122f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param verbose: Set to True to print out all devserver calls. 123f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """ 124f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi calls = sorted(calls, key=lambda c: c.devserver) 125f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for devserver,calls_for_devserver in groupby(calls, lambda c: c.devserver): 126f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi calls_for_devserver = list(calls_for_devserver) 127f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print '='*80 128f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print devserver 129f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print '='*80 130f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print 'Number of calls: %d' % len(calls_for_devserver) 131f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print ('Number of unique images: %d' % 132f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi len(set([call.image for call in calls_for_devserver]))) 133f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if verbose: 134f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for call in sorted(calls_for_devserver, 135f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi key=lambda c: c.time_recorded): 136f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print ('%s %s %s' % (call.time_recorded, call.image, 137f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi ', '.join(call.artifacts))) 138f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 139f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 140f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shidef detect_duplicated_stage(calls): 141f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """Detect any artifact for same build was staged in multiple devservers. 142f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 143f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi @param calls: A list of devserver stage requests. 144f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """ 145f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print '\nDetecting artifacts staged in multiple devservers.' 146f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi calls = sorted(calls, key=lambda c: c.image) 147f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi # Count how many times a devserver staged duplicated artifacts. A number 148f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi # significantly larger then others can indicate that the devserver failed 149f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi # check_health too often and needs to be removed from production. 150f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi duplicated_stage_count = {} 151f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for image,calls_for_image in groupby(calls, lambda c: c.image): 152f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi calls_for_image = list(calls_for_image) 153f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi devservers = set([call.devserver for call in calls_for_image]) 154f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if len(devservers) > 1: 155f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print '='*80 156f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print image 157f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print '='*80 158f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi calls_for_image = sorted(calls_for_image, key=lambda c: c.devserver) 159f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for devserver,calls_for_devserver in groupby(calls_for_image, 160f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi lambda c: c.devserver): 161f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi timestamps = [c.time_recorded for c in calls_for_devserver] 162f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print ('%s: %-3d requests %s -- %s' % 163f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi (devserver, len(timestamps), min(timestamps), 164f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi max(timestamps))) 165f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi duplicated_stage_count[devserver] = ( 166f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi duplicated_stage_count.get(devserver, 0) + 1) 167f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print '\nCount of images with duplicated stages on each devserver:' 168f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi counts = sorted(duplicated_stage_count.iteritems(), 169f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi key=operator.itemgetter(1), reverse=True) 170f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for k,v in counts: 171f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print '%-15s: %d' % (k, v) 172f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 173f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 174f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shidef main(): 175f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi """main script. """ 176f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi t_now = time.time() 177f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi t_now_minus_one_day = t_now - 3600 * 24 178f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser = argparse.ArgumentParser() 179f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('-l', type=float, dest='last', 180f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help='last hours to search results across', 181f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=None) 182f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('--start', type=str, dest='start', 183f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help=('Enter start time as: yyyy-mm-dd hh-mm-ss,' 184f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'defualts to 24h ago. This option is ignored when' 185f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi ' -l is used.'), 186f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=time_utils.epoch_time_to_date_string( 187f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi t_now_minus_one_day)) 188f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('--end', type=str, dest='end', 189f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,' 190f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'defualts to current time. This option is ignored' 191f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi ' when -l is used.'), 192f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=time_utils.epoch_time_to_date_string(t_now)) 193f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('--devservers', nargs='+', dest='devservers', 194f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help=('Enter space deliminated devservers. Default are' 195f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi ' all devservers specified in global config.'), 196f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=[]) 197f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('--artifact_filters', nargs='+', 198f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi dest='artifact_filters', 199f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help=('Enter space deliminated filters on artifact ' 200f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'name. For example "autotest test_suites". The ' 201f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'filter does not support regex.'), 202f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=[]) 203f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('--image_filters', nargs='+', dest='image_filters', 204f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help=('Enter space deliminated filters on image name. ' 205f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'For example "nyan 38 6566", search will use ' 206f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'regex to match each filter. Do not use filters ' 207f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'with mixed letter and number, e.g., R38.'), 208f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=[]) 209f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('-d', '--detect_duplicated_stage', action='store_true', 210f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi dest='detect_duplicated_stage', 211f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help=('Set to True to detect if an artifacts for a same' 212f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi ' build was staged in multiple devservers. ' 213f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'Default is True.'), 214f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=False) 215f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi parser.add_argument('-v', action='store_true', dest='verbose', 216f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi default=False, 217f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi help='-v to print out ALL entries.') 218f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi options = parser.parse_args() 219f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if options.verbose: 220f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi logging.getLogger().setLevel(logging.INFO) 221f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 222f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if options.last: 223f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi end_time = datetime.datetime.now() 224f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi start_time = end_time - datetime.timedelta(seconds=3600 * options.last) 225f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi else: 226f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi start_time = datetime.datetime.strptime(options.start, 227f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi time_utils.TIME_FMT) 228f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi end_time = datetime.datetime.strptime(options.end, time_utils.TIME_FMT) 229f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi logging.info('Searching devserver calls from %s to %s', start_time, 230f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi end_time) 231f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 232f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi devservers = options.devservers 233f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if not devservers: 234f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi devserver_urls = global_config.global_config.get_config_value( 235f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 'CROS', 'dev_server', type=list, default=[]) 236f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi devservers = [] 237f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for url in devserver_urls: 238f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi match = re.match('http://([^:]*):*\d*', url) 239f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi devservers.append(match.groups(0)[0] if match else url) 240f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi logging.info('Found devservers: %s', devservers) 241f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 242f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi regex_constraints = [] 243f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for filter in options.image_filters: 244f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi regex_constraints.append(('image', '.*%s.*' % filter)) 245f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi calls = [] 246f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi for devserver in devservers: 247f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi calls.extend(get_calls(start_time, end_time, options.artifact_filters, 248f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi regex_constraints, devserver=devserver)) 249f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 250f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi print_call_details(calls, options.verbose) 251f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 252f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi if options.detect_duplicated_stage: 253f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi detect_duplicated_stage(calls) 254f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 255f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi 256f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shiif __name__ == '__main__': 257f8b71d1c0d37ed681b1fc129468a9c2e2d0ecbdbDan Shi main() 258