168368d75cc58205305a960d368c6fc6debd7b576Caroline Tice# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 268368d75cc58205305a960d368c6fc6debd7b576Caroline Tice# Use of this source code is governed by a BSD-style license that can be 368368d75cc58205305a960d368c6fc6debd7b576Caroline Tice# found in the LICENSE file. 468368d75cc58205305a960d368c6fc6debd7b576Caroline Ticeimport logging 54876fb68355faa7c435ef8f5dc4619f040f63988Caroline Ticeimport re 668368d75cc58205305a960d368c6fc6debd7b576Caroline Ticeimport os 768368d75cc58205305a960d368c6fc6debd7b576Caroline Ticeimport StringIO 868368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 968368d75cc58205305a960d368c6fc6debd7b576Caroline Ticeimport common 1068368d75cc58205305a960d368c6fc6debd7b576Caroline Ticefrom autotest_lib.client.common_lib import error 1168368d75cc58205305a960d368c6fc6debd7b576Caroline Ticefrom autotest_lib.server import test 1268368d75cc58205305a960d368c6fc6debd7b576Caroline Ticefrom autotest_lib.server import utils 13bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Ticefrom autotest_lib.site_utils import test_runner_utils 14bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Ticefrom autotest_lib.server.cros import telemetry_runner 1568368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 16185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 1768368d75cc58205305a960d368c6fc6debd7b576Caroline TiceTELEMETRY_TIMEOUT_MINS = 60 18185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan HuangCHROME_SRC_ROOT = '/var/cache/chromeos-cache/distfiles/target/' 19b51531db1b04a36c9b6b8ae9928847f05698409cTing-Yuan HuangCLIENT_CHROME_ROOT = '/usr/local/telemetry/src' 20b51531db1b04a36c9b6b8ae9928847f05698409cTing-Yuan HuangRUN_BENCHMARK = 'tools/perf/run_benchmark' 2168368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 22bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline TiceRSA_KEY = '-i %s' % test_runner_utils.TEST_KEY_PATH 23bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline TiceDUT_CHROME_RESULTS_DIR = '/usr/local/telemetry/src/tools/perf' 24bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice 254876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice# Result Statuses 264876fb68355faa7c435ef8f5dc4619f040f63988Caroline TiceSUCCESS_STATUS = 'SUCCESS' 274876fb68355faa7c435ef8f5dc4619f040f63988Caroline TiceWARNING_STATUS = 'WARNING' 284876fb68355faa7c435ef8f5dc4619f040f63988Caroline TiceFAILED_STATUS = 'FAILED' 294876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice 304876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice# Regex for the RESULT output lines understood by chrome buildbot. 314876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice# Keep in sync with 324876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice# chromium/tools/build/scripts/slave/performance_log_processor.py. 334876fb68355faa7c435ef8f5dc4619f040f63988Caroline TiceRESULTS_REGEX = re.compile(r'(?P<IMPORTANT>\*)?RESULT ' 344876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= ' 354876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice r'(?P<VALUE>[\{\[]?[-\d\., ]+[\}\]]?)(' 364876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice r' ?(?P<UNITS>.+))?') 374876fb68355faa7c435ef8f5dc4619f040f63988Caroline TiceHISTOGRAM_REGEX = re.compile(r'(?P<IMPORTANT>\*)?HISTOGRAM ' 384876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= ' 394876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice r'(?P<VALUE_JSON>{.*})(?P<UNITS>.+)?') 404876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice 41185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 42185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huangdef _find_chrome_root_dir(): 43185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # Look for chrome source root, either externally mounted, or inside 44185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # the chroot. Prefer chrome-src-internal source tree to chrome-src. 45185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang sources_list = ('chrome-src-internal', 'chrome-src') 46185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 47185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang dir_list = [os.path.join(CHROME_SRC_ROOT, x) for x in sources_list] 48185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang if 'CHROME_ROOT' in os.environ: 49185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang dir_list.insert(0, os.environ['CHROME_ROOT']) 50185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 51185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang for dir in dir_list: 52185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang if os.path.exists(dir): 53185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang chrome_root_dir = dir 54185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang break 55185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang else: 56185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang raise error.TestError('Chrome source directory not found.') 57185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 58185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang logging.info('Using Chrome source tree at %s', chrome_root_dir) 59b51531db1b04a36c9b6b8ae9928847f05698409cTing-Yuan Huang return os.path.join(chrome_root_dir, 'src') 60185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 61185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 62185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huangdef _ensure_deps(dut, test_name): 6368368d75cc58205305a960d368c6fc6debd7b576Caroline Tice """ 64185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang Ensure the dependencies are locally available on DUT. 6568368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 66185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang @param dut: The autotest host object representing DUT. 67185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang @param test_name: Name of the telemetry test. 6868368d75cc58205305a960d368c6fc6debd7b576Caroline Tice """ 69185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # Get DEPs using host's telemetry. 70185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang chrome_root_dir = _find_chrome_root_dir() 71b51531db1b04a36c9b6b8ae9928847f05698409cTing-Yuan Huang format_string = ('python %s/tools/perf/fetch_benchmark_deps.py %s') 72185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang command = format_string % (chrome_root_dir, test_name) 73185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang logging.info('Getting DEPs: %s', command) 74185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stdout = StringIO.StringIO() 75185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stderr = StringIO.StringIO() 76185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang try: 77185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang result = utils.run(command, stdout_tee=stdout, 78185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stderr_tee=stderr) 79185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang except error.CmdError as e: 80185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang logging.debug('Error occurred getting DEPs: %s\n %s\n', 81185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stdout.getvalue(), stderr.getvalue()) 82185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang raise error.TestFail('Error occurred while getting DEPs.') 83185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 84185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # Download DEPs to DUT. 85185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # send_file() relies on rsync over ssh. Couldn't be better. 86185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stdout_str = stdout.getvalue() 87185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stdout.close() 88185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stderr.close() 89185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang for dep in stdout_str.split(): 90185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang src = os.path.join(chrome_root_dir, dep) 91185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang dst = os.path.join(CLIENT_CHROME_ROOT, dep) 92185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang if not os.path.isfile(src): 93185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang raise error.TestFail('Error occurred while saving DEPs.') 94185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang logging.info('Copying: %s -> %s', src, dst) 95185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang try: 96185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang dut.send_file(src, dst) 97185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang except: 98185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang raise error.TestFail('Error occurred while sending DEPs to dut.\n') 9968368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 10068368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 101bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Ticeclass telemetry_Crosperf(test.test): 102bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice """Run one or more telemetry benchmarks under the crosperf script.""" 103bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice version = 1 1044876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice 105bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice def scp_telemetry_results(self, client_ip, dut): 106bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice """Copy telemetry results from dut. 1074876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice 108bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice @param client_ip: The ip address of the DUT 109bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice @param dut: The autotest host object representing DUT. 1104876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice 111bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice @returns status code for scp command. 1124876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice """ 113bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice cmd=[] 114bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice src = ('root@%s:%s/results-chart.json' % 115bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice (dut.hostname if dut else client_ip, DUT_CHROME_RESULTS_DIR)) 116bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice cmd.extend(['scp', telemetry_runner.DUT_SCP_OPTIONS, RSA_KEY, '-v', 117bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice src, self.resultsdir]) 118bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice command = ' '.join(cmd) 119bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice 120bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice logging.debug('Retrieving Results: %s', command) 121bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice try: 122bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice result = utils.run(command, 123bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice timeout=TELEMETRY_TIMEOUT_MINS * 60) 124bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice exit_code = result.exit_status 125bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice except Exception as e: 126db23d7aae74ac1e8b4e3e2b82303bee2241de949Caroline Tice logging.error('Failed to retrieve results: %s', e) 127db23d7aae74ac1e8b4e3e2b82303bee2241de949Caroline Tice raise 1284876fb68355faa7c435ef8f5dc4619f040f63988Caroline Tice 129bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice logging.debug('command return value: %d', exit_code) 130bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice return exit_code 131185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 132185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang def run_once(self, args, client_ip='', dut=None): 13368368d75cc58205305a960d368c6fc6debd7b576Caroline Tice """ 13468368d75cc58205305a960d368c6fc6debd7b576Caroline Tice Run a single telemetry test. 13568368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 13668368d75cc58205305a960d368c6fc6debd7b576Caroline Tice @param args: A dictionary of the arguments that were passed 13768368d75cc58205305a960d368c6fc6debd7b576Caroline Tice to this test. 138185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang @param client_ip: The ip address of the DUT 139185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang @param dut: The autotest host object representing DUT. 14068368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 141185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang @returns A TelemetryResult instance with the results of this execution. 14268368d75cc58205305a960d368c6fc6debd7b576Caroline Tice """ 143185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang test_name = args['test'] 144185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang test_args = '' 145d2eb716ddcc092e37eb36024c43ca7a348606d99Caroline Tice if 'test_args' in args: 146185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang test_args = args['test_args'] 147185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 148185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # Decide whether the test will run locally or by a remote server. 149db23d7aae74ac1e8b4e3e2b82303bee2241de949Caroline Tice if args.get('run_local', 'false').lower() == 'true': 150185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # The telemetry scripts will run on DUT. 151185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang _ensure_deps(dut, test_name) 152bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice format_string = ('python %s --browser=system ' 153bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice '--output-format=chartjson %s %s') 154185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang command = format_string % (os.path.join(CLIENT_CHROME_ROOT, 155185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang RUN_BENCHMARK), 156185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang test_args, test_name) 157185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang runner = dut 1584b996a37f51d98f2101d97686615e7b35115a4e9Caroline Tice else: 159185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # The telemetry scripts will run on server. 160185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang format_string = ('python %s --browser=cros-chrome --remote=%s ' 161bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice '--output-format=chartjson %s %s') 162185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang command = format_string % (os.path.join(_find_chrome_root_dir(), 163185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang RUN_BENCHMARK), 164185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang client_ip, test_args, test_name) 165185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang runner = utils 166185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 167185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang # Run the test. 168185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stdout = StringIO.StringIO() 169185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stderr = StringIO.StringIO() 17068368d75cc58205305a960d368c6fc6debd7b576Caroline Tice try: 171185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang logging.info('CMD: %s', command) 172185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang result = runner.run(command, stdout_tee=stdout, stderr_tee=stderr, 173185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang timeout=TELEMETRY_TIMEOUT_MINS*60) 17468368d75cc58205305a960d368c6fc6debd7b576Caroline Tice exit_code = result.exit_status 17568368d75cc58205305a960d368c6fc6debd7b576Caroline Tice except error.CmdError as e: 17668368d75cc58205305a960d368c6fc6debd7b576Caroline Tice logging.debug('Error occurred executing telemetry.') 17768368d75cc58205305a960d368c6fc6debd7b576Caroline Tice exit_code = e.result_obj.exit_status 178185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang raise error.TestFail('An error occurred while executing ' 179185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang 'telemetry test.') 180185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang finally: 181185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stdout_str = stdout.getvalue() 182185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stderr_str = stderr.getvalue() 183185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stdout.close() 184185702cf8f71cc1b116a30f50c866996d6381a33Ting-Yuan Huang stderr.close() 18575187bce207199265e6a2e6b12bb390ade48f785Ting-Yuan Huang logging.info('Telemetry completed with exit code: %d.' 18675187bce207199265e6a2e6b12bb390ade48f785Ting-Yuan Huang '\nstdout:%s\nstderr:%s', exit_code, 18775187bce207199265e6a2e6b12bb390ade48f785Ting-Yuan Huang stdout_str, stderr_str) 18868368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 189bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice # Copy the results-chart.json file into the test_that results 190bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice # directory. 191db23d7aae74ac1e8b4e3e2b82303bee2241de949Caroline Tice if args.get('run_local', 'false').lower() == 'true': 192bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice result = self.scp_telemetry_results(client_ip, dut) 193bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice else: 194bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice src_dir = os.path.dirname(os.path.join(_find_chrome_root_dir(), 195bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice RUN_BENCHMARK)) 196bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice 197bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice filepath = os.path.join(src_dir, 'results-chart.json') 198bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice if os.path.exists(filepath): 199bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice command = 'cp %s %s' % (filepath, self.resultsdir) 200bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice result = utils.run(command) 201bcf13d8ebbfb09634439cced6e482d354b9f4a7cCaroline Tice else: 202db23d7aae74ac1e8b4e3e2b82303bee2241de949Caroline Tice raise IOError('Missing results file: %s' % filepath) 20368368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 20468368d75cc58205305a960d368c6fc6debd7b576Caroline Tice 205b6f0c4d6418f0f790c5b1f2157908862573ab705Caroline Tice return result 206