results_cache_unittest.py revision 5775e7e3e85529f18220d5d6bc47df6c2fc79a6a
1#!/usr/bin/env python2 2 3# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Module of result cache unittest.""" 7 8from __future__ import print_function 9 10import mock 11import os 12import tempfile 13import unittest 14 15import image_checksummer 16import machine_manager 17import test_flag 18 19from label import MockLabel 20from results_cache import CacheConditions 21from results_cache import Result 22from results_cache import ResultsCache 23from results_cache import TelemetryResult 24from cros_utils import command_executer 25from cros_utils import logger 26from cros_utils import misc 27 28OUTPUT = """CMD (True): ./test_that.sh\ 29 --remote=172.17.128.241 --board=lumpy LibCBench 30CMD (None): cd /usr/local/google/home/yunlian/gd/src/build/images/lumpy/latest/../../../../..; cros_sdk -- ./in_chroot_cmd6X7Cxu.sh 31Identity added: /tmp/test_that.PO1234567/autotest_key (/tmp/test_that.PO1234567/autotest_key) 32INFO : Using emerged autotests already installed at /build/lumpy/usr/local/autotest. 33 34INFO : Running the following control files 1 times: 35INFO : * 'client/site_tests/platform_LibCBench/control' 36 37INFO : Running client test client/site_tests/platform_LibCBench/control 38./server/autoserv -m 172.17.128.241 --ssh-port 22 -c client/site_tests/platform_LibCBench/control -r /tmp/test_that.PO1234567/platform_LibCBench --test-retry=0 --args 39ERROR:root:import statsd failed, no stats will be reported. 4014:20:22 INFO | Results placed in /tmp/test_that.PO1234567/platform_LibCBench 4114:20:22 INFO | Processing control file 4214:20:23 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_VIIP67ssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 4314:20:23 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. 4414:20:23 INFO | INFO ---- ---- kernel=3.8.11 localtime=May 22 14:20:23 timestamp=1369257623 4514:20:23 INFO | Installing autotest on 172.17.128.241 4614:20:23 INFO | Using installation dir /usr/local/autotest 4714:20:23 WARNI| No job_repo_url for <remote host: 172.17.128.241> 4814:20:23 INFO | Could not install autotest using the packaging system: No repos to install an autotest client from. Trying other methods 4914:20:23 INFO | Installation of autotest completed 5014:20:24 WARNI| No job_repo_url for <remote host: 172.17.128.241> 5114:20:24 INFO | Executing /usr/local/autotest/bin/autotest /usr/local/autotest/control phase 0 5214:20:24 INFO | Entered autotestd_monitor. 5314:20:24 INFO | Finished launching tail subprocesses. 5414:20:24 INFO | Finished waiting on autotestd to start. 5514:20:26 INFO | START ---- ---- timestamp=1369257625 localtime=May 22 14:20:25 5614:20:26 INFO | START platform_LibCBench platform_LibCBench timestamp=1369257625 localtime=May 22 14:20:25 5714:20:30 INFO | GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 completed successfully 5814:20:30 INFO | END GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 5914:20:31 INFO | END GOOD ---- ---- timestamp=1369257630 localtime=May 22 14:20:30 6014:20:31 INFO | Got lock of exit_code_file. 6114:20:31 INFO | Released lock of exit_code_file and closed it. 62OUTPUT: ============================== 63OUTPUT: Current time: 2013-05-22 14:20:32.818831 Elapsed: 0:01:30 ETA: Unknown 64Done: 0% [ ] 65OUTPUT: Thread Status: 66RUNNING: 1 ('ttt: LibCBench (1)' 0:01:21) 67Machine Status: 68Machine Thread Lock Status Checksum 69172.17.128.241 ttt: LibCBench (1) True RUNNING 3ba9f2ecbb222f20887daea5583d86ba 70 71OUTPUT: ============================== 7214:20:33 INFO | Killing child processes. 7314:20:33 INFO | Client complete 7414:20:33 INFO | Finished processing control file 7514:20:33 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_aVJUgmssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 7614:20:33 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. 77 78INFO : Test results: 79------------------------------------------------------------------- 80platform_LibCBench [ PASSED ] 81platform_LibCBench/platform_LibCBench [ PASSED ] 82platform_LibCBench/platform_LibCBench b_malloc_big1__0_ 0.00375231466667 83platform_LibCBench/platform_LibCBench b_malloc_big2__0_ 0.002951359 84platform_LibCBench/platform_LibCBench b_malloc_bubble__0_ 0.015066374 85platform_LibCBench/platform_LibCBench b_malloc_sparse__0_ 0.015053784 86platform_LibCBench/platform_LibCBench b_malloc_thread_local__0_ 0.01138439 87platform_LibCBench/platform_LibCBench b_malloc_thread_stress__0_ 0.0367894733333 88platform_LibCBench/platform_LibCBench b_malloc_tiny1__0_ 0.000768474333333 89platform_LibCBench/platform_LibCBench b_malloc_tiny2__0_ 0.000581407333333 90platform_LibCBench/platform_LibCBench b_pthread_create_serial1__0_ 0.0291785246667 91platform_LibCBench/platform_LibCBench b_pthread_createjoin_serial1__0_ 0.031907936 92platform_LibCBench/platform_LibCBench b_pthread_createjoin_serial2__0_ 0.043485347 93platform_LibCBench/platform_LibCBench b_pthread_uselesslock__0_ 0.0294113346667 94platform_LibCBench/platform_LibCBench b_regex_compile____a_b_c__d_b__ 0.00529833933333 95platform_LibCBench/platform_LibCBench b_regex_search____a_b_c__d_b__ 0.00165455066667 96platform_LibCBench/platform_LibCBench b_regex_search___a_25_b__ 0.0496191923333 97platform_LibCBench/platform_LibCBench b_stdio_putcgetc__0_ 0.100005711667 98platform_LibCBench/platform_LibCBench b_stdio_putcgetc_unlocked__0_ 0.0371443833333 99platform_LibCBench/platform_LibCBench b_string_memset__0_ 0.00275405066667 100platform_LibCBench/platform_LibCBench b_string_strchr__0_ 0.00456903 101platform_LibCBench/platform_LibCBench b_string_strlen__0_ 0.044893587 102platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac__ 0.118360778 103platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaac__ 0.068957325 104platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaacccccccccccc__ 0.0135694476667 105platform_LibCBench/platform_LibCBench b_string_strstr___abcdefghijklmnopqrstuvwxyz__ 0.0134553343333 106platform_LibCBench/platform_LibCBench b_string_strstr___azbycxdwevfugthsirjqkplomn__ 0.0133123556667 107platform_LibCBench/platform_LibCBench b_utf8_bigbuf__0_ 0.0473772253333 108platform_LibCBench/platform_LibCBench b_utf8_onebyone__0_ 0.130938538333 109------------------------------------------------------------------- 110Total PASS: 2/2 (100%) 111 112INFO : Elapsed time: 0m16s 113""" 114 115error = """ 116ERROR: Identity added: /tmp/test_that.Z4Ld/autotest_key (/tmp/test_that.Z4Ld/autotest_key) 117INFO : Using emerged autotests already installed at /build/lumpy/usr/local/autotest. 118INFO : Running the following control files 1 times: 119INFO : * 'client/site_tests/platform_LibCBench/control' 120INFO : Running client test client/site_tests/platform_LibCBench/control 121INFO : Test results: 122INFO : Elapsed time: 0m18s 123""" 124 125keyvals = { 126 '': 'PASS', 127 'b_stdio_putcgetc__0_': '0.100005711667', 128 'b_string_strstr___azbycxdwevfugthsirjqkplomn__': '0.0133123556667', 129 'b_malloc_thread_local__0_': '0.01138439', 130 'b_string_strlen__0_': '0.044893587', 131 'b_malloc_sparse__0_': '0.015053784', 132 'b_string_memset__0_': '0.00275405066667', 133 'platform_LibCBench': 'PASS', 134 'b_pthread_uselesslock__0_': '0.0294113346667', 135 'b_string_strchr__0_': '0.00456903', 136 'b_pthread_create_serial1__0_': '0.0291785246667', 137 'b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac__': '0.118360778', 138 'b_string_strstr___aaaaaaaaaaaaaacccccccccccc__': '0.0135694476667', 139 'b_pthread_createjoin_serial1__0_': '0.031907936', 140 'b_malloc_thread_stress__0_': '0.0367894733333', 141 'b_regex_search____a_b_c__d_b__': '0.00165455066667', 142 'b_malloc_bubble__0_': '0.015066374', 143 'b_malloc_big2__0_': '0.002951359', 144 'b_stdio_putcgetc_unlocked__0_': '0.0371443833333', 145 'b_pthread_createjoin_serial2__0_': '0.043485347', 146 'b_regex_search___a_25_b__': '0.0496191923333', 147 'b_utf8_bigbuf__0_': '0.0473772253333', 148 'b_malloc_big1__0_': '0.00375231466667', 149 'b_regex_compile____a_b_c__d_b__': '0.00529833933333', 150 'b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaac__': '0.068957325', 151 'b_malloc_tiny2__0_': '0.000581407333333', 152 'b_utf8_onebyone__0_': '0.130938538333', 153 'b_malloc_tiny1__0_': '0.000768474333333', 154 'b_string_strstr___abcdefghijklmnopqrstuvwxyz__': '0.0134553343333' 155} 156 157TMP_DIR1 = '/tmp/tmpAbcXyz' 158 159 160class MockResult(Result): 161 """Mock result class.""" 162 163 def __init__(self, mylogger, label, logging_level, machine): 164 super(MockResult, self).__init__(mylogger, label, logging_level, machine) 165 166 def FindFilesInResultsDir(self, find_args): 167 return '' 168 169 # pylint: disable=arguments-differ 170 def GetKeyvals(self, temp=False): 171 if temp: 172 pass 173 return keyvals 174 175 176class ResultTest(unittest.TestCase): 177 """Result test class.""" 178 179 def __init__(self, *args, **kwargs): 180 super(ResultTest, self).__init__(*args, **kwargs) 181 self.callFakeProcessResults = False 182 self.fakeCacheReturnResult = None 183 self.callGetResultsDir = False 184 self.callProcessResults = False 185 self.callGetPerfReportFiles = False 186 self.kv_dict = None 187 self.tmpdir = '' 188 self.callGetNewKeyvals = False 189 self.callGetResultsFile = False 190 self.callGetPerfDataFiles = False 191 self.args = None 192 self.callGatherPerfResults = False 193 self.mock_logger = mock.Mock(spec=logger.Logger) 194 self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 195 self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', 196 '/tmp', 'lumpy', 'remote', 'image_args', 197 'cache_dir', 'average', 'gcc', None) 198 199 def testCreateFromRun(self): 200 result = MockResult.CreateFromRun(logger.GetLogger(), 'average', 201 self.mock_label, 'remote1', OUTPUT, error, 202 0, True, 0) 203 self.assertEqual(result.keyvals, keyvals) 204 self.assertEqual(result.chroot_results_dir, 205 '/tmp/test_that.PO1234567/platform_LibCBench') 206 self.assertEqual(result.results_dir, 207 '/tmp/chroot/tmp/test_that.PO1234567/platform_LibCBench') 208 self.assertEqual(result.retval, 0) 209 210 def setUp(self): 211 self.result = Result(self.mock_logger, self.mock_label, 'average', 212 self.mock_cmd_exec) 213 214 @mock.patch.object(os.path, 'isdir') 215 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 216 @mock.patch.object(command_executer.CommandExecuter, 'CopyFiles') 217 def test_copy_files_to(self, mock_copyfiles, mock_runcmd, mock_isdir): 218 219 files = ['src_file_1', 'src_file_2', 'src_file_3'] 220 dest_dir = '/tmp/test' 221 self.mock_cmd_exec.RunCommand = mock_runcmd 222 self.mock_cmd_exec.CopyFiles = mock_copyfiles 223 224 mock_copyfiles.return_value = 0 225 226 #test 1. dest_dir exists; CopyFiles returns 0. 227 mock_isdir.return_value = True 228 self.result.CopyFilesTo(dest_dir, files) 229 self.assertEqual(mock_runcmd.call_count, 0) 230 self.assertEqual(mock_copyfiles.call_count, 3) 231 first_args = mock_copyfiles.call_args_list[0][0] 232 second_args = mock_copyfiles.call_args_list[1][0] 233 third_args = mock_copyfiles.call_args_list[2][0] 234 self.assertEqual(first_args, ('src_file_1', '/tmp/test/src_file_1.0')) 235 self.assertEqual(second_args, ('src_file_2', '/tmp/test/src_file_2.0')) 236 self.assertEqual(third_args, ('src_file_3', '/tmp/test/src_file_3.0')) 237 238 mock_runcmd.reset_mock() 239 mock_copyfiles.reset_mock() 240 #test 2. dest_dir does not exist; CopyFiles returns 0. 241 mock_isdir.return_value = False 242 self.result.CopyFilesTo(dest_dir, files) 243 self.assertEqual(mock_runcmd.call_count, 3) 244 self.assertEqual(mock_copyfiles.call_count, 3) 245 self.assertEqual(mock_runcmd.call_args_list[0], 246 mock_runcmd.call_args_list[1]) 247 self.assertEqual(mock_runcmd.call_args_list[0], 248 mock_runcmd.call_args_list[2]) 249 self.assertEqual(mock_runcmd.call_args_list[0][0], ('mkdir -p /tmp/test',)) 250 251 #test 3. CopyFiles returns 1 (fails). 252 mock_copyfiles.return_value = 1 253 self.assertRaises(Exception, self.result.CopyFilesTo, dest_dir, files) 254 255 @mock.patch.object(Result, 'CopyFilesTo') 256 def test_copy_results_to(self, mockCopyFilesTo): 257 perf_data_files = [ 258 '/tmp/perf.data.0', '/tmp/perf.data.1', '/tmp/perf.data.2' 259 ] 260 perf_report_files = [ 261 '/tmp/perf.report.0', '/tmp/perf.report.1', '/tmp/perf.report.2' 262 ] 263 264 self.result.perf_data_files = perf_data_files 265 self.result.perf_report_files = perf_report_files 266 267 self.result.CopyFilesTo = mockCopyFilesTo 268 self.result.CopyResultsTo('/tmp/results/') 269 self.assertEqual(mockCopyFilesTo.call_count, 2) 270 self.assertEqual(len(mockCopyFilesTo.call_args_list), 2) 271 self.assertEqual(mockCopyFilesTo.call_args_list[0][0], 272 ('/tmp/results/', perf_data_files)) 273 self.assertEqual(mockCopyFilesTo.call_args_list[1][0], 274 ('/tmp/results/', perf_report_files)) 275 276 def test_get_new_keyvals(self): 277 kv_dict = {} 278 279 def FakeGetDataMeasurementsFiles(): 280 filename = os.path.join(os.getcwd(), 'unittest_keyval_file.txt') 281 return [filename] 282 283 self.result.GetDataMeasurementsFiles = FakeGetDataMeasurementsFiles 284 kv_dict2, udict = self.result.GetNewKeyvals(kv_dict) 285 self.assertEqual(kv_dict2, { 286 u'Box2D__Box2D': 4775, 287 u'Mandreel__Mandreel': 6620, 288 u'Gameboy__Gameboy': 9901, 289 u'Crypto__Crypto': 8737, 290 u'telemetry_page_measurement_results__num_errored': 0, 291 u'telemetry_page_measurement_results__num_failed': 0, 292 u'PdfJS__PdfJS': 6455, 293 u'Total__Score': 7918, 294 u'EarleyBoyer__EarleyBoyer': 14340, 295 u'MandreelLatency__MandreelLatency': 5188, 296 u'CodeLoad__CodeLoad': 6271, 297 u'DeltaBlue__DeltaBlue': 14401, 298 u'Typescript__Typescript': 9815, 299 u'SplayLatency__SplayLatency': 7653, 300 u'zlib__zlib': 16094, 301 u'Richards__Richards': 10358, 302 u'RegExp__RegExp': 1765, 303 u'NavierStokes__NavierStokes': 9815, 304 u'Splay__Splay': 4425, 305 u'RayTrace__RayTrace': 16600 306 }) 307 self.assertEqual(udict, { 308 u'Box2D__Box2D': u'score', 309 u'Mandreel__Mandreel': u'score', 310 u'Gameboy__Gameboy': u'score', 311 u'Crypto__Crypto': u'score', 312 u'telemetry_page_measurement_results__num_errored': u'count', 313 u'telemetry_page_measurement_results__num_failed': u'count', 314 u'PdfJS__PdfJS': u'score', 315 u'Total__Score': u'score', 316 u'EarleyBoyer__EarleyBoyer': u'score', 317 u'MandreelLatency__MandreelLatency': u'score', 318 u'CodeLoad__CodeLoad': u'score', 319 u'DeltaBlue__DeltaBlue': u'score', 320 u'Typescript__Typescript': u'score', 321 u'SplayLatency__SplayLatency': u'score', 322 u'zlib__zlib': u'score', 323 u'Richards__Richards': u'score', 324 u'RegExp__RegExp': u'score', 325 u'NavierStokes__NavierStokes': u'score', 326 u'Splay__Splay': u'score', 327 u'RayTrace__RayTrace': u'score' 328 }) 329 330 def test_append_telemetry_units(self): 331 kv_dict = { 332 u'Box2D__Box2D': 4775, 333 u'Mandreel__Mandreel': 6620, 334 u'Gameboy__Gameboy': 9901, 335 u'Crypto__Crypto': 8737, 336 u'PdfJS__PdfJS': 6455, 337 u'Total__Score': 7918, 338 u'EarleyBoyer__EarleyBoyer': 14340, 339 u'MandreelLatency__MandreelLatency': 5188, 340 u'CodeLoad__CodeLoad': 6271, 341 u'DeltaBlue__DeltaBlue': 14401, 342 u'Typescript__Typescript': 9815, 343 u'SplayLatency__SplayLatency': 7653, 344 u'zlib__zlib': 16094, 345 u'Richards__Richards': 10358, 346 u'RegExp__RegExp': 1765, 347 u'NavierStokes__NavierStokes': 9815, 348 u'Splay__Splay': 4425, 349 u'RayTrace__RayTrace': 16600 350 } 351 units_dict = { 352 u'Box2D__Box2D': u'score', 353 u'Mandreel__Mandreel': u'score', 354 u'Gameboy__Gameboy': u'score', 355 u'Crypto__Crypto': u'score', 356 u'PdfJS__PdfJS': u'score', 357 u'Total__Score': u'score', 358 u'EarleyBoyer__EarleyBoyer': u'score', 359 u'MandreelLatency__MandreelLatency': u'score', 360 u'CodeLoad__CodeLoad': u'score', 361 u'DeltaBlue__DeltaBlue': u'score', 362 u'Typescript__Typescript': u'score', 363 u'SplayLatency__SplayLatency': u'score', 364 u'zlib__zlib': u'score', 365 u'Richards__Richards': u'score', 366 u'RegExp__RegExp': u'score', 367 u'NavierStokes__NavierStokes': u'score', 368 u'Splay__Splay': u'score', 369 u'RayTrace__RayTrace': u'score' 370 } 371 372 results_dict = self.result.AppendTelemetryUnits(kv_dict, units_dict) 373 self.assertEqual(results_dict, { 374 u'Box2D__Box2D': [4775, u'score'], 375 u'Splay__Splay': [4425, u'score'], 376 u'Gameboy__Gameboy': [9901, u'score'], 377 u'Crypto__Crypto': [8737, u'score'], 378 u'PdfJS__PdfJS': [6455, u'score'], 379 u'Total__Score': [7918, u'score'], 380 u'EarleyBoyer__EarleyBoyer': [14340, u'score'], 381 u'MandreelLatency__MandreelLatency': [5188, u'score'], 382 u'DeltaBlue__DeltaBlue': [14401, u'score'], 383 u'SplayLatency__SplayLatency': [7653, u'score'], 384 u'Mandreel__Mandreel': [6620, u'score'], 385 u'Richards__Richards': [10358, u'score'], 386 u'zlib__zlib': [16094, u'score'], 387 u'CodeLoad__CodeLoad': [6271, u'score'], 388 u'Typescript__Typescript': [9815, u'score'], 389 u'RegExp__RegExp': [1765, u'score'], 390 u'RayTrace__RayTrace': [16600, u'score'], 391 u'NavierStokes__NavierStokes': [9815, u'score'] 392 }) 393 394 @mock.patch.object(misc, 'GetInsideChrootPath') 395 @mock.patch.object(tempfile, 'mkdtemp') 396 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 397 @mock.patch.object(command_executer.CommandExecuter, 398 'ChrootRunCommandWOutput') 399 def test_get_keyvals(self, mock_chrootruncmd, mock_runcmd, mock_mkdtemp, 400 mock_getpath): 401 402 self.kv_dict = {} 403 self.callGetNewKeyvals = False 404 405 def reset(): 406 self.kv_dict = {} 407 self.callGetNewKeyvals = False 408 mock_chrootruncmd.reset_mock() 409 mock_runcmd.reset_mock() 410 mock_mkdtemp.reset_mock() 411 mock_getpath.reset_mock() 412 413 def FakeGetNewKeyvals(kv_dict): 414 self.kv_dict = kv_dict 415 self.callGetNewKeyvals = True 416 return_kvdict = {'first_time': 680, 'Total': 10} 417 return_udict = {'first_time': 'ms', 'Total': 'score'} 418 return return_kvdict, return_udict 419 420 mock_mkdtemp.return_value = TMP_DIR1 421 mock_chrootruncmd.return_value = [ 422 '', ('%s,PASS\n%s/telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), '' 423 ] 424 mock_getpath.return_value = TMP_DIR1 425 self.result.ce.ChrootRunCommandWOutput = mock_chrootruncmd 426 self.result.ce.RunCommand = mock_runcmd 427 self.result.GetNewKeyvals = FakeGetNewKeyvals 428 self.result.suite = 'telemetry_Crosperf' 429 self.result.results_dir = '/tmp/test_that_resultsNmq' 430 431 # Test 1. no self.temp_dir. 432 res = self.result.GetKeyvals() 433 self.assertTrue(self.callGetNewKeyvals) 434 self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) 435 self.assertEqual(mock_runcmd.call_count, 1) 436 self.assertEqual(mock_runcmd.call_args_list[0][0], 437 ('cp -r /tmp/test_that_resultsNmq/* %s' % TMP_DIR1,)) 438 self.assertEqual(mock_chrootruncmd.call_count, 1) 439 self.assertEqual(mock_chrootruncmd.call_args_list[0][0], ( 440 '/tmp', ('python generate_test_report --no-color --csv %s') % TMP_DIR1)) 441 self.assertEqual(mock_getpath.call_count, 1) 442 self.assertEqual(mock_mkdtemp.call_count, 1) 443 self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) 444 445 # Test 2. self.temp_dir 446 reset() 447 mock_chrootruncmd.return_value = [ 448 '', ('/tmp/tmpJCajRG,PASS\n/tmp/tmpJCajRG/' 449 'telemetry_Crosperf,PASS\n'), '' 450 ] 451 mock_getpath.return_value = '/tmp/tmpJCajRG' 452 self.result.temp_dir = '/tmp/tmpJCajRG' 453 res = self.result.GetKeyvals() 454 self.assertEqual(mock_runcmd.call_count, 0) 455 self.assertEqual(mock_mkdtemp.call_count, 0) 456 self.assertEqual(mock_chrootruncmd.call_count, 1) 457 self.assertTrue(self.callGetNewKeyvals) 458 self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) 459 self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) 460 461 # Test 3. suite != telemetry_Crosperf. Normally this would be for 462 # running non-Telemetry autotests, such as BootPerfServer. In this test 463 # case, the keyvals we have set up were returned from a Telemetry test run; 464 # so this pass is basically testing that we don't append the units to the 465 # test results (which we do for Telemetry autotest runs). 466 reset() 467 self.result.suite = '' 468 res = self.result.GetKeyvals() 469 self.assertEqual(res, {'Total': 10, 'first_time': 680}) 470 471 def test_get_results_dir(self): 472 473 self.result.out = '' 474 self.assertRaises(Exception, self.result.GetResultsDir) 475 476 self.result.out = OUTPUT 477 resdir = self.result.GetResultsDir() 478 self.assertEqual(resdir, '/tmp/test_that.PO1234567/platform_LibCBench') 479 480 @mock.patch.object(command_executer.CommandExecuter, 'RunCommandGeneric') 481 def test_find_files_in_results_dir(self, mock_runcmd): 482 483 self.result.results_dir = None 484 res = self.result.FindFilesInResultsDir('-name perf.data') 485 self.assertIsNone(res) 486 487 self.result.ce.RunCommand = mock_runcmd 488 self.result.results_dir = '/tmp/test_results' 489 mock_runcmd.return_value = [0, '/tmp/test_results/perf.data', ''] 490 res = self.result.FindFilesInResultsDir('-name perf.data') 491 self.assertEqual(mock_runcmd.call_count, 1) 492 self.assertEqual(mock_runcmd.call_args_list[0][0], 493 ('find /tmp/test_results -name perf.data',)) 494 self.assertEqual(res, '/tmp/test_results/perf.data') 495 496 mock_runcmd.reset_mock() 497 mock_runcmd.return_value = [1, '', ''] 498 self.assertRaises(Exception, self.result.FindFilesInResultsDir, 499 '-name perf.data') 500 501 @mock.patch.object(Result, 'FindFilesInResultsDir') 502 def test_get_perf_data_files(self, mock_findfiles): 503 self.args = None 504 505 mock_findfiles.return_value = 'line1\nline1\n' 506 self.result.FindFilesInResultsDir = mock_findfiles 507 res = self.result.GetPerfDataFiles() 508 self.assertEqual(res, ['line1', 'line1']) 509 self.assertEqual(mock_findfiles.call_args_list[0][0], ('-name perf.data',)) 510 511 def test_get_perf_report_files(self): 512 self.args = None 513 514 def FakeFindFiles(find_args): 515 self.args = find_args 516 return 'line1\nline1\n' 517 518 self.result.FindFilesInResultsDir = FakeFindFiles 519 res = self.result.GetPerfReportFiles() 520 self.assertEqual(res, ['line1', 'line1']) 521 self.assertEqual(self.args, '-name perf.data.report') 522 523 def test_get_data_measurement_files(self): 524 self.args = None 525 526 def FakeFindFiles(find_args): 527 self.args = find_args 528 return 'line1\nline1\n' 529 530 self.result.FindFilesInResultsDir = FakeFindFiles 531 res = self.result.GetDataMeasurementsFiles() 532 self.assertEqual(res, ['line1', 'line1']) 533 self.assertEqual(self.args, '-name perf_measurements') 534 535 @mock.patch.object(misc, 'GetInsideChrootPath') 536 @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') 537 def test_generate_perf_report_files(self, mock_chrootruncmd, mock_getpath): 538 fake_file = '/usr/chromeos/chroot/tmp/results/fake_file' 539 self.result.perf_data_files = ['/tmp/results/perf.data'] 540 self.result.board = 'lumpy' 541 mock_getpath.return_value = fake_file 542 self.result.ce.ChrootRunCommand = mock_chrootruncmd 543 tmp = self.result.GeneratePerfReportFiles() 544 self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file]) 545 self.assertEqual(mock_chrootruncmd.call_args_list[0][0], 546 ('/tmp', 547 ('/usr/sbin/perf report -n --symfs /build/lumpy ' 548 '--vmlinux /build/lumpy/usr/lib/debug/boot/vmlinux ' 549 '--kallsyms /build/lumpy/boot/System.map-* -i ' 550 '%s --stdio > %s') % (fake_file, fake_file))) 551 552 @mock.patch.object(misc, 'GetOutsideChrootPath') 553 def test_populate_from_run(self, mock_getpath): 554 555 def FakeGetResultsDir(): 556 self.callGetResultsDir = True 557 return '/tmp/results_dir' 558 559 def FakeGetResultsFile(): 560 self.callGetResultsFile = True 561 return [] 562 563 def FakeGetPerfDataFiles(): 564 self.callGetPerfDataFiles = True 565 return [] 566 567 def FakeGetPerfReportFiles(): 568 self.callGetPerfReportFiles = True 569 return [] 570 571 def FakeProcessResults(show_results=False): 572 if show_results: 573 pass 574 self.callProcessResults = True 575 576 if mock_getpath: 577 pass 578 mock.get_path = '/tmp/chromeos/tmp/results_dir' 579 self.result.chromeos_root = '/tmp/chromeos' 580 581 self.callGetResultsDir = False 582 self.callGetResultsFile = False 583 self.callGetPerfDataFiles = False 584 self.callGetPerfReportFiles = False 585 self.callProcessResults = False 586 587 self.result.GetResultsDir = FakeGetResultsDir 588 self.result.GetResultsFile = FakeGetResultsFile 589 self.result.GetPerfDataFiles = FakeGetPerfDataFiles 590 self.result.GeneratePerfReportFiles = FakeGetPerfReportFiles 591 self.result.ProcessResults = FakeProcessResults 592 593 self.result.PopulateFromRun(OUTPUT, '', 0, 'test', 'telemetry_Crosperf') 594 self.assertTrue(self.callGetResultsDir) 595 self.assertTrue(self.callGetResultsFile) 596 self.assertTrue(self.callGetPerfDataFiles) 597 self.assertTrue(self.callGetPerfReportFiles) 598 self.assertTrue(self.callProcessResults) 599 600 def test_process_results(self): 601 602 def FakeGetKeyvals(show_all=False): 603 if show_all: 604 return {'first_time': 680, 'Total': 10} 605 else: 606 return {'Total': 10} 607 608 def FakeGatherPerfResults(): 609 self.callGatherPerfResults = True 610 611 self.callGatherPerfResults = False 612 613 self.result.GetKeyvals = FakeGetKeyvals 614 self.result.GatherPerfResults = FakeGatherPerfResults 615 616 self.result.retval = 0 617 self.result.ProcessResults() 618 self.assertTrue(self.callGatherPerfResults) 619 self.assertEqual(len(self.result.keyvals), 2) 620 self.assertEqual(self.result.keyvals, {'Total': 10, 'retval': 0}) 621 622 self.result.retval = 1 623 self.result.ProcessResults() 624 self.assertEqual(len(self.result.keyvals), 2) 625 self.assertEqual(self.result.keyvals, {'Total': 10, 'retval': 1}) 626 627 @mock.patch.object(misc, 'GetInsideChrootPath') 628 @mock.patch.object(command_executer.CommandExecuter, 629 'ChrootRunCommandWOutput') 630 def test_populate_from_cache_dir(self, mock_runchrootcmd, mock_getpath): 631 632 # pylint: disable=redefined-builtin 633 def FakeMkdtemp(dir=None): 634 if dir: 635 pass 636 return self.tmpdir 637 638 current_path = os.getcwd() 639 cache_dir = os.path.join(current_path, 'test_cache/test_input') 640 self.result.ce = command_executer.GetCommandExecuter(log_level='average') 641 self.result.ce.ChrootRunCommandWOutput = mock_runchrootcmd 642 mock_runchrootcmd.return_value = [ 643 '', ('%s,PASS\n%s/\telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), 644 '' 645 ] 646 mock_getpath.return_value = TMP_DIR1 647 self.tmpdir = tempfile.mkdtemp() 648 save_real_mkdtemp = tempfile.mkdtemp 649 tempfile.mkdtemp = FakeMkdtemp 650 651 self.result.PopulateFromCacheDir(cache_dir, 'sunspider', 652 'telemetry_Crosperf') 653 self.assertEqual(self.result.keyvals, { 654 u'Total__Total': [444.0, u'ms'], 655 u'regexp-dna__regexp-dna': [16.2, u'ms'], 656 u'telemetry_page_measurement_results__num_failed': [0, u'count'], 657 u'telemetry_page_measurement_results__num_errored': [0, u'count'], 658 u'string-fasta__string-fasta': [23.2, u'ms'], 659 u'crypto-sha1__crypto-sha1': [11.6, u'ms'], 660 u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': [3.2, u'ms'], 661 u'access-nsieve__access-nsieve': [7.9, u'ms'], 662 u'bitops-nsieve-bits__bitops-nsieve-bits': [9.4, u'ms'], 663 u'string-validate-input__string-validate-input': [19.3, u'ms'], 664 u'3d-raytrace__3d-raytrace': [24.7, u'ms'], 665 u'3d-cube__3d-cube': [28.0, u'ms'], 666 u'string-unpack-code__string-unpack-code': [46.7, u'ms'], 667 u'date-format-tofte__date-format-tofte': [26.3, u'ms'], 668 u'math-partial-sums__math-partial-sums': [22.0, u'ms'], 669 '\telemetry_Crosperf': ['PASS', ''], 670 u'crypto-aes__crypto-aes': [15.2, u'ms'], 671 u'bitops-bitwise-and__bitops-bitwise-and': [8.4, u'ms'], 672 u'crypto-md5__crypto-md5': [10.5, u'ms'], 673 u'string-tagcloud__string-tagcloud': [52.8, u'ms'], 674 u'access-nbody__access-nbody': [8.5, u'ms'], 675 'retval': 0, 676 u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'], 677 u'math-cordic__math-cordic': [8.7, u'ms'], 678 u'access-binary-trees__access-binary-trees': [4.5, u'ms'], 679 u'controlflow-recursive__controlflow-recursive': [4.4, u'ms'], 680 u'access-fannkuch__access-fannkuch': [17.8, u'ms'], 681 u'string-base64__string-base64': [16.0, u'ms'], 682 u'date-format-xparb__date-format-xparb': [20.9, u'ms'], 683 u'3d-morph__3d-morph': [22.1, u'ms'], 684 u'bitops-bits-in-byte__bitops-bits-in-byte': [9.1, u'ms'] 685 }) 686 687 # Clean up after test. 688 tempfile.mkdtemp = save_real_mkdtemp 689 command = 'rm -Rf %s' % self.tmpdir 690 self.result.ce.RunCommand(command) 691 692 @mock.patch.object(misc, 'GetRoot') 693 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 694 def test_cleanup(self, mock_runcmd, mock_getroot): 695 696 # Test 1. 'rm_chroot_tmp' is True; self.results_dir exists; 697 # self.temp_dir exists; results_dir name contains 'test_that_results_'. 698 mock_getroot.return_value = ['/tmp/tmp_AbcXyz', 'test_that_results_fake'] 699 self.result.ce.RunCommand = mock_runcmd 700 self.result.results_dir = 'test_results_dir' 701 self.result.temp_dir = 'testtemp_dir' 702 self.result.CleanUp(True) 703 self.assertEqual(mock_getroot.call_count, 1) 704 self.assertEqual(mock_runcmd.call_count, 2) 705 self.assertEqual(mock_runcmd.call_args_list[0][0], 706 ('rm -rf test_results_dir',)) 707 self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) 708 709 # Test 2. Same, except ath results_dir name does not contain 710 # 'test_that_results_' 711 mock_getroot.reset_mock() 712 mock_runcmd.reset_mock() 713 mock_getroot.return_value = ['/tmp/tmp_AbcXyz', 'other_results_fake'] 714 self.result.ce.RunCommand = mock_runcmd 715 self.result.results_dir = 'test_results_dir' 716 self.result.temp_dir = 'testtemp_dir' 717 self.result.CleanUp(True) 718 self.assertEqual(mock_getroot.call_count, 1) 719 self.assertEqual(mock_runcmd.call_count, 2) 720 self.assertEqual(mock_runcmd.call_args_list[0][0], 721 ('rm -rf /tmp/tmp_AbcXyz',)) 722 self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) 723 724 # Test 3. mock_getroot returns nothing; 'rm_chroot_tmp' is False. 725 mock_getroot.reset_mock() 726 mock_runcmd.reset_mock() 727 self.result.CleanUp(False) 728 self.assertEqual(mock_getroot.call_count, 0) 729 self.assertEqual(mock_runcmd.call_count, 1) 730 self.assertEqual(mock_runcmd.call_args_list[0][0], ('rm -rf testtemp_dir',)) 731 732 # Test 4. 'rm_chroot_tmp' is True, but result_dir & temp_dir are None. 733 mock_getroot.reset_mock() 734 mock_runcmd.reset_mock() 735 self.result.results_dir = None 736 self.result.temp_dir = None 737 self.result.CleanUp(True) 738 self.assertEqual(mock_getroot.call_count, 0) 739 self.assertEqual(mock_runcmd.call_count, 0) 740 741 @mock.patch.object(misc, 'GetInsideChrootPath') 742 @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') 743 def test_store_to_cache_dir(self, mock_chrootruncmd, mock_getpath): 744 745 def FakeMkdtemp(directory=''): 746 if directory: 747 pass 748 return self.tmpdir 749 750 if mock_chrootruncmd or mock_getpath: 751 pass 752 current_path = os.getcwd() 753 cache_dir = os.path.join(current_path, 'test_cache/test_output') 754 755 self.result.ce = command_executer.GetCommandExecuter(log_level='average') 756 self.result.out = OUTPUT 757 self.result.err = error 758 self.result.retval = 0 759 self.tmpdir = tempfile.mkdtemp() 760 if not os.path.exists(self.tmpdir): 761 os.makedirs(self.tmpdir) 762 self.result.results_dir = os.path.join(os.getcwd(), 'test_cache') 763 save_real_mkdtemp = tempfile.mkdtemp 764 tempfile.mkdtemp = FakeMkdtemp 765 766 mock_mm = machine_manager.MockMachineManager('/tmp/chromeos_root', 0, 767 'average', '') 768 mock_mm.machine_checksum_string['mock_label'] = 'fake_machine_checksum123' 769 770 mock_keylist = ['key1', 'key2', 'key3'] 771 test_flag.SetTestMode(True) 772 self.result.StoreToCacheDir(cache_dir, mock_mm, mock_keylist) 773 774 # Check that the correct things were written to the 'cache'. 775 test_dir = os.path.join(os.getcwd(), 'test_cache/test_output') 776 base_dir = os.path.join(os.getcwd(), 'test_cache/compare_output') 777 self.assertTrue(os.path.exists(os.path.join(test_dir, 'autotest.tbz2'))) 778 self.assertTrue(os.path.exists(os.path.join(test_dir, 'machine.txt'))) 779 self.assertTrue(os.path.exists(os.path.join(test_dir, 'results.txt'))) 780 781 f1 = os.path.join(test_dir, 'machine.txt') 782 f2 = os.path.join(base_dir, 'machine.txt') 783 cmd = 'diff %s %s' % (f1, f2) 784 [_, out, _] = self.result.ce.RunCommandWOutput(cmd) 785 self.assertEqual(len(out), 0) 786 787 f1 = os.path.join(test_dir, 'results.txt') 788 f2 = os.path.join(base_dir, 'results.txt') 789 cmd = 'diff %s %s' % (f1, f2) 790 [_, out, _] = self.result.ce.RunCommandWOutput(cmd) 791 self.assertEqual(len(out), 0) 792 793 # Clean up after test. 794 tempfile.mkdtemp = save_real_mkdtemp 795 command = 'rm %s/*' % test_dir 796 self.result.ce.RunCommand(command) 797 798 799TELEMETRY_RESULT_KEYVALS = { 800 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 801 'math-cordic (ms)': 802 '11.4', 803 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 804 'access-nbody (ms)': 805 '6.9', 806 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 807 'access-fannkuch (ms)': 808 '26.3', 809 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 810 'math-spectral-norm (ms)': 811 '6.3', 812 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 813 'bitops-nsieve-bits (ms)': 814 '9.3', 815 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 816 'math-partial-sums (ms)': 817 '32.8', 818 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 819 'regexp-dna (ms)': 820 '16.1', 821 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 822 '3d-cube (ms)': 823 '42.7', 824 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 825 'crypto-md5 (ms)': 826 '10.8', 827 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 828 'crypto-sha1 (ms)': 829 '12.4', 830 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 831 'string-tagcloud (ms)': 832 '47.2', 833 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 834 'string-fasta (ms)': 835 '36.3', 836 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 837 'access-binary-trees (ms)': 838 '7.3', 839 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 840 'date-format-xparb (ms)': 841 '138.1', 842 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 843 'crypto-aes (ms)': 844 '19.2', 845 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 846 'Total (ms)': 847 '656.5', 848 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 849 'string-base64 (ms)': 850 '17.5', 851 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 852 'string-validate-input (ms)': 853 '24.8', 854 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 855 '3d-raytrace (ms)': 856 '28.7', 857 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 858 'controlflow-recursive (ms)': 859 '5.3', 860 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 861 'bitops-bits-in-byte (ms)': 862 '9.8', 863 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 864 '3d-morph (ms)': 865 '50.2', 866 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 867 'bitops-bitwise-and (ms)': 868 '8.8', 869 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 870 'access-nsieve (ms)': 871 '8.6', 872 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 873 'date-format-tofte (ms)': 874 '31.2', 875 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 876 'bitops-3bit-bits-in-byte (ms)': 877 '3.5', 878 'retval': 879 0, 880 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 881 'string-unpack-code (ms)': 882 '45.0' 883} 884 885PURE_TELEMETRY_OUTPUT = """ 886page_name,3d-cube (ms),3d-morph (ms),3d-raytrace (ms),Total (ms),access-binary-trees (ms),access-fannkuch (ms),access-nbody (ms),access-nsieve (ms),bitops-3bit-bits-in-byte (ms),bitops-bits-in-byte (ms),bitops-bitwise-and (ms),bitops-nsieve-bits (ms),controlflow-recursive (ms),crypto-aes (ms),crypto-md5 (ms),crypto-sha1 (ms),date-format-tofte (ms),date-format-xparb (ms),math-cordic (ms),math-partial-sums (ms),math-spectral-norm (ms),regexp-dna (ms),string-base64 (ms),string-fasta (ms),string-tagcloud (ms),string-unpack-code (ms),string-validate-input (ms)\r\nhttp://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html,42.7,50.2,28.7,656.5,7.3,26.3,6.9,8.6,3.5,9.8,8.8,9.3,5.3,19.2,10.8,12.4,31.2,138.1,11.4,32.8,6.3,16.1,17.5,36.3,47.2,45.0,24.8\r 887""" 888 889 890class TelemetryResultTest(unittest.TestCase): 891 """Telemetry result test.""" 892 893 def __init__(self, *args, **kwargs): 894 super(TelemetryResultTest, self).__init__(*args, **kwargs) 895 self.callFakeProcessResults = False 896 self.result = None 897 self.mock_logger = mock.Mock(spec=logger.Logger) 898 self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 899 self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', 900 '/tmp', 'lumpy', 'remote', 'image_args', 901 'cache_dir', 'average', 'gcc', None) 902 self.mock_machine = machine_manager.MockCrosMachine('falco.cros', 903 '/tmp/chromeos', 904 'average') 905 906 def test_populate_from_run(self): 907 908 def FakeProcessResults(): 909 self.callFakeProcessResults = True 910 911 self.callFakeProcessResults = False 912 self.result = TelemetryResult(self.mock_logger, self.mock_label, 'average', 913 self.mock_cmd_exec) 914 self.result.ProcessResults = FakeProcessResults 915 self.result.PopulateFromRun(OUTPUT, error, 3, 'fake_test', 916 'telemetry_Crosperf') 917 self.assertTrue(self.callFakeProcessResults) 918 self.assertEqual(self.result.out, OUTPUT) 919 self.assertEqual(self.result.err, error) 920 self.assertEqual(self.result.retval, 3) 921 922 def test_populate_from_cache_dir_and_process_results(self): 923 924 self.result = TelemetryResult(self.mock_logger, self.mock_label, 'average', 925 self.mock_machine) 926 current_path = os.getcwd() 927 cache_dir = os.path.join(current_path, 928 'test_cache/test_puretelemetry_input') 929 self.result.PopulateFromCacheDir(cache_dir, '', '') 930 self.assertEqual(self.result.out.strip(), PURE_TELEMETRY_OUTPUT.strip()) 931 self.assertEqual(self.result.err, '') 932 self.assertEqual(self.result.retval, 0) 933 self.assertEqual(self.result.keyvals, TELEMETRY_RESULT_KEYVALS) 934 935 936class ResultsCacheTest(unittest.TestCase): 937 """Resultcache test class.""" 938 939 def __init__(self, *args, **kwargs): 940 super(ResultsCacheTest, self).__init__(*args, **kwargs) 941 self.fakeCacheReturnResult = None 942 self.mock_logger = mock.Mock(spec=logger.Logger) 943 self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', 944 '/tmp', 'lumpy', 'remote', 'image_args', 945 'cache_dir', 'average', 'gcc', None) 946 947 def setUp(self): 948 self.results_cache = ResultsCache() 949 950 mock_machine = machine_manager.MockCrosMachine('falco.cros', 951 '/tmp/chromeos', 'average') 952 953 mock_mm = machine_manager.MockMachineManager('/tmp/chromeos_root', 0, 954 'average', '') 955 mock_mm.machine_checksum_string['mock_label'] = 'fake_machine_checksum123' 956 957 self.results_cache.Init( 958 self.mock_label.chromeos_image, 959 self.mock_label.chromeos_root, 960 'sunspider', 961 1, # benchmark_run.iteration, 962 '', # benchmark_run.test_args, 963 '', # benchmark_run.profiler_args, 964 mock_mm, 965 mock_machine, 966 self.mock_label.board, 967 [CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH], 968 self.mock_logger, 969 'average', 970 self.mock_label, 971 '', # benchmark_run.share_cache 972 'telemetry_Crosperf', 973 True, # benchmark_run.show_all_results 974 False) # benchmark_run.run_local 975 976 @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') 977 def test_get_cache_dir_for_write(self, mock_checksum): 978 979 def FakeGetMachines(label): 980 if label: 981 pass 982 m1 = machine_manager.MockCrosMachine('lumpy1.cros', 983 self.results_cache.chromeos_root, 984 'average') 985 m2 = machine_manager.MockCrosMachine('lumpy2.cros', 986 self.results_cache.chromeos_root, 987 'average') 988 return [m1, m2] 989 990 mock_checksum.return_value = 'FakeImageChecksumabc123' 991 self.results_cache.machine_manager.GetMachines = FakeGetMachines 992 self.results_cache.machine_manager.machine_checksum['mock_label'] = \ 993 'FakeMachineChecksumabc987' 994 # Based on the label, benchmark and machines, get the directory in which 995 # to store the cache information for this test run. 996 result_path = self.results_cache.GetCacheDirForWrite() 997 # Verify that the returned directory is correct (since the label 998 # contained a cache_dir, named 'cache_dir', that's what is expected in 999 # the result, rather than '~/cros_scratch'). 1000 comp_path = os.path.join(os.getcwd(), 1001 'cache_dir/54524606abaae4fdf7b02f49f7ae7127_' 1002 'sunspider_1_fda29412ceccb72977516c4785d08e2c_' 1003 'FakeImageChecksumabc123_FakeMachineChecksum' 1004 'abc987__6') 1005 self.assertEqual(result_path, comp_path) 1006 1007 def test_form_cache_dir(self): 1008 # This is very similar to the previous test (FormCacheDir is called 1009 # from GetCacheDirForWrite). 1010 cache_key_list = ('54524606abaae4fdf7b02f49f7ae7127', 'sunspider', '1', 1011 '7215ee9c7d9dc229d2921a40e899ec5f', 1012 'FakeImageChecksumabc123', '*', '*', '6') 1013 path = self.results_cache.FormCacheDir(cache_key_list) 1014 self.assertEqual(len(path), 1) 1015 path1 = path[0] 1016 test_dirname = ('54524606abaae4fdf7b02f49f7ae7127_sunspider_1_7215ee9' 1017 'c7d9dc229d2921a40e899ec5f_FakeImageChecksumabc123_*_*_6') 1018 comp_path = os.path.join(os.getcwd(), 'cache_dir', test_dirname) 1019 self.assertEqual(path1, comp_path) 1020 1021 @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') 1022 def test_get_cache_key_list(self, mock_checksum): 1023 # This tests the mechanism that generates the various pieces of the 1024 # cache directory name, based on various conditions. 1025 1026 def FakeGetMachines(label): 1027 if label: 1028 pass 1029 m1 = machine_manager.MockCrosMachine('lumpy1.cros', 1030 self.results_cache.chromeos_root, 1031 'average') 1032 m2 = machine_manager.MockCrosMachine('lumpy2.cros', 1033 self.results_cache.chromeos_root, 1034 'average') 1035 return [m1, m2] 1036 1037 mock_checksum.return_value = 'FakeImageChecksumabc123' 1038 self.results_cache.machine_manager.GetMachines = FakeGetMachines 1039 self.results_cache.machine_manager.machine_checksum['mock_label'] = \ 1040 'FakeMachineChecksumabc987' 1041 1042 # Test 1. Generating cache name for reading (not writing). 1043 key_list = self.results_cache.GetCacheKeyList(True) 1044 self.assertEqual(key_list[0], '*') # Machine checksum value, for read. 1045 self.assertEqual(key_list[1], 'sunspider') 1046 self.assertEqual(key_list[2], '1') 1047 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1048 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1049 self.assertEqual(key_list[5], '*') 1050 self.assertEqual(key_list[6], '*') 1051 self.assertEqual(key_list[7], '6') 1052 1053 # Test 2. Generating cache name for writing, with local image type. 1054 key_list = self.results_cache.GetCacheKeyList(False) 1055 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1056 self.assertEqual(key_list[1], 'sunspider') 1057 self.assertEqual(key_list[2], '1') 1058 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1059 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1060 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1061 self.assertEqual(key_list[6], '') 1062 self.assertEqual(key_list[7], '6') 1063 1064 # Test 3. Generating cache name for writing, with trybot image type. 1065 self.results_cache.label.image_type = 'trybot' 1066 key_list = self.results_cache.GetCacheKeyList(False) 1067 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1068 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1069 self.assertEqual(key_list[4], '54524606abaae4fdf7b02f49f7ae7127') 1070 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1071 1072 # Test 4. Generating cache name for writing, with official image type. 1073 self.results_cache.label.image_type = 'official' 1074 key_list = self.results_cache.GetCacheKeyList(False) 1075 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1076 self.assertEqual(key_list[1], 'sunspider') 1077 self.assertEqual(key_list[2], '1') 1078 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1079 self.assertEqual(key_list[4], '*') 1080 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1081 self.assertEqual(key_list[6], '') 1082 self.assertEqual(key_list[7], '6') 1083 1084 # Test 5. Generating cache name for writing, with local image type, and 1085 # specifying that the image path must match the cached image path. 1086 self.results_cache.label.image_type = 'local' 1087 self.results_cache.cache_conditions.append(CacheConditions.IMAGE_PATH_MATCH) 1088 key_list = self.results_cache.GetCacheKeyList(False) 1089 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1090 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1091 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1092 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1093 1094 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 1095 @mock.patch.object(os.path, 'isdir') 1096 @mock.patch.object(Result, 'CreateFromCacheHit') 1097 def test_read_result(self, mock_create, mock_isdir, mock_runcmd): 1098 1099 self.fakeCacheReturnResult = None 1100 1101 def FakeGetCacheDirForRead(): 1102 return self.fakeCacheReturnResult 1103 1104 def FakeGetCacheDirForWrite(): 1105 return self.fakeCacheReturnResult 1106 1107 mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 1108 fake_result = Result(self.mock_logger, self.mock_label, 'average', 1109 mock_cmd_exec) 1110 fake_result.retval = 0 1111 1112 # Set up results_cache _GetCacheDirFor{Read,Write} to return 1113 # self.fakeCacheReturnResult, which is initially None (see above). 1114 # So initially, no cache dir is returned. 1115 self.results_cache.GetCacheDirForRead = FakeGetCacheDirForRead 1116 self.results_cache.GetCacheDirForWrite = FakeGetCacheDirForWrite 1117 1118 mock_isdir.return_value = True 1119 save_cc = [ 1120 CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH 1121 ] 1122 self.results_cache.cache_conditions.append(CacheConditions.FALSE) 1123 1124 # Test 1. CacheCondition.FALSE, which means do not read from the cache. 1125 # (force re-running of test). Result should be None. 1126 res = self.results_cache.ReadResult() 1127 self.assertIsNone(res) 1128 self.assertEqual(mock_runcmd.call_count, 1) 1129 1130 # Test 2. Remove CacheCondition.FALSE. Result should still be None, 1131 # because GetCacheDirForRead is returning None at the moment. 1132 mock_runcmd.reset_mock() 1133 self.results_cache.cache_conditions = save_cc 1134 res = self.results_cache.ReadResult() 1135 self.assertIsNone(res) 1136 self.assertEqual(mock_runcmd.call_count, 0) 1137 1138 # Test 3. Now set up cache dir to be returned by GetCacheDirForRead. 1139 # Since cache_dir is found, will call Result.CreateFromCacheHit, which 1140 # which will actually all our mock_create and should return fake_result. 1141 self.fakeCacheReturnResult = 'fake/cache/dir' 1142 mock_create.return_value = fake_result 1143 res = self.results_cache.ReadResult() 1144 self.assertEqual(mock_runcmd.call_count, 0) 1145 self.assertEqual(res, fake_result) 1146 1147 # Test 4. os.path.isdir(cache_dir) will now return false, so result 1148 # should be None again (no cache found). 1149 mock_isdir.return_value = False 1150 res = self.results_cache.ReadResult() 1151 self.assertEqual(mock_runcmd.call_count, 0) 1152 self.assertIsNone(res) 1153 1154 # Test 5. os.path.isdir returns true, but mock_create now returns None 1155 # (the call to CreateFromCacheHit returns None), so overal result is None. 1156 mock_isdir.return_value = True 1157 mock_create.return_value = None 1158 res = self.results_cache.ReadResult() 1159 self.assertEqual(mock_runcmd.call_count, 0) 1160 self.assertIsNone(res) 1161 1162 # Test 6. Everything works 'as expected', result should be fake_result. 1163 mock_create.return_value = fake_result 1164 res = self.results_cache.ReadResult() 1165 self.assertEqual(mock_runcmd.call_count, 0) 1166 self.assertEqual(res, fake_result) 1167 1168 # Test 7. The run failed; result should be None. 1169 mock_create.return_value = fake_result 1170 fake_result.retval = 1 1171 self.results_cache.cache_conditions.append(CacheConditions.RUN_SUCCEEDED) 1172 res = self.results_cache.ReadResult() 1173 self.assertEqual(mock_runcmd.call_count, 0) 1174 self.assertIsNone(res) 1175 1176 1177if __name__ == '__main__': 1178 unittest.main() 1179