results_cache_unittest.py revision eddb06396dc17fcec418c3aadc234f737b63c876
1#!/usr/bin/python2 2 3# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Module of result cache unittest.""" 8 9from __future__ import print_function 10 11import mock 12import os 13import tempfile 14import unittest 15 16import image_checksummer 17import machine_manager 18import test_flag 19 20from label import MockLabel 21from results_cache import CacheConditions 22from results_cache import Result 23from results_cache import ResultsCache 24from results_cache import TelemetryResult 25from cros_utils import command_executer 26from cros_utils import logger 27from cros_utils import misc 28 29OUTPUT = """CMD (True): ./test_that.sh\ 30 --remote=172.17.128.241 --board=lumpy LibCBench 31CMD (None): cd /usr/local/google/home/yunlian/gd/src/build/images/lumpy/latest/../../../../..; cros_sdk -- ./in_chroot_cmd6X7Cxu.sh 32Identity added: /tmp/test_that.PO1234567/autotest_key (/tmp/test_that.PO1234567/autotest_key) 33INFO : Using emerged autotests already installed at /build/lumpy/usr/local/autotest. 34 35INFO : Running the following control files 1 times: 36INFO : * 'client/site_tests/platform_LibCBench/control' 37 38INFO : Running client test client/site_tests/platform_LibCBench/control 39./server/autoserv -m 172.17.128.241 --ssh-port 22 -c client/site_tests/platform_LibCBench/control -r /tmp/test_that.PO1234567/platform_LibCBench --test-retry=0 --args 40ERROR:root:import statsd failed, no stats will be reported. 4114:20:22 INFO | Results placed in /tmp/test_that.PO1234567/platform_LibCBench 4214:20:22 INFO | Processing control file 4314:20:23 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_VIIP67ssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 4414:20:23 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. 4514:20:23 INFO | INFO ---- ---- kernel=3.8.11 localtime=May 22 14:20:23 timestamp=1369257623 4614:20:23 INFO | Installing autotest on 172.17.128.241 4714:20:23 INFO | Using installation dir /usr/local/autotest 4814:20:23 WARNI| No job_repo_url for <remote host: 172.17.128.241> 4914:20:23 INFO | Could not install autotest using the packaging system: No repos to install an autotest client from. Trying other methods 5014:20:23 INFO | Installation of autotest completed 5114:20:24 WARNI| No job_repo_url for <remote host: 172.17.128.241> 5214:20:24 INFO | Executing /usr/local/autotest/bin/autotest /usr/local/autotest/control phase 0 5314:20:24 INFO | Entered autotestd_monitor. 5414:20:24 INFO | Finished launching tail subprocesses. 5514:20:24 INFO | Finished waiting on autotestd to start. 5614:20:26 INFO | START ---- ---- timestamp=1369257625 localtime=May 22 14:20:25 5714:20:26 INFO | START platform_LibCBench platform_LibCBench timestamp=1369257625 localtime=May 22 14:20:25 5814:20:30 INFO | GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 completed successfully 5914:20:30 INFO | END GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 6014:20:31 INFO | END GOOD ---- ---- timestamp=1369257630 localtime=May 22 14:20:30 6114:20:31 INFO | Got lock of exit_code_file. 6214:20:31 INFO | Released lock of exit_code_file and closed it. 63OUTPUT: ============================== 64OUTPUT: Current time: 2013-05-22 14:20:32.818831 Elapsed: 0:01:30 ETA: Unknown 65Done: 0% [ ] 66OUTPUT: Thread Status: 67RUNNING: 1 ('ttt: LibCBench (1)' 0:01:21) 68Machine Status: 69Machine Thread Lock Status Checksum 70172.17.128.241 ttt: LibCBench (1) True RUNNING 3ba9f2ecbb222f20887daea5583d86ba 71 72OUTPUT: ============================== 7314:20:33 INFO | Killing child processes. 7414:20:33 INFO | Client complete 7514:20:33 INFO | Finished processing control file 7614:20:33 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_aVJUgmssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 7714:20:33 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. 78 79INFO : Test results: 80------------------------------------------------------------------- 81platform_LibCBench [ PASSED ] 82platform_LibCBench/platform_LibCBench [ PASSED ] 83platform_LibCBench/platform_LibCBench b_malloc_big1__0_ 0.00375231466667 84platform_LibCBench/platform_LibCBench b_malloc_big2__0_ 0.002951359 85platform_LibCBench/platform_LibCBench b_malloc_bubble__0_ 0.015066374 86platform_LibCBench/platform_LibCBench b_malloc_sparse__0_ 0.015053784 87platform_LibCBench/platform_LibCBench b_malloc_thread_local__0_ 0.01138439 88platform_LibCBench/platform_LibCBench b_malloc_thread_stress__0_ 0.0367894733333 89platform_LibCBench/platform_LibCBench b_malloc_tiny1__0_ 0.000768474333333 90platform_LibCBench/platform_LibCBench b_malloc_tiny2__0_ 0.000581407333333 91platform_LibCBench/platform_LibCBench b_pthread_create_serial1__0_ 0.0291785246667 92platform_LibCBench/platform_LibCBench b_pthread_createjoin_serial1__0_ 0.031907936 93platform_LibCBench/platform_LibCBench b_pthread_createjoin_serial2__0_ 0.043485347 94platform_LibCBench/platform_LibCBench b_pthread_uselesslock__0_ 0.0294113346667 95platform_LibCBench/platform_LibCBench b_regex_compile____a_b_c__d_b__ 0.00529833933333 96platform_LibCBench/platform_LibCBench b_regex_search____a_b_c__d_b__ 0.00165455066667 97platform_LibCBench/platform_LibCBench b_regex_search___a_25_b__ 0.0496191923333 98platform_LibCBench/platform_LibCBench b_stdio_putcgetc__0_ 0.100005711667 99platform_LibCBench/platform_LibCBench b_stdio_putcgetc_unlocked__0_ 0.0371443833333 100platform_LibCBench/platform_LibCBench b_string_memset__0_ 0.00275405066667 101platform_LibCBench/platform_LibCBench b_string_strchr__0_ 0.00456903 102platform_LibCBench/platform_LibCBench b_string_strlen__0_ 0.044893587 103platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac__ 0.118360778 104platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaac__ 0.068957325 105platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaacccccccccccc__ 0.0135694476667 106platform_LibCBench/platform_LibCBench b_string_strstr___abcdefghijklmnopqrstuvwxyz__ 0.0134553343333 107platform_LibCBench/platform_LibCBench b_string_strstr___azbycxdwevfugthsirjqkplomn__ 0.0133123556667 108platform_LibCBench/platform_LibCBench b_utf8_bigbuf__0_ 0.0473772253333 109platform_LibCBench/platform_LibCBench b_utf8_onebyone__0_ 0.130938538333 110------------------------------------------------------------------- 111Total PASS: 2/2 (100%) 112 113INFO : Elapsed time: 0m16s 114""" 115 116error = """ 117ERROR: Identity added: /tmp/test_that.Z4Ld/autotest_key (/tmp/test_that.Z4Ld/autotest_key) 118INFO : Using emerged autotests already installed at /build/lumpy/usr/local/autotest. 119INFO : Running the following control files 1 times: 120INFO : * 'client/site_tests/platform_LibCBench/control' 121INFO : Running client test client/site_tests/platform_LibCBench/control 122INFO : Test results: 123INFO : Elapsed time: 0m18s 124""" 125 126keyvals = {'': 'PASS', 127 'b_stdio_putcgetc__0_': '0.100005711667', 128 'b_string_strstr___azbycxdwevfugthsirjqkplomn__': '0.0133123556667', 129 'b_malloc_thread_local__0_': '0.01138439', 130 'b_string_strlen__0_': '0.044893587', 131 'b_malloc_sparse__0_': '0.015053784', 132 'b_string_memset__0_': '0.00275405066667', 133 'platform_LibCBench': 'PASS', 134 'b_pthread_uselesslock__0_': '0.0294113346667', 135 'b_string_strchr__0_': '0.00456903', 136 'b_pthread_create_serial1__0_': '0.0291785246667', 137 'b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac__': 138 '0.118360778', 139 'b_string_strstr___aaaaaaaaaaaaaacccccccccccc__': '0.0135694476667', 140 'b_pthread_createjoin_serial1__0_': '0.031907936', 141 'b_malloc_thread_stress__0_': '0.0367894733333', 142 'b_regex_search____a_b_c__d_b__': '0.00165455066667', 143 'b_malloc_bubble__0_': '0.015066374', 144 'b_malloc_big2__0_': '0.002951359', 145 'b_stdio_putcgetc_unlocked__0_': '0.0371443833333', 146 'b_pthread_createjoin_serial2__0_': '0.043485347', 147 'b_regex_search___a_25_b__': '0.0496191923333', 148 'b_utf8_bigbuf__0_': '0.0473772253333', 149 'b_malloc_big1__0_': '0.00375231466667', 150 'b_regex_compile____a_b_c__d_b__': '0.00529833933333', 151 'b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaac__': '0.068957325', 152 'b_malloc_tiny2__0_': '0.000581407333333', 153 'b_utf8_onebyone__0_': '0.130938538333', 154 'b_malloc_tiny1__0_': '0.000768474333333', 155 'b_string_strstr___abcdefghijklmnopqrstuvwxyz__': '0.0134553343333'} 156 157TMP_DIR1 = '/tmp/tmpAbcXyz' 158 159 160class MockResult(Result): 161 """Mock result class.""" 162 def __init__(self, mylogger, label, logging_level, machine): 163 super(MockResult, self).__init__(mylogger, label, logging_level, machine) 164 165 def FindFilesInResultsDir(self, find_args): 166 return '' 167 168 # pylint: disable=arguments-differ 169 def GetKeyvals(self, temp=False): 170 if temp: 171 pass 172 return keyvals 173 174 175class ResultTest(unittest.TestCase): 176 """Result test class.""" 177 178 def __init__(self, *args, **kwargs): 179 super(ResultTest, self).__init__(*args, **kwargs) 180 self.callFakeProcessResults = False 181 self.fakeCacheReturnResult = None 182 self.callGetResultsDir = False 183 self.callProcessResults = False 184 self.callGetPerfReportFiles = False 185 self.kv_dict = None 186 self.tmpdir = '' 187 self.callGetNewKeyvals = False 188 self.callGetPerfDataFiles = False 189 self.args = None 190 self.callGatherPerfResults = False 191 self.mock_logger = mock.Mock(spec=logger.Logger) 192 self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 193 self.mock_label = MockLabel('mock_label', 'chromeos_image', '/tmp', 'lumpy', 194 'remote', 'image_args', 'cache_dir', 'average', 195 'gcc', None) 196 197 def testCreateFromRun(self): 198 result = MockResult.CreateFromRun(logger.GetLogger(), 'average', 199 self.mock_label, 'remote1', OUTPUT, error, 200 0, True, 0) 201 self.assertEqual(result.keyvals, keyvals) 202 self.assertEqual(result.chroot_results_dir, 203 '/tmp/test_that.PO1234567/platform_LibCBench') 204 self.assertEqual(result.results_dir, 205 '/tmp/chroot/tmp/test_that.PO1234567/platform_LibCBench') 206 self.assertEqual(result.retval, 0) 207 208 def setUp(self): 209 self.result = Result(self.mock_logger, self.mock_label, 'average', 210 self.mock_cmd_exec) 211 212 @mock.patch.object(os.path, 'isdir') 213 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 214 @mock.patch.object(command_executer.CommandExecuter, 'CopyFiles') 215 def test_copy_files_to(self, mock_copyfiles, mock_runcmd, mock_isdir): 216 217 files = ['src_file_1', 'src_file_2', 'src_file_3'] 218 dest_dir = '/tmp/test' 219 self.mock_cmd_exec.RunCommand = mock_runcmd 220 self.mock_cmd_exec.CopyFiles = mock_copyfiles 221 222 mock_copyfiles.return_value = 0 223 224 #test 1. dest_dir exists; CopyFiles returns 0. 225 mock_isdir.return_value = True 226 self.result.CopyFilesTo(dest_dir, files) 227 self.assertEqual(mock_runcmd.call_count, 0) 228 self.assertEqual(mock_copyfiles.call_count, 3) 229 first_args = mock_copyfiles.call_args_list[0][0] 230 second_args = mock_copyfiles.call_args_list[1][0] 231 third_args = mock_copyfiles.call_args_list[2][0] 232 self.assertEqual(first_args, ('src_file_1', '/tmp/test/src_file_1.0')) 233 self.assertEqual(second_args, ('src_file_2', '/tmp/test/src_file_2.0')) 234 self.assertEqual(third_args, ('src_file_3', '/tmp/test/src_file_3.0')) 235 236 mock_runcmd.reset_mock() 237 mock_copyfiles.reset_mock() 238 #test 2. dest_dir does not exist; CopyFiles returns 0. 239 mock_isdir.return_value = False 240 self.result.CopyFilesTo(dest_dir, files) 241 self.assertEqual(mock_runcmd.call_count, 3) 242 self.assertEqual(mock_copyfiles.call_count, 3) 243 self.assertEqual(mock_runcmd.call_args_list[0], 244 mock_runcmd.call_args_list[1]) 245 self.assertEqual(mock_runcmd.call_args_list[0], 246 mock_runcmd.call_args_list[2]) 247 self.assertEqual(mock_runcmd.call_args_list[0][0], ('mkdir -p /tmp/test',)) 248 249 #test 3. CopyFiles returns 1 (fails). 250 mock_copyfiles.return_value = 1 251 self.assertRaises(Exception, self.result.CopyFilesTo, dest_dir, files) 252 253 @mock.patch.object(Result, 'CopyFilesTo') 254 def test_copy_results_to(self, mockCopyFilesTo): 255 perf_data_files = ['/tmp/perf.data.0', '/tmp/perf.data.1', 256 '/tmp/perf.data.2'] 257 perf_report_files = ['/tmp/perf.report.0', '/tmp/perf.report.1', 258 '/tmp/perf.report.2'] 259 260 self.result.perf_data_files = perf_data_files 261 self.result.perf_report_files = perf_report_files 262 263 self.result.CopyFilesTo = mockCopyFilesTo 264 self.result.CopyResultsTo('/tmp/results/') 265 self.assertEqual(mockCopyFilesTo.call_count, 2) 266 self.assertEqual(len(mockCopyFilesTo.call_args_list), 2) 267 self.assertEqual(mockCopyFilesTo.call_args_list[0][0], 268 ('/tmp/results/', perf_data_files)) 269 self.assertEqual(mockCopyFilesTo.call_args_list[1][0], 270 ('/tmp/results/', perf_report_files)) 271 272 def test_get_new_keyvals(self): 273 kv_dict = {} 274 275 def FakeGetDataMeasurementsFiles(): 276 filename = os.path.join(os.getcwd(), 'unittest_keyval_file.txt') 277 return [filename] 278 279 self.result.GetDataMeasurementsFiles = FakeGetDataMeasurementsFiles 280 kv_dict2, udict = self.result.GetNewKeyvals(kv_dict) 281 self.assertEqual(kv_dict2, 282 {u'Box2D__Box2D': 4775, 283 u'Mandreel__Mandreel': 6620, 284 u'Gameboy__Gameboy': 9901, 285 u'Crypto__Crypto': 8737, 286 u'telemetry_page_measurement_results__num_errored': 0, 287 u'telemetry_page_measurement_results__num_failed': 0, 288 u'PdfJS__PdfJS': 6455, 289 u'Total__Score': 7918, 290 u'EarleyBoyer__EarleyBoyer': 14340, 291 u'MandreelLatency__MandreelLatency': 5188, 292 u'CodeLoad__CodeLoad': 6271, 293 u'DeltaBlue__DeltaBlue': 14401, 294 u'Typescript__Typescript': 9815, 295 u'SplayLatency__SplayLatency': 7653, 296 u'zlib__zlib': 16094, 297 u'Richards__Richards': 10358, 298 u'RegExp__RegExp': 1765, 299 u'NavierStokes__NavierStokes': 9815, 300 u'Splay__Splay': 4425, 301 u'RayTrace__RayTrace': 16600}) 302 self.assertEqual( 303 udict, {u'Box2D__Box2D': u'score', 304 u'Mandreel__Mandreel': u'score', 305 u'Gameboy__Gameboy': u'score', 306 u'Crypto__Crypto': u'score', 307 u'telemetry_page_measurement_results__num_errored': u'count', 308 u'telemetry_page_measurement_results__num_failed': u'count', 309 u'PdfJS__PdfJS': u'score', 310 u'Total__Score': u'score', 311 u'EarleyBoyer__EarleyBoyer': u'score', 312 u'MandreelLatency__MandreelLatency': u'score', 313 u'CodeLoad__CodeLoad': u'score', 314 u'DeltaBlue__DeltaBlue': u'score', 315 u'Typescript__Typescript': u'score', 316 u'SplayLatency__SplayLatency': u'score', 317 u'zlib__zlib': u'score', 318 u'Richards__Richards': u'score', 319 u'RegExp__RegExp': u'score', 320 u'NavierStokes__NavierStokes': u'score', 321 u'Splay__Splay': u'score', 322 u'RayTrace__RayTrace': u'score'}) 323 324 def test_append_telemetry_units(self): 325 kv_dict = {u'Box2D__Box2D': 4775, 326 u'Mandreel__Mandreel': 6620, 327 u'Gameboy__Gameboy': 9901, 328 u'Crypto__Crypto': 8737, 329 u'PdfJS__PdfJS': 6455, 330 u'Total__Score': 7918, 331 u'EarleyBoyer__EarleyBoyer': 14340, 332 u'MandreelLatency__MandreelLatency': 5188, 333 u'CodeLoad__CodeLoad': 6271, 334 u'DeltaBlue__DeltaBlue': 14401, 335 u'Typescript__Typescript': 9815, 336 u'SplayLatency__SplayLatency': 7653, 337 u'zlib__zlib': 16094, 338 u'Richards__Richards': 10358, 339 u'RegExp__RegExp': 1765, 340 u'NavierStokes__NavierStokes': 9815, 341 u'Splay__Splay': 4425, 342 u'RayTrace__RayTrace': 16600} 343 units_dict = {u'Box2D__Box2D': u'score', 344 u'Mandreel__Mandreel': u'score', 345 u'Gameboy__Gameboy': u'score', 346 u'Crypto__Crypto': u'score', 347 u'PdfJS__PdfJS': u'score', 348 u'Total__Score': u'score', 349 u'EarleyBoyer__EarleyBoyer': u'score', 350 u'MandreelLatency__MandreelLatency': u'score', 351 u'CodeLoad__CodeLoad': u'score', 352 u'DeltaBlue__DeltaBlue': u'score', 353 u'Typescript__Typescript': u'score', 354 u'SplayLatency__SplayLatency': u'score', 355 u'zlib__zlib': u'score', 356 u'Richards__Richards': u'score', 357 u'RegExp__RegExp': u'score', 358 u'NavierStokes__NavierStokes': u'score', 359 u'Splay__Splay': u'score', 360 u'RayTrace__RayTrace': u'score'} 361 362 results_dict = self.result.AppendTelemetryUnits(kv_dict, units_dict) 363 self.assertEqual(results_dict, 364 {u'Box2D__Box2D': [4775, u'score'], 365 u'Splay__Splay': [4425, u'score'], 366 u'Gameboy__Gameboy': [9901, u'score'], 367 u'Crypto__Crypto': [8737, u'score'], 368 u'PdfJS__PdfJS': [6455, u'score'], 369 u'Total__Score': [7918, u'score'], 370 u'EarleyBoyer__EarleyBoyer': [14340, u'score'], 371 u'MandreelLatency__MandreelLatency': [5188, u'score'], 372 u'DeltaBlue__DeltaBlue': [14401, u'score'], 373 u'SplayLatency__SplayLatency': [7653, u'score'], 374 u'Mandreel__Mandreel': [6620, u'score'], 375 u'Richards__Richards': [10358, u'score'], 376 u'zlib__zlib': [16094, u'score'], 377 u'CodeLoad__CodeLoad': [6271, u'score'], 378 u'Typescript__Typescript': [9815, u'score'], 379 u'RegExp__RegExp': [1765, u'score'], 380 u'RayTrace__RayTrace': [16600, u'score'], 381 u'NavierStokes__NavierStokes': [9815, u'score']}) 382 383 @mock.patch.object(misc, 'GetInsideChrootPath') 384 @mock.patch.object(tempfile, 'mkdtemp') 385 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 386 @mock.patch.object(command_executer.CommandExecuter, 387 'ChrootRunCommandWOutput') 388 def test_get_keyvals(self, mock_chrootruncmd, mock_runcmd, mock_mkdtemp, 389 mock_getpath): 390 391 self.kv_dict = {} 392 self.callGetNewKeyvals = False 393 394 def reset(): 395 self.kv_dict = {} 396 self.callGetNewKeyvals = False 397 mock_chrootruncmd.reset_mock() 398 mock_runcmd.reset_mock() 399 mock_mkdtemp.reset_mock() 400 mock_getpath.reset_mock() 401 402 def FakeGetNewKeyvals(kv_dict): 403 self.kv_dict = kv_dict 404 self.callGetNewKeyvals = True 405 return_kvdict = {'first_time': 680, 'Total': 10} 406 return_udict = {'first_time': 'ms', 'Total': 'score'} 407 return return_kvdict, return_udict 408 409 mock_mkdtemp.return_value = TMP_DIR1 410 mock_chrootruncmd.return_value = ['', 411 ('%s,PASS\n%s/telemetry_Crosperf,PASS\n') 412 % (TMP_DIR1, TMP_DIR1), ''] 413 mock_getpath.return_value = TMP_DIR1 414 self.result.ce.ChrootRunCommandWOutput = mock_chrootruncmd 415 self.result.ce.RunCommand = mock_runcmd 416 self.result.GetNewKeyvals = FakeGetNewKeyvals 417 self.result.suite = 'telemetry_Crosperf' 418 self.result.results_dir = '/tmp/test_that_resultsNmq' 419 420 # Test 1. no self.temp_dir. 421 res = self.result.GetKeyvals() 422 self.assertTrue(self.callGetNewKeyvals) 423 self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) 424 self.assertEqual(mock_runcmd.call_count, 1) 425 self.assertEqual(mock_runcmd.call_args_list[0][0], 426 ('cp -r /tmp/test_that_resultsNmq/* %s' % TMP_DIR1,)) 427 self.assertEqual(mock_chrootruncmd.call_count, 1) 428 self.assertEqual(mock_chrootruncmd.call_args_list[0][0], ( 429 '/tmp', ('python generate_test_report --no-color --csv %s') % TMP_DIR1)) 430 self.assertEqual(mock_getpath.call_count, 1) 431 self.assertEqual(mock_mkdtemp.call_count, 1) 432 self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) 433 434 # Test 2. self.temp_dir 435 reset() 436 mock_chrootruncmd.return_value = ['', 437 ('/tmp/tmpJCajRG,PASS\n/tmp/tmpJCajRG/' 438 'telemetry_Crosperf,PASS\n'), ''] 439 mock_getpath.return_value = '/tmp/tmpJCajRG' 440 self.result.temp_dir = '/tmp/tmpJCajRG' 441 res = self.result.GetKeyvals() 442 self.assertEqual(mock_runcmd.call_count, 0) 443 self.assertEqual(mock_mkdtemp.call_count, 0) 444 self.assertEqual(mock_chrootruncmd.call_count, 1) 445 self.assertTrue(self.callGetNewKeyvals) 446 self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) 447 self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) 448 449 # Test 3. suite != telemetry_Crosperf. Normally this would be for 450 # running non-Telemetry autotests, such as BootPerfServer. In this test 451 # case, the keyvals we have set up were returned from a Telemetry test run; 452 # so this pass is basically testing that we don't append the units to the 453 # test results (which we do for Telemetry autotest runs). 454 reset() 455 self.result.suite = '' 456 res = self.result.GetKeyvals() 457 self.assertEqual(res, {'Total': 10, 'first_time': 680}) 458 459 def test_get_results_dir(self): 460 461 self.result.out = '' 462 self.assertRaises(Exception, self.result.GetResultsDir) 463 464 self.result.out = OUTPUT 465 resdir = self.result.GetResultsDir() 466 self.assertEqual(resdir, '/tmp/test_that.PO1234567/platform_LibCBench') 467 468 @mock.patch.object(command_executer.CommandExecuter, 'RunCommandGeneric') 469 def test_find_files_in_results_dir(self, mock_runcmd): 470 471 self.result.results_dir = None 472 res = self.result.FindFilesInResultsDir('-name perf.data') 473 self.assertIsNone(res) 474 475 self.result.ce.RunCommand = mock_runcmd 476 self.result.results_dir = '/tmp/test_results' 477 mock_runcmd.return_value = [0, '/tmp/test_results/perf.data', ''] 478 res = self.result.FindFilesInResultsDir('-name perf.data') 479 self.assertEqual(mock_runcmd.call_count, 1) 480 self.assertEqual(mock_runcmd.call_args_list[0][0], 481 ('find /tmp/test_results -name perf.data',)) 482 self.assertEqual(res, '/tmp/test_results/perf.data') 483 484 mock_runcmd.reset_mock() 485 mock_runcmd.return_value = [1, '', ''] 486 self.assertRaises(Exception, self.result.FindFilesInResultsDir, 487 '-name perf.data') 488 489 @mock.patch.object(Result, 'FindFilesInResultsDir') 490 def test_get_perf_data_files(self, mock_findfiles): 491 self.args = None 492 493 mock_findfiles.return_value = 'line1\nline1\n' 494 self.result.FindFilesInResultsDir = mock_findfiles 495 res = self.result.GetPerfDataFiles() 496 self.assertEqual(res, ['line1', 'line1']) 497 self.assertEqual(mock_findfiles.call_args_list[0][0], ('-name perf.data',)) 498 499 def test_get_perf_report_files(self): 500 self.args = None 501 502 def FakeFindFiles(find_args): 503 self.args = find_args 504 return 'line1\nline1\n' 505 506 self.result.FindFilesInResultsDir = FakeFindFiles 507 res = self.result.GetPerfReportFiles() 508 self.assertEqual(res, ['line1', 'line1']) 509 self.assertEqual(self.args, '-name perf.data.report') 510 511 def test_get_data_measurement_files(self): 512 self.args = None 513 514 def FakeFindFiles(find_args): 515 self.args = find_args 516 return 'line1\nline1\n' 517 518 self.result.FindFilesInResultsDir = FakeFindFiles 519 res = self.result.GetDataMeasurementsFiles() 520 self.assertEqual(res, ['line1', 'line1']) 521 self.assertEqual(self.args, '-name perf_measurements') 522 523 @mock.patch.object(misc, 'GetInsideChrootPath') 524 @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') 525 def test_generate_perf_report_files(self, mock_chrootruncmd, mock_getpath): 526 fake_file = '/usr/chromeos/chroot/tmp/results/fake_file' 527 self.result.perf_data_files = ['/tmp/results/perf.data'] 528 self.result.board = 'lumpy' 529 mock_getpath.return_value = fake_file 530 self.result.ce.ChrootRunCommand = mock_chrootruncmd 531 tmp = self.result.GeneratePerfReportFiles() 532 self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file]) 533 self.assertEqual(mock_chrootruncmd.call_args_list[0][0], 534 ('/tmp', 535 ('/tmp/perf.static report -n --symfs /build/lumpy ' 536 '--vmlinux /build/lumpy/usr/lib/debug/boot/vmlinux ' 537 '--kallsyms /build/lumpy/boot/System.map-* -i ' 538 '%s --stdio > %s') % (fake_file, fake_file))) 539 540 @mock.patch.object(misc, 'GetOutsideChrootPath') 541 def test_populate_from_run(self, mock_getpath): 542 543 def FakeGetResultsDir(): 544 self.callGetResultsDir = True 545 return '/tmp/results_dir' 546 547 def FakeGetPerfDataFiles(): 548 self.callGetPerfDataFiles = True 549 return [] 550 551 def FakeGetPerfReportFiles(): 552 self.callGetPerfReportFiles = True 553 return [] 554 555 def FakeProcessResults(show_results=False): 556 if show_results: 557 pass 558 self.callProcessResults = True 559 560 if mock_getpath: 561 pass 562 mock.get_path = '/tmp/chromeos/tmp/results_dir' 563 self.result.chromeos_root = '/tmp/chromeos' 564 565 self.callGetResultsDir = False 566 self.callGetPerfDataFiles = False 567 self.callGetPerfReportFiles = False 568 self.callProcessResults = False 569 570 self.result.GetResultsDir = FakeGetResultsDir 571 self.result.GetPerfDataFiles = FakeGetPerfDataFiles 572 self.result.GeneratePerfReportFiles = FakeGetPerfReportFiles 573 self.result.ProcessResults = FakeProcessResults 574 575 self.result.PopulateFromRun(OUTPUT, '', 0, 'test', 576 'telemetry_Crosperf') 577 self.assertTrue(self.callGetResultsDir) 578 self.assertTrue(self.callGetPerfDataFiles) 579 self.assertTrue(self.callGetPerfReportFiles) 580 self.assertTrue(self.callProcessResults) 581 582 def test_process_results(self): 583 584 def FakeGetKeyvals(show_all=False): 585 if show_all: 586 return {'first_time': 680, 'Total': 10} 587 else: 588 return {'Total': 10} 589 590 def FakeGatherPerfResults(): 591 self.callGatherPerfResults = True 592 593 self.callGatherPerfResults = False 594 595 self.result.GetKeyvals = FakeGetKeyvals 596 self.result.GatherPerfResults = FakeGatherPerfResults 597 598 self.result.retval = 0 599 self.result.ProcessResults() 600 self.assertTrue(self.callGatherPerfResults) 601 self.assertEqual(len(self.result.keyvals), 2) 602 self.assertEqual(self.result.keyvals, {'Total': 10, 603 'retval': 0}) 604 605 self.result.retval = 1 606 self.result.ProcessResults() 607 self.assertEqual(len(self.result.keyvals), 2) 608 self.assertEqual(self.result.keyvals, {'Total': 10, 'retval': 1}) 609 610 @mock.patch.object(misc, 'GetInsideChrootPath') 611 @mock.patch.object(command_executer.CommandExecuter, 612 'ChrootRunCommandWOutput') 613 def test_populate_from_cache_dir(self, mock_runchrootcmd, mock_getpath): 614 615 # pylint: disable=redefined-builtin 616 def FakeMkdtemp(dir=None): 617 if dir: 618 pass 619 return self.tmpdir 620 621 current_path = os.getcwd() 622 cache_dir = os.path.join(current_path, 'test_cache/test_input') 623 self.result.ce = command_executer.GetCommandExecuter(log_level='average') 624 self.result.ce.ChrootRunCommandWOutput = mock_runchrootcmd 625 mock_runchrootcmd.return_value = ['', 626 ('%s,PASS\n%s/\telemetry_Crosperf,PASS\n') 627 % (TMP_DIR1, TMP_DIR1), ''] 628 mock_getpath.return_value = TMP_DIR1 629 self.tmpdir = tempfile.mkdtemp() 630 save_real_mkdtemp = tempfile.mkdtemp 631 tempfile.mkdtemp = FakeMkdtemp 632 633 self.result.PopulateFromCacheDir(cache_dir, 'sunspider', 634 'telemetry_Crosperf') 635 self.assertEqual( 636 self.result.keyvals, 637 {u'Total__Total': [444.0, u'ms'], 638 u'regexp-dna__regexp-dna': [16.2, u'ms'], 639 u'telemetry_page_measurement_results__num_failed': [0, u'count'], 640 u'telemetry_page_measurement_results__num_errored': [0, u'count'], 641 u'string-fasta__string-fasta': [23.2, u'ms'], 642 u'crypto-sha1__crypto-sha1': [11.6, u'ms'], 643 u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': [3.2, u'ms'], 644 u'access-nsieve__access-nsieve': [7.9, u'ms'], 645 u'bitops-nsieve-bits__bitops-nsieve-bits': [9.4, u'ms'], 646 u'string-validate-input__string-validate-input': [19.3, u'ms'], 647 u'3d-raytrace__3d-raytrace': [24.7, u'ms'], 648 u'3d-cube__3d-cube': [28.0, u'ms'], 649 u'string-unpack-code__string-unpack-code': [46.7, u'ms'], 650 u'date-format-tofte__date-format-tofte': [26.3, u'ms'], 651 u'math-partial-sums__math-partial-sums': [22.0, u'ms'], 652 '\telemetry_Crosperf': ['PASS', ''], 653 u'crypto-aes__crypto-aes': [15.2, u'ms'], 654 u'bitops-bitwise-and__bitops-bitwise-and': [8.4, u'ms'], 655 u'crypto-md5__crypto-md5': [10.5, u'ms'], 656 u'string-tagcloud__string-tagcloud': [52.8, u'ms'], 657 u'access-nbody__access-nbody': [8.5, u'ms'], 658 'retval': 0, 659 u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'], 660 u'math-cordic__math-cordic': [8.7, u'ms'], 661 u'access-binary-trees__access-binary-trees': [4.5, u'ms'], 662 u'controlflow-recursive__controlflow-recursive': [4.4, u'ms'], 663 u'access-fannkuch__access-fannkuch': [17.8, u'ms'], 664 u'string-base64__string-base64': [16.0, u'ms'], 665 u'date-format-xparb__date-format-xparb': [20.9, u'ms'], 666 u'3d-morph__3d-morph': [22.1, u'ms'], 667 u'bitops-bits-in-byte__bitops-bits-in-byte': [9.1, u'ms']}) 668 669 # Clean up after test. 670 tempfile.mkdtemp = save_real_mkdtemp 671 command = 'rm -Rf %s' % self.tmpdir 672 self.result.ce.RunCommand(command) 673 674 @mock.patch.object(misc, 'GetRoot') 675 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 676 def test_cleanup(self, mock_runcmd, mock_getroot): 677 678 # Test 1. 'rm_chroot_tmp' is True; self.results_dir exists; 679 # self.temp_dir exists; results_dir name contains 'test_that_results_'. 680 mock_getroot.return_value = ['/tmp/tmp_AbcXyz', 'test_that_results_fake'] 681 self.result.ce.RunCommand = mock_runcmd 682 self.result.results_dir = 'test_results_dir' 683 self.result.temp_dir = 'testtemp_dir' 684 self.result.CleanUp(True) 685 self.assertEqual(mock_getroot.call_count, 1) 686 self.assertEqual(mock_runcmd.call_count, 2) 687 self.assertEqual(mock_runcmd.call_args_list[0][0], 688 ('rm -rf test_results_dir',)) 689 self.assertEqual(mock_runcmd.call_args_list[1][0], 690 ('rm -rf testtemp_dir',)) 691 692 # Test 2. Same, except ath results_dir name does not contain 693 # 'test_that_results_' 694 mock_getroot.reset_mock() 695 mock_runcmd.reset_mock() 696 mock_getroot.return_value = ['/tmp/tmp_AbcXyz', 'other_results_fake'] 697 self.result.ce.RunCommand = mock_runcmd 698 self.result.results_dir = 'test_results_dir' 699 self.result.temp_dir = 'testtemp_dir' 700 self.result.CleanUp(True) 701 self.assertEqual(mock_getroot.call_count, 1) 702 self.assertEqual(mock_runcmd.call_count, 2) 703 self.assertEqual(mock_runcmd.call_args_list[0][0], 704 ('rm -rf /tmp/tmp_AbcXyz',)) 705 self.assertEqual(mock_runcmd.call_args_list[1][0], 706 ('rm -rf testtemp_dir',)) 707 708 # Test 3. mock_getroot returns nothing; 'rm_chroot_tmp' is False. 709 mock_getroot.reset_mock() 710 mock_runcmd.reset_mock() 711 self.result.CleanUp(False) 712 self.assertEqual(mock_getroot.call_count, 0) 713 self.assertEqual(mock_runcmd.call_count, 1) 714 self.assertEqual(mock_runcmd.call_args_list[0][0], 715 ('rm -rf testtemp_dir',)) 716 717 # Test 4. 'rm_chroot_tmp' is True, but result_dir & temp_dir are None. 718 mock_getroot.reset_mock() 719 mock_runcmd.reset_mock() 720 self.result.results_dir = None 721 self.result.temp_dir = None 722 self.result.CleanUp(True) 723 self.assertEqual(mock_getroot.call_count, 0) 724 self.assertEqual(mock_runcmd.call_count, 0) 725 726 @mock.patch.object(misc, 'GetInsideChrootPath') 727 @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') 728 def test_store_to_cache_dir(self, mock_chrootruncmd, mock_getpath): 729 730 def FakeMkdtemp(directory=''): 731 if directory: 732 pass 733 return self.tmpdir 734 735 if mock_chrootruncmd or mock_getpath: 736 pass 737 current_path = os.getcwd() 738 cache_dir = os.path.join(current_path, 'test_cache/test_output') 739 740 self.result.ce = command_executer.GetCommandExecuter(log_level='average') 741 self.result.out = OUTPUT 742 self.result.err = error 743 self.result.retval = 0 744 self.tmpdir = tempfile.mkdtemp() 745 if not os.path.exists(self.tmpdir): 746 os.makedirs(self.tmpdir) 747 self.result.results_dir = os.path.join(os.getcwd(), 'test_cache') 748 save_real_mkdtemp = tempfile.mkdtemp 749 tempfile.mkdtemp = FakeMkdtemp 750 751 mock_mm = machine_manager.MockMachineManager('/tmp/chromeos_root', 0, 752 'average', '') 753 mock_mm.machine_checksum_string['mock_label'] = 'fake_machine_checksum123' 754 755 mock_keylist = ['key1', 'key2', 'key3'] 756 test_flag.SetTestMode(True) 757 self.result.StoreToCacheDir(cache_dir, mock_mm, mock_keylist) 758 759 # Check that the correct things were written to the 'cache'. 760 test_dir = os.path.join(os.getcwd(), 'test_cache/test_output') 761 base_dir = os.path.join(os.getcwd(), 'test_cache/compare_output') 762 self.assertTrue(os.path.exists(os.path.join(test_dir, 'autotest.tbz2'))) 763 self.assertTrue(os.path.exists(os.path.join(test_dir, 'machine.txt'))) 764 self.assertTrue(os.path.exists(os.path.join(test_dir, 'results.txt'))) 765 766 f1 = os.path.join(test_dir, 'machine.txt') 767 f2 = os.path.join(base_dir, 'machine.txt') 768 cmd = 'diff %s %s' % (f1, f2) 769 [_, out, _] = self.result.ce.RunCommandWOutput(cmd) 770 self.assertEqual(len(out), 0) 771 772 f1 = os.path.join(test_dir, 'results.txt') 773 f2 = os.path.join(base_dir, 'results.txt') 774 cmd = 'diff %s %s' % (f1, f2) 775 [_, out, _] = self.result.ce.RunCommandWOutput(cmd) 776 self.assertEqual(len(out), 0) 777 778 # Clean up after test. 779 tempfile.mkdtemp = save_real_mkdtemp 780 command = 'rm %s/*' % test_dir 781 self.result.ce.RunCommand(command) 782 783 784TELEMETRY_RESULT_KEYVALS = { 785 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 786 'math-cordic (ms)': 787 '11.4', 788 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 789 'access-nbody (ms)': 790 '6.9', 791 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 792 'access-fannkuch (ms)': 793 '26.3', 794 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 795 'math-spectral-norm (ms)': 796 '6.3', 797 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 798 'bitops-nsieve-bits (ms)': 799 '9.3', 800 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 801 'math-partial-sums (ms)': 802 '32.8', 803 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 804 'regexp-dna (ms)': 805 '16.1', 806 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 807 '3d-cube (ms)': 808 '42.7', 809 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 810 'crypto-md5 (ms)': 811 '10.8', 812 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 813 'crypto-sha1 (ms)': 814 '12.4', 815 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 816 'string-tagcloud (ms)': 817 '47.2', 818 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 819 'string-fasta (ms)': 820 '36.3', 821 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 822 'access-binary-trees (ms)': 823 '7.3', 824 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 825 'date-format-xparb (ms)': 826 '138.1', 827 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 828 'crypto-aes (ms)': 829 '19.2', 830 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 831 'Total (ms)': 832 '656.5', 833 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 834 'string-base64 (ms)': 835 '17.5', 836 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 837 'string-validate-input (ms)': 838 '24.8', 839 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 840 '3d-raytrace (ms)': 841 '28.7', 842 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 843 'controlflow-recursive (ms)': 844 '5.3', 845 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 846 'bitops-bits-in-byte (ms)': 847 '9.8', 848 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 849 '3d-morph (ms)': 850 '50.2', 851 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 852 'bitops-bitwise-and (ms)': 853 '8.8', 854 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 855 'access-nsieve (ms)': 856 '8.6', 857 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 858 'date-format-tofte (ms)': 859 '31.2', 860 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 861 'bitops-3bit-bits-in-byte (ms)': 862 '3.5', 863 'retval': 0, 864 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 865 'string-unpack-code (ms)': 866 '45.0' 867} 868 869PURE_TELEMETRY_OUTPUT = """ 870page_name,3d-cube (ms),3d-morph (ms),3d-raytrace (ms),Total (ms),access-binary-trees (ms),access-fannkuch (ms),access-nbody (ms),access-nsieve (ms),bitops-3bit-bits-in-byte (ms),bitops-bits-in-byte (ms),bitops-bitwise-and (ms),bitops-nsieve-bits (ms),controlflow-recursive (ms),crypto-aes (ms),crypto-md5 (ms),crypto-sha1 (ms),date-format-tofte (ms),date-format-xparb (ms),math-cordic (ms),math-partial-sums (ms),math-spectral-norm (ms),regexp-dna (ms),string-base64 (ms),string-fasta (ms),string-tagcloud (ms),string-unpack-code (ms),string-validate-input (ms)\r\nhttp://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html,42.7,50.2,28.7,656.5,7.3,26.3,6.9,8.6,3.5,9.8,8.8,9.3,5.3,19.2,10.8,12.4,31.2,138.1,11.4,32.8,6.3,16.1,17.5,36.3,47.2,45.0,24.8\r 871""" 872 873 874class TelemetryResultTest(unittest.TestCase): 875 """Telemetry result test.""" 876 877 def __init__(self, *args, **kwargs): 878 super(TelemetryResultTest, self).__init__(*args, **kwargs) 879 self.callFakeProcessResults = False 880 self.result = None 881 self.mock_logger = mock.Mock(spec=logger.Logger) 882 self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 883 self.mock_label = MockLabel('mock_label', 'chromeos_image', '/tmp', 884 'lumpy', 'remote', 'image_args', 'cache_dir', 885 'average', 'gcc', None) 886 self.mock_machine = machine_manager.MockCrosMachine('falco.cros', 887 '/tmp/chromeos', 888 'average') 889 890 def test_populate_from_run(self): 891 892 def FakeProcessResults(): 893 self.callFakeProcessResults = True 894 895 self.callFakeProcessResults = False 896 self.result = TelemetryResult(self.mock_logger, self.mock_label, 'average', 897 self.mock_cmd_exec) 898 self.result.ProcessResults = FakeProcessResults 899 self.result.PopulateFromRun(OUTPUT, error, 3, 'fake_test', 900 'telemetry_Crosperf') 901 self.assertTrue(self.callFakeProcessResults) 902 self.assertEqual(self.result.out, OUTPUT) 903 self.assertEqual(self.result.err, error) 904 self.assertEqual(self.result.retval, 3) 905 906 def test_populate_from_cache_dir_and_process_results(self): 907 908 self.result = TelemetryResult(self.mock_logger, self.mock_label, 'average', 909 self.mock_machine) 910 current_path = os.getcwd() 911 cache_dir = os.path.join(current_path, 912 'test_cache/test_puretelemetry_input') 913 self.result.PopulateFromCacheDir(cache_dir, '', '') 914 self.assertEqual(self.result.out.strip(), PURE_TELEMETRY_OUTPUT.strip()) 915 self.assertEqual(self.result.err, '') 916 self.assertEqual(self.result.retval, 0) 917 self.assertEqual(self.result.keyvals, TELEMETRY_RESULT_KEYVALS) 918 919 920class ResultsCacheTest(unittest.TestCase): 921 """Resultcache test class.""" 922 923 def __init__(self, *args, **kwargs): 924 super(ResultsCacheTest, self).__init__(*args, **kwargs) 925 self.fakeCacheReturnResult = None 926 self.mock_logger = mock.Mock(spec=logger.Logger) 927 self.mock_label = MockLabel('mock_label', 'chromeos_image', '/tmp', 'lumpy', 928 'remote', 'image_args', 'cache_dir', 'average', 929 'gcc', None) 930 931 def setUp(self): 932 self.results_cache = ResultsCache() 933 934 mock_machine = machine_manager.MockCrosMachine('falco.cros', 935 '/tmp/chromeos', 'average') 936 937 mock_mm = machine_manager.MockMachineManager('/tmp/chromeos_root', 0, 938 'average', '') 939 mock_mm.machine_checksum_string['mock_label'] = 'fake_machine_checksum123' 940 941 self.results_cache.Init(self.mock_label.chromeos_image, 942 self.mock_label.chromeos_root, 943 'sunspider', 944 1, # benchmark_run.iteration, 945 '', # benchmark_run.test_args, 946 '', # benchmark_run.profiler_args, 947 mock_mm, 948 mock_machine, 949 self.mock_label.board, 950 [CacheConditions.CACHE_FILE_EXISTS, 951 CacheConditions.CHECKSUMS_MATCH], 952 self.mock_logger, 953 'average', 954 self.mock_label, 955 '', # benchmark_run.share_cache 956 'telemetry_Crosperf', 957 True, # benchmark_run.show_all_results 958 False) # benchmark_run.run_local 959 960 @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') 961 def test_get_cache_dir_for_write(self, mock_checksum): 962 963 def FakeGetMachines(label): 964 if label: 965 pass 966 m1 = machine_manager.MockCrosMachine( 967 'lumpy1.cros', self.results_cache.chromeos_root, 'average') 968 m2 = machine_manager.MockCrosMachine( 969 'lumpy2.cros', self.results_cache.chromeos_root, 'average') 970 return [m1, m2] 971 972 mock_checksum.return_value = 'FakeImageChecksumabc123' 973 self.results_cache.machine_manager.GetMachines = FakeGetMachines 974 self.results_cache.machine_manager.machine_checksum['mock_label'] = \ 975 'FakeMachineChecksumabc987' 976 # Based on the label, benchmark and machines, get the directory in which 977 # to store the cache information for this test run. 978 result_path = self.results_cache.GetCacheDirForWrite() 979 # Verify that the returned directory is correct (since the label 980 # contained a cache_dir, named 'cache_dir', that's what is expected in 981 # the result, rather than '~/cros_scratch'). 982 comp_path = os.path.join(os.getcwd(), 983 'cache_dir/54524606abaae4fdf7b02f49f7ae7127_' 984 'sunspider_1_fda29412ceccb72977516c4785d08e2c_' 985 'FakeImageChecksumabc123_FakeMachineChecksum' 986 'abc987__6') 987 self.assertEqual(result_path, comp_path) 988 989 def test_form_cache_dir(self): 990 # This is very similar to the previous test (FormCacheDir is called 991 # from GetCacheDirForWrite). 992 cache_key_list = ('54524606abaae4fdf7b02f49f7ae7127', 'sunspider', '1', 993 '7215ee9c7d9dc229d2921a40e899ec5f', 994 'FakeImageChecksumabc123', '*', '*', '6') 995 path = self.results_cache.FormCacheDir(cache_key_list) 996 self.assertEqual(len(path), 1) 997 path1 = path[0] 998 test_dirname = ('54524606abaae4fdf7b02f49f7ae7127_sunspider_1_7215ee9' 999 'c7d9dc229d2921a40e899ec5f_FakeImageChecksumabc123_*_*_6') 1000 comp_path = os.path.join(os.getcwd(), 'cache_dir', test_dirname) 1001 self.assertEqual(path1, comp_path) 1002 1003 @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') 1004 def test_get_cache_key_list(self, mock_checksum): 1005 # This tests the mechanism that generates the various pieces of the 1006 # cache directory name, based on various conditions. 1007 1008 def FakeGetMachines(label): 1009 if label: 1010 pass 1011 m1 = machine_manager.MockCrosMachine( 1012 'lumpy1.cros', self.results_cache.chromeos_root, 'average') 1013 m2 = machine_manager.MockCrosMachine( 1014 'lumpy2.cros', self.results_cache.chromeos_root, 'average') 1015 return [m1, m2] 1016 1017 mock_checksum.return_value = 'FakeImageChecksumabc123' 1018 self.results_cache.machine_manager.GetMachines = FakeGetMachines 1019 self.results_cache.machine_manager.machine_checksum['mock_label'] = \ 1020 'FakeMachineChecksumabc987' 1021 1022 # Test 1. Generating cache name for reading (not writing). 1023 key_list = self.results_cache.GetCacheKeyList(True) 1024 self.assertEqual(key_list[0], '*') # Machine checksum value, for read. 1025 self.assertEqual(key_list[1], 'sunspider') 1026 self.assertEqual(key_list[2], '1') 1027 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1028 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1029 self.assertEqual(key_list[5], '*') 1030 self.assertEqual(key_list[6], '*') 1031 self.assertEqual(key_list[7], '6') 1032 1033 # Test 2. Generating cache name for writing, with local image type. 1034 key_list = self.results_cache.GetCacheKeyList(False) 1035 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1036 self.assertEqual(key_list[1], 'sunspider') 1037 self.assertEqual(key_list[2], '1') 1038 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1039 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1040 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1041 self.assertEqual(key_list[6], '') 1042 self.assertEqual(key_list[7], '6') 1043 1044 # Test 3. Generating cache name for writing, with trybot image type. 1045 self.results_cache.label.image_type = 'trybot' 1046 key_list = self.results_cache.GetCacheKeyList(False) 1047 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1048 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1049 self.assertEqual(key_list[4], '54524606abaae4fdf7b02f49f7ae7127') 1050 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1051 1052 # Test 4. Generating cache name for writing, with official image type. 1053 self.results_cache.label.image_type = 'official' 1054 key_list = self.results_cache.GetCacheKeyList(False) 1055 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1056 self.assertEqual(key_list[1], 'sunspider') 1057 self.assertEqual(key_list[2], '1') 1058 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1059 self.assertEqual(key_list[4], '*') 1060 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1061 self.assertEqual(key_list[6], '') 1062 self.assertEqual(key_list[7], '6') 1063 1064 # Test 5. Generating cache name for writing, with local image type, and 1065 # specifying that the image path must match the cached image path. 1066 self.results_cache.label.image_type = 'local' 1067 self.results_cache.cache_conditions.append(CacheConditions.IMAGE_PATH_MATCH) 1068 key_list = self.results_cache.GetCacheKeyList(False) 1069 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1070 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1071 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1072 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1073 1074 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 1075 @mock.patch.object(os.path, 'isdir') 1076 @mock.patch.object(Result, 'CreateFromCacheHit') 1077 def test_read_result(self, mock_create, mock_isdir, mock_runcmd): 1078 1079 self.fakeCacheReturnResult = None 1080 1081 def FakeGetCacheDirForRead(): 1082 return self.fakeCacheReturnResult 1083 1084 def FakeGetCacheDirForWrite(): 1085 return self.fakeCacheReturnResult 1086 1087 mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 1088 fake_result = Result(self.mock_logger, self.mock_label, 'average', 1089 mock_cmd_exec) 1090 fake_result.retval = 0 1091 1092 # Set up results_cache _GetCacheDirFor{Read,Write} to return 1093 # self.fakeCacheReturnResult, which is initially None (see above). 1094 # So initially, no cache dir is returned. 1095 self.results_cache.GetCacheDirForRead = FakeGetCacheDirForRead 1096 self.results_cache.GetCacheDirForWrite = FakeGetCacheDirForWrite 1097 1098 mock_isdir.return_value = True 1099 save_cc = [CacheConditions.CACHE_FILE_EXISTS, 1100 CacheConditions.CHECKSUMS_MATCH] 1101 self.results_cache.cache_conditions.append(CacheConditions.FALSE) 1102 1103 # Test 1. CacheCondition.FALSE, which means do not read from the cache. 1104 # (force re-running of test). Result should be None. 1105 res = self.results_cache.ReadResult() 1106 self.assertIsNone(res) 1107 self.assertEqual(mock_runcmd.call_count, 1) 1108 1109 # Test 2. Remove CacheCondition.FALSE. Result should still be None, 1110 # because GetCacheDirForRead is returning None at the moment. 1111 mock_runcmd.reset_mock() 1112 self.results_cache.cache_conditions = save_cc 1113 res = self.results_cache.ReadResult() 1114 self.assertIsNone(res) 1115 self.assertEqual(mock_runcmd.call_count, 0) 1116 1117 # Test 3. Now set up cache dir to be returned by GetCacheDirForRead. 1118 # Since cache_dir is found, will call Result.CreateFromCacheHit, which 1119 # which will actually all our mock_create and should return fake_result. 1120 self.fakeCacheReturnResult = 'fake/cache/dir' 1121 mock_create.return_value = fake_result 1122 res = self.results_cache.ReadResult() 1123 self.assertEqual(mock_runcmd.call_count, 0) 1124 self.assertEqual(res, fake_result) 1125 1126 # Test 4. os.path.isdir(cache_dir) will now return false, so result 1127 # should be None again (no cache found). 1128 mock_isdir.return_value = False 1129 res = self.results_cache.ReadResult() 1130 self.assertEqual(mock_runcmd.call_count, 0) 1131 self.assertIsNone(res) 1132 1133 # Test 5. os.path.isdir returns true, but mock_create now returns None 1134 # (the call to CreateFromCacheHit returns None), so overal result is None. 1135 mock_isdir.return_value = True 1136 mock_create.return_value = None 1137 res = self.results_cache.ReadResult() 1138 self.assertEqual(mock_runcmd.call_count, 0) 1139 self.assertIsNone(res) 1140 1141 # Test 6. Everything works 'as expected', result should be fake_result. 1142 mock_create.return_value = fake_result 1143 res = self.results_cache.ReadResult() 1144 self.assertEqual(mock_runcmd.call_count, 0) 1145 self.assertEqual(res, fake_result) 1146 1147 # Test 7. The run failed; result should be None. 1148 mock_create.return_value = fake_result 1149 fake_result.retval = 1 1150 self.results_cache.cache_conditions.append(CacheConditions.RUN_SUCCEEDED) 1151 res = self.results_cache.ReadResult() 1152 self.assertEqual(mock_runcmd.call_count, 0) 1153 self.assertIsNone(res) 1154 1155 1156if __name__ == '__main__': 1157 unittest.main() 1158