1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# pylint: disable=unused-argument 6 7import os 8import unittest 9 10from py_utils import cloud_storage 11import mock 12from pyfakefs import fake_filesystem_unittest 13from pyfakefs import fake_filesystem 14from pyfakefs import fake_filesystem_glob 15 16import dependency_manager 17from dependency_manager import uploader 18 19 20class BaseConfigCreationAndUpdateUnittests(fake_filesystem_unittest.TestCase): 21 def setUp(self): 22 self.addTypeEqualityFunc(uploader.CloudStorageUploader, 23 uploader.CloudStorageUploader.__eq__) 24 self.setUpPyfakefs() 25 self.dependencies = { 26 'dep1': {'cloud_storage_bucket': 'bucket1', 27 'cloud_storage_base_folder': 'dependencies_folder', 28 'file_info': { 29 'plat1': { 30 'cloud_storage_hash': 'hash11', 31 'download_path': '../../relative/dep1/path1'}, 32 'plat2': { 33 'cloud_storage_hash': 'hash12', 34 'download_path': '../../relative/dep1/path2'}}}, 35 'dep2': {'cloud_storage_bucket': 'bucket2', 36 'file_info': { 37 'plat1': { 38 'cloud_storage_hash': 'hash21', 39 'download_path': '../../relative/dep2/path1'}, 40 'plat2': { 41 'cloud_storage_hash': 'hash22', 42 'download_path': '../../relative/dep2/path2'}}}} 43 44 self.expected_file_lines = [ 45 # pylint: disable=bad-continuation 46 '{', '"config_type": "BaseConfig",', '"dependencies": {', 47 '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', 48 '"cloud_storage_bucket": "bucket1",', '"file_info": {', 49 '"plat1": {', '"cloud_storage_hash": "hash11",', 50 '"download_path": "../../relative/dep1/path1"', '},', 51 '"plat2": {', '"cloud_storage_hash": "hash12",', 52 '"download_path": "../../relative/dep1/path2"', '}', '}', '},', 53 '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', 54 '"plat1": {', '"cloud_storage_hash": "hash21",', 55 '"download_path": "../../relative/dep2/path1"', '},', 56 '"plat2": {', '"cloud_storage_hash": "hash22",', 57 '"download_path": "../../relative/dep2/path2"', '}', '}', '}', 58 '}', '}'] 59 60 self.file_path = os.path.abspath(os.path.join( 61 'path', 'to', 'config', 'file')) 62 63 self.new_dep_path = 'path/to/new/dep' 64 self.fs.CreateFile(self.new_dep_path) 65 self.new_dep_hash = 'A23B56B7F23E798601F' 66 self.new_dependencies = { 67 'dep1': {'cloud_storage_bucket': 'bucket1', 68 'cloud_storage_base_folder': 'dependencies_folder', 69 'file_info': { 70 'plat1': { 71 'cloud_storage_hash': 'hash11', 72 'download_path': '../../relative/dep1/path1'}, 73 'plat2': { 74 'cloud_storage_hash': self.new_dep_hash, 75 'download_path': '../../relative/dep1/path2'}}}, 76 'dep2': {'cloud_storage_bucket': 'bucket2', 77 'file_info': { 78 'plat1': { 79 'cloud_storage_hash': 'hash21', 80 'download_path': '../../relative/dep2/path1'}, 81 'plat2': { 82 'cloud_storage_hash': 'hash22', 83 'download_path': '../../relative/dep2/path2'}}}} 84 self.new_bucket = 'bucket1' 85 self.new_remote_path = 'dependencies_folder/dep1_%s' % self.new_dep_hash 86 self.new_pending_upload = uploader.CloudStorageUploader( 87 self.new_bucket, self.new_remote_path, self.new_dep_path) 88 self.expected_new_backup_path = '.'.join([self.new_remote_path, 'old']) 89 self.new_expected_file_lines = [ 90 # pylint: disable=bad-continuation 91 '{', '"config_type": "BaseConfig",', '"dependencies": {', 92 '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', 93 '"cloud_storage_bucket": "bucket1",', '"file_info": {', 94 '"plat1": {', '"cloud_storage_hash": "hash11",', 95 '"download_path": "../../relative/dep1/path1"', '},', 96 '"plat2": {', '"cloud_storage_hash": "%s",' % self.new_dep_hash, 97 '"download_path": "../../relative/dep1/path2"', '}', '}', '},', 98 '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', 99 '"plat1": {', '"cloud_storage_hash": "hash21",', 100 '"download_path": "../../relative/dep2/path1"', '},', 101 '"plat2": {', '"cloud_storage_hash": "hash22",', 102 '"download_path": "../../relative/dep2/path2"', '}', '}', '}', 103 '}', '}'] 104 105 self.final_dep_path = 'path/to/final/dep' 106 self.fs.CreateFile(self.final_dep_path) 107 self.final_dep_hash = 'B34662F23B56B7F98601F' 108 self.final_bucket = 'bucket2' 109 self.final_remote_path = 'dep1_%s' % self.final_dep_hash 110 self.final_pending_upload = uploader.CloudStorageUploader( 111 self.final_bucket, self.final_remote_path, self.final_dep_path) 112 self.expected_final_backup_path = '.'.join([self.final_remote_path, 113 'old']) 114 self.final_dependencies = { 115 'dep1': {'cloud_storage_bucket': 'bucket1', 116 'cloud_storage_base_folder': 'dependencies_folder', 117 'file_info': { 118 'plat1': { 119 'cloud_storage_hash': 'hash11', 120 'download_path': '../../relative/dep1/path1'}, 121 'plat2': { 122 'cloud_storage_hash': self.new_dep_hash, 123 'download_path': '../../relative/dep1/path2'}}}, 124 'dep2': {'cloud_storage_bucket': 'bucket2', 125 'file_info': { 126 'plat1': { 127 'cloud_storage_hash': self.final_dep_hash, 128 'download_path': '../../relative/dep2/path1'}, 129 'plat2': { 130 'cloud_storage_hash': 'hash22', 131 'download_path': '../../relative/dep2/path2'}}}} 132 self.final_expected_file_lines = [ 133 # pylint: disable=bad-continuation 134 '{', '"config_type": "BaseConfig",', '"dependencies": {', 135 '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', 136 '"cloud_storage_bucket": "bucket1",', '"file_info": {', 137 '"plat1": {', '"cloud_storage_hash": "hash11",', 138 '"download_path": "../../relative/dep1/path1"', '},', 139 '"plat2": {', '"cloud_storage_hash": "%s",' % self.new_dep_hash, 140 '"download_path": "../../relative/dep1/path2"', '}', '}', '},', 141 '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', 142 '"plat1": {', '"cloud_storage_hash": "%s",' % self.final_dep_hash, 143 '"download_path": "../../relative/dep2/path1"', '},', 144 '"plat2": {', '"cloud_storage_hash": "hash22",', 145 '"download_path": "../../relative/dep2/path2"', '}', '}', '}', 146 '}', '}'] 147 148 149 def tearDown(self): 150 self.tearDownPyfakefs() 151 152 # Init is not meant to be overridden, so we should be mocking the 153 # base_config's json module, even in subclasses. 154 def testCreateEmptyConfig(self): 155 expected_file_lines = ['{', 156 '"config_type": "BaseConfig",', 157 '"dependencies": {}', 158 '}'] 159 config = dependency_manager.BaseConfig(self.file_path, writable=True) 160 161 file_module = fake_filesystem.FakeFileOpen(self.fs) 162 for line in file_module(self.file_path): 163 self.assertEqual(expected_file_lines.pop(0), line.strip()) 164 self.fs.CloseOpenFile(file_module(self.file_path)) 165 self.assertEqual({}, config._config_data) 166 self.assertEqual(self.file_path, config._config_path) 167 168 def testCreateEmptyConfigError(self): 169 self.assertRaises(dependency_manager.EmptyConfigError, 170 dependency_manager.BaseConfig, self.file_path) 171 172 def testCloudStorageRemotePath(self): 173 dependency = 'dep_name' 174 cs_hash = self.new_dep_hash 175 cs_base_folder = 'dependency_remote_folder' 176 expected_remote_path = '%s/%s_%s' % (cs_base_folder, dependency, cs_hash) 177 remote_path = dependency_manager.BaseConfig._CloudStorageRemotePath( 178 dependency, cs_hash, cs_base_folder) 179 self.assertEqual(expected_remote_path, remote_path) 180 181 cs_base_folder = 'dependency_remote_folder' 182 expected_remote_path = '%s_%s' % (dependency, cs_hash) 183 remote_path = dependency_manager.BaseConfig._CloudStorageRemotePath( 184 dependency, cs_hash, cs_base_folder) 185 186 def testGetEmptyJsonDict(self): 187 expected_json_dict = {'config_type': 'BaseConfig', 188 'dependencies': {}} 189 json_dict = dependency_manager.BaseConfig._GetJsonDict() 190 self.assertEqual(expected_json_dict, json_dict) 191 192 def testGetNonEmptyJsonDict(self): 193 expected_json_dict = {"config_type": "BaseConfig", 194 "dependencies": self.dependencies} 195 json_dict = dependency_manager.BaseConfig._GetJsonDict(self.dependencies) 196 self.assertEqual(expected_json_dict, json_dict) 197 198 def testWriteEmptyConfigToFile(self): 199 expected_file_lines = ['{', '"config_type": "BaseConfig",', 200 '"dependencies": {}', '}'] 201 self.assertFalse(os.path.exists(self.file_path)) 202 dependency_manager.BaseConfig._WriteConfigToFile(self.file_path) 203 self.assertTrue(os.path.exists(self.file_path)) 204 file_module = fake_filesystem.FakeFileOpen(self.fs) 205 for line in file_module(self.file_path): 206 self.assertEqual(expected_file_lines.pop(0), line.strip()) 207 self.fs.CloseOpenFile(file_module(self.file_path)) 208 209 def testWriteNonEmptyConfigToFile(self): 210 self.assertFalse(os.path.exists(self.file_path)) 211 dependency_manager.BaseConfig._WriteConfigToFile(self.file_path, 212 self.dependencies) 213 self.assertTrue(os.path.exists(self.file_path)) 214 expected_file_lines = list(self.expected_file_lines) 215 file_module = fake_filesystem.FakeFileOpen(self.fs) 216 for line in file_module(self.file_path): 217 self.assertEqual(expected_file_lines.pop(0), line.strip()) 218 self.fs.CloseOpenFile(file_module(self.file_path)) 219 220 @mock.patch('dependency_manager.uploader.cloud_storage') 221 def testExecuteUpdateJobsNoOp(self, uploader_cs_mock): 222 self.fs.CreateFile(self.file_path, 223 contents='\n'.join(self.expected_file_lines)) 224 config = dependency_manager.BaseConfig(self.file_path, writable=True) 225 226 self.assertFalse(config.ExecuteUpdateJobs()) 227 self.assertFalse(config._IsDirty()) 228 self.assertFalse(config._pending_uploads) 229 self.assertEqual(self.dependencies, config._config_data) 230 file_module = fake_filesystem.FakeFileOpen(self.fs) 231 expected_file_lines = list(self.expected_file_lines) 232 for line in file_module(self.file_path): 233 self.assertEqual(expected_file_lines.pop(0), line.strip()) 234 self.fs.CloseOpenFile(file_module(self.file_path)) 235 236 @mock.patch('dependency_manager.uploader.cloud_storage') 237 def testExecuteUpdateJobsFailureOnInsertNoCSCollision( 238 self, uploader_cs_mock): 239 uploader_cs_mock.Exists.return_value = False 240 uploader_cs_mock.Insert.side_effect = cloud_storage.CloudStorageError 241 self.fs.CreateFile(self.file_path, 242 contents='\n'.join(self.expected_file_lines)) 243 config = dependency_manager.BaseConfig(self.file_path, writable=True) 244 config._config_data = self.new_dependencies.copy() 245 config._is_dirty = True 246 config._pending_uploads = [self.new_pending_upload] 247 self.assertEqual(self.new_dependencies, config._config_data) 248 self.assertTrue(config._is_dirty) 249 self.assertEqual(1, len(config._pending_uploads)) 250 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 251 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 252 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 253 self.new_dep_path)] 254 expected_copy_calls = [] 255 expected_delete_calls = [] 256 257 self.assertRaises(cloud_storage.CloudStorageError, 258 config.ExecuteUpdateJobs) 259 self.assertTrue(config._is_dirty) 260 self.assertEqual(1, len(config._pending_uploads)) 261 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 262 self.assertEqual(self.new_dependencies, config._config_data) 263 file_module = fake_filesystem.FakeFileOpen(self.fs) 264 expected_file_lines = list(self.expected_file_lines) 265 for line in file_module(self.file_path): 266 self.assertEqual(expected_file_lines.pop(0), line.strip()) 267 self.fs.CloseOpenFile(file_module(self.file_path)) 268 self.assertEqual(1, len(config._pending_uploads)) 269 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 270 self.assertEqual(expected_insert_calls, 271 uploader_cs_mock.Insert.call_args_list) 272 self.assertEqual(expected_exists_calls, 273 uploader_cs_mock.Exists.call_args_list) 274 self.assertEqual(expected_copy_calls, 275 uploader_cs_mock.Copy.call_args_list) 276 self.assertEqual(expected_delete_calls, 277 uploader_cs_mock.Delete.call_args_list) 278 279 @mock.patch('dependency_manager.uploader.cloud_storage') 280 def testExecuteUpdateJobsFailureOnInsertCSCollisionForce( 281 self, uploader_cs_mock): 282 uploader_cs_mock.Exists.return_value = True 283 uploader_cs_mock.Insert.side_effect = cloud_storage.CloudStorageError 284 self.fs.CreateFile(self.file_path, 285 contents='\n'.join(self.expected_file_lines)) 286 config = dependency_manager.BaseConfig(self.file_path, writable=True) 287 config._config_data = self.new_dependencies.copy() 288 config._is_dirty = True 289 config._pending_uploads = [self.new_pending_upload] 290 self.assertEqual(self.new_dependencies, config._config_data) 291 self.assertTrue(config._is_dirty) 292 self.assertEqual(1, len(config._pending_uploads)) 293 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 294 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 295 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 296 self.new_dep_path)] 297 expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, 298 self.new_remote_path, 299 self.expected_new_backup_path), 300 mock.call(self.new_bucket, self.new_bucket, 301 self.expected_new_backup_path, 302 self.new_remote_path)] 303 expected_delete_calls = [] 304 305 self.assertRaises(cloud_storage.CloudStorageError, 306 config.ExecuteUpdateJobs, force=True) 307 self.assertTrue(config._is_dirty) 308 self.assertEqual(1, len(config._pending_uploads)) 309 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 310 self.assertEqual(self.new_dependencies, config._config_data) 311 file_module = fake_filesystem.FakeFileOpen(self.fs) 312 expected_file_lines = list(self.expected_file_lines) 313 for line in file_module(self.file_path): 314 self.assertEqual(expected_file_lines.pop(0), line.strip()) 315 self.fs.CloseOpenFile(file_module(self.file_path)) 316 self.assertEqual(1, len(config._pending_uploads)) 317 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 318 self.assertEqual(expected_insert_calls, 319 uploader_cs_mock.Insert.call_args_list) 320 self.assertEqual(expected_exists_calls, 321 uploader_cs_mock.Exists.call_args_list) 322 self.assertEqual(expected_copy_calls, 323 uploader_cs_mock.Copy.call_args_list) 324 self.assertEqual(expected_delete_calls, 325 uploader_cs_mock.Delete.call_args_list) 326 327 @mock.patch('dependency_manager.uploader.cloud_storage') 328 def testExecuteUpdateJobsFailureOnInsertCSCollisionNoForce( 329 self, uploader_cs_mock): 330 uploader_cs_mock.Exists.return_value = True 331 uploader_cs_mock.Insert.side_effect = cloud_storage.CloudStorageError 332 self.fs.CreateFile(self.file_path, 333 contents='\n'.join(self.expected_file_lines)) 334 config = dependency_manager.BaseConfig(self.file_path, writable=True) 335 config._config_data = self.new_dependencies.copy() 336 config._is_dirty = True 337 config._pending_uploads = [self.new_pending_upload] 338 self.assertEqual(self.new_dependencies, config._config_data) 339 self.assertTrue(config._is_dirty) 340 self.assertEqual(1, len(config._pending_uploads)) 341 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 342 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 343 expected_insert_calls = [] 344 expected_copy_calls = [] 345 expected_delete_calls = [] 346 347 self.assertRaises(cloud_storage.CloudStorageError, 348 config.ExecuteUpdateJobs) 349 self.assertTrue(config._is_dirty) 350 self.assertEqual(1, len(config._pending_uploads)) 351 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 352 self.assertEqual(self.new_dependencies, config._config_data) 353 file_module = fake_filesystem.FakeFileOpen(self.fs) 354 expected_file_lines = list(self.expected_file_lines) 355 for line in file_module(self.file_path): 356 self.assertEqual(expected_file_lines.pop(0), line.strip()) 357 self.fs.CloseOpenFile(file_module(self.file_path)) 358 self.assertEqual(1, len(config._pending_uploads)) 359 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 360 self.assertEqual(expected_insert_calls, 361 uploader_cs_mock.Insert.call_args_list) 362 self.assertEqual(expected_exists_calls, 363 uploader_cs_mock.Exists.call_args_list) 364 self.assertEqual(expected_copy_calls, 365 uploader_cs_mock.Copy.call_args_list) 366 self.assertEqual(expected_delete_calls, 367 uploader_cs_mock.Delete.call_args_list) 368 369 @mock.patch('dependency_manager.uploader.cloud_storage') 370 def testExecuteUpdateJobsFailureOnCopy( 371 self, uploader_cs_mock): 372 uploader_cs_mock.Exists.return_value = True 373 uploader_cs_mock.Copy.side_effect = cloud_storage.CloudStorageError 374 self.fs.CreateFile(self.file_path, 375 contents='\n'.join(self.expected_file_lines)) 376 config = dependency_manager.BaseConfig(self.file_path, writable=True) 377 config._config_data = self.new_dependencies.copy() 378 config._is_dirty = True 379 config._pending_uploads = [self.new_pending_upload] 380 self.assertEqual(self.new_dependencies, config._config_data) 381 self.assertTrue(config._is_dirty) 382 self.assertEqual(1, len(config._pending_uploads)) 383 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 384 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 385 expected_insert_calls = [] 386 expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, 387 self.new_remote_path, 388 self.expected_new_backup_path)] 389 expected_delete_calls = [] 390 391 self.assertRaises(cloud_storage.CloudStorageError, 392 config.ExecuteUpdateJobs, force=True) 393 self.assertTrue(config._is_dirty) 394 self.assertEqual(1, len(config._pending_uploads)) 395 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 396 self.assertEqual(self.new_dependencies, config._config_data) 397 file_module = fake_filesystem.FakeFileOpen(self.fs) 398 expected_file_lines = list(self.expected_file_lines) 399 for line in file_module(self.file_path): 400 self.assertEqual(expected_file_lines.pop(0), line.strip()) 401 self.fs.CloseOpenFile(file_module(self.file_path)) 402 self.assertEqual(1, len(config._pending_uploads)) 403 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 404 self.assertEqual(expected_insert_calls, 405 uploader_cs_mock.Insert.call_args_list) 406 self.assertEqual(expected_exists_calls, 407 uploader_cs_mock.Exists.call_args_list) 408 self.assertEqual(expected_copy_calls, 409 uploader_cs_mock.Copy.call_args_list) 410 self.assertEqual(expected_delete_calls, 411 uploader_cs_mock.Delete.call_args_list) 412 413 @mock.patch('dependency_manager.uploader.cloud_storage') 414 def testExecuteUpdateJobsFailureOnSecondInsertNoCSCollision( 415 self, uploader_cs_mock): 416 uploader_cs_mock.Exists.return_value = False 417 uploader_cs_mock.Insert.side_effect = [ 418 True, cloud_storage.CloudStorageError] 419 self.fs.CreateFile(self.file_path, 420 contents='\n'.join(self.expected_file_lines)) 421 config = dependency_manager.BaseConfig(self.file_path, writable=True) 422 config._config_data = self.new_dependencies.copy() 423 config._is_dirty = True 424 config._pending_uploads = [self.new_pending_upload, 425 self.final_pending_upload] 426 self.assertEqual(self.new_dependencies, config._config_data) 427 self.assertTrue(config._is_dirty) 428 self.assertEqual(2, len(config._pending_uploads)) 429 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 430 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 431 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), 432 mock.call(self.final_bucket, 433 self.final_remote_path)] 434 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 435 self.new_dep_path), 436 mock.call(self.final_bucket, 437 self.final_remote_path, 438 self.final_dep_path)] 439 expected_copy_calls = [] 440 expected_delete_calls = [mock.call(self.new_bucket, self.new_remote_path)] 441 442 self.assertRaises(cloud_storage.CloudStorageError, 443 config.ExecuteUpdateJobs) 444 self.assertTrue(config._is_dirty) 445 self.assertEqual(2, len(config._pending_uploads)) 446 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 447 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 448 self.assertEqual(self.new_dependencies, config._config_data) 449 file_module = fake_filesystem.FakeFileOpen(self.fs) 450 expected_file_lines = list(self.expected_file_lines) 451 for line in file_module(self.file_path): 452 self.assertEqual(expected_file_lines.pop(0), line.strip()) 453 self.fs.CloseOpenFile(file_module(self.file_path)) 454 self.assertEqual(expected_insert_calls, 455 uploader_cs_mock.Insert.call_args_list) 456 self.assertEqual(expected_exists_calls, 457 uploader_cs_mock.Exists.call_args_list) 458 self.assertEqual(expected_copy_calls, 459 uploader_cs_mock.Copy.call_args_list) 460 self.assertEqual(expected_delete_calls, 461 uploader_cs_mock.Delete.call_args_list) 462 463 @mock.patch('dependency_manager.uploader.cloud_storage') 464 def testExecuteUpdateJobsFailureOnSecondInsertCSCollisionForce( 465 self, uploader_cs_mock): 466 uploader_cs_mock.Exists.return_value = True 467 uploader_cs_mock.Insert.side_effect = [ 468 True, cloud_storage.CloudStorageError] 469 self.fs.CreateFile(self.file_path, 470 contents='\n'.join(self.expected_file_lines)) 471 config = dependency_manager.BaseConfig(self.file_path, writable=True) 472 config._config_data = self.new_dependencies.copy() 473 config._is_dirty = True 474 config._pending_uploads = [self.new_pending_upload, 475 self.final_pending_upload] 476 self.assertEqual(self.new_dependencies, config._config_data) 477 self.assertTrue(config._is_dirty) 478 self.assertEqual(2, len(config._pending_uploads)) 479 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 480 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 481 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), 482 mock.call(self.final_bucket, 483 self.final_remote_path)] 484 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 485 self.new_dep_path), 486 mock.call(self.final_bucket, 487 self.final_remote_path, 488 self.final_dep_path)] 489 expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, 490 self.new_remote_path, 491 self.expected_new_backup_path), 492 mock.call(self.final_bucket, self.final_bucket, 493 self.final_remote_path, 494 self.expected_final_backup_path), 495 mock.call(self.final_bucket, self.final_bucket, 496 self.expected_final_backup_path, 497 self.final_remote_path), 498 mock.call(self.new_bucket, self.new_bucket, 499 self.expected_new_backup_path, 500 self.new_remote_path)] 501 expected_delete_calls = [] 502 503 self.assertRaises(cloud_storage.CloudStorageError, 504 config.ExecuteUpdateJobs, force=True) 505 self.assertTrue(config._is_dirty) 506 self.assertEqual(2, len(config._pending_uploads)) 507 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 508 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 509 self.assertEqual(self.new_dependencies, config._config_data) 510 file_module = fake_filesystem.FakeFileOpen(self.fs) 511 expected_file_lines = list(self.expected_file_lines) 512 for line in file_module(self.file_path): 513 self.assertEqual(expected_file_lines.pop(0), line.strip()) 514 self.fs.CloseOpenFile(file_module(self.file_path)) 515 self.assertEqual(expected_insert_calls, 516 uploader_cs_mock.Insert.call_args_list) 517 self.assertEqual(expected_exists_calls, 518 uploader_cs_mock.Exists.call_args_list) 519 self.assertEqual(expected_copy_calls, 520 uploader_cs_mock.Copy.call_args_list) 521 self.assertEqual(expected_delete_calls, 522 uploader_cs_mock.Delete.call_args_list) 523 524 @mock.patch('dependency_manager.uploader.cloud_storage') 525 def testExecuteUpdateJobsFailureOnSecondInsertFirstCSCollisionForce( 526 self, uploader_cs_mock): 527 uploader_cs_mock.Exists.side_effect = [True, False, True] 528 uploader_cs_mock.Insert.side_effect = [ 529 True, cloud_storage.CloudStorageError] 530 self.fs.CreateFile(self.file_path, 531 contents='\n'.join(self.expected_file_lines)) 532 config = dependency_manager.BaseConfig(self.file_path, writable=True) 533 config._config_data = self.new_dependencies.copy() 534 config._is_dirty = True 535 config._pending_uploads = [self.new_pending_upload, 536 self.final_pending_upload] 537 self.assertEqual(self.new_dependencies, config._config_data) 538 self.assertTrue(config._is_dirty) 539 self.assertEqual(2, len(config._pending_uploads)) 540 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 541 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 542 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), 543 mock.call(self.final_bucket, 544 self.final_remote_path)] 545 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 546 self.new_dep_path), 547 mock.call(self.final_bucket, 548 self.final_remote_path, 549 self.final_dep_path)] 550 expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, 551 self.new_remote_path, 552 self.expected_new_backup_path), 553 mock.call(self.new_bucket, self.new_bucket, 554 self.expected_new_backup_path, 555 self.new_remote_path)] 556 expected_delete_calls = [] 557 558 self.assertRaises(cloud_storage.CloudStorageError, 559 config.ExecuteUpdateJobs, force=True) 560 self.assertTrue(config._is_dirty) 561 self.assertEqual(2, len(config._pending_uploads)) 562 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 563 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 564 self.assertEqual(self.new_dependencies, config._config_data) 565 file_module = fake_filesystem.FakeFileOpen(self.fs) 566 expected_file_lines = list(self.expected_file_lines) 567 for line in file_module(self.file_path): 568 self.assertEqual(expected_file_lines.pop(0), line.strip()) 569 self.fs.CloseOpenFile(file_module(self.file_path)) 570 self.assertEqual(expected_insert_calls, 571 uploader_cs_mock.Insert.call_args_list) 572 self.assertEqual(expected_exists_calls, 573 uploader_cs_mock.Exists.call_args_list) 574 self.assertEqual(expected_copy_calls, 575 uploader_cs_mock.Copy.call_args_list) 576 self.assertEqual(expected_delete_calls, 577 uploader_cs_mock.Delete.call_args_list) 578 579 @mock.patch('dependency_manager.uploader.cloud_storage') 580 def testExecuteUpdateJobsFailureOnFirstCSCollisionNoForce( 581 self, uploader_cs_mock): 582 uploader_cs_mock.Exists.side_effect = [True, False, True] 583 uploader_cs_mock.Insert.side_effect = [ 584 True, cloud_storage.CloudStorageError] 585 self.fs.CreateFile(self.file_path, 586 contents='\n'.join(self.expected_file_lines)) 587 config = dependency_manager.BaseConfig(self.file_path, writable=True) 588 config._config_data = self.new_dependencies.copy() 589 config._is_dirty = True 590 config._pending_uploads = [self.new_pending_upload, 591 self.final_pending_upload] 592 self.assertEqual(self.new_dependencies, config._config_data) 593 self.assertTrue(config._is_dirty) 594 self.assertEqual(2, len(config._pending_uploads)) 595 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 596 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 597 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 598 expected_insert_calls = [] 599 expected_copy_calls = [] 600 expected_delete_calls = [] 601 602 self.assertRaises(cloud_storage.CloudStorageError, 603 config.ExecuteUpdateJobs) 604 self.assertTrue(config._is_dirty) 605 self.assertEqual(2, len(config._pending_uploads)) 606 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 607 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 608 self.assertEqual(self.new_dependencies, config._config_data) 609 file_module = fake_filesystem.FakeFileOpen(self.fs) 610 expected_file_lines = list(self.expected_file_lines) 611 for line in file_module(self.file_path): 612 self.assertEqual(expected_file_lines.pop(0), line.strip()) 613 self.fs.CloseOpenFile(file_module(self.file_path)) 614 self.assertEqual(expected_insert_calls, 615 uploader_cs_mock.Insert.call_args_list) 616 self.assertEqual(expected_exists_calls, 617 uploader_cs_mock.Exists.call_args_list) 618 self.assertEqual(expected_copy_calls, 619 uploader_cs_mock.Copy.call_args_list) 620 self.assertEqual(expected_delete_calls, 621 uploader_cs_mock.Delete.call_args_list) 622 623 @mock.patch('dependency_manager.uploader.cloud_storage') 624 def testExecuteUpdateJobsFailureOnSecondCopyCSCollision( 625 self, uploader_cs_mock): 626 uploader_cs_mock.Exists.return_value = True 627 uploader_cs_mock.Insert.return_value = True 628 uploader_cs_mock.Copy.side_effect = [ 629 True, cloud_storage.CloudStorageError, True] 630 self.fs.CreateFile(self.file_path, 631 contents='\n'.join(self.expected_file_lines)) 632 config = dependency_manager.BaseConfig(self.file_path, writable=True) 633 config._config_data = self.new_dependencies.copy() 634 config._is_dirty = True 635 config._pending_uploads = [self.new_pending_upload, 636 self.final_pending_upload] 637 self.assertEqual(self.new_dependencies, config._config_data) 638 self.assertTrue(config._is_dirty) 639 self.assertEqual(2, len(config._pending_uploads)) 640 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 641 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 642 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), 643 mock.call(self.final_bucket, 644 self.final_remote_path)] 645 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 646 self.new_dep_path)] 647 expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, 648 self.new_remote_path, 649 self.expected_new_backup_path), 650 mock.call(self.final_bucket, self.final_bucket, 651 self.final_remote_path, 652 self.expected_final_backup_path), 653 mock.call(self.new_bucket, self.new_bucket, 654 self.expected_new_backup_path, 655 self.new_remote_path)] 656 expected_delete_calls = [] 657 658 self.assertRaises(cloud_storage.CloudStorageError, 659 config.ExecuteUpdateJobs, force=True) 660 self.assertTrue(config._is_dirty) 661 self.assertEqual(2, len(config._pending_uploads)) 662 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 663 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 664 self.assertEqual(self.new_dependencies, config._config_data) 665 file_module = fake_filesystem.FakeFileOpen(self.fs) 666 expected_file_lines = list(self.expected_file_lines) 667 for line in file_module(self.file_path): 668 self.assertEqual(expected_file_lines.pop(0), line.strip()) 669 self.fs.CloseOpenFile(file_module(self.file_path)) 670 self.assertEqual(expected_insert_calls, 671 uploader_cs_mock.Insert.call_args_list) 672 self.assertEqual(expected_exists_calls, 673 uploader_cs_mock.Exists.call_args_list) 674 self.assertEqual(expected_copy_calls, 675 uploader_cs_mock.Copy.call_args_list) 676 self.assertEqual(expected_delete_calls, 677 uploader_cs_mock.Delete.call_args_list) 678 679 @mock.patch('dependency_manager.uploader.cloud_storage') 680 def testExecuteUpdateJobsFailureOnSecondCopyNoCSCollisionForce( 681 self, uploader_cs_mock): 682 uploader_cs_mock.Exists.side_effect = [False, True, False] 683 uploader_cs_mock.Copy.side_effect = cloud_storage.CloudStorageError 684 self.fs.CreateFile(self.file_path, 685 contents='\n'.join(self.expected_file_lines)) 686 config = dependency_manager.BaseConfig(self.file_path, writable=True) 687 config._config_data = self.new_dependencies.copy() 688 config._is_dirty = True 689 config._pending_uploads = [self.new_pending_upload, 690 self.final_pending_upload] 691 self.assertEqual(self.new_dependencies, config._config_data) 692 self.assertTrue(config._is_dirty) 693 self.assertEqual(2, len(config._pending_uploads)) 694 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 695 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 696 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), 697 mock.call(self.final_bucket, 698 self.final_remote_path)] 699 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 700 self.new_dep_path)] 701 expected_copy_calls = [mock.call(self.final_bucket, self.final_bucket, 702 self.final_remote_path, 703 self.expected_final_backup_path)] 704 expected_delete_calls = [mock.call(self.new_bucket, self.new_remote_path)] 705 706 self.assertRaises(cloud_storage.CloudStorageError, 707 config.ExecuteUpdateJobs, force=True) 708 self.assertTrue(config._is_dirty) 709 self.assertEqual(2, len(config._pending_uploads)) 710 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 711 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 712 self.assertEqual(self.new_dependencies, config._config_data) 713 file_module = fake_filesystem.FakeFileOpen(self.fs) 714 expected_file_lines = list(self.expected_file_lines) 715 for line in file_module(self.file_path): 716 self.assertEqual(expected_file_lines.pop(0), line.strip()) 717 self.fs.CloseOpenFile(file_module(self.file_path)) 718 self.assertEqual(expected_insert_calls, 719 uploader_cs_mock.Insert.call_args_list) 720 self.assertEqual(expected_exists_calls, 721 uploader_cs_mock.Exists.call_args_list) 722 self.assertEqual(expected_copy_calls, 723 uploader_cs_mock.Copy.call_args_list) 724 self.assertEqual(expected_delete_calls, 725 uploader_cs_mock.Delete.call_args_list) 726 727 @mock.patch('dependency_manager.uploader.cloud_storage') 728 def testExecuteUpdateJobsFailureOnSecondCopyNoCSCollisionNoForce( 729 self, uploader_cs_mock): 730 uploader_cs_mock.Exists.side_effect = [False, True, False] 731 uploader_cs_mock.Copy.side_effect = cloud_storage.CloudStorageError 732 self.fs.CreateFile(self.file_path, 733 contents='\n'.join(self.expected_file_lines)) 734 config = dependency_manager.BaseConfig(self.file_path, writable=True) 735 config._config_data = self.new_dependencies.copy() 736 config._is_dirty = True 737 config._pending_uploads = [self.new_pending_upload, 738 self.final_pending_upload] 739 self.assertEqual(self.new_dependencies, config._config_data) 740 self.assertTrue(config._is_dirty) 741 self.assertEqual(2, len(config._pending_uploads)) 742 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 743 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 744 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), 745 mock.call(self.final_bucket, 746 self.final_remote_path)] 747 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 748 self.new_dep_path)] 749 expected_copy_calls = [] 750 expected_delete_calls = [mock.call(self.new_bucket, self.new_remote_path)] 751 752 self.assertRaises(cloud_storage.CloudStorageError, 753 config.ExecuteUpdateJobs) 754 self.assertTrue(config._is_dirty) 755 self.assertEqual(2, len(config._pending_uploads)) 756 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 757 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 758 self.assertEqual(self.new_dependencies, config._config_data) 759 file_module = fake_filesystem.FakeFileOpen(self.fs) 760 expected_file_lines = list(self.expected_file_lines) 761 for line in file_module(self.file_path): 762 self.assertEqual(expected_file_lines.pop(0), line.strip()) 763 self.fs.CloseOpenFile(file_module(self.file_path)) 764 self.assertEqual(expected_insert_calls, 765 uploader_cs_mock.Insert.call_args_list) 766 self.assertEqual(expected_exists_calls, 767 uploader_cs_mock.Exists.call_args_list) 768 self.assertEqual(expected_copy_calls, 769 uploader_cs_mock.Copy.call_args_list) 770 self.assertEqual(expected_delete_calls, 771 uploader_cs_mock.Delete.call_args_list) 772 773 @mock.patch('dependency_manager.uploader.cloud_storage') 774 def testExecuteUpdateJobsSuccessOnePendingDepNoCloudStorageCollision( 775 self, uploader_cs_mock): 776 uploader_cs_mock.Exists.return_value = False 777 self.fs.CreateFile(self.file_path, 778 contents='\n'.join(self.expected_file_lines)) 779 config = dependency_manager.BaseConfig(self.file_path, writable=True) 780 config._config_data = self.new_dependencies.copy() 781 config._pending_uploads = [self.new_pending_upload] 782 self.assertEqual(self.new_dependencies, config._config_data) 783 self.assertTrue(config._IsDirty()) 784 self.assertEqual(1, len(config._pending_uploads)) 785 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 786 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 787 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 788 self.new_dep_path)] 789 expected_copy_calls = [] 790 expected_delete_calls = [] 791 792 self.assertTrue(config.ExecuteUpdateJobs()) 793 self.assertFalse(config._IsDirty()) 794 self.assertFalse(config._pending_uploads) 795 self.assertEqual(self.new_dependencies, config._config_data) 796 file_module = fake_filesystem.FakeFileOpen(self.fs) 797 expected_file_lines = list(self.new_expected_file_lines) 798 for line in file_module(self.file_path): 799 self.assertEqual(expected_file_lines.pop(0), line.strip()) 800 self.fs.CloseOpenFile(file_module(self.file_path)) 801 self.assertFalse(config._pending_uploads) 802 self.assertEqual(expected_insert_calls, 803 uploader_cs_mock.Insert.call_args_list) 804 self.assertEqual(expected_exists_calls, 805 uploader_cs_mock.Exists.call_args_list) 806 self.assertEqual(expected_copy_calls, 807 uploader_cs_mock.Copy.call_args_list) 808 self.assertEqual(expected_delete_calls, 809 uploader_cs_mock.Delete.call_args_list) 810 811 @mock.patch('dependency_manager.uploader.cloud_storage') 812 def testExecuteUpdateJobsSuccessOnePendingDepCloudStorageCollision( 813 self, uploader_cs_mock): 814 uploader_cs_mock.Exists.return_value = True 815 self.fs.CreateFile(self.file_path, 816 contents='\n'.join(self.expected_file_lines)) 817 config = dependency_manager.BaseConfig(self.file_path, writable=True) 818 config._config_data = self.new_dependencies.copy() 819 config._pending_uploads = [self.new_pending_upload] 820 self.assertEqual(self.new_dependencies, config._config_data) 821 self.assertTrue(config._IsDirty()) 822 self.assertEqual(1, len(config._pending_uploads)) 823 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 824 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 825 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 826 self.new_dep_path)] 827 expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, 828 self.new_remote_path, 829 self.expected_new_backup_path)] 830 831 self.assertTrue(config.ExecuteUpdateJobs(force=True)) 832 self.assertFalse(config._IsDirty()) 833 self.assertFalse(config._pending_uploads) 834 self.assertEqual(self.new_dependencies, config._config_data) 835 file_module = fake_filesystem.FakeFileOpen(self.fs) 836 expected_file_lines = list(self.new_expected_file_lines) 837 for line in file_module(self.file_path): 838 self.assertEqual(expected_file_lines.pop(0), line.strip()) 839 self.fs.CloseOpenFile(file_module(self.file_path)) 840 self.assertFalse(config._pending_uploads) 841 self.assertEqual(expected_insert_calls, 842 uploader_cs_mock.Insert.call_args_list) 843 self.assertEqual(expected_exists_calls, 844 uploader_cs_mock.Exists.call_args_list) 845 self.assertEqual(expected_copy_calls, 846 uploader_cs_mock.Copy.call_args_list) 847 848 @mock.patch('dependency_manager.uploader.cloud_storage') 849 def testExecuteUpdateJobsErrorOnePendingDepCloudStorageCollisionNoForce( 850 self, uploader_cs_mock): 851 uploader_cs_mock.Exists.return_value = True 852 self.fs.CreateFile(self.file_path, 853 contents='\n'.join(self.expected_file_lines)) 854 config = dependency_manager.BaseConfig(self.file_path, writable=True) 855 config._config_data = self.new_dependencies.copy() 856 config._is_dirty = True 857 config._pending_uploads = [self.new_pending_upload] 858 self.assertEqual(self.new_dependencies, config._config_data) 859 self.assertTrue(config._is_dirty) 860 self.assertEqual(1, len(config._pending_uploads)) 861 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 862 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] 863 expected_insert_calls = [] 864 expected_copy_calls = [] 865 866 self.assertRaises(dependency_manager.CloudStorageUploadConflictError, 867 config.ExecuteUpdateJobs) 868 self.assertTrue(config._is_dirty) 869 self.assertTrue(config._pending_uploads) 870 self.assertEqual(self.new_dependencies, config._config_data) 871 self.assertEqual(1, len(config._pending_uploads)) 872 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 873 file_module = fake_filesystem.FakeFileOpen(self.fs) 874 expected_file_lines = list(self.expected_file_lines) 875 for line in file_module(self.file_path): 876 self.assertEqual(expected_file_lines.pop(0), line.strip()) 877 self.fs.CloseOpenFile(file_module(self.file_path)) 878 self.assertEqual(expected_insert_calls, 879 uploader_cs_mock.Insert.call_args_list) 880 self.assertEqual(expected_exists_calls, 881 uploader_cs_mock.Exists.call_args_list) 882 self.assertEqual(expected_copy_calls, 883 uploader_cs_mock.Copy.call_args_list) 884 885 @mock.patch('dependency_manager.uploader.cloud_storage') 886 def testExecuteUpdateJobsSuccessMultiplePendingDepsOneCloudStorageCollision( 887 self, uploader_cs_mock): 888 uploader_cs_mock.Exists.side_effect = [False, True] 889 self.fs.CreateFile(self.file_path, 890 contents='\n'.join(self.expected_file_lines)) 891 config = dependency_manager.BaseConfig(self.file_path, writable=True) 892 config._config_data = self.final_dependencies.copy() 893 config._pending_uploads = [self.new_pending_upload, 894 self.final_pending_upload] 895 self.assertEqual(self.final_dependencies, config._config_data) 896 self.assertTrue(config._IsDirty()) 897 self.assertEqual(2, len(config._pending_uploads)) 898 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 899 self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) 900 901 expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), 902 mock.call(self.final_bucket, 903 self.final_remote_path)] 904 expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, 905 self.new_dep_path), 906 mock.call(self.final_bucket, 907 self.final_remote_path, 908 self.final_dep_path)] 909 expected_copy_calls = [mock.call(self.final_bucket, self.final_bucket, 910 self.final_remote_path, 911 self.expected_final_backup_path)] 912 913 self.assertTrue(config.ExecuteUpdateJobs(force=True)) 914 self.assertFalse(config._IsDirty()) 915 self.assertFalse(config._pending_uploads) 916 self.assertEqual(self.final_dependencies, config._config_data) 917 file_module = fake_filesystem.FakeFileOpen(self.fs) 918 expected_file_lines = list(self.final_expected_file_lines) 919 for line in file_module(self.file_path): 920 self.assertEqual(expected_file_lines.pop(0), line.strip()) 921 self.fs.CloseOpenFile(file_module(self.file_path)) 922 self.assertFalse(config._pending_uploads) 923 self.assertEqual(expected_insert_calls, 924 uploader_cs_mock.Insert.call_args_list) 925 self.assertEqual(expected_exists_calls, 926 uploader_cs_mock.Exists.call_args_list) 927 self.assertEqual(expected_copy_calls, 928 uploader_cs_mock.Copy.call_args_list) 929 930 @mock.patch('dependency_manager.uploader.cloud_storage') 931 def testUpdateCloudStorageDependenciesReadOnlyConfig( 932 self, uploader_cs_mock): 933 self.fs.CreateFile(self.file_path, 934 contents='\n'.join(self.expected_file_lines)) 935 config = dependency_manager.BaseConfig(self.file_path) 936 with self.assertRaises(dependency_manager.ReadWriteError): 937 config.AddCloudStorageDependencyUpdateJob( 938 'dep', 'plat', 'path') 939 with self.assertRaises(dependency_manager.ReadWriteError): 940 config.AddCloudStorageDependencyUpdateJob( 941 'dep', 'plat', 'path', version='1.2.3') 942 with self.assertRaises(dependency_manager.ReadWriteError): 943 config.AddCloudStorageDependencyUpdateJob( 944 'dep', 'plat', 'path', execute_job=False) 945 with self.assertRaises(dependency_manager.ReadWriteError): 946 config.AddCloudStorageDependencyUpdateJob( 947 'dep', 'plat', 'path', version='1.2.3', execute_job=False) 948 949 @mock.patch('dependency_manager.uploader.cloud_storage') 950 def testUpdateCloudStorageDependenciesMissingDependency( 951 self, uploader_cs_mock): 952 self.fs.CreateFile(self.file_path, 953 contents='\n'.join(self.expected_file_lines)) 954 config = dependency_manager.BaseConfig(self.file_path, writable=True) 955 self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, 956 'dep', 'plat', 'path') 957 self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, 958 'dep', 'plat', 'path', version='1.2.3') 959 self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, 960 'dep', 'plat', 'path', execute_job=False) 961 self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, 962 'dep', 'plat', 'path', version='1.2.3', execute_job=False) 963 964 @mock.patch('dependency_manager.uploader.cloud_storage') 965 @mock.patch('dependency_manager.base_config.cloud_storage') 966 def testUpdateCloudStorageDependenciesWrite( 967 self, base_config_cs_mock, uploader_cs_mock): 968 expected_dependencies = self.dependencies 969 self.fs.CreateFile(self.file_path, 970 contents='\n'.join(self.expected_file_lines)) 971 config = dependency_manager.BaseConfig(self.file_path, writable=True) 972 self.assertFalse(config._IsDirty()) 973 self.assertEqual(expected_dependencies, config._config_data) 974 975 base_config_cs_mock.CalculateHash.return_value = self.new_dep_hash 976 uploader_cs_mock.Exists.return_value = False 977 expected_dependencies = self.new_dependencies 978 config.AddCloudStorageDependencyUpdateJob( 979 'dep1', 'plat2', self.new_dep_path, execute_job=True) 980 self.assertFalse(config._IsDirty()) 981 self.assertFalse(config._pending_uploads) 982 self.assertEqual(expected_dependencies, config._config_data) 983 # check that file contents has been updated 984 file_module = fake_filesystem.FakeFileOpen(self.fs) 985 expected_file_lines = list(self.new_expected_file_lines) 986 for line in file_module(self.file_path): 987 self.assertEqual(expected_file_lines.pop(0), line.strip()) 988 self.fs.CloseOpenFile(file_module(self.file_path)) 989 990 expected_dependencies = self.final_dependencies 991 base_config_cs_mock.CalculateHash.return_value = self.final_dep_hash 992 config.AddCloudStorageDependencyUpdateJob( 993 'dep2', 'plat1', self.final_dep_path, execute_job=True) 994 self.assertFalse(config._IsDirty()) 995 self.assertFalse(config._pending_uploads) 996 self.assertEqual(expected_dependencies, config._config_data) 997 # check that file contents has been updated 998 expected_file_lines = list(self.final_expected_file_lines) 999 file_module = fake_filesystem.FakeFileOpen(self.fs) 1000 for line in file_module(self.file_path): 1001 self.assertEqual(expected_file_lines.pop(0), line.strip()) 1002 self.fs.CloseOpenFile(file_module(self.file_path)) 1003 1004 @mock.patch('dependency_manager.uploader.cloud_storage') 1005 @mock.patch('dependency_manager.base_config.cloud_storage') 1006 def testUpdateCloudStorageDependenciesNoWrite( 1007 self, base_config_cs_mock, uploader_cs_mock): 1008 self.fs.CreateFile(self.file_path, 1009 contents='\n'.join(self.expected_file_lines)) 1010 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1011 1012 self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, 1013 'dep', 'plat', 'path') 1014 self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, 1015 'dep', 'plat', 'path', version='1.2.3') 1016 1017 expected_dependencies = self.dependencies 1018 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1019 self.assertFalse(config._IsDirty()) 1020 self.assertFalse(config._pending_uploads) 1021 self.assertEqual(expected_dependencies, config._config_data) 1022 1023 base_config_cs_mock.CalculateHash.return_value = self.new_dep_hash 1024 uploader_cs_mock.Exists.return_value = False 1025 expected_dependencies = self.new_dependencies 1026 config.AddCloudStorageDependencyUpdateJob( 1027 'dep1', 'plat2', self.new_dep_path, execute_job=False) 1028 self.assertTrue(config._IsDirty()) 1029 self.assertEqual(1, len(config._pending_uploads)) 1030 self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) 1031 self.assertEqual(expected_dependencies, config._config_data) 1032 # check that file contents have not been updated. 1033 expected_file_lines = list(self.expected_file_lines) 1034 file_module = fake_filesystem.FakeFileOpen(self.fs) 1035 for line in file_module(self.file_path): 1036 self.assertEqual(expected_file_lines.pop(0), line.strip()) 1037 self.fs.CloseOpenFile(file_module(self.file_path)) 1038 1039 expected_dependencies = self.final_dependencies 1040 base_config_cs_mock.CalculateHash.return_value = self.final_dep_hash 1041 config.AddCloudStorageDependencyUpdateJob( 1042 'dep2', 'plat1', self.final_dep_path, execute_job=False) 1043 self.assertTrue(config._IsDirty()) 1044 self.assertEqual(expected_dependencies, config._config_data) 1045 # check that file contents have not been updated. 1046 expected_file_lines = list(self.expected_file_lines) 1047 file_module = fake_filesystem.FakeFileOpen(self.fs) 1048 for line in file_module(self.file_path): 1049 self.assertEqual(expected_file_lines.pop(0), line.strip()) 1050 self.fs.CloseOpenFile(file_module(self.file_path)) 1051 1052 1053class BaseConfigDataManipulationUnittests(fake_filesystem_unittest.TestCase): 1054 def setUp(self): 1055 self.addTypeEqualityFunc(uploader.CloudStorageUploader, 1056 uploader.CloudStorageUploader.__eq__) 1057 self.setUpPyfakefs() 1058 1059 self.cs_bucket = 'bucket1' 1060 self.cs_base_folder = 'dependencies_folder' 1061 self.cs_hash = 'hash12' 1062 self.download_path = '../../relative/dep1/path2' 1063 self.local_paths = ['../../../relative/local/path21', 1064 '../../../relative/local/path22'] 1065 self.platform_dict = {'cloud_storage_hash': self.cs_hash, 1066 'download_path': self.download_path, 1067 'local_paths': self.local_paths} 1068 self.dependencies = { 1069 'dep1': { 1070 'cloud_storage_bucket': self.cs_bucket, 1071 'cloud_storage_base_folder': self.cs_base_folder, 1072 'file_info': { 1073 'plat1': { 1074 'cloud_storage_hash': 'hash11', 1075 'download_path': '../../relative/dep1/path1', 1076 'local_paths': ['../../../relative/local/path11', 1077 '../../../relative/local/path12']}, 1078 'plat2': self.platform_dict 1079 } 1080 }, 1081 'dep2': { 1082 'cloud_storage_bucket': 'bucket2', 1083 'file_info': { 1084 'plat1': { 1085 'cloud_storage_hash': 'hash21', 1086 'download_path': '../../relative/dep2/path1', 1087 'local_paths': ['../../../relative/local/path31', 1088 '../../../relative/local/path32']}, 1089 'plat2': { 1090 'cloud_storage_hash': 'hash22', 1091 'download_path': '../../relative/dep2/path2'}}}} 1092 1093 self.file_path = os.path.abspath(os.path.join( 1094 'path', 'to', 'config', 'file')) 1095 1096 1097 self.expected_file_lines = [ 1098 # pylint: disable=bad-continuation 1099 '{', '"config_type": "BaseConfig",', '"dependencies": {', 1100 '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', 1101 '"cloud_storage_bucket": "bucket1",', '"file_info": {', 1102 '"plat1": {', '"cloud_storage_hash": "hash11",', 1103 '"download_path": "../../relative/dep1/path1",', 1104 '"local_paths": [', '"../../../relative/local/path11",', 1105 '"../../../relative/local/path12"', ']', '},', 1106 '"plat2": {', '"cloud_storage_hash": "hash12",', 1107 '"download_path": "../../relative/dep1/path2",', 1108 '"local_paths": [', '"../../../relative/local/path21",', 1109 '"../../../relative/local/path22"', ']', 1110 '}', '}', '},', 1111 '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', 1112 '"plat1": {', '"cloud_storage_hash": "hash21",', 1113 '"download_path": "../../relative/dep2/path1",', 1114 '"local_paths": [', '"../../../relative/local/path31",', 1115 '"../../../relative/local/path32"', ']', '},', 1116 '"plat2": {', '"cloud_storage_hash": "hash22",', 1117 '"download_path": "../../relative/dep2/path2"', '}', '}', '}', 1118 '}', '}'] 1119 self.fs.CreateFile(self.file_path, 1120 contents='\n'.join(self.expected_file_lines)) 1121 1122 1123 def testSetPlatformDataFailureNotWritable(self): 1124 config = dependency_manager.BaseConfig(self.file_path) 1125 self.assertRaises( 1126 dependency_manager.ReadWriteError, config._SetPlatformData, 1127 'dep1', 'plat1', 'cloud_storage_bucket', 'new_bucket') 1128 self.assertEqual(self.dependencies, config._config_data) 1129 1130 def testSetPlatformDataFailure(self): 1131 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1132 self.assertRaises(ValueError, config._SetPlatformData, 'missing_dep', 1133 'plat2', 'cloud_storage_bucket', 'new_bucket') 1134 self.assertEqual(self.dependencies, config._config_data) 1135 self.assertRaises(ValueError, config._SetPlatformData, 'dep1', 1136 'missing_plat', 'cloud_storage_bucket', 'new_bucket') 1137 self.assertEqual(self.dependencies, config._config_data) 1138 1139 1140 def testSetPlatformDataCloudStorageBucketSuccess(self): 1141 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1142 updated_cs_dependencies = { 1143 'dep1': {'cloud_storage_bucket': 'new_bucket', 1144 'cloud_storage_base_folder': 'dependencies_folder', 1145 'file_info': { 1146 'plat1': { 1147 'cloud_storage_hash': 'hash11', 1148 'download_path': '../../relative/dep1/path1', 1149 'local_paths': ['../../../relative/local/path11', 1150 '../../../relative/local/path12']}, 1151 'plat2': { 1152 'cloud_storage_hash': 'hash12', 1153 'download_path': '../../relative/dep1/path2', 1154 'local_paths': ['../../../relative/local/path21', 1155 '../../../relative/local/path22']}}}, 1156 'dep2': {'cloud_storage_bucket': 'bucket2', 1157 'file_info': { 1158 'plat1': { 1159 'cloud_storage_hash': 'hash21', 1160 'download_path': '../../relative/dep2/path1', 1161 'local_paths': ['../../../relative/local/path31', 1162 '../../../relative/local/path32']}, 1163 'plat2': { 1164 'cloud_storage_hash': 'hash22', 1165 'download_path': '../../relative/dep2/path2'}}}} 1166 config._SetPlatformData('dep1', 'plat2', 'cloud_storage_bucket', 1167 'new_bucket') 1168 self.assertEqual(updated_cs_dependencies, config._config_data) 1169 1170 def testSetPlatformDataCloudStorageBaseFolderSuccess(self): 1171 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1172 updated_cs_dependencies = { 1173 'dep1': {'cloud_storage_bucket': 'bucket1', 1174 'cloud_storage_base_folder': 'new_dependencies_folder', 1175 'file_info': { 1176 'plat1': { 1177 'cloud_storage_hash': 'hash11', 1178 'download_path': '../../relative/dep1/path1', 1179 'local_paths': ['../../../relative/local/path11', 1180 '../../../relative/local/path12']}, 1181 'plat2': { 1182 'cloud_storage_hash': 'hash12', 1183 'download_path': '../../relative/dep1/path2', 1184 'local_paths': ['../../../relative/local/path21', 1185 '../../../relative/local/path22']}}}, 1186 'dep2': {'cloud_storage_bucket': 'bucket2', 1187 'file_info': { 1188 'plat1': { 1189 'cloud_storage_hash': 'hash21', 1190 'download_path': '../../relative/dep2/path1', 1191 'local_paths': ['../../../relative/local/path31', 1192 '../../../relative/local/path32']}, 1193 'plat2': { 1194 'cloud_storage_hash': 'hash22', 1195 'download_path': '../../relative/dep2/path2'}}}} 1196 config._SetPlatformData('dep1', 'plat2', 'cloud_storage_base_folder', 1197 'new_dependencies_folder') 1198 self.assertEqual(updated_cs_dependencies, config._config_data) 1199 1200 def testSetPlatformDataHashSuccess(self): 1201 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1202 updated_cs_dependencies = { 1203 'dep1': {'cloud_storage_bucket': 'bucket1', 1204 'cloud_storage_base_folder': 'dependencies_folder', 1205 'file_info': { 1206 'plat1': { 1207 'cloud_storage_hash': 'hash11', 1208 'download_path': '../../relative/dep1/path1', 1209 'local_paths': ['../../../relative/local/path11', 1210 '../../../relative/local/path12']}, 1211 'plat2': { 1212 'cloud_storage_hash': 'new_hash', 1213 'download_path': '../../relative/dep1/path2', 1214 'local_paths': ['../../../relative/local/path21', 1215 '../../../relative/local/path22']}}}, 1216 'dep2': {'cloud_storage_bucket': 'bucket2', 1217 'file_info': { 1218 'plat1': { 1219 'cloud_storage_hash': 'hash21', 1220 'download_path': '../../relative/dep2/path1', 1221 'local_paths': ['../../../relative/local/path31', 1222 '../../../relative/local/path32']}, 1223 'plat2': { 1224 'cloud_storage_hash': 'hash22', 1225 'download_path': '../../relative/dep2/path2'}}}} 1226 config._SetPlatformData('dep1', 'plat2', 'cloud_storage_hash', 1227 'new_hash') 1228 self.assertEqual(updated_cs_dependencies, config._config_data) 1229 1230 def testSetPlatformDataDownloadPathSuccess(self): 1231 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1232 updated_cs_dependencies = { 1233 'dep1': {'cloud_storage_bucket': 'bucket1', 1234 'cloud_storage_base_folder': 'dependencies_folder', 1235 'file_info': { 1236 'plat1': { 1237 'cloud_storage_hash': 'hash11', 1238 'download_path': '../../relative/dep1/path1', 1239 'local_paths': ['../../../relative/local/path11', 1240 '../../../relative/local/path12']}, 1241 'plat2': { 1242 'cloud_storage_hash': 'hash12', 1243 'download_path': '../../new/dep1/path2', 1244 'local_paths': ['../../../relative/local/path21', 1245 '../../../relative/local/path22']}}}, 1246 'dep2': {'cloud_storage_bucket': 'bucket2', 1247 'file_info': { 1248 'plat1': { 1249 'cloud_storage_hash': 'hash21', 1250 'download_path': '../../relative/dep2/path1', 1251 'local_paths': ['../../../relative/local/path31', 1252 '../../../relative/local/path32']}, 1253 'plat2': { 1254 'cloud_storage_hash': 'hash22', 1255 'download_path': '../../relative/dep2/path2'}}}} 1256 config._SetPlatformData('dep1', 'plat2', 'download_path', 1257 '../../new/dep1/path2') 1258 self.assertEqual(updated_cs_dependencies, config._config_data) 1259 1260 def testSetPlatformDataLocalPathSuccess(self): 1261 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1262 updated_cs_dependencies = { 1263 'dep1': {'cloud_storage_bucket': 'bucket1', 1264 'cloud_storage_base_folder': 'dependencies_folder', 1265 'file_info': { 1266 'plat1': { 1267 'cloud_storage_hash': 'hash11', 1268 'download_path': '../../relative/dep1/path1', 1269 'local_paths': ['../../../relative/local/path11', 1270 '../../../relative/local/path12']}, 1271 'plat2': { 1272 'cloud_storage_hash': 'hash12', 1273 'download_path': '../../relative/dep1/path2', 1274 'local_paths': ['../../new/relative/local/path21', 1275 '../../new/relative/local/path22']}}}, 1276 'dep2': {'cloud_storage_bucket': 'bucket2', 1277 'file_info': { 1278 'plat1': { 1279 'cloud_storage_hash': 'hash21', 1280 'download_path': '../../relative/dep2/path1', 1281 'local_paths': ['../../../relative/local/path31', 1282 '../../../relative/local/path32']}, 1283 'plat2': { 1284 'cloud_storage_hash': 'hash22', 1285 'download_path': '../../relative/dep2/path2'}}}} 1286 config._SetPlatformData('dep1', 'plat2', 'local_paths', 1287 ['../../new/relative/local/path21', 1288 '../../new/relative/local/path22']) 1289 self.assertEqual(updated_cs_dependencies, config._config_data) 1290 1291 def testGetPlatformDataFailure(self): 1292 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1293 self.assertRaises(ValueError, config._GetPlatformData, 'missing_dep', 1294 'plat2', 'cloud_storage_bucket') 1295 self.assertEqual(self.dependencies, config._config_data) 1296 self.assertRaises(ValueError, config._GetPlatformData, 'dep1', 1297 'missing_plat', 'cloud_storage_bucket') 1298 self.assertEqual(self.dependencies, config._config_data) 1299 1300 def testGetPlatformDataDictSuccess(self): 1301 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1302 self.assertEqual(self.platform_dict, 1303 config._GetPlatformData('dep1', 'plat2')) 1304 self.assertEqual(self.dependencies, config._config_data) 1305 1306 def testGetPlatformDataCloudStorageBucketSuccess(self): 1307 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1308 self.assertEqual(self.cs_bucket, config._GetPlatformData( 1309 'dep1', 'plat2', 'cloud_storage_bucket')) 1310 self.assertEqual(self.dependencies, config._config_data) 1311 1312 def testGetPlatformDataCloudStorageBaseFolderSuccess(self): 1313 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1314 self.assertEqual(self.cs_base_folder, config._GetPlatformData( 1315 'dep1', 'plat2', 'cloud_storage_base_folder')) 1316 self.assertEqual(self.dependencies, config._config_data) 1317 1318 def testGetPlatformDataHashSuccess(self): 1319 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1320 self.assertEqual(self.cs_hash, config._GetPlatformData( 1321 'dep1', 'plat2', 'cloud_storage_hash')) 1322 self.assertEqual(self.dependencies, config._config_data) 1323 1324 def testGetPlatformDataDownloadPathSuccess(self): 1325 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1326 self.assertEqual(self.download_path, config._GetPlatformData( 1327 'dep1', 'plat2', 'download_path')) 1328 self.assertEqual(self.dependencies, config._config_data) 1329 1330 def testGetPlatformDataLocalPathSuccess(self): 1331 config = dependency_manager.BaseConfig(self.file_path, writable=True) 1332 self.assertEqual(self.local_paths, config._GetPlatformData( 1333 'dep1', 'plat2', 'local_paths')) 1334 self.assertEqual(self.dependencies, config._config_data) 1335 1336class BaseConfigTest(unittest.TestCase): 1337 """ Subclassable unittests for BaseConfig. 1338 For subclasses: override setUp, GetConfigDataFromDict, 1339 and EndToEndExpectedConfigData as needed. 1340 1341 setUp must set the following properties: 1342 self.config_type: String returnedd from GetConfigType in config subclass. 1343 self.config_class: the class for the config subclass. 1344 self.config_module: importable module for the config subclass. 1345 self.empty_dict: expected dictionary for an empty config, as it would be 1346 stored in a json file. 1347 self.one_dep_dict: example dictionary for a config with one dependency, 1348 as it would be stored in a json file. 1349 """ 1350 def setUp(self): 1351 self.config_type = 'BaseConfig' 1352 self.config_class = dependency_manager.BaseConfig 1353 self.config_module = 'dependency_manager.base_config' 1354 1355 self.empty_dict = {'config_type': self.config_type, 1356 'dependencies': {}} 1357 1358 dependency_dict = { 1359 'dep': { 1360 'cloud_storage_base_folder': 'cs_base_folder1', 1361 'cloud_storage_bucket': 'bucket1', 1362 'file_info': { 1363 'plat1_arch1': { 1364 'cloud_storage_hash': 'hash111', 1365 'download_path': 'download_path111', 1366 'cs_remote_path': 'cs_path111', 1367 'version_in_cs': 'version_111', 1368 'local_paths': ['local_path1110', 'local_path1111'] 1369 }, 1370 'plat1_arch2': { 1371 'cloud_storage_hash': 'hash112', 1372 'download_path': 'download_path112', 1373 'cs_remote_path': 'cs_path112', 1374 'local_paths': ['local_path1120', 'local_path1121'] 1375 }, 1376 'win_arch1': { 1377 'cloud_storage_hash': 'hash1w1', 1378 'download_path': 'download\\path\\1w1', 1379 'cs_remote_path': 'cs_path1w1', 1380 'local_paths': ['local\\path\\1w10', 'local\\path\\1w11'] 1381 }, 1382 'all_the_variables': { 1383 'cloud_storage_hash': 'hash111', 1384 'download_path': 'download_path111', 1385 'cs_remote_path': 'cs_path111', 1386 'version_in_cs': 'version_111', 1387 'path_within_archive': 'path/within/archive', 1388 'local_paths': ['local_path1110', 'local_path1111'] 1389 } 1390 } 1391 } 1392 } 1393 self.one_dep_dict = {'config_type': self.config_type, 1394 'dependencies': dependency_dict} 1395 1396 def GetConfigDataFromDict(self, config_dict): 1397 return config_dict.get('dependencies', {}) 1398 1399 @mock.patch('os.path') 1400 @mock.patch('__builtin__.open') 1401 def testInitBaseProperties(self, open_mock, path_mock): 1402 # Init is not meant to be overridden, so we should be mocking the 1403 # base_config's json module, even in subclasses. 1404 json_module = 'dependency_manager.base_config.json' 1405 with mock.patch(json_module) as json_mock: 1406 json_mock.load.return_value = self.empty_dict.copy() 1407 config = self.config_class('file_path') 1408 self.assertEqual('file_path', config._config_path) 1409 self.assertEqual(self.config_type, config.GetConfigType()) 1410 self.assertEqual(self.GetConfigDataFromDict(self.empty_dict), 1411 config._config_data) 1412 1413 1414 @mock.patch('dependency_manager.dependency_info.DependencyInfo') 1415 @mock.patch('os.path') 1416 @mock.patch('__builtin__.open') 1417 def testInitWithDependencies(self, open_mock, path_mock, dep_info_mock): 1418 # Init is not meant to be overridden, so we should be mocking the 1419 # base_config's json module, even in subclasses. 1420 json_module = 'dependency_manager.base_config.json' 1421 with mock.patch(json_module) as json_mock: 1422 json_mock.load.return_value = self.one_dep_dict 1423 config = self.config_class('file_path') 1424 self.assertEqual('file_path', config._config_path) 1425 self.assertEqual(self.config_type, config.GetConfigType()) 1426 self.assertEqual(self.GetConfigDataFromDict(self.one_dep_dict), 1427 config._config_data) 1428 1429 def testFormatPath(self): 1430 self.assertEqual(None, self.config_class._FormatPath(None)) 1431 self.assertEqual('', self.config_class._FormatPath('')) 1432 self.assertEqual('some_string', 1433 self.config_class._FormatPath('some_string')) 1434 1435 expected_path = os.path.join('some', 'file', 'path') 1436 self.assertEqual(expected_path, 1437 self.config_class._FormatPath('some/file/path')) 1438 self.assertEqual(expected_path, 1439 self.config_class._FormatPath('some\\file\\path')) 1440 1441 @mock.patch('dependency_manager.base_config.json') 1442 @mock.patch('dependency_manager.dependency_info.DependencyInfo') 1443 @mock.patch('os.path.exists') 1444 @mock.patch('__builtin__.open') 1445 def testIterDependenciesError( 1446 self, open_mock, exists_mock, dep_info_mock, json_mock): 1447 # Init is not meant to be overridden, so we should be mocking the 1448 # base_config's json module, even in subclasses. 1449 json_mock.load.return_value = self.one_dep_dict 1450 config = self.config_class('file_path', writable=True) 1451 self.assertEqual(self.GetConfigDataFromDict(self.one_dep_dict), 1452 config._config_data) 1453 self.assertTrue(config._writable) 1454 with self.assertRaises(dependency_manager.ReadWriteError): 1455 for _ in config.IterDependencyInfo(): 1456 pass 1457 1458 @mock.patch('dependency_manager.base_config.json') 1459 @mock.patch('dependency_manager.dependency_info.DependencyInfo') 1460 @mock.patch('os.path.exists') 1461 @mock.patch('__builtin__.open') 1462 def testIterDependencies( 1463 self, open_mock, exists_mock, dep_info_mock, json_mock): 1464 json_mock.load.return_value = self.one_dep_dict 1465 config = self.config_class('file_path') 1466 self.assertEqual(self.GetConfigDataFromDict(self.one_dep_dict), 1467 config._config_data) 1468 expected_dep_info = ['dep_info0', 'dep_info1', 'dep_info2'] 1469 dep_info_mock.side_effect = expected_dep_info 1470 expected_calls = [ 1471 mock.call('dep', 'plat1_arch1', 'file_path', cs_bucket='bucket1', 1472 cs_hash='hash111', download_path='download_path111', 1473 cs_remote_path='cs_path111', 1474 local_paths=['local_path1110', 'local_path1111']), 1475 mock.call('dep', 'plat1_arch1', 'file_path', cs_bucket='bucket1', 1476 cs_hash='hash112', download_path='download_path112', 1477 cs_remote_path='cs_path112', 1478 local_paths=['local_path1120', 'local_path1121']), 1479 mock.call('dep', 'win_arch1', 'file_path', cs_bucket='bucket1', 1480 cs_hash='hash1w1', 1481 download_path=os.path.join('download', 'path', '1w1'), 1482 cs_remote_path='cs_path1w1', 1483 local_paths=[os.path.join('download', 'path', '1w10'), 1484 os.path.join('download', 'path', '1w11')])] 1485 deps_seen = [] 1486 for dep_info in config.IterDependencyInfo(): 1487 deps_seen.append(dep_info) 1488 dep_info_mock.assert_call_args(expected_calls) 1489 self.assertItemsEqual(expected_dep_info, deps_seen) 1490 1491 @mock.patch('dependency_manager.base_config.json') 1492 @mock.patch('os.path.exists') 1493 @mock.patch('__builtin__.open') 1494 def testIterDependenciesStaleGlob(self, open_mock, exists_mock, json_mock): 1495 json_mock.load.return_value = self.one_dep_dict 1496 config = self.config_class('file_path') 1497 1498 abspath = os.path.abspath 1499 should_match = set(map(abspath, [ 1500 'dep_all_the_variables_0123456789abcdef0123456789abcdef01234567', 1501 'dep_all_the_variables_123456789abcdef0123456789abcdef012345678'])) 1502 # Not testing case changes, because Windows is case-insensitive. 1503 should_not_match = set(map(abspath, [ 1504 # A configuration that doesn't unzip shouldn't clear any stale unzips. 1505 'dep_plat1_arch1_0123456789abcdef0123456789abcdef01234567', 1506 # "Hash" component less than 40 characters (not a valid SHA1 hash). 1507 'dep_all_the_variables_0123456789abcdef0123456789abcdef0123456', 1508 # "Hash" component greater than 40 characters (not a valid SHA1 hash). 1509 'dep_all_the_variables_0123456789abcdef0123456789abcdef012345678', 1510 # "Hash" component not comprised of hex (not a valid SHA1 hash). 1511 'dep_all_the_variables_0123456789gggggg0123456789gggggg01234567'])) 1512 1513 # Create a fake filesystem just for glob to use 1514 fake_fs = fake_filesystem.FakeFilesystem() 1515 fake_glob = fake_filesystem_glob.FakeGlobModule(fake_fs) 1516 for stale_dir in set.union(should_match, should_not_match): 1517 fake_fs.CreateDirectory(stale_dir) 1518 fake_fs.CreateFile(os.path.join(stale_dir, 'some_file')) 1519 1520 for dep_info in config.IterDependencyInfo(): 1521 if dep_info.platform == 'all_the_variables': 1522 cs_info = dep_info.cloud_storage_info 1523 actual_glob = cs_info._archive_info._stale_unzip_path_glob 1524 actual_matches = set(fake_glob.glob(actual_glob)) 1525 self.assertItemsEqual(should_match, actual_matches) 1526