1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import sys 7import unittest 8 9import mock 10from pyfakefs import fake_filesystem_unittest 11 12 13from catapult_base import cloud_storage 14from catapult_base import util 15 16 17def _FakeReadHash(_): 18 return 'hashthis!' 19 20 21def _FakeCalulateHashMatchesRead(_): 22 return 'hashthis!' 23 24 25def _FakeCalulateHashNewHash(_): 26 return 'omgnewhash' 27 28 29class CloudStorageUnitTest(fake_filesystem_unittest.TestCase): 30 31 def setUp(self): 32 self.original_environ = os.environ.copy() 33 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '' 34 self.setUpPyfakefs() 35 self.fs.CreateFile( 36 os.path.join(util.GetCatapultDir(), 'third_party', 'gsutil', 'gsutil')) 37 38 def CreateFiles(self, file_paths): 39 for f in file_paths: 40 self.fs.CreateFile(f) 41 42 def tearDown(self): 43 self.tearDownPyfakefs() 44 os.environ = self.original_environ 45 46 def _FakeRunCommand(self, cmd): 47 pass 48 49 def _FakeGet(self, bucket, remote_path, local_path): 50 pass 51 52 def _AssertRunCommandRaisesError(self, communicate_strs, error): 53 with mock.patch('catapult_base.cloud_storage.subprocess.Popen') as popen: 54 p_mock = mock.Mock() 55 popen.return_value = p_mock 56 p_mock.returncode = 1 57 for stderr in communicate_strs: 58 p_mock.communicate.return_value = ('', stderr) 59 self.assertRaises(error, cloud_storage._RunCommand, []) 60 61 def testRunCommandCredentialsError(self): 62 strs = ['You are attempting to access protected data with no configured', 63 'Failure: No handler was ready to authenticate.'] 64 self._AssertRunCommandRaisesError(strs, cloud_storage.CredentialsError) 65 66 def testRunCommandPermissionError(self): 67 strs = ['status=403', 'status 403', '403 Forbidden'] 68 self._AssertRunCommandRaisesError(strs, cloud_storage.PermissionError) 69 70 def testRunCommandNotFoundError(self): 71 strs = ['InvalidUriError', 'No such object', 'No URLs matched', 72 'One or more URLs matched no', 'InvalidUriError'] 73 self._AssertRunCommandRaisesError(strs, cloud_storage.NotFoundError) 74 75 def testRunCommandServerError(self): 76 strs = ['500 Internal Server Error'] 77 self._AssertRunCommandRaisesError(strs, cloud_storage.ServerError) 78 79 def testRunCommandGenericError(self): 80 strs = ['Random string'] 81 self._AssertRunCommandRaisesError(strs, cloud_storage.CloudStorageError) 82 83 def testInsertCreatesValidCloudUrl(self): 84 orig_run_command = cloud_storage._RunCommand 85 try: 86 cloud_storage._RunCommand = self._FakeRunCommand 87 remote_path = 'test-remote-path.html' 88 local_path = 'test-local-path.html' 89 cloud_url = cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET, 90 remote_path, local_path) 91 self.assertEqual('https://console.developers.google.com/m/cloudstorage' 92 '/b/chromium-telemetry/o/test-remote-path.html', 93 cloud_url) 94 finally: 95 cloud_storage._RunCommand = orig_run_command 96 97 @mock.patch('catapult_base.cloud_storage.subprocess') 98 def testExistsReturnsFalse(self, subprocess_mock): 99 p_mock = mock.Mock() 100 subprocess_mock.Popen.return_value = p_mock 101 p_mock.communicate.return_value = ( 102 '', 103 'CommandException: One or more URLs matched no objects.\n') 104 p_mock.returncode_result = 1 105 self.assertFalse(cloud_storage.Exists('fake bucket', 106 'fake remote path')) 107 108 @mock.patch('catapult_base.cloud_storage.CalculateHash') 109 @mock.patch('catapult_base.cloud_storage._GetLocked') 110 @mock.patch('catapult_base.cloud_storage._PseudoFileLock') 111 @mock.patch('catapult_base.cloud_storage.os.path') 112 def testGetIfHashChanged(self, path_mock, unused_lock_mock, get_mock, 113 calc_hash_mock): 114 path_mock.exists.side_effect = [False, True, True] 115 calc_hash_mock.return_value = 'hash' 116 117 # The file at |local_path| doesn't exist. We should download file from cs. 118 ret = cloud_storage.GetIfHashChanged( 119 'remote_path', 'local_path', 'cs_bucket', 'hash') 120 self.assertTrue(ret) 121 get_mock.assert_called_once_with('cs_bucket', 'remote_path', 'local_path') 122 get_mock.reset_mock() 123 self.assertFalse(calc_hash_mock.call_args) 124 calc_hash_mock.reset_mock() 125 126 # A local file exists at |local_path| but has the wrong hash. 127 # We should download file from cs. 128 ret = cloud_storage.GetIfHashChanged( 129 'remote_path', 'local_path', 'cs_bucket', 'new_hash') 130 self.assertTrue(ret) 131 get_mock.assert_called_once_with('cs_bucket', 'remote_path', 'local_path') 132 get_mock.reset_mock() 133 calc_hash_mock.assert_called_once_with('local_path') 134 calc_hash_mock.reset_mock() 135 136 # Downloaded file exists locally and has the right hash. Don't download. 137 ret = cloud_storage.GetIfHashChanged( 138 'remote_path', 'local_path', 'cs_bucket', 'hash') 139 self.assertFalse(get_mock.call_args) 140 self.assertFalse(ret) 141 calc_hash_mock.reset_mock() 142 get_mock.reset_mock() 143 144 @mock.patch('catapult_base.cloud_storage._PseudoFileLock') 145 def testGetIfChanged(self, unused_lock_mock): 146 orig_get = cloud_storage._GetLocked 147 orig_read_hash = cloud_storage.ReadHash 148 orig_calculate_hash = cloud_storage.CalculateHash 149 cloud_storage.ReadHash = _FakeReadHash 150 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 151 file_path = 'test-file-path.wpr' 152 hash_path = file_path + '.sha1' 153 try: 154 cloud_storage._GetLocked = self._FakeGet 155 # hash_path doesn't exist. 156 self.assertFalse(cloud_storage.GetIfChanged(file_path, 157 cloud_storage.PUBLIC_BUCKET)) 158 # hash_path exists, but file_path doesn't. 159 self.CreateFiles([hash_path]) 160 self.assertTrue(cloud_storage.GetIfChanged(file_path, 161 cloud_storage.PUBLIC_BUCKET)) 162 # hash_path and file_path exist, and have same hash. 163 self.CreateFiles([file_path]) 164 self.assertFalse(cloud_storage.GetIfChanged(file_path, 165 cloud_storage.PUBLIC_BUCKET)) 166 # hash_path and file_path exist, and have different hashes. 167 cloud_storage.CalculateHash = _FakeCalulateHashNewHash 168 self.assertTrue(cloud_storage.GetIfChanged(file_path, 169 cloud_storage.PUBLIC_BUCKET)) 170 finally: 171 cloud_storage._GetLocked = orig_get 172 cloud_storage.CalculateHash = orig_calculate_hash 173 cloud_storage.ReadHash = orig_read_hash 174 175 @unittest.skipIf(sys.platform.startswith('win'), 176 'https://github.com/catapult-project/catapult/issues/1861') 177 def testGetFilesInDirectoryIfChanged(self): 178 self.CreateFiles([ 179 'real_dir_path/dir1/1file1.sha1', 180 'real_dir_path/dir1/1file2.txt', 181 'real_dir_path/dir1/1file3.sha1', 182 'real_dir_path/dir2/2file.txt', 183 'real_dir_path/dir3/3file1.sha1']) 184 185 def IncrementFilesUpdated(*_): 186 IncrementFilesUpdated.files_updated += 1 187 IncrementFilesUpdated.files_updated = 0 188 orig_get_if_changed = cloud_storage.GetIfChanged 189 cloud_storage.GetIfChanged = IncrementFilesUpdated 190 try: 191 self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged, 192 os.path.abspath(os.sep), cloud_storage.PUBLIC_BUCKET) 193 self.assertEqual(0, IncrementFilesUpdated.files_updated) 194 self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged, 195 'fake_dir_path', cloud_storage.PUBLIC_BUCKET) 196 self.assertEqual(0, IncrementFilesUpdated.files_updated) 197 cloud_storage.GetFilesInDirectoryIfChanged('real_dir_path', 198 cloud_storage.PUBLIC_BUCKET) 199 self.assertEqual(3, IncrementFilesUpdated.files_updated) 200 finally: 201 cloud_storage.GetIfChanged = orig_get_if_changed 202 203 def testCopy(self): 204 orig_run_command = cloud_storage._RunCommand 205 206 def AssertCorrectRunCommandArgs(args): 207 self.assertEqual(expected_args, args) 208 cloud_storage._RunCommand = AssertCorrectRunCommandArgs 209 expected_args = ['cp', 'gs://bucket1/remote_path1', 210 'gs://bucket2/remote_path2'] 211 try: 212 cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') 213 finally: 214 cloud_storage._RunCommand = orig_run_command 215 216 217 @mock.patch('catapult_base.cloud_storage._PseudoFileLock') 218 def testDisableCloudStorageIo(self, unused_lock_mock): 219 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1' 220 dir_path = 'real_dir_path' 221 self.fs.CreateDirectory(dir_path) 222 file_path = os.path.join(dir_path, 'file1') 223 file_path_sha = file_path + '.sha1' 224 self.CreateFiles([file_path, file_path_sha]) 225 with open(file_path_sha, 'w') as f: 226 f.write('hash1234') 227 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 228 cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') 229 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 230 cloud_storage.Get('bucket', 'foo', file_path) 231 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 232 cloud_storage.GetIfChanged(file_path, 'foo') 233 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 234 cloud_storage.GetIfHashChanged('bar', file_path, 'bucket', 'hash1234') 235 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 236 cloud_storage.Insert('bucket', 'foo', file_path) 237 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 238 cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket') 239