1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import os
6import sys
7import unittest
8
9import mock
10from pyfakefs import fake_filesystem_unittest
11
12
13from catapult_base import cloud_storage
14from catapult_base import util
15
16
17def _FakeReadHash(_):
18  return 'hashthis!'
19
20
21def _FakeCalulateHashMatchesRead(_):
22  return 'hashthis!'
23
24
25def _FakeCalulateHashNewHash(_):
26  return 'omgnewhash'
27
28
29class CloudStorageUnitTest(fake_filesystem_unittest.TestCase):
30
31  def setUp(self):
32    self.original_environ = os.environ.copy()
33    os.environ['DISABLE_CLOUD_STORAGE_IO'] = ''
34    self.setUpPyfakefs()
35    self.fs.CreateFile(
36        os.path.join(util.GetCatapultDir(), 'third_party', 'gsutil', 'gsutil'))
37
38  def CreateFiles(self, file_paths):
39    for f in file_paths:
40      self.fs.CreateFile(f)
41
42  def tearDown(self):
43    self.tearDownPyfakefs()
44    os.environ = self.original_environ
45
46  def _FakeRunCommand(self, cmd):
47    pass
48
49  def _FakeGet(self, bucket, remote_path, local_path):
50    pass
51
52  def _AssertRunCommandRaisesError(self, communicate_strs, error):
53    with mock.patch('catapult_base.cloud_storage.subprocess.Popen') as popen:
54      p_mock = mock.Mock()
55      popen.return_value = p_mock
56      p_mock.returncode = 1
57      for stderr in communicate_strs:
58        p_mock.communicate.return_value = ('', stderr)
59        self.assertRaises(error, cloud_storage._RunCommand, [])
60
61  def testRunCommandCredentialsError(self):
62    strs = ['You are attempting to access protected data with no configured',
63            'Failure: No handler was ready to authenticate.']
64    self._AssertRunCommandRaisesError(strs, cloud_storage.CredentialsError)
65
66  def testRunCommandPermissionError(self):
67    strs = ['status=403', 'status 403', '403 Forbidden']
68    self._AssertRunCommandRaisesError(strs, cloud_storage.PermissionError)
69
70  def testRunCommandNotFoundError(self):
71    strs = ['InvalidUriError', 'No such object', 'No URLs matched',
72            'One or more URLs matched no', 'InvalidUriError']
73    self._AssertRunCommandRaisesError(strs, cloud_storage.NotFoundError)
74
75  def testRunCommandServerError(self):
76    strs = ['500 Internal Server Error']
77    self._AssertRunCommandRaisesError(strs, cloud_storage.ServerError)
78
79  def testRunCommandGenericError(self):
80    strs = ['Random string']
81    self._AssertRunCommandRaisesError(strs, cloud_storage.CloudStorageError)
82
83  def testInsertCreatesValidCloudUrl(self):
84    orig_run_command = cloud_storage._RunCommand
85    try:
86      cloud_storage._RunCommand = self._FakeRunCommand
87      remote_path = 'test-remote-path.html'
88      local_path = 'test-local-path.html'
89      cloud_url = cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET,
90                                       remote_path, local_path)
91      self.assertEqual('https://console.developers.google.com/m/cloudstorage'
92                       '/b/chromium-telemetry/o/test-remote-path.html',
93                       cloud_url)
94    finally:
95      cloud_storage._RunCommand = orig_run_command
96
97  @mock.patch('catapult_base.cloud_storage.subprocess')
98  def testExistsReturnsFalse(self, subprocess_mock):
99    p_mock = mock.Mock()
100    subprocess_mock.Popen.return_value = p_mock
101    p_mock.communicate.return_value = (
102        '',
103        'CommandException: One or more URLs matched no objects.\n')
104    p_mock.returncode_result = 1
105    self.assertFalse(cloud_storage.Exists('fake bucket',
106                                          'fake remote path'))
107
108  @mock.patch('catapult_base.cloud_storage.CalculateHash')
109  @mock.patch('catapult_base.cloud_storage._GetLocked')
110  @mock.patch('catapult_base.cloud_storage._PseudoFileLock')
111  @mock.patch('catapult_base.cloud_storage.os.path')
112  def testGetIfHashChanged(self, path_mock, unused_lock_mock, get_mock,
113                           calc_hash_mock):
114    path_mock.exists.side_effect = [False, True, True]
115    calc_hash_mock.return_value = 'hash'
116
117    # The file at |local_path| doesn't exist. We should download file from cs.
118    ret = cloud_storage.GetIfHashChanged(
119        'remote_path', 'local_path', 'cs_bucket', 'hash')
120    self.assertTrue(ret)
121    get_mock.assert_called_once_with('cs_bucket', 'remote_path', 'local_path')
122    get_mock.reset_mock()
123    self.assertFalse(calc_hash_mock.call_args)
124    calc_hash_mock.reset_mock()
125
126    # A local file exists at |local_path| but has the wrong hash.
127    # We should download file from cs.
128    ret = cloud_storage.GetIfHashChanged(
129        'remote_path', 'local_path', 'cs_bucket', 'new_hash')
130    self.assertTrue(ret)
131    get_mock.assert_called_once_with('cs_bucket', 'remote_path', 'local_path')
132    get_mock.reset_mock()
133    calc_hash_mock.assert_called_once_with('local_path')
134    calc_hash_mock.reset_mock()
135
136    # Downloaded file exists locally and has the right hash. Don't download.
137    ret = cloud_storage.GetIfHashChanged(
138        'remote_path', 'local_path', 'cs_bucket', 'hash')
139    self.assertFalse(get_mock.call_args)
140    self.assertFalse(ret)
141    calc_hash_mock.reset_mock()
142    get_mock.reset_mock()
143
144  @mock.patch('catapult_base.cloud_storage._PseudoFileLock')
145  def testGetIfChanged(self, unused_lock_mock):
146    orig_get = cloud_storage._GetLocked
147    orig_read_hash = cloud_storage.ReadHash
148    orig_calculate_hash = cloud_storage.CalculateHash
149    cloud_storage.ReadHash = _FakeReadHash
150    cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
151    file_path = 'test-file-path.wpr'
152    hash_path = file_path + '.sha1'
153    try:
154      cloud_storage._GetLocked = self._FakeGet
155      # hash_path doesn't exist.
156      self.assertFalse(cloud_storage.GetIfChanged(file_path,
157                                                  cloud_storage.PUBLIC_BUCKET))
158      # hash_path exists, but file_path doesn't.
159      self.CreateFiles([hash_path])
160      self.assertTrue(cloud_storage.GetIfChanged(file_path,
161                                                 cloud_storage.PUBLIC_BUCKET))
162      # hash_path and file_path exist, and have same hash.
163      self.CreateFiles([file_path])
164      self.assertFalse(cloud_storage.GetIfChanged(file_path,
165                                                  cloud_storage.PUBLIC_BUCKET))
166      # hash_path and file_path exist, and have different hashes.
167      cloud_storage.CalculateHash = _FakeCalulateHashNewHash
168      self.assertTrue(cloud_storage.GetIfChanged(file_path,
169                                                 cloud_storage.PUBLIC_BUCKET))
170    finally:
171      cloud_storage._GetLocked = orig_get
172      cloud_storage.CalculateHash = orig_calculate_hash
173      cloud_storage.ReadHash = orig_read_hash
174
175  @unittest.skipIf(sys.platform.startswith('win'),
176                   'https://github.com/catapult-project/catapult/issues/1861')
177  def testGetFilesInDirectoryIfChanged(self):
178    self.CreateFiles([
179        'real_dir_path/dir1/1file1.sha1',
180        'real_dir_path/dir1/1file2.txt',
181        'real_dir_path/dir1/1file3.sha1',
182        'real_dir_path/dir2/2file.txt',
183        'real_dir_path/dir3/3file1.sha1'])
184
185    def IncrementFilesUpdated(*_):
186      IncrementFilesUpdated.files_updated += 1
187    IncrementFilesUpdated.files_updated = 0
188    orig_get_if_changed = cloud_storage.GetIfChanged
189    cloud_storage.GetIfChanged = IncrementFilesUpdated
190    try:
191      self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
192                        os.path.abspath(os.sep), cloud_storage.PUBLIC_BUCKET)
193      self.assertEqual(0, IncrementFilesUpdated.files_updated)
194      self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
195                        'fake_dir_path', cloud_storage.PUBLIC_BUCKET)
196      self.assertEqual(0, IncrementFilesUpdated.files_updated)
197      cloud_storage.GetFilesInDirectoryIfChanged('real_dir_path',
198                                                 cloud_storage.PUBLIC_BUCKET)
199      self.assertEqual(3, IncrementFilesUpdated.files_updated)
200    finally:
201      cloud_storage.GetIfChanged = orig_get_if_changed
202
203  def testCopy(self):
204    orig_run_command = cloud_storage._RunCommand
205
206    def AssertCorrectRunCommandArgs(args):
207      self.assertEqual(expected_args, args)
208    cloud_storage._RunCommand = AssertCorrectRunCommandArgs
209    expected_args = ['cp', 'gs://bucket1/remote_path1',
210                     'gs://bucket2/remote_path2']
211    try:
212      cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
213    finally:
214      cloud_storage._RunCommand = orig_run_command
215
216
217  @mock.patch('catapult_base.cloud_storage._PseudoFileLock')
218  def testDisableCloudStorageIo(self, unused_lock_mock):
219    os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1'
220    dir_path = 'real_dir_path'
221    self.fs.CreateDirectory(dir_path)
222    file_path = os.path.join(dir_path, 'file1')
223    file_path_sha = file_path + '.sha1'
224    self.CreateFiles([file_path, file_path_sha])
225    with open(file_path_sha, 'w') as f:
226      f.write('hash1234')
227    with self.assertRaises(cloud_storage.CloudStorageIODisabled):
228      cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
229    with self.assertRaises(cloud_storage.CloudStorageIODisabled):
230      cloud_storage.Get('bucket', 'foo', file_path)
231    with self.assertRaises(cloud_storage.CloudStorageIODisabled):
232      cloud_storage.GetIfChanged(file_path, 'foo')
233    with self.assertRaises(cloud_storage.CloudStorageIODisabled):
234      cloud_storage.GetIfHashChanged('bar', file_path, 'bucket', 'hash1234')
235    with self.assertRaises(cloud_storage.CloudStorageIODisabled):
236      cloud_storage.Insert('bucket', 'foo', file_path)
237    with self.assertRaises(cloud_storage.CloudStorageIODisabled):
238      cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket')
239