1# -*- coding: utf-8 -*- 2# Copyright 2010 Google Inc. All Rights Reserved. 3# 4# Permission is hereby granted, free of charge, to any person obtaining a 5# copy of this software and associated documentation files (the 6# "Software"), to deal in the Software without restriction, including 7# without limitation the rights to use, copy, modify, merge, publish, dis- 8# tribute, sublicense, and/or sell copies of the Software, and to permit 9# persons to whom the Software is furnished to do so, subject to the fol- 10# lowing conditions: 11# 12# The above copyright notice and this permission notice shall be included 13# in all copies or substantial portions of the Software. 14# 15# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 17# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 18# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21# IN THE SOFTWARE. 22"""Tests for gsutil naming logic. 23 24The test code in this file runs against an in-memory storage service mock, 25so runs very quickly. This is valuable for testing changes that impact the 26naming rules, since those rules are complex and it's useful to be able to 27make small incremental changes and rerun the tests frequently. Additional 28end-to-end tests (which send traffic to the production Google Cloud Storage 29service) are available via the gsutil test command. 30""" 31 32from __future__ import absolute_import 33 34import gzip 35import os 36import StringIO 37 38from gslib import copy_helper 39from gslib.cloud_api import NotFoundException 40from gslib.cloud_api import ServiceException 41from gslib.exception import CommandException 42from gslib.storage_url import StorageUrlFromString 43import gslib.tests.testcase as testcase 44from gslib.tests.util import ObjectToURI as suri 45from gslib.tests.util import SetBotoConfigForTest 46from gslib.util import UTF8 47 48 49def _Overwrite(fp): 50 """Overwrite first byte in an open file and flush contents.""" 51 fp.seek(0) 52 fp.write('x') 53 fp.flush() 54 55 56def _Append(fp): 57 """Append a byte at end of an open file and flush contents.""" 58 fp.seek(0, 2) 59 fp.write('x') 60 fp.flush() 61 62 63# TODO: Re-enable SequentialAndParallelTransfer decorator on tests in this file 64# once we refactor to a thread-safe mock storage service implementation. 65class GsutilNamingTests(testcase.GsUtilUnitTestCase): 66 """Unit tests for gsutil naming logic.""" 67 68 def testGetPathBeforeFinalDir(self): 69 """Tests GetPathBeforeFinalDir() (unit test).""" 70 self.assertEqual( 71 'gs://', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString( 72 'gs://bucket/'))) 73 self.assertEqual( 74 'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString( 75 'gs://bucket/dir/'))) 76 self.assertEqual( 77 'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString( 78 'gs://bucket/dir'))) 79 self.assertEqual( 80 'gs://bucket/dir', copy_helper.GetPathBeforeFinalDir( 81 StorageUrlFromString('gs://bucket/dir/obj'))) 82 src_dir = self.CreateTempDir() 83 subdir = os.path.join(src_dir, 'subdir') 84 os.mkdir(subdir) 85 self.assertEqual(suri(src_dir), 86 copy_helper.GetPathBeforeFinalDir( 87 StorageUrlFromString(suri(subdir)))) 88 89 # @SequentialAndParallelTransfer 90 def testCopyingTopLevelFileToBucket(self): 91 """Tests copying one top-level file to a bucket.""" 92 src_file = self.CreateTempFile(file_name='f0') 93 dst_bucket_uri = self.CreateBucket() 94 self.RunCommand('cp', [src_file, suri(dst_bucket_uri)]) 95 actual = list(self._test_wildcard_iterator( 96 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 97 self.assertEqual(1, len(actual)) 98 self.assertEqual('f0', actual[0].root_object.name) 99 100 # @SequentialAndParallelTransfer 101 def testCopyingMultipleFilesToBucket(self): 102 """Tests copying multiple files to a bucket.""" 103 src_file0 = self.CreateTempFile(file_name='f0') 104 src_file1 = self.CreateTempFile(file_name='f1') 105 dst_bucket_uri = self.CreateBucket() 106 self.RunCommand('cp', [src_file0, src_file1, suri(dst_bucket_uri)]) 107 actual = set(str(u) for u in self._test_wildcard_iterator( 108 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 109 expected = set([ 110 suri(dst_bucket_uri, 'f0'), 111 suri(dst_bucket_uri, 'f1'), 112 ]) 113 self.assertEqual(expected, actual) 114 115 # @SequentialAndParallelTransfer 116 def testCopyingNestedFileToBucketSubdir(self): 117 """Tests copying a nested file to a bucket subdir. 118 119 Tests that we correctly translate local FS-specific delimiters ('\' on 120 Windows) to bucket delimiter (/). 121 """ 122 tmpdir = self.CreateTempDir() 123 subdir = os.path.join(tmpdir, 'subdir') 124 os.mkdir(subdir) 125 src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='') 126 dst_bucket_uri = self.CreateBucket() 127 # Make an object under subdir so next copy will treat subdir as a subdir. 128 self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir/a')]) 129 self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir')]) 130 actual = set(str(u) for u in self._test_wildcard_iterator( 131 suri(dst_bucket_uri, '**')).IterObjects()) 132 expected = set([ 133 suri(dst_bucket_uri, 'subdir', 'a'), 134 suri(dst_bucket_uri, 'subdir', 'obj'), 135 ]) 136 self.assertEqual(expected, actual) 137 138 # @SequentialAndParallelTransfer 139 def testCopyingAbsolutePathDirToBucket(self): 140 """Tests recursively copying absolute path directory to a bucket.""" 141 dst_bucket_uri = self.CreateBucket() 142 src_dir_root = self.CreateTempDir(test_files=[ 143 'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')]) 144 self.RunCommand('cp', ['-R', src_dir_root, suri(dst_bucket_uri)]) 145 actual = set(str(u) for u in self._test_wildcard_iterator( 146 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 147 src_tmpdir = os.path.split(src_dir_root)[1] 148 expected = set([ 149 suri(dst_bucket_uri, src_tmpdir, 'f0'), 150 suri(dst_bucket_uri, src_tmpdir, 'f1'), 151 suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), 152 suri(dst_bucket_uri, src_tmpdir, 'dir0', 'dir1', 'nested')]) 153 self.assertEqual(expected, actual) 154 155 # @SequentialAndParallelTransfer 156 def testCopyingRelativePathDirToBucket(self): 157 """Tests recursively copying relative directory to a bucket.""" 158 dst_bucket_uri = self.CreateBucket() 159 src_dir = self.CreateTempDir(test_files=[('dir0', 'f1')]) 160 self.RunCommand('cp', ['-R', 'dir0', suri(dst_bucket_uri)], cwd=src_dir) 161 actual = set(str(u) for u in self._test_wildcard_iterator( 162 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 163 expected = set([suri(dst_bucket_uri, 'dir0', 'f1')]) 164 self.assertEqual(expected, actual) 165 166 # @SequentialAndParallelTransfer 167 def testCopyingRelPathSubDirToBucketSubdirWithDollarFolderObj(self): 168 """Tests recursively copying relative sub-directory to bucket subdir. 169 170 Subdir is signified by a $folder$ object. 171 """ 172 # Create a $folder$ object to simulate a folder created by the legacy GCS 173 # console (or various other tools), which gsutil understands to mean there 174 # is a folder into which the object is being copied. 175 dst_bucket_uri = self.CreateBucket() 176 self.CreateObject(bucket_uri=dst_bucket_uri, object_name='abc_$folder$', 177 contents='') 178 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) 179 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'), 180 suri(dst_bucket_uri, 'abc')], cwd=src_dir) 181 actual = set(str(u) for u in self._test_wildcard_iterator( 182 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 183 expected = set([suri(dst_bucket_uri, 'abc_$folder$'), 184 suri(dst_bucket_uri, 'abc', 'dir1', 'f1')]) 185 self.assertEqual(expected, actual) 186 187 # @SequentialAndParallelTransfer 188 def testCopyingRelativePathSubDirToBucketSubdirSignifiedBySlash(self): 189 """Tests recursively copying relative sub-directory to bucket subdir. 190 191 Subdir is signified by a / object. 192 """ 193 dst_bucket_uri = self.CreateBucket() 194 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) 195 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'), 196 suri(dst_bucket_uri, 'abc') + '/'], cwd=src_dir) 197 actual = set(str(u) for u in self._test_wildcard_iterator( 198 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 199 expected = set([suri(dst_bucket_uri, 'abc', 'dir1', 'f1')]) 200 self.assertEqual(expected, actual) 201 202 # @SequentialAndParallelTransfer 203 def testCopyingRelativePathSubDirToBucket(self): 204 """Tests recursively copying relative sub-directory to a bucket.""" 205 dst_bucket_uri = self.CreateBucket() 206 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) 207 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'), 208 suri(dst_bucket_uri)], cwd=src_dir) 209 actual = set(str(u) for u in self._test_wildcard_iterator( 210 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 211 expected = set([suri(dst_bucket_uri, 'dir1', 'f1')]) 212 self.assertEqual(expected, actual) 213 214 # @SequentialAndParallelTransfer 215 def testCopyingDotSlashToBucket(self): 216 """Tests copying ./ to a bucket produces expected naming.""" 217 # When running a command like gsutil cp -r . gs://dest we expect the dest 218 # obj names to be of the form gs://dest/abc, not gs://dest/./abc. 219 dst_bucket_uri = self.CreateBucket() 220 src_dir = self.CreateTempDir(test_files=['foo']) 221 for rel_src_dir in ['.', '.%s' % os.sep]: 222 self.RunCommand('cp', ['-R', rel_src_dir, suri(dst_bucket_uri)], 223 cwd=src_dir) 224 actual = set(str(u) for u in self._test_wildcard_iterator( 225 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 226 expected = set([suri(dst_bucket_uri, 'foo')]) 227 self.assertEqual(expected, actual) 228 229 # @SequentialAndParallelTransfer 230 def testCopyingDirContainingOneFileToBucket(self): 231 """Tests copying a directory containing 1 file to a bucket. 232 233 We test this case to ensure that correct bucket handling isn't dependent 234 on the copy being treated as a multi-source copy. 235 """ 236 dst_bucket_uri = self.CreateBucket() 237 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')]) 238 self.RunCommand('cp', ['-R', os.path.join(src_dir, 'dir0', 'dir1'), 239 suri(dst_bucket_uri)]) 240 actual = list((str(u) for u in self._test_wildcard_iterator( 241 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))) 242 self.assertEqual(1, len(actual)) 243 self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), actual[0]) 244 245 def testCopyingBucketToDir(self): 246 """Tests copying from a bucket to a directory.""" 247 src_bucket_uri = self.CreateBucket(test_objects=['foo', 'dir/foo2']) 248 dst_dir = self.CreateTempDir() 249 # Mock objects don't support hash digestion. 250 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): 251 self.RunCommand('cp', ['-R', suri(src_bucket_uri), dst_dir]) 252 actual = set(str(u) for u in self._test_wildcard_iterator( 253 '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) 254 expected = set([suri(dst_dir, src_bucket_uri.bucket_name, 'foo'), 255 suri(dst_dir, src_bucket_uri.bucket_name, 'dir', 'foo2')]) 256 self.assertEqual(expected, actual) 257 258 def testCopyingBucketToBucket(self): 259 """Tests copying from a bucket-only URI to a bucket.""" 260 src_bucket_uri = self.CreateBucket(test_objects=['foo', 'dir/foo2']) 261 dst_bucket_uri = self.CreateBucket() 262 self.RunCommand('cp', ['-R', suri(src_bucket_uri), suri(dst_bucket_uri)]) 263 actual = set(str(u) for u in self._test_wildcard_iterator( 264 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 265 expected = set([ 266 suri(dst_bucket_uri, src_bucket_uri.bucket_name, 'foo'), 267 suri(dst_bucket_uri, src_bucket_uri.bucket_name, 'dir', 'foo2')]) 268 self.assertEqual(expected, actual) 269 270 def testCopyingDirectoryToDirectory(self): 271 """Tests copying from a directory to a directory.""" 272 src_dir = self.CreateTempDir(test_files=['foo', ('dir', 'foo2')]) 273 dst_dir = self.CreateTempDir() 274 self.RunCommand('cp', ['-R', src_dir, dst_dir]) 275 actual = set(str(u) for u in self._test_wildcard_iterator( 276 '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) 277 src_dir_base = os.path.split(src_dir)[1] 278 expected = set([suri(dst_dir, src_dir_base, 'foo'), 279 suri(dst_dir, src_dir_base, 'dir', 'foo2')]) 280 self.assertEqual(expected, actual) 281 282 def testCopyingFilesAndDirNonRecursive(self): 283 """Tests copying containing files and a directory without -R.""" 284 src_dir = self.CreateTempDir(test_files=['foo', 'bar', ('d1', 'f2'), 285 ('d2', 'f3'), ('d3', 'd4', 'f4')]) 286 dst_dir = self.CreateTempDir() 287 self.RunCommand('cp', ['%s%s*' % (src_dir, os.sep), dst_dir]) 288 actual = set(str(u) for u in self._test_wildcard_iterator( 289 '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) 290 expected = set([suri(dst_dir, 'foo'), suri(dst_dir, 'bar')]) 291 self.assertEqual(expected, actual) 292 293 def testCopyingFileToDir(self): 294 """Tests copying one file to a directory.""" 295 src_file = self.CreateTempFile(file_name='foo') 296 dst_dir = self.CreateTempDir() 297 self.RunCommand('cp', [src_file, dst_dir]) 298 actual = list(self._test_wildcard_iterator( 299 '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) 300 self.assertEqual(1, len(actual)) 301 self.assertEqual(suri(dst_dir, 'foo'), str(actual[0])) 302 303 # @SequentialAndParallelTransfer 304 def testCopyingFileToObjectWithConsecutiveSlashes(self): 305 """Tests copying a file to an object containing consecutive slashes.""" 306 src_file = self.CreateTempFile(file_name='f0') 307 dst_bucket_uri = self.CreateBucket() 308 self.RunCommand('cp', [src_file, suri(dst_bucket_uri) + '//obj']) 309 actual = list(self._test_wildcard_iterator( 310 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 311 self.assertEqual(1, len(actual)) 312 self.assertEqual('/obj', actual[0].root_object.name) 313 314 def testCopyingCompressedFileToBucket(self): 315 """Tests copying one file with compression to a bucket.""" 316 src_file = self.CreateTempFile(contents='plaintext', file_name='f2.txt') 317 dst_bucket_uri = self.CreateBucket() 318 self.RunCommand('cp', ['-z', 'txt', src_file, suri(dst_bucket_uri)],) 319 actual = list(self._test_wildcard_iterator( 320 suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) 321 self.assertEqual(1, len(actual)) 322 actual_obj = actual[0].root_object 323 self.assertEqual('f2.txt', actual_obj.name) 324 self.assertEqual('gzip', actual_obj.contentEncoding) 325 326 stdout = self.RunCommand('cat', [suri(dst_bucket_uri, 'f2.txt')], 327 return_stdout=True) 328 329 f = gzip.GzipFile(fileobj=StringIO.StringIO(stdout), mode='rb') 330 try: 331 self.assertEqual(f.read(), 'plaintext') 332 finally: 333 f.close() 334 335 def testCopyingObjectToObject(self): 336 """Tests copying an object to an object.""" 337 src_bucket_uri = self.CreateBucket(test_objects=['obj']) 338 dst_bucket_uri = self.CreateBucket() 339 self.RunCommand('cp', [suri(src_bucket_uri, 'obj'), suri(dst_bucket_uri)]) 340 actual = list(self._test_wildcard_iterator( 341 suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) 342 self.assertEqual(1, len(actual)) 343 self.assertEqual('obj', actual[0].root_object.name) 344 345 def testCopyingObjectToObjectUsingDestWildcard(self): 346 """Tests copying an object to an object using a dest wildcard.""" 347 src_bucket_uri = self.CreateBucket(test_objects=['obj']) 348 dst_bucket_uri = self.CreateBucket(test_objects=['dstobj']) 349 self.RunCommand('cp', [suri(src_bucket_uri, 'obj'), 350 '%s*' % dst_bucket_uri.uri]) 351 actual = set(str(u) for u in self._test_wildcard_iterator( 352 suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) 353 expected = set([suri(dst_bucket_uri, 'dstobj')]) 354 self.assertEqual(expected, actual) 355 356 def testCopyingObjsAndFilesToDir(self): 357 """Tests copying objects and files to a directory.""" 358 src_bucket_uri = self.CreateBucket(test_objects=['f1']) 359 src_dir = self.CreateTempDir(test_files=['f2']) 360 dst_dir = self.CreateTempDir() 361 # Mock objects don't support hash digestion. 362 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): 363 self.RunCommand('cp', ['-R', suri(src_bucket_uri, '**'), 364 os.path.join(src_dir, '**'), dst_dir]) 365 actual = set(str(u) for u in self._test_wildcard_iterator( 366 os.path.join(dst_dir, '**')).IterAll(expand_top_level_buckets=True)) 367 expected = set([suri(dst_dir, 'f1'), suri(dst_dir, 'f2')]) 368 self.assertEqual(expected, actual) 369 370 def testCopyingObjToDot(self): 371 """Tests that copying an object to . or ./ downloads to correct name.""" 372 src_bucket_uri = self.CreateBucket(test_objects=['f1']) 373 dst_dir = self.CreateTempDir() 374 for final_char in ('/', ''): 375 # Mock objects don't support hash digestion. 376 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): 377 self.RunCommand('cp', [suri(src_bucket_uri, 'f1'), '.%s' % final_char], 378 cwd=dst_dir) 379 actual = set() 380 for dirname, dirnames, filenames in os.walk(dst_dir): 381 for subdirname in dirnames: 382 actual.add(os.path.join(dirname, subdirname)) 383 for filename in filenames: 384 actual.add(os.path.join(dirname, filename)) 385 expected = set([os.path.join(dst_dir, 'f1')]) 386 self.assertEqual(expected, actual) 387 388 # @SequentialAndParallelTransfer 389 def testCopyingObjsAndFilesToBucket(self): 390 """Tests copying objects and files to a bucket.""" 391 src_bucket_uri = self.CreateBucket(test_objects=['f1']) 392 src_dir = self.CreateTempDir(test_files=['f2']) 393 dst_bucket_uri = self.CreateBucket() 394 self.RunCommand('cp', ['-R', suri(src_bucket_uri, '**'), 395 '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)]) 396 actual = set(str(u) for u in self._test_wildcard_iterator( 397 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 398 expected = set([suri(dst_bucket_uri, 'f1'), suri(dst_bucket_uri, 'f2')]) 399 self.assertEqual(expected, actual) 400 401 # @SequentialAndParallelTransfer 402 def testCopyingSubdirRecursiveToNonexistentSubdir(self): 403 """Tests copying a directory with a single file recursively to a bucket. 404 405 The file should end up in a new bucket subdirectory with the file's 406 directory structure starting below the recursive copy point, as in Unix cp. 407 408 Example: 409 filepath: dir1/dir2/foo 410 cp -r dir1 dir3 411 Results in dir3/dir2/foo being created. 412 """ 413 src_dir = self.CreateTempDir() 414 self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo') 415 dst_bucket_uri = self.CreateBucket() 416 self.RunCommand('cp', ['-R', src_dir + '/dir1', 417 suri(dst_bucket_uri, 'dir3')]) 418 actual = set(str(u) for u in self._test_wildcard_iterator( 419 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 420 expected = set([suri(dst_bucket_uri, 'dir3/dir2/foo')]) 421 self.assertEqual(expected, actual) 422 423 def testAttemptDirCopyWithoutRecursion(self): 424 """Tests copying a directory without -R.""" 425 src_dir = self.CreateTempDir(test_files=1) 426 dst_dir = self.CreateTempDir() 427 try: 428 self.RunCommand('cp', [src_dir, dst_dir]) 429 self.fail('Did not get expected CommandException') 430 except CommandException, e: 431 self.assertIn('No URLs matched', e.reason) 432 433 def testNonRecursiveFileAndSameNameSubdir(self): 434 """Tests copying a file and subdirectory of the same name without -R.""" 435 src_bucket_uri = self.CreateBucket(test_objects=['f1', 'f1/f2']) 436 dst_dir = self.CreateTempDir() 437 # Mock objects don't support hash digestion. 438 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): 439 self.RunCommand('cp', [suri(src_bucket_uri, 'f1'), dst_dir]) 440 actual = list(self._test_wildcard_iterator( 441 '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) 442 self.assertEqual(1, len(actual)) 443 self.assertEqual(suri(dst_dir, 'f1'), str(actual[0])) 444 # TODO: Assert that we omit the prefix here when unit_testcase supports 445 # returning stderr. 446 447 def testAttemptCopyingProviderOnlySrc(self): 448 """Attempts to copy a src specified as a provider-only URI.""" 449 src_bucket_uri = self.CreateBucket() 450 try: 451 self.RunCommand('cp', ['gs://', suri(src_bucket_uri)]) 452 self.fail('Did not get expected CommandException') 453 except CommandException, e: 454 self.assertIn('provider-only', e.reason) 455 456 def testAttemptCopyingOverlappingSrcDstFile(self): 457 """Attempts to an object atop itself.""" 458 src_file = self.CreateTempFile() 459 try: 460 self.RunCommand('cp', [src_file, src_file]) 461 self.fail('Did not get expected CommandException') 462 except CommandException, e: 463 self.assertIn('are the same file - abort', e.reason) 464 465 def testAttemptCopyingToMultiMatchWildcard(self): 466 """Attempts to copy where dst wildcard matches >1 obj.""" 467 src_bucket_uri = self.CreateBucket(test_objects=2) 468 try: 469 self.RunCommand('cp', [suri(src_bucket_uri, 'obj0'), 470 suri(src_bucket_uri, '*')]) 471 self.fail('Did not get expected CommandException') 472 except CommandException, e: 473 self.assertNotEqual(e.reason.find('must match exactly 1 URL'), -1) 474 475 def testAttemptCopyingMultiObjsToFile(self): 476 """Attempts to copy multiple objects to a file.""" 477 src_bucket_uri = self.CreateBucket(test_objects=2) 478 dst_file = self.CreateTempFile() 479 try: 480 self.RunCommand('cp', ['-R', suri(src_bucket_uri, '*'), dst_file]) 481 self.fail('Did not get expected CommandException') 482 except CommandException, e: 483 self.assertIn('must name a directory, bucket, or', e.reason) 484 485 def testAttemptCopyingWithFileDirConflict(self): 486 """Attempts to copy objects that cause a file/directory conflict.""" 487 # Create objects with name conflicts (a/b and a). Use 'dst' bucket because 488 # it gets cleared after each test. 489 bucket_uri = self.CreateBucket() 490 self.CreateObject(bucket_uri=bucket_uri, object_name='a') 491 self.CreateObject(bucket_uri=bucket_uri, object_name='b/a') 492 dst_dir = self.CreateTempDir() 493 try: 494 self.RunCommand('cp', ['-R', suri(bucket_uri), dst_dir]) 495 self.fail('Did not get expected CommandException') 496 except CommandException, e: 497 self.assertNotEqual('exists where a directory needs to be created', 498 e.reason) 499 500 def testAttemptCopyingWithDirFileConflict(self): 501 """Attempts to copy an object that causes a directory/file conflict.""" 502 # Create an object that conflicts with dest subdir. 503 tmpdir = self.CreateTempDir() 504 os.mkdir(os.path.join(tmpdir, 'abc')) 505 src_uri = self.CreateObject(object_name='abc', contents='bar') 506 try: 507 self.RunCommand('cp', [suri(src_uri), tmpdir + '/']) 508 self.fail('Did not get expected CommandException') 509 except CommandException, e: 510 self.assertNotEqual('where the file needs to be created', e.reason) 511 512 def testWildcardMoveWithinBucket(self): 513 """Attempts to move using src wildcard that overlaps dest object. 514 515 We want to ensure that this doesn't stomp the result data. 516 """ 517 dst_bucket_uri = self.CreateBucket(test_objects=['old']) 518 self.RunCommand('mv', [suri(dst_bucket_uri, 'old*'), 519 suri(dst_bucket_uri, 'new')]) 520 actual = set(str(u) for u in self._test_wildcard_iterator( 521 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 522 expected = set([suri(dst_bucket_uri, 'new')]) 523 self.assertEqual(expected, actual) 524 525 def testLsNonExistentObjectWithPrefixName(self): 526 """Test ls of non-existent obj that matches prefix of existing objs.""" 527 # Use an object name that matches a prefix of other names at that level, to 528 # ensure the ls subdir handling logic doesn't pick up anything extra. 529 src_bucket_uri = self.CreateBucket(test_objects=['obj_with_suffix']) 530 try: 531 self.RunCommand('ls', [suri(src_bucket_uri, 'obj')]) 532 except CommandException, e: 533 self.assertIn('matched no objects', e.reason) 534 535 def testLsBucketNonRecursive(self): 536 """Test that ls of a bucket returns expected results.""" 537 src_bucket_uri = self.CreateBucket(test_objects=['foo1', 'd0/foo2', 538 'd1/d2/foo3']) 539 output = self.RunCommand('ls', [suri(src_bucket_uri, '*')], 540 return_stdout=True) 541 expected = set([suri(src_bucket_uri, 'foo1'), 542 suri(src_bucket_uri, 'd1', ':'), 543 suri(src_bucket_uri, 'd1', 'd2') + src_bucket_uri.delim, 544 suri(src_bucket_uri, 'd0', ':'), 545 suri(src_bucket_uri, 'd0', 'foo2')]) 546 expected.add('') # Blank line between subdir listings. 547 actual = set(output.split('\n')) 548 self.assertEqual(expected, actual) 549 550 def testLsBucketRecursive(self): 551 """Test that ls -R of a bucket returns expected results.""" 552 src_bucket_uri = self.CreateBucket(test_objects=['foo1', 'd0/foo2', 553 'd1/d2/foo3']) 554 output = self.RunCommand('ls', ['-R', suri(src_bucket_uri, '*')], 555 return_stdout=True) 556 expected = set([suri(src_bucket_uri, 'foo1'), 557 suri(src_bucket_uri, 'd1', ':'), 558 suri(src_bucket_uri, 'd1', 'd2', ':'), 559 suri(src_bucket_uri, 'd1', 'd2', 'foo3'), 560 suri(src_bucket_uri, 'd0', ':'), 561 suri(src_bucket_uri, 'd0', 'foo2')]) 562 expected.add('') # Blank line between subdir listings. 563 actual = set(output.split('\n')) 564 self.assertEqual(expected, actual) 565 566 def testLsBucketRecursiveWithLeadingSlashObjectName(self): 567 """Test that ls -R of a bucket with an object that has leading slash.""" 568 dst_bucket_uri = self.CreateBucket(test_objects=['f0']) 569 output = self.RunCommand('ls', ['-R', suri(dst_bucket_uri) + '*'], 570 return_stdout=True) 571 expected = set([suri(dst_bucket_uri, 'f0')]) 572 expected.add('') # Blank line between subdir listings. 573 actual = set(output.split('\n')) 574 self.assertEqual(expected, actual) 575 576 def testLsBucketSubdirNonRecursive(self): 577 """Test that ls of a bucket subdir returns expected results.""" 578 src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/foo', 579 'src_subdir/nested/foo2']) 580 output = self.RunCommand('ls', [suri(src_bucket_uri, 'src_subdir')], 581 return_stdout=True) 582 expected = set([ 583 suri(src_bucket_uri, 'src_subdir', 'foo'), 584 suri(src_bucket_uri, 'src_subdir', 'nested') + src_bucket_uri.delim]) 585 expected.add('') # Blank line between subdir listings. 586 actual = set(output.split('\n')) 587 self.assertEqual(expected, actual) 588 589 def testLsBucketSubdirRecursive(self): 590 """Test that ls -R of a bucket subdir returns expected results.""" 591 src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/foo', 592 'src_subdir/nested/foo2']) 593 for final_char in ('/', ''): 594 output = self.RunCommand( 595 'ls', ['-R', suri(src_bucket_uri, 'src_subdir') + final_char], 596 return_stdout=True) 597 expected = set([ 598 suri(src_bucket_uri, 'src_subdir', ':'), 599 suri(src_bucket_uri, 'src_subdir', 'foo'), 600 suri(src_bucket_uri, 'src_subdir', 'nested', ':'), 601 suri(src_bucket_uri, 'src_subdir', 'nested', 'foo2')]) 602 expected.add('') # Blank line between subdir listings. 603 actual = set(output.split('\n')) 604 self.assertEqual(expected, actual) 605 606 def testSetAclOnBucketRuns(self): 607 """Test that the 'acl set' command basically runs.""" 608 # We don't test reading back the acl (via 'acl get' command) because at 609 # present MockStorageService doesn't translate canned ACLs into actual ACL 610 # XML. 611 src_bucket_uri = self.CreateBucket() 612 self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri)]) 613 614 def testSetAclOnWildcardNamedBucketRuns(self): 615 """Test that 'acl set' basically runs against wildcard-named bucket.""" 616 # We don't test reading back the acl (via 'acl get' command) because at 617 # present MockStorageService doesn't translate canned ACLs into actual ACL 618 # XML. 619 src_bucket_uri = self.CreateBucket(test_objects=['f0']) 620 self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri)[:-2] + '*']) 621 622 def testSetAclOnObjectRuns(self): 623 """Test that the 'acl set' command basically runs.""" 624 src_bucket_uri = self.CreateBucket(test_objects=['f0']) 625 self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri, '*')]) 626 627 def testSetDefAclOnBucketRuns(self): 628 """Test that the 'defacl set' command basically runs.""" 629 src_bucket_uri = self.CreateBucket() 630 self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri)]) 631 632 def testSetDefAclOnObjectFails(self): 633 """Test that the 'defacl set' command fails when run against an object.""" 634 src_bucket_uri = self.CreateBucket() 635 try: 636 self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri, '*')]) 637 self.fail('Did not get expected CommandException') 638 except CommandException, e: 639 self.assertIn('URL must name a bucket', e.reason) 640 641 # @SequentialAndParallelTransfer 642 def testMinusDOptionWorks(self): 643 """Tests using gsutil -D option.""" 644 src_file = self.CreateTempFile(file_name='f0') 645 dst_bucket_uri = self.CreateBucket() 646 self.RunCommand('cp', [src_file, suri(dst_bucket_uri)], debug=3) 647 actual = list(self._test_wildcard_iterator( 648 suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) 649 self.assertEqual(1, len(actual)) 650 self.assertEqual('f0', actual[0].root_object.name) 651 652 # @SequentialAndParallelTransfer 653 def testFlatCopyingObjsAndFilesToBucketSubDir(self): 654 """Tests copying flatly listed objects and files to bucket subdir.""" 655 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'd0/f1', 'd1/d2/f2']) 656 src_dir = self.CreateTempDir(test_files=['f3', ('d3', 'f4'), 657 ('d4', 'd5', 'f5')]) 658 dst_bucket_uri = self.CreateBucket(test_objects=['dst_subdir0/existing', 659 'dst_subdir1/existing']) 660 # Test with and without final slash on dest subdir. 661 for i, final_char in enumerate(('/', '')): 662 self.RunCommand( 663 'cp', ['-R', suri(src_bucket_uri, '**'), os.path.join(src_dir, '**'), 664 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char]) 665 666 actual = set(str(u) for u in self._test_wildcard_iterator( 667 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 668 expected = set() 669 for i in range(2): 670 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing')) 671 for j in range(6): 672 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'f%d' % j)) 673 self.assertEqual(expected, actual) 674 675 # @SequentialAndParallelTransfer 676 def testRecursiveCopyObjsAndFilesToExistingBucketSubDir(self): 677 """Tests recursive copy of objects and files to existing bucket subdir.""" 678 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1']) 679 dst_bucket_uri = self.CreateBucket(test_objects=[ 680 'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj']) 681 src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')]) 682 # Test with and without final slash on dest subdir. 683 for i, final_char in enumerate(('/', '')): 684 self.RunCommand( 685 'cp', ['-R', suri(src_bucket_uri), src_dir, 686 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char]) 687 actual = set(str(u) for u in self._test_wildcard_iterator( 688 suri(dst_bucket_uri, 'dst_subdir%d' % i, '**')).IterAll( 689 expand_top_level_buckets=True)) 690 tmp_dirname = os.path.split(src_dir)[1] 691 bucketname = src_bucket_uri.bucket_name 692 expected = set([ 693 suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing_obj'), 694 suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'f0'), 695 suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'nested', 'f1'), 696 suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'f2'), 697 suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'nested', 'f3') 698 ]) 699 self.assertEqual(expected, actual) 700 701 # @SequentialAndParallelTransfer 702 def testRecursiveCopyObjsAndFilesToNonExistentBucketSubDir(self): 703 """Tests recursive copy of objs + files to non-existent bucket subdir.""" 704 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1']) 705 src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')]) 706 dst_bucket_uri = self.CreateBucket() 707 self.RunCommand('cp', ['-R', src_dir, suri(src_bucket_uri), 708 suri(dst_bucket_uri, 'dst_subdir')]) 709 actual = set(str(u) for u in self._test_wildcard_iterator( 710 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 711 expected = set([suri(dst_bucket_uri, 'dst_subdir', 'f0'), 712 suri(dst_bucket_uri, 'dst_subdir', 'nested', 'f1'), 713 suri(dst_bucket_uri, 'dst_subdir', 'f2'), 714 suri(dst_bucket_uri, 'dst_subdir', 'nested', 'f3')]) 715 self.assertEqual(expected, actual) 716 717 def testCopyingBucketSubDirToDir(self): 718 """Tests copying a bucket subdir to a directory.""" 719 src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/obj']) 720 dst_dir = self.CreateTempDir() 721 # Test with and without final slash on dest subdir. 722 for (final_src_char, final_dst_char) in ( 723 ('', ''), ('', '/'), ('/', ''), ('/', '/')): 724 # Mock objects don't support hash digestion. 725 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): 726 self.RunCommand( 727 'cp', ['-R', suri(src_bucket_uri, 'src_subdir') + final_src_char, 728 dst_dir + final_dst_char]) 729 actual = set(str(u) for u in self._test_wildcard_iterator( 730 '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) 731 expected = set([suri(dst_dir, 'src_subdir', 'obj')]) 732 self.assertEqual(expected, actual) 733 734 def testCopyingWildcardSpecifiedBucketSubDirToExistingDir(self): 735 """Tests copying a wildcard-specified bucket subdir to a directory.""" 736 src_bucket_uri = self.CreateBucket( 737 test_objects=['src_sub0dir/foo', 'src_sub1dir/foo', 'src_sub2dir/foo', 738 'src_sub3dir/foo']) 739 dst_dir = self.CreateTempDir() 740 # Test with and without final slash on dest subdir. 741 for i, (final_src_char, final_dst_char) in enumerate(( 742 ('', ''), ('', '/'), ('/', ''), ('/', '/'))): 743 # Mock objects don't support hash digestion. 744 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): 745 self.RunCommand( 746 'cp', ['-R', suri(src_bucket_uri, 'src_sub%d*' % i) + 747 final_src_char, dst_dir + final_dst_char]) 748 actual = set(str(u) for u in self._test_wildcard_iterator( 749 os.path.join(dst_dir, 'src_sub%ddir' % i, '**')).IterAll( 750 expand_top_level_buckets=True)) 751 expected = set([suri(dst_dir, 'src_sub%ddir' % i, 'foo')]) 752 self.assertEqual(expected, actual) 753 754 def testCopyingBucketSubDirToDirFailsWithoutMinusR(self): 755 """Tests for failure when attempting bucket subdir copy without -R.""" 756 src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/obj']) 757 dst_dir = self.CreateTempDir() 758 try: 759 self.RunCommand( 760 'cp', [suri(src_bucket_uri, 'src_subdir'), dst_dir]) 761 self.fail('Did not get expected CommandException') 762 except CommandException, e: 763 self.assertIn('No URLs matched', e.reason) 764 765 def testCopyingBucketSubDirToBucketSubDir(self): 766 """Tests copying a bucket subdir to another bucket subdir.""" 767 src_bucket_uri = self.CreateBucket( 768 test_objects=['src_subdir_%d/obj' % i for i in range(4)]) 769 dst_bucket_uri = self.CreateBucket( 770 test_objects=['dst_subdir_%d/obj2' % i for i in range(4)]) 771 # Test with and without final slash on dest subdir. 772 for i, (final_src_char, final_dst_char) in enumerate(( 773 ('', ''), ('', '/'), ('/', ''), ('/', '/'))): 774 self.RunCommand( 775 'cp', ['-R', 776 suri(src_bucket_uri, 'src_subdir_%d' % i) + final_src_char, 777 suri(dst_bucket_uri, 'dst_subdir_%d' % i) + final_dst_char]) 778 actual = set(str(u) for u in self._test_wildcard_iterator( 779 suri(dst_bucket_uri, 'dst_subdir_%d' % i, '**')).IterAll( 780 expand_top_level_buckets=True)) 781 expected = set([suri(dst_bucket_uri, 'dst_subdir_%d' % i, 782 'src_subdir_%d' % i, 'obj'), 783 suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'obj2')]) 784 self.assertEqual(expected, actual) 785 786 def testCopyingBucketSubDirToBucketSubDirWithNested(self): 787 """Tests copying a bucket subdir to another bucket subdir with nesting.""" 788 src_bucket_uri = self.CreateBucket( 789 test_objects=['src_subdir_%d/obj' % i for i in range(4)] + 790 ['src_subdir_%d/nested/obj' % i for i in range(4)]) 791 dst_bucket_uri = self.CreateBucket( 792 test_objects=['dst_subdir_%d/obj2' % i for i in range(4)]) 793 # Test with and without final slash on dest subdir. 794 for i, (final_src_char, final_dst_char) in enumerate(( 795 ('', ''), ('', '/'), ('/', ''), ('/', '/'))): 796 self.RunCommand( 797 'cp', ['-R', 798 suri(src_bucket_uri, 'src_subdir_%d' % i) + final_src_char, 799 suri(dst_bucket_uri, 'dst_subdir_%d' % i) + final_dst_char]) 800 actual = set(str(u) for u in self._test_wildcard_iterator( 801 suri(dst_bucket_uri, 'dst_subdir_%d' % i, '**')).IterAll( 802 expand_top_level_buckets=True)) 803 expected = set([suri(dst_bucket_uri, 'dst_subdir_%d' % i, 804 'src_subdir_%d' % i, 'obj'), 805 suri(dst_bucket_uri, 'dst_subdir_%d' % i, 806 'src_subdir_%d' % i, 'nested', 'obj'), 807 suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'obj2')]) 808 self.assertEqual(expected, actual) 809 810 def testMovingBucketSubDirToExistingBucketSubDir(self): 811 """Tests moving a bucket subdir to a existing bucket subdir.""" 812 src_objs = ['foo'] 813 for i in range(4): 814 src_objs.extend(['src_subdir%d/foo2' % i, 'src_subdir%d/nested/foo3' % i]) 815 src_bucket_uri = self.CreateBucket(test_objects=src_objs) 816 dst_bucket_uri = self.CreateBucket( 817 test_objects=['dst_subdir%d/existing' % i for i in range(4)]) 818 # Test with and without final slash on dest subdir. 819 for i, (final_src_char, final_dst_char) in enumerate(( 820 ('', ''), ('', '/'), ('/', ''), ('/', '/'))): 821 self.RunCommand( 822 'mv', [suri(src_bucket_uri, 'src_subdir%d' % i) + final_src_char, 823 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char]) 824 825 actual = set(str(u) for u in self._test_wildcard_iterator( 826 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 827 expected = set() 828 for i in range(4): 829 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing')) 830 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'src_subdir%d' %i, 831 'foo2')) 832 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'src_subdir%d' %i, 833 'nested', 'foo3')) 834 self.assertEqual(expected, actual) 835 836 def testCopyingObjectToBucketSubDir(self): 837 """Tests copying an object to a bucket subdir.""" 838 src_bucket_uri = self.CreateBucket(test_objects=['obj0']) 839 dst_bucket_uri = self.CreateBucket(test_objects=['dir0/existing', 840 'dir1/existing']) 841 # Test with and without final slash on dest subdir. 842 for i, final_dst_char in enumerate(('', '/')): 843 self.RunCommand('cp', [ 844 suri(src_bucket_uri, 'obj0'), 845 suri(dst_bucket_uri, 'dir%d' % i) + final_dst_char]) 846 actual = set(str(u) for u in self._test_wildcard_iterator( 847 suri(dst_bucket_uri, 'dir%d' % i, '**')).IterAll( 848 expand_top_level_buckets=True)) 849 expected = set([suri(dst_bucket_uri, 'dir%d' % i, 'obj0'), 850 suri(dst_bucket_uri, 'dir%d' % i, 'existing')]) 851 self.assertEqual(expected, actual) 852 853 # @SequentialAndParallelTransfer 854 def testCopyingWildcardedFilesToBucketSubDir(self): 855 """Tests copying wildcarded files to a bucket subdir.""" 856 dst_bucket_uri = self.CreateBucket(test_objects=['subdir0/existing', 857 'subdir1/existing']) 858 src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2']) 859 # Test with and without final slash on dest subdir. 860 for i, final_dst_char in enumerate(('', '/')): 861 self.RunCommand( 862 'cp', [os.path.join(src_dir, 'f?'), 863 suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char]) 864 actual = set(str(u) for u in self._test_wildcard_iterator( 865 suri(dst_bucket_uri, 'subdir%d' % i, '**')).IterAll( 866 expand_top_level_buckets=True)) 867 expected = set([suri(dst_bucket_uri, 'subdir%d' % i, 'existing'), 868 suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), 869 suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), 870 suri(dst_bucket_uri, 'subdir%d' % i, 'f2')]) 871 self.assertEqual(expected, actual) 872 873 # @SequentialAndParallelTransfer 874 def testCopyingOneNestedFileToBucketSubDir(self): 875 """Tests copying one nested file to a bucket subdir.""" 876 dst_bucket_uri = self.CreateBucket(test_objects=['d0/placeholder', 877 'd1/placeholder']) 878 src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')]) 879 # Test with and without final slash on dest subdir. 880 for i, final_dst_char in enumerate(('', '/')): 881 self.RunCommand('cp', ['-r', suri(src_dir, 'd3'), 882 suri(dst_bucket_uri, 'd%d' % i) + final_dst_char]) 883 actual = set(str(u) for u in self._test_wildcard_iterator( 884 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 885 expected = set([ 886 suri(dst_bucket_uri, 'd0', 'placeholder'), 887 suri(dst_bucket_uri, 'd1', 'placeholder'), 888 suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'), 889 suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1')]) 890 self.assertEqual(expected, actual) 891 892 def testMovingWildcardedFilesToNonExistentBucketSubDir(self): 893 """Tests moving files to a non-existent bucket subdir.""" 894 # This tests for how we allow users to do something like: 895 # gsutil cp *.txt gs://bucket/dir 896 # where *.txt matches more than 1 file and gs://bucket/dir 897 # doesn't exist as a subdir. 898 # 899 src_bucket_uri = self.CreateBucket(test_objects=[ 900 'f0f0', 'f0f1', 'f1f0', 'f1f1']) 901 dst_bucket_uri = self.CreateBucket(test_objects=[ 902 'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj']) 903 # Test with and without final slash on dest subdir. 904 for i, final_dst_char in enumerate(('', '/')): 905 # Copy some files into place in dst bucket. 906 self.RunCommand( 907 'cp', [suri(src_bucket_uri, 'f%df*' % i), 908 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char]) 909 # Now do the move test. 910 self.RunCommand( 911 'mv', [suri(src_bucket_uri, 'f%d*' % i), 912 suri(dst_bucket_uri, 'nonexisting%d' % i) + final_dst_char]) 913 914 actual = set(str(u) for u in self._test_wildcard_iterator( 915 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 916 expected = set([ 917 suri(dst_bucket_uri, 'dst_subdir0', 'existing_obj'), 918 suri(dst_bucket_uri, 'dst_subdir0', 'f0f0'), 919 suri(dst_bucket_uri, 'dst_subdir0', 'f0f1'), 920 suri(dst_bucket_uri, 'nonexisting0', 'f0f0'), 921 suri(dst_bucket_uri, 'nonexisting0', 'f0f1'), 922 suri(dst_bucket_uri, 'dst_subdir1', 'existing_obj'), 923 suri(dst_bucket_uri, 'dst_subdir1', 'f1f0'), 924 suri(dst_bucket_uri, 'dst_subdir1', 'f1f1'), 925 suri(dst_bucket_uri, 'nonexisting1', 'f1f0'), 926 suri(dst_bucket_uri, 'nonexisting1', 'f1f1')]) 927 self.assertEqual(expected, actual) 928 929 def testMovingObjectToBucketSubDir(self): 930 """Tests moving an object to a bucket subdir.""" 931 src_bucket_uri = self.CreateBucket(test_objects=['obj0', 'obj1']) 932 dst_bucket_uri = self.CreateBucket(test_objects=[ 933 'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj']) 934 # Test with and without final slash on dest subdir. 935 for i, final_dst_char in enumerate(('', '/')): 936 self.RunCommand( 937 'mv', [suri(src_bucket_uri, 'obj%d' % i), 938 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char]) 939 940 actual = set(str(u) for u in self._test_wildcard_iterator( 941 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 942 expected = set([ 943 suri(dst_bucket_uri, 'dst_subdir0', 'existing_obj'), 944 suri(dst_bucket_uri, 'dst_subdir0', 'obj0'), 945 suri(dst_bucket_uri, 'dst_subdir1', 'existing_obj'), 946 suri(dst_bucket_uri, 'dst_subdir1', 'obj1')]) 947 self.assertEqual(expected, actual) 948 949 actual = set(str(u) for u in self._test_wildcard_iterator( 950 suri(src_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 951 self.assertEqual(actual, set()) 952 953 def testWildcardSrcSubDirMoveDisallowed(self): 954 """Tests moving a bucket subdir specified by wildcard is disallowed.""" 955 src_bucket_uri = self.CreateBucket(test_objects=['dir/foo1']) 956 dst_bucket_uri = self.CreateBucket(test_objects=['dir/foo2']) 957 try: 958 self.RunCommand( 959 'mv', [suri(src_bucket_uri, 'dir*'), suri(dst_bucket_uri, 'dir')]) 960 self.fail('Did not get expected CommandException') 961 except CommandException, e: 962 self.assertIn('mv command disallows naming', e.reason) 963 964 def testMovingBucketSubDirToNonExistentBucketSubDir(self): 965 """Tests moving a bucket subdir to a non-existent bucket subdir.""" 966 src_bucket = self.CreateBucket(test_objects=[ 967 'foo', 'src_subdir0/foo2', 'src_subdir0/nested/foo3', 968 'src_subdir1/foo2', 'src_subdir1/nested/foo3']) 969 dst_bucket = self.CreateBucket() 970 # Test with and without final slash on dest subdir. 971 for i, final_src_char in enumerate(('', '/')): 972 self.RunCommand( 973 'mv', [suri(src_bucket, 'src_subdir%d' % i) + final_src_char, 974 suri(dst_bucket, 'dst_subdir%d' % i)]) 975 976 actual = set(str(u) for u in self._test_wildcard_iterator( 977 suri(dst_bucket, '**')).IterAll(expand_top_level_buckets=True)) 978 # Unlike the case with copying, with mv we expect renaming to occur 979 # at the level of the src subdir, vs appending that subdir beneath the 980 # dst subdir like is done for copying. 981 expected = set([suri(dst_bucket, 'dst_subdir0', 'foo2'), 982 suri(dst_bucket, 'dst_subdir1', 'foo2'), 983 suri(dst_bucket, 'dst_subdir0', 'nested', 'foo3'), 984 suri(dst_bucket, 'dst_subdir1', 'nested', 'foo3')]) 985 self.assertEqual(expected, actual) 986 987 def testRemovingBucketSubDir(self): 988 """Tests removing a bucket subdir.""" 989 dst_bucket_uri = self.CreateBucket(test_objects=[ 990 'f0', 'dir0/f1', 'dir0/nested/f2', 'dir1/f1', 'dir1/nested/f2']) 991 # Test with and without final slash on dest subdir. 992 for i, final_src_char in enumerate(('', '/')): 993 # Test removing bucket subdir. 994 self.RunCommand( 995 'rm', ['-R', suri(dst_bucket_uri, 'dir%d' % i) + final_src_char]) 996 actual = set(str(u) for u in self._test_wildcard_iterator( 997 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 998 expected = set([suri(dst_bucket_uri, 'f0')]) 999 self.assertEqual(expected, actual) 1000 1001 def testRecursiveRemoveObjsInBucket(self): 1002 """Tests removing all objects in bucket via rm -R gs://bucket.""" 1003 bucket_uris = [ 1004 self.CreateBucket(test_objects=['f0', 'dir/f1', 'dir/nested/f2']), 1005 self.CreateBucket(test_objects=['f0', 'dir/f1', 'dir/nested/f2'])] 1006 # Test with and without final slash on dest subdir. 1007 for i, final_src_char in enumerate(('', '/')): 1008 # Test removing all objects via rm -R. 1009 self.RunCommand('rm', ['-R', suri(bucket_uris[i]) + final_src_char]) 1010 try: 1011 self.RunCommand('ls', [suri(bucket_uris[i])]) 1012 # Ensure exception is raised. 1013 self.assertTrue(False) 1014 except NotFoundException, e: 1015 self.assertEqual(e.status, 404) 1016 1017 def testUnicodeArgs(self): 1018 """Tests that you can list an object with unicode characters.""" 1019 object_name = u'フォ' 1020 bucket_uri = self.CreateBucket() 1021 self.CreateObject(bucket_uri=bucket_uri, object_name=object_name, 1022 contents='foo') 1023 object_name_bytes = object_name.encode(UTF8) 1024 stdout = self.RunCommand('ls', [suri(bucket_uri, object_name_bytes)], 1025 return_stdout=True) 1026 self.assertIn(object_name_bytes, stdout) 1027 1028 def testRecursiveListTrailingSlash(self): 1029 bucket_uri = self.CreateBucket() 1030 obj_uri = self.CreateObject( 1031 bucket_uri=bucket_uri, object_name='/', contents='foo') 1032 stdout = self.RunCommand('ls', ['-R', suri(bucket_uri)], return_stdout=True) 1033 # Note: The suri function normalizes the URI, so the double slash gets 1034 # removed. 1035 self.assertEqual(stdout.splitlines(), [suri(obj_uri) + '/:', 1036 suri(obj_uri) + '/']) 1037 1038 def FinalObjNameComponent(self, uri): 1039 """For gs://bucket/abc/def/ghi returns ghi.""" 1040 return uri.uri.rpartition('/')[-1] 1041 1042 def testFileContainingColon(self): 1043 url_str = 'abc:def' 1044 url = StorageUrlFromString(url_str) 1045 self.assertEqual('file', url.scheme) 1046 self.assertEqual('file://%s' % url_str, url.url_string) 1047 1048 1049# TODO: These should all be moved to their own test_*.py testing files. 1050class GsUtilCommandTests(testcase.GsUtilUnitTestCase): 1051 """Basic sanity check tests to make sure commands run.""" 1052 1053 def testDisableLoggingCommandRuns(self): 1054 """Test that the 'logging set off' command basically runs.""" 1055 src_bucket_uri = self.CreateBucket() 1056 self.RunCommand('logging', ['set', 'off', suri(src_bucket_uri)]) 1057 1058 def testEnableLoggingCommandRuns(self): 1059 """Test that the 'logging set on' command basically runs.""" 1060 src_bucket_uri = self.CreateBucket() 1061 self.RunCommand('logging', ['set', 'on', '-b', 'gs://log_bucket', 1062 suri(src_bucket_uri)]) 1063 1064 def testHelpCommandDoesntRaise(self): 1065 """Test that the help command doesn't raise (sanity checks all help).""" 1066 # Unset PAGER if defined, so help output paginating into $PAGER doesn't 1067 # cause test to pause. 1068 if 'PAGER' in os.environ: 1069 del os.environ['PAGER'] 1070 self.RunCommand('help', []) 1071 1072 def testCatCommandRuns(self): 1073 """Test that the cat command basically runs.""" 1074 src_uri = self.CreateObject(contents='foo') 1075 stdout = self.RunCommand('cat', [suri(src_uri)], return_stdout=True) 1076 self.assertEqual(stdout, 'foo') 1077 1078 def testGetLoggingCommandRuns(self): 1079 """Test that the 'logging get' command basically runs.""" 1080 src_bucket_uri = self.CreateBucket() 1081 self.RunCommand('logging', ['get', suri(src_bucket_uri)]) 1082 1083 def testMakeBucketsCommand(self): 1084 """Test mb on existing bucket.""" 1085 dst_bucket_uri = self.CreateBucket() 1086 try: 1087 self.RunCommand('mb', [suri(dst_bucket_uri)]) 1088 self.fail('Did not get expected StorageCreateError') 1089 except ServiceException, e: 1090 self.assertEqual(e.status, 409) 1091 1092 def testRemoveObjsCommand(self): 1093 """Test rm command on non-existent object.""" 1094 dst_bucket_uri = self.CreateBucket() 1095 try: 1096 self.RunCommand('rm', [suri(dst_bucket_uri, 'non_existent')]) 1097 self.fail('Did not get expected CommandException') 1098 except CommandException, e: 1099 self.assertIn('No URLs matched', e.reason) 1100 1101 # Now that gsutil ver computes a checksum it adds 1-3 seconds to test run 1102 # time (for in memory mocked tests that otherwise take ~ 0.1 seconds). Since 1103 # it provides very little test value, we're leaving this test commented out. 1104 # def testVerCommmandRuns(self): 1105 # """Test that the Ver command basically runs""" 1106 # self.RunCommand('ver', []) 1107