18d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# -*- coding: utf-8 -*- 28d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# Copyright 2011 Google Inc. All Rights Reserved. 38d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# 48d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# Licensed under the Apache License, Version 2.0 (the "License"); 58d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# you may not use this file except in compliance with the License. 68d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# You may obtain a copy of the License at 78d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# 88d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# http://www.apache.org/licenses/LICENSE-2.0 98d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# 108d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# Unless required by applicable law or agreed to in writing, software 118d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# distributed under the License is distributed on an "AS IS" BASIS, 128d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 138d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# See the License for the specific language governing permissions and 148d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# limitations under the License. 158d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi"""Implementation of Unix-like rm command for cloud storage providers.""" 168d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 178d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom __future__ import absolute_import 188d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 19cef7893435aa41160dd1255c43cb8498279738ccChris Craikfrom gslib.cloud_api import BucketNotFoundException 208d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.cloud_api import NotEmptyException 21cef7893435aa41160dd1255c43cb8498279738ccChris Craikfrom gslib.cloud_api import NotFoundException 228d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.cloud_api import ServiceException 238d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.command import Command 248d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.command import GetFailureCount 258d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.command import ResetFailureCount 268d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.command_argument import CommandArgument 278d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.cs_api_map import ApiSelector 288d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.exception import CommandException 298d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.name_expansion import NameExpansionIterator 308d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.storage_url import StorageUrlFromString 318d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.translation_helper import PreconditionsFromHeaders 328d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.util import GetCloudApiInstance 338d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.util import NO_MAX 348d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.util import Retry 358d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoifrom gslib.util import StdinIterator 368d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 378d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 388d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi_SYNOPSIS = """ 398d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm [-f] [-r] url... 408d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm [-f] [-r] -I 418d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi""" 428d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 438d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi_DETAILED_HELP_TEXT = (""" 448d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi<B>SYNOPSIS</B> 458d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi""" + _SYNOPSIS + """ 468d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 478d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 488d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi<B>DESCRIPTION</B> 498d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi The gsutil rm command removes objects. 508d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi For example, the command: 518d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 528d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm gs://bucket/subdir/* 538d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 548d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi will remove all objects in gs://bucket/subdir, but not in any of its 558d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi sub-directories. In contrast: 568d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 578d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm gs://bucket/subdir/** 588d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 598d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi will remove all objects under gs://bucket/subdir or any of its 608d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi subdirectories. 618d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 628d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi You can also use the -r option to specify recursive object deletion. Thus, for 638d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi example, either of the following two commands will remove gs://bucket/subdir 648d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi and all objects and subdirectories under it: 658d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 668d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm gs://bucket/subdir** 678d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm -r gs://bucket/subdir 688d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 698d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi The -r option will also delete all object versions in the subdirectory for 708d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi versioning-enabled buckets, whereas the ** command will only delete the live 718d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi version of each object in the subdirectory. 728d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 738d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi Running gsutil rm -r on a bucket will delete all versions of all objects in 748d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi the bucket, and then delete the bucket: 758d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 768d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm -r gs://bucket 778d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 788d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi If you want to delete all objects in the bucket, but not the bucket itself, 798d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi this command will work: 808d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 818d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm gs://bucket/** 828d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 838d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi If you have a large number of objects to remove you might want to use the 848d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil -m option, to perform a parallel (multi-threaded/multi-processing) 858d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi removes: 868d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 878d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil -m rm -r gs://my_bucket/subdir 888d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 898d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi You can pass a list of URLs (one per line) to remove on stdin instead of as 908d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi command line arguments by using the -I option. This allows you to use gsutil 918d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi in a pipeline to remove objects identified by a program, such as: 928d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 938d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi some_program | gsutil -m rm -I 948d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 958d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi The contents of stdin can name cloud URLs and wildcards of cloud URLs. 968d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 978d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi Note that gsutil rm will refuse to remove files from the local 988d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi file system. For example this will fail: 998d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1008d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil rm *.txt 1018d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1028d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi WARNING: Object removal cannot be undone. Google Cloud Storage is designed 1038d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi to give developers a high amount of flexibility and control over their data, 1048d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi and Google maintains strict controls over the processing and purging of 1058d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi deleted data. To protect yourself from mistakes, you can configure object 1068d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi versioning on your bucket(s). See 'gsutil help versions' for details. 1078d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1088d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1098d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi<B>DATA RESTORATION FROM ACCIDENTAL DELETION OR OVERWRITES</B> 1108d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi KandoiGoogle Cloud Storage does not provide support for restoring data lost 1118d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoior overwritten due to customer errors. If you have concerns that your 1128d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoiapplication software (or your users) may at some point erroneously delete or 1138d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoioverwrite data, you can protect yourself from that risk by enabling Object 1148d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi KandoiVersioning (see "gsutil help versioning"). Doing so increases storage costs, 1158d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoiwhich can be partially mitigated by configuring Lifecycle Management to delete 1168d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoiolder object versions (see "gsutil help lifecycle"). 1178d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1188d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1198d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi<B>OPTIONS</B> 1208d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi -f Continues silently (without printing error messages) despite 1218d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi errors when removing multiple objects. If some of the objects 1228d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi could not be removed, gsutil's exit status will be non-zero even 1238d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if this flag is set. This option is implicitly set when running 1248d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi "gsutil -m rm ...". 1258d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1268d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi -I Causes gsutil to read the list of objects to remove from stdin. 1278d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi This allows you to run a program that generates the list of 1288d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi objects to remove. 1298d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1308d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi -R, -r Causes bucket or bucket subdirectory contents (all objects and 1318d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi subdirectories that it contains) to be removed recursively. If 1328d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi used with a bucket-only URL (like gs://bucket), after deleting 1338d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi objects and subdirectories gsutil will delete the bucket. The -r 1348d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi flag implies the -a flag and will delete all object versions. 1358d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1368d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi -a Delete all versions of an object. 1378d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi""") 1388d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1398d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1408d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoidef _RemoveExceptionHandler(cls, e): 1418d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi """Simple exception handler to allow post-completion status.""" 1428d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if not cls.continue_on_error: 1438d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi cls.logger.error(str(e)) 144cef7893435aa41160dd1255c43cb8498279738ccChris Craik # TODO: Use shared state to track missing bucket names when we get a 145cef7893435aa41160dd1255c43cb8498279738ccChris Craik # BucketNotFoundException. Then improve bucket removal logic and exception 146cef7893435aa41160dd1255c43cb8498279738ccChris Craik # messages. 147cef7893435aa41160dd1255c43cb8498279738ccChris Craik if isinstance(e, BucketNotFoundException): 148cef7893435aa41160dd1255c43cb8498279738ccChris Craik cls.bucket_not_found_count += 1 149cef7893435aa41160dd1255c43cb8498279738ccChris Craik cls.logger.error(str(e)) 150cef7893435aa41160dd1255c43cb8498279738ccChris Craik else: 151cef7893435aa41160dd1255c43cb8498279738ccChris Craik cls.op_failure_count += 1 1528d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1538d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1548d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi# pylint: disable=unused-argument 1558d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoidef _RemoveFoldersExceptionHandler(cls, e): 1568d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi """When removing folders, we don't mind if none exist.""" 1578d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if (isinstance(e, CommandException.__class__) and 158cef7893435aa41160dd1255c43cb8498279738ccChris Craik 'No URLs matched' in e.message) or isinstance(e, NotFoundException): 1598d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi pass 1608d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi else: 1618d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi raise e 1628d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1638d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1648d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoidef _RemoveFuncWrapper(cls, name_expansion_result, thread_state=None): 1658d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi cls.RemoveFunc(name_expansion_result, thread_state=thread_state) 1668d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1678d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1688d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoiclass RmCommand(Command): 1698d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi """Implementation of gsutil rm command.""" 1708d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1718d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Command specification. See base class for documentation. 1728d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi command_spec = Command.CreateCommandSpec( 1738d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 'rm', 1748d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi command_name_aliases=['del', 'delete', 'remove'], 1758d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi usage_synopsis=_SYNOPSIS, 1768d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi min_args=0, 1778d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi max_args=NO_MAX, 1788d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi supported_sub_args='afIrR', 1798d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi file_url_ok=False, 1808d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi provider_url_ok=False, 1818d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi urls_start_arg=0, 1828d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gs_api_support=[ApiSelector.XML, ApiSelector.JSON], 1838d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gs_default_api=ApiSelector.JSON, 1848d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi argparse_arguments=[ 1858d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi CommandArgument.MakeZeroOrMoreCloudURLsArgument() 1868d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi ] 1878d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi ) 1888d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Help specification. See help_provider.py for documentation. 1898d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi help_spec = Command.HelpSpec( 1908d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi help_name='rm', 1918d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi help_name_aliases=['del', 'delete', 'remove'], 1928d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi help_type='command_help', 1938d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi help_one_line_summary='Remove objects', 1948d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi help_text=_DETAILED_HELP_TEXT, 1958d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi subcommand_help_text={}, 1968d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi ) 1978d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 1988d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi def RunCommand(self): 1998d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi """Command entry point for the rm command.""" 2008d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # self.recursion_requested is initialized in command.py (so it can be 2018d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # checked in parent class for all commands). 202cef7893435aa41160dd1255c43cb8498279738ccChris Craik self.continue_on_error = self.parallel_operations 2038d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.read_args_from_stdin = False 2048d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.all_versions = False 2058d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if self.sub_opts: 2068d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi for o, unused_a in self.sub_opts: 2078d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if o == '-a': 2088d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.all_versions = True 2098d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi elif o == '-f': 2108d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.continue_on_error = True 2118d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi elif o == '-I': 2128d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.read_args_from_stdin = True 2138d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi elif o == '-r' or o == '-R': 2148d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.recursion_requested = True 2158d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.all_versions = True 2168d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 2178d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if self.read_args_from_stdin: 2188d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if self.args: 2198d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi raise CommandException('No arguments allowed with the -I flag.') 2208d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi url_strs = StdinIterator() 2218d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi else: 2228d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if not self.args: 2238d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi raise CommandException('The rm command (without -I) expects at ' 2248d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 'least one URL.') 2258d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi url_strs = self.args 2268d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 227cef7893435aa41160dd1255c43cb8498279738ccChris Craik # Tracks if any deletes failed. 228cef7893435aa41160dd1255c43cb8498279738ccChris Craik self.op_failure_count = 0 229cef7893435aa41160dd1255c43cb8498279738ccChris Craik 230cef7893435aa41160dd1255c43cb8498279738ccChris Craik # Tracks if any buckets were missing. 231cef7893435aa41160dd1255c43cb8498279738ccChris Craik self.bucket_not_found_count = 0 232cef7893435aa41160dd1255c43cb8498279738ccChris Craik 2338d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi bucket_urls_to_delete = [] 2348d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi bucket_strings_to_delete = [] 2358d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if self.recursion_requested: 2368d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi bucket_fields = ['id'] 2378d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi for url_str in url_strs: 2388d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi url = StorageUrlFromString(url_str) 2398d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if url.IsBucket() or url.IsProvider(): 2408d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi for blr in self.WildcardIterator(url_str).IterBuckets( 2418d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi bucket_fields=bucket_fields): 2428d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi bucket_urls_to_delete.append(blr.storage_url) 2438d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi bucket_strings_to_delete.append(url_str) 2448d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 2458d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.preconditions = PreconditionsFromHeaders(self.headers or {}) 2468d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 2478d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi try: 2488d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Expand wildcards, dirs, buckets, and bucket subdirs in URLs. 2498d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi name_expansion_iterator = NameExpansionIterator( 2508d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.command_name, self.debug, self.logger, self.gsutil_api, 2518d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi url_strs, self.recursion_requested, project_id=self.project_id, 2528d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi all_versions=self.all_versions, 2538d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi continue_on_error=self.continue_on_error or self.parallel_operations) 2548d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 2558d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Perform remove requests in parallel (-m) mode, if requested, using 2568d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # configured number of parallel processes and threads. Otherwise, 2578d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # perform requests with sequential function calls in current process. 2588d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.Apply(_RemoveFuncWrapper, name_expansion_iterator, 2598d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi _RemoveExceptionHandler, 260cef7893435aa41160dd1255c43cb8498279738ccChris Craik fail_on_error=(not self.continue_on_error), 261cef7893435aa41160dd1255c43cb8498279738ccChris Craik shared_attrs=['op_failure_count', 'bucket_not_found_count']) 2628d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 2638d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Assuming the bucket has versioning enabled, url's that don't map to 2648d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # objects should throw an error even with all_versions, since the prior 2658d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # round of deletes only sends objects to a history table. 2668d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # This assumption that rm -a is only called for versioned buckets should be 2678d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # corrected, but the fix is non-trivial. 2688d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi except CommandException as e: 2698d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Don't raise if there are buckets to delete -- it's valid to say: 2708d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # gsutil rm -r gs://some_bucket 2718d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # if the bucket is empty. 2728d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if not bucket_urls_to_delete and not self.continue_on_error: 2738d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi raise 2748d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Reset the failure count if we failed due to an empty bucket that we're 2758d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # going to delete. 2768d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi msg = 'No URLs matched: ' 2778d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if msg in str(e): 2788d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi parts = str(e).split(msg) 2798d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if len(parts) == 2 and parts[1] in bucket_strings_to_delete: 2808d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi ResetFailureCount() 281cef7893435aa41160dd1255c43cb8498279738ccChris Craik else: 282cef7893435aa41160dd1255c43cb8498279738ccChris Craik raise 2838d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi except ServiceException, e: 2848d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if not self.continue_on_error: 2858d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi raise 2868d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 287cef7893435aa41160dd1255c43cb8498279738ccChris Craik if self.bucket_not_found_count: 288cef7893435aa41160dd1255c43cb8498279738ccChris Craik raise CommandException('Encountered non-existent bucket during listing') 289cef7893435aa41160dd1255c43cb8498279738ccChris Craik 290cef7893435aa41160dd1255c43cb8498279738ccChris Craik if self.op_failure_count and not self.continue_on_error: 2918d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi raise CommandException('Some files could not be removed.') 2928d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 2938d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # If this was a gsutil rm -r command covering any bucket subdirs, 2948d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # remove any dir_$folder$ objects (which are created by various web UI 2958d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # tools to simulate folders). 2968d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if self.recursion_requested: 2978d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi had_previous_failures = GetFailureCount() > 0 2988d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi folder_object_wildcards = [] 2998d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi for url_str in url_strs: 3008d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi url = StorageUrlFromString(url_str) 3018d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if url.IsObject(): 3028d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi folder_object_wildcards.append('%s**_$folder$' % url_str) 3038d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if folder_object_wildcards: 3048d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.continue_on_error = True 3058d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi try: 3068d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi name_expansion_iterator = NameExpansionIterator( 3078d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.command_name, self.debug, 3088d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.logger, self.gsutil_api, 3098d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi folder_object_wildcards, self.recursion_requested, 3108d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi project_id=self.project_id, 3118d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi all_versions=self.all_versions) 3128d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # When we're removing folder objects, always continue on error 3138d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.Apply(_RemoveFuncWrapper, name_expansion_iterator, 3148d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi _RemoveFoldersExceptionHandler, 3158d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi fail_on_error=False) 3168d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi except CommandException as e: 3178d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Ignore exception from name expansion due to an absent folder file. 3188d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if not e.reason.startswith('No URLs matched:'): 3198d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi raise 3208d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi if not had_previous_failures: 3218d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi ResetFailureCount() 3228d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 3238d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi # Now that all data has been deleted, delete any bucket URLs. 3248d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi for url in bucket_urls_to_delete: 3258d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.logger.info('Removing %s...', url) 3268d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 3278d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi @Retry(NotEmptyException, tries=3, timeout_secs=1) 3288d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi def BucketDeleteWithRetry(): 3298d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme) 3308d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 3318d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi BucketDeleteWithRetry() 3328d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 333cef7893435aa41160dd1255c43cb8498279738ccChris Craik if self.op_failure_count: 334cef7893435aa41160dd1255c43cb8498279738ccChris Craik plural_str = 's' if self.op_failure_count else '' 335cef7893435aa41160dd1255c43cb8498279738ccChris Craik raise CommandException('%d file%s/object%s could not be removed.' % ( 336cef7893435aa41160dd1255c43cb8498279738ccChris Craik self.op_failure_count, plural_str, plural_str)) 337cef7893435aa41160dd1255c43cb8498279738ccChris Craik 3388d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi return 0 3398d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 3408d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi def RemoveFunc(self, name_expansion_result, thread_state=None): 3418d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) 3428d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 3438d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi exp_src_url = name_expansion_result.expanded_storage_url 3448d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi self.logger.info('Removing %s...', exp_src_url) 3458d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi gsutil_api.DeleteObject( 3468d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi exp_src_url.bucket_name, exp_src_url.object_name, 3478d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi preconditions=self.preconditions, generation=exp_src_url.generation, 3488d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi provider=exp_src_url.scheme) 3498d2b206a675ec20ea07100c35df34e65ee1e45e8Ruchi Kandoi 350