1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright 2013 Google Inc. All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Main module for Google Cloud Storage command line tool.""" 17 18from __future__ import absolute_import 19 20import ConfigParser 21import datetime 22import errno 23import getopt 24import logging 25import os 26import re 27import signal 28import socket 29import sys 30import textwrap 31import traceback 32 33# Load the gsutil version number and append it to boto.UserAgent so the value is 34# set before anything instantiates boto. This has to run after THIRD_PARTY_DIR 35# is modified (done in gsutil.py) but before any calls are made that would cause 36# boto.s3.Connection to be loaded - otherwise the Connection class would end up 37# with a static reference to the pre-modified version of the UserAgent field, 38# so boto requests would not include gsutil/version# in the UserAgent string. 39import boto 40import gslib 41# TODO: gsutil-beta: Cloud SDK scans for this string and performs 42# substitution; ensure this works with both apitools and boto. 43boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform) 44if os.environ.get('CLOUDSDK_WRAPPER') == '1': 45 boto.UserAgent += ' Cloud SDK Command Line Tool' 46 if os.environ.get('CLOUDSDK_VERSION'): 47 boto.UserAgent += ' %s' % os.environ.get('CLOUDSDK_VERSION') 48 49# pylint: disable=g-bad-import-order 50# pylint: disable=g-import-not-at-top 51import httplib2 52import oauth2client 53from gslib import wildcard_iterator 54from gslib.cloud_api import AccessDeniedException 55from gslib.cloud_api import ArgumentException 56from gslib.cloud_api import BadRequestException 57from gslib.cloud_api import ProjectIdException 58from gslib.cloud_api import ServiceException 59from gslib.command_runner import CommandRunner 60import gslib.exception 61from gslib.exception import CommandException 62import apitools.base.py.exceptions as apitools_exceptions 63from gslib.util import CreateLock 64from gslib.util import GetBotoConfigFileList 65from gslib.util import GetCertsFile 66from gslib.util import GetCleanupFiles 67from gslib.util import GsutilStreamHandler 68from gslib.util import ProxyInfoFromEnvironmentVar 69from gslib.sig_handling import GetCaughtSignals 70from gslib.sig_handling import InitializeSignalHandling 71from gslib.sig_handling import RegisterSignalHandler 72 73GSUTIL_CLIENT_ID = '909320924072.apps.googleusercontent.com' 74# Google OAuth2 clients always have a secret, even if the client is an installed 75# application/utility such as gsutil. Of course, in such cases the "secret" is 76# actually publicly known; security depends entirely on the secrecy of refresh 77# tokens, which effectively become bearer tokens. 78GSUTIL_CLIENT_NOTSOSECRET = 'p3RlpR10xMFh9ZXBS/ZNLYUu' 79if os.environ.get('CLOUDSDK_WRAPPER') == '1': 80 # Cloud SDK installs have a separate client ID / secret. 81 GSUTIL_CLIENT_ID = '32555940559.apps.googleusercontent.com' 82 GSUTIL_CLIENT_NOTSOSECRET = 'ZmssLNjJy2998hD4CTg2ejr2' 83 84CONFIG_KEYS_TO_REDACT = ['proxy', 'proxy_port', 'proxy_user', 'proxy_pass'] 85 86 87# We don't use the oauth2 authentication plugin directly; importing it here 88# ensures that it's loaded and available by default when an operation requiring 89# authentication is performed. 90try: 91 # pylint: disable=unused-import,g-import-not-at-top 92 import gcs_oauth2_boto_plugin 93except ImportError: 94 pass 95 96DEBUG_WARNING = """ 97***************************** WARNING ***************************** 98*** You are running gsutil with debug output enabled. 99*** Be aware that debug output includes authentication credentials. 100*** Make sure to remove the value of the Authorization header for 101*** each HTTP request printed to the console prior to posting to 102*** a public medium such as a forum post or Stack Overflow. 103***************************** WARNING ***************************** 104""".lstrip() 105 106TRACE_WARNING = """ 107***************************** WARNING ***************************** 108*** You are running gsutil with trace output enabled. 109*** Be aware that trace output includes authentication credentials 110*** and may include the contents of any files accessed during the trace. 111***************************** WARNING ***************************** 112""".lstrip() 113 114HTTP_WARNING = """ 115***************************** WARNING ***************************** 116*** You are running gsutil with the "https_validate_certificates" config 117*** variable set to False. This option should always be set to True in 118*** production environments to protect against man-in-the-middle attacks, 119*** and leaking of user data. 120***************************** WARNING ***************************** 121""".lstrip() 122 123debug = 0 124test_exception_traces = False 125 126 127# pylint: disable=unused-argument 128def _CleanupSignalHandler(signal_num, cur_stack_frame): 129 """Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM.""" 130 _Cleanup() 131 132 133def _Cleanup(): 134 for fname in GetCleanupFiles(): 135 try: 136 os.unlink(fname) 137 except: # pylint: disable=bare-except 138 pass 139 140 141def _OutputAndExit(message): 142 """Outputs message and exists with code 1.""" 143 from gslib.util import UTF8 # pylint: disable=g-import-not-at-top 144 if debug >= 2 or test_exception_traces: 145 stack_trace = traceback.format_exc() 146 err = ('DEBUG: Exception stack trace:\n %s\n' % 147 re.sub('\\n', '\n ', stack_trace)) 148 else: 149 err = '%s\n' % message 150 try: 151 sys.stderr.write(err.encode(UTF8)) 152 except UnicodeDecodeError: 153 # Can happen when outputting invalid Unicode filenames. 154 sys.stderr.write(err) 155 sys.exit(1) 156 157 158def _OutputUsageAndExit(command_runner): 159 command_runner.RunNamedCommand('help') 160 sys.exit(1) 161 162 163class GsutilFormatter(logging.Formatter): 164 """A logging.Formatter that supports logging microseconds (%f).""" 165 166 def formatTime(self, record, datefmt=None): 167 if datefmt: 168 return datetime.datetime.fromtimestamp(record.created).strftime(datefmt) 169 170 # Use default implementation if datefmt is not specified. 171 return super(GsutilFormatter, self).formatTime(record, datefmt=datefmt) 172 173 174def _ConfigureLogging(level=logging.INFO): 175 """Similar to logging.basicConfig() except it always adds a handler.""" 176 log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s' 177 date_format = '%m%d %H:%M:%S.%f' 178 formatter = GsutilFormatter(fmt=log_format, datefmt=date_format) 179 handler = GsutilStreamHandler() 180 handler.setFormatter(formatter) 181 root_logger = logging.getLogger() 182 root_logger.addHandler(handler) 183 root_logger.setLevel(level) 184 185 186def main(): 187 InitializeSignalHandling() 188 # Any modules used in initializing multiprocessing variables must be 189 # imported after importing gslib.__main__. 190 # pylint: disable=redefined-outer-name,g-import-not-at-top 191 import gslib.boto_translation 192 import gslib.command 193 import gslib.util 194 from gslib.util import BOTO_IS_SECURE 195 from gslib.util import CERTIFICATE_VALIDATION_ENABLED 196 # pylint: disable=unused-variable 197 from gcs_oauth2_boto_plugin import oauth2_client 198 from apitools.base.py import credentials_lib 199 # pylint: enable=unused-variable 200 from gslib.util import CheckMultiprocessingAvailableAndInit 201 if CheckMultiprocessingAvailableAndInit().is_available: 202 # These setup methods must be called, and, on Windows, they can only be 203 # called from within an "if __name__ == '__main__':" block. 204 gslib.command.InitializeMultiprocessingVariables() 205 gslib.boto_translation.InitializeMultiprocessingVariables() 206 else: 207 gslib.command.InitializeThreadingVariables() 208 209 # This needs to be done after gslib.util.InitializeMultiprocessingVariables(), 210 # since otherwise we can't call gslib.util.CreateLock. 211 try: 212 # pylint: disable=unused-import,g-import-not-at-top 213 import gcs_oauth2_boto_plugin 214 gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( 215 GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET) 216 gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock()) 217 credentials_lib.SetCredentialsCacheFileLock(CreateLock()) 218 except ImportError: 219 pass 220 221 global debug 222 global test_exception_traces 223 224 if not (2, 6) <= sys.version_info[:3] < (3,): 225 raise gslib.exception.CommandException( 226 'gsutil requires python 2.6 or 2.7.') 227 228 # In gsutil 4.0 and beyond, we don't use the boto library for the JSON 229 # API. However, we still store gsutil configuration data in the .boto 230 # config file for compatibility with previous versions and user convenience. 231 # Many users have a .boto configuration file from previous versions, and it 232 # is useful to have all of the configuration for gsutil stored in one place. 233 command_runner = CommandRunner() 234 if not BOTO_IS_SECURE: 235 raise CommandException('\n'.join(textwrap.wrap( 236 'Your boto configuration has is_secure = False. Gsutil cannot be ' 237 'run this way, for security reasons.'))) 238 239 headers = {} 240 parallel_operations = False 241 quiet = False 242 version = False 243 debug = 0 244 trace_token = None 245 test_exception_traces = False 246 247 # If user enters no commands just print the usage info. 248 if len(sys.argv) == 1: 249 sys.argv.append('help') 250 251 # Change the default of the 'https_validate_certificates' boto option to 252 # True (it is currently False in boto). 253 if not boto.config.has_option('Boto', 'https_validate_certificates'): 254 if not boto.config.has_section('Boto'): 255 boto.config.add_section('Boto') 256 boto.config.setbool('Boto', 'https_validate_certificates', True) 257 258 gslib.util.certs_file_lock = CreateLock() 259 for signal_num in GetCaughtSignals(): 260 RegisterSignalHandler(signal_num, _CleanupSignalHandler) 261 GetCertsFile() 262 263 try: 264 try: 265 opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq', 266 ['debug', 'detailedDebug', 'version', 'option', 267 'help', 'header', 'multithreaded', 'quiet', 268 'testexceptiontraces', 'trace-token=']) 269 except getopt.GetoptError as e: 270 _HandleCommandException(gslib.exception.CommandException(e.msg)) 271 for o, a in opts: 272 if o in ('-d', '--debug'): 273 # Passing debug=2 causes boto to include httplib header output. 274 debug = 3 275 elif o in ('-D', '--detailedDebug'): 276 # We use debug level 3 to ask gsutil code to output more detailed 277 # debug output. This is a bit of a hack since it overloads the same 278 # flag that was originally implemented for boto use. And we use -DD 279 # to ask for really detailed debugging (i.e., including HTTP payload). 280 if debug == 3: 281 debug = 4 282 else: 283 debug = 3 284 elif o in ('-?', '--help'): 285 _OutputUsageAndExit(command_runner) 286 elif o in ('-h', '--header'): 287 (hdr_name, _, hdr_val) = a.partition(':') 288 if not hdr_name: 289 _OutputUsageAndExit(command_runner) 290 headers[hdr_name.lower()] = hdr_val 291 elif o in ('-m', '--multithreaded'): 292 parallel_operations = True 293 elif o in ('-q', '--quiet'): 294 quiet = True 295 elif o in ('-v', '--version'): 296 version = True 297 elif o == '--trace-token': 298 trace_token = a 299 elif o == '--testexceptiontraces': # Hidden flag for integration tests. 300 test_exception_traces = True 301 elif o in ('-o', '--option'): 302 (opt_section_name, _, opt_value) = a.partition('=') 303 if not opt_section_name: 304 _OutputUsageAndExit(command_runner) 305 (opt_section, _, opt_name) = opt_section_name.partition(':') 306 if not opt_section or not opt_name: 307 _OutputUsageAndExit(command_runner) 308 if not boto.config.has_section(opt_section): 309 boto.config.add_section(opt_section) 310 boto.config.set(opt_section, opt_name, opt_value) 311 httplib2.debuglevel = debug 312 if trace_token: 313 sys.stderr.write(TRACE_WARNING) 314 if debug > 1: 315 sys.stderr.write(DEBUG_WARNING) 316 if debug >= 2: 317 _ConfigureLogging(level=logging.DEBUG) 318 command_runner.RunNamedCommand('ver', ['-l']) 319 config_items = [] 320 try: 321 config_items.extend(boto.config.items('Boto')) 322 config_items.extend(boto.config.items('GSUtil')) 323 except ConfigParser.NoSectionError: 324 pass 325 for i in xrange(len(config_items)): 326 config_item_key = config_items[i][0] 327 if config_item_key in CONFIG_KEYS_TO_REDACT: 328 config_items[i] = (config_item_key, 'REDACTED') 329 sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) 330 sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList()) 331 sys.stderr.write('config: %s\n' % str(config_items)) 332 elif quiet: 333 _ConfigureLogging(level=logging.WARNING) 334 else: 335 _ConfigureLogging(level=logging.INFO) 336 # oauth2client uses info logging in places that would better 337 # correspond to gsutil's debug logging (e.g., when refreshing 338 # access tokens). 339 oauth2client.client.logger.setLevel(logging.WARNING) 340 341 if not CERTIFICATE_VALIDATION_ENABLED: 342 sys.stderr.write(HTTP_WARNING) 343 344 if version: 345 command_name = 'version' 346 elif not args: 347 command_name = 'help' 348 else: 349 command_name = args[0] 350 351 _CheckAndWarnForProxyDifferences() 352 353 if os.environ.get('_ARGCOMPLETE', '0') == '1': 354 return _PerformTabCompletion(command_runner) 355 356 return _RunNamedCommandAndHandleExceptions( 357 command_runner, command_name, args=args[1:], headers=headers, 358 debug_level=debug, trace_token=trace_token, 359 parallel_operations=parallel_operations) 360 finally: 361 _Cleanup() 362 363 364def _CheckAndWarnForProxyDifferences(): 365 # If there are both boto config and environment variable config present for 366 # proxies, unset the environment variable and warn if it differs. 367 boto_port = boto.config.getint('Boto', 'proxy_port', 0) 368 if boto.config.get('Boto', 'proxy', None) or boto_port: 369 for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: 370 if proxy_env_var in os.environ and os.environ[proxy_env_var]: 371 differing_values = [] 372 proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) 373 if proxy_info.proxy_host != boto.config.get('Boto', 'proxy', None): 374 differing_values.append( 375 'Boto proxy host: "%s" differs from %s proxy host: "%s"' % 376 (boto.config.get('Boto', 'proxy', None), proxy_env_var, 377 proxy_info.proxy_host)) 378 if (proxy_info.proxy_user != 379 boto.config.get('Boto', 'proxy_user', None)): 380 differing_values.append( 381 'Boto proxy user: "%s" differs from %s proxy user: "%s"' % 382 (boto.config.get('Boto', 'proxy_user', None), proxy_env_var, 383 proxy_info.proxy_user)) 384 if (proxy_info.proxy_pass != 385 boto.config.get('Boto', 'proxy_pass', None)): 386 differing_values.append( 387 'Boto proxy password differs from %s proxy password' % 388 proxy_env_var) 389 # Only compare ports if at least one is present, since the 390 # boto logic for selecting default ports has not yet executed. 391 if ((proxy_info.proxy_port or boto_port) and 392 proxy_info.proxy_port != boto_port): 393 differing_values.append( 394 'Boto proxy port: "%s" differs from %s proxy port: "%s"' % 395 (boto_port, proxy_env_var, proxy_info.proxy_port)) 396 if differing_values: 397 sys.stderr.write('\n'.join(textwrap.wrap( 398 'WARNING: Proxy configuration is present in both the %s ' 399 'environment variable and boto configuration, but ' 400 'configuration differs. boto configuration proxy values will ' 401 'be used. Differences detected:' % proxy_env_var))) 402 sys.stderr.write('\n%s\n' % '\n'.join(differing_values)) 403 # Regardless of whether the proxy configuration values matched, 404 # delete the environment variable so as not to confuse boto. 405 del os.environ[proxy_env_var] 406 407 408def _HandleUnknownFailure(e): 409 # Called if we fall through all known/handled exceptions. 410 _OutputAndExit('Failure: %s.' % e) 411 412 413def _HandleCommandException(e): 414 if e.informational: 415 _OutputAndExit(e.reason) 416 else: 417 _OutputAndExit('CommandException: %s' % e.reason) 418 419 420# pylint: disable=unused-argument 421def _HandleControlC(signal_num, cur_stack_frame): 422 """Called when user hits ^C. 423 424 This function prints a brief message instead of the normal Python stack trace 425 (unless -D option is used). 426 427 Args: 428 signal_num: Signal that was caught. 429 cur_stack_frame: Unused. 430 """ 431 if debug >= 2: 432 stack_trace = ''.join(traceback.format_list(traceback.extract_stack())) 433 _OutputAndExit( 434 'DEBUG: Caught signal %d - Exception stack trace:\n' 435 ' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace))) 436 else: 437 _OutputAndExit('Caught signal %d - exiting' % signal_num) 438 439 440def _HandleSigQuit(signal_num, cur_stack_frame): 441 """Called when user hits ^\\, so we can force breakpoint a running gsutil.""" 442 import pdb # pylint: disable=g-import-not-at-top 443 pdb.set_trace() 444 445 446def _ConstructAccountProblemHelp(reason): 447 """Constructs a help string for an access control error. 448 449 Args: 450 reason: e.reason string from caught exception. 451 452 Returns: 453 Contructed help text. 454 """ 455 default_project_id = boto.config.get_value('GSUtil', 'default_project_id') 456 # pylint: disable=line-too-long, g-inconsistent-quotes 457 acct_help = ( 458 "Your request resulted in an AccountProblem (403) error. Usually this " 459 "happens if you attempt to create a bucket without first having " 460 "enabled billing for the project you are using. Please ensure billing is " 461 "enabled for your project by following the instructions at " 462 "`Google Developers Console<https://developers.google.com/console/help/billing>`. ") 463 if default_project_id: 464 acct_help += ( 465 "In the project overview, ensure that the Project Number listed for " 466 "your project matches the project ID (%s) from your boto config file. " 467 % default_project_id) 468 acct_help += ( 469 "If the above doesn't resolve your AccountProblem, please send mail to " 470 "gs-team@google.com requesting assistance, noting the exact command you " 471 "ran, the fact that you received a 403 AccountProblem error, and your " 472 "project ID. Please do not post your project ID on StackOverflow. " 473 "Note: It's possible to use Google Cloud Storage without enabling " 474 "billing if you're only listing or reading objects for which you're " 475 "authorized, or if you're uploading objects to a bucket billed to a " 476 "project that has billing enabled. But if you're attempting to create " 477 "buckets or upload objects to a bucket owned by your own project, you " 478 "must first enable billing for that project.") 479 return acct_help 480 481 482def _CheckAndHandleCredentialException(e, args): 483 # Provide detail to users who have no boto config file (who might previously 484 # have been using gsutil only for accessing publicly readable buckets and 485 # objects). 486 # pylint: disable=g-import-not-at-top 487 from gslib.util import HasConfiguredCredentials 488 if (not HasConfiguredCredentials() and 489 not boto.config.get_value('Tests', 'bypass_anonymous_access_warning', 490 False)): 491 # The check above allows tests to assert that we get a particular, 492 # expected failure, rather than always encountering this error message 493 # when there are no configured credentials. This allows tests to 494 # simulate a second user without permissions, without actually requiring 495 # two separate configured users. 496 if os.environ.get('CLOUDSDK_WRAPPER') == '1': 497 _OutputAndExit('\n'.join(textwrap.wrap( 498 'You are attempting to access protected data with no configured ' 499 'credentials. Please visit ' 500 'https://cloud.google.com/console#/project and sign up for an ' 501 'account, and then run the "gcloud auth login" command to ' 502 'configure gsutil to use these credentials.'))) 503 else: 504 _OutputAndExit('\n'.join(textwrap.wrap( 505 'You are attempting to access protected data with no configured ' 506 'credentials. Please visit ' 507 'https://cloud.google.com/console#/project and sign up for an ' 508 'account, and then run the "gsutil config" command to configure ' 509 'gsutil to use these credentials.'))) 510 elif (e.reason and 511 (e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or 512 'account for the specified project has been disabled' in e.reason) 513 and ','.join(args).find('gs://') != -1): 514 _OutputAndExit('\n'.join(textwrap.wrap( 515 _ConstructAccountProblemHelp(e.reason)))) 516 517 518def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None, 519 headers=None, debug_level=0, 520 trace_token=None, 521 parallel_operations=False): 522 """Runs the command with the given command runner and arguments.""" 523 # pylint: disable=g-import-not-at-top 524 from gslib.util import GetConfigFilePath 525 from gslib.util import IS_WINDOWS 526 from gslib.util import IsRunningInteractively 527 try: 528 # Catch ^C so we can print a brief message instead of the normal Python 529 # stack trace. Register as a final signal handler because this handler kills 530 # the main gsutil process (so it must run last). 531 RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True) 532 # Catch ^\ so we can force a breakpoint in a running gsutil. 533 if not IS_WINDOWS: 534 RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit) 535 return command_runner.RunNamedCommand(command_name, args, headers, 536 debug_level, trace_token, 537 parallel_operations) 538 except AttributeError as e: 539 if str(e).find('secret_access_key') != -1: 540 _OutputAndExit('Missing credentials for the given URI(s). Does your ' 541 'boto config file contain all needed credentials?') 542 else: 543 _OutputAndExit(str(e)) 544 except gslib.exception.CommandException as e: 545 _HandleCommandException(e) 546 except getopt.GetoptError as e: 547 _HandleCommandException(gslib.exception.CommandException(e.msg)) 548 except boto.exception.InvalidUriError as e: 549 _OutputAndExit('InvalidUriError: %s.' % e.message) 550 except gslib.exception.InvalidUrlError as e: 551 _OutputAndExit('InvalidUrlError: %s.' % e.message) 552 except boto.auth_handler.NotReadyToAuthenticate: 553 _OutputAndExit('NotReadyToAuthenticate') 554 except OSError as e: 555 _OutputAndExit('OSError: %s.' % e.strerror) 556 except IOError as e: 557 if (e.errno == errno.EPIPE or (IS_WINDOWS and e.errno == errno.EINVAL) 558 and not IsRunningInteractively()): 559 # If we get a pipe error, this just means that the pipe to stdout or 560 # stderr is broken. This can happen if the user pipes gsutil to a command 561 # that doesn't use the entire output stream. Instead of raising an error, 562 # just swallow it up and exit cleanly. 563 sys.exit(0) 564 else: 565 raise 566 except wildcard_iterator.WildcardException as e: 567 _OutputAndExit(e.reason) 568 except ProjectIdException as e: 569 _OutputAndExit( 570 'You are attempting to perform an operation that requires a ' 571 'project id, with none configured. Please re-run ' 572 'gsutil config and make sure to follow the instructions for ' 573 'finding and entering your default project id.') 574 except BadRequestException as e: 575 if e.reason == 'MissingSecurityHeader': 576 _CheckAndHandleCredentialException(e, args) 577 _OutputAndExit(e) 578 except AccessDeniedException as e: 579 _CheckAndHandleCredentialException(e, args) 580 _OutputAndExit(e) 581 except ArgumentException as e: 582 _OutputAndExit(e) 583 except ServiceException as e: 584 _OutputAndExit(e) 585 except apitools_exceptions.HttpError as e: 586 # These should usually be retried by the underlying implementation or 587 # wrapped by CloudApi ServiceExceptions, but if we do get them, 588 # print something useful. 589 _OutputAndExit('HttpError: %s, %s' % (getattr(e.response, 'status', ''), 590 e.content or '')) 591 except socket.error as e: 592 if e.args[0] == errno.EPIPE: 593 # Retrying with a smaller file (per suggestion below) works because 594 # the library code send loop (in boto/s3/key.py) can get through the 595 # entire file and then request the HTTP response before the socket 596 # gets closed and the response lost. 597 _OutputAndExit( 598 'Got a "Broken pipe" error. This can happen to clients using Python ' 599 '2.x, when the server sends an error response and then closes the ' 600 'socket (see http://bugs.python.org/issue5542). If you are trying to ' 601 'upload a large object you might retry with a small (say 200k) ' 602 'object, and see if you get a more specific error code.' 603 ) 604 else: 605 _HandleUnknownFailure(e) 606 except Exception as e: 607 # Check for two types of errors related to service accounts. These errors 608 # appear to be the same except for their messages, but they are caused by 609 # different problems and both have unhelpful error messages. Moreover, 610 # the error type belongs to PyOpenSSL, which is not necessarily installed. 611 if 'mac verify failure' in str(e): 612 _OutputAndExit( 613 'Encountered an error while refreshing access token. ' 614 'If you are using a service account,\nplease verify that the ' 615 'gs_service_key_file_password field in your config file,' 616 '\n%s, is correct.' % GetConfigFilePath()) 617 elif 'asn1 encoding routines' in str(e): 618 _OutputAndExit( 619 'Encountered an error while refreshing access token. ' 620 'If you are using a service account,\nplease verify that the ' 621 'gs_service_key_file field in your config file,\n%s, is correct.' 622 % GetConfigFilePath()) 623 _HandleUnknownFailure(e) 624 625 626def _PerformTabCompletion(command_runner): 627 """Performs gsutil-specific tab completion for the shell.""" 628 # argparse and argcomplete are bundled with the Google Cloud SDK. 629 # When gsutil is invoked from the Google Cloud SDK, both should be available. 630 try: 631 import argcomplete 632 import argparse 633 except ImportError as e: 634 _OutputAndExit('A library required for performing tab completion was' 635 ' not found.\nCause: %s' % e) 636 parser = argparse.ArgumentParser(add_help=False) 637 subparsers = parser.add_subparsers() 638 command_runner.ConfigureCommandArgumentParsers(subparsers) 639 argcomplete.autocomplete(parser, exit_method=sys.exit) 640 641 return 0 642 643if __name__ == '__main__': 644 sys.exit(main()) 645