1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# Copyright 2013 Google Inc. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Main module for Google Cloud Storage command line tool."""
17
18from __future__ import absolute_import
19
20import ConfigParser
21import datetime
22import errno
23import getopt
24import logging
25import os
26import re
27import signal
28import socket
29import sys
30import textwrap
31import traceback
32
33# Load the gsutil version number and append it to boto.UserAgent so the value is
34# set before anything instantiates boto. This has to run after THIRD_PARTY_DIR
35# is modified (done in gsutil.py) but before any calls are made that would cause
36# boto.s3.Connection to be loaded - otherwise the Connection class would end up
37# with a static reference to the pre-modified version of the UserAgent field,
38# so boto requests would not include gsutil/version# in the UserAgent string.
39import boto
40import gslib
41# TODO: gsutil-beta: Cloud SDK scans for this string and performs
42# substitution; ensure this works with both apitools and boto.
43boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform)
44if os.environ.get('CLOUDSDK_WRAPPER') == '1':
45  boto.UserAgent += ' Cloud SDK Command Line Tool'
46  if os.environ.get('CLOUDSDK_VERSION'):
47    boto.UserAgent += ' %s' % os.environ.get('CLOUDSDK_VERSION')
48
49# pylint: disable=g-bad-import-order
50# pylint: disable=g-import-not-at-top
51import httplib2
52import oauth2client
53from gslib import wildcard_iterator
54from gslib.cloud_api import AccessDeniedException
55from gslib.cloud_api import ArgumentException
56from gslib.cloud_api import BadRequestException
57from gslib.cloud_api import ProjectIdException
58from gslib.cloud_api import ServiceException
59from gslib.command_runner import CommandRunner
60import gslib.exception
61from gslib.exception import CommandException
62import apitools.base.py.exceptions as apitools_exceptions
63from gslib.util import CreateLock
64from gslib.util import GetBotoConfigFileList
65from gslib.util import GetCertsFile
66from gslib.util import GetCleanupFiles
67from gslib.util import GsutilStreamHandler
68from gslib.util import ProxyInfoFromEnvironmentVar
69from gslib.sig_handling import GetCaughtSignals
70from gslib.sig_handling import InitializeSignalHandling
71from gslib.sig_handling import RegisterSignalHandler
72
73GSUTIL_CLIENT_ID = '909320924072.apps.googleusercontent.com'
74# Google OAuth2 clients always have a secret, even if the client is an installed
75# application/utility such as gsutil.  Of course, in such cases the "secret" is
76# actually publicly known; security depends entirely on the secrecy of refresh
77# tokens, which effectively become bearer tokens.
78GSUTIL_CLIENT_NOTSOSECRET = 'p3RlpR10xMFh9ZXBS/ZNLYUu'
79if os.environ.get('CLOUDSDK_WRAPPER') == '1':
80  # Cloud SDK installs have a separate client ID / secret.
81  GSUTIL_CLIENT_ID = '32555940559.apps.googleusercontent.com'
82  GSUTIL_CLIENT_NOTSOSECRET = 'ZmssLNjJy2998hD4CTg2ejr2'
83
84CONFIG_KEYS_TO_REDACT = ['proxy', 'proxy_port', 'proxy_user', 'proxy_pass']
85
86
87# We don't use the oauth2 authentication plugin directly; importing it here
88# ensures that it's loaded and available by default when an operation requiring
89# authentication is performed.
90try:
91  # pylint: disable=unused-import,g-import-not-at-top
92  import gcs_oauth2_boto_plugin
93except ImportError:
94  pass
95
96DEBUG_WARNING = """
97***************************** WARNING *****************************
98*** You are running gsutil with debug output enabled.
99*** Be aware that debug output includes authentication credentials.
100*** Make sure to remove the value of the Authorization header for
101*** each HTTP request printed to the console prior to posting to
102*** a public medium such as a forum post or Stack Overflow.
103***************************** WARNING *****************************
104""".lstrip()
105
106TRACE_WARNING = """
107***************************** WARNING *****************************
108*** You are running gsutil with trace output enabled.
109*** Be aware that trace output includes authentication credentials
110*** and may include the contents of any files accessed during the trace.
111***************************** WARNING *****************************
112""".lstrip()
113
114HTTP_WARNING = """
115***************************** WARNING *****************************
116*** You are running gsutil with the "https_validate_certificates" config
117*** variable set to False. This option should always be set to True in
118*** production environments to protect against man-in-the-middle attacks,
119*** and leaking of user data.
120***************************** WARNING *****************************
121""".lstrip()
122
123debug = 0
124test_exception_traces = False
125
126
127# pylint: disable=unused-argument
128def _CleanupSignalHandler(signal_num, cur_stack_frame):
129  """Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM."""
130  _Cleanup()
131
132
133def _Cleanup():
134  for fname in GetCleanupFiles():
135    try:
136      os.unlink(fname)
137    except:  # pylint: disable=bare-except
138      pass
139
140
141def _OutputAndExit(message):
142  """Outputs message and exists with code 1."""
143  from gslib.util import UTF8  # pylint: disable=g-import-not-at-top
144  if debug >= 2 or test_exception_traces:
145    stack_trace = traceback.format_exc()
146    err = ('DEBUG: Exception stack trace:\n    %s\n' %
147           re.sub('\\n', '\n    ', stack_trace))
148  else:
149    err = '%s\n' % message
150  try:
151    sys.stderr.write(err.encode(UTF8))
152  except UnicodeDecodeError:
153    # Can happen when outputting invalid Unicode filenames.
154    sys.stderr.write(err)
155  sys.exit(1)
156
157
158def _OutputUsageAndExit(command_runner):
159  command_runner.RunNamedCommand('help')
160  sys.exit(1)
161
162
163class GsutilFormatter(logging.Formatter):
164  """A logging.Formatter that supports logging microseconds (%f)."""
165
166  def formatTime(self, record, datefmt=None):
167    if datefmt:
168      return datetime.datetime.fromtimestamp(record.created).strftime(datefmt)
169
170    # Use default implementation if datefmt is not specified.
171    return super(GsutilFormatter, self).formatTime(record, datefmt=datefmt)
172
173
174def _ConfigureLogging(level=logging.INFO):
175  """Similar to logging.basicConfig() except it always adds a handler."""
176  log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s'
177  date_format = '%m%d %H:%M:%S.%f'
178  formatter = GsutilFormatter(fmt=log_format, datefmt=date_format)
179  handler = GsutilStreamHandler()
180  handler.setFormatter(formatter)
181  root_logger = logging.getLogger()
182  root_logger.addHandler(handler)
183  root_logger.setLevel(level)
184
185
186def main():
187  InitializeSignalHandling()
188  # Any modules used in initializing multiprocessing variables must be
189  # imported after importing gslib.__main__.
190  # pylint: disable=redefined-outer-name,g-import-not-at-top
191  import gslib.boto_translation
192  import gslib.command
193  import gslib.util
194  from gslib.util import BOTO_IS_SECURE
195  from gslib.util import CERTIFICATE_VALIDATION_ENABLED
196  # pylint: disable=unused-variable
197  from gcs_oauth2_boto_plugin import oauth2_client
198  from apitools.base.py import credentials_lib
199  # pylint: enable=unused-variable
200  from gslib.util import CheckMultiprocessingAvailableAndInit
201  if CheckMultiprocessingAvailableAndInit().is_available:
202    # These setup methods must be called, and, on Windows, they can only be
203    # called from within an "if __name__ == '__main__':" block.
204    gslib.command.InitializeMultiprocessingVariables()
205    gslib.boto_translation.InitializeMultiprocessingVariables()
206  else:
207    gslib.command.InitializeThreadingVariables()
208
209  # This needs to be done after gslib.util.InitializeMultiprocessingVariables(),
210  # since otherwise we can't call gslib.util.CreateLock.
211  try:
212    # pylint: disable=unused-import,g-import-not-at-top
213    import gcs_oauth2_boto_plugin
214    gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
215        GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET)
216    gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock())
217    credentials_lib.SetCredentialsCacheFileLock(CreateLock())
218  except ImportError:
219    pass
220
221  global debug
222  global test_exception_traces
223
224  if not (2, 6) <= sys.version_info[:3] < (3,):
225    raise gslib.exception.CommandException(
226        'gsutil requires python 2.6 or 2.7.')
227
228  # In gsutil 4.0 and beyond, we don't use the boto library for the JSON
229  # API. However, we still store gsutil configuration data in the .boto
230  # config file for compatibility with previous versions and user convenience.
231  # Many users have a .boto configuration file from previous versions, and it
232  # is useful to have all of the configuration for gsutil stored in one place.
233  command_runner = CommandRunner()
234  if not BOTO_IS_SECURE:
235    raise CommandException('\n'.join(textwrap.wrap(
236        'Your boto configuration has is_secure = False. Gsutil cannot be '
237        'run this way, for security reasons.')))
238
239  headers = {}
240  parallel_operations = False
241  quiet = False
242  version = False
243  debug = 0
244  trace_token = None
245  test_exception_traces = False
246
247  # If user enters no commands just print the usage info.
248  if len(sys.argv) == 1:
249    sys.argv.append('help')
250
251  # Change the default of the 'https_validate_certificates' boto option to
252  # True (it is currently False in boto).
253  if not boto.config.has_option('Boto', 'https_validate_certificates'):
254    if not boto.config.has_section('Boto'):
255      boto.config.add_section('Boto')
256    boto.config.setbool('Boto', 'https_validate_certificates', True)
257
258  gslib.util.certs_file_lock = CreateLock()
259  for signal_num in GetCaughtSignals():
260    RegisterSignalHandler(signal_num, _CleanupSignalHandler)
261  GetCertsFile()
262
263  try:
264    try:
265      opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq',
266                                 ['debug', 'detailedDebug', 'version', 'option',
267                                  'help', 'header', 'multithreaded', 'quiet',
268                                  'testexceptiontraces', 'trace-token='])
269    except getopt.GetoptError as e:
270      _HandleCommandException(gslib.exception.CommandException(e.msg))
271    for o, a in opts:
272      if o in ('-d', '--debug'):
273        # Passing debug=2 causes boto to include httplib header output.
274        debug = 3
275      elif o in ('-D', '--detailedDebug'):
276        # We use debug level 3 to ask gsutil code to output more detailed
277        # debug output. This is a bit of a hack since it overloads the same
278        # flag that was originally implemented for boto use. And we use -DD
279        # to ask for really detailed debugging (i.e., including HTTP payload).
280        if debug == 3:
281          debug = 4
282        else:
283          debug = 3
284      elif o in ('-?', '--help'):
285        _OutputUsageAndExit(command_runner)
286      elif o in ('-h', '--header'):
287        (hdr_name, _, hdr_val) = a.partition(':')
288        if not hdr_name:
289          _OutputUsageAndExit(command_runner)
290        headers[hdr_name.lower()] = hdr_val
291      elif o in ('-m', '--multithreaded'):
292        parallel_operations = True
293      elif o in ('-q', '--quiet'):
294        quiet = True
295      elif o in ('-v', '--version'):
296        version = True
297      elif o == '--trace-token':
298        trace_token = a
299      elif o == '--testexceptiontraces':  # Hidden flag for integration tests.
300        test_exception_traces = True
301      elif o in ('-o', '--option'):
302        (opt_section_name, _, opt_value) = a.partition('=')
303        if not opt_section_name:
304          _OutputUsageAndExit(command_runner)
305        (opt_section, _, opt_name) = opt_section_name.partition(':')
306        if not opt_section or not opt_name:
307          _OutputUsageAndExit(command_runner)
308        if not boto.config.has_section(opt_section):
309          boto.config.add_section(opt_section)
310        boto.config.set(opt_section, opt_name, opt_value)
311    httplib2.debuglevel = debug
312    if trace_token:
313      sys.stderr.write(TRACE_WARNING)
314    if debug > 1:
315      sys.stderr.write(DEBUG_WARNING)
316    if debug >= 2:
317      _ConfigureLogging(level=logging.DEBUG)
318      command_runner.RunNamedCommand('ver', ['-l'])
319      config_items = []
320      try:
321        config_items.extend(boto.config.items('Boto'))
322        config_items.extend(boto.config.items('GSUtil'))
323      except ConfigParser.NoSectionError:
324        pass
325      for i in xrange(len(config_items)):
326        config_item_key = config_items[i][0]
327        if config_item_key in CONFIG_KEYS_TO_REDACT:
328          config_items[i] = (config_item_key, 'REDACTED')
329      sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
330      sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList())
331      sys.stderr.write('config: %s\n' % str(config_items))
332    elif quiet:
333      _ConfigureLogging(level=logging.WARNING)
334    else:
335      _ConfigureLogging(level=logging.INFO)
336      # oauth2client uses info logging in places that would better
337      # correspond to gsutil's debug logging (e.g., when refreshing
338      # access tokens).
339      oauth2client.client.logger.setLevel(logging.WARNING)
340
341    if not CERTIFICATE_VALIDATION_ENABLED:
342      sys.stderr.write(HTTP_WARNING)
343
344    if version:
345      command_name = 'version'
346    elif not args:
347      command_name = 'help'
348    else:
349      command_name = args[0]
350
351    _CheckAndWarnForProxyDifferences()
352
353    if os.environ.get('_ARGCOMPLETE', '0') == '1':
354      return _PerformTabCompletion(command_runner)
355
356    return _RunNamedCommandAndHandleExceptions(
357        command_runner, command_name, args=args[1:], headers=headers,
358        debug_level=debug, trace_token=trace_token,
359        parallel_operations=parallel_operations)
360  finally:
361    _Cleanup()
362
363
364def _CheckAndWarnForProxyDifferences():
365  # If there are both boto config and environment variable config present for
366  # proxies, unset the environment variable and warn if it differs.
367  boto_port = boto.config.getint('Boto', 'proxy_port', 0)
368  if boto.config.get('Boto', 'proxy', None) or boto_port:
369    for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
370      if proxy_env_var in os.environ and os.environ[proxy_env_var]:
371        differing_values = []
372        proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
373        if proxy_info.proxy_host != boto.config.get('Boto', 'proxy', None):
374          differing_values.append(
375              'Boto proxy host: "%s" differs from %s proxy host: "%s"' %
376              (boto.config.get('Boto', 'proxy', None), proxy_env_var,
377               proxy_info.proxy_host))
378        if (proxy_info.proxy_user !=
379            boto.config.get('Boto', 'proxy_user', None)):
380          differing_values.append(
381              'Boto proxy user: "%s" differs from %s proxy user: "%s"' %
382              (boto.config.get('Boto', 'proxy_user', None), proxy_env_var,
383               proxy_info.proxy_user))
384        if (proxy_info.proxy_pass !=
385            boto.config.get('Boto', 'proxy_pass', None)):
386          differing_values.append(
387              'Boto proxy password differs from %s proxy password' %
388              proxy_env_var)
389        # Only compare ports if at least one is present, since the
390        # boto logic for selecting default ports has not yet executed.
391        if ((proxy_info.proxy_port or boto_port) and
392            proxy_info.proxy_port != boto_port):
393          differing_values.append(
394              'Boto proxy port: "%s" differs from %s proxy port: "%s"' %
395              (boto_port, proxy_env_var, proxy_info.proxy_port))
396        if differing_values:
397          sys.stderr.write('\n'.join(textwrap.wrap(
398              'WARNING: Proxy configuration is present in both the %s '
399              'environment variable and boto configuration, but '
400              'configuration differs. boto configuration proxy values will '
401              'be used. Differences detected:' % proxy_env_var)))
402          sys.stderr.write('\n%s\n' % '\n'.join(differing_values))
403        # Regardless of whether the proxy configuration values matched,
404        # delete the environment variable so as not to confuse boto.
405        del os.environ[proxy_env_var]
406
407
408def _HandleUnknownFailure(e):
409  # Called if we fall through all known/handled exceptions.
410  _OutputAndExit('Failure: %s.' % e)
411
412
413def _HandleCommandException(e):
414  if e.informational:
415    _OutputAndExit(e.reason)
416  else:
417    _OutputAndExit('CommandException: %s' % e.reason)
418
419
420# pylint: disable=unused-argument
421def _HandleControlC(signal_num, cur_stack_frame):
422  """Called when user hits ^C.
423
424  This function prints a brief message instead of the normal Python stack trace
425  (unless -D option is used).
426
427  Args:
428    signal_num: Signal that was caught.
429    cur_stack_frame: Unused.
430  """
431  if debug >= 2:
432    stack_trace = ''.join(traceback.format_list(traceback.extract_stack()))
433    _OutputAndExit(
434        'DEBUG: Caught signal %d - Exception stack trace:\n'
435        '    %s' % (signal_num, re.sub('\\n', '\n    ', stack_trace)))
436  else:
437    _OutputAndExit('Caught signal %d - exiting' % signal_num)
438
439
440def _HandleSigQuit(signal_num, cur_stack_frame):
441  """Called when user hits ^\\, so we can force breakpoint a running gsutil."""
442  import pdb  # pylint: disable=g-import-not-at-top
443  pdb.set_trace()
444
445
446def _ConstructAccountProblemHelp(reason):
447  """Constructs a help string for an access control error.
448
449  Args:
450    reason: e.reason string from caught exception.
451
452  Returns:
453    Contructed help text.
454  """
455  default_project_id = boto.config.get_value('GSUtil', 'default_project_id')
456  # pylint: disable=line-too-long, g-inconsistent-quotes
457  acct_help = (
458      "Your request resulted in an AccountProblem (403) error. Usually this "
459      "happens if you attempt to create a bucket without first having "
460      "enabled billing for the project you are using. Please ensure billing is "
461      "enabled for your project by following the instructions at "
462      "`Google Developers Console<https://developers.google.com/console/help/billing>`. ")
463  if default_project_id:
464    acct_help += (
465        "In the project overview, ensure that the Project Number listed for "
466        "your project matches the project ID (%s) from your boto config file. "
467        % default_project_id)
468  acct_help += (
469      "If the above doesn't resolve your AccountProblem, please send mail to "
470      "gs-team@google.com requesting assistance, noting the exact command you "
471      "ran, the fact that you received a 403 AccountProblem error, and your "
472      "project ID. Please do not post your project ID on StackOverflow. "
473      "Note: It's possible to use Google Cloud Storage without enabling "
474      "billing if you're only listing or reading objects for which you're "
475      "authorized, or if you're uploading objects to a bucket billed to a "
476      "project that has billing enabled. But if you're attempting to create "
477      "buckets or upload objects to a bucket owned by your own project, you "
478      "must first enable billing for that project.")
479  return acct_help
480
481
482def _CheckAndHandleCredentialException(e, args):
483  # Provide detail to users who have no boto config file (who might previously
484  # have been using gsutil only for accessing publicly readable buckets and
485  # objects).
486  # pylint: disable=g-import-not-at-top
487  from gslib.util import HasConfiguredCredentials
488  if (not HasConfiguredCredentials() and
489      not boto.config.get_value('Tests', 'bypass_anonymous_access_warning',
490                                False)):
491    # The check above allows tests to assert that we get a particular,
492    # expected failure, rather than always encountering this error message
493    # when there are no configured credentials. This allows tests to
494    # simulate a second user without permissions, without actually requiring
495    # two separate configured users.
496    if os.environ.get('CLOUDSDK_WRAPPER') == '1':
497      _OutputAndExit('\n'.join(textwrap.wrap(
498          'You are attempting to access protected data with no configured '
499          'credentials. Please visit '
500          'https://cloud.google.com/console#/project and sign up for an '
501          'account, and then run the "gcloud auth login" command to '
502          'configure gsutil to use these credentials.')))
503    else:
504      _OutputAndExit('\n'.join(textwrap.wrap(
505          'You are attempting to access protected data with no configured '
506          'credentials. Please visit '
507          'https://cloud.google.com/console#/project and sign up for an '
508          'account, and then run the "gsutil config" command to configure '
509          'gsutil to use these credentials.')))
510  elif (e.reason and
511        (e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or
512         'account for the specified project has been disabled' in e.reason)
513        and ','.join(args).find('gs://') != -1):
514    _OutputAndExit('\n'.join(textwrap.wrap(
515        _ConstructAccountProblemHelp(e.reason))))
516
517
518def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
519                                        headers=None, debug_level=0,
520                                        trace_token=None,
521                                        parallel_operations=False):
522  """Runs the command with the given command runner and arguments."""
523  # pylint: disable=g-import-not-at-top
524  from gslib.util import GetConfigFilePath
525  from gslib.util import IS_WINDOWS
526  from gslib.util import IsRunningInteractively
527  try:
528    # Catch ^C so we can print a brief message instead of the normal Python
529    # stack trace. Register as a final signal handler because this handler kills
530    # the main gsutil process (so it must run last).
531    RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True)
532    # Catch ^\ so we can force a breakpoint in a running gsutil.
533    if not IS_WINDOWS:
534      RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit)
535    return command_runner.RunNamedCommand(command_name, args, headers,
536                                          debug_level, trace_token,
537                                          parallel_operations)
538  except AttributeError as e:
539    if str(e).find('secret_access_key') != -1:
540      _OutputAndExit('Missing credentials for the given URI(s). Does your '
541                     'boto config file contain all needed credentials?')
542    else:
543      _OutputAndExit(str(e))
544  except gslib.exception.CommandException as e:
545    _HandleCommandException(e)
546  except getopt.GetoptError as e:
547    _HandleCommandException(gslib.exception.CommandException(e.msg))
548  except boto.exception.InvalidUriError as e:
549    _OutputAndExit('InvalidUriError: %s.' % e.message)
550  except gslib.exception.InvalidUrlError as e:
551    _OutputAndExit('InvalidUrlError: %s.' % e.message)
552  except boto.auth_handler.NotReadyToAuthenticate:
553    _OutputAndExit('NotReadyToAuthenticate')
554  except OSError as e:
555    _OutputAndExit('OSError: %s.' % e.strerror)
556  except IOError as e:
557    if (e.errno == errno.EPIPE or (IS_WINDOWS and e.errno == errno.EINVAL)
558        and not IsRunningInteractively()):
559      # If we get a pipe error, this just means that the pipe to stdout or
560      # stderr is broken. This can happen if the user pipes gsutil to a command
561      # that doesn't use the entire output stream. Instead of raising an error,
562      # just swallow it up and exit cleanly.
563      sys.exit(0)
564    else:
565      raise
566  except wildcard_iterator.WildcardException as e:
567    _OutputAndExit(e.reason)
568  except ProjectIdException as e:
569    _OutputAndExit(
570        'You are attempting to perform an operation that requires a '
571        'project id, with none configured. Please re-run '
572        'gsutil config and make sure to follow the instructions for '
573        'finding and entering your default project id.')
574  except BadRequestException as e:
575    if e.reason == 'MissingSecurityHeader':
576      _CheckAndHandleCredentialException(e, args)
577    _OutputAndExit(e)
578  except AccessDeniedException as e:
579    _CheckAndHandleCredentialException(e, args)
580    _OutputAndExit(e)
581  except ArgumentException as e:
582    _OutputAndExit(e)
583  except ServiceException as e:
584    _OutputAndExit(e)
585  except apitools_exceptions.HttpError as e:
586    # These should usually be retried by the underlying implementation or
587    # wrapped by CloudApi ServiceExceptions, but if we do get them,
588    # print something useful.
589    _OutputAndExit('HttpError: %s, %s' % (getattr(e.response, 'status', ''),
590                                          e.content or ''))
591  except socket.error as e:
592    if e.args[0] == errno.EPIPE:
593      # Retrying with a smaller file (per suggestion below) works because
594      # the library code send loop (in boto/s3/key.py) can get through the
595      # entire file and then request the HTTP response before the socket
596      # gets closed and the response lost.
597      _OutputAndExit(
598          'Got a "Broken pipe" error. This can happen to clients using Python '
599          '2.x, when the server sends an error response and then closes the '
600          'socket (see http://bugs.python.org/issue5542). If you are trying to '
601          'upload a large object you might retry with a small (say 200k) '
602          'object, and see if you get a more specific error code.'
603      )
604    else:
605      _HandleUnknownFailure(e)
606  except Exception as e:
607    # Check for two types of errors related to service accounts. These errors
608    # appear to be the same except for their messages, but they are caused by
609    # different problems and both have unhelpful error messages. Moreover,
610    # the error type belongs to PyOpenSSL, which is not necessarily installed.
611    if 'mac verify failure' in str(e):
612      _OutputAndExit(
613          'Encountered an error while refreshing access token. '
614          'If you are using a service account,\nplease verify that the '
615          'gs_service_key_file_password field in your config file,'
616          '\n%s, is correct.' % GetConfigFilePath())
617    elif 'asn1 encoding routines' in str(e):
618      _OutputAndExit(
619          'Encountered an error while refreshing access token. '
620          'If you are using a service account,\nplease verify that the '
621          'gs_service_key_file field in your config file,\n%s, is correct.'
622          % GetConfigFilePath())
623    _HandleUnknownFailure(e)
624
625
626def _PerformTabCompletion(command_runner):
627  """Performs gsutil-specific tab completion for the shell."""
628  # argparse and argcomplete are bundled with the Google Cloud SDK.
629  # When gsutil is invoked from the Google Cloud SDK, both should be available.
630  try:
631    import argcomplete
632    import argparse
633  except ImportError as e:
634    _OutputAndExit('A library required for performing tab completion was'
635                   ' not found.\nCause: %s' % e)
636  parser = argparse.ArgumentParser(add_help=False)
637  subparsers = parser.add_subparsers()
638  command_runner.ConfigureCommandArgumentParsers(subparsers)
639  argcomplete.autocomplete(parser, exit_method=sys.exit)
640
641  return 0
642
643if __name__ == '__main__':
644  sys.exit(main())
645