1#!/usr/bin/env python
2# Copyright 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Script that attempts to push to a special git repository to verify that git
7credentials are configured correctly. It also verifies that gclient solution is
8configured to use git checkout.
9
10It will be added as gclient hook shortly before Chromium switches to git and
11removed after the switch.
12
13When running as hook in *.corp.google.com network it will also report status
14of the push attempt to the server (on appengine), so that chrome-infra team can
15collect information about misconfigured Git accounts.
16"""
17
18import contextlib
19import datetime
20import errno
21import getpass
22import json
23import logging
24import netrc
25import optparse
26import os
27import pprint
28import shutil
29import socket
30import ssl
31import subprocess
32import sys
33import tempfile
34import time
35import urllib2
36import urlparse
37
38
39# Absolute path to src/ directory.
40REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
41
42# Absolute path to a file with gclient solutions.
43GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient')
44
45# Incremented whenever some changes to scrip logic are made. Change in version
46# will cause the check to be rerun on next gclient runhooks invocation.
47CHECKER_VERSION = 1
48
49# Do not attempt to upload a report after this date.
50UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1)
51
52# URL to POST json with results to.
53MOTHERSHIP_URL = (
54    'https://chromium-git-access.appspot.com/'
55    'git_access/api/v1/reports/access_check')
56
57# Repository to push test commits to.
58TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test'
59
60# Git-compatible gclient solution.
61GOOD_GCLIENT_SOLUTION = {
62  'name': 'src',
63  'deps_file': 'DEPS',
64  'managed': False,
65  'url': 'https://chromium.googlesource.com/chromium/src.git',
66}
67
68# Possible chunks of git push response in case .netrc is misconfigured.
69BAD_ACL_ERRORS = (
70  '(prohibited by Gerrit)',
71  'does not match your user account',
72  'Git repository not found',
73  'Invalid user name or password',
74  'Please make sure you have the correct access rights',
75)
76
77# Git executable to call.
78GIT_EXE = 'git.bat' if sys.platform == 'win32' else 'git'
79
80
81def is_on_bot():
82  """True when running under buildbot."""
83  return os.environ.get('CHROME_HEADLESS') == '1'
84
85
86def is_in_google_corp():
87  """True when running in google corp network."""
88  try:
89    return socket.getfqdn().endswith('.corp.google.com')
90  except socket.error:
91    logging.exception('Failed to get FQDN')
92    return False
93
94
95def is_using_git():
96  """True if git checkout is used."""
97  return os.path.exists(os.path.join(REPO_ROOT, '.git', 'objects'))
98
99
100def is_using_svn():
101  """True if svn checkout is used."""
102  return os.path.exists(os.path.join(REPO_ROOT, '.svn'))
103
104
105def read_git_config(prop):
106  """Reads git config property of src.git repo.
107
108  Returns empty string in case of errors.
109  """
110  try:
111    proc = subprocess.Popen(
112        [GIT_EXE, 'config', prop], stdout=subprocess.PIPE, cwd=REPO_ROOT)
113    out, _ = proc.communicate()
114    return out.strip()
115  except OSError as exc:
116    if exc.errno != errno.ENOENT:
117      logging.exception('Unexpected error when calling git')
118    return ''
119
120
121def read_netrc_user(netrc_obj, host):
122  """Reads 'user' field of a host entry in netrc.
123
124  Returns empty string if netrc is missing, or host is not there.
125  """
126  if not netrc_obj:
127    return ''
128  entry = netrc_obj.authenticators(host)
129  if not entry:
130    return ''
131  return entry[0]
132
133
134def get_git_version():
135  """Returns version of git or None if git is not available."""
136  try:
137    proc = subprocess.Popen([GIT_EXE, '--version'], stdout=subprocess.PIPE)
138    out, _ = proc.communicate()
139    return out.strip() if proc.returncode == 0 else ''
140  except OSError as exc:
141    if exc.errno != errno.ENOENT:
142      logging.exception('Unexpected error when calling git')
143    return ''
144
145
146def read_gclient_solution():
147  """Read information about 'src' gclient solution from .gclient file.
148
149  Returns tuple:
150    (url, deps_file, managed)
151    or
152    (None, None, None) if no such solution.
153  """
154  try:
155    env = {}
156    execfile(GCLIENT_CONFIG, env, env)
157    for sol in (env.get('solutions') or []):
158      if sol.get('name') == 'src':
159        return sol.get('url'), sol.get('deps_file'), sol.get('managed')
160    return None, None, None
161  except Exception:
162    logging.exception('Failed to read .gclient solution')
163    return None, None, None
164
165
166def read_git_insteadof(host):
167  """Reads relevant insteadOf config entries."""
168  try:
169    proc = subprocess.Popen([GIT_EXE, 'config', '-l'], stdout=subprocess.PIPE)
170    out, _ = proc.communicate()
171    lines = []
172    for line in out.strip().split('\n'):
173      line = line.lower()
174      if 'insteadof=' in line and host in line:
175        lines.append(line)
176    return '\n'.join(lines)
177  except OSError as exc:
178    if exc.errno != errno.ENOENT:
179      logging.exception('Unexpected error when calling git')
180    return ''
181
182
183def scan_configuration():
184  """Scans local environment for git related configuration values."""
185  # Git checkout?
186  is_git = is_using_git()
187
188  # On Windows HOME should be set.
189  if 'HOME' in os.environ:
190    netrc_path = os.path.join(
191        os.environ['HOME'],
192        '_netrc' if sys.platform.startswith('win') else '.netrc')
193  else:
194    netrc_path = None
195
196  # Netrc exists?
197  is_using_netrc = netrc_path and os.path.exists(netrc_path)
198
199  # Read it.
200  netrc_obj = None
201  if is_using_netrc:
202    try:
203      netrc_obj = netrc.netrc(netrc_path)
204    except Exception:
205      logging.exception('Failed to read netrc from %s', netrc_path)
206      netrc_obj = None
207
208  # Read gclient 'src' solution.
209  gclient_url, gclient_deps, gclient_managed = read_gclient_solution()
210
211  return {
212    'checker_version': CHECKER_VERSION,
213    'is_git': is_git,
214    'is_home_set': 'HOME' in os.environ,
215    'is_using_netrc': is_using_netrc,
216    'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0,
217    'git_version': get_git_version(),
218    'platform': sys.platform,
219    'username': getpass.getuser(),
220    'git_user_email': read_git_config('user.email') if is_git else '',
221    'git_user_name': read_git_config('user.name') if is_git else '',
222    'git_insteadof': read_git_insteadof('chromium.googlesource.com'),
223    'chromium_netrc_email':
224        read_netrc_user(netrc_obj, 'chromium.googlesource.com'),
225    'chrome_internal_netrc_email':
226        read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'),
227    'gclient_deps': gclient_deps,
228    'gclient_managed': gclient_managed,
229    'gclient_url': gclient_url,
230  }
231
232
233def last_configuration_path():
234  """Path to store last checked configuration."""
235  if is_using_git():
236    return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json')
237  elif is_using_svn():
238    return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json')
239  else:
240    return os.path.join(REPO_ROOT, '.check_git_push_access_conf.json')
241
242
243def read_last_configuration():
244  """Reads last checked configuration if it exists."""
245  try:
246    with open(last_configuration_path(), 'r') as f:
247      return json.load(f)
248  except (IOError, ValueError):
249    return None
250
251
252def write_last_configuration(conf):
253  """Writes last checked configuration to a file."""
254  try:
255    with open(last_configuration_path(), 'w') as f:
256      json.dump(conf, f, indent=2, sort_keys=True)
257  except IOError:
258    logging.exception('Failed to write JSON to %s', path)
259
260
261@contextlib.contextmanager
262def temp_directory():
263  """Creates a temp directory, then nukes it."""
264  tmp = tempfile.mkdtemp()
265  try:
266    yield tmp
267  finally:
268    try:
269      shutil.rmtree(tmp)
270    except (OSError, IOError):
271      logging.exception('Failed to remove temp directory %s', tmp)
272
273
274class Runner(object):
275  """Runs a bunch of commands in some directory, collects logs from them."""
276
277  def __init__(self, cwd, verbose):
278    self.cwd = cwd
279    self.verbose = verbose
280    self.log = []
281
282  def run(self, cmd):
283    self.append_to_log('> ' + ' '.join(cmd))
284    retcode = -1
285    try:
286      proc = subprocess.Popen(
287          cmd,
288          stdout=subprocess.PIPE,
289          stderr=subprocess.STDOUT,
290          cwd=self.cwd)
291      out, _ = proc.communicate()
292      out = out.strip()
293      retcode = proc.returncode
294    except OSError as exc:
295      out = str(exc)
296    if retcode:
297      out += '\n(exit code: %d)' % retcode
298    self.append_to_log(out)
299    return retcode
300
301  def append_to_log(self, text):
302    if text:
303      self.log.append(text)
304      if self.verbose:
305        logging.warning(text)
306
307
308def check_git_config(conf, report_url, verbose):
309  """Attempts to push to a git repository, reports results to a server.
310
311  Returns True if the check finished without incidents (push itself may
312  have failed) and should NOT be retried on next invocation of the hook.
313  """
314  # Don't even try to push if netrc is not configured.
315  if not conf['chromium_netrc_email']:
316    return upload_report(
317        conf,
318        report_url,
319        verbose,
320        push_works=False,
321        push_log='',
322        push_duration_ms=0)
323
324  # Ref to push to, each user has its own ref.
325  ref = 'refs/push-test/%s' % conf['chromium_netrc_email']
326
327  push_works = False
328  flake = False
329  started = time.time()
330  try:
331    logging.warning('Checking push access to the git repository...')
332    with temp_directory() as tmp:
333      # Prepare a simple commit on a new timeline.
334      runner = Runner(tmp, verbose)
335      runner.run([GIT_EXE, 'init', '.'])
336      if conf['git_user_name']:
337        runner.run([GIT_EXE, 'config', 'user.name', conf['git_user_name']])
338      if conf['git_user_email']:
339        runner.run([GIT_EXE, 'config', 'user.email', conf['git_user_email']])
340      with open(os.path.join(tmp, 'timestamp'), 'w') as f:
341        f.write(str(int(time.time() * 1000)))
342      runner.run([GIT_EXE, 'add', 'timestamp'])
343      runner.run([GIT_EXE, 'commit', '-m', 'Push test.'])
344      # Try to push multiple times if it fails due to issues other than ACLs.
345      attempt = 0
346      while attempt < 5:
347        attempt += 1
348        logging.info('Pushing to %s %s', TEST_REPO_URL, ref)
349        ret = runner.run(
350            [GIT_EXE, 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f'])
351        if not ret:
352          push_works = True
353          break
354        if any(x in runner.log[-1] for x in BAD_ACL_ERRORS):
355          push_works = False
356          break
357  except Exception:
358    logging.exception('Unexpected exception when pushing')
359    flake = True
360
361  if push_works:
362    logging.warning('Git push works!')
363  else:
364    logging.warning(
365        'Git push doesn\'t work, which is fine if you are not a committer.')
366
367  uploaded = upload_report(
368      conf,
369      report_url,
370      verbose,
371      push_works=push_works,
372      push_log='\n'.join(runner.log),
373      push_duration_ms=int((time.time() - started) * 1000))
374  return uploaded and not flake
375
376
377def check_gclient_config(conf):
378  """Shows warning if gclient solution is not properly configured for git."""
379  # Ignore configs that do not have 'src' solution at all.
380  if not conf['gclient_url']:
381    return
382  current = {
383    'name': 'src',
384    'deps_file': conf['gclient_deps'] or 'DEPS',
385    'managed': conf['gclient_managed'] or False,
386    'url': conf['gclient_url'],
387  }
388  # After depot_tools r291592 both DEPS and .DEPS.git are valid.
389  good = GOOD_GCLIENT_SOLUTION.copy()
390  good['deps_file'] = current['deps_file']
391  if current == good:
392    return
393  # Show big warning if url or deps_file is wrong.
394  if current['url'] != good['url'] or current['deps_file'] != good['deps_file']:
395    print '-' * 80
396    print 'Your gclient solution is not set to use supported git workflow!'
397    print
398    print 'Your \'src\' solution (in %s):' % GCLIENT_CONFIG
399    print pprint.pformat(current, indent=2)
400    print
401    print 'Correct \'src\' solution to use git:'
402    print pprint.pformat(good, indent=2)
403    print
404    print 'Please update your .gclient file ASAP.'
405    print '-' * 80
406  # Show smaller (additional) warning about managed workflow.
407  if current['managed']:
408    print '-' * 80
409    print (
410        'You are using managed gclient mode with git, which was deprecated '
411        'on 8/22/13:')
412    print (
413        'https://groups.google.com/a/chromium.org/'
414        'forum/#!topic/chromium-dev/n9N5N3JL2_U')
415    print
416    print (
417        'It is strongly advised to switch to unmanaged mode. For more '
418        'information about managed mode and reasons for its deprecation see:')
419    print 'http://www.chromium.org/developers/how-tos/get-the-code#Managed_mode'
420    print
421    print (
422        'There\'s also a large suite of tools to assist managing git '
423        'checkouts.\nSee \'man depot_tools\' (or read '
424        'depot_tools/man/html/depot_tools.html).')
425    print '-' * 80
426
427
428def upload_report(
429    conf, report_url, verbose, push_works, push_log, push_duration_ms):
430  """Posts report to the server, returns True if server accepted it.
431
432  Uploads the report only if script is running in Google corp network. Otherwise
433  just prints the report.
434  """
435  report = conf.copy()
436  report.update(
437      push_works=push_works,
438      push_log=push_log,
439      push_duration_ms=push_duration_ms)
440
441  as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True)
442  if verbose:
443    print 'Status of git push attempt:'
444    print as_bytes
445
446  # Do not upload it outside of corp or if server side is already disabled.
447  if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS:
448    if verbose:
449      print (
450          'You can send the above report to chrome-git-migration@google.com '
451          'if you need help to set up you committer git account.')
452    return True
453
454  req = urllib2.Request(
455      url=report_url,
456      data=as_bytes,
457      headers={'Content-Type': 'application/json; charset=utf-8'})
458
459  attempt = 0
460  success = False
461  while not success and attempt < 10:
462    attempt += 1
463    try:
464      logging.warning(
465          'Attempting to upload the report to %s...',
466          urlparse.urlparse(report_url).netloc)
467      resp = urllib2.urlopen(req, timeout=5)
468      report_id = None
469      try:
470        report_id = json.load(resp)['report_id']
471      except (ValueError, TypeError, KeyError):
472        pass
473      logging.warning('Report uploaded: %s', report_id)
474      success = True
475    except (urllib2.URLError, socket.error, ssl.SSLError) as exc:
476      logging.warning('Failed to upload the report: %s', exc)
477  return success
478
479
480def main(args):
481  parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
482  parser.add_option(
483      '--running-as-hook',
484      action='store_true',
485      help='Set when invoked from gclient hook')
486  parser.add_option(
487      '--report-url',
488      default=MOTHERSHIP_URL,
489      help='URL to submit the report to')
490  parser.add_option(
491      '--verbose',
492      action='store_true',
493      help='More logging')
494  options, args = parser.parse_args()
495  if args:
496    parser.error('Unknown argument %s' % args)
497  logging.basicConfig(
498      format='%(message)s',
499      level=logging.INFO if options.verbose else logging.WARN)
500
501  # When invoked not as a hook, always run the check.
502  if not options.running_as_hook:
503    config = scan_configuration()
504    check_gclient_config(config)
505    check_git_config(config, options.report_url, True)
506    return 0
507
508  # Always do nothing on bots.
509  if is_on_bot():
510    return 0
511
512  # Read current config, verify gclient solution looks correct.
513  config = scan_configuration()
514  check_gclient_config(config)
515
516  # Do not attempt to push from non-google owned machines.
517  if not is_in_google_corp():
518    logging.info('Skipping git push check: non *.corp.google.com machine.')
519    return 0
520
521  # Skip git push check if current configuration was already checked.
522  if config == read_last_configuration():
523    logging.info('Check already performed, skipping.')
524    return 0
525
526  # Run the check. Mark configuration as checked only on success. Ignore any
527  # exceptions or errors. This check must not break gclient runhooks.
528  try:
529    ok = check_git_config(config, options.report_url, False)
530    if ok:
531      write_last_configuration(config)
532    else:
533      logging.warning('Check failed and will be retried on the next run')
534  except Exception:
535    logging.exception('Unexpected exception when performing git access check')
536  return 0
537
538
539if __name__ == '__main__':
540  sys.exit(main(sys.argv[1:]))
541