1#!/usr/bin/env python 2# Copyright 2014 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Script that attempts to push to a special git repository to verify that git 7credentials are configured correctly. It also verifies that gclient solution is 8configured to use git checkout. 9 10It will be added as gclient hook shortly before Chromium switches to git and 11removed after the switch. 12 13When running as hook in *.corp.google.com network it will also report status 14of the push attempt to the server (on appengine), so that chrome-infra team can 15collect information about misconfigured Git accounts. 16""" 17 18import contextlib 19import datetime 20import errno 21import getpass 22import json 23import logging 24import netrc 25import optparse 26import os 27import pprint 28import shutil 29import socket 30import ssl 31import subprocess 32import sys 33import tempfile 34import time 35import urllib2 36import urlparse 37 38 39# Absolute path to src/ directory. 40REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 41 42# Absolute path to a file with gclient solutions. 43GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient') 44 45# Incremented whenever some changes to scrip logic are made. Change in version 46# will cause the check to be rerun on next gclient runhooks invocation. 47CHECKER_VERSION = 1 48 49# Do not attempt to upload a report after this date. 50UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1) 51 52# URL to POST json with results to. 53MOTHERSHIP_URL = ( 54 'https://chromium-git-access.appspot.com/' 55 'git_access/api/v1/reports/access_check') 56 57# Repository to push test commits to. 58TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test' 59 60# Git-compatible gclient solution. 61GOOD_GCLIENT_SOLUTION = { 62 'name': 'src', 63 'deps_file': 'DEPS', 64 'managed': False, 65 'url': 'https://chromium.googlesource.com/chromium/src.git', 66} 67 68# Possible chunks of git push response in case .netrc is misconfigured. 69BAD_ACL_ERRORS = ( 70 '(prohibited by Gerrit)', 71 'does not match your user account', 72 'Git repository not found', 73 'Invalid user name or password', 74 'Please make sure you have the correct access rights', 75) 76 77# Git executable to call. 78GIT_EXE = 'git.bat' if sys.platform == 'win32' else 'git' 79 80 81def is_on_bot(): 82 """True when running under buildbot.""" 83 return os.environ.get('CHROME_HEADLESS') == '1' 84 85 86def is_in_google_corp(): 87 """True when running in google corp network.""" 88 try: 89 return socket.getfqdn().endswith('.corp.google.com') 90 except socket.error: 91 logging.exception('Failed to get FQDN') 92 return False 93 94 95def is_using_git(): 96 """True if git checkout is used.""" 97 return os.path.exists(os.path.join(REPO_ROOT, '.git', 'objects')) 98 99 100def is_using_svn(): 101 """True if svn checkout is used.""" 102 return os.path.exists(os.path.join(REPO_ROOT, '.svn')) 103 104 105def read_git_config(prop): 106 """Reads git config property of src.git repo. 107 108 Returns empty string in case of errors. 109 """ 110 try: 111 proc = subprocess.Popen( 112 [GIT_EXE, 'config', prop], stdout=subprocess.PIPE, cwd=REPO_ROOT) 113 out, _ = proc.communicate() 114 return out.strip() 115 except OSError as exc: 116 if exc.errno != errno.ENOENT: 117 logging.exception('Unexpected error when calling git') 118 return '' 119 120 121def read_netrc_user(netrc_obj, host): 122 """Reads 'user' field of a host entry in netrc. 123 124 Returns empty string if netrc is missing, or host is not there. 125 """ 126 if not netrc_obj: 127 return '' 128 entry = netrc_obj.authenticators(host) 129 if not entry: 130 return '' 131 return entry[0] 132 133 134def get_git_version(): 135 """Returns version of git or None if git is not available.""" 136 try: 137 proc = subprocess.Popen([GIT_EXE, '--version'], stdout=subprocess.PIPE) 138 out, _ = proc.communicate() 139 return out.strip() if proc.returncode == 0 else '' 140 except OSError as exc: 141 if exc.errno != errno.ENOENT: 142 logging.exception('Unexpected error when calling git') 143 return '' 144 145 146def read_gclient_solution(): 147 """Read information about 'src' gclient solution from .gclient file. 148 149 Returns tuple: 150 (url, deps_file, managed) 151 or 152 (None, None, None) if no such solution. 153 """ 154 try: 155 env = {} 156 execfile(GCLIENT_CONFIG, env, env) 157 for sol in (env.get('solutions') or []): 158 if sol.get('name') == 'src': 159 return sol.get('url'), sol.get('deps_file'), sol.get('managed') 160 return None, None, None 161 except Exception: 162 logging.exception('Failed to read .gclient solution') 163 return None, None, None 164 165 166def read_git_insteadof(host): 167 """Reads relevant insteadOf config entries.""" 168 try: 169 proc = subprocess.Popen([GIT_EXE, 'config', '-l'], stdout=subprocess.PIPE) 170 out, _ = proc.communicate() 171 lines = [] 172 for line in out.strip().split('\n'): 173 line = line.lower() 174 if 'insteadof=' in line and host in line: 175 lines.append(line) 176 return '\n'.join(lines) 177 except OSError as exc: 178 if exc.errno != errno.ENOENT: 179 logging.exception('Unexpected error when calling git') 180 return '' 181 182 183def scan_configuration(): 184 """Scans local environment for git related configuration values.""" 185 # Git checkout? 186 is_git = is_using_git() 187 188 # On Windows HOME should be set. 189 if 'HOME' in os.environ: 190 netrc_path = os.path.join( 191 os.environ['HOME'], 192 '_netrc' if sys.platform.startswith('win') else '.netrc') 193 else: 194 netrc_path = None 195 196 # Netrc exists? 197 is_using_netrc = netrc_path and os.path.exists(netrc_path) 198 199 # Read it. 200 netrc_obj = None 201 if is_using_netrc: 202 try: 203 netrc_obj = netrc.netrc(netrc_path) 204 except Exception: 205 logging.exception('Failed to read netrc from %s', netrc_path) 206 netrc_obj = None 207 208 # Read gclient 'src' solution. 209 gclient_url, gclient_deps, gclient_managed = read_gclient_solution() 210 211 return { 212 'checker_version': CHECKER_VERSION, 213 'is_git': is_git, 214 'is_home_set': 'HOME' in os.environ, 215 'is_using_netrc': is_using_netrc, 216 'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0, 217 'git_version': get_git_version(), 218 'platform': sys.platform, 219 'username': getpass.getuser(), 220 'git_user_email': read_git_config('user.email') if is_git else '', 221 'git_user_name': read_git_config('user.name') if is_git else '', 222 'git_insteadof': read_git_insteadof('chromium.googlesource.com'), 223 'chromium_netrc_email': 224 read_netrc_user(netrc_obj, 'chromium.googlesource.com'), 225 'chrome_internal_netrc_email': 226 read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'), 227 'gclient_deps': gclient_deps, 228 'gclient_managed': gclient_managed, 229 'gclient_url': gclient_url, 230 } 231 232 233def last_configuration_path(): 234 """Path to store last checked configuration.""" 235 if is_using_git(): 236 return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json') 237 elif is_using_svn(): 238 return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json') 239 else: 240 return os.path.join(REPO_ROOT, '.check_git_push_access_conf.json') 241 242 243def read_last_configuration(): 244 """Reads last checked configuration if it exists.""" 245 try: 246 with open(last_configuration_path(), 'r') as f: 247 return json.load(f) 248 except (IOError, ValueError): 249 return None 250 251 252def write_last_configuration(conf): 253 """Writes last checked configuration to a file.""" 254 try: 255 with open(last_configuration_path(), 'w') as f: 256 json.dump(conf, f, indent=2, sort_keys=True) 257 except IOError: 258 logging.exception('Failed to write JSON to %s', path) 259 260 261@contextlib.contextmanager 262def temp_directory(): 263 """Creates a temp directory, then nukes it.""" 264 tmp = tempfile.mkdtemp() 265 try: 266 yield tmp 267 finally: 268 try: 269 shutil.rmtree(tmp) 270 except (OSError, IOError): 271 logging.exception('Failed to remove temp directory %s', tmp) 272 273 274class Runner(object): 275 """Runs a bunch of commands in some directory, collects logs from them.""" 276 277 def __init__(self, cwd, verbose): 278 self.cwd = cwd 279 self.verbose = verbose 280 self.log = [] 281 282 def run(self, cmd): 283 self.append_to_log('> ' + ' '.join(cmd)) 284 retcode = -1 285 try: 286 proc = subprocess.Popen( 287 cmd, 288 stdout=subprocess.PIPE, 289 stderr=subprocess.STDOUT, 290 cwd=self.cwd) 291 out, _ = proc.communicate() 292 out = out.strip() 293 retcode = proc.returncode 294 except OSError as exc: 295 out = str(exc) 296 if retcode: 297 out += '\n(exit code: %d)' % retcode 298 self.append_to_log(out) 299 return retcode 300 301 def append_to_log(self, text): 302 if text: 303 self.log.append(text) 304 if self.verbose: 305 logging.warning(text) 306 307 308def check_git_config(conf, report_url, verbose): 309 """Attempts to push to a git repository, reports results to a server. 310 311 Returns True if the check finished without incidents (push itself may 312 have failed) and should NOT be retried on next invocation of the hook. 313 """ 314 # Don't even try to push if netrc is not configured. 315 if not conf['chromium_netrc_email']: 316 return upload_report( 317 conf, 318 report_url, 319 verbose, 320 push_works=False, 321 push_log='', 322 push_duration_ms=0) 323 324 # Ref to push to, each user has its own ref. 325 ref = 'refs/push-test/%s' % conf['chromium_netrc_email'] 326 327 push_works = False 328 flake = False 329 started = time.time() 330 try: 331 logging.warning('Checking push access to the git repository...') 332 with temp_directory() as tmp: 333 # Prepare a simple commit on a new timeline. 334 runner = Runner(tmp, verbose) 335 runner.run([GIT_EXE, 'init', '.']) 336 if conf['git_user_name']: 337 runner.run([GIT_EXE, 'config', 'user.name', conf['git_user_name']]) 338 if conf['git_user_email']: 339 runner.run([GIT_EXE, 'config', 'user.email', conf['git_user_email']]) 340 with open(os.path.join(tmp, 'timestamp'), 'w') as f: 341 f.write(str(int(time.time() * 1000))) 342 runner.run([GIT_EXE, 'add', 'timestamp']) 343 runner.run([GIT_EXE, 'commit', '-m', 'Push test.']) 344 # Try to push multiple times if it fails due to issues other than ACLs. 345 attempt = 0 346 while attempt < 5: 347 attempt += 1 348 logging.info('Pushing to %s %s', TEST_REPO_URL, ref) 349 ret = runner.run( 350 [GIT_EXE, 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f']) 351 if not ret: 352 push_works = True 353 break 354 if any(x in runner.log[-1] for x in BAD_ACL_ERRORS): 355 push_works = False 356 break 357 except Exception: 358 logging.exception('Unexpected exception when pushing') 359 flake = True 360 361 if push_works: 362 logging.warning('Git push works!') 363 else: 364 logging.warning( 365 'Git push doesn\'t work, which is fine if you are not a committer.') 366 367 uploaded = upload_report( 368 conf, 369 report_url, 370 verbose, 371 push_works=push_works, 372 push_log='\n'.join(runner.log), 373 push_duration_ms=int((time.time() - started) * 1000)) 374 return uploaded and not flake 375 376 377def check_gclient_config(conf): 378 """Shows warning if gclient solution is not properly configured for git.""" 379 # Ignore configs that do not have 'src' solution at all. 380 if not conf['gclient_url']: 381 return 382 current = { 383 'name': 'src', 384 'deps_file': conf['gclient_deps'] or 'DEPS', 385 'managed': conf['gclient_managed'] or False, 386 'url': conf['gclient_url'], 387 } 388 # After depot_tools r291592 both DEPS and .DEPS.git are valid. 389 good = GOOD_GCLIENT_SOLUTION.copy() 390 good['deps_file'] = current['deps_file'] 391 if current == good: 392 return 393 # Show big warning if url or deps_file is wrong. 394 if current['url'] != good['url'] or current['deps_file'] != good['deps_file']: 395 print '-' * 80 396 print 'Your gclient solution is not set to use supported git workflow!' 397 print 398 print 'Your \'src\' solution (in %s):' % GCLIENT_CONFIG 399 print pprint.pformat(current, indent=2) 400 print 401 print 'Correct \'src\' solution to use git:' 402 print pprint.pformat(good, indent=2) 403 print 404 print 'Please update your .gclient file ASAP.' 405 print '-' * 80 406 # Show smaller (additional) warning about managed workflow. 407 if current['managed']: 408 print '-' * 80 409 print ( 410 'You are using managed gclient mode with git, which was deprecated ' 411 'on 8/22/13:') 412 print ( 413 'https://groups.google.com/a/chromium.org/' 414 'forum/#!topic/chromium-dev/n9N5N3JL2_U') 415 print 416 print ( 417 'It is strongly advised to switch to unmanaged mode. For more ' 418 'information about managed mode and reasons for its deprecation see:') 419 print 'http://www.chromium.org/developers/how-tos/get-the-code#Managed_mode' 420 print 421 print ( 422 'There\'s also a large suite of tools to assist managing git ' 423 'checkouts.\nSee \'man depot_tools\' (or read ' 424 'depot_tools/man/html/depot_tools.html).') 425 print '-' * 80 426 427 428def upload_report( 429 conf, report_url, verbose, push_works, push_log, push_duration_ms): 430 """Posts report to the server, returns True if server accepted it. 431 432 Uploads the report only if script is running in Google corp network. Otherwise 433 just prints the report. 434 """ 435 report = conf.copy() 436 report.update( 437 push_works=push_works, 438 push_log=push_log, 439 push_duration_ms=push_duration_ms) 440 441 as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True) 442 if verbose: 443 print 'Status of git push attempt:' 444 print as_bytes 445 446 # Do not upload it outside of corp or if server side is already disabled. 447 if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS: 448 if verbose: 449 print ( 450 'You can send the above report to chrome-git-migration@google.com ' 451 'if you need help to set up you committer git account.') 452 return True 453 454 req = urllib2.Request( 455 url=report_url, 456 data=as_bytes, 457 headers={'Content-Type': 'application/json; charset=utf-8'}) 458 459 attempt = 0 460 success = False 461 while not success and attempt < 10: 462 attempt += 1 463 try: 464 logging.warning( 465 'Attempting to upload the report to %s...', 466 urlparse.urlparse(report_url).netloc) 467 resp = urllib2.urlopen(req, timeout=5) 468 report_id = None 469 try: 470 report_id = json.load(resp)['report_id'] 471 except (ValueError, TypeError, KeyError): 472 pass 473 logging.warning('Report uploaded: %s', report_id) 474 success = True 475 except (urllib2.URLError, socket.error, ssl.SSLError) as exc: 476 logging.warning('Failed to upload the report: %s', exc) 477 return success 478 479 480def main(args): 481 parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) 482 parser.add_option( 483 '--running-as-hook', 484 action='store_true', 485 help='Set when invoked from gclient hook') 486 parser.add_option( 487 '--report-url', 488 default=MOTHERSHIP_URL, 489 help='URL to submit the report to') 490 parser.add_option( 491 '--verbose', 492 action='store_true', 493 help='More logging') 494 options, args = parser.parse_args() 495 if args: 496 parser.error('Unknown argument %s' % args) 497 logging.basicConfig( 498 format='%(message)s', 499 level=logging.INFO if options.verbose else logging.WARN) 500 501 # When invoked not as a hook, always run the check. 502 if not options.running_as_hook: 503 config = scan_configuration() 504 check_gclient_config(config) 505 check_git_config(config, options.report_url, True) 506 return 0 507 508 # Always do nothing on bots. 509 if is_on_bot(): 510 return 0 511 512 # Read current config, verify gclient solution looks correct. 513 config = scan_configuration() 514 check_gclient_config(config) 515 516 # Do not attempt to push from non-google owned machines. 517 if not is_in_google_corp(): 518 logging.info('Skipping git push check: non *.corp.google.com machine.') 519 return 0 520 521 # Skip git push check if current configuration was already checked. 522 if config == read_last_configuration(): 523 logging.info('Check already performed, skipping.') 524 return 0 525 526 # Run the check. Mark configuration as checked only on success. Ignore any 527 # exceptions or errors. This check must not break gclient runhooks. 528 try: 529 ok = check_git_config(config, options.report_url, False) 530 if ok: 531 write_last_configuration(config) 532 else: 533 logging.warning('Check failed and will be retried on the next run') 534 except Exception: 535 logging.exception('Unexpected exception when performing git access check') 536 return 0 537 538 539if __name__ == '__main__': 540 sys.exit(main(sys.argv[1:])) 541