refresh_servlet.py revision 1320f92c476a1ad9d19dba2a48c72b75566198e9
1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import traceback 6 7from app_yaml_helper import AppYamlHelper 8from appengine_wrappers import IsDeadlineExceededError, logservice 9from branch_utility import BranchUtility 10from compiled_file_system import CompiledFileSystem 11from custom_logger import CustomLogger 12from data_source_registry import CreateDataSource 13from environment import GetAppVersion 14from file_system import IsFileSystemThrottledError 15from future import Future 16from gcs_file_system_provider import CloudStorageFileSystemProvider 17from github_file_system_provider import GithubFileSystemProvider 18from host_file_system_provider import HostFileSystemProvider 19from object_store_creator import ObjectStoreCreator 20from server_instance import ServerInstance 21from servlet import Servlet, Request, Response 22from timer import Timer, TimerClosure 23 24 25 26_log = CustomLogger('refresh') 27 28 29class RefreshServlet(Servlet): 30 '''Servlet which refreshes a single data source. 31 ''' 32 def __init__(self, request, delegate_for_test=None): 33 Servlet.__init__(self, request) 34 self._delegate = delegate_for_test or RefreshServlet.Delegate() 35 36 class Delegate(object): 37 '''RefreshServlet's runtime dependencies. Override for testing. 38 ''' 39 def CreateBranchUtility(self, object_store_creator): 40 return BranchUtility.Create(object_store_creator) 41 42 def CreateHostFileSystemProvider(self, 43 object_store_creator, 44 pinned_commit=None): 45 return HostFileSystemProvider(object_store_creator, 46 pinned_commit=pinned_commit) 47 48 def CreateGithubFileSystemProvider(self, object_store_creator): 49 return GithubFileSystemProvider(object_store_creator) 50 51 def CreateGCSFileSystemProvider(self, object_store_creator): 52 return CloudStorageFileSystemProvider(object_store_creator) 53 54 def GetAppVersion(self): 55 return GetAppVersion() 56 57 def Get(self): 58 # Manually flush logs at the end of the run. However, sometimes 59 # even that isn't enough, which is why in this file we use the 60 # custom logger and make it flush the log every time its used. 61 logservice.AUTOFLUSH_ENABLED = False 62 try: 63 return self._GetImpl() 64 except BaseException: 65 _log.error('Caught top-level exception! %s', traceback.format_exc()) 66 finally: 67 logservice.flush() 68 69 def _GetImpl(self): 70 path = self._request.path.strip('/') 71 parts = self._request.path.split('/', 1) 72 source_name = parts[0] 73 if len(parts) == 2: 74 source_path = parts[1] 75 else: 76 source_path = None 77 78 _log.info('starting refresh of %s DataSource %s' % 79 (source_name, '' if source_path is None else '[%s]' % source_path)) 80 81 if 'commit' in self._request.arguments: 82 commit = self._request.arguments['commit'] 83 else: 84 _log.warning('No commit given; refreshing from master. ' 85 'This is probably NOT what you want.') 86 commit = None 87 88 server_instance = self._CreateServerInstance(commit) 89 success = True 90 try: 91 if source_name == 'platform_bundle': 92 data_source = server_instance.platform_bundle 93 elif source_name == 'content_providers': 94 data_source = server_instance.content_providers 95 else: 96 data_source = CreateDataSource(source_name, server_instance) 97 98 class_name = data_source.__class__.__name__ 99 refresh_future = data_source.Refresh(source_path) 100 assert isinstance(refresh_future, Future), ( 101 '%s.Refresh() did not return a Future' % class_name) 102 timer = Timer() 103 try: 104 refresh_future.Get() 105 except Exception as e: 106 _log.error('%s: error %s' % (class_name, traceback.format_exc())) 107 success = False 108 if IsFileSystemThrottledError(e): 109 return Response.ThrottledError('Throttled') 110 raise 111 finally: 112 _log.info('Refreshing %s took %s' % 113 (class_name, timer.Stop().FormatElapsed())) 114 115 except: 116 success = False 117 # This should never actually happen. 118 _log.error('uncaught error: %s' % traceback.format_exc()) 119 raise 120 finally: 121 _log.info('finished (%s)', 'success' if success else 'FAILED') 122 return (Response.Ok('Success') if success else 123 Response.InternalError('Failure')) 124 125 def _CreateServerInstance(self, commit): 126 '''Creates a ServerInstance pinned to |commit|, or HEAD if None. 127 NOTE: If passed None it's likely that during the cron run patches will be 128 submitted at HEAD, which may change data underneath the cron run. 129 ''' 130 object_store_creator = ObjectStoreCreator(start_empty=True) 131 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) 132 host_file_system_provider = self._delegate.CreateHostFileSystemProvider( 133 object_store_creator, pinned_commit=commit) 134 github_file_system_provider = self._delegate.CreateGithubFileSystemProvider( 135 object_store_creator) 136 gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider( 137 object_store_creator) 138 return ServerInstance(object_store_creator, 139 CompiledFileSystem.Factory(object_store_creator), 140 branch_utility, 141 host_file_system_provider, 142 github_file_system_provider, 143 gcs_file_system_provider) 144