1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import traceback
6
7from app_yaml_helper import AppYamlHelper
8from appengine_wrappers import IsDeadlineExceededError, logservice
9from branch_utility import BranchUtility
10from compiled_file_system import CompiledFileSystem
11from custom_logger import CustomLogger
12from data_source_registry import CreateDataSource
13from environment import GetAppVersion
14from file_system import IsFileSystemThrottledError
15from future import Future
16from gcs_file_system_provider import CloudStorageFileSystemProvider
17from github_file_system_provider import GithubFileSystemProvider
18from host_file_system_provider import HostFileSystemProvider
19from object_store_creator import ObjectStoreCreator
20from server_instance import ServerInstance
21from servlet import Servlet, Request, Response
22from timer import Timer, TimerClosure
23
24
25
26_log = CustomLogger('refresh')
27
28
29class RefreshServlet(Servlet):
30  '''Servlet which refreshes a single data source.
31  '''
32  def __init__(self, request, delegate_for_test=None):
33    Servlet.__init__(self, request)
34    self._delegate = delegate_for_test or RefreshServlet.Delegate()
35
36  class Delegate(object):
37    '''RefreshServlet's runtime dependencies. Override for testing.
38    '''
39    def CreateBranchUtility(self, object_store_creator):
40      return BranchUtility.Create(object_store_creator)
41
42    def CreateHostFileSystemProvider(self,
43                                     object_store_creator,
44                                     pinned_commit=None):
45      return HostFileSystemProvider(object_store_creator,
46                                    pinned_commit=pinned_commit)
47
48    def CreateGithubFileSystemProvider(self, object_store_creator):
49      return GithubFileSystemProvider(object_store_creator)
50
51    def CreateGCSFileSystemProvider(self, object_store_creator):
52      return CloudStorageFileSystemProvider(object_store_creator)
53
54    def GetAppVersion(self):
55      return GetAppVersion()
56
57  def Get(self):
58    # Manually flush logs at the end of the run. However, sometimes
59    # even that isn't enough, which is why in this file we use the
60    # custom logger and make it flush the log every time its used.
61    logservice.AUTOFLUSH_ENABLED = False
62    try:
63      return self._GetImpl()
64    except BaseException:
65      _log.error('Caught top-level exception! %s', traceback.format_exc())
66    finally:
67      logservice.flush()
68
69  def _GetImpl(self):
70    path = self._request.path.strip('/')
71    parts = self._request.path.split('/', 1)
72    source_name = parts[0]
73    if len(parts) == 2:
74      source_path = parts[1]
75    else:
76      source_path = None
77
78    _log.info('starting refresh of %s DataSource %s' %
79        (source_name, '' if source_path is None else '[%s]' % source_path))
80
81    if 'commit' in self._request.arguments:
82      commit = self._request.arguments['commit']
83    else:
84      _log.warning('No commit given; refreshing from master. '
85                   'This is probably NOT what you want.')
86      commit = None
87
88    server_instance = self._CreateServerInstance(commit)
89    success = True
90    try:
91      if source_name == 'platform_bundle':
92        data_source = server_instance.platform_bundle
93      elif source_name == 'content_providers':
94        data_source = server_instance.content_providers
95      else:
96        data_source = CreateDataSource(source_name, server_instance)
97
98      class_name = data_source.__class__.__name__
99      refresh_future = data_source.Refresh(source_path)
100      assert isinstance(refresh_future, Future), (
101          '%s.Refresh() did not return a Future' % class_name)
102      timer = Timer()
103      try:
104        refresh_future.Get()
105      except Exception as e:
106        _log.error('%s: error %s' % (class_name, traceback.format_exc()))
107        success = False
108        if IsFileSystemThrottledError(e):
109          return Response.ThrottledError('Throttled')
110        raise
111      finally:
112        _log.info('Refreshing %s took %s' %
113            (class_name, timer.Stop().FormatElapsed()))
114
115    except:
116      success = False
117      # This should never actually happen.
118      _log.error('uncaught error: %s' % traceback.format_exc())
119      raise
120    finally:
121      _log.info('finished (%s)', 'success' if success else 'FAILED')
122      return (Response.Ok('Success') if success else
123              Response.InternalError('Failure'))
124
125  def _CreateServerInstance(self, commit):
126    '''Creates a ServerInstance pinned to |commit|, or HEAD if None.
127    NOTE: If passed None it's likely that during the cron run patches will be
128    submitted at HEAD, which may change data underneath the cron run.
129    '''
130    object_store_creator = ObjectStoreCreator(start_empty=True)
131    branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
132    host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
133        object_store_creator, pinned_commit=commit)
134    github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
135        object_store_creator)
136    gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider(
137        object_store_creator)
138    return ServerInstance(object_store_creator,
139                          CompiledFileSystem.Factory(object_store_creator),
140                          branch_utility,
141                          host_file_system_provider,
142                          github_file_system_provider,
143                          gcs_file_system_provider)
144