cron_servlet.py revision a36e5920737c6adbddd3e43b760e5de8431db6e0 
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import time
7import traceback
8
9from app_yaml_helper import AppYamlHelper
10from appengine_wrappers import (
11    GetAppVersion, DeadlineExceededError, IsDevServer, logservice)
12from branch_utility import BranchUtility
13from compiled_file_system import CompiledFileSystem
14from empty_dir_file_system import EmptyDirFileSystem
15from file_system_util import CreateURLsFromPaths
16from github_file_system import GithubFileSystem
17from host_file_system_creator import HostFileSystemCreator
18from object_store_creator import ObjectStoreCreator
19from render_servlet import RenderServlet
20from server_instance import ServerInstance
21from servlet import Servlet, Request, Response
22import svn_constants
23
24class _SingletonRenderServletDelegate(RenderServlet.Delegate):
25  def __init__(self, server_instance):
26    self._server_instance = server_instance
27
28  def CreateServerInstance(self):
29    return self._server_instance
30
31class CronServlet(Servlet):
32  '''Servlet which runs a cron job.
33  '''
34  def __init__(self, request, delegate_for_test=None):
35    Servlet.__init__(self, request)
36    self._delegate = delegate_for_test or CronServlet.Delegate()
37
38  class Delegate(object):
39    '''CronServlet's runtime dependencies. Override for testing.
40    '''
41    def CreateBranchUtility(self, object_store_creator):
42      return BranchUtility.Create(object_store_creator)
43
44    def CreateHostFileSystemCreator(self, object_store_creator):
45      return HostFileSystemCreator(object_store_creator)
46
47    def CreateAppSamplesFileSystem(self, object_store_creator):
48      # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's
49      # not supported yet (see comment there).
50      return (EmptyDirFileSystem() if IsDevServer() else
51              GithubFileSystem.Create(object_store_creator))
52
53    def GetAppVersion(self):
54      return GetAppVersion()
55
56  def Get(self):
57    # Crons often time out, and when they do *and* then eventually try to
58    # flush logs they die. Turn off autoflush and manually do so at the end.
59    logservice.AUTOFLUSH_ENABLED = False
60    try:
61      return self._GetImpl()
62    finally:
63      logservice.flush()
64
65  def _GetImpl(self):
66    # Cron strategy:
67    #
68    # Find all public template files and static files, and render them. Most of
69    # the time these won't have changed since the last cron run, so it's a
70    # little wasteful, but hopefully rendering is really fast (if it isn't we
71    # have a problem).
72    logging.info('cron: starting')
73
74    # This is returned every time RenderServlet wants to create a new
75    # ServerInstance.
76    server_instance = self._GetSafeServerInstance()
77
78    def get_via_render_servlet(path):
79      request = Request(path, self._request.host, self._request.headers)
80      delegate = _SingletonRenderServletDelegate(server_instance)
81      return RenderServlet(request, delegate).Get()
82
83    def run_cron_for_dir(d, path_prefix=''):
84      success = True
85      start_time = time.time()
86      files = dict(
87          CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix))
88      logging.info('cron: rendering %s files from %s...' % (len(files), d))
89      try:
90        for i, path in enumerate(files):
91          error = None
92          try:
93            response = get_via_render_servlet(path)
94            if response.status != 200:
95              error = 'Got %s response' % response.status
96          except DeadlineExceededError:
97            logging.error(
98                'cron: deadline exceeded rendering %s (%s of %s): %s' % (
99                    path, i + 1, len(files), traceback.format_exc()))
100            raise
101          except error:
102            pass
103          if error:
104            logging.error('cron: error rendering %s: %s' % (path, error))
105            success = False
106      finally:
107        logging.info('cron: rendering %s files from %s took %s seconds' % (
108            len(files), d, time.time() - start_time))
109      return success
110
111    success = True
112    try:
113      # Render all of the publicly accessible files.
114      cron_runs = [
115        # Note: rendering the public templates will pull in all of the private
116        # templates.
117        (svn_constants.PUBLIC_TEMPLATE_PATH, ''),
118        # Note: rendering the public templates will have pulled in the .js
119        # and manifest.json files (for listing examples on the API reference
120        # pages), but there are still images, CSS, etc.
121        (svn_constants.STATIC_PATH, 'static/'),
122      ]
123      if not IsDevServer():
124        cron_runs.append(
125            (svn_constants.EXAMPLES_PATH, 'extensions/examples/'))
126
127      # Note: don't try to short circuit any of this stuff. We want to run
128      # the cron for all the directories regardless of intermediate
129      # failures.
130      for path, path_prefix in cron_runs:
131        success = run_cron_for_dir(path, path_prefix=path_prefix) and success
132
133      # TODO(kalman): Generic way for classes to request cron access. The next
134      # two special cases are ugly. It would potentially greatly speed up cron
135      # runs, too.
136
137      # Extension examples have zip files too. Well, so do apps, but the app
138      # file system doesn't get the Offline treatment so they don't need cron.
139      if not IsDevServer():
140        manifest_json = 'manifest.json'
141        example_zips = []
142        for root, _, files in server_instance.host_file_system.Walk(
143            svn_constants.EXAMPLES_PATH):
144          example_zips.extend(
145              root + '.zip' for name in files if name == manifest_json)
146        logging.info('cron: rendering %s example zips...' % len(example_zips))
147        start_time = time.time()
148        try:
149          success = success and all(
150              get_via_render_servlet('extensions/examples/%s' % z).status == 200
151              for z in example_zips)
152        finally:
153          logging.info('cron: rendering %s example zips took %s seconds' % (
154              len(example_zips), time.time() - start_time))
155
156    except DeadlineExceededError:
157      success = False
158
159    logging.info('cron: running Redirector cron...')
160    server_instance.redirector.Cron()
161
162    logging.info('cron: finished (%s)' % ('success' if success else 'failure',))
163
164    return (Response.Ok('Success') if success else
165            Response.InternalError('Failure'))
166
167  def _GetSafeServerInstance(self):
168    '''Returns a ServerInstance with a host file system at a safe revision,
169    meaning the last revision that the current running version of the server
170    existed.
171    '''
172    delegate = self._delegate
173    server_instance_at_head = self._CreateServerInstance(None)
174
175    app_yaml_handler = AppYamlHelper(
176        svn_constants.APP_YAML_PATH,
177        server_instance_at_head.host_file_system,
178        server_instance_at_head.object_store_creator,
179        server_instance_at_head.host_file_system_creator)
180
181    if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
182      # TODO(kalman): return a new ServerInstance at an explicit revision in
183      # case the HEAD version changes underneath us.
184      return server_instance_at_head
185
186    # The version in app.yaml is greater than the currently running app's.
187    # The safe version is the one before it changed.
188    safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
189        delegate.GetAppVersion()) - 1
190
191    logging.info('cron: app version %s is out of date, safe is %s' % (
192        delegate.GetAppVersion(), safe_revision))
193
194    return self._CreateServerInstance(safe_revision)
195
196  def _CreateServerInstance(self, revision):
197    object_store_creator = ObjectStoreCreator(start_empty=True)
198    branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
199    host_file_system_creator = self._delegate.CreateHostFileSystemCreator(
200        object_store_creator)
201    host_file_system = host_file_system_creator.Create(revision=revision)
202    app_samples_file_system = self._delegate.CreateAppSamplesFileSystem(
203        object_store_creator)
204    compiled_host_fs_factory = CompiledFileSystem.Factory(
205        host_file_system,
206        object_store_creator)
207    return ServerInstance(object_store_creator,
208                          host_file_system,
209                          app_samples_file_system,
210                          '',
211                          compiled_host_fs_factory,
212                          branch_utility,
213                          host_file_system_creator)
214