1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import base64
6import logging
7import posixpath
8import time
9
10from appengine_wrappers import urlfetch
11from environment import GetAppVersion
12from future import Future
13
14
15_MAX_RETRIES = 5
16_RETRY_DELAY_SECONDS = 30
17
18
19def _MakeHeaders(username, password, access_token):
20  headers = {
21    'User-Agent': 'Chromium docserver %s' % GetAppVersion(),
22    'Cache-Control': 'max-age=0',
23  }
24  if username is not None and password is not None:
25    headers['Authorization'] = 'Basic %s' % base64.b64encode(
26        '%s:%s' % (username, password))
27  if access_token is not None:
28    headers['Authorization'] = 'OAuth %s' % access_token
29  return headers
30
31
32class AppEngineUrlFetcher(object):
33  """A wrapper around the App Engine urlfetch module that allows for easy
34  async fetches.
35  """
36  def __init__(self, base_path=None):
37    assert base_path is None or not base_path.endswith('/'), base_path
38    self._base_path = base_path
39    self._retries_left = _MAX_RETRIES
40
41  def Fetch(self, url, username=None, password=None, access_token=None):
42    """Fetches a file synchronously.
43    """
44    return urlfetch.fetch(self._FromBasePath(url),
45                          deadline=20,
46                          headers=_MakeHeaders(username,
47                                               password,
48                                               access_token))
49
50  def FetchAsync(self, url, username=None, password=None, access_token=None):
51    """Fetches a file asynchronously, and returns a Future with the result.
52    """
53    def process_result(result):
54      if result.status_code == 429:
55        if self._retries_left == 0:
56          logging.error('Still throttled. Giving up.')
57          return result
58        self._retries_left -= 1
59        logging.info('Throttled. Trying again in %s seconds.' %
60                     _RETRY_DELAY_SECONDS)
61        time.sleep(_RETRY_DELAY_SECONDS)
62        return self.FetchAsync(url, username, password, access_token).Get()
63      return result
64
65    rpc = urlfetch.create_rpc(deadline=20)
66    urlfetch.make_fetch_call(rpc,
67                             self._FromBasePath(url),
68                             headers=_MakeHeaders(username,
69                                                  password,
70                                                  access_token))
71    return Future(callback=lambda: process_result(rpc.get_result()))
72
73  def _FromBasePath(self, url):
74    assert not url.startswith('/'), url
75    if self._base_path is not None:
76      url = posixpath.join(self._base_path, url) if url else self._base_path
77    return url
78