github_file_system.py revision c2e0dbddbe15c98d52c4786dac06cb8952a8ae6d 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import json
6import logging
7import os
8
9import appengine_blobstore as blobstore
10from appengine_wrappers import GetAppVersion, urlfetch
11from file_system import FileSystem, StatInfo
12from future import Future
13from object_store_creator import ObjectStoreCreator
14from StringIO import StringIO
15from zipfile import ZipFile, BadZipfile
16
17ZIP_KEY = 'zipball'
18USERNAME = None
19PASSWORD = None
20
21def _MakeBlobstoreKey(version):
22  return ZIP_KEY + '.' + str(version)
23
24class _AsyncFetchFutureZip(object):
25  def __init__(self,
26               fetcher,
27               username,
28               password,
29               blobstore,
30               key_to_set,
31               key_to_delete=None):
32    self._fetcher = fetcher
33    self._fetch = fetcher.FetchAsync(ZIP_KEY,
34                                     username=username,
35                                     password=password)
36    self._blobstore = blobstore
37    self._key_to_set = key_to_set
38    self._key_to_delete = key_to_delete
39
40  def Get(self):
41    try:
42      result = self._fetch.Get()
43      # Check if Github authentication failed.
44      if result.status_code == 401:
45        logging.error('Github authentication failed for %s, falling back to '
46                      'unauthenticated.' % USERNAME)
47        blob = self._fetcher.Fetch(ZIP_KEY).content
48      else:
49        blob = result.content
50    except urlfetch.DownloadError as e:
51      logging.error('Bad github zip file: %s' % e)
52      return None
53    if self._key_to_delete is not None:
54      self._blobstore.Delete(_MakeBlobstoreKey(self._key_to_delete),
55                             blobstore.BLOBSTORE_GITHUB)
56    try:
57      return_zip = ZipFile(StringIO(blob))
58    except BadZipfile as e:
59      logging.error('Bad github zip file: %s' % e)
60      return None
61
62    self._blobstore.Set(_MakeBlobstoreKey(self._key_to_set),
63                        blob,
64                        blobstore.BLOBSTORE_GITHUB)
65    return return_zip
66
67class GithubFileSystem(FileSystem):
68  """FileSystem implementation which fetches resources from github.
69  """
70  def __init__(self, fetcher, blobstore):
71    # The password store is the same for all branches and versions.
72    password_store = (ObjectStoreCreator.GlobalFactory()
73        .Create(GithubFileSystem).Create(category='password'))
74    if USERNAME is None:
75      password_data = password_store.GetMulti(('username', 'password')).Get()
76      self._username, self._password = (password_data.get('username'),
77                                        password_data.get('password'))
78    else:
79      password_store.SetMulti({'username': USERNAME, 'password': PASSWORD})
80      self._username, self._password = (USERNAME, PASSWORD)
81
82    self._fetcher = fetcher
83    self._blobstore = blobstore
84    self._stat_object_store = (ObjectStoreCreator.SharedFactory(GetAppVersion())
85        .Create(GithubFileSystem).Create())
86    self._version = None
87    self._GetZip(self.Stat(ZIP_KEY).version)
88
89  def _GetZip(self, version):
90    blob = self._blobstore.Get(_MakeBlobstoreKey(version),
91                               blobstore.BLOBSTORE_GITHUB)
92    if blob is not None:
93      try:
94        self._zip_file = Future(value=ZipFile(StringIO(blob)))
95      except BadZipfile as e:
96        self._blobstore.Delete(_MakeBlobstoreKey(version),
97                               blobstore.BLOBSTORE_GITHUB)
98        logging.error('Bad github zip file: %s' % e)
99        self._zip_file = Future(value=None)
100    else:
101      self._zip_file = Future(
102          delegate=_AsyncFetchFutureZip(self._fetcher,
103                                        self._username,
104                                        self._password,
105                                        self._blobstore,
106                                        version,
107                                        key_to_delete=self._version))
108    self._version = version
109
110  def _ReadFile(self, path):
111    try:
112      zip_file = self._zip_file.Get()
113    except Exception as e:
114      logging.error('Github ReadFile error: %s' % e)
115      return ''
116    if zip_file is None:
117      logging.error('Bad github zip file.')
118      return ''
119    prefix = zip_file.namelist()[0][:-1]
120    return zip_file.read(prefix + path)
121
122  def _ListDir(self, path):
123    try:
124      zip_file = self._zip_file.Get()
125    except Exception as e:
126      logging.error('Github ListDir error: %s' % e)
127      return []
128    if zip_file is None:
129      logging.error('Bad github zip file.')
130      return []
131    filenames = zip_file.namelist()
132    # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
133    filenames = [f[len(filenames[0]) - 1:] for f in filenames]
134    # Remove the path of the directory we're listing from the filenames.
135    filenames = [f[len(path):] for f in filenames
136                 if f != path and f.startswith(path)]
137    # Remove all files not directly in this directory.
138    return [f for f in filenames if f[:-1].count('/') == 0]
139
140  def Read(self, paths, binary=False):
141    version = self.Stat(ZIP_KEY).version
142    if version != self._version:
143      self._GetZip(version)
144    result = {}
145    for path in paths:
146      if path.endswith('/'):
147        result[path] = self._ListDir(path)
148      else:
149        result[path] = self._ReadFile(path)
150    return Future(value=result)
151
152  def _DefaultStat(self, path):
153    version = 0
154    # TODO(kalman): we should replace all of this by wrapping the
155    # GithubFileSystem in a CachingFileSystem. A lot of work has been put into
156    # CFS to be robust, and GFS is missing out.
157    # For example: the following line is wrong, but it could be moot.
158    self._stat_object_store.Set(path, version)
159    return StatInfo(version)
160
161  def Stat(self, path):
162    version = self._stat_object_store.Get(path).Get()
163    if version is not None:
164      return StatInfo(version)
165    try:
166      result = self._fetcher.Fetch('commits/HEAD',
167                                   username=USERNAME,
168                                   password=PASSWORD)
169    except urlfetch.DownloadError as e:
170      logging.error('GithubFileSystem Stat: %s' % e)
171      return self._DefaultStat(path)
172    # Check if Github authentication failed.
173    if result.status_code == 401:
174      logging.error('Github authentication failed for %s, falling back to '
175                    'unauthenticated.' % USERNAME)
176      try:
177        result = self._fetcher.Fetch('commits/HEAD')
178      except urlfetch.DownloadError as e:
179        logging.error('GithubFileSystem Stat: %s' % e)
180        return self._DefaultStat(path)
181    version = (json.loads(result.content).get('commit', {})
182                                         .get('tree', {})
183                                         .get('sha', None))
184    # Check if the JSON was valid, and set to 0 if not.
185    if version is not None:
186      self._stat_object_store.Set(path, version)
187    else:
188      logging.warning('Problem fetching commit hash from github.')
189      return self._DefaultStat(path)
190    return StatInfo(version)
191