1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import sys
6
7from docs_server_utils import ToUnicode
8from file_system import FileNotFoundError
9from future import Future
10from path_util import AssertIsDirectory, AssertIsFile, ToDirectory
11from third_party.json_schema_compiler import json_parse
12from third_party.json_schema_compiler.memoize import memoize
13from third_party.motemplate import Motemplate
14
15
16_CACHEABLE_FUNCTIONS = set()
17_SINGLE_FILE_FUNCTIONS = set()
18
19
20def _GetUnboundFunction(fn):
21  '''Functions bound to an object are separate from the unbound
22  defintion. This causes issues when checking for cache membership,
23  so always get the unbound function, if possible.
24  '''
25  return getattr(fn, 'im_func', fn)
26
27
28def Cache(fn):
29  '''A decorator which can be applied to the compilation function
30  passed to CompiledFileSystem.Create, indicating that file/list data
31  should be cached.
32
33  This decorator should be listed first in any list of decorators, along
34  with the SingleFile decorator below.
35  '''
36  _CACHEABLE_FUNCTIONS.add(_GetUnboundFunction(fn))
37  return fn
38
39
40def SingleFile(fn):
41  '''A decorator which can be optionally applied to the compilation function
42  passed to CompiledFileSystem.Create, indicating that the function only
43  needs access to the file which is given in the function's callback. When
44  this is the case some optimisations can be done.
45
46  Note that this decorator must be listed first in any list of decorators to
47  have any effect.
48  '''
49  _SINGLE_FILE_FUNCTIONS.add(_GetUnboundFunction(fn))
50  return fn
51
52
53def Unicode(fn):
54  '''A decorator which can be optionally applied to the compilation function
55  passed to CompiledFileSystem.Create, indicating that the function processes
56  the file's data as Unicode text.
57  '''
58
59  # The arguments passed to fn can be (self, path, data) or (path, data). In
60  # either case the last argument is |data|, which should be converted to
61  # Unicode.
62  def convert_args(args):
63    args = list(args)
64    args[-1] = ToUnicode(args[-1])
65    return args
66
67  return lambda *args: fn(*convert_args(args))
68
69
70class _CacheEntry(object):
71  def __init__(self, cache_data, version):
72
73    self.cache_data = cache_data
74    self.version = version
75
76
77class CompiledFileSystem(object):
78  '''This class caches FileSystem data that has been processed.
79  '''
80
81  class Factory(object):
82    '''A class to build a CompiledFileSystem backed by |file_system|.
83    '''
84
85    def __init__(self, object_store_creator):
86      self._object_store_creator = object_store_creator
87
88    def Create(self, file_system, compilation_function, cls, category=None):
89      '''Creates a CompiledFileSystem view over |file_system| that populates
90      its cache by calling |compilation_function| with (path, data), where
91      |data| is the data that was fetched from |path| in |file_system|.
92
93      The namespace for the compiled file system is derived similar to
94      ObjectStoreCreator: from |cls| along with an optional |category|.
95      '''
96      assert isinstance(cls, type)
97      assert not cls.__name__[0].islower()  # guard against non-class types
98      full_name = [cls.__name__, file_system.GetIdentity()]
99      if category is not None:
100        full_name.append(category)
101      def create_object_store(my_category):
102        # The read caches can start populated (start_empty=False) because file
103        # updates are picked up by the stat - but only if the compilation
104        # function is affected by a single file. If the compilation function is
105        # affected by other files (e.g. compiling a list of APIs available to
106        # extensions may be affected by both a features file and the list of
107        # files in the API directory) then this optimisation won't work.
108        return self._object_store_creator.Create(
109            CompiledFileSystem,
110            category='/'.join(full_name + [my_category]),
111            start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS)
112      return CompiledFileSystem(file_system,
113                                compilation_function,
114                                create_object_store('file'),
115                                create_object_store('list'))
116
117    @memoize
118    def ForJson(self, file_system):
119      '''A CompiledFileSystem specifically for parsing JSON configuration data.
120      These are memoized over file systems tied to different branches.
121      '''
122      return self.Create(file_system,
123                         Cache(SingleFile(lambda _, data:
124                             json_parse.Parse(ToUnicode(data)))),
125                         CompiledFileSystem,
126                         category='json')
127
128    @memoize
129    def ForTemplates(self, file_system):
130      '''Creates a CompiledFileSystem for parsing templates.
131      '''
132      return self.Create(
133          file_system,
134          SingleFile(lambda path, text: Motemplate(ToUnicode(text), name=path)),
135          CompiledFileSystem)
136
137    @memoize
138    def ForUnicode(self, file_system):
139      '''Creates a CompiledFileSystem for Unicode text processing.
140      '''
141      return self.Create(
142        file_system,
143        SingleFile(lambda _, text: ToUnicode(text)),
144        CompiledFileSystem,
145        category='text')
146
147  def __init__(self,
148               file_system,
149               compilation_function,
150               file_object_store,
151               list_object_store):
152    self._file_system = file_system
153    self._compilation_function = compilation_function
154    self._file_object_store = file_object_store
155    self._list_object_store = list_object_store
156
157  def _Get(self, store, key):
158    if _GetUnboundFunction(self._compilation_function) in _CACHEABLE_FUNCTIONS:
159      return store.Get(key)
160    return Future(value=None)
161
162  def _Set(self, store, key, value):
163    if _GetUnboundFunction(self._compilation_function) in _CACHEABLE_FUNCTIONS:
164      store.Set(key, value)
165
166  def _RecursiveList(self, path):
167    '''Returns a Future containing the recursive directory listing of |path| as
168    a flat list of paths.
169    '''
170    def split_dirs_from_files(paths):
171      '''Returns a tuple (dirs, files) where |dirs| contains the directory
172      names in |paths| and |files| contains the files.
173      '''
174      result = [], []
175      for path in paths:
176        result[0 if path.endswith('/') else 1].append(path)
177      return result
178
179    def add_prefix(prefix, paths):
180      return [prefix + path for path in paths]
181
182    # Read in the initial list of files. Do this eagerly (i.e. not part of the
183    # asynchronous Future contract) because there's a greater chance to
184    # parallelise fetching with the second layer (can fetch multiple paths).
185    try:
186      first_layer_dirs, first_layer_files = split_dirs_from_files(
187          self._file_system.ReadSingle(path).Get())
188    except FileNotFoundError:
189      return Future(exc_info=sys.exc_info())
190
191    if not first_layer_dirs:
192      return Future(value=first_layer_files)
193
194    def get_from_future_listing(listings):
195      '''Recursively lists files from directory listing |futures|.
196      '''
197      dirs, files = [], []
198      for dir_name, listing in listings.iteritems():
199        new_dirs, new_files = split_dirs_from_files(listing)
200        # |dirs| are paths for reading. Add the full prefix relative to
201        # |path| so that |file_system| can find the files.
202        dirs += add_prefix(dir_name, new_dirs)
203        # |files| are not for reading, they are for returning to the caller.
204        # This entire function set (i.e. GetFromFileListing) is defined to
205        # not include the fetched-path in the result, however, |dir_name|
206        # will be prefixed with |path|. Strip it.
207        assert dir_name.startswith(path)
208        files += add_prefix(dir_name[len(path):], new_files)
209      if dirs:
210        files += self._file_system.Read(dirs).Then(
211            get_from_future_listing).Get()
212      return files
213
214    return self._file_system.Read(add_prefix(path, first_layer_dirs)).Then(
215        lambda results: first_layer_files + get_from_future_listing(results))
216
217  def GetFromFile(self, path, skip_not_found=False):
218    '''Calls |compilation_function| on the contents of the file at |path|.
219    If |skip_not_found| is True, then None is passed to |compilation_function|.
220    '''
221    AssertIsFile(path)
222
223    try:
224      version = self._file_system.Stat(path).version
225    except FileNotFoundError:
226      if skip_not_found:
227        version = None
228      else:
229        return Future(exc_info=sys.exc_info())
230
231    cache_entry = self._Get(self._file_object_store, path).Get()
232    if (cache_entry is not None) and (version == cache_entry.version):
233      return Future(value=cache_entry.cache_data)
234
235    def compile_(files):
236      cache_data = self._compilation_function(path, files)
237      self._Set(self._file_object_store, path, _CacheEntry(cache_data, version))
238      return cache_data
239
240    return self._file_system.ReadSingle(
241        path, skip_not_found=skip_not_found).Then(compile_)
242
243  def GetFromFileListing(self, path):
244    '''Calls |compilation_function| on the listing of the files at |path|.
245    Assumes that the path given is to a directory.
246    '''
247    AssertIsDirectory(path)
248
249    try:
250      version = self._file_system.Stat(path).version
251    except FileNotFoundError:
252      return Future(exc_info=sys.exc_info())
253
254    cache_entry = self._Get(self._list_object_store, path).Get()
255    if (cache_entry is not None) and (version == cache_entry.version):
256      return Future(value=cache_entry.cache_data)
257
258    def compile_(files):
259      cache_data = self._compilation_function(path, files)
260      self._Set(self._list_object_store, path, _CacheEntry(cache_data, version))
261      return cache_data
262    return self._RecursiveList(path).Then(compile_)
263
264  # _GetFileVersionFromCache and _GetFileListingVersionFromCache are exposed
265  # *only* so that ChainedCompiledFileSystem can optimise its caches. *Do not*
266  # use these methods otherwise, they don't do what you want. Use
267  # FileSystem.Stat on the FileSystem that this CompiledFileSystem uses.
268
269  def _GetFileVersionFromCache(self, path):
270    cache_entry = self._Get(self._file_object_store, path).Get()
271    if cache_entry is not None:
272      return Future(value=cache_entry.version)
273    stat_future = self._file_system.StatAsync(path)
274    return Future(callback=lambda: stat_future.Get().version)
275
276  def _GetFileListingVersionFromCache(self, path):
277    path = ToDirectory(path)
278    cache_entry = self._Get(self._list_object_store, path).Get()
279    if cache_entry is not None:
280      return Future(value=cache_entry.version)
281    stat_future = self._file_system.StatAsync(path)
282    return Future(callback=lambda: stat_future.Get().version)
283
284  def GetIdentity(self):
285    return self._file_system.GetIdentity()
286