caching_rietveld_patcher.py revision ca12bfac764ba476d6cd062bf1dde12cc64c3f40 
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from datetime import datetime, timedelta
6from file_system import FileNotFoundError, ToUnicode
7from future import Future
8from patcher import Patcher
9
10_VERSION_CACHE_MAXAGE = timedelta(seconds=5)
11
12''' Append @version for keys to distinguish between different patchsets of
13an issue.
14'''
15def _MakeKey(path, version):
16  return '%s@%s' % (path, version)
17
18def _ToObjectStoreValue(raw_value, version):
19  return dict((_MakeKey(key, version), raw_value[key])
20              for key in raw_value)
21
22def _FromObjectStoreValue(raw_value, binary):
23  return dict((key[0:key.rfind('@')], _HandleBinary(raw_value[key], binary))
24              for key in raw_value)
25
26def _HandleBinary(data, binary):
27  return data if binary else ToUnicode(data)
28
29class _AsyncUncachedFuture(object):
30  def __init__(self,
31               version,
32               paths,
33               binary,
34               cached_value,
35               missing_paths,
36               fetch_delegate,
37               object_store):
38    self._version = version
39    self._paths = paths
40    self._binary = binary
41    self._cached_value = cached_value
42    self._missing_paths = missing_paths
43    self._fetch_delegate = fetch_delegate
44    self._object_store = object_store
45
46  def Get(self):
47    uncached_raw_value = self._fetch_delegate.Get()
48    self._object_store.SetMulti(_ToObjectStoreValue(uncached_raw_value,
49                                                    self._version))
50
51    for path in self._missing_paths:
52      if uncached_raw_value.get(path) is None:
53        raise FileNotFoundError('File %s was not found in the patch.' % path)
54      self._cached_value[path] = _HandleBinary(uncached_raw_value[path],
55                                               self._binary)
56
57    return self._cached_value
58
59class CachingRietveldPatcher(Patcher):
60  ''' CachingRietveldPatcher implements a caching layer on top of |patcher|.
61  In theory, it can be used with any class that implements Patcher. But this
62  class assumes that applying to all patched files at once is more efficient
63  than applying to individual files.
64  '''
65  def __init__(self,
66               rietveld_patcher,
67               object_store_creator,
68               test_datetime=datetime):
69    self._patcher = rietveld_patcher
70    def create_object_store(category):
71      return object_store_creator.Create(
72          CachingRietveldPatcher,
73          category='%s/%s' % (rietveld_patcher.GetIdentity(), category))
74    self._version_object_store = create_object_store('version')
75    self._list_object_store = create_object_store('list')
76    self._file_object_store = create_object_store('file')
77    self._datetime = test_datetime
78
79  def GetVersion(self):
80    key = 'version'
81    value = self._version_object_store.Get(key).Get()
82    if value is not None:
83      version, time = value
84      if self._datetime.now() - time < _VERSION_CACHE_MAXAGE:
85        return version
86
87    version = self._patcher.GetVersion()
88    self._version_object_store.Set(key,
89                                (version, self._datetime.now()))
90    return version
91
92  def GetPatchedFiles(self, version=None):
93    if version is None:
94      version = self.GetVersion()
95    patched_files = self._list_object_store.Get(version).Get()
96    if patched_files is not None:
97      return patched_files
98
99    patched_files = self._patcher.GetPatchedFiles(version)
100    self._list_object_store.Set(version, patched_files)
101    return patched_files
102
103  def Apply(self, paths, file_system, binary=False, version=None):
104    if version is None:
105      version = self.GetVersion()
106    added, deleted, modified = self.GetPatchedFiles(version)
107    cached_value = _FromObjectStoreValue(self._file_object_store.
108        GetMulti([_MakeKey(path, version) for path in paths]).Get(), binary)
109    missing_paths = list(set(paths) - set(cached_value.keys()))
110    if len(missing_paths) == 0:
111      return Future(value=cached_value)
112
113    # binary is explicitly set to True. Here we are applying the patch to
114    # ALL patched files without a way to know whether individual files are
115    # binary or not. Therefore all data cached must be binary. When reading
116    # from the cache with binary=False, it will be converted to Unicode by
117    # _HandleBinary.
118    return _AsyncUncachedFuture(version,
119                                paths,
120                                binary,
121                                cached_value,
122                                missing_paths,
123                                self._patcher.Apply(set(added) | set(modified),
124                                                    None,
125                                                    True,
126                                                    version),
127                                self._file_object_store)
128
129  def GetIdentity(self):
130    return self._patcher.GetIdentity()
131