1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Caches processed query results in memcache and datastore.
6
7Memcache is not very reliable for the perf dashboard. Prometheus team explained
8that memcache is LRU and shared between multiple applications, so their activity
9may result in our data being evicted. To prevent this, we cache processed
10query results in the data store. Using NDB, the values are also cached in
11memcache if possible. This improves performance because doing a get()
12for a key which has a single BlobProperty is much quicker than a complex query
13over a large dataset.
14(Background: http://g/prometheus-discuss/othVtufGIyM/wjAS5djyG8kJ)
15
16When an item is cached, layered_cache does the following:
171) Namespaces the key based on whether datastore_hooks says the request is
18internal_only.
192) Pickles the value (memcache does this internally), and adds a data store
20entity with the key and a BlobProperty with the pickled value.
21
22Retrieving values checks memcache via NDB first, and if datastore is used it
23unpickles.
24
25When an item is removed from the the cache, it is removed from both internal and
26external caches, since removals are usually caused by large changes that affect
27both caches.
28
29Although this module contains ndb.Model classes, these are not intended
30to be used directly by other modules.
31"""
32
33import cPickle
34import datetime
35import logging
36
37from google.appengine.api import datastore_errors
38from google.appengine.ext import ndb
39
40from dashboard import datastore_hooks
41from dashboard import request_handler
42
43
44class DeleteExpiredEntitiesHandler(request_handler.RequestHandler):
45  """URL endpoint for a cron job to delete expired entities from datastore."""
46
47  def get(self):
48    """This get handler is called from cron.
49
50    It deletes only expired CachedPickledString entities from the datastore.
51    """
52    DeleteAllExpiredEntities()
53
54
55class CachedPickledString(ndb.Model):
56  value = ndb.BlobProperty()
57  expire_time = ndb.DateTimeProperty()
58
59  @classmethod
60  def GetExpiredKeys(cls):
61    """Gets keys of expired entities.
62
63    Returns:
64      List of keys for items which are expired.
65    """
66    current_time = datetime.datetime.now()
67    query = cls.query(cls.expire_time < current_time)
68    query = query.filter(cls.expire_time != None)
69    return query.fetch(keys_only=True)
70
71
72def _NamespaceKey(key, namespace=None):
73  if not namespace:
74    namespace = datastore_hooks.GetNamespace()
75  return '%s__%s' % (namespace, key)
76
77
78def Prewarm(keys):
79  """Prewarms the NDB in-context cache by doing async_get for the keys.
80
81  For requests like /add_point which can get/set dozens of keys, contention
82  occasionally causes the gets to take several seconds. But they will be
83  cached in context by NDB if they are requested at the start of the request.
84
85  Args:
86    keys: List of string keys.
87  """
88  to_get = []
89  for key in keys:
90    to_get.append(ndb.Key('CachedPickledString',
91                          _NamespaceKey(key, datastore_hooks.EXTERNAL)))
92    to_get.append(ndb.Key('CachedPickledString',
93                          _NamespaceKey(key, datastore_hooks.INTERNAL)))
94  ndb.get_multi_async(to_get)
95
96
97def Get(key):
98  """Gets the value from the datastore."""
99  namespaced_key = _NamespaceKey(key)
100  entity = ndb.Key('CachedPickledString', namespaced_key).get(
101      read_policy=ndb.EVENTUAL_CONSISTENCY)
102  if entity:
103    return cPickle.loads(entity.value)
104  return None
105
106
107def GetExternal(key):
108  """Gets the value from the datastore for the externally namespaced key."""
109  namespaced_key = _NamespaceKey(key, datastore_hooks.EXTERNAL)
110  entity = ndb.Key('CachedPickledString', namespaced_key).get(
111      read_policy=ndb.EVENTUAL_CONSISTENCY)
112  if entity:
113    return cPickle.loads(entity.value)
114  return None
115
116
117def Set(key, value, days_to_keep=None, namespace=None):
118  """Sets the value in the datastore.
119
120  Args:
121    key: The key name, which will be namespaced.
122    value: The value to set.
123    days_to_keep: Number of days to keep entity in datastore, default is None.
124    Entity will not expire when this value is 0 or None.
125    namespace: Optional namespace, otherwise namespace will be retrieved
126        using datastore_hooks.GetNamespace().
127  """
128  # When number of days to keep is given, calculate expiration time for
129  # the entity and store it in datastore.
130  # Once the entity expires, it will be deleted from the datastore.
131  expire_time = None
132  if days_to_keep:
133    expire_time = datetime.datetime.now() + datetime.timedelta(
134        days=days_to_keep)
135  namespaced_key = _NamespaceKey(key, namespace)
136
137  try:
138    CachedPickledString(id=namespaced_key,
139                        value=cPickle.dumps(value),
140                        expire_time=expire_time).put()
141  except datastore_errors.BadRequestError as e:
142    logging.warning('BadRequestError for key %s: %s', key, e)
143
144
145def SetExternal(key, value, days_to_keep=None):
146  """Sets the value in the datastore for the externally namespaced key.
147
148  Needed for things like /add_point that update internal/external data at the
149  same time.
150
151  Args:
152    key: The key name, which will be namespaced as externally_visible.
153    value: The value to set.
154    days_to_keep: Number of days to keep entity in datastore, default is None.
155        Entity will not expire when this value is 0 or None.
156  """
157  Set(key, value, days_to_keep, datastore_hooks.EXTERNAL)
158
159
160def Delete(key):
161  """Clears the value from the datastore."""
162  internal_key = _NamespaceKey(key, namespace=datastore_hooks.INTERNAL)
163  external_key = _NamespaceKey(key, namespace=datastore_hooks.EXTERNAL)
164  ndb.delete_multi([ndb.Key('CachedPickledString', internal_key),
165                    ndb.Key('CachedPickledString', external_key)])
166
167
168def DeleteAllExpiredEntities():
169  """Deletes all expired entities from the datastore."""
170  ndb.delete_multi(CachedPickledString.GetExpiredKeys())
171