1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Caches processed query results in memcache and datastore. 6 7Memcache is not very reliable for the perf dashboard. Prometheus team explained 8that memcache is LRU and shared between multiple applications, so their activity 9may result in our data being evicted. To prevent this, we cache processed 10query results in the data store. Using NDB, the values are also cached in 11memcache if possible. This improves performance because doing a get() 12for a key which has a single BlobProperty is much quicker than a complex query 13over a large dataset. 14(Background: http://g/prometheus-discuss/othVtufGIyM/wjAS5djyG8kJ) 15 16When an item is cached, layered_cache does the following: 171) Namespaces the key based on whether datastore_hooks says the request is 18internal_only. 192) Pickles the value (memcache does this internally), and adds a data store 20entity with the key and a BlobProperty with the pickled value. 21 22Retrieving values checks memcache via NDB first, and if datastore is used it 23unpickles. 24 25When an item is removed from the the cache, it is removed from both internal and 26external caches, since removals are usually caused by large changes that affect 27both caches. 28 29Although this module contains ndb.Model classes, these are not intended 30to be used directly by other modules. 31""" 32 33import cPickle 34import datetime 35import logging 36 37from google.appengine.api import datastore_errors 38from google.appengine.ext import ndb 39 40from dashboard import datastore_hooks 41from dashboard import request_handler 42 43 44class DeleteExpiredEntitiesHandler(request_handler.RequestHandler): 45 """URL endpoint for a cron job to delete expired entities from datastore.""" 46 47 def get(self): 48 """This get handler is called from cron. 49 50 It deletes only expired CachedPickledString entities from the datastore. 51 """ 52 DeleteAllExpiredEntities() 53 54 55class CachedPickledString(ndb.Model): 56 value = ndb.BlobProperty() 57 expire_time = ndb.DateTimeProperty() 58 59 @classmethod 60 def GetExpiredKeys(cls): 61 """Gets keys of expired entities. 62 63 Returns: 64 List of keys for items which are expired. 65 """ 66 current_time = datetime.datetime.now() 67 query = cls.query(cls.expire_time < current_time) 68 query = query.filter(cls.expire_time != None) 69 return query.fetch(keys_only=True) 70 71 72def _NamespaceKey(key, namespace=None): 73 if not namespace: 74 namespace = datastore_hooks.GetNamespace() 75 return '%s__%s' % (namespace, key) 76 77 78def Prewarm(keys): 79 """Prewarms the NDB in-context cache by doing async_get for the keys. 80 81 For requests like /add_point which can get/set dozens of keys, contention 82 occasionally causes the gets to take several seconds. But they will be 83 cached in context by NDB if they are requested at the start of the request. 84 85 Args: 86 keys: List of string keys. 87 """ 88 to_get = [] 89 for key in keys: 90 to_get.append(ndb.Key('CachedPickledString', 91 _NamespaceKey(key, datastore_hooks.EXTERNAL))) 92 to_get.append(ndb.Key('CachedPickledString', 93 _NamespaceKey(key, datastore_hooks.INTERNAL))) 94 ndb.get_multi_async(to_get) 95 96 97def Get(key): 98 """Gets the value from the datastore.""" 99 namespaced_key = _NamespaceKey(key) 100 entity = ndb.Key('CachedPickledString', namespaced_key).get( 101 read_policy=ndb.EVENTUAL_CONSISTENCY) 102 if entity: 103 return cPickle.loads(entity.value) 104 return None 105 106 107def GetExternal(key): 108 """Gets the value from the datastore for the externally namespaced key.""" 109 namespaced_key = _NamespaceKey(key, datastore_hooks.EXTERNAL) 110 entity = ndb.Key('CachedPickledString', namespaced_key).get( 111 read_policy=ndb.EVENTUAL_CONSISTENCY) 112 if entity: 113 return cPickle.loads(entity.value) 114 return None 115 116 117def Set(key, value, days_to_keep=None, namespace=None): 118 """Sets the value in the datastore. 119 120 Args: 121 key: The key name, which will be namespaced. 122 value: The value to set. 123 days_to_keep: Number of days to keep entity in datastore, default is None. 124 Entity will not expire when this value is 0 or None. 125 namespace: Optional namespace, otherwise namespace will be retrieved 126 using datastore_hooks.GetNamespace(). 127 """ 128 # When number of days to keep is given, calculate expiration time for 129 # the entity and store it in datastore. 130 # Once the entity expires, it will be deleted from the datastore. 131 expire_time = None 132 if days_to_keep: 133 expire_time = datetime.datetime.now() + datetime.timedelta( 134 days=days_to_keep) 135 namespaced_key = _NamespaceKey(key, namespace) 136 137 try: 138 CachedPickledString(id=namespaced_key, 139 value=cPickle.dumps(value), 140 expire_time=expire_time).put() 141 except datastore_errors.BadRequestError as e: 142 logging.warning('BadRequestError for key %s: %s', key, e) 143 144 145def SetExternal(key, value, days_to_keep=None): 146 """Sets the value in the datastore for the externally namespaced key. 147 148 Needed for things like /add_point that update internal/external data at the 149 same time. 150 151 Args: 152 key: The key name, which will be namespaced as externally_visible. 153 value: The value to set. 154 days_to_keep: Number of days to keep entity in datastore, default is None. 155 Entity will not expire when this value is 0 or None. 156 """ 157 Set(key, value, days_to_keep, datastore_hooks.EXTERNAL) 158 159 160def Delete(key): 161 """Clears the value from the datastore.""" 162 internal_key = _NamespaceKey(key, namespace=datastore_hooks.INTERNAL) 163 external_key = _NamespaceKey(key, namespace=datastore_hooks.EXTERNAL) 164 ndb.delete_multi([ndb.Key('CachedPickledString', internal_key), 165 ndb.Key('CachedPickledString', external_key)]) 166 167 168def DeleteAllExpiredEntities(): 169 """Deletes all expired entities from the datastore.""" 170 ndb.delete_multi(CachedPickledString.GetExpiredKeys()) 171