DiskBasedCache.java revision b9b8dc3d98fb1a8c3f02c2c2fcc18cbd344c05cb
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.volley.toolbox; 18 19import android.os.SystemClock; 20 21import com.android.volley.Cache; 22import com.android.volley.VolleyLog; 23 24import java.io.BufferedInputStream; 25import java.io.EOFException; 26import java.io.File; 27import java.io.FileInputStream; 28import java.io.FileOutputStream; 29import java.io.FilterInputStream; 30import java.io.IOException; 31import java.io.InputStream; 32import java.io.OutputStream; 33import java.util.Collections; 34import java.util.HashMap; 35import java.util.Iterator; 36import java.util.LinkedHashMap; 37import java.util.Map; 38 39/** 40 * Cache implementation that caches files directly onto the hard disk in the specified 41 * directory. The default disk usage size is 5MB, but is configurable. 42 */ 43public class DiskBasedCache implements Cache { 44 45 /** Map of the Key, CacheHeader pairs */ 46 private final Map<String, CacheHeader> mEntries = 47 new LinkedHashMap<String, CacheHeader>(16, .75f, true); 48 49 /** Total amount of space currently used by the cache in bytes. */ 50 private long mTotalSize = 0; 51 52 /** The root directory to use for the cache. */ 53 private final File mRootDirectory; 54 55 /** The maximum size of the cache in bytes. */ 56 private final int mMaxCacheSizeInBytes; 57 58 /** Default maximum disk usage in bytes. */ 59 private static final int DEFAULT_DISK_USAGE_BYTES = 5 * 1024 * 1024; 60 61 /** High water mark percentage for the cache */ 62 private static final float HYSTERESIS_FACTOR = 0.9f; 63 64 /** Magic number for current version of cache file format. */ 65 private static final int CACHE_MAGIC = 0x20140623; 66 67 /** 68 * Constructs an instance of the DiskBasedCache at the specified directory. 69 * @param rootDirectory The root directory of the cache. 70 * @param maxCacheSizeInBytes The maximum size of the cache in bytes. 71 */ 72 public DiskBasedCache(File rootDirectory, int maxCacheSizeInBytes) { 73 mRootDirectory = rootDirectory; 74 mMaxCacheSizeInBytes = maxCacheSizeInBytes; 75 } 76 77 /** 78 * Constructs an instance of the DiskBasedCache at the specified directory using 79 * the default maximum cache size of 5MB. 80 * @param rootDirectory The root directory of the cache. 81 */ 82 public DiskBasedCache(File rootDirectory) { 83 this(rootDirectory, DEFAULT_DISK_USAGE_BYTES); 84 } 85 86 /** 87 * Clears the cache. Deletes all cached files from disk. 88 */ 89 @Override 90 public synchronized void clear() { 91 File[] files = mRootDirectory.listFiles(); 92 if (files != null) { 93 for (File file : files) { 94 file.delete(); 95 } 96 } 97 mEntries.clear(); 98 mTotalSize = 0; 99 VolleyLog.d("Cache cleared."); 100 } 101 102 /** 103 * Returns the cache entry with the specified key if it exists, null otherwise. 104 */ 105 @Override 106 public synchronized Entry get(String key) { 107 CacheHeader entry = mEntries.get(key); 108 // if the entry does not exist, return. 109 if (entry == null) { 110 return null; 111 } 112 113 File file = getFileForKey(key); 114 CountingInputStream cis = null; 115 try { 116 cis = new CountingInputStream(new FileInputStream(file)); 117 CacheHeader.readHeader(cis); // eat header 118 byte[] data = streamToBytes(cis, (int) (file.length() - cis.bytesRead)); 119 return entry.toCacheEntry(data); 120 } catch (IOException e) { 121 VolleyLog.d("%s: %s", file.getAbsolutePath(), e.toString()); 122 remove(key); 123 return null; 124 } finally { 125 if (cis != null) { 126 try { 127 cis.close(); 128 } catch (IOException ioe) { 129 return null; 130 } 131 } 132 } 133 } 134 135 /** 136 * Initializes the DiskBasedCache by scanning for all files currently in the 137 * specified root directory. Creates the root directory if necessary. 138 */ 139 @Override 140 public synchronized void initialize() { 141 if (!mRootDirectory.exists()) { 142 if (!mRootDirectory.mkdirs()) { 143 VolleyLog.e("Unable to create cache dir %s", mRootDirectory.getAbsolutePath()); 144 } 145 return; 146 } 147 148 File[] files = mRootDirectory.listFiles(); 149 if (files == null) { 150 return; 151 } 152 for (File file : files) { 153 BufferedInputStream fis = null; 154 try { 155 fis = new BufferedInputStream(new FileInputStream(file)); 156 CacheHeader entry = CacheHeader.readHeader(fis); 157 entry.size = file.length(); 158 putEntry(entry.key, entry); 159 } catch (IOException e) { 160 if (file != null) { 161 file.delete(); 162 } 163 } finally { 164 try { 165 if (fis != null) { 166 fis.close(); 167 } 168 } catch (IOException ignored) { } 169 } 170 } 171 } 172 173 /** 174 * Invalidates an entry in the cache. 175 * @param key Cache key 176 * @param fullExpire True to fully expire the entry, false to soft expire 177 */ 178 @Override 179 public synchronized void invalidate(String key, boolean fullExpire) { 180 Entry entry = get(key); 181 if (entry != null) { 182 entry.softTtl = 0; 183 if (fullExpire) { 184 entry.ttl = 0; 185 } 186 put(key, entry); 187 } 188 189 } 190 191 /** 192 * Puts the entry with the specified key into the cache. 193 */ 194 @Override 195 public synchronized void put(String key, Entry entry) { 196 pruneIfNeeded(entry.data.length); 197 File file = getFileForKey(key); 198 try { 199 FileOutputStream fos = new FileOutputStream(file); 200 CacheHeader e = new CacheHeader(key, entry); 201 boolean success = e.writeHeader(fos); 202 if (!success) { 203 fos.close(); 204 VolleyLog.d("Failed to write header for %s", file.getAbsolutePath()); 205 throw new IOException(); 206 } 207 fos.write(entry.data); 208 fos.close(); 209 putEntry(key, e); 210 return; 211 } catch (IOException e) { 212 } 213 boolean deleted = file.delete(); 214 if (!deleted) { 215 VolleyLog.d("Could not clean up file %s", file.getAbsolutePath()); 216 } 217 } 218 219 /** 220 * Removes the specified key from the cache if it exists. 221 */ 222 @Override 223 public synchronized void remove(String key) { 224 boolean deleted = getFileForKey(key).delete(); 225 removeEntry(key); 226 if (!deleted) { 227 VolleyLog.d("Could not delete cache entry for key=%s, filename=%s", 228 key, getFilenameForKey(key)); 229 } 230 } 231 232 /** 233 * Creates a pseudo-unique filename for the specified cache key. 234 * @param key The key to generate a file name for. 235 * @return A pseudo-unique filename. 236 */ 237 private String getFilenameForKey(String key) { 238 int firstHalfLength = key.length() / 2; 239 String localFilename = String.valueOf(key.substring(0, firstHalfLength).hashCode()); 240 localFilename += String.valueOf(key.substring(firstHalfLength).hashCode()); 241 return localFilename; 242 } 243 244 /** 245 * Returns a file object for the given cache key. 246 */ 247 public File getFileForKey(String key) { 248 return new File(mRootDirectory, getFilenameForKey(key)); 249 } 250 251 /** 252 * Prunes the cache to fit the amount of bytes specified. 253 * @param neededSpace The amount of bytes we are trying to fit into the cache. 254 */ 255 private void pruneIfNeeded(int neededSpace) { 256 if ((mTotalSize + neededSpace) < mMaxCacheSizeInBytes) { 257 return; 258 } 259 if (VolleyLog.DEBUG) { 260 VolleyLog.v("Pruning old cache entries."); 261 } 262 263 long before = mTotalSize; 264 int prunedFiles = 0; 265 long startTime = SystemClock.elapsedRealtime(); 266 267 Iterator<Map.Entry<String, CacheHeader>> iterator = mEntries.entrySet().iterator(); 268 while (iterator.hasNext()) { 269 Map.Entry<String, CacheHeader> entry = iterator.next(); 270 CacheHeader e = entry.getValue(); 271 boolean deleted = getFileForKey(e.key).delete(); 272 if (deleted) { 273 mTotalSize -= e.size; 274 } else { 275 VolleyLog.d("Could not delete cache entry for key=%s, filename=%s", 276 e.key, getFilenameForKey(e.key)); 277 } 278 iterator.remove(); 279 prunedFiles++; 280 281 if ((mTotalSize + neededSpace) < mMaxCacheSizeInBytes * HYSTERESIS_FACTOR) { 282 break; 283 } 284 } 285 286 if (VolleyLog.DEBUG) { 287 VolleyLog.v("pruned %d files, %d bytes, %d ms", 288 prunedFiles, (mTotalSize - before), SystemClock.elapsedRealtime() - startTime); 289 } 290 } 291 292 /** 293 * Puts the entry with the specified key into the cache. 294 * @param key The key to identify the entry by. 295 * @param entry The entry to cache. 296 */ 297 private void putEntry(String key, CacheHeader entry) { 298 if (!mEntries.containsKey(key)) { 299 mTotalSize += entry.size; 300 } else { 301 CacheHeader oldEntry = mEntries.get(key); 302 mTotalSize += (entry.size - oldEntry.size); 303 } 304 mEntries.put(key, entry); 305 } 306 307 /** 308 * Removes the entry identified by 'key' from the cache. 309 */ 310 private void removeEntry(String key) { 311 CacheHeader entry = mEntries.get(key); 312 if (entry != null) { 313 mTotalSize -= entry.size; 314 mEntries.remove(key); 315 } 316 } 317 318 /** 319 * Reads the contents of an InputStream into a byte[]. 320 * */ 321 private static byte[] streamToBytes(InputStream in, int length) throws IOException { 322 byte[] bytes = new byte[length]; 323 int count; 324 int pos = 0; 325 while (pos < length && ((count = in.read(bytes, pos, length - pos)) != -1)) { 326 pos += count; 327 } 328 if (pos != length) { 329 throw new IOException("Expected " + length + " bytes, read " + pos + " bytes"); 330 } 331 return bytes; 332 } 333 334 /** 335 * Handles holding onto the cache headers for an entry. 336 */ 337 // Visible for testing. 338 static class CacheHeader { 339 /** The size of the data identified by this CacheHeader. (This is not 340 * serialized to disk. */ 341 public long size; 342 343 /** The key that identifies the cache entry. */ 344 public String key; 345 346 /** ETag for cache coherence. */ 347 public String etag; 348 349 /** Date of this response as reported by the server. */ 350 public long serverDate; 351 352 /** TTL for this record. */ 353 public long ttl; 354 355 /** Soft TTL for this record. */ 356 public long softTtl; 357 358 /** Headers from the response resulting in this cache entry. */ 359 public Map<String, String> responseHeaders; 360 361 private CacheHeader() { } 362 363 /** 364 * Instantiates a new CacheHeader object 365 * @param key The key that identifies the cache entry 366 * @param entry The cache entry. 367 */ 368 public CacheHeader(String key, Entry entry) { 369 this.key = key; 370 this.size = entry.data.length; 371 this.etag = entry.etag; 372 this.serverDate = entry.serverDate; 373 this.ttl = entry.ttl; 374 this.softTtl = entry.softTtl; 375 this.responseHeaders = entry.responseHeaders; 376 } 377 378 /** 379 * Reads the header off of an InputStream and returns a CacheHeader object. 380 * @param is The InputStream to read from. 381 * @throws IOException 382 */ 383 public static CacheHeader readHeader(InputStream is) throws IOException { 384 CacheHeader entry = new CacheHeader(); 385 int magic = readInt(is); 386 if (magic != CACHE_MAGIC) { 387 // don't bother deleting, it'll get pruned eventually 388 throw new IOException(); 389 } 390 entry.key = readString(is); 391 entry.etag = readString(is); 392 if (entry.etag.equals("")) { 393 entry.etag = null; 394 } 395 entry.serverDate = readLong(is); 396 entry.ttl = readLong(is); 397 entry.softTtl = readLong(is); 398 entry.responseHeaders = readStringStringMap(is); 399 return entry; 400 } 401 402 /** 403 * Creates a cache entry for the specified data. 404 */ 405 public Entry toCacheEntry(byte[] data) { 406 Entry e = new Entry(); 407 e.data = data; 408 e.etag = etag; 409 e.serverDate = serverDate; 410 e.ttl = ttl; 411 e.softTtl = softTtl; 412 e.responseHeaders = responseHeaders; 413 return e; 414 } 415 416 417 /** 418 * Writes the contents of this CacheHeader to the specified OutputStream. 419 */ 420 public boolean writeHeader(OutputStream os) { 421 try { 422 writeInt(os, CACHE_MAGIC); 423 writeString(os, key); 424 writeString(os, etag == null ? "" : etag); 425 writeLong(os, serverDate); 426 writeLong(os, ttl); 427 writeLong(os, softTtl); 428 writeStringStringMap(responseHeaders, os); 429 os.flush(); 430 return true; 431 } catch (IOException e) { 432 VolleyLog.d("%s", e.toString()); 433 return false; 434 } 435 } 436 437 } 438 439 private static class CountingInputStream extends FilterInputStream { 440 private int bytesRead = 0; 441 442 private CountingInputStream(InputStream in) { 443 super(in); 444 } 445 446 @Override 447 public int read() throws IOException { 448 int result = super.read(); 449 if (result != -1) { 450 bytesRead++; 451 } 452 return result; 453 } 454 455 @Override 456 public int read(byte[] buffer, int offset, int count) throws IOException { 457 int result = super.read(buffer, offset, count); 458 if (result != -1) { 459 bytesRead += result; 460 } 461 return result; 462 } 463 } 464 465 /* 466 * Homebrewed simple serialization system used for reading and writing cache 467 * headers on disk. Once upon a time, this used the standard Java 468 * Object{Input,Output}Stream, but the default implementation relies heavily 469 * on reflection (even for standard types) and generates a ton of garbage. 470 */ 471 472 /** 473 * Simple wrapper around {@link InputStream#read()} that throws EOFException 474 * instead of returning -1. 475 */ 476 private static int read(InputStream is) throws IOException { 477 int b = is.read(); 478 if (b == -1) { 479 throw new EOFException(); 480 } 481 return b; 482 } 483 484 static void writeInt(OutputStream os, int n) throws IOException { 485 os.write((n >> 0) & 0xff); 486 os.write((n >> 8) & 0xff); 487 os.write((n >> 16) & 0xff); 488 os.write((n >> 24) & 0xff); 489 } 490 491 static int readInt(InputStream is) throws IOException { 492 int n = 0; 493 n |= (read(is) << 0); 494 n |= (read(is) << 8); 495 n |= (read(is) << 16); 496 n |= (read(is) << 24); 497 return n; 498 } 499 500 static void writeLong(OutputStream os, long n) throws IOException { 501 os.write((byte)(n >>> 0)); 502 os.write((byte)(n >>> 8)); 503 os.write((byte)(n >>> 16)); 504 os.write((byte)(n >>> 24)); 505 os.write((byte)(n >>> 32)); 506 os.write((byte)(n >>> 40)); 507 os.write((byte)(n >>> 48)); 508 os.write((byte)(n >>> 56)); 509 } 510 511 static long readLong(InputStream is) throws IOException { 512 long n = 0; 513 n |= ((read(is) & 0xFFL) << 0); 514 n |= ((read(is) & 0xFFL) << 8); 515 n |= ((read(is) & 0xFFL) << 16); 516 n |= ((read(is) & 0xFFL) << 24); 517 n |= ((read(is) & 0xFFL) << 32); 518 n |= ((read(is) & 0xFFL) << 40); 519 n |= ((read(is) & 0xFFL) << 48); 520 n |= ((read(is) & 0xFFL) << 56); 521 return n; 522 } 523 524 static void writeString(OutputStream os, String s) throws IOException { 525 byte[] b = s.getBytes("UTF-8"); 526 writeLong(os, b.length); 527 os.write(b, 0, b.length); 528 } 529 530 static String readString(InputStream is) throws IOException { 531 int n = (int) readLong(is); 532 byte[] b = streamToBytes(is, n); 533 return new String(b, "UTF-8"); 534 } 535 536 static void writeStringStringMap(Map<String, String> map, OutputStream os) throws IOException { 537 if (map != null) { 538 writeInt(os, map.size()); 539 for (Map.Entry<String, String> entry : map.entrySet()) { 540 writeString(os, entry.getKey()); 541 writeString(os, entry.getValue()); 542 } 543 } else { 544 writeInt(os, 0); 545 } 546 } 547 548 static Map<String, String> readStringStringMap(InputStream is) throws IOException { 549 int size = readInt(is); 550 Map<String, String> result = (size == 0) 551 ? Collections.<String, String>emptyMap() 552 : new HashMap<String, String>(size); 553 for (int i = 0; i < size; i++) { 554 String key = readString(is).intern(); 555 String value = readString(is).intern(); 556 result.put(key, value); 557 } 558 return result; 559 } 560 561 562} 563