1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.volley.toolbox; 18 19import android.os.SystemClock; 20 21import com.android.volley.Cache; 22import com.android.volley.VolleyLog; 23 24import java.io.BufferedInputStream; 25import java.io.BufferedOutputStream; 26import java.io.EOFException; 27import java.io.File; 28import java.io.FileInputStream; 29import java.io.FileOutputStream; 30import java.io.FilterInputStream; 31import java.io.IOException; 32import java.io.InputStream; 33import java.io.OutputStream; 34import java.util.Collections; 35import java.util.HashMap; 36import java.util.Iterator; 37import java.util.LinkedHashMap; 38import java.util.Map; 39 40/** 41 * Cache implementation that caches files directly onto the hard disk in the specified 42 * directory. The default disk usage size is 5MB, but is configurable. 43 */ 44public class DiskBasedCache implements Cache { 45 46 /** Map of the Key, CacheHeader pairs */ 47 private final Map<String, CacheHeader> mEntries = 48 new LinkedHashMap<String, CacheHeader>(16, .75f, true); 49 50 /** Total amount of space currently used by the cache in bytes. */ 51 private long mTotalSize = 0; 52 53 /** The root directory to use for the cache. */ 54 private final File mRootDirectory; 55 56 /** The maximum size of the cache in bytes. */ 57 private final int mMaxCacheSizeInBytes; 58 59 /** Default maximum disk usage in bytes. */ 60 private static final int DEFAULT_DISK_USAGE_BYTES = 5 * 1024 * 1024; 61 62 /** High water mark percentage for the cache */ 63 private static final float HYSTERESIS_FACTOR = 0.9f; 64 65 /** Magic number for current version of cache file format. */ 66 private static final int CACHE_MAGIC = 0x20150306; 67 68 /** 69 * Constructs an instance of the DiskBasedCache at the specified directory. 70 * @param rootDirectory The root directory of the cache. 71 * @param maxCacheSizeInBytes The maximum size of the cache in bytes. 72 */ 73 public DiskBasedCache(File rootDirectory, int maxCacheSizeInBytes) { 74 mRootDirectory = rootDirectory; 75 mMaxCacheSizeInBytes = maxCacheSizeInBytes; 76 } 77 78 /** 79 * Constructs an instance of the DiskBasedCache at the specified directory using 80 * the default maximum cache size of 5MB. 81 * @param rootDirectory The root directory of the cache. 82 */ 83 public DiskBasedCache(File rootDirectory) { 84 this(rootDirectory, DEFAULT_DISK_USAGE_BYTES); 85 } 86 87 /** 88 * Clears the cache. Deletes all cached files from disk. 89 */ 90 @Override 91 public synchronized void clear() { 92 File[] files = mRootDirectory.listFiles(); 93 if (files != null) { 94 for (File file : files) { 95 file.delete(); 96 } 97 } 98 mEntries.clear(); 99 mTotalSize = 0; 100 VolleyLog.d("Cache cleared."); 101 } 102 103 /** 104 * Returns the cache entry with the specified key if it exists, null otherwise. 105 */ 106 @Override 107 public synchronized Entry get(String key) { 108 CacheHeader entry = mEntries.get(key); 109 // if the entry does not exist, return. 110 if (entry == null) { 111 return null; 112 } 113 114 File file = getFileForKey(key); 115 CountingInputStream cis = null; 116 try { 117 cis = new CountingInputStream(new BufferedInputStream(new FileInputStream(file))); 118 CacheHeader.readHeader(cis); // eat header 119 byte[] data = streamToBytes(cis, (int) (file.length() - cis.bytesRead)); 120 return entry.toCacheEntry(data); 121 } catch (IOException e) { 122 VolleyLog.d("%s: %s", file.getAbsolutePath(), e.toString()); 123 remove(key); 124 return null; 125 } catch (NegativeArraySizeException e) { 126 VolleyLog.d("%s: %s", file.getAbsolutePath(), e.toString()); 127 remove(key); 128 return null; 129 } finally { 130 if (cis != null) { 131 try { 132 cis.close(); 133 } catch (IOException ioe) { 134 return null; 135 } 136 } 137 } 138 } 139 140 /** 141 * Initializes the DiskBasedCache by scanning for all files currently in the 142 * specified root directory. Creates the root directory if necessary. 143 */ 144 @Override 145 public synchronized void initialize() { 146 if (!mRootDirectory.exists()) { 147 if (!mRootDirectory.mkdirs()) { 148 VolleyLog.e("Unable to create cache dir %s", mRootDirectory.getAbsolutePath()); 149 } 150 return; 151 } 152 153 File[] files = mRootDirectory.listFiles(); 154 if (files == null) { 155 return; 156 } 157 for (File file : files) { 158 BufferedInputStream fis = null; 159 try { 160 fis = new BufferedInputStream(new FileInputStream(file)); 161 CacheHeader entry = CacheHeader.readHeader(fis); 162 entry.size = file.length(); 163 putEntry(entry.key, entry); 164 } catch (IOException e) { 165 if (file != null) { 166 file.delete(); 167 } 168 } finally { 169 try { 170 if (fis != null) { 171 fis.close(); 172 } 173 } catch (IOException ignored) { } 174 } 175 } 176 } 177 178 /** 179 * Invalidates an entry in the cache. 180 * @param key Cache key 181 * @param fullExpire True to fully expire the entry, false to soft expire 182 */ 183 @Override 184 public synchronized void invalidate(String key, boolean fullExpire) { 185 Entry entry = get(key); 186 if (entry != null) { 187 entry.softTtl = 0; 188 if (fullExpire) { 189 entry.ttl = 0; 190 } 191 put(key, entry); 192 } 193 194 } 195 196 /** 197 * Puts the entry with the specified key into the cache. 198 */ 199 @Override 200 public synchronized void put(String key, Entry entry) { 201 pruneIfNeeded(entry.data.length); 202 File file = getFileForKey(key); 203 try { 204 BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(file)); 205 CacheHeader e = new CacheHeader(key, entry); 206 boolean success = e.writeHeader(fos); 207 if (!success) { 208 fos.close(); 209 VolleyLog.d("Failed to write header for %s", file.getAbsolutePath()); 210 throw new IOException(); 211 } 212 fos.write(entry.data); 213 fos.close(); 214 putEntry(key, e); 215 return; 216 } catch (IOException e) { 217 } 218 boolean deleted = file.delete(); 219 if (!deleted) { 220 VolleyLog.d("Could not clean up file %s", file.getAbsolutePath()); 221 } 222 } 223 224 /** 225 * Removes the specified key from the cache if it exists. 226 */ 227 @Override 228 public synchronized void remove(String key) { 229 boolean deleted = getFileForKey(key).delete(); 230 removeEntry(key); 231 if (!deleted) { 232 VolleyLog.d("Could not delete cache entry for key=%s, filename=%s", 233 key, getFilenameForKey(key)); 234 } 235 } 236 237 /** 238 * Creates a pseudo-unique filename for the specified cache key. 239 * @param key The key to generate a file name for. 240 * @return A pseudo-unique filename. 241 */ 242 private String getFilenameForKey(String key) { 243 int firstHalfLength = key.length() / 2; 244 String localFilename = String.valueOf(key.substring(0, firstHalfLength).hashCode()); 245 localFilename += String.valueOf(key.substring(firstHalfLength).hashCode()); 246 return localFilename; 247 } 248 249 /** 250 * Returns a file object for the given cache key. 251 */ 252 public File getFileForKey(String key) { 253 return new File(mRootDirectory, getFilenameForKey(key)); 254 } 255 256 /** 257 * Prunes the cache to fit the amount of bytes specified. 258 * @param neededSpace The amount of bytes we are trying to fit into the cache. 259 */ 260 private void pruneIfNeeded(int neededSpace) { 261 if ((mTotalSize + neededSpace) < mMaxCacheSizeInBytes) { 262 return; 263 } 264 if (VolleyLog.DEBUG) { 265 VolleyLog.v("Pruning old cache entries."); 266 } 267 268 long before = mTotalSize; 269 int prunedFiles = 0; 270 long startTime = SystemClock.elapsedRealtime(); 271 272 Iterator<Map.Entry<String, CacheHeader>> iterator = mEntries.entrySet().iterator(); 273 while (iterator.hasNext()) { 274 Map.Entry<String, CacheHeader> entry = iterator.next(); 275 CacheHeader e = entry.getValue(); 276 boolean deleted = getFileForKey(e.key).delete(); 277 if (deleted) { 278 mTotalSize -= e.size; 279 } else { 280 VolleyLog.d("Could not delete cache entry for key=%s, filename=%s", 281 e.key, getFilenameForKey(e.key)); 282 } 283 iterator.remove(); 284 prunedFiles++; 285 286 if ((mTotalSize + neededSpace) < mMaxCacheSizeInBytes * HYSTERESIS_FACTOR) { 287 break; 288 } 289 } 290 291 if (VolleyLog.DEBUG) { 292 VolleyLog.v("pruned %d files, %d bytes, %d ms", 293 prunedFiles, (mTotalSize - before), SystemClock.elapsedRealtime() - startTime); 294 } 295 } 296 297 /** 298 * Puts the entry with the specified key into the cache. 299 * @param key The key to identify the entry by. 300 * @param entry The entry to cache. 301 */ 302 private void putEntry(String key, CacheHeader entry) { 303 if (!mEntries.containsKey(key)) { 304 mTotalSize += entry.size; 305 } else { 306 CacheHeader oldEntry = mEntries.get(key); 307 mTotalSize += (entry.size - oldEntry.size); 308 } 309 mEntries.put(key, entry); 310 } 311 312 /** 313 * Removes the entry identified by 'key' from the cache. 314 */ 315 private void removeEntry(String key) { 316 CacheHeader entry = mEntries.get(key); 317 if (entry != null) { 318 mTotalSize -= entry.size; 319 mEntries.remove(key); 320 } 321 } 322 323 /** 324 * Reads the contents of an InputStream into a byte[]. 325 * */ 326 private static byte[] streamToBytes(InputStream in, int length) throws IOException { 327 byte[] bytes = new byte[length]; 328 int count; 329 int pos = 0; 330 while (pos < length && ((count = in.read(bytes, pos, length - pos)) != -1)) { 331 pos += count; 332 } 333 if (pos != length) { 334 throw new IOException("Expected " + length + " bytes, read " + pos + " bytes"); 335 } 336 return bytes; 337 } 338 339 /** 340 * Handles holding onto the cache headers for an entry. 341 */ 342 // Visible for testing. 343 static class CacheHeader { 344 /** The size of the data identified by this CacheHeader. (This is not 345 * serialized to disk. */ 346 public long size; 347 348 /** The key that identifies the cache entry. */ 349 public String key; 350 351 /** ETag for cache coherence. */ 352 public String etag; 353 354 /** Date of this response as reported by the server. */ 355 public long serverDate; 356 357 /** The last modified date for the requested object. */ 358 public long lastModified; 359 360 /** TTL for this record. */ 361 public long ttl; 362 363 /** Soft TTL for this record. */ 364 public long softTtl; 365 366 /** Headers from the response resulting in this cache entry. */ 367 public Map<String, String> responseHeaders; 368 369 private CacheHeader() { } 370 371 /** 372 * Instantiates a new CacheHeader object 373 * @param key The key that identifies the cache entry 374 * @param entry The cache entry. 375 */ 376 public CacheHeader(String key, Entry entry) { 377 this.key = key; 378 this.size = entry.data.length; 379 this.etag = entry.etag; 380 this.serverDate = entry.serverDate; 381 this.lastModified = entry.lastModified; 382 this.ttl = entry.ttl; 383 this.softTtl = entry.softTtl; 384 this.responseHeaders = entry.responseHeaders; 385 } 386 387 /** 388 * Reads the header off of an InputStream and returns a CacheHeader object. 389 * @param is The InputStream to read from. 390 * @throws IOException 391 */ 392 public static CacheHeader readHeader(InputStream is) throws IOException { 393 CacheHeader entry = new CacheHeader(); 394 int magic = readInt(is); 395 if (magic != CACHE_MAGIC) { 396 // don't bother deleting, it'll get pruned eventually 397 throw new IOException(); 398 } 399 entry.key = readString(is); 400 entry.etag = readString(is); 401 if (entry.etag.equals("")) { 402 entry.etag = null; 403 } 404 entry.serverDate = readLong(is); 405 entry.lastModified = readLong(is); 406 entry.ttl = readLong(is); 407 entry.softTtl = readLong(is); 408 entry.responseHeaders = readStringStringMap(is); 409 410 return entry; 411 } 412 413 /** 414 * Creates a cache entry for the specified data. 415 */ 416 public Entry toCacheEntry(byte[] data) { 417 Entry e = new Entry(); 418 e.data = data; 419 e.etag = etag; 420 e.serverDate = serverDate; 421 e.lastModified = lastModified; 422 e.ttl = ttl; 423 e.softTtl = softTtl; 424 e.responseHeaders = responseHeaders; 425 return e; 426 } 427 428 429 /** 430 * Writes the contents of this CacheHeader to the specified OutputStream. 431 */ 432 public boolean writeHeader(OutputStream os) { 433 try { 434 writeInt(os, CACHE_MAGIC); 435 writeString(os, key); 436 writeString(os, etag == null ? "" : etag); 437 writeLong(os, serverDate); 438 writeLong(os, lastModified); 439 writeLong(os, ttl); 440 writeLong(os, softTtl); 441 writeStringStringMap(responseHeaders, os); 442 os.flush(); 443 return true; 444 } catch (IOException e) { 445 VolleyLog.d("%s", e.toString()); 446 return false; 447 } 448 } 449 450 } 451 452 private static class CountingInputStream extends FilterInputStream { 453 private int bytesRead = 0; 454 455 private CountingInputStream(InputStream in) { 456 super(in); 457 } 458 459 @Override 460 public int read() throws IOException { 461 int result = super.read(); 462 if (result != -1) { 463 bytesRead++; 464 } 465 return result; 466 } 467 468 @Override 469 public int read(byte[] buffer, int offset, int count) throws IOException { 470 int result = super.read(buffer, offset, count); 471 if (result != -1) { 472 bytesRead += result; 473 } 474 return result; 475 } 476 } 477 478 /* 479 * Homebrewed simple serialization system used for reading and writing cache 480 * headers on disk. Once upon a time, this used the standard Java 481 * Object{Input,Output}Stream, but the default implementation relies heavily 482 * on reflection (even for standard types) and generates a ton of garbage. 483 */ 484 485 /** 486 * Simple wrapper around {@link InputStream#read()} that throws EOFException 487 * instead of returning -1. 488 */ 489 private static int read(InputStream is) throws IOException { 490 int b = is.read(); 491 if (b == -1) { 492 throw new EOFException(); 493 } 494 return b; 495 } 496 497 static void writeInt(OutputStream os, int n) throws IOException { 498 os.write((n >> 0) & 0xff); 499 os.write((n >> 8) & 0xff); 500 os.write((n >> 16) & 0xff); 501 os.write((n >> 24) & 0xff); 502 } 503 504 static int readInt(InputStream is) throws IOException { 505 int n = 0; 506 n |= (read(is) << 0); 507 n |= (read(is) << 8); 508 n |= (read(is) << 16); 509 n |= (read(is) << 24); 510 return n; 511 } 512 513 static void writeLong(OutputStream os, long n) throws IOException { 514 os.write((byte)(n >>> 0)); 515 os.write((byte)(n >>> 8)); 516 os.write((byte)(n >>> 16)); 517 os.write((byte)(n >>> 24)); 518 os.write((byte)(n >>> 32)); 519 os.write((byte)(n >>> 40)); 520 os.write((byte)(n >>> 48)); 521 os.write((byte)(n >>> 56)); 522 } 523 524 static long readLong(InputStream is) throws IOException { 525 long n = 0; 526 n |= ((read(is) & 0xFFL) << 0); 527 n |= ((read(is) & 0xFFL) << 8); 528 n |= ((read(is) & 0xFFL) << 16); 529 n |= ((read(is) & 0xFFL) << 24); 530 n |= ((read(is) & 0xFFL) << 32); 531 n |= ((read(is) & 0xFFL) << 40); 532 n |= ((read(is) & 0xFFL) << 48); 533 n |= ((read(is) & 0xFFL) << 56); 534 return n; 535 } 536 537 static void writeString(OutputStream os, String s) throws IOException { 538 byte[] b = s.getBytes("UTF-8"); 539 writeLong(os, b.length); 540 os.write(b, 0, b.length); 541 } 542 543 static String readString(InputStream is) throws IOException { 544 int n = (int) readLong(is); 545 byte[] b = streamToBytes(is, n); 546 return new String(b, "UTF-8"); 547 } 548 549 static void writeStringStringMap(Map<String, String> map, OutputStream os) throws IOException { 550 if (map != null) { 551 writeInt(os, map.size()); 552 for (Map.Entry<String, String> entry : map.entrySet()) { 553 writeString(os, entry.getKey()); 554 writeString(os, entry.getValue()); 555 } 556 } else { 557 writeInt(os, 0); 558 } 559 } 560 561 static Map<String, String> readStringStringMap(InputStream is) throws IOException { 562 int size = readInt(is); 563 Map<String, String> result = (size == 0) 564 ? Collections.<String, String>emptyMap() 565 : new HashMap<String, String>(size); 566 for (int i = 0; i < size; i++) { 567 String key = readString(is).intern(); 568 String value = readString(is).intern(); 569 result.put(key, value); 570 } 571 return result; 572 } 573 574 575} 576