leveldb_database.cc revision 5d1f7b1de12d16ceb2c938c56701a3e8bfa558f7 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8
9#include "base/basictypes.h"
10#include "base/logging.h"
11#include "base/memory/scoped_ptr.h"
12#include "base/metrics/histogram.h"
13#include "base/strings/string16.h"
14#include "base/strings/string_piece.h"
15#include "base/strings/stringprintf.h"
16#include "base/strings/utf_string_conversions.h"
17#include "base/sys_info.h"
18#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
19#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
20#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
21#include "third_party/leveldatabase/env_chromium.h"
22#include "third_party/leveldatabase/env_idb.h"
23#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
24#include "third_party/leveldatabase/src/include/leveldb/comparator.h"
25#include "third_party/leveldatabase/src/include/leveldb/db.h"
26#include "third_party/leveldatabase/src/include/leveldb/env.h"
27#include "third_party/leveldatabase/src/include/leveldb/slice.h"
28
29using base::StringPiece;
30
31namespace content {
32
33// Forcing flushes to disk at the end of a transaction guarantees that the
34// data hit disk, but drastically impacts throughput when the filesystem is
35// busy with background compactions. Not syncing trades off reliability for
36// performance. Note that background compactions which move data from the
37// log to SSTs are always done with reliable writes.
38//
39// Sync writes are necessary on Windows for quota calculations; POSIX
40// calculates file sizes correctly even when not synced to disk.
41#if defined(OS_WIN)
42static const bool kSyncWrites = true;
43#else
44// TODO(dgrogan): Either remove the #if block or change this back to false.
45// See http://crbug.com/338385.
46static const bool kSyncWrites = true;
47#endif
48
49static leveldb::Slice MakeSlice(const StringPiece& s) {
50  return leveldb::Slice(s.begin(), s.size());
51}
52
53static StringPiece MakeStringPiece(const leveldb::Slice& s) {
54  return StringPiece(s.data(), s.size());
55}
56
57class ComparatorAdapter : public leveldb::Comparator {
58 public:
59  explicit ComparatorAdapter(const LevelDBComparator* comparator)
60      : comparator_(comparator) {}
61
62  virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
63      OVERRIDE {
64    return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
65  }
66
67  virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
68
69  // TODO(jsbell): Support the methods below in the future.
70  virtual void FindShortestSeparator(std::string* start,
71                                     const leveldb::Slice& limit) const
72      OVERRIDE {}
73  virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
74
75 private:
76  const LevelDBComparator* comparator_;
77};
78
79LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
80    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
81
82LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
83
84LevelDBDatabase::LevelDBDatabase() {}
85
86LevelDBDatabase::~LevelDBDatabase() {
87  // db_'s destructor uses comparator_adapter_; order of deletion is important.
88  db_.reset();
89  comparator_adapter_.reset();
90  env_.reset();
91}
92
93static leveldb::Status OpenDB(leveldb::Comparator* comparator,
94                              leveldb::Env* env,
95                              const base::FilePath& path,
96                              leveldb::DB** db) {
97  leveldb::Options options;
98  options.comparator = comparator;
99  options.create_if_missing = true;
100  options.paranoid_checks = true;
101  options.compression = leveldb::kSnappyCompression;
102
103  // For info about the troubles we've run into with this parameter, see:
104  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
105  options.max_open_files = 80;
106  options.env = env;
107
108  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
109  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
110}
111
112bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
113  leveldb::Options options;
114  options.env = leveldb::IDBEnv();
115  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
116  const leveldb::Status s =
117      leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
118  return s.ok();
119}
120
121namespace {
122class LockImpl : public LevelDBLock {
123 public:
124  explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
125      : env_(env), lock_(lock) {}
126  virtual ~LockImpl() { env_->UnlockFile(lock_); }
127 private:
128  leveldb::Env* env_;
129  leveldb::FileLock* lock_;
130};
131}
132
133scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
134    const base::FilePath& file_name) {
135  leveldb::Env* env = leveldb::IDBEnv();
136  base::FilePath lock_path = file_name.AppendASCII("LOCK");
137  leveldb::FileLock* lock = NULL;
138  leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
139  if (!status.ok())
140    return scoped_ptr<LevelDBLock>();
141  DCHECK(lock);
142  return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
143}
144
145static int CheckFreeSpace(const char* const type,
146                          const base::FilePath& file_name) {
147  std::string name =
148      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
149  int64 free_disk_space_in_k_bytes =
150      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
151  if (free_disk_space_in_k_bytes < 0) {
152    base::Histogram::FactoryGet(
153        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
154        1,
155        2 /*boundary*/,
156        2 /*boundary*/ + 1,
157        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
158    return -1;
159  }
160  int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
161                                       ? INT_MAX
162                                       : free_disk_space_in_k_bytes;
163  const uint64 histogram_max = static_cast<uint64>(1e9);
164  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
165  base::Histogram::FactoryGet(name,
166                              1,
167                              histogram_max,
168                              11 /*buckets*/,
169                              base::HistogramBase::kUmaTargetedHistogramFlag)
170      ->Add(clamped_disk_space_k_bytes);
171  return clamped_disk_space_k_bytes;
172}
173
174static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
175                                            const leveldb::Status& s) {
176  leveldb_env::MethodID method;
177  int error = -1;
178  leveldb_env::ErrorParsingResult result =
179      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
180  if (result == leveldb_env::NONE)
181    return;
182  std::string method_histogram_name(histogram_name);
183  method_histogram_name.append(".EnvMethod");
184  base::LinearHistogram::FactoryGet(
185      method_histogram_name,
186      1,
187      leveldb_env::kNumEntries,
188      leveldb_env::kNumEntries + 1,
189      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
190
191  std::string error_histogram_name(histogram_name);
192
193  if (result == leveldb_env::METHOD_AND_PFE) {
194    DCHECK(error < 0);
195    error_histogram_name.append(std::string(".PFE.") +
196                                leveldb_env::MethodIDToString(method));
197    base::LinearHistogram::FactoryGet(
198        error_histogram_name,
199        1,
200        -base::PLATFORM_FILE_ERROR_MAX,
201        -base::PLATFORM_FILE_ERROR_MAX + 1,
202        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
203  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
204    error_histogram_name.append(std::string(".Errno.") +
205                                leveldb_env::MethodIDToString(method));
206    base::LinearHistogram::FactoryGet(
207        error_histogram_name,
208        1,
209        ERANGE + 1,
210        ERANGE + 2,
211        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
212  }
213}
214
215static void ParseAndHistogramCorruptionDetails(
216    const std::string& histogram_name,
217    const leveldb::Status& status) {
218  int error = leveldb_env::GetCorruptionCode(status);
219  DCHECK(error >= 0);
220  std::string corruption_histogram_name(histogram_name);
221  corruption_histogram_name.append(".Corruption");
222  const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
223  base::LinearHistogram::FactoryGet(
224      corruption_histogram_name,
225      1,
226      kNumPatterns,
227      kNumPatterns + 1,
228      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
229}
230
231static void HistogramLevelDBError(const std::string& histogram_name,
232                                  const leveldb::Status& s) {
233  if (s.ok()) {
234    NOTREACHED();
235    return;
236  }
237  enum {
238    LEVEL_DB_NOT_FOUND,
239    LEVEL_DB_CORRUPTION,
240    LEVEL_DB_IO_ERROR,
241    LEVEL_DB_OTHER,
242    LEVEL_DB_MAX_ERROR
243  };
244  int leveldb_error = LEVEL_DB_OTHER;
245  if (s.IsNotFound())
246    leveldb_error = LEVEL_DB_NOT_FOUND;
247  else if (s.IsCorruption())
248    leveldb_error = LEVEL_DB_CORRUPTION;
249  else if (s.IsIOError())
250    leveldb_error = LEVEL_DB_IO_ERROR;
251  base::Histogram::FactoryGet(histogram_name,
252                              1,
253                              LEVEL_DB_MAX_ERROR,
254                              LEVEL_DB_MAX_ERROR + 1,
255                              base::HistogramBase::kUmaTargetedHistogramFlag)
256      ->Add(leveldb_error);
257  if (s.IsIOError())
258    ParseAndHistogramIOErrorDetails(histogram_name, s);
259  else
260    ParseAndHistogramCorruptionDetails(histogram_name, s);
261}
262
263leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
264                                      const LevelDBComparator* comparator,
265                                      scoped_ptr<LevelDBDatabase>* result,
266                                      bool* is_disk_full) {
267  scoped_ptr<ComparatorAdapter> comparator_adapter(
268      new ComparatorAdapter(comparator));
269
270  leveldb::DB* db;
271  const leveldb::Status s =
272      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
273
274  if (!s.ok()) {
275    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
276    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
277    // Disks with <100k of free space almost never succeed in opening a
278    // leveldb database.
279    if (is_disk_full)
280      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
281
282    LOG(ERROR) << "Failed to open LevelDB database from "
283               << file_name.AsUTF8Unsafe() << "," << s.ToString();
284    return s;
285  }
286
287  CheckFreeSpace("Success", file_name);
288
289  (*result).reset(new LevelDBDatabase);
290  (*result)->db_ = make_scoped_ptr(db);
291  (*result)->comparator_adapter_ = comparator_adapter.Pass();
292  (*result)->comparator_ = comparator;
293
294  return s;
295}
296
297scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
298    const LevelDBComparator* comparator) {
299  scoped_ptr<ComparatorAdapter> comparator_adapter(
300      new ComparatorAdapter(comparator));
301  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
302
303  leveldb::DB* db;
304  const leveldb::Status s = OpenDB(
305      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
306
307  if (!s.ok()) {
308    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
309    return scoped_ptr<LevelDBDatabase>();
310  }
311
312  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
313  result->env_ = in_memory_env.Pass();
314  result->db_ = make_scoped_ptr(db);
315  result->comparator_adapter_ = comparator_adapter.Pass();
316  result->comparator_ = comparator;
317
318  return result.Pass();
319}
320
321bool LevelDBDatabase::Put(const StringPiece& key, std::string* value) {
322  leveldb::WriteOptions write_options;
323  write_options.sync = kSyncWrites;
324
325  const leveldb::Status s =
326      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
327  if (s.ok())
328    return true;
329  LOG(ERROR) << "LevelDB put failed: " << s.ToString();
330  return false;
331}
332
333bool LevelDBDatabase::Remove(const StringPiece& key) {
334  leveldb::WriteOptions write_options;
335  write_options.sync = kSyncWrites;
336
337  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
338  if (s.ok())
339    return true;
340  if (s.IsNotFound())
341    return false;
342  LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
343  return false;
344}
345
346bool LevelDBDatabase::Get(const StringPiece& key,
347                          std::string* value,
348                          bool* found,
349                          const LevelDBSnapshot* snapshot) {
350  *found = false;
351  leveldb::ReadOptions read_options;
352  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
353                                         // performance impact is too great.
354  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
355
356  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
357  if (s.ok()) {
358    *found = true;
359    return true;
360  }
361  if (s.IsNotFound())
362    return true;
363  HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
364  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
365  return false;
366}
367
368bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
369  leveldb::WriteOptions write_options;
370  write_options.sync = kSyncWrites;
371
372  const leveldb::Status s =
373      db_->Write(write_options, write_batch.write_batch_.get());
374  if (s.ok())
375    return true;
376  HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
377  LOG(ERROR) << "LevelDB write failed: " << s.ToString();
378  return false;
379}
380
381namespace {
382class IteratorImpl : public LevelDBIterator {
383 public:
384  virtual ~IteratorImpl() {}
385
386  virtual bool IsValid() const OVERRIDE;
387  virtual void SeekToLast() OVERRIDE;
388  virtual void Seek(const StringPiece& target) OVERRIDE;
389  virtual void Next() OVERRIDE;
390  virtual void Prev() OVERRIDE;
391  virtual StringPiece Key() const OVERRIDE;
392  virtual StringPiece Value() const OVERRIDE;
393
394 private:
395  friend class content::LevelDBDatabase;
396  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
397  void CheckStatus();
398
399  scoped_ptr<leveldb::Iterator> iterator_;
400};
401}
402
403IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
404    : iterator_(it.Pass()) {}
405
406void IteratorImpl::CheckStatus() {
407  const leveldb::Status s = iterator_->status();
408  if (!s.ok())
409    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
410}
411
412bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
413
414void IteratorImpl::SeekToLast() {
415  iterator_->SeekToLast();
416  CheckStatus();
417}
418
419void IteratorImpl::Seek(const StringPiece& target) {
420  iterator_->Seek(MakeSlice(target));
421  CheckStatus();
422}
423
424void IteratorImpl::Next() {
425  DCHECK(IsValid());
426  iterator_->Next();
427  CheckStatus();
428}
429
430void IteratorImpl::Prev() {
431  DCHECK(IsValid());
432  iterator_->Prev();
433  CheckStatus();
434}
435
436StringPiece IteratorImpl::Key() const {
437  DCHECK(IsValid());
438  return MakeStringPiece(iterator_->key());
439}
440
441StringPiece IteratorImpl::Value() const {
442  DCHECK(IsValid());
443  return MakeStringPiece(iterator_->value());
444}
445
446scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
447    const LevelDBSnapshot* snapshot) {
448  leveldb::ReadOptions read_options;
449  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
450                                         // performance impact is too great.
451  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
452
453  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
454  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
455}
456
457const LevelDBComparator* LevelDBDatabase::Comparator() const {
458  return comparator_;
459}
460
461void LevelDBDatabase::Compact(const base::StringPiece& start,
462                              const base::StringPiece& stop) {
463  const leveldb::Slice start_slice = MakeSlice(start);
464  const leveldb::Slice stop_slice = MakeSlice(stop);
465  db_->CompactRange(&start_slice, &stop_slice);
466}
467
468}  // namespace content
469