leveldb_database.cc revision 1e9bf3e0803691d0a228da41fc608347b6db4340 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8
9#include "base/basictypes.h"
10#include "base/logging.h"
11#include "base/memory/scoped_ptr.h"
12#include "base/metrics/histogram.h"
13#include "base/strings/string16.h"
14#include "base/strings/string_piece.h"
15#include "base/strings/stringprintf.h"
16#include "base/strings/utf_string_conversions.h"
17#include "base/sys_info.h"
18#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
19#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
20#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
21#include "third_party/leveldatabase/env_chromium.h"
22#include "third_party/leveldatabase/env_idb.h"
23#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
24#include "third_party/leveldatabase/src/include/leveldb/comparator.h"
25#include "third_party/leveldatabase/src/include/leveldb/db.h"
26#include "third_party/leveldatabase/src/include/leveldb/env.h"
27#include "third_party/leveldatabase/src/include/leveldb/slice.h"
28
29using base::StringPiece;
30
31namespace content {
32
33// Forcing flushes to disk at the end of a transaction guarantees that the
34// data hit disk, but drastically impacts throughput when the filesystem is
35// busy with background compactions. Not syncing trades off reliability for
36// performance. Note that background compactions which move data from the
37// log to SSTs are still done with reliable writes.
38//
39// Sync writes are necessary on Windows for quota calculations; POSIX
40// calculates file sizes correctly even when not synced to disk.
41#if defined(OS_WIN)
42static const bool kSyncWrites = true;
43#else
44static const bool kSyncWrites = false;
45#endif
46
47static leveldb::Slice MakeSlice(const StringPiece& s) {
48  return leveldb::Slice(s.begin(), s.size());
49}
50
51static StringPiece MakeStringPiece(const leveldb::Slice& s) {
52  return StringPiece(s.data(), s.size());
53}
54
55class ComparatorAdapter : public leveldb::Comparator {
56 public:
57  explicit ComparatorAdapter(const LevelDBComparator* comparator)
58      : comparator_(comparator) {}
59
60  virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
61      OVERRIDE {
62    return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
63  }
64
65  virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
66
67  // TODO(jsbell): Support the methods below in the future.
68  virtual void FindShortestSeparator(std::string* start,
69                                     const leveldb::Slice& limit) const
70      OVERRIDE {}
71  virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
72
73 private:
74  const LevelDBComparator* comparator_;
75};
76
77LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
78    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
79
80LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
81
82LevelDBDatabase::LevelDBDatabase() {}
83
84LevelDBDatabase::~LevelDBDatabase() {
85  // db_'s destructor uses comparator_adapter_; order of deletion is important.
86  db_.reset();
87  comparator_adapter_.reset();
88  env_.reset();
89}
90
91static leveldb::Status OpenDB(leveldb::Comparator* comparator,
92                              leveldb::Env* env,
93                              const base::FilePath& path,
94                              leveldb::DB** db) {
95  leveldb::Options options;
96  options.comparator = comparator;
97  options.create_if_missing = true;
98  options.paranoid_checks = true;
99  options.compression = leveldb::kSnappyCompression;
100
101  // For info about the troubles we've run into with this parameter, see:
102  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
103  options.max_open_files = 80;
104  options.env = env;
105
106  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
107  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
108}
109
110bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
111  leveldb::Options options;
112  options.env = leveldb::IDBEnv();
113  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
114  const leveldb::Status s =
115      leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
116  return s.ok();
117}
118
119namespace {
120class LockImpl : public LevelDBLock {
121 public:
122  explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
123      : env_(env), lock_(lock) {}
124  virtual ~LockImpl() { env_->UnlockFile(lock_); }
125 private:
126  leveldb::Env* env_;
127  leveldb::FileLock* lock_;
128};
129}
130
131scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
132    const base::FilePath& file_name) {
133  leveldb::Env* env = leveldb::IDBEnv();
134  base::FilePath lock_path = file_name.AppendASCII("LOCK");
135  leveldb::FileLock* lock = NULL;
136  leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
137  if (!status.ok())
138    return scoped_ptr<LevelDBLock>();
139  DCHECK(lock);
140  return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
141}
142
143static int CheckFreeSpace(const char* const type,
144                          const base::FilePath& file_name) {
145  std::string name =
146      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
147  int64 free_disk_space_in_k_bytes =
148      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
149  if (free_disk_space_in_k_bytes < 0) {
150    base::Histogram::FactoryGet(
151        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
152        1,
153        2 /*boundary*/,
154        2 /*boundary*/ + 1,
155        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
156    return -1;
157  }
158  int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
159                                       ? INT_MAX
160                                       : free_disk_space_in_k_bytes;
161  const uint64 histogram_max = static_cast<uint64>(1e9);
162  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
163  base::Histogram::FactoryGet(name,
164                              1,
165                              histogram_max,
166                              11 /*buckets*/,
167                              base::HistogramBase::kUmaTargetedHistogramFlag)
168      ->Add(clamped_disk_space_k_bytes);
169  return clamped_disk_space_k_bytes;
170}
171
172static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
173                                            const leveldb::Status& s) {
174  leveldb_env::MethodID method;
175  int error = -1;
176  leveldb_env::ErrorParsingResult result =
177      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
178  if (result == leveldb_env::NONE)
179    return;
180  std::string method_histogram_name(histogram_name);
181  method_histogram_name.append(".EnvMethod");
182  base::LinearHistogram::FactoryGet(
183      method_histogram_name,
184      1,
185      leveldb_env::kNumEntries,
186      leveldb_env::kNumEntries + 1,
187      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
188
189  std::string error_histogram_name(histogram_name);
190
191  if (result == leveldb_env::METHOD_AND_PFE) {
192    DCHECK(error < 0);
193    error_histogram_name.append(std::string(".PFE.") +
194                                leveldb_env::MethodIDToString(method));
195    base::LinearHistogram::FactoryGet(
196        error_histogram_name,
197        1,
198        -base::PLATFORM_FILE_ERROR_MAX,
199        -base::PLATFORM_FILE_ERROR_MAX + 1,
200        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
201  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
202    error_histogram_name.append(std::string(".Errno.") +
203                                leveldb_env::MethodIDToString(method));
204    base::LinearHistogram::FactoryGet(
205        error_histogram_name,
206        1,
207        ERANGE + 1,
208        ERANGE + 2,
209        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
210  }
211}
212
213static void ParseAndHistogramCorruptionDetails(
214    const std::string& histogram_name,
215    const leveldb::Status& status) {
216  int error = leveldb_env::ParseCorruptionMessage(status);
217  DCHECK(error >= 0);
218  std::string corruption_histogram_name(histogram_name);
219  corruption_histogram_name.append(".Corruption");
220  const int kNumPatterns = leveldb_env::GetNumCorruptionPatterns();
221  base::LinearHistogram::FactoryGet(
222      corruption_histogram_name,
223      1,
224      kNumPatterns,
225      kNumPatterns + 1,
226      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
227}
228
229static void HistogramLevelDBError(const std::string& histogram_name,
230                                  const leveldb::Status& s) {
231  if (s.ok()) {
232    NOTREACHED();
233    return;
234  }
235  enum {
236    LEVEL_DB_NOT_FOUND,
237    LEVEL_DB_CORRUPTION,
238    LEVEL_DB_IO_ERROR,
239    LEVEL_DB_OTHER,
240    LEVEL_DB_MAX_ERROR
241  };
242  int leveldb_error = LEVEL_DB_OTHER;
243  if (s.IsNotFound())
244    leveldb_error = LEVEL_DB_NOT_FOUND;
245  else if (s.IsCorruption())
246    leveldb_error = LEVEL_DB_CORRUPTION;
247  else if (s.IsIOError())
248    leveldb_error = LEVEL_DB_IO_ERROR;
249  base::Histogram::FactoryGet(histogram_name,
250                              1,
251                              LEVEL_DB_MAX_ERROR,
252                              LEVEL_DB_MAX_ERROR + 1,
253                              base::HistogramBase::kUmaTargetedHistogramFlag)
254      ->Add(leveldb_error);
255  if (s.IsIOError())
256    ParseAndHistogramIOErrorDetails(histogram_name, s);
257  else
258    ParseAndHistogramCorruptionDetails(histogram_name, s);
259}
260
261leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
262                                      const LevelDBComparator* comparator,
263                                      scoped_ptr<LevelDBDatabase>* result,
264                                      bool* is_disk_full) {
265  scoped_ptr<ComparatorAdapter> comparator_adapter(
266      new ComparatorAdapter(comparator));
267
268  leveldb::DB* db;
269  const leveldb::Status s =
270      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
271
272  if (!s.ok()) {
273    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
274    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
275    // Disks with <100k of free space almost never succeed in opening a
276    // leveldb database.
277    if (is_disk_full)
278      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
279
280    LOG(ERROR) << "Failed to open LevelDB database from "
281               << file_name.AsUTF8Unsafe() << "," << s.ToString();
282    return s;
283  }
284
285  CheckFreeSpace("Success", file_name);
286
287  (*result).reset(new LevelDBDatabase);
288  (*result)->db_ = make_scoped_ptr(db);
289  (*result)->comparator_adapter_ = comparator_adapter.Pass();
290  (*result)->comparator_ = comparator;
291
292  return s;
293}
294
295scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
296    const LevelDBComparator* comparator) {
297  scoped_ptr<ComparatorAdapter> comparator_adapter(
298      new ComparatorAdapter(comparator));
299  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
300
301  leveldb::DB* db;
302  const leveldb::Status s = OpenDB(
303      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
304
305  if (!s.ok()) {
306    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
307    return scoped_ptr<LevelDBDatabase>();
308  }
309
310  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
311  result->env_ = in_memory_env.Pass();
312  result->db_ = make_scoped_ptr(db);
313  result->comparator_adapter_ = comparator_adapter.Pass();
314  result->comparator_ = comparator;
315
316  return result.Pass();
317}
318
319bool LevelDBDatabase::Put(const StringPiece& key, std::string* value) {
320  leveldb::WriteOptions write_options;
321  write_options.sync = kSyncWrites;
322
323  const leveldb::Status s =
324      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
325  if (s.ok())
326    return true;
327  LOG(ERROR) << "LevelDB put failed: " << s.ToString();
328  return false;
329}
330
331bool LevelDBDatabase::Remove(const StringPiece& key) {
332  leveldb::WriteOptions write_options;
333  write_options.sync = kSyncWrites;
334
335  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
336  if (s.ok())
337    return true;
338  if (s.IsNotFound())
339    return false;
340  LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
341  return false;
342}
343
344bool LevelDBDatabase::Get(const StringPiece& key,
345                          std::string* value,
346                          bool* found,
347                          const LevelDBSnapshot* snapshot) {
348  *found = false;
349  leveldb::ReadOptions read_options;
350  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
351                                         // performance impact is too great.
352  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
353
354  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
355  if (s.ok()) {
356    *found = true;
357    return true;
358  }
359  if (s.IsNotFound())
360    return true;
361  HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
362  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
363  return false;
364}
365
366bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
367  leveldb::WriteOptions write_options;
368  write_options.sync = kSyncWrites;
369
370  const leveldb::Status s =
371      db_->Write(write_options, write_batch.write_batch_.get());
372  if (s.ok())
373    return true;
374  HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
375  LOG(ERROR) << "LevelDB write failed: " << s.ToString();
376  return false;
377}
378
379namespace {
380class IteratorImpl : public LevelDBIterator {
381 public:
382  virtual ~IteratorImpl() {}
383
384  virtual bool IsValid() const OVERRIDE;
385  virtual void SeekToLast() OVERRIDE;
386  virtual void Seek(const StringPiece& target) OVERRIDE;
387  virtual void Next() OVERRIDE;
388  virtual void Prev() OVERRIDE;
389  virtual StringPiece Key() const OVERRIDE;
390  virtual StringPiece Value() const OVERRIDE;
391
392 private:
393  friend class content::LevelDBDatabase;
394  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
395  void CheckStatus();
396
397  scoped_ptr<leveldb::Iterator> iterator_;
398};
399}
400
401IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
402    : iterator_(it.Pass()) {}
403
404void IteratorImpl::CheckStatus() {
405  const leveldb::Status s = iterator_->status();
406  if (!s.ok())
407    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
408}
409
410bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
411
412void IteratorImpl::SeekToLast() {
413  iterator_->SeekToLast();
414  CheckStatus();
415}
416
417void IteratorImpl::Seek(const StringPiece& target) {
418  iterator_->Seek(MakeSlice(target));
419  CheckStatus();
420}
421
422void IteratorImpl::Next() {
423  DCHECK(IsValid());
424  iterator_->Next();
425  CheckStatus();
426}
427
428void IteratorImpl::Prev() {
429  DCHECK(IsValid());
430  iterator_->Prev();
431  CheckStatus();
432}
433
434StringPiece IteratorImpl::Key() const {
435  DCHECK(IsValid());
436  return MakeStringPiece(iterator_->key());
437}
438
439StringPiece IteratorImpl::Value() const {
440  DCHECK(IsValid());
441  return MakeStringPiece(iterator_->value());
442}
443
444scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
445    const LevelDBSnapshot* snapshot) {
446  leveldb::ReadOptions read_options;
447  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
448                                         // performance impact is too great.
449  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
450
451  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
452  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
453}
454
455const LevelDBComparator* LevelDBDatabase::Comparator() const {
456  return comparator_;
457}
458
459}  // namespace content
460