leveldb_database.cc revision e5d81f57cb97b3b6b7fccc9c5610d21eb81db09d
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8
9#include "base/basictypes.h"
10#include "base/files/file.h"
11#include "base/logging.h"
12#include "base/memory/scoped_ptr.h"
13#include "base/metrics/histogram.h"
14#include "base/strings/string16.h"
15#include "base/strings/string_piece.h"
16#include "base/strings/stringprintf.h"
17#include "base/strings/utf_string_conversions.h"
18#include "base/sys_info.h"
19#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
20#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
21#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
22#include "third_party/leveldatabase/env_chromium.h"
23#include "third_party/leveldatabase/env_idb.h"
24#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
25#include "third_party/leveldatabase/src/include/leveldb/db.h"
26#include "third_party/leveldatabase/src/include/leveldb/env.h"
27#include "third_party/leveldatabase/src/include/leveldb/slice.h"
28
29using base::StringPiece;
30
31namespace content {
32
33// Forcing flushes to disk at the end of a transaction guarantees that the
34// data hit disk, but drastically impacts throughput when the filesystem is
35// busy with background compactions. Not syncing trades off reliability for
36// performance. Note that background compactions which move data from the
37// log to SSTs are always done with reliable writes.
38//
39// Sync writes are necessary on Windows for quota calculations; POSIX
40// calculates file sizes correctly even when not synced to disk.
41#if defined(OS_WIN)
42static const bool kSyncWrites = true;
43#else
44// TODO(dgrogan): Either remove the #if block or change this back to false.
45// See http://crbug.com/338385.
46static const bool kSyncWrites = true;
47#endif
48
49static leveldb::Slice MakeSlice(const StringPiece& s) {
50  return leveldb::Slice(s.begin(), s.size());
51}
52
53static StringPiece MakeStringPiece(const leveldb::Slice& s) {
54  return StringPiece(s.data(), s.size());
55}
56
57LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
58    const LevelDBComparator* comparator)
59    : comparator_(comparator) {}
60
61int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
62                                                const leveldb::Slice& b) const {
63  return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
64}
65
66const char* LevelDBDatabase::ComparatorAdapter::Name() const {
67  return comparator_->Name();
68}
69
70// TODO(jsbell): Support the methods below in the future.
71void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
72    std::string* start,
73    const leveldb::Slice& limit) const {}
74
75void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
76    std::string* key) const {}
77
78LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
79    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
80
81LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
82
83LevelDBDatabase::LevelDBDatabase() {}
84
85LevelDBDatabase::~LevelDBDatabase() {
86  // db_'s destructor uses comparator_adapter_; order of deletion is important.
87  db_.reset();
88  comparator_adapter_.reset();
89  env_.reset();
90}
91
92static leveldb::Status OpenDB(leveldb::Comparator* comparator,
93                              leveldb::Env* env,
94                              const base::FilePath& path,
95                              leveldb::DB** db) {
96  leveldb::Options options;
97  options.comparator = comparator;
98  options.create_if_missing = true;
99  options.paranoid_checks = true;
100  options.compression = leveldb::kSnappyCompression;
101
102  // For info about the troubles we've run into with this parameter, see:
103  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
104  options.max_open_files = 80;
105  options.env = env;
106
107  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
108  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
109}
110
111leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
112  leveldb::Options options;
113  options.env = leveldb::IDBEnv();
114  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
115  return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
116}
117
118namespace {
119class LockImpl : public LevelDBLock {
120 public:
121  explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
122      : env_(env), lock_(lock) {}
123  virtual ~LockImpl() { env_->UnlockFile(lock_); }
124 private:
125  leveldb::Env* env_;
126  leveldb::FileLock* lock_;
127};
128}
129
130scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
131    const base::FilePath& file_name) {
132  leveldb::Env* env = leveldb::IDBEnv();
133  base::FilePath lock_path = file_name.AppendASCII("LOCK");
134  leveldb::FileLock* lock = NULL;
135  leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
136  if (!status.ok())
137    return scoped_ptr<LevelDBLock>();
138  DCHECK(lock);
139  return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
140}
141
142static int CheckFreeSpace(const char* const type,
143                          const base::FilePath& file_name) {
144  std::string name =
145      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
146  int64 free_disk_space_in_k_bytes =
147      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
148  if (free_disk_space_in_k_bytes < 0) {
149    base::Histogram::FactoryGet(
150        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
151        1,
152        2 /*boundary*/,
153        2 /*boundary*/ + 1,
154        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
155    return -1;
156  }
157  int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
158                                       ? INT_MAX
159                                       : free_disk_space_in_k_bytes;
160  const uint64 histogram_max = static_cast<uint64>(1e9);
161  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
162  base::Histogram::FactoryGet(name,
163                              1,
164                              histogram_max,
165                              11 /*buckets*/,
166                              base::HistogramBase::kUmaTargetedHistogramFlag)
167      ->Add(clamped_disk_space_k_bytes);
168  return clamped_disk_space_k_bytes;
169}
170
171static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
172                                            const leveldb::Status& s) {
173  leveldb_env::MethodID method;
174  int error = -1;
175  leveldb_env::ErrorParsingResult result =
176      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
177  if (result == leveldb_env::NONE)
178    return;
179  std::string method_histogram_name(histogram_name);
180  method_histogram_name.append(".EnvMethod");
181  base::LinearHistogram::FactoryGet(
182      method_histogram_name,
183      1,
184      leveldb_env::kNumEntries,
185      leveldb_env::kNumEntries + 1,
186      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
187
188  std::string error_histogram_name(histogram_name);
189
190  if (result == leveldb_env::METHOD_AND_PFE) {
191    DCHECK(error < 0);
192    error_histogram_name.append(std::string(".PFE.") +
193                                leveldb_env::MethodIDToString(method));
194    base::LinearHistogram::FactoryGet(
195        error_histogram_name,
196        1,
197        -base::File::FILE_ERROR_MAX,
198        -base::File::FILE_ERROR_MAX + 1,
199        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
200  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
201    error_histogram_name.append(std::string(".Errno.") +
202                                leveldb_env::MethodIDToString(method));
203    base::LinearHistogram::FactoryGet(
204        error_histogram_name,
205        1,
206        ERANGE + 1,
207        ERANGE + 2,
208        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
209  }
210}
211
212static void ParseAndHistogramCorruptionDetails(
213    const std::string& histogram_name,
214    const leveldb::Status& status) {
215  int error = leveldb_env::GetCorruptionCode(status);
216  DCHECK(error >= 0);
217  std::string corruption_histogram_name(histogram_name);
218  corruption_histogram_name.append(".Corruption");
219  const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
220  base::LinearHistogram::FactoryGet(
221      corruption_histogram_name,
222      1,
223      kNumPatterns,
224      kNumPatterns + 1,
225      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
226}
227
228static void HistogramLevelDBError(const std::string& histogram_name,
229                                  const leveldb::Status& s) {
230  if (s.ok()) {
231    NOTREACHED();
232    return;
233  }
234  enum {
235    LEVEL_DB_NOT_FOUND,
236    LEVEL_DB_CORRUPTION,
237    LEVEL_DB_IO_ERROR,
238    LEVEL_DB_OTHER,
239    LEVEL_DB_MAX_ERROR
240  };
241  int leveldb_error = LEVEL_DB_OTHER;
242  if (s.IsNotFound())
243    leveldb_error = LEVEL_DB_NOT_FOUND;
244  else if (s.IsCorruption())
245    leveldb_error = LEVEL_DB_CORRUPTION;
246  else if (s.IsIOError())
247    leveldb_error = LEVEL_DB_IO_ERROR;
248  base::Histogram::FactoryGet(histogram_name,
249                              1,
250                              LEVEL_DB_MAX_ERROR,
251                              LEVEL_DB_MAX_ERROR + 1,
252                              base::HistogramBase::kUmaTargetedHistogramFlag)
253      ->Add(leveldb_error);
254  if (s.IsIOError())
255    ParseAndHistogramIOErrorDetails(histogram_name, s);
256  else
257    ParseAndHistogramCorruptionDetails(histogram_name, s);
258}
259
260leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
261                                      const LevelDBComparator* comparator,
262                                      scoped_ptr<LevelDBDatabase>* result,
263                                      bool* is_disk_full) {
264  scoped_ptr<ComparatorAdapter> comparator_adapter(
265      new ComparatorAdapter(comparator));
266
267  leveldb::DB* db;
268  const leveldb::Status s =
269      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
270
271  if (!s.ok()) {
272    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
273    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
274    // Disks with <100k of free space almost never succeed in opening a
275    // leveldb database.
276    if (is_disk_full)
277      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
278
279    LOG(ERROR) << "Failed to open LevelDB database from "
280               << file_name.AsUTF8Unsafe() << "," << s.ToString();
281    return s;
282  }
283
284  CheckFreeSpace("Success", file_name);
285
286  (*result).reset(new LevelDBDatabase);
287  (*result)->db_ = make_scoped_ptr(db);
288  (*result)->comparator_adapter_ = comparator_adapter.Pass();
289  (*result)->comparator_ = comparator;
290
291  return s;
292}
293
294scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
295    const LevelDBComparator* comparator) {
296  scoped_ptr<ComparatorAdapter> comparator_adapter(
297      new ComparatorAdapter(comparator));
298  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
299
300  leveldb::DB* db;
301  const leveldb::Status s = OpenDB(
302      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
303
304  if (!s.ok()) {
305    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
306    return scoped_ptr<LevelDBDatabase>();
307  }
308
309  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
310  result->env_ = in_memory_env.Pass();
311  result->db_ = make_scoped_ptr(db);
312  result->comparator_adapter_ = comparator_adapter.Pass();
313  result->comparator_ = comparator;
314
315  return result.Pass();
316}
317
318leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
319                                     std::string* value) {
320  leveldb::WriteOptions write_options;
321  write_options.sync = kSyncWrites;
322
323  const leveldb::Status s =
324      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
325  if (!s.ok())
326    LOG(ERROR) << "LevelDB put failed: " << s.ToString();
327  return s;
328}
329
330leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
331  leveldb::WriteOptions write_options;
332  write_options.sync = kSyncWrites;
333
334  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
335  if (!s.IsNotFound())
336    LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
337  return s;
338}
339
340leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
341                                     std::string* value,
342                                     bool* found,
343                                     const LevelDBSnapshot* snapshot) {
344  *found = false;
345  leveldb::ReadOptions read_options;
346  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
347                                         // performance impact is too great.
348  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
349
350  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
351  if (s.ok()) {
352    *found = true;
353    return s;
354  }
355  if (s.IsNotFound())
356    return leveldb::Status::OK();
357  HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
358  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
359  return s;
360}
361
362leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
363  leveldb::WriteOptions write_options;
364  write_options.sync = kSyncWrites;
365
366  const leveldb::Status s =
367      db_->Write(write_options, write_batch.write_batch_.get());
368  if (!s.ok()) {
369    HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
370    LOG(ERROR) << "LevelDB write failed: " << s.ToString();
371  }
372  return s;
373}
374
375namespace {
376class IteratorImpl : public LevelDBIterator {
377 public:
378  virtual ~IteratorImpl() {}
379
380  virtual bool IsValid() const OVERRIDE;
381  virtual void SeekToLast() OVERRIDE;
382  virtual void Seek(const StringPiece& target) OVERRIDE;
383  virtual void Next() OVERRIDE;
384  virtual void Prev() OVERRIDE;
385  virtual StringPiece Key() const OVERRIDE;
386  virtual StringPiece Value() const OVERRIDE;
387
388 private:
389  friend class content::LevelDBDatabase;
390  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
391  void CheckStatus();
392
393  scoped_ptr<leveldb::Iterator> iterator_;
394};
395}
396
397IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
398    : iterator_(it.Pass()) {}
399
400void IteratorImpl::CheckStatus() {
401  const leveldb::Status s = iterator_->status();
402  if (!s.ok())
403    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
404}
405
406bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
407
408void IteratorImpl::SeekToLast() {
409  iterator_->SeekToLast();
410  CheckStatus();
411}
412
413void IteratorImpl::Seek(const StringPiece& target) {
414  iterator_->Seek(MakeSlice(target));
415  CheckStatus();
416}
417
418void IteratorImpl::Next() {
419  DCHECK(IsValid());
420  iterator_->Next();
421  CheckStatus();
422}
423
424void IteratorImpl::Prev() {
425  DCHECK(IsValid());
426  iterator_->Prev();
427  CheckStatus();
428}
429
430StringPiece IteratorImpl::Key() const {
431  DCHECK(IsValid());
432  return MakeStringPiece(iterator_->key());
433}
434
435StringPiece IteratorImpl::Value() const {
436  DCHECK(IsValid());
437  return MakeStringPiece(iterator_->value());
438}
439
440scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
441    const LevelDBSnapshot* snapshot) {
442  leveldb::ReadOptions read_options;
443  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
444                                         // performance impact is too great.
445  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
446
447  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
448  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
449}
450
451const LevelDBComparator* LevelDBDatabase::Comparator() const {
452  return comparator_;
453}
454
455void LevelDBDatabase::Compact(const base::StringPiece& start,
456                              const base::StringPiece& stop) {
457  const leveldb::Slice start_slice = MakeSlice(start);
458  const leveldb::Slice stop_slice = MakeSlice(stop);
459  db_->CompactRange(&start_slice, &stop_slice);
460}
461
462void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
463
464}  // namespace content
465