leveldb_database.cc revision a1401311d1ab56c4ed0a474bd38c108f75cb0cd9 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8
9#include "base/basictypes.h"
10#include "base/files/file.h"
11#include "base/logging.h"
12#include "base/memory/scoped_ptr.h"
13#include "base/metrics/histogram.h"
14#include "base/strings/string16.h"
15#include "base/strings/string_piece.h"
16#include "base/strings/stringprintf.h"
17#include "base/strings/utf_string_conversions.h"
18#include "base/sys_info.h"
19#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
20#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
21#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
22#include "third_party/leveldatabase/env_chromium.h"
23#include "third_party/leveldatabase/env_idb.h"
24#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
25#include "third_party/leveldatabase/src/include/leveldb/comparator.h"
26#include "third_party/leveldatabase/src/include/leveldb/db.h"
27#include "third_party/leveldatabase/src/include/leveldb/env.h"
28#include "third_party/leveldatabase/src/include/leveldb/slice.h"
29
30using base::StringPiece;
31
32namespace content {
33
34// Forcing flushes to disk at the end of a transaction guarantees that the
35// data hit disk, but drastically impacts throughput when the filesystem is
36// busy with background compactions. Not syncing trades off reliability for
37// performance. Note that background compactions which move data from the
38// log to SSTs are always done with reliable writes.
39//
40// Sync writes are necessary on Windows for quota calculations; POSIX
41// calculates file sizes correctly even when not synced to disk.
42#if defined(OS_WIN)
43static const bool kSyncWrites = true;
44#else
45// TODO(dgrogan): Either remove the #if block or change this back to false.
46// See http://crbug.com/338385.
47static const bool kSyncWrites = true;
48#endif
49
50static leveldb::Slice MakeSlice(const StringPiece& s) {
51  return leveldb::Slice(s.begin(), s.size());
52}
53
54static StringPiece MakeStringPiece(const leveldb::Slice& s) {
55  return StringPiece(s.data(), s.size());
56}
57
58class ComparatorAdapter : public leveldb::Comparator {
59 public:
60  explicit ComparatorAdapter(const LevelDBComparator* comparator)
61      : comparator_(comparator) {}
62
63  virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
64      OVERRIDE {
65    return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
66  }
67
68  virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
69
70  // TODO(jsbell): Support the methods below in the future.
71  virtual void FindShortestSeparator(std::string* start,
72                                     const leveldb::Slice& limit) const
73      OVERRIDE {}
74  virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
75
76 private:
77  const LevelDBComparator* comparator_;
78};
79
80LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
81    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
82
83LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
84
85LevelDBDatabase::LevelDBDatabase() {}
86
87LevelDBDatabase::~LevelDBDatabase() {
88  // db_'s destructor uses comparator_adapter_; order of deletion is important.
89  db_.reset();
90  comparator_adapter_.reset();
91  env_.reset();
92}
93
94static leveldb::Status OpenDB(leveldb::Comparator* comparator,
95                              leveldb::Env* env,
96                              const base::FilePath& path,
97                              leveldb::DB** db) {
98  leveldb::Options options;
99  options.comparator = comparator;
100  options.create_if_missing = true;
101  options.paranoid_checks = true;
102  options.compression = leveldb::kSnappyCompression;
103
104  // For info about the troubles we've run into with this parameter, see:
105  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
106  options.max_open_files = 80;
107  options.env = env;
108
109  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
110  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
111}
112
113leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
114  leveldb::Options options;
115  options.env = leveldb::IDBEnv();
116  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
117  return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
118}
119
120namespace {
121class LockImpl : public LevelDBLock {
122 public:
123  explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
124      : env_(env), lock_(lock) {}
125  virtual ~LockImpl() { env_->UnlockFile(lock_); }
126 private:
127  leveldb::Env* env_;
128  leveldb::FileLock* lock_;
129};
130}
131
132scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
133    const base::FilePath& file_name) {
134  leveldb::Env* env = leveldb::IDBEnv();
135  base::FilePath lock_path = file_name.AppendASCII("LOCK");
136  leveldb::FileLock* lock = NULL;
137  leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
138  if (!status.ok())
139    return scoped_ptr<LevelDBLock>();
140  DCHECK(lock);
141  return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
142}
143
144static int CheckFreeSpace(const char* const type,
145                          const base::FilePath& file_name) {
146  std::string name =
147      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
148  int64 free_disk_space_in_k_bytes =
149      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
150  if (free_disk_space_in_k_bytes < 0) {
151    base::Histogram::FactoryGet(
152        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
153        1,
154        2 /*boundary*/,
155        2 /*boundary*/ + 1,
156        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
157    return -1;
158  }
159  int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
160                                       ? INT_MAX
161                                       : free_disk_space_in_k_bytes;
162  const uint64 histogram_max = static_cast<uint64>(1e9);
163  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
164  base::Histogram::FactoryGet(name,
165                              1,
166                              histogram_max,
167                              11 /*buckets*/,
168                              base::HistogramBase::kUmaTargetedHistogramFlag)
169      ->Add(clamped_disk_space_k_bytes);
170  return clamped_disk_space_k_bytes;
171}
172
173static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
174                                            const leveldb::Status& s) {
175  leveldb_env::MethodID method;
176  int error = -1;
177  leveldb_env::ErrorParsingResult result =
178      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
179  if (result == leveldb_env::NONE)
180    return;
181  std::string method_histogram_name(histogram_name);
182  method_histogram_name.append(".EnvMethod");
183  base::LinearHistogram::FactoryGet(
184      method_histogram_name,
185      1,
186      leveldb_env::kNumEntries,
187      leveldb_env::kNumEntries + 1,
188      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
189
190  std::string error_histogram_name(histogram_name);
191
192  if (result == leveldb_env::METHOD_AND_PFE) {
193    DCHECK(error < 0);
194    error_histogram_name.append(std::string(".PFE.") +
195                                leveldb_env::MethodIDToString(method));
196    base::LinearHistogram::FactoryGet(
197        error_histogram_name,
198        1,
199        -base::File::FILE_ERROR_MAX,
200        -base::File::FILE_ERROR_MAX + 1,
201        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
202  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
203    error_histogram_name.append(std::string(".Errno.") +
204                                leveldb_env::MethodIDToString(method));
205    base::LinearHistogram::FactoryGet(
206        error_histogram_name,
207        1,
208        ERANGE + 1,
209        ERANGE + 2,
210        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
211  }
212}
213
214static void ParseAndHistogramCorruptionDetails(
215    const std::string& histogram_name,
216    const leveldb::Status& status) {
217  int error = leveldb_env::GetCorruptionCode(status);
218  DCHECK(error >= 0);
219  std::string corruption_histogram_name(histogram_name);
220  corruption_histogram_name.append(".Corruption");
221  const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
222  base::LinearHistogram::FactoryGet(
223      corruption_histogram_name,
224      1,
225      kNumPatterns,
226      kNumPatterns + 1,
227      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
228}
229
230static void HistogramLevelDBError(const std::string& histogram_name,
231                                  const leveldb::Status& s) {
232  if (s.ok()) {
233    NOTREACHED();
234    return;
235  }
236  enum {
237    LEVEL_DB_NOT_FOUND,
238    LEVEL_DB_CORRUPTION,
239    LEVEL_DB_IO_ERROR,
240    LEVEL_DB_OTHER,
241    LEVEL_DB_MAX_ERROR
242  };
243  int leveldb_error = LEVEL_DB_OTHER;
244  if (s.IsNotFound())
245    leveldb_error = LEVEL_DB_NOT_FOUND;
246  else if (s.IsCorruption())
247    leveldb_error = LEVEL_DB_CORRUPTION;
248  else if (s.IsIOError())
249    leveldb_error = LEVEL_DB_IO_ERROR;
250  base::Histogram::FactoryGet(histogram_name,
251                              1,
252                              LEVEL_DB_MAX_ERROR,
253                              LEVEL_DB_MAX_ERROR + 1,
254                              base::HistogramBase::kUmaTargetedHistogramFlag)
255      ->Add(leveldb_error);
256  if (s.IsIOError())
257    ParseAndHistogramIOErrorDetails(histogram_name, s);
258  else
259    ParseAndHistogramCorruptionDetails(histogram_name, s);
260}
261
262leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
263                                      const LevelDBComparator* comparator,
264                                      scoped_ptr<LevelDBDatabase>* result,
265                                      bool* is_disk_full) {
266  scoped_ptr<ComparatorAdapter> comparator_adapter(
267      new ComparatorAdapter(comparator));
268
269  leveldb::DB* db;
270  const leveldb::Status s =
271      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
272
273  if (!s.ok()) {
274    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
275    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
276    // Disks with <100k of free space almost never succeed in opening a
277    // leveldb database.
278    if (is_disk_full)
279      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
280
281    LOG(ERROR) << "Failed to open LevelDB database from "
282               << file_name.AsUTF8Unsafe() << "," << s.ToString();
283    return s;
284  }
285
286  CheckFreeSpace("Success", file_name);
287
288  (*result).reset(new LevelDBDatabase);
289  (*result)->db_ = make_scoped_ptr(db);
290  (*result)->comparator_adapter_ = comparator_adapter.Pass();
291  (*result)->comparator_ = comparator;
292
293  return s;
294}
295
296scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
297    const LevelDBComparator* comparator) {
298  scoped_ptr<ComparatorAdapter> comparator_adapter(
299      new ComparatorAdapter(comparator));
300  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
301
302  leveldb::DB* db;
303  const leveldb::Status s = OpenDB(
304      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
305
306  if (!s.ok()) {
307    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
308    return scoped_ptr<LevelDBDatabase>();
309  }
310
311  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
312  result->env_ = in_memory_env.Pass();
313  result->db_ = make_scoped_ptr(db);
314  result->comparator_adapter_ = comparator_adapter.Pass();
315  result->comparator_ = comparator;
316
317  return result.Pass();
318}
319
320leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
321                                     std::string* value) {
322  leveldb::WriteOptions write_options;
323  write_options.sync = kSyncWrites;
324
325  const leveldb::Status s =
326      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
327  if (!s.ok())
328    LOG(ERROR) << "LevelDB put failed: " << s.ToString();
329  return s;
330}
331
332leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
333  leveldb::WriteOptions write_options;
334  write_options.sync = kSyncWrites;
335
336  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
337  if (!s.IsNotFound())
338    LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
339  return s;
340}
341
342leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
343                                     std::string* value,
344                                     bool* found,
345                                     const LevelDBSnapshot* snapshot) {
346  *found = false;
347  leveldb::ReadOptions read_options;
348  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
349                                         // performance impact is too great.
350  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
351
352  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
353  if (s.ok()) {
354    *found = true;
355    return s;
356  }
357  if (s.IsNotFound())
358    return leveldb::Status::OK();
359  HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
360  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
361  return s;
362}
363
364leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
365  leveldb::WriteOptions write_options;
366  write_options.sync = kSyncWrites;
367
368  const leveldb::Status s =
369      db_->Write(write_options, write_batch.write_batch_.get());
370  if (!s.ok()) {
371    HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
372    LOG(ERROR) << "LevelDB write failed: " << s.ToString();
373  }
374  return s;
375}
376
377namespace {
378class IteratorImpl : public LevelDBIterator {
379 public:
380  virtual ~IteratorImpl() {}
381
382  virtual bool IsValid() const OVERRIDE;
383  virtual void SeekToLast() OVERRIDE;
384  virtual void Seek(const StringPiece& target) OVERRIDE;
385  virtual void Next() OVERRIDE;
386  virtual void Prev() OVERRIDE;
387  virtual StringPiece Key() const OVERRIDE;
388  virtual StringPiece Value() const OVERRIDE;
389
390 private:
391  friend class content::LevelDBDatabase;
392  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
393  void CheckStatus();
394
395  scoped_ptr<leveldb::Iterator> iterator_;
396};
397}
398
399IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
400    : iterator_(it.Pass()) {}
401
402void IteratorImpl::CheckStatus() {
403  const leveldb::Status s = iterator_->status();
404  if (!s.ok())
405    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
406}
407
408bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
409
410void IteratorImpl::SeekToLast() {
411  iterator_->SeekToLast();
412  CheckStatus();
413}
414
415void IteratorImpl::Seek(const StringPiece& target) {
416  iterator_->Seek(MakeSlice(target));
417  CheckStatus();
418}
419
420void IteratorImpl::Next() {
421  DCHECK(IsValid());
422  iterator_->Next();
423  CheckStatus();
424}
425
426void IteratorImpl::Prev() {
427  DCHECK(IsValid());
428  iterator_->Prev();
429  CheckStatus();
430}
431
432StringPiece IteratorImpl::Key() const {
433  DCHECK(IsValid());
434  return MakeStringPiece(iterator_->key());
435}
436
437StringPiece IteratorImpl::Value() const {
438  DCHECK(IsValid());
439  return MakeStringPiece(iterator_->value());
440}
441
442scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
443    const LevelDBSnapshot* snapshot) {
444  leveldb::ReadOptions read_options;
445  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
446                                         // performance impact is too great.
447  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
448
449  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
450  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
451}
452
453const LevelDBComparator* LevelDBDatabase::Comparator() const {
454  return comparator_;
455}
456
457void LevelDBDatabase::Compact(const base::StringPiece& start,
458                              const base::StringPiece& stop) {
459  const leveldb::Slice start_slice = MakeSlice(start);
460  const leveldb::Slice stop_slice = MakeSlice(stop);
461  db_->CompactRange(&start_slice, &stop_slice);
462}
463
464}  // namespace content
465