leveldb_database.cc revision 7dbb3d5cf0c15f500944d211057644d6a2f37371 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8
9#include "base/basictypes.h"
10#include "base/logging.h"
11#include "base/memory/scoped_ptr.h"
12#include "base/metrics/histogram.h"
13#include "base/strings/string16.h"
14#include "base/strings/string_piece.h"
15#include "base/strings/stringprintf.h"
16#include "base/strings/utf_string_conversions.h"
17#include "base/sys_info.h"
18#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
19#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
20#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
21#include "third_party/leveldatabase/env_chromium.h"
22#include "third_party/leveldatabase/env_idb.h"
23#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
24#include "third_party/leveldatabase/src/include/leveldb/comparator.h"
25#include "third_party/leveldatabase/src/include/leveldb/db.h"
26#include "third_party/leveldatabase/src/include/leveldb/env.h"
27#include "third_party/leveldatabase/src/include/leveldb/slice.h"
28
29using base::StringPiece;
30
31namespace content {
32
33static leveldb::Slice MakeSlice(const StringPiece& s) {
34  return leveldb::Slice(s.begin(), s.size());
35}
36
37static StringPiece MakeStringPiece(const leveldb::Slice& s) {
38  return StringPiece(s.data(), s.size());
39}
40
41class ComparatorAdapter : public leveldb::Comparator {
42 public:
43  explicit ComparatorAdapter(const LevelDBComparator* comparator)
44      : comparator_(comparator) {}
45
46  virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
47      OVERRIDE {
48    return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
49  }
50
51  virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
52
53  // TODO(jsbell): Support the methods below in the future.
54  virtual void FindShortestSeparator(std::string* start,
55                                     const leveldb::Slice& limit) const
56      OVERRIDE {}
57  virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
58
59 private:
60  const LevelDBComparator* comparator_;
61};
62
63LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
64    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
65
66LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
67
68LevelDBDatabase::LevelDBDatabase() {}
69
70LevelDBDatabase::~LevelDBDatabase() {
71  // db_'s destructor uses comparator_adapter_; order of deletion is important.
72  db_.reset();
73  comparator_adapter_.reset();
74  env_.reset();
75}
76
77static leveldb::Status OpenDB(leveldb::Comparator* comparator,
78                              leveldb::Env* env,
79                              const base::FilePath& path,
80                              leveldb::DB** db) {
81  leveldb::Options options;
82  options.comparator = comparator;
83  options.create_if_missing = true;
84  options.paranoid_checks = true;
85
86  // Marking compression as explicitly off so snappy support can be
87  // compiled in for other leveldb clients without implicitly enabling
88  // it for IndexedDB. http://crbug.com/81384
89  options.compression = leveldb::kNoCompression;
90
91  // For info about the troubles we've run into with this parameter, see:
92  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
93  options.max_open_files = 80;
94  options.env = env;
95
96  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
97  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
98}
99
100bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
101  leveldb::Options options;
102  options.env = leveldb::IDBEnv();
103  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
104  const leveldb::Status s =
105      leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
106  return s.ok();
107}
108
109static int CheckFreeSpace(const char* const type,
110                          const base::FilePath& file_name) {
111  std::string name =
112      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
113  int64 free_disk_space_in_k_bytes =
114      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
115  if (free_disk_space_in_k_bytes < 0) {
116    base::Histogram::FactoryGet(
117        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
118        1,
119        2 /*boundary*/,
120        2 /*boundary*/ + 1,
121        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
122    return -1;
123  }
124  int clamped_disk_space_k_bytes =
125      free_disk_space_in_k_bytes > INT_MAX ? INT_MAX
126                                           : free_disk_space_in_k_bytes;
127  const uint64 histogram_max = static_cast<uint64>(1e9);
128  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
129  base::Histogram::FactoryGet(name,
130                              1,
131                              histogram_max,
132                              11 /*buckets*/,
133                              base::HistogramBase::kUmaTargetedHistogramFlag)
134      ->Add(clamped_disk_space_k_bytes);
135  return clamped_disk_space_k_bytes;
136}
137
138static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
139                                            const leveldb::Status& s) {
140  leveldb_env::MethodID method;
141  int error = -1;
142  leveldb_env::ErrorParsingResult result =
143      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
144  if (result == leveldb_env::NONE)
145    return;
146  std::string method_histogram_name(histogram_name);
147  method_histogram_name.append(".EnvMethod");
148  base::LinearHistogram::FactoryGet(
149      method_histogram_name,
150      1,
151      leveldb_env::kNumEntries,
152      leveldb_env::kNumEntries + 1,
153      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
154
155  std::string error_histogram_name(histogram_name);
156
157  if (result == leveldb_env::METHOD_AND_PFE) {
158    DCHECK(error < 0);
159    error_histogram_name.append(std::string(".PFE.") +
160                                leveldb_env::MethodIDToString(method));
161    base::LinearHistogram::FactoryGet(
162        error_histogram_name,
163        1,
164        -base::PLATFORM_FILE_ERROR_MAX,
165        -base::PLATFORM_FILE_ERROR_MAX + 1,
166        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
167  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
168    error_histogram_name.append(std::string(".Errno.") +
169                                leveldb_env::MethodIDToString(method));
170    base::LinearHistogram::FactoryGet(
171        error_histogram_name,
172        1,
173        ERANGE + 1,
174        ERANGE + 2,
175        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
176  }
177}
178
179static void ParseAndHistogramCorruptionDetails(
180    const std::string& histogram_name,
181    const leveldb::Status& status) {
182  DCHECK(!status.IsIOError());
183  DCHECK(!status.ok());
184  const int kOtherError = 0;
185  int error = kOtherError;
186  const std::string& str_error = status.ToString();
187  // Keep in sync with LevelDBCorruptionTypes in histograms.xml.
188  const char* patterns[] = {
189    "missing files",
190    "log record too small",
191    "corrupted internal key",
192    "partial record",
193    "missing start of fragmented record",
194    "error in middle of record",
195    "unknown record type",
196    "truncated record at end",
197    "bad record length",
198    "VersionEdit",
199    "FileReader invoked with unexpected value",
200    "corrupted key",
201    "CURRENT file does not end with newline",
202    "no meta-nextfile entry",
203    "no meta-lognumber entry",
204    "no last-sequence-number entry",
205    "malformed WriteBatch",
206    "bad WriteBatch Put",
207    "bad WriteBatch Delete",
208    "unknown WriteBatch tag",
209    "WriteBatch has wrong count",
210    "bad entry in block",
211    "bad block contents",
212    "bad block handle",
213    "truncated block read",
214    "block checksum mismatch",
215    "checksum mismatch",
216    "corrupted compressed block contents",
217    "bad block type",
218    "bad magic number",
219    "file is too short",
220  };
221  const size_t kNumPatterns = arraysize(patterns);
222  for (size_t i = 0; i < kNumPatterns; ++i) {
223    if (str_error.find(patterns[i]) != std::string::npos) {
224      error = i + 1;
225      break;
226    }
227  }
228  DCHECK(error >= 0);
229  std::string corruption_histogram_name(histogram_name);
230  corruption_histogram_name.append(".Corruption");
231  base::LinearHistogram::FactoryGet(
232      corruption_histogram_name,
233      1,
234      kNumPatterns + 1,
235      kNumPatterns + 2,
236      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
237}
238
239static void HistogramLevelDBError(const std::string& histogram_name,
240                                  const leveldb::Status& s) {
241  if (s.ok()) {
242    NOTREACHED();
243    return;
244  }
245  enum {
246    LEVEL_DB_NOT_FOUND,
247    LEVEL_DB_CORRUPTION,
248    LEVEL_DB_IO_ERROR,
249    LEVEL_DB_OTHER,
250    LEVEL_DB_MAX_ERROR
251  };
252  int leveldb_error = LEVEL_DB_OTHER;
253  if (s.IsNotFound())
254    leveldb_error = LEVEL_DB_NOT_FOUND;
255  else if (s.IsCorruption())
256    leveldb_error = LEVEL_DB_CORRUPTION;
257  else if (s.IsIOError())
258    leveldb_error = LEVEL_DB_IO_ERROR;
259  base::Histogram::FactoryGet(histogram_name,
260                              1,
261                              LEVEL_DB_MAX_ERROR,
262                              LEVEL_DB_MAX_ERROR + 1,
263                              base::HistogramBase::kUmaTargetedHistogramFlag)
264      ->Add(leveldb_error);
265  if (s.IsIOError())
266    ParseAndHistogramIOErrorDetails(histogram_name, s);
267  else
268    ParseAndHistogramCorruptionDetails(histogram_name, s);
269}
270
271scoped_ptr<LevelDBDatabase> LevelDBDatabase::Open(
272    const base::FilePath& file_name,
273    const LevelDBComparator* comparator,
274    bool* is_disk_full) {
275  scoped_ptr<ComparatorAdapter> comparator_adapter(
276      new ComparatorAdapter(comparator));
277
278  leveldb::DB* db;
279  const leveldb::Status s =
280      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
281
282  if (!s.ok()) {
283    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
284    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
285    // Disks with <100k of free space almost never succeed in opening a
286    // leveldb database.
287    if (is_disk_full)
288      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
289
290    LOG(ERROR) << "Failed to open LevelDB database from "
291               << file_name.AsUTF8Unsafe() << "," << s.ToString();
292    return scoped_ptr<LevelDBDatabase>();
293  }
294
295  CheckFreeSpace("Success", file_name);
296
297  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
298  result->db_ = make_scoped_ptr(db);
299  result->comparator_adapter_ = comparator_adapter.Pass();
300  result->comparator_ = comparator;
301
302  return result.Pass();
303}
304
305scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
306    const LevelDBComparator* comparator) {
307  scoped_ptr<ComparatorAdapter> comparator_adapter(
308      new ComparatorAdapter(comparator));
309  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
310
311  leveldb::DB* db;
312  const leveldb::Status s = OpenDB(
313      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
314
315  if (!s.ok()) {
316    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
317    return scoped_ptr<LevelDBDatabase>();
318  }
319
320  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
321  result->env_ = in_memory_env.Pass();
322  result->db_ = make_scoped_ptr(db);
323  result->comparator_adapter_ = comparator_adapter.Pass();
324  result->comparator_ = comparator;
325
326  return result.Pass();
327}
328
329bool LevelDBDatabase::Put(const StringPiece& key, std::string* value) {
330  leveldb::WriteOptions write_options;
331  write_options.sync = true;
332
333  const leveldb::Status s =
334      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
335  if (s.ok())
336    return true;
337  LOG(ERROR) << "LevelDB put failed: " << s.ToString();
338  return false;
339}
340
341bool LevelDBDatabase::Remove(const StringPiece& key) {
342  leveldb::WriteOptions write_options;
343  write_options.sync = true;
344
345  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
346  if (s.ok())
347    return true;
348  if (s.IsNotFound())
349    return false;
350  LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
351  return false;
352}
353
354bool LevelDBDatabase::Get(const StringPiece& key,
355                          std::string* value,
356                          bool* found,
357                          const LevelDBSnapshot* snapshot) {
358  *found = false;
359  leveldb::ReadOptions read_options;
360  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
361                                         // performance impact is too great.
362  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
363
364  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
365  if (s.ok()) {
366    *found = true;
367    return true;
368  }
369  if (s.IsNotFound())
370    return true;
371  HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
372  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
373  return false;
374}
375
376bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
377  leveldb::WriteOptions write_options;
378  write_options.sync = true;
379
380  const leveldb::Status s =
381      db_->Write(write_options, write_batch.write_batch_.get());
382  if (s.ok())
383    return true;
384  HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
385  LOG(ERROR) << "LevelDB write failed: " << s.ToString();
386  return false;
387}
388
389namespace {
390class IteratorImpl : public LevelDBIterator {
391 public:
392  virtual ~IteratorImpl() {}
393
394  virtual bool IsValid() const OVERRIDE;
395  virtual void SeekToLast() OVERRIDE;
396  virtual void Seek(const StringPiece& target) OVERRIDE;
397  virtual void Next() OVERRIDE;
398  virtual void Prev() OVERRIDE;
399  virtual StringPiece Key() const OVERRIDE;
400  virtual StringPiece Value() const OVERRIDE;
401
402 private:
403  friend class content::LevelDBDatabase;
404  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
405  void CheckStatus();
406
407  scoped_ptr<leveldb::Iterator> iterator_;
408};
409}
410
411IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
412    : iterator_(it.Pass()) {}
413
414void IteratorImpl::CheckStatus() {
415  const leveldb::Status s = iterator_->status();
416  if (!s.ok())
417    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
418}
419
420bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
421
422void IteratorImpl::SeekToLast() {
423  iterator_->SeekToLast();
424  CheckStatus();
425}
426
427void IteratorImpl::Seek(const StringPiece& target) {
428  iterator_->Seek(MakeSlice(target));
429  CheckStatus();
430}
431
432void IteratorImpl::Next() {
433  DCHECK(IsValid());
434  iterator_->Next();
435  CheckStatus();
436}
437
438void IteratorImpl::Prev() {
439  DCHECK(IsValid());
440  iterator_->Prev();
441  CheckStatus();
442}
443
444StringPiece IteratorImpl::Key() const {
445  DCHECK(IsValid());
446  return MakeStringPiece(iterator_->key());
447}
448
449StringPiece IteratorImpl::Value() const {
450  DCHECK(IsValid());
451  return MakeStringPiece(iterator_->value());
452}
453
454scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
455    const LevelDBSnapshot* snapshot) {
456  leveldb::ReadOptions read_options;
457  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
458                                         // performance impact is too great.
459  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
460
461  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
462  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
463}
464
465const LevelDBComparator* LevelDBDatabase::Comparator() const {
466  return comparator_;
467}
468
469}  // namespace content
470