leveldb_database.cc revision ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8
9#include "base/basictypes.h"
10#include "base/logging.h"
11#include "base/memory/scoped_ptr.h"
12#include "base/metrics/histogram.h"
13#include "base/strings/string16.h"
14#include "base/strings/string_piece.h"
15#include "base/strings/stringprintf.h"
16#include "base/strings/utf_string_conversions.h"
17#include "base/sys_info.h"
18#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
19#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
20#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
21#include "third_party/leveldatabase/env_chromium.h"
22#include "third_party/leveldatabase/env_idb.h"
23#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
24#include "third_party/leveldatabase/src/include/leveldb/comparator.h"
25#include "third_party/leveldatabase/src/include/leveldb/db.h"
26#include "third_party/leveldatabase/src/include/leveldb/env.h"
27#include "third_party/leveldatabase/src/include/leveldb/slice.h"
28
29using base::StringPiece;
30
31namespace content {
32
33static leveldb::Slice MakeSlice(const StringPiece& s) {
34  return leveldb::Slice(s.begin(), s.size());
35}
36
37static StringPiece MakeStringPiece(const leveldb::Slice& s) {
38  return StringPiece(s.data(), s.size());
39}
40
41class ComparatorAdapter : public leveldb::Comparator {
42 public:
43  explicit ComparatorAdapter(const LevelDBComparator* comparator)
44      : comparator_(comparator) {}
45
46  virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
47      OVERRIDE {
48    return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
49  }
50
51  virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
52
53  // TODO(jsbell): Support the methods below in the future.
54  virtual void FindShortestSeparator(std::string* start,
55                                     const leveldb::Slice& limit) const
56      OVERRIDE {}
57  virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
58
59 private:
60  const LevelDBComparator* comparator_;
61};
62
63LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
64    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
65
66LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
67
68LevelDBDatabase::LevelDBDatabase() {}
69
70LevelDBDatabase::~LevelDBDatabase() {
71  // db_'s destructor uses comparator_adapter_; order of deletion is important.
72  db_.reset();
73  comparator_adapter_.reset();
74  env_.reset();
75}
76
77static leveldb::Status OpenDB(leveldb::Comparator* comparator,
78                              leveldb::Env* env,
79                              const base::FilePath& path,
80                              leveldb::DB** db) {
81  leveldb::Options options;
82  options.comparator = comparator;
83  options.create_if_missing = true;
84  options.paranoid_checks = true;
85
86  // Marking compression as explicitly off so snappy support can be
87  // compiled in for other leveldb clients without implicitly enabling
88  // it for IndexedDB. http://crbug.com/81384
89  options.compression = leveldb::kNoCompression;
90
91  // For info about the troubles we've run into with this parameter, see:
92  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
93  options.max_open_files = 80;
94  options.env = env;
95
96  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
97  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
98}
99
100bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
101  leveldb::Options options;
102  options.env = leveldb::IDBEnv();
103  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
104  const leveldb::Status s =
105      leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
106  return s.ok();
107}
108
109static int CheckFreeSpace(const char* const type,
110                          const base::FilePath& file_name) {
111  std::string name =
112      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
113  int64 free_disk_space_in_k_bytes =
114      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
115  if (free_disk_space_in_k_bytes < 0) {
116    base::Histogram::FactoryGet(
117        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
118        1,
119        2 /*boundary*/,
120        2 /*boundary*/ + 1,
121        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
122    return -1;
123  }
124  int clamped_disk_space_k_bytes =
125      free_disk_space_in_k_bytes > INT_MAX ? INT_MAX
126                                           : free_disk_space_in_k_bytes;
127  const uint64 histogram_max = static_cast<uint64>(1e9);
128  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
129  base::Histogram::FactoryGet(name,
130                              1,
131                              histogram_max,
132                              11 /*buckets*/,
133                              base::HistogramBase::kUmaTargetedHistogramFlag)
134      ->Add(clamped_disk_space_k_bytes);
135  return clamped_disk_space_k_bytes;
136}
137
138static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
139                                            const leveldb::Status& s) {
140  leveldb_env::MethodID method;
141  int error = -1;
142  leveldb_env::ErrorParsingResult result =
143      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
144  if (result == leveldb_env::NONE)
145    return;
146  std::string method_histogram_name(histogram_name);
147  method_histogram_name.append(".EnvMethod");
148  base::LinearHistogram::FactoryGet(
149      method_histogram_name,
150      1,
151      leveldb_env::kNumEntries,
152      leveldb_env::kNumEntries + 1,
153      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
154
155  std::string error_histogram_name(histogram_name);
156
157  if (result == leveldb_env::METHOD_AND_PFE) {
158    DCHECK(error < 0);
159    error_histogram_name.append(std::string(".PFE.") +
160                                leveldb_env::MethodIDToString(method));
161    base::LinearHistogram::FactoryGet(
162        error_histogram_name,
163        1,
164        -base::PLATFORM_FILE_ERROR_MAX,
165        -base::PLATFORM_FILE_ERROR_MAX + 1,
166        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
167  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
168    error_histogram_name.append(std::string(".Errno.") +
169                                leveldb_env::MethodIDToString(method));
170    base::LinearHistogram::FactoryGet(
171        error_histogram_name,
172        1,
173        ERANGE + 1,
174        ERANGE + 2,
175        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
176  }
177}
178
179static void ParseAndHistogramCorruptionDetails(
180    const std::string& histogram_name,
181    const leveldb::Status& status) {
182  DCHECK(!status.IsIOError());
183  DCHECK(!status.ok());
184  const int kOtherError = 0;
185  int error = kOtherError;
186  const std::string& str_error = status.ToString();
187  // Keep in sync with LevelDBCorruptionTypes in histograms.xml.
188  const char* patterns[] = {
189    "missing files",
190    "log record too small",
191    "corrupted internal key",
192    "partial record",
193    "missing start of fragmented record",
194    "error in middle of record",
195    "unknown record type",
196    "truncated record at end",
197    "bad record length",
198    "VersionEdit",
199    "FileReader invoked with unexpected value",
200    "corrupted key",
201    "CURRENT file does not end with newline",
202    "no meta-nextfile entry",
203    "no meta-lognumber entry",
204    "no last-sequence-number entry",
205    "malformed WriteBatch",
206    "bad WriteBatch Put",
207    "bad WriteBatch Delete",
208    "unknown WriteBatch tag",
209    "WriteBatch has wrong count",
210    "bad entry in block",
211    "bad block contents",
212    "bad block handle",
213    "truncated block read",
214    "block checksum mismatch",
215    "checksum mismatch",
216    "corrupted compressed block contents",
217    "bad block type",
218    "bad magic number",
219    "file is too short",
220  };
221  const size_t kNumPatterns = arraysize(patterns);
222  for (size_t i = 0; i < kNumPatterns; ++i) {
223    if (str_error.find(patterns[i]) != std::string::npos) {
224      error = i + 1;
225      break;
226    }
227  }
228  DCHECK(error >= 0);
229  std::string corruption_histogram_name(histogram_name);
230  corruption_histogram_name.append(".Corruption");
231  base::LinearHistogram::FactoryGet(
232      corruption_histogram_name,
233      1,
234      kNumPatterns + 1,
235      kNumPatterns + 2,
236      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
237}
238
239static void HistogramLevelDBError(const std::string& histogram_name,
240                                  const leveldb::Status& s) {
241  if (s.ok()) {
242    NOTREACHED();
243    return;
244  }
245  enum {
246    LEVEL_DB_NOT_FOUND,
247    LEVEL_DB_CORRUPTION,
248    LEVEL_DB_IO_ERROR,
249    LEVEL_DB_OTHER,
250    LEVEL_DB_MAX_ERROR
251  };
252  int leveldb_error = LEVEL_DB_OTHER;
253  if (s.IsNotFound())
254    leveldb_error = LEVEL_DB_NOT_FOUND;
255  else if (s.IsCorruption())
256    leveldb_error = LEVEL_DB_CORRUPTION;
257  else if (s.IsIOError())
258    leveldb_error = LEVEL_DB_IO_ERROR;
259  base::Histogram::FactoryGet(histogram_name,
260                              1,
261                              LEVEL_DB_MAX_ERROR,
262                              LEVEL_DB_MAX_ERROR + 1,
263                              base::HistogramBase::kUmaTargetedHistogramFlag)
264      ->Add(leveldb_error);
265  if (s.IsIOError())
266    ParseAndHistogramIOErrorDetails(histogram_name, s);
267  else
268    ParseAndHistogramCorruptionDetails(histogram_name, s);
269}
270
271leveldb::Status LevelDBDatabase::Open(
272    const base::FilePath& file_name,
273    const LevelDBComparator* comparator,
274    scoped_ptr<LevelDBDatabase>* result,
275    bool* is_disk_full) {
276  scoped_ptr<ComparatorAdapter> comparator_adapter(
277      new ComparatorAdapter(comparator));
278
279  leveldb::DB* db;
280  const leveldb::Status s =
281      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
282
283  if (!s.ok()) {
284    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
285    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
286    // Disks with <100k of free space almost never succeed in opening a
287    // leveldb database.
288    if (is_disk_full)
289      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
290
291    LOG(ERROR) << "Failed to open LevelDB database from "
292               << file_name.AsUTF8Unsafe() << "," << s.ToString();
293    return s;
294  }
295
296  CheckFreeSpace("Success", file_name);
297
298  (*result).reset(new LevelDBDatabase);
299  (*result)->db_ = make_scoped_ptr(db);
300  (*result)->comparator_adapter_ = comparator_adapter.Pass();
301  (*result)->comparator_ = comparator;
302
303  return s;
304}
305
306scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
307    const LevelDBComparator* comparator) {
308  scoped_ptr<ComparatorAdapter> comparator_adapter(
309      new ComparatorAdapter(comparator));
310  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
311
312  leveldb::DB* db;
313  const leveldb::Status s = OpenDB(
314      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
315
316  if (!s.ok()) {
317    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
318    return scoped_ptr<LevelDBDatabase>();
319  }
320
321  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
322  result->env_ = in_memory_env.Pass();
323  result->db_ = make_scoped_ptr(db);
324  result->comparator_adapter_ = comparator_adapter.Pass();
325  result->comparator_ = comparator;
326
327  return result.Pass();
328}
329
330bool LevelDBDatabase::Put(const StringPiece& key, std::string* value) {
331  leveldb::WriteOptions write_options;
332  write_options.sync = true;
333
334  const leveldb::Status s =
335      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
336  if (s.ok())
337    return true;
338  LOG(ERROR) << "LevelDB put failed: " << s.ToString();
339  return false;
340}
341
342bool LevelDBDatabase::Remove(const StringPiece& key) {
343  leveldb::WriteOptions write_options;
344  write_options.sync = true;
345
346  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
347  if (s.ok())
348    return true;
349  if (s.IsNotFound())
350    return false;
351  LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
352  return false;
353}
354
355bool LevelDBDatabase::Get(const StringPiece& key,
356                          std::string* value,
357                          bool* found,
358                          const LevelDBSnapshot* snapshot) {
359  *found = false;
360  leveldb::ReadOptions read_options;
361  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
362                                         // performance impact is too great.
363  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
364
365  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
366  if (s.ok()) {
367    *found = true;
368    return true;
369  }
370  if (s.IsNotFound())
371    return true;
372  HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
373  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
374  return false;
375}
376
377bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
378  leveldb::WriteOptions write_options;
379  write_options.sync = true;
380
381  const leveldb::Status s =
382      db_->Write(write_options, write_batch.write_batch_.get());
383  if (s.ok())
384    return true;
385  HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
386  LOG(ERROR) << "LevelDB write failed: " << s.ToString();
387  return false;
388}
389
390namespace {
391class IteratorImpl : public LevelDBIterator {
392 public:
393  virtual ~IteratorImpl() {}
394
395  virtual bool IsValid() const OVERRIDE;
396  virtual void SeekToLast() OVERRIDE;
397  virtual void Seek(const StringPiece& target) OVERRIDE;
398  virtual void Next() OVERRIDE;
399  virtual void Prev() OVERRIDE;
400  virtual StringPiece Key() const OVERRIDE;
401  virtual StringPiece Value() const OVERRIDE;
402
403 private:
404  friend class content::LevelDBDatabase;
405  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
406  void CheckStatus();
407
408  scoped_ptr<leveldb::Iterator> iterator_;
409};
410}
411
412IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
413    : iterator_(it.Pass()) {}
414
415void IteratorImpl::CheckStatus() {
416  const leveldb::Status s = iterator_->status();
417  if (!s.ok())
418    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
419}
420
421bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
422
423void IteratorImpl::SeekToLast() {
424  iterator_->SeekToLast();
425  CheckStatus();
426}
427
428void IteratorImpl::Seek(const StringPiece& target) {
429  iterator_->Seek(MakeSlice(target));
430  CheckStatus();
431}
432
433void IteratorImpl::Next() {
434  DCHECK(IsValid());
435  iterator_->Next();
436  CheckStatus();
437}
438
439void IteratorImpl::Prev() {
440  DCHECK(IsValid());
441  iterator_->Prev();
442  CheckStatus();
443}
444
445StringPiece IteratorImpl::Key() const {
446  DCHECK(IsValid());
447  return MakeStringPiece(iterator_->key());
448}
449
450StringPiece IteratorImpl::Value() const {
451  DCHECK(IsValid());
452  return MakeStringPiece(iterator_->value());
453}
454
455scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
456    const LevelDBSnapshot* snapshot) {
457  leveldb::ReadOptions read_options;
458  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
459                                         // performance impact is too great.
460  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
461
462  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
463  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
464}
465
466const LevelDBComparator* LevelDBDatabase::Comparator() const {
467  return comparator_;
468}
469
470}  // namespace content
471