leveldb_database.cc revision eb525c5499e34cc9c4b825d6d9e75bb07cc06ace 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8#include <string>
9
10#include "base/basictypes.h"
11#include "base/logging.h"
12#include "base/memory/scoped_ptr.h"
13#include "base/metrics/histogram.h"
14#include "base/strings/string16.h"
15#include "base/strings/stringprintf.h"
16#include "base/strings/utf_string_conversions.h"
17#include "base/sys_info.h"
18#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
19#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
20#include "content/browser/indexed_db/leveldb/leveldb_slice.h"
21#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
22#include "third_party/leveldatabase/env_chromium.h"
23#include "third_party/leveldatabase/env_idb.h"
24#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
25#include "third_party/leveldatabase/src/include/leveldb/comparator.h"
26#include "third_party/leveldatabase/src/include/leveldb/db.h"
27#include "third_party/leveldatabase/src/include/leveldb/env.h"
28#include "third_party/leveldatabase/src/include/leveldb/slice.h"
29
30namespace content {
31
32static leveldb::Slice MakeSlice(const std::vector<char>& value) {
33  DCHECK_GT(value.size(), static_cast<size_t>(0));
34  return leveldb::Slice(&*value.begin(), value.size());
35}
36
37static leveldb::Slice MakeSlice(const LevelDBSlice& s) {
38  return leveldb::Slice(s.begin(), s.end() - s.begin());
39}
40
41static LevelDBSlice MakeLevelDBSlice(const leveldb::Slice& s) {
42  return LevelDBSlice(s.data(), s.data() + s.size());
43}
44
45class ComparatorAdapter : public leveldb::Comparator {
46 public:
47  explicit ComparatorAdapter(const LevelDBComparator* comparator)
48      : comparator_(comparator) {}
49
50  virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
51      OVERRIDE {
52    return comparator_->Compare(MakeLevelDBSlice(a), MakeLevelDBSlice(b));
53  }
54
55  virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
56
57  // TODO(jsbell): Support the methods below in the future.
58  virtual void FindShortestSeparator(std::string* start,
59                                     const leveldb::Slice& limit) const
60      OVERRIDE {}
61  virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
62
63 private:
64  const LevelDBComparator* comparator_;
65};
66
67LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
68    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
69
70LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
71
72LevelDBDatabase::LevelDBDatabase() {}
73
74LevelDBDatabase::~LevelDBDatabase() {
75  // db_'s destructor uses comparator_adapter_; order of deletion is important.
76  db_.reset();
77  comparator_adapter_.reset();
78  env_.reset();
79}
80
81static leveldb::Status OpenDB(leveldb::Comparator* comparator,
82                              leveldb::Env* env,
83                              const base::FilePath& path,
84                              leveldb::DB** db) {
85  leveldb::Options options;
86  options.comparator = comparator;
87  options.create_if_missing = true;
88  options.paranoid_checks = true;
89
90  // Marking compression as explicitly off so snappy support can be
91  // compiled in for other leveldb clients without implicitly enabling
92  // it for IndexedDB. http://crbug.com/81384
93  options.compression = leveldb::kNoCompression;
94
95  // For info about the troubles we've run into with this parameter, see:
96  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
97  options.max_open_files = 80;
98  options.env = env;
99
100  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
101  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
102}
103
104bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
105  leveldb::Options options;
106  options.env = leveldb::IDBEnv();
107  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
108  const leveldb::Status s =
109      leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
110  return s.ok();
111}
112
113static int CheckFreeSpace(const char* const type,
114                          const base::FilePath& file_name) {
115  std::string name =
116      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
117  int64 free_disk_space_in_k_bytes =
118      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
119  if (free_disk_space_in_k_bytes < 0) {
120    base::Histogram::FactoryGet(
121        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
122        1,
123        2 /*boundary*/,
124        2 /*boundary*/ + 1,
125        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
126    return -1;
127  }
128  int clamped_disk_space_k_bytes =
129      free_disk_space_in_k_bytes > INT_MAX ? INT_MAX
130                                           : free_disk_space_in_k_bytes;
131  const uint64 histogram_max = static_cast<uint64>(1e9);
132  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
133  base::Histogram::FactoryGet(name,
134                              1,
135                              histogram_max,
136                              11 /*buckets*/,
137                              base::HistogramBase::kUmaTargetedHistogramFlag)
138      ->Add(clamped_disk_space_k_bytes);
139  return clamped_disk_space_k_bytes;
140}
141
142static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
143                                            const leveldb::Status& s) {
144  leveldb_env::MethodID method;
145  int error = -1;
146  leveldb_env::ErrorParsingResult result =
147      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
148  if (result == leveldb_env::NONE)
149    return;
150  std::string method_histogram_name(histogram_name);
151  method_histogram_name.append(".EnvMethod");
152  base::LinearHistogram::FactoryGet(
153      method_histogram_name,
154      1,
155      leveldb_env::kNumEntries,
156      leveldb_env::kNumEntries + 1,
157      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
158
159  std::string error_histogram_name(histogram_name);
160
161  if (result == leveldb_env::METHOD_AND_PFE) {
162    DCHECK(error < 0);
163    error_histogram_name.append(std::string(".PFE.") +
164                                leveldb_env::MethodIDToString(method));
165    base::LinearHistogram::FactoryGet(
166        error_histogram_name,
167        1,
168        -base::PLATFORM_FILE_ERROR_MAX,
169        -base::PLATFORM_FILE_ERROR_MAX + 1,
170        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
171  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
172    error_histogram_name.append(std::string(".Errno.") +
173                                leveldb_env::MethodIDToString(method));
174    base::LinearHistogram::FactoryGet(
175        error_histogram_name,
176        1,
177        ERANGE + 1,
178        ERANGE + 2,
179        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
180  }
181}
182
183static void ParseAndHistogramCorruptionDetails(
184    const std::string& histogram_name,
185    const leveldb::Status& status) {
186  DCHECK(!status.IsIOError());
187  DCHECK(!status.ok());
188  const int kOtherError = 0;
189  int error = kOtherError;
190  const std::string& str_error = status.ToString();
191  // Keep in sync with LevelDBCorruptionTypes in histograms.xml.
192  const char* patterns[] = {
193    "missing files",
194    "log record too small",
195    "corrupted internal key",
196    "partial record",
197    "missing start of fragmented record",
198    "error in middle of record",
199    "unknown record type",
200    "truncated record at end",
201    "bad record length",
202    "VersionEdit",
203    "FileReader invoked with unexpected value",
204    "corrupted key",
205    "CURRENT file does not end with newline",
206    "no meta-nextfile entry",
207    "no meta-lognumber entry",
208    "no last-sequence-number entry",
209    "malformed WriteBatch",
210    "bad WriteBatch Put",
211    "bad WriteBatch Delete",
212    "unknown WriteBatch tag",
213    "WriteBatch has wrong count",
214    "bad entry in block",
215    "bad block contents",
216    "bad block handle",
217    "truncated block read",
218    "block checksum mismatch",
219    "checksum mismatch",
220    "corrupted compressed block contents",
221    "bad block type",
222    "bad magic number",
223    "file is too short",
224  };
225  const size_t kNumPatterns = arraysize(patterns);
226  for (size_t i = 0; i < kNumPatterns; ++i) {
227    if (str_error.find(patterns[i]) != std::string::npos) {
228      error = i + 1;
229      break;
230    }
231  }
232  DCHECK(error >= 0);
233  std::string corruption_histogram_name(histogram_name);
234  corruption_histogram_name.append(".Corruption");
235  base::LinearHistogram::FactoryGet(
236      corruption_histogram_name,
237      1,
238      kNumPatterns + 1,
239      kNumPatterns + 2,
240      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
241}
242
243static void HistogramLevelDBError(const std::string& histogram_name,
244                                  const leveldb::Status& s) {
245  if (s.ok()) {
246    NOTREACHED();
247    return;
248  }
249  enum {
250    LEVEL_DB_NOT_FOUND,
251    LEVEL_DB_CORRUPTION,
252    LEVEL_DB_IO_ERROR,
253    LEVEL_DB_OTHER,
254    LEVEL_DB_MAX_ERROR
255  };
256  int leveldb_error = LEVEL_DB_OTHER;
257  if (s.IsNotFound())
258    leveldb_error = LEVEL_DB_NOT_FOUND;
259  else if (s.IsCorruption())
260    leveldb_error = LEVEL_DB_CORRUPTION;
261  else if (s.IsIOError())
262    leveldb_error = LEVEL_DB_IO_ERROR;
263  base::Histogram::FactoryGet(histogram_name,
264                              1,
265                              LEVEL_DB_MAX_ERROR,
266                              LEVEL_DB_MAX_ERROR + 1,
267                              base::HistogramBase::kUmaTargetedHistogramFlag)
268      ->Add(leveldb_error);
269  if (s.IsIOError())
270    ParseAndHistogramIOErrorDetails(histogram_name, s);
271  else
272    ParseAndHistogramCorruptionDetails(histogram_name, s);
273}
274
275scoped_ptr<LevelDBDatabase> LevelDBDatabase::Open(
276    const base::FilePath& file_name,
277    const LevelDBComparator* comparator,
278    bool* is_disk_full) {
279  scoped_ptr<ComparatorAdapter> comparator_adapter(
280      new ComparatorAdapter(comparator));
281
282  leveldb::DB* db;
283  const leveldb::Status s =
284      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
285
286  if (!s.ok()) {
287    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
288    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
289    // Disks with <100k of free space almost never succeed in opening a
290    // leveldb database.
291    if (is_disk_full)
292      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
293
294    LOG(ERROR) << "Failed to open LevelDB database from "
295               << file_name.AsUTF8Unsafe() << "," << s.ToString();
296    return scoped_ptr<LevelDBDatabase>();
297  }
298
299  CheckFreeSpace("Success", file_name);
300
301  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
302  result->db_ = make_scoped_ptr(db);
303  result->comparator_adapter_ = comparator_adapter.Pass();
304  result->comparator_ = comparator;
305
306  return result.Pass();
307}
308
309scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
310    const LevelDBComparator* comparator) {
311  scoped_ptr<ComparatorAdapter> comparator_adapter(
312      new ComparatorAdapter(comparator));
313  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
314
315  leveldb::DB* db;
316  const leveldb::Status s = OpenDB(
317      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
318
319  if (!s.ok()) {
320    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
321    return scoped_ptr<LevelDBDatabase>();
322  }
323
324  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
325  result->env_ = in_memory_env.Pass();
326  result->db_ = make_scoped_ptr(db);
327  result->comparator_adapter_ = comparator_adapter.Pass();
328  result->comparator_ = comparator;
329
330  return result.Pass();
331}
332
333bool LevelDBDatabase::Put(const LevelDBSlice& key,
334                          std::vector<char>* value) {
335  leveldb::WriteOptions write_options;
336  write_options.sync = true;
337
338  const leveldb::Status s =
339      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
340  if (s.ok())
341    return true;
342  LOG(ERROR) << "LevelDB put failed: " << s.ToString();
343  return false;
344}
345
346bool LevelDBDatabase::Remove(const LevelDBSlice& key) {
347  leveldb::WriteOptions write_options;
348  write_options.sync = true;
349
350  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
351  if (s.ok())
352    return true;
353  if (s.IsNotFound())
354    return false;
355  LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
356  return false;
357}
358
359bool LevelDBDatabase::Get(const LevelDBSlice& key,
360                          std::string* value,
361                          bool* found,
362                          const LevelDBSnapshot* snapshot) {
363  *found = false;
364  leveldb::ReadOptions read_options;
365  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
366                                         // performance impact is too great.
367  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
368
369  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
370  if (s.ok()) {
371    *found = true;
372    return true;
373  }
374  if (s.IsNotFound())
375    return true;
376  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
377  return false;
378}
379
380bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
381  leveldb::WriteOptions write_options;
382  write_options.sync = true;
383
384  const leveldb::Status s =
385      db_->Write(write_options, write_batch.write_batch_.get());
386  if (s.ok())
387    return true;
388  HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
389  LOG(ERROR) << "LevelDB write failed: " << s.ToString();
390  return false;
391}
392
393namespace {
394class IteratorImpl : public LevelDBIterator {
395 public:
396  virtual ~IteratorImpl() {}
397
398  virtual bool IsValid() const OVERRIDE;
399  virtual void SeekToLast() OVERRIDE;
400  virtual void Seek(const LevelDBSlice& target) OVERRIDE;
401  virtual void Next() OVERRIDE;
402  virtual void Prev() OVERRIDE;
403  virtual LevelDBSlice Key() const OVERRIDE;
404  virtual LevelDBSlice Value() const OVERRIDE;
405
406 private:
407  friend class content::LevelDBDatabase;
408  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
409  void CheckStatus();
410
411  scoped_ptr<leveldb::Iterator> iterator_;
412};
413}
414
415IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
416    : iterator_(it.Pass()) {}
417
418void IteratorImpl::CheckStatus() {
419  const leveldb::Status s = iterator_->status();
420  if (!s.ok())
421    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
422}
423
424bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
425
426void IteratorImpl::SeekToLast() {
427  iterator_->SeekToLast();
428  CheckStatus();
429}
430
431void IteratorImpl::Seek(const LevelDBSlice& target) {
432  iterator_->Seek(MakeSlice(target));
433  CheckStatus();
434}
435
436void IteratorImpl::Next() {
437  DCHECK(IsValid());
438  iterator_->Next();
439  CheckStatus();
440}
441
442void IteratorImpl::Prev() {
443  DCHECK(IsValid());
444  iterator_->Prev();
445  CheckStatus();
446}
447
448LevelDBSlice IteratorImpl::Key() const {
449  DCHECK(IsValid());
450  return MakeLevelDBSlice(iterator_->key());
451}
452
453LevelDBSlice IteratorImpl::Value() const {
454  DCHECK(IsValid());
455  return MakeLevelDBSlice(iterator_->value());
456}
457
458scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
459    const LevelDBSnapshot* snapshot) {
460  leveldb::ReadOptions read_options;
461  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
462                                         // performance impact is too great.
463  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
464
465  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
466  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
467}
468
469const LevelDBComparator* LevelDBDatabase::Comparator() const {
470  return comparator_;
471}
472
473}  // namespace content
474