leveldb_database.cc revision 7d4cd473f85ac64c3747c96c277f9e506a0d2246 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <string>
8
9#include "base/basictypes.h"
10#include "base/logging.h"
11#include "base/memory/scoped_ptr.h"
12#include "base/metrics/histogram.h"
13#include "base/strings/string16.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/sys_info.h"
16#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
17#include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
18#include "content/browser/indexed_db/leveldb/leveldb_slice.h"
19#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
20#include "third_party/leveldatabase/env_chromium.h"
21#include "third_party/leveldatabase/env_idb.h"
22#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
23#include "third_party/leveldatabase/src/include/leveldb/comparator.h"
24#include "third_party/leveldatabase/src/include/leveldb/db.h"
25#include "third_party/leveldatabase/src/include/leveldb/env.h"
26#include "third_party/leveldatabase/src/include/leveldb/slice.h"
27
28namespace content {
29
30static leveldb::Slice MakeSlice(const std::vector<char>& value) {
31  DCHECK_GT(value.size(), static_cast<size_t>(0));
32  return leveldb::Slice(&*value.begin(), value.size());
33}
34
35static leveldb::Slice MakeSlice(const LevelDBSlice& s) {
36  return leveldb::Slice(s.begin(), s.end() - s.begin());
37}
38
39static LevelDBSlice MakeLevelDBSlice(const leveldb::Slice& s) {
40  return LevelDBSlice(s.data(), s.data() + s.size());
41}
42
43class ComparatorAdapter : public leveldb::Comparator {
44 public:
45  explicit ComparatorAdapter(const LevelDBComparator* comparator)
46      : comparator_(comparator) {}
47
48  virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
49      OVERRIDE {
50    return comparator_->Compare(MakeLevelDBSlice(a), MakeLevelDBSlice(b));
51  }
52
53  virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
54
55  // TODO(jsbell): Support the methods below in the future.
56  virtual void FindShortestSeparator(std::string* start,
57                                     const leveldb::Slice& limit) const
58      OVERRIDE {}
59  virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
60
61 private:
62  const LevelDBComparator* comparator_;
63};
64
65LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
66    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
67
68LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
69
70LevelDBDatabase::LevelDBDatabase() {}
71
72LevelDBDatabase::~LevelDBDatabase() {
73  // db_'s destructor uses comparator_adapter_; order of deletion is important.
74  db_.reset();
75  comparator_adapter_.reset();
76  env_.reset();
77}
78
79static leveldb::Status OpenDB(leveldb::Comparator* comparator,
80                              leveldb::Env* env,
81                              const base::FilePath& path,
82                              leveldb::DB** db) {
83  leveldb::Options options;
84  options.comparator = comparator;
85  options.create_if_missing = true;
86  options.paranoid_checks = true;
87
88  // Marking compression as explicitly off so snappy support can be
89  // compiled in for other leveldb clients without implicitly enabling
90  // it for IndexedDB. http://crbug.com/81384
91  options.compression = leveldb::kNoCompression;
92
93  // 20 max_open_files is the minimum LevelDB allows but its cache behaves
94  // poorly with less than 4 files per shard. As of this writing the latest
95  // leveldb (1.9) hardcodes 16 shards. See
96  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
97  options.max_open_files = 80;
98  options.env = env;
99
100  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
101  return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
102}
103
104bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
105  leveldb::Options options;
106  options.env = leveldb::IDBEnv();
107  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
108  const leveldb::Status s =
109      leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
110  return s.ok();
111}
112
113static int CheckFreeSpace(const char* type, const base::FilePath& file_name) {
114  // TODO(dgrogan): Change string16 -> std::string.
115  string16 name = ASCIIToUTF16("WebCore.IndexedDB.LevelDB.Open") +
116                  ASCIIToUTF16(type) + ASCIIToUTF16("FreeDiskSpace");
117  int64 free_disk_space_in_k_bytes =
118      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
119  if (free_disk_space_in_k_bytes < 0) {
120    base::Histogram::FactoryGet(
121        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
122        1,
123        2 /*boundary*/,
124        2 /*boundary*/ + 1,
125        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
126    return -1;
127  }
128  int clamped_disk_space_k_bytes =
129      free_disk_space_in_k_bytes > INT_MAX ? INT_MAX
130                                           : free_disk_space_in_k_bytes;
131  const uint64 histogram_max = static_cast<uint64>(1e9);
132  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
133  base::Histogram::FactoryGet(UTF16ToUTF8(name),
134                              1,
135                              histogram_max,
136                              11 /*buckets*/,
137                              base::HistogramBase::kUmaTargetedHistogramFlag)
138      ->Add(clamped_disk_space_k_bytes);
139  return clamped_disk_space_k_bytes;
140}
141
142static void HistogramLevelDBError(const std::string& histogram_name,
143                                  const leveldb::Status& s) {
144  DCHECK(!s.ok());
145  enum {
146    LEVEL_DB_NOT_FOUND,
147    LEVEL_DB_CORRUPTION,
148    LEVEL_DB_IO_ERROR,
149    LEVEL_DB_OTHER,
150    LEVEL_DB_MAX_ERROR
151  };
152  int leveldb_error = LEVEL_DB_OTHER;
153  if (s.IsNotFound())
154    leveldb_error = LEVEL_DB_NOT_FOUND;
155  else if (s.IsCorruption())
156    leveldb_error = LEVEL_DB_CORRUPTION;
157  else if (s.IsIOError())
158    leveldb_error = LEVEL_DB_IO_ERROR;
159  base::Histogram::FactoryGet(histogram_name,
160                              1,
161                              LEVEL_DB_MAX_ERROR,
162                              LEVEL_DB_MAX_ERROR + 1,
163                              base::HistogramBase::kUmaTargetedHistogramFlag)
164      ->Add(leveldb_error);
165
166  // The code above histograms the type of error. The code below tries to
167  // histogram the method where the error occurred in ChromiumEnv and, in most
168  // cases, the exact error encountered.
169  leveldb_env::MethodID method;
170  int error = -1;
171  leveldb_env::ErrorParsingResult result =
172      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
173  if (result == leveldb_env::NONE)
174    return;
175  std::string method_histogram_name(histogram_name);
176  method_histogram_name.append(".EnvMethod");
177  base::LinearHistogram::FactoryGet(
178      method_histogram_name,
179      1,
180      leveldb_env::kNumEntries,
181      leveldb_env::kNumEntries + 1,
182      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
183}
184
185scoped_ptr<LevelDBDatabase> LevelDBDatabase::Open(
186    const base::FilePath& file_name,
187    const LevelDBComparator* comparator,
188    bool* is_disk_full) {
189  scoped_ptr<ComparatorAdapter> comparator_adapter(
190      new ComparatorAdapter(comparator));
191
192  leveldb::DB* db;
193  const leveldb::Status s =
194      OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
195
196  if (!s.ok()) {
197    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
198    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
199    // Disks with <100k of free space almost never succeed in opening a
200    // leveldb database.
201    if (is_disk_full)
202      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
203
204    LOG(ERROR) << "Failed to open LevelDB database from "
205               << file_name.AsUTF8Unsafe() << "," << s.ToString();
206    return scoped_ptr<LevelDBDatabase>();
207  }
208
209  CheckFreeSpace("Success", file_name);
210
211  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
212  result->db_ = make_scoped_ptr(db);
213  result->comparator_adapter_ = comparator_adapter.Pass();
214  result->comparator_ = comparator;
215
216  return result.Pass();
217}
218
219scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
220    const LevelDBComparator* comparator) {
221  scoped_ptr<ComparatorAdapter> comparator_adapter(
222      new ComparatorAdapter(comparator));
223  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
224
225  leveldb::DB* db;
226  const leveldb::Status s = OpenDB(
227      comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
228
229  if (!s.ok()) {
230    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
231    return scoped_ptr<LevelDBDatabase>();
232  }
233
234  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
235  result->env_ = in_memory_env.Pass();
236  result->db_ = make_scoped_ptr(db);
237  result->comparator_adapter_ = comparator_adapter.Pass();
238  result->comparator_ = comparator;
239
240  return result.Pass();
241}
242
243bool LevelDBDatabase::Put(const LevelDBSlice& key,
244                          std::vector<char>* value) {
245  leveldb::WriteOptions write_options;
246  write_options.sync = true;
247
248  const leveldb::Status s =
249      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
250  if (s.ok())
251    return true;
252  LOG(ERROR) << "LevelDB put failed: " << s.ToString();
253  return false;
254}
255
256bool LevelDBDatabase::Remove(const LevelDBSlice& key) {
257  leveldb::WriteOptions write_options;
258  write_options.sync = true;
259
260  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
261  if (s.ok())
262    return true;
263  if (s.IsNotFound())
264    return false;
265  LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
266  return false;
267}
268
269bool LevelDBDatabase::Get(const LevelDBSlice& key,
270                          std::string* value,
271                          bool* found,
272                          const LevelDBSnapshot* snapshot) {
273  *found = false;
274  leveldb::ReadOptions read_options;
275  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
276                                         // performance impact is too great.
277  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
278
279  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
280  if (s.ok()) {
281    *found = true;
282    return true;
283  }
284  if (s.IsNotFound())
285    return true;
286  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
287  return false;
288}
289
290bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
291  leveldb::WriteOptions write_options;
292  write_options.sync = true;
293
294  const leveldb::Status s =
295      db_->Write(write_options, write_batch.write_batch_.get());
296  if (s.ok())
297    return true;
298  HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
299  LOG(ERROR) << "LevelDB write failed: " << s.ToString();
300  return false;
301}
302
303namespace {
304class IteratorImpl : public LevelDBIterator {
305 public:
306  virtual ~IteratorImpl() {}
307
308  virtual bool IsValid() const OVERRIDE;
309  virtual void SeekToLast() OVERRIDE;
310  virtual void Seek(const LevelDBSlice& target) OVERRIDE;
311  virtual void Next() OVERRIDE;
312  virtual void Prev() OVERRIDE;
313  virtual LevelDBSlice Key() const OVERRIDE;
314  virtual LevelDBSlice Value() const OVERRIDE;
315
316 private:
317  friend class content::LevelDBDatabase;
318  explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
319  void CheckStatus();
320
321  scoped_ptr<leveldb::Iterator> iterator_;
322};
323}
324
325IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
326    : iterator_(it.Pass()) {}
327
328void IteratorImpl::CheckStatus() {
329  const leveldb::Status s = iterator_->status();
330  if (!s.ok())
331    LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
332}
333
334bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
335
336void IteratorImpl::SeekToLast() {
337  iterator_->SeekToLast();
338  CheckStatus();
339}
340
341void IteratorImpl::Seek(const LevelDBSlice& target) {
342  iterator_->Seek(MakeSlice(target));
343  CheckStatus();
344}
345
346void IteratorImpl::Next() {
347  DCHECK(IsValid());
348  iterator_->Next();
349  CheckStatus();
350}
351
352void IteratorImpl::Prev() {
353  DCHECK(IsValid());
354  iterator_->Prev();
355  CheckStatus();
356}
357
358LevelDBSlice IteratorImpl::Key() const {
359  DCHECK(IsValid());
360  return MakeLevelDBSlice(iterator_->key());
361}
362
363LevelDBSlice IteratorImpl::Value() const {
364  DCHECK(IsValid());
365  return MakeLevelDBSlice(iterator_->value());
366}
367
368scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
369    const LevelDBSnapshot* snapshot) {
370  leveldb::ReadOptions read_options;
371  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
372                                         // performance impact is too great.
373  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
374  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
375  if (!i)  // TODO(jsbell): Double check if we actually need to check this.
376    return scoped_ptr<LevelDBIterator>();
377  return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
378}
379
380const LevelDBComparator* LevelDBDatabase::Comparator() const {
381  return comparator_;
382}
383
384}  // namespace content
385