leveldb_database.cc revision 1320f92c476a1ad9d19dba2a48c72b75566198e9 
1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/leveldb/leveldb_database.h"
6
7#include <cerrno>
8
9#include "base/basictypes.h"
10#include "base/files/file.h"
11#include "base/logging.h"
12#include "base/memory/scoped_ptr.h"
13#include "base/metrics/histogram.h"
14#include "base/strings/string16.h"
15#include "base/strings/string_piece.h"
16#include "base/strings/stringprintf.h"
17#include "base/strings/utf_string_conversions.h"
18#include "base/sys_info.h"
19#include "content/browser/indexed_db/indexed_db_class_factory.h"
20#include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
21#include "content/browser/indexed_db/leveldb/leveldb_iterator_impl.h"
22#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
23#include "third_party/leveldatabase/env_chromium.h"
24#include "third_party/leveldatabase/env_idb.h"
25#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
26#include "third_party/leveldatabase/src/include/leveldb/db.h"
27#include "third_party/leveldatabase/src/include/leveldb/env.h"
28#include "third_party/leveldatabase/src/include/leveldb/filter_policy.h"
29#include "third_party/leveldatabase/src/include/leveldb/slice.h"
30
31using base::StringPiece;
32
33namespace content {
34
35// Forcing flushes to disk at the end of a transaction guarantees that the
36// data hit disk, but drastically impacts throughput when the filesystem is
37// busy with background compactions. Not syncing trades off reliability for
38// performance. Note that background compactions which move data from the
39// log to SSTs are always done with reliable writes.
40//
41// Sync writes are necessary on Windows for quota calculations; POSIX
42// calculates file sizes correctly even when not synced to disk.
43#if defined(OS_WIN)
44static const bool kSyncWrites = true;
45#else
46// TODO(dgrogan): Either remove the #if block or change this back to false.
47// See http://crbug.com/338385.
48static const bool kSyncWrites = true;
49#endif
50
51static leveldb::Slice MakeSlice(const StringPiece& s) {
52  return leveldb::Slice(s.begin(), s.size());
53}
54
55static StringPiece MakeStringPiece(const leveldb::Slice& s) {
56  return StringPiece(s.data(), s.size());
57}
58
59LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
60    const LevelDBComparator* comparator)
61    : comparator_(comparator) {}
62
63int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
64                                                const leveldb::Slice& b) const {
65  return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
66}
67
68const char* LevelDBDatabase::ComparatorAdapter::Name() const {
69  return comparator_->Name();
70}
71
72// TODO(jsbell): Support the methods below in the future.
73void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
74    std::string* start,
75    const leveldb::Slice& limit) const {}
76
77void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
78    std::string* key) const {}
79
80LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
81    : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
82
83LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
84
85LevelDBDatabase::LevelDBDatabase() {}
86
87LevelDBDatabase::~LevelDBDatabase() {
88  // db_'s destructor uses comparator_adapter_; order of deletion is important.
89  db_.reset();
90  comparator_adapter_.reset();
91  env_.reset();
92}
93
94static leveldb::Status OpenDB(
95    leveldb::Comparator* comparator,
96    leveldb::Env* env,
97    const base::FilePath& path,
98    leveldb::DB** db,
99    scoped_ptr<const leveldb::FilterPolicy>* filter_policy) {
100  filter_policy->reset(leveldb::NewBloomFilterPolicy(10));
101  leveldb::Options options;
102  options.comparator = comparator;
103  options.create_if_missing = true;
104  options.paranoid_checks = true;
105  options.filter_policy = filter_policy->get();
106  options.compression = leveldb::kSnappyCompression;
107
108  // For info about the troubles we've run into with this parameter, see:
109  // https://code.google.com/p/chromium/issues/detail?id=227313#c11
110  options.max_open_files = 80;
111  options.env = env;
112
113  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
114  leveldb::Status s = leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
115
116  return s;
117}
118
119leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
120  leveldb::Options options;
121  options.env = leveldb::IDBEnv();
122  // ChromiumEnv assumes UTF8, converts back to FilePath before using.
123  return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
124}
125
126namespace {
127class LockImpl : public LevelDBLock {
128 public:
129  explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
130      : env_(env), lock_(lock) {}
131  virtual ~LockImpl() { env_->UnlockFile(lock_); }
132 private:
133  leveldb::Env* env_;
134  leveldb::FileLock* lock_;
135
136  DISALLOW_COPY_AND_ASSIGN(LockImpl);
137};
138}  // namespace
139
140scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
141    const base::FilePath& file_name) {
142  leveldb::Env* env = leveldb::IDBEnv();
143  base::FilePath lock_path = file_name.AppendASCII("LOCK");
144  leveldb::FileLock* lock = NULL;
145  leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
146  if (!status.ok())
147    return scoped_ptr<LevelDBLock>();
148  DCHECK(lock);
149  return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
150}
151
152static int CheckFreeSpace(const char* const type,
153                          const base::FilePath& file_name) {
154  std::string name =
155      std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
156  int64 free_disk_space_in_k_bytes =
157      base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
158  if (free_disk_space_in_k_bytes < 0) {
159    base::Histogram::FactoryGet(
160        "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
161        1,
162        2 /*boundary*/,
163        2 /*boundary*/ + 1,
164        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
165    return -1;
166  }
167  int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
168                                       ? INT_MAX
169                                       : free_disk_space_in_k_bytes;
170  const uint64 histogram_max = static_cast<uint64>(1e9);
171  COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
172  base::Histogram::FactoryGet(name,
173                              1,
174                              histogram_max,
175                              11 /*buckets*/,
176                              base::HistogramBase::kUmaTargetedHistogramFlag)
177      ->Add(clamped_disk_space_k_bytes);
178  return clamped_disk_space_k_bytes;
179}
180
181static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
182                                            const leveldb::Status& s) {
183  leveldb_env::MethodID method;
184  int error = -1;
185  leveldb_env::ErrorParsingResult result =
186      leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
187  if (result == leveldb_env::NONE)
188    return;
189  std::string method_histogram_name(histogram_name);
190  method_histogram_name.append(".EnvMethod");
191  base::LinearHistogram::FactoryGet(
192      method_histogram_name,
193      1,
194      leveldb_env::kNumEntries,
195      leveldb_env::kNumEntries + 1,
196      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
197
198  std::string error_histogram_name(histogram_name);
199
200  if (result == leveldb_env::METHOD_AND_PFE) {
201    DCHECK_LT(error, 0);
202    error_histogram_name.append(std::string(".PFE.") +
203                                leveldb_env::MethodIDToString(method));
204    base::LinearHistogram::FactoryGet(
205        error_histogram_name,
206        1,
207        -base::File::FILE_ERROR_MAX,
208        -base::File::FILE_ERROR_MAX + 1,
209        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
210  } else if (result == leveldb_env::METHOD_AND_ERRNO) {
211    error_histogram_name.append(std::string(".Errno.") +
212                                leveldb_env::MethodIDToString(method));
213    base::LinearHistogram::FactoryGet(
214        error_histogram_name,
215        1,
216        ERANGE + 1,
217        ERANGE + 2,
218        base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
219  }
220}
221
222static void ParseAndHistogramCorruptionDetails(
223    const std::string& histogram_name,
224    const leveldb::Status& status) {
225  int error = leveldb_env::GetCorruptionCode(status);
226  DCHECK_GE(error, 0);
227  std::string corruption_histogram_name(histogram_name);
228  corruption_histogram_name.append(".Corruption");
229  const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
230  base::LinearHistogram::FactoryGet(
231      corruption_histogram_name,
232      1,
233      kNumPatterns,
234      kNumPatterns + 1,
235      base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
236}
237
238static void HistogramLevelDBError(const std::string& histogram_name,
239                                  const leveldb::Status& s) {
240  if (s.ok()) {
241    NOTREACHED();
242    return;
243  }
244  enum {
245    LEVEL_DB_NOT_FOUND,
246    LEVEL_DB_CORRUPTION,
247    LEVEL_DB_IO_ERROR,
248    LEVEL_DB_OTHER,
249    LEVEL_DB_MAX_ERROR
250  };
251  int leveldb_error = LEVEL_DB_OTHER;
252  if (s.IsNotFound())
253    leveldb_error = LEVEL_DB_NOT_FOUND;
254  else if (s.IsCorruption())
255    leveldb_error = LEVEL_DB_CORRUPTION;
256  else if (s.IsIOError())
257    leveldb_error = LEVEL_DB_IO_ERROR;
258  base::Histogram::FactoryGet(histogram_name,
259                              1,
260                              LEVEL_DB_MAX_ERROR,
261                              LEVEL_DB_MAX_ERROR + 1,
262                              base::HistogramBase::kUmaTargetedHistogramFlag)
263      ->Add(leveldb_error);
264  if (s.IsIOError())
265    ParseAndHistogramIOErrorDetails(histogram_name, s);
266  else
267    ParseAndHistogramCorruptionDetails(histogram_name, s);
268}
269
270leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
271                                      const LevelDBComparator* comparator,
272                                      scoped_ptr<LevelDBDatabase>* result,
273                                      bool* is_disk_full) {
274  base::TimeTicks begin_time = base::TimeTicks::Now();
275
276  scoped_ptr<ComparatorAdapter> comparator_adapter(
277      new ComparatorAdapter(comparator));
278
279  leveldb::DB* db;
280  scoped_ptr<const leveldb::FilterPolicy> filter_policy;
281  const leveldb::Status s = OpenDB(comparator_adapter.get(),
282                                   leveldb::IDBEnv(),
283                                   file_name,
284                                   &db,
285                                   &filter_policy);
286
287  if (!s.ok()) {
288    HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
289    int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
290    // Disks with <100k of free space almost never succeed in opening a
291    // leveldb database.
292    if (is_disk_full)
293      *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
294
295    LOG(ERROR) << "Failed to open LevelDB database from "
296               << file_name.AsUTF8Unsafe() << "," << s.ToString();
297    return s;
298  }
299
300  UMA_HISTOGRAM_MEDIUM_TIMES("WebCore.IndexedDB.LevelDB.OpenTime",
301                             base::TimeTicks::Now() - begin_time);
302
303  CheckFreeSpace("Success", file_name);
304
305  (*result).reset(new LevelDBDatabase);
306  (*result)->db_ = make_scoped_ptr(db);
307  (*result)->comparator_adapter_ = comparator_adapter.Pass();
308  (*result)->comparator_ = comparator;
309  (*result)->filter_policy_ = filter_policy.Pass();
310
311  return s;
312}
313
314scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
315    const LevelDBComparator* comparator) {
316  scoped_ptr<ComparatorAdapter> comparator_adapter(
317      new ComparatorAdapter(comparator));
318  scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
319
320  leveldb::DB* db;
321  scoped_ptr<const leveldb::FilterPolicy> filter_policy;
322  const leveldb::Status s = OpenDB(comparator_adapter.get(),
323                                   in_memory_env.get(),
324                                   base::FilePath(),
325                                   &db,
326                                   &filter_policy);
327
328  if (!s.ok()) {
329    LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
330    return scoped_ptr<LevelDBDatabase>();
331  }
332
333  scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
334  result->env_ = in_memory_env.Pass();
335  result->db_ = make_scoped_ptr(db);
336  result->comparator_adapter_ = comparator_adapter.Pass();
337  result->comparator_ = comparator;
338  result->filter_policy_ = filter_policy.Pass();
339
340  return result.Pass();
341}
342
343leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
344                                     std::string* value) {
345  base::TimeTicks begin_time = base::TimeTicks::Now();
346
347  leveldb::WriteOptions write_options;
348  write_options.sync = kSyncWrites;
349
350  const leveldb::Status s =
351      db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
352  if (!s.ok())
353    LOG(ERROR) << "LevelDB put failed: " << s.ToString();
354  else
355    UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.PutTime",
356                        base::TimeTicks::Now() - begin_time);
357  return s;
358}
359
360leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
361  leveldb::WriteOptions write_options;
362  write_options.sync = kSyncWrites;
363
364  const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
365  if (!s.IsNotFound())
366    LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
367  return s;
368}
369
370leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
371                                     std::string* value,
372                                     bool* found,
373                                     const LevelDBSnapshot* snapshot) {
374  *found = false;
375  leveldb::ReadOptions read_options;
376  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
377                                         // performance impact is too great.
378  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
379
380  const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
381  if (s.ok()) {
382    *found = true;
383    return s;
384  }
385  if (s.IsNotFound())
386    return leveldb::Status::OK();
387  HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
388  LOG(ERROR) << "LevelDB get failed: " << s.ToString();
389  return s;
390}
391
392leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
393  base::TimeTicks begin_time = base::TimeTicks::Now();
394  leveldb::WriteOptions write_options;
395  write_options.sync = kSyncWrites;
396
397  const leveldb::Status s =
398      db_->Write(write_options, write_batch.write_batch_.get());
399  if (!s.ok()) {
400    HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
401    LOG(ERROR) << "LevelDB write failed: " << s.ToString();
402  } else {
403    UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.WriteTime",
404                        base::TimeTicks::Now() - begin_time);
405  }
406  return s;
407}
408
409scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
410    const LevelDBSnapshot* snapshot) {
411  leveldb::ReadOptions read_options;
412  read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
413                                         // performance impact is too great.
414  read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
415
416  scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
417  return scoped_ptr<LevelDBIterator>(
418      IndexedDBClassFactory::Get()->CreateIteratorImpl(i.Pass()));
419}
420
421const LevelDBComparator* LevelDBDatabase::Comparator() const {
422  return comparator_;
423}
424
425void LevelDBDatabase::Compact(const base::StringPiece& start,
426                              const base::StringPiece& stop) {
427  const leveldb::Slice start_slice = MakeSlice(start);
428  const leveldb::Slice stop_slice = MakeSlice(stop);
429  // NULL batch means just wait for earlier writes to be done
430  db_->Write(leveldb::WriteOptions(), NULL);
431  db_->CompactRange(&start_slice, &stop_slice);
432}
433
434void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
435
436}  // namespace content
437