leveldb_database.cc revision e5d81f57cb97b3b6b7fccc9c5610d21eb81db09d
1// Copyright (c) 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/browser/indexed_db/leveldb/leveldb_database.h" 6 7#include <cerrno> 8 9#include "base/basictypes.h" 10#include "base/files/file.h" 11#include "base/logging.h" 12#include "base/memory/scoped_ptr.h" 13#include "base/metrics/histogram.h" 14#include "base/strings/string16.h" 15#include "base/strings/string_piece.h" 16#include "base/strings/stringprintf.h" 17#include "base/strings/utf_string_conversions.h" 18#include "base/sys_info.h" 19#include "content/browser/indexed_db/leveldb/leveldb_comparator.h" 20#include "content/browser/indexed_db/leveldb/leveldb_iterator.h" 21#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h" 22#include "third_party/leveldatabase/env_chromium.h" 23#include "third_party/leveldatabase/env_idb.h" 24#include "third_party/leveldatabase/src/helpers/memenv/memenv.h" 25#include "third_party/leveldatabase/src/include/leveldb/db.h" 26#include "third_party/leveldatabase/src/include/leveldb/env.h" 27#include "third_party/leveldatabase/src/include/leveldb/slice.h" 28 29using base::StringPiece; 30 31namespace content { 32 33// Forcing flushes to disk at the end of a transaction guarantees that the 34// data hit disk, but drastically impacts throughput when the filesystem is 35// busy with background compactions. Not syncing trades off reliability for 36// performance. Note that background compactions which move data from the 37// log to SSTs are always done with reliable writes. 38// 39// Sync writes are necessary on Windows for quota calculations; POSIX 40// calculates file sizes correctly even when not synced to disk. 41#if defined(OS_WIN) 42static const bool kSyncWrites = true; 43#else 44// TODO(dgrogan): Either remove the #if block or change this back to false. 45// See http://crbug.com/338385. 46static const bool kSyncWrites = true; 47#endif 48 49static leveldb::Slice MakeSlice(const StringPiece& s) { 50 return leveldb::Slice(s.begin(), s.size()); 51} 52 53static StringPiece MakeStringPiece(const leveldb::Slice& s) { 54 return StringPiece(s.data(), s.size()); 55} 56 57LevelDBDatabase::ComparatorAdapter::ComparatorAdapter( 58 const LevelDBComparator* comparator) 59 : comparator_(comparator) {} 60 61int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a, 62 const leveldb::Slice& b) const { 63 return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b)); 64} 65 66const char* LevelDBDatabase::ComparatorAdapter::Name() const { 67 return comparator_->Name(); 68} 69 70// TODO(jsbell): Support the methods below in the future. 71void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator( 72 std::string* start, 73 const leveldb::Slice& limit) const {} 74 75void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor( 76 std::string* key) const {} 77 78LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db) 79 : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {} 80 81LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); } 82 83LevelDBDatabase::LevelDBDatabase() {} 84 85LevelDBDatabase::~LevelDBDatabase() { 86 // db_'s destructor uses comparator_adapter_; order of deletion is important. 87 db_.reset(); 88 comparator_adapter_.reset(); 89 env_.reset(); 90} 91 92static leveldb::Status OpenDB(leveldb::Comparator* comparator, 93 leveldb::Env* env, 94 const base::FilePath& path, 95 leveldb::DB** db) { 96 leveldb::Options options; 97 options.comparator = comparator; 98 options.create_if_missing = true; 99 options.paranoid_checks = true; 100 options.compression = leveldb::kSnappyCompression; 101 102 // For info about the troubles we've run into with this parameter, see: 103 // https://code.google.com/p/chromium/issues/detail?id=227313#c11 104 options.max_open_files = 80; 105 options.env = env; 106 107 // ChromiumEnv assumes UTF8, converts back to FilePath before using. 108 return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db); 109} 110 111leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) { 112 leveldb::Options options; 113 options.env = leveldb::IDBEnv(); 114 // ChromiumEnv assumes UTF8, converts back to FilePath before using. 115 return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options); 116} 117 118namespace { 119class LockImpl : public LevelDBLock { 120 public: 121 explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock) 122 : env_(env), lock_(lock) {} 123 virtual ~LockImpl() { env_->UnlockFile(lock_); } 124 private: 125 leveldb::Env* env_; 126 leveldb::FileLock* lock_; 127}; 128} 129 130scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting( 131 const base::FilePath& file_name) { 132 leveldb::Env* env = leveldb::IDBEnv(); 133 base::FilePath lock_path = file_name.AppendASCII("LOCK"); 134 leveldb::FileLock* lock = NULL; 135 leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock); 136 if (!status.ok()) 137 return scoped_ptr<LevelDBLock>(); 138 DCHECK(lock); 139 return scoped_ptr<LevelDBLock>(new LockImpl(env, lock)); 140} 141 142static int CheckFreeSpace(const char* const type, 143 const base::FilePath& file_name) { 144 std::string name = 145 std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace"; 146 int64 free_disk_space_in_k_bytes = 147 base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024; 148 if (free_disk_space_in_k_bytes < 0) { 149 base::Histogram::FactoryGet( 150 "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure", 151 1, 152 2 /*boundary*/, 153 2 /*boundary*/ + 1, 154 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/); 155 return -1; 156 } 157 int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX 158 ? INT_MAX 159 : free_disk_space_in_k_bytes; 160 const uint64 histogram_max = static_cast<uint64>(1e9); 161 COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big); 162 base::Histogram::FactoryGet(name, 163 1, 164 histogram_max, 165 11 /*buckets*/, 166 base::HistogramBase::kUmaTargetedHistogramFlag) 167 ->Add(clamped_disk_space_k_bytes); 168 return clamped_disk_space_k_bytes; 169} 170 171static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name, 172 const leveldb::Status& s) { 173 leveldb_env::MethodID method; 174 int error = -1; 175 leveldb_env::ErrorParsingResult result = 176 leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error); 177 if (result == leveldb_env::NONE) 178 return; 179 std::string method_histogram_name(histogram_name); 180 method_histogram_name.append(".EnvMethod"); 181 base::LinearHistogram::FactoryGet( 182 method_histogram_name, 183 1, 184 leveldb_env::kNumEntries, 185 leveldb_env::kNumEntries + 1, 186 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method); 187 188 std::string error_histogram_name(histogram_name); 189 190 if (result == leveldb_env::METHOD_AND_PFE) { 191 DCHECK(error < 0); 192 error_histogram_name.append(std::string(".PFE.") + 193 leveldb_env::MethodIDToString(method)); 194 base::LinearHistogram::FactoryGet( 195 error_histogram_name, 196 1, 197 -base::File::FILE_ERROR_MAX, 198 -base::File::FILE_ERROR_MAX + 1, 199 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error); 200 } else if (result == leveldb_env::METHOD_AND_ERRNO) { 201 error_histogram_name.append(std::string(".Errno.") + 202 leveldb_env::MethodIDToString(method)); 203 base::LinearHistogram::FactoryGet( 204 error_histogram_name, 205 1, 206 ERANGE + 1, 207 ERANGE + 2, 208 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error); 209 } 210} 211 212static void ParseAndHistogramCorruptionDetails( 213 const std::string& histogram_name, 214 const leveldb::Status& status) { 215 int error = leveldb_env::GetCorruptionCode(status); 216 DCHECK(error >= 0); 217 std::string corruption_histogram_name(histogram_name); 218 corruption_histogram_name.append(".Corruption"); 219 const int kNumPatterns = leveldb_env::GetNumCorruptionCodes(); 220 base::LinearHistogram::FactoryGet( 221 corruption_histogram_name, 222 1, 223 kNumPatterns, 224 kNumPatterns + 1, 225 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error); 226} 227 228static void HistogramLevelDBError(const std::string& histogram_name, 229 const leveldb::Status& s) { 230 if (s.ok()) { 231 NOTREACHED(); 232 return; 233 } 234 enum { 235 LEVEL_DB_NOT_FOUND, 236 LEVEL_DB_CORRUPTION, 237 LEVEL_DB_IO_ERROR, 238 LEVEL_DB_OTHER, 239 LEVEL_DB_MAX_ERROR 240 }; 241 int leveldb_error = LEVEL_DB_OTHER; 242 if (s.IsNotFound()) 243 leveldb_error = LEVEL_DB_NOT_FOUND; 244 else if (s.IsCorruption()) 245 leveldb_error = LEVEL_DB_CORRUPTION; 246 else if (s.IsIOError()) 247 leveldb_error = LEVEL_DB_IO_ERROR; 248 base::Histogram::FactoryGet(histogram_name, 249 1, 250 LEVEL_DB_MAX_ERROR, 251 LEVEL_DB_MAX_ERROR + 1, 252 base::HistogramBase::kUmaTargetedHistogramFlag) 253 ->Add(leveldb_error); 254 if (s.IsIOError()) 255 ParseAndHistogramIOErrorDetails(histogram_name, s); 256 else 257 ParseAndHistogramCorruptionDetails(histogram_name, s); 258} 259 260leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name, 261 const LevelDBComparator* comparator, 262 scoped_ptr<LevelDBDatabase>* result, 263 bool* is_disk_full) { 264 scoped_ptr<ComparatorAdapter> comparator_adapter( 265 new ComparatorAdapter(comparator)); 266 267 leveldb::DB* db; 268 const leveldb::Status s = 269 OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db); 270 271 if (!s.ok()) { 272 HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s); 273 int free_space_k_bytes = CheckFreeSpace("Failure", file_name); 274 // Disks with <100k of free space almost never succeed in opening a 275 // leveldb database. 276 if (is_disk_full) 277 *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100; 278 279 LOG(ERROR) << "Failed to open LevelDB database from " 280 << file_name.AsUTF8Unsafe() << "," << s.ToString(); 281 return s; 282 } 283 284 CheckFreeSpace("Success", file_name); 285 286 (*result).reset(new LevelDBDatabase); 287 (*result)->db_ = make_scoped_ptr(db); 288 (*result)->comparator_adapter_ = comparator_adapter.Pass(); 289 (*result)->comparator_ = comparator; 290 291 return s; 292} 293 294scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory( 295 const LevelDBComparator* comparator) { 296 scoped_ptr<ComparatorAdapter> comparator_adapter( 297 new ComparatorAdapter(comparator)); 298 scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv())); 299 300 leveldb::DB* db; 301 const leveldb::Status s = OpenDB( 302 comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db); 303 304 if (!s.ok()) { 305 LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString(); 306 return scoped_ptr<LevelDBDatabase>(); 307 } 308 309 scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase); 310 result->env_ = in_memory_env.Pass(); 311 result->db_ = make_scoped_ptr(db); 312 result->comparator_adapter_ = comparator_adapter.Pass(); 313 result->comparator_ = comparator; 314 315 return result.Pass(); 316} 317 318leveldb::Status LevelDBDatabase::Put(const StringPiece& key, 319 std::string* value) { 320 leveldb::WriteOptions write_options; 321 write_options.sync = kSyncWrites; 322 323 const leveldb::Status s = 324 db_->Put(write_options, MakeSlice(key), MakeSlice(*value)); 325 if (!s.ok()) 326 LOG(ERROR) << "LevelDB put failed: " << s.ToString(); 327 return s; 328} 329 330leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) { 331 leveldb::WriteOptions write_options; 332 write_options.sync = kSyncWrites; 333 334 const leveldb::Status s = db_->Delete(write_options, MakeSlice(key)); 335 if (!s.IsNotFound()) 336 LOG(ERROR) << "LevelDB remove failed: " << s.ToString(); 337 return s; 338} 339 340leveldb::Status LevelDBDatabase::Get(const StringPiece& key, 341 std::string* value, 342 bool* found, 343 const LevelDBSnapshot* snapshot) { 344 *found = false; 345 leveldb::ReadOptions read_options; 346 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the 347 // performance impact is too great. 348 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0; 349 350 const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value); 351 if (s.ok()) { 352 *found = true; 353 return s; 354 } 355 if (s.IsNotFound()) 356 return leveldb::Status::OK(); 357 HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s); 358 LOG(ERROR) << "LevelDB get failed: " << s.ToString(); 359 return s; 360} 361 362leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) { 363 leveldb::WriteOptions write_options; 364 write_options.sync = kSyncWrites; 365 366 const leveldb::Status s = 367 db_->Write(write_options, write_batch.write_batch_.get()); 368 if (!s.ok()) { 369 HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s); 370 LOG(ERROR) << "LevelDB write failed: " << s.ToString(); 371 } 372 return s; 373} 374 375namespace { 376class IteratorImpl : public LevelDBIterator { 377 public: 378 virtual ~IteratorImpl() {} 379 380 virtual bool IsValid() const OVERRIDE; 381 virtual void SeekToLast() OVERRIDE; 382 virtual void Seek(const StringPiece& target) OVERRIDE; 383 virtual void Next() OVERRIDE; 384 virtual void Prev() OVERRIDE; 385 virtual StringPiece Key() const OVERRIDE; 386 virtual StringPiece Value() const OVERRIDE; 387 388 private: 389 friend class content::LevelDBDatabase; 390 explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator); 391 void CheckStatus(); 392 393 scoped_ptr<leveldb::Iterator> iterator_; 394}; 395} 396 397IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it) 398 : iterator_(it.Pass()) {} 399 400void IteratorImpl::CheckStatus() { 401 const leveldb::Status s = iterator_->status(); 402 if (!s.ok()) 403 LOG(ERROR) << "LevelDB iterator error: " << s.ToString(); 404} 405 406bool IteratorImpl::IsValid() const { return iterator_->Valid(); } 407 408void IteratorImpl::SeekToLast() { 409 iterator_->SeekToLast(); 410 CheckStatus(); 411} 412 413void IteratorImpl::Seek(const StringPiece& target) { 414 iterator_->Seek(MakeSlice(target)); 415 CheckStatus(); 416} 417 418void IteratorImpl::Next() { 419 DCHECK(IsValid()); 420 iterator_->Next(); 421 CheckStatus(); 422} 423 424void IteratorImpl::Prev() { 425 DCHECK(IsValid()); 426 iterator_->Prev(); 427 CheckStatus(); 428} 429 430StringPiece IteratorImpl::Key() const { 431 DCHECK(IsValid()); 432 return MakeStringPiece(iterator_->key()); 433} 434 435StringPiece IteratorImpl::Value() const { 436 DCHECK(IsValid()); 437 return MakeStringPiece(iterator_->value()); 438} 439 440scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator( 441 const LevelDBSnapshot* snapshot) { 442 leveldb::ReadOptions read_options; 443 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the 444 // performance impact is too great. 445 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0; 446 447 scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options)); 448 return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass())); 449} 450 451const LevelDBComparator* LevelDBDatabase::Comparator() const { 452 return comparator_; 453} 454 455void LevelDBDatabase::Compact(const base::StringPiece& start, 456 const base::StringPiece& stop) { 457 const leveldb::Slice start_slice = MakeSlice(start); 458 const leveldb::Slice stop_slice = MakeSlice(stop); 459 db_->CompactRange(&start_slice, &stop_slice); 460} 461 462void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); } 463 464} // namespace content 465