leveldb_database.cc revision 116680a4aac90f2aa7413d9095a592090648e557
1// Copyright (c) 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/browser/indexed_db/leveldb/leveldb_database.h" 6 7#include <cerrno> 8 9#include "base/basictypes.h" 10#include "base/files/file.h" 11#include "base/logging.h" 12#include "base/memory/scoped_ptr.h" 13#include "base/metrics/histogram.h" 14#include "base/strings/string16.h" 15#include "base/strings/string_piece.h" 16#include "base/strings/stringprintf.h" 17#include "base/strings/utf_string_conversions.h" 18#include "base/sys_info.h" 19#include "content/browser/indexed_db/indexed_db_class_factory.h" 20#include "content/browser/indexed_db/leveldb/leveldb_comparator.h" 21#include "content/browser/indexed_db/leveldb/leveldb_iterator_impl.h" 22#include "content/browser/indexed_db/leveldb/leveldb_write_batch.h" 23#include "third_party/leveldatabase/env_chromium.h" 24#include "third_party/leveldatabase/env_idb.h" 25#include "third_party/leveldatabase/src/helpers/memenv/memenv.h" 26#include "third_party/leveldatabase/src/include/leveldb/db.h" 27#include "third_party/leveldatabase/src/include/leveldb/env.h" 28#include "third_party/leveldatabase/src/include/leveldb/slice.h" 29 30using base::StringPiece; 31 32namespace content { 33 34// Forcing flushes to disk at the end of a transaction guarantees that the 35// data hit disk, but drastically impacts throughput when the filesystem is 36// busy with background compactions. Not syncing trades off reliability for 37// performance. Note that background compactions which move data from the 38// log to SSTs are always done with reliable writes. 39// 40// Sync writes are necessary on Windows for quota calculations; POSIX 41// calculates file sizes correctly even when not synced to disk. 42#if defined(OS_WIN) 43static const bool kSyncWrites = true; 44#else 45// TODO(dgrogan): Either remove the #if block or change this back to false. 46// See http://crbug.com/338385. 47static const bool kSyncWrites = true; 48#endif 49 50static leveldb::Slice MakeSlice(const StringPiece& s) { 51 return leveldb::Slice(s.begin(), s.size()); 52} 53 54static StringPiece MakeStringPiece(const leveldb::Slice& s) { 55 return StringPiece(s.data(), s.size()); 56} 57 58LevelDBDatabase::ComparatorAdapter::ComparatorAdapter( 59 const LevelDBComparator* comparator) 60 : comparator_(comparator) {} 61 62int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a, 63 const leveldb::Slice& b) const { 64 return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b)); 65} 66 67const char* LevelDBDatabase::ComparatorAdapter::Name() const { 68 return comparator_->Name(); 69} 70 71// TODO(jsbell): Support the methods below in the future. 72void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator( 73 std::string* start, 74 const leveldb::Slice& limit) const {} 75 76void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor( 77 std::string* key) const {} 78 79LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db) 80 : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {} 81 82LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); } 83 84LevelDBDatabase::LevelDBDatabase() {} 85 86LevelDBDatabase::~LevelDBDatabase() { 87 // db_'s destructor uses comparator_adapter_; order of deletion is important. 88 db_.reset(); 89 comparator_adapter_.reset(); 90 env_.reset(); 91} 92 93static leveldb::Status OpenDB(leveldb::Comparator* comparator, 94 leveldb::Env* env, 95 const base::FilePath& path, 96 leveldb::DB** db) { 97 leveldb::Options options; 98 options.comparator = comparator; 99 options.create_if_missing = true; 100 options.paranoid_checks = true; 101 options.compression = leveldb::kSnappyCompression; 102 103 // For info about the troubles we've run into with this parameter, see: 104 // https://code.google.com/p/chromium/issues/detail?id=227313#c11 105 options.max_open_files = 80; 106 options.env = env; 107 108 // ChromiumEnv assumes UTF8, converts back to FilePath before using. 109 return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db); 110} 111 112leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) { 113 leveldb::Options options; 114 options.env = leveldb::IDBEnv(); 115 // ChromiumEnv assumes UTF8, converts back to FilePath before using. 116 return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options); 117} 118 119namespace { 120class LockImpl : public LevelDBLock { 121 public: 122 explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock) 123 : env_(env), lock_(lock) {} 124 virtual ~LockImpl() { env_->UnlockFile(lock_); } 125 private: 126 leveldb::Env* env_; 127 leveldb::FileLock* lock_; 128 129 DISALLOW_COPY_AND_ASSIGN(LockImpl); 130}; 131} // namespace 132 133scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting( 134 const base::FilePath& file_name) { 135 leveldb::Env* env = leveldb::IDBEnv(); 136 base::FilePath lock_path = file_name.AppendASCII("LOCK"); 137 leveldb::FileLock* lock = NULL; 138 leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock); 139 if (!status.ok()) 140 return scoped_ptr<LevelDBLock>(); 141 DCHECK(lock); 142 return scoped_ptr<LevelDBLock>(new LockImpl(env, lock)); 143} 144 145static int CheckFreeSpace(const char* const type, 146 const base::FilePath& file_name) { 147 std::string name = 148 std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace"; 149 int64 free_disk_space_in_k_bytes = 150 base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024; 151 if (free_disk_space_in_k_bytes < 0) { 152 base::Histogram::FactoryGet( 153 "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure", 154 1, 155 2 /*boundary*/, 156 2 /*boundary*/ + 1, 157 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/); 158 return -1; 159 } 160 int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX 161 ? INT_MAX 162 : free_disk_space_in_k_bytes; 163 const uint64 histogram_max = static_cast<uint64>(1e9); 164 COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big); 165 base::Histogram::FactoryGet(name, 166 1, 167 histogram_max, 168 11 /*buckets*/, 169 base::HistogramBase::kUmaTargetedHistogramFlag) 170 ->Add(clamped_disk_space_k_bytes); 171 return clamped_disk_space_k_bytes; 172} 173 174static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name, 175 const leveldb::Status& s) { 176 leveldb_env::MethodID method; 177 int error = -1; 178 leveldb_env::ErrorParsingResult result = 179 leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error); 180 if (result == leveldb_env::NONE) 181 return; 182 std::string method_histogram_name(histogram_name); 183 method_histogram_name.append(".EnvMethod"); 184 base::LinearHistogram::FactoryGet( 185 method_histogram_name, 186 1, 187 leveldb_env::kNumEntries, 188 leveldb_env::kNumEntries + 1, 189 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method); 190 191 std::string error_histogram_name(histogram_name); 192 193 if (result == leveldb_env::METHOD_AND_PFE) { 194 DCHECK_LT(error, 0); 195 error_histogram_name.append(std::string(".PFE.") + 196 leveldb_env::MethodIDToString(method)); 197 base::LinearHistogram::FactoryGet( 198 error_histogram_name, 199 1, 200 -base::File::FILE_ERROR_MAX, 201 -base::File::FILE_ERROR_MAX + 1, 202 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error); 203 } else if (result == leveldb_env::METHOD_AND_ERRNO) { 204 error_histogram_name.append(std::string(".Errno.") + 205 leveldb_env::MethodIDToString(method)); 206 base::LinearHistogram::FactoryGet( 207 error_histogram_name, 208 1, 209 ERANGE + 1, 210 ERANGE + 2, 211 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error); 212 } 213} 214 215static void ParseAndHistogramCorruptionDetails( 216 const std::string& histogram_name, 217 const leveldb::Status& status) { 218 int error = leveldb_env::GetCorruptionCode(status); 219 DCHECK_GE(error, 0); 220 std::string corruption_histogram_name(histogram_name); 221 corruption_histogram_name.append(".Corruption"); 222 const int kNumPatterns = leveldb_env::GetNumCorruptionCodes(); 223 base::LinearHistogram::FactoryGet( 224 corruption_histogram_name, 225 1, 226 kNumPatterns, 227 kNumPatterns + 1, 228 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error); 229} 230 231static void HistogramLevelDBError(const std::string& histogram_name, 232 const leveldb::Status& s) { 233 if (s.ok()) { 234 NOTREACHED(); 235 return; 236 } 237 enum { 238 LEVEL_DB_NOT_FOUND, 239 LEVEL_DB_CORRUPTION, 240 LEVEL_DB_IO_ERROR, 241 LEVEL_DB_OTHER, 242 LEVEL_DB_MAX_ERROR 243 }; 244 int leveldb_error = LEVEL_DB_OTHER; 245 if (s.IsNotFound()) 246 leveldb_error = LEVEL_DB_NOT_FOUND; 247 else if (s.IsCorruption()) 248 leveldb_error = LEVEL_DB_CORRUPTION; 249 else if (s.IsIOError()) 250 leveldb_error = LEVEL_DB_IO_ERROR; 251 base::Histogram::FactoryGet(histogram_name, 252 1, 253 LEVEL_DB_MAX_ERROR, 254 LEVEL_DB_MAX_ERROR + 1, 255 base::HistogramBase::kUmaTargetedHistogramFlag) 256 ->Add(leveldb_error); 257 if (s.IsIOError()) 258 ParseAndHistogramIOErrorDetails(histogram_name, s); 259 else 260 ParseAndHistogramCorruptionDetails(histogram_name, s); 261} 262 263leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name, 264 const LevelDBComparator* comparator, 265 scoped_ptr<LevelDBDatabase>* result, 266 bool* is_disk_full) { 267 scoped_ptr<ComparatorAdapter> comparator_adapter( 268 new ComparatorAdapter(comparator)); 269 270 leveldb::DB* db; 271 const leveldb::Status s = 272 OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db); 273 274 if (!s.ok()) { 275 HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s); 276 int free_space_k_bytes = CheckFreeSpace("Failure", file_name); 277 // Disks with <100k of free space almost never succeed in opening a 278 // leveldb database. 279 if (is_disk_full) 280 *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100; 281 282 LOG(ERROR) << "Failed to open LevelDB database from " 283 << file_name.AsUTF8Unsafe() << "," << s.ToString(); 284 return s; 285 } 286 287 CheckFreeSpace("Success", file_name); 288 289 (*result).reset(new LevelDBDatabase); 290 (*result)->db_ = make_scoped_ptr(db); 291 (*result)->comparator_adapter_ = comparator_adapter.Pass(); 292 (*result)->comparator_ = comparator; 293 294 return s; 295} 296 297scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory( 298 const LevelDBComparator* comparator) { 299 scoped_ptr<ComparatorAdapter> comparator_adapter( 300 new ComparatorAdapter(comparator)); 301 scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv())); 302 303 leveldb::DB* db; 304 const leveldb::Status s = OpenDB( 305 comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db); 306 307 if (!s.ok()) { 308 LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString(); 309 return scoped_ptr<LevelDBDatabase>(); 310 } 311 312 scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase); 313 result->env_ = in_memory_env.Pass(); 314 result->db_ = make_scoped_ptr(db); 315 result->comparator_adapter_ = comparator_adapter.Pass(); 316 result->comparator_ = comparator; 317 318 return result.Pass(); 319} 320 321leveldb::Status LevelDBDatabase::Put(const StringPiece& key, 322 std::string* value) { 323 leveldb::WriteOptions write_options; 324 write_options.sync = kSyncWrites; 325 326 const leveldb::Status s = 327 db_->Put(write_options, MakeSlice(key), MakeSlice(*value)); 328 if (!s.ok()) 329 LOG(ERROR) << "LevelDB put failed: " << s.ToString(); 330 return s; 331} 332 333leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) { 334 leveldb::WriteOptions write_options; 335 write_options.sync = kSyncWrites; 336 337 const leveldb::Status s = db_->Delete(write_options, MakeSlice(key)); 338 if (!s.IsNotFound()) 339 LOG(ERROR) << "LevelDB remove failed: " << s.ToString(); 340 return s; 341} 342 343leveldb::Status LevelDBDatabase::Get(const StringPiece& key, 344 std::string* value, 345 bool* found, 346 const LevelDBSnapshot* snapshot) { 347 *found = false; 348 leveldb::ReadOptions read_options; 349 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the 350 // performance impact is too great. 351 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0; 352 353 const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value); 354 if (s.ok()) { 355 *found = true; 356 return s; 357 } 358 if (s.IsNotFound()) 359 return leveldb::Status::OK(); 360 HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s); 361 LOG(ERROR) << "LevelDB get failed: " << s.ToString(); 362 return s; 363} 364 365leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) { 366 leveldb::WriteOptions write_options; 367 write_options.sync = kSyncWrites; 368 369 const leveldb::Status s = 370 db_->Write(write_options, write_batch.write_batch_.get()); 371 if (!s.ok()) { 372 HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s); 373 LOG(ERROR) << "LevelDB write failed: " << s.ToString(); 374 } 375 return s; 376} 377 378scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator( 379 const LevelDBSnapshot* snapshot) { 380 leveldb::ReadOptions read_options; 381 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the 382 // performance impact is too great. 383 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0; 384 385 scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options)); 386 return scoped_ptr<LevelDBIterator>( 387 IndexedDBClassFactory::Get()->CreateIteratorImpl(i.Pass())); 388} 389 390const LevelDBComparator* LevelDBDatabase::Comparator() const { 391 return comparator_; 392} 393 394void LevelDBDatabase::Compact(const base::StringPiece& start, 395 const base::StringPiece& stop) { 396 const leveldb::Slice start_slice = MakeSlice(start); 397 const leveldb::Slice stop_slice = MakeSlice(stop); 398 db_->CompactRange(&start_slice, &stop_slice); 399} 400 401void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); } 402 403} // namespace content 404