1// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file. See the AUTHORS file for names of contributors.
4
5#include <errno.h>
6#include <stdio.h>
7#include <string.h>
8
9#include <deque>
10
11#include "base/at_exit.h"
12#include "base/debug/trace_event.h"
13#include "base/file_util.h"
14#include "base/files/file_enumerator.h"
15#include "base/files/file_path.h"
16#include "base/lazy_instance.h"
17#include "base/memory/ref_counted.h"
18#include "base/message_loop/message_loop.h"
19#include "base/metrics/histogram.h"
20#include "base/platform_file.h"
21#include "base/posix/eintr_wrapper.h"
22#include "base/strings/utf_string_conversions.h"
23#include "base/synchronization/lock.h"
24#include "base/sys_info.h"
25#include "base/threading/platform_thread.h"
26#include "base/threading/thread.h"
27#include "chromium_logger.h"
28#include "env_chromium.h"
29#include "leveldb/env.h"
30#include "leveldb/slice.h"
31#include "port/port.h"
32#include "third_party/re2/re2/re2.h"
33#include "util/logging.h"
34
35#if defined(OS_WIN)
36#include <io.h>
37#include "base/win/win_util.h"
38#endif
39
40#if defined(OS_POSIX)
41#include <dirent.h>
42#include <fcntl.h>
43#include <sys/resource.h>
44#include <sys/time.h>
45#endif
46
47using namespace leveldb;
48
49namespace leveldb_env {
50
51namespace {
52
53const base::FilePath::CharType backup_table_extension[] =
54    FILE_PATH_LITERAL(".bak");
55const base::FilePath::CharType table_extension[] = FILE_PATH_LITERAL(".ldb");
56
57#if (defined(OS_POSIX) && !defined(OS_LINUX)) || defined(OS_WIN)
58// The following are glibc-specific
59
60size_t fread_unlocked(void *ptr, size_t size, size_t n, FILE *file) {
61  return fread(ptr, size, n, file);
62}
63
64size_t fwrite_unlocked(const void *ptr, size_t size, size_t n, FILE *file) {
65  return fwrite(ptr, size, n, file);
66}
67
68int fflush_unlocked(FILE *file) {
69  return fflush(file);
70}
71
72#if !defined(OS_ANDROID)
73int fdatasync(int fildes) {
74#if defined(OS_WIN)
75  return _commit(fildes);
76#else
77  return HANDLE_EINTR(fsync(fildes));
78#endif
79}
80#endif
81
82#endif
83
84// Wide-char safe fopen wrapper.
85FILE* fopen_internal(const char* fname, const char* mode) {
86#if defined(OS_WIN)
87  return _wfopen(UTF8ToUTF16(fname).c_str(), ASCIIToUTF16(mode).c_str());
88#else
89  return fopen(fname, mode);
90#endif
91}
92
93base::FilePath CreateFilePath(const std::string& file_path) {
94#if defined(OS_WIN)
95  return base::FilePath(UTF8ToUTF16(file_path));
96#else
97  return base::FilePath(file_path);
98#endif
99}
100
101static const base::FilePath::CharType kLevelDBTestDirectoryPrefix[]
102    = FILE_PATH_LITERAL("leveldb-test-");
103
104const char* PlatformFileErrorString(const ::base::PlatformFileError& error) {
105  switch (error) {
106    case ::base::PLATFORM_FILE_ERROR_FAILED:
107      return "No further details.";
108    case ::base::PLATFORM_FILE_ERROR_IN_USE:
109      return "File currently in use.";
110    case ::base::PLATFORM_FILE_ERROR_EXISTS:
111      return "File already exists.";
112    case ::base::PLATFORM_FILE_ERROR_NOT_FOUND:
113      return "File not found.";
114    case ::base::PLATFORM_FILE_ERROR_ACCESS_DENIED:
115      return "Access denied.";
116    case ::base::PLATFORM_FILE_ERROR_TOO_MANY_OPENED:
117      return "Too many files open.";
118    case ::base::PLATFORM_FILE_ERROR_NO_MEMORY:
119      return "Out of memory.";
120    case ::base::PLATFORM_FILE_ERROR_NO_SPACE:
121      return "No space left on drive.";
122    case ::base::PLATFORM_FILE_ERROR_NOT_A_DIRECTORY:
123      return "Not a directory.";
124    case ::base::PLATFORM_FILE_ERROR_INVALID_OPERATION:
125      return "Invalid operation.";
126    case ::base::PLATFORM_FILE_ERROR_SECURITY:
127      return "Security error.";
128    case ::base::PLATFORM_FILE_ERROR_ABORT:
129      return "File operation aborted.";
130    case ::base::PLATFORM_FILE_ERROR_NOT_A_FILE:
131      return "The supplied path was not a file.";
132    case ::base::PLATFORM_FILE_ERROR_NOT_EMPTY:
133      return "The file was not empty.";
134    case ::base::PLATFORM_FILE_ERROR_INVALID_URL:
135      return "Invalid URL.";
136    case ::base::PLATFORM_FILE_ERROR_IO:
137      return "OS or hardware error.";
138    case ::base::PLATFORM_FILE_OK:
139      return "OK.";
140    case ::base::PLATFORM_FILE_ERROR_MAX:
141      NOTREACHED();
142  }
143  NOTIMPLEMENTED();
144  return "Unknown error.";
145}
146
147class ChromiumSequentialFile: public SequentialFile {
148 private:
149  std::string filename_;
150  FILE* file_;
151  const UMALogger* uma_logger_;
152
153 public:
154  ChromiumSequentialFile(const std::string& fname, FILE* f,
155                         const UMALogger* uma_logger)
156      : filename_(fname), file_(f), uma_logger_(uma_logger) { }
157  virtual ~ChromiumSequentialFile() { fclose(file_); }
158
159  virtual Status Read(size_t n, Slice* result, char* scratch) {
160    Status s;
161    size_t r = fread_unlocked(scratch, 1, n, file_);
162    *result = Slice(scratch, r);
163    if (r < n) {
164      if (feof(file_)) {
165        // We leave status as ok if we hit the end of the file
166      } else {
167        // A partial read with an error: return a non-ok status
168        s = MakeIOError(filename_, strerror(errno), kSequentialFileRead, errno);
169        uma_logger_->RecordErrorAt(kSequentialFileRead);
170      }
171    }
172    return s;
173  }
174
175  virtual Status Skip(uint64_t n) {
176    if (fseek(file_, n, SEEK_CUR)) {
177      int saved_errno = errno;
178      uma_logger_->RecordErrorAt(kSequentialFileSkip);
179      return MakeIOError(
180          filename_, strerror(saved_errno), kSequentialFileSkip, saved_errno);
181    }
182    return Status::OK();
183  }
184};
185
186class ChromiumRandomAccessFile: public RandomAccessFile {
187 private:
188  std::string filename_;
189  ::base::PlatformFile file_;
190  const UMALogger* uma_logger_;
191
192 public:
193  ChromiumRandomAccessFile(const std::string& fname, ::base::PlatformFile file,
194                           const UMALogger* uma_logger)
195      : filename_(fname), file_(file), uma_logger_(uma_logger) { }
196  virtual ~ChromiumRandomAccessFile() { ::base::ClosePlatformFile(file_); }
197
198  virtual Status Read(uint64_t offset, size_t n, Slice* result,
199                      char* scratch) const {
200    Status s;
201    int r = ::base::ReadPlatformFile(file_, offset, scratch, n);
202    *result = Slice(scratch, (r < 0) ? 0 : r);
203    if (r < 0) {
204      // An error: return a non-ok status
205      s = MakeIOError(
206          filename_, "Could not perform read", kRandomAccessFileRead);
207      uma_logger_->RecordErrorAt(kRandomAccessFileRead);
208    }
209    return s;
210  }
211};
212
213class ChromiumFileLock : public FileLock {
214 public:
215  ::base::PlatformFile file_;
216  std::string name_;
217};
218
219class Retrier {
220 public:
221  Retrier(MethodID method, RetrierProvider* provider)
222      : start_(base::TimeTicks::Now()),
223        limit_(start_ + base::TimeDelta::FromMilliseconds(
224                            provider->MaxRetryTimeMillis())),
225        last_(start_),
226        time_to_sleep_(base::TimeDelta::FromMilliseconds(10)),
227        success_(true),
228        method_(method),
229        last_error_(base::PLATFORM_FILE_OK),
230        provider_(provider) {}
231  ~Retrier() {
232    if (success_) {
233      provider_->GetRetryTimeHistogram(method_)->AddTime(last_ - start_);
234      if (last_error_ != base::PLATFORM_FILE_OK) {
235        DCHECK(last_error_ < 0);
236        provider_->GetRecoveredFromErrorHistogram(method_)->Add(-last_error_);
237      }
238    }
239  }
240  bool ShouldKeepTrying(base::PlatformFileError last_error) {
241    DCHECK_NE(last_error, base::PLATFORM_FILE_OK);
242    last_error_ = last_error;
243    if (last_ < limit_) {
244      base::PlatformThread::Sleep(time_to_sleep_);
245      last_ = base::TimeTicks::Now();
246      return true;
247    }
248    success_ = false;
249    return false;
250  }
251
252 private:
253  base::TimeTicks start_;
254  base::TimeTicks limit_;
255  base::TimeTicks last_;
256  base::TimeDelta time_to_sleep_;
257  bool success_;
258  MethodID method_;
259  base::PlatformFileError last_error_;
260  RetrierProvider* provider_;
261};
262
263class IDBEnv : public ChromiumEnv {
264 public:
265  IDBEnv() : ChromiumEnv() {
266    name_ = "LevelDBEnv.IDB";
267    make_backup_ = true;
268  }
269};
270
271::base::LazyInstance<IDBEnv>::Leaky idb_env = LAZY_INSTANCE_INITIALIZER;
272
273::base::LazyInstance<ChromiumEnv>::Leaky default_env =
274    LAZY_INSTANCE_INITIALIZER;
275
276}  // unnamed namespace
277
278const char* MethodIDToString(MethodID method) {
279  switch (method) {
280    case kSequentialFileRead:
281      return "SequentialFileRead";
282    case kSequentialFileSkip:
283      return "SequentialFileSkip";
284    case kRandomAccessFileRead:
285      return "RandomAccessFileRead";
286    case kWritableFileAppend:
287      return "WritableFileAppend";
288    case kWritableFileClose:
289      return "WritableFileClose";
290    case kWritableFileFlush:
291      return "WritableFileFlush";
292    case kWritableFileSync:
293      return "WritableFileSync";
294    case kNewSequentialFile:
295      return "NewSequentialFile";
296    case kNewRandomAccessFile:
297      return "NewRandomAccessFile";
298    case kNewWritableFile:
299      return "NewWritableFile";
300    case kDeleteFile:
301      return "DeleteFile";
302    case kCreateDir:
303      return "CreateDir";
304    case kDeleteDir:
305      return "DeleteDir";
306    case kGetFileSize:
307      return "GetFileSize";
308    case kRenameFile:
309      return "RenameFile";
310    case kLockFile:
311      return "LockFile";
312    case kUnlockFile:
313      return "UnlockFile";
314    case kGetTestDirectory:
315      return "GetTestDirectory";
316    case kNewLogger:
317      return "NewLogger";
318    case kSyncParent:
319      return "SyncParent";
320    case kGetChildren:
321      return "GetChildren";
322    case kNumEntries:
323      NOTREACHED();
324      return "kNumEntries";
325  }
326  NOTREACHED();
327  return "Unknown";
328}
329
330Status MakeIOError(Slice filename,
331                   const char* message,
332                   MethodID method,
333                   int saved_errno) {
334  char buf[512];
335  snprintf(buf,
336           sizeof(buf),
337           "%s (ChromeMethodErrno: %d::%s::%d)",
338           message,
339           method,
340           MethodIDToString(method),
341           saved_errno);
342  return Status::IOError(filename, buf);
343}
344
345Status MakeIOError(Slice filename,
346                   const char* message,
347                   MethodID method,
348                   base::PlatformFileError error) {
349  DCHECK(error < 0);
350  char buf[512];
351  snprintf(buf,
352           sizeof(buf),
353           "%s (ChromeMethodPFE: %d::%s::%d)",
354           message,
355           method,
356           MethodIDToString(method),
357           -error);
358  return Status::IOError(filename, buf);
359}
360
361Status MakeIOError(Slice filename, const char* message, MethodID method) {
362  char buf[512];
363  snprintf(buf,
364           sizeof(buf),
365           "%s (ChromeMethodOnly: %d::%s)",
366           message,
367           method,
368           MethodIDToString(method));
369  return Status::IOError(filename, buf);
370}
371
372ErrorParsingResult ParseMethodAndError(const char* string,
373                                       MethodID* method_param,
374                                       int* error) {
375  int method;
376  if (RE2::PartialMatch(string, "ChromeMethodOnly: (\\d+)", &method)) {
377    *method_param = static_cast<MethodID>(method);
378    return METHOD_ONLY;
379  }
380  if (RE2::PartialMatch(
381          string, "ChromeMethodPFE: (\\d+)::.*::(\\d+)", &method, error)) {
382    *error = -*error;
383    *method_param = static_cast<MethodID>(method);
384    return METHOD_AND_PFE;
385  }
386  if (RE2::PartialMatch(
387          string, "ChromeMethodErrno: (\\d+)::.*::(\\d+)", &method, error)) {
388    *method_param = static_cast<MethodID>(method);
389    return METHOD_AND_ERRNO;
390  }
391  return NONE;
392}
393
394// Keep in sync with LevelDBCorruptionTypes in histograms.xml. Also, don't
395// change the order because indices into this array have been recorded in uma
396// histograms.
397const char* patterns[] = {
398  "missing files",
399  "log record too small",
400  "corrupted internal key",
401  "partial record",
402  "missing start of fragmented record",
403  "error in middle of record",
404  "unknown record type",
405  "truncated record at end",
406  "bad record length",
407  "VersionEdit",
408  "FileReader invoked with unexpected value",
409  "corrupted key",
410  "CURRENT file does not end with newline",
411  "no meta-nextfile entry",
412  "no meta-lognumber entry",
413  "no last-sequence-number entry",
414  "malformed WriteBatch",
415  "bad WriteBatch Put",
416  "bad WriteBatch Delete",
417  "unknown WriteBatch tag",
418  "WriteBatch has wrong count",
419  "bad entry in block",
420  "bad block contents",
421  "bad block handle",
422  "truncated block read",
423  "block checksum mismatch",
424  "checksum mismatch",
425  "corrupted compressed block contents",
426  "bad block type",
427  "bad magic number",
428  "file is too short",
429};
430
431// Returns 1-based index into the above array or 0 if nothing matches.
432int GetCorruptionCode(const leveldb::Status& status) {
433  DCHECK(!status.IsIOError());
434  DCHECK(!status.ok());
435  const int kOtherError = 0;
436  int error = kOtherError;
437  const std::string& str_error = status.ToString();
438  const size_t kNumPatterns = arraysize(patterns);
439  for (size_t i = 0; i < kNumPatterns; ++i) {
440    if (str_error.find(patterns[i]) != std::string::npos) {
441      error = i + 1;
442      break;
443    }
444  }
445  return error;
446}
447
448int GetNumCorruptionCodes() {
449  // + 1 for the "other" error that is returned when a corruption message
450  // doesn't match any of the patterns.
451  return arraysize(patterns) + 1;
452}
453
454std::string GetCorruptionMessage(const leveldb::Status& status) {
455  int code = GetCorruptionCode(status);
456  if (code == 0)
457    return "Unknown corruption";
458  return patterns[code - 1];
459}
460
461bool IndicatesDiskFull(const leveldb::Status& status) {
462  if (status.ok())
463    return false;
464  leveldb_env::MethodID method;
465  int error = -1;
466  leveldb_env::ErrorParsingResult result = leveldb_env::ParseMethodAndError(
467      status.ToString().c_str(), &method, &error);
468  return (result == leveldb_env::METHOD_AND_PFE &&
469          static_cast<base::PlatformFileError>(error) ==
470              base::PLATFORM_FILE_ERROR_NO_SPACE) ||
471         (result == leveldb_env::METHOD_AND_ERRNO && error == ENOSPC);
472}
473
474bool IsIOError(const leveldb::Status& status) {
475  leveldb_env::MethodID method;
476  int error = -1;
477  leveldb_env::ErrorParsingResult result = leveldb_env::ParseMethodAndError(
478      status.ToString().c_str(), &method, &error);
479  return result != leveldb_env::NONE;
480}
481
482bool IsCorruption(const leveldb::Status& status) {
483  // LevelDB returns InvalidArgument when an sst file is truncated but there is
484  // no IsInvalidArgument() accessor defined.
485  return status.IsCorruption() || (!status.ok() && !IsIOError(status));
486}
487
488std::string FilePathToString(const base::FilePath& file_path) {
489#if defined(OS_WIN)
490  return UTF16ToUTF8(file_path.value());
491#else
492  return file_path.value();
493#endif
494}
495
496ChromiumWritableFile::ChromiumWritableFile(const std::string& fname,
497                                           FILE* f,
498                                           const UMALogger* uma_logger,
499                                           WriteTracker* tracker,
500                                           bool make_backup)
501    : filename_(fname),
502      file_(f),
503      uma_logger_(uma_logger),
504      tracker_(tracker),
505      file_type_(kOther),
506      make_backup_(make_backup) {
507  base::FilePath path = base::FilePath::FromUTF8Unsafe(fname);
508  if (FilePathToString(path.BaseName()).find("MANIFEST") == 0)
509    file_type_ = kManifest;
510  else if (path.MatchesExtension(table_extension))
511    file_type_ = kTable;
512  if (file_type_ != kManifest)
513    tracker_->DidCreateNewFile(filename_);
514  parent_dir_ = FilePathToString(CreateFilePath(fname).DirName());
515}
516
517ChromiumWritableFile::~ChromiumWritableFile() {
518  if (file_ != NULL) {
519    // Ignoring any potential errors
520    fclose(file_);
521  }
522}
523
524Status ChromiumWritableFile::SyncParent() {
525  Status s;
526#if !defined(OS_WIN)
527  TRACE_EVENT0("leveldb", "SyncParent");
528
529  int parent_fd =
530      HANDLE_EINTR(open(parent_dir_.c_str(), O_RDONLY));
531  if (parent_fd < 0) {
532    int saved_errno = errno;
533    return MakeIOError(
534        parent_dir_, strerror(saved_errno), kSyncParent, saved_errno);
535  }
536  if (HANDLE_EINTR(fsync(parent_fd)) != 0) {
537    int saved_errno = errno;
538    s = MakeIOError(
539        parent_dir_, strerror(saved_errno), kSyncParent, saved_errno);
540  };
541  close(parent_fd);
542#endif
543  return s;
544}
545
546Status ChromiumWritableFile::Append(const Slice& data) {
547  if (file_type_ == kManifest && tracker_->DoesDirNeedSync(filename_)) {
548    Status s = SyncParent();
549    if (!s.ok())
550      return s;
551    tracker_->DidSyncDir(filename_);
552  }
553
554  size_t r = fwrite_unlocked(data.data(), 1, data.size(), file_);
555  if (r != data.size()) {
556    int saved_errno = errno;
557    uma_logger_->RecordOSError(kWritableFileAppend, saved_errno);
558    return MakeIOError(
559        filename_, strerror(saved_errno), kWritableFileAppend, saved_errno);
560  }
561  return Status::OK();
562}
563
564Status ChromiumWritableFile::Close() {
565  Status result;
566  if (fclose(file_) != 0) {
567    result = MakeIOError(filename_, strerror(errno), kWritableFileClose, errno);
568    uma_logger_->RecordErrorAt(kWritableFileClose);
569  }
570  file_ = NULL;
571  return result;
572}
573
574Status ChromiumWritableFile::Flush() {
575  Status result;
576  if (HANDLE_EINTR(fflush_unlocked(file_))) {
577    int saved_errno = errno;
578    result = MakeIOError(
579        filename_, strerror(saved_errno), kWritableFileFlush, saved_errno);
580    uma_logger_->RecordOSError(kWritableFileFlush, saved_errno);
581  }
582  return result;
583}
584
585static bool MakeBackup(const std::string& fname) {
586  base::FilePath original_table_name = CreateFilePath(fname);
587  base::FilePath backup_table_name =
588      original_table_name.ReplaceExtension(backup_table_extension);
589  return base::CopyFile(original_table_name, backup_table_name);
590}
591
592Status ChromiumWritableFile::Sync() {
593  TRACE_EVENT0("leveldb", "ChromiumEnv::Sync");
594  Status result;
595  int error = 0;
596
597  if (HANDLE_EINTR(fflush_unlocked(file_)))
598    error = errno;
599  // Sync even if fflush gave an error; perhaps the data actually got out,
600  // even though something went wrong.
601  if (fdatasync(fileno(file_)) && !error)
602    error = errno;
603  // Report the first error we found.
604  if (error) {
605    result = MakeIOError(filename_, strerror(error), kWritableFileSync, error);
606    uma_logger_->RecordErrorAt(kWritableFileSync);
607  } else if (make_backup_ && file_type_ == kTable) {
608    bool success = MakeBackup(filename_);
609    uma_logger_->RecordBackupResult(success);
610  }
611  return result;
612}
613
614ChromiumEnv::ChromiumEnv()
615    : name_("LevelDBEnv"),
616      make_backup_(false),
617      bgsignal_(&mu_),
618      started_bgthread_(false),
619      kMaxRetryTimeMillis(1000) {
620}
621
622ChromiumEnv::~ChromiumEnv() {
623  // In chromium, ChromiumEnv is leaked. It'd be nice to add NOTREACHED here to
624  // ensure that behavior isn't accidentally changed, but there's an instance in
625  // a unit test that is deleted.
626}
627
628Status ChromiumEnv::NewSequentialFile(const std::string& fname,
629                                      SequentialFile** result) {
630  FILE* f = fopen_internal(fname.c_str(), "rb");
631  if (f == NULL) {
632    *result = NULL;
633    int saved_errno = errno;
634    RecordOSError(kNewSequentialFile, saved_errno);
635    return MakeIOError(
636        fname, strerror(saved_errno), kNewSequentialFile, saved_errno);
637  } else {
638    *result = new ChromiumSequentialFile(fname, f, this);
639    return Status::OK();
640  }
641}
642
643void ChromiumEnv::RecordOpenFilesLimit(const std::string& type) {
644#if defined(OS_POSIX)
645  struct rlimit nofile;
646  if (getrlimit(RLIMIT_NOFILE, &nofile))
647    return;
648  GetMaxFDHistogram(type)->Add(nofile.rlim_cur);
649#endif
650}
651
652Status ChromiumEnv::NewRandomAccessFile(const std::string& fname,
653                                        RandomAccessFile** result) {
654  int flags = ::base::PLATFORM_FILE_READ | ::base::PLATFORM_FILE_OPEN;
655  bool created;
656  ::base::PlatformFileError error_code;
657  ::base::PlatformFile file = ::base::CreatePlatformFile(
658      CreateFilePath(fname), flags, &created, &error_code);
659  if (error_code == ::base::PLATFORM_FILE_OK) {
660    *result = new ChromiumRandomAccessFile(fname, file, this);
661    RecordOpenFilesLimit("Success");
662    return Status::OK();
663  }
664  if (error_code == ::base::PLATFORM_FILE_ERROR_TOO_MANY_OPENED)
665    RecordOpenFilesLimit("TooManyOpened");
666  else
667    RecordOpenFilesLimit("OtherError");
668  *result = NULL;
669  RecordOSError(kNewRandomAccessFile, error_code);
670  return MakeIOError(fname,
671                     PlatformFileErrorString(error_code),
672                     kNewRandomAccessFile,
673                     error_code);
674}
675
676Status ChromiumEnv::NewWritableFile(const std::string& fname,
677                                    WritableFile** result) {
678  *result = NULL;
679  FILE* f = fopen_internal(fname.c_str(), "wb");
680  if (f == NULL) {
681    int saved_errno = errno;
682    RecordErrorAt(kNewWritableFile);
683    return MakeIOError(
684        fname, strerror(saved_errno), kNewWritableFile, saved_errno);
685  } else {
686    *result = new ChromiumWritableFile(fname, f, this, this, make_backup_);
687    return Status::OK();
688  }
689}
690
691bool ChromiumEnv::FileExists(const std::string& fname) {
692  return ::base::PathExists(CreateFilePath(fname));
693}
694
695base::FilePath ChromiumEnv::RestoreFromBackup(const base::FilePath& base_name) {
696  base::FilePath table_name =
697      base_name.AddExtension(table_extension);
698  bool result = base::CopyFile(base_name.AddExtension(backup_table_extension),
699                               table_name);
700  std::string uma_name(name_);
701  uma_name.append(".TableRestore");
702  base::BooleanHistogram::FactoryGet(
703      uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
704  return table_name;
705}
706
707void ChromiumEnv::RestoreIfNecessary(const std::string& dir,
708                                     std::vector<std::string>* result) {
709  std::set<base::FilePath> tables_found;
710  std::set<base::FilePath> backups_found;
711  for (std::vector<std::string>::iterator it = result->begin();
712       it != result->end();
713       ++it) {
714    base::FilePath current = CreateFilePath(*it);
715    if (current.MatchesExtension(table_extension))
716      tables_found.insert(current.RemoveExtension());
717    if (current.MatchesExtension(backup_table_extension))
718      backups_found.insert(current.RemoveExtension());
719  }
720  std::set<base::FilePath> backups_only;
721  std::set_difference(backups_found.begin(),
722                      backups_found.end(),
723                      tables_found.begin(),
724                      tables_found.end(),
725                      std::inserter(backups_only, backups_only.begin()));
726  if (backups_only.size()) {
727    std::string uma_name(name_);
728    uma_name.append(".MissingFiles");
729    int num_missing_files =
730        backups_only.size() > INT_MAX ? INT_MAX : backups_only.size();
731    base::Histogram::FactoryGet(uma_name,
732                                1 /*min*/,
733                                100 /*max*/,
734                                8 /*num_buckets*/,
735                                base::Histogram::kUmaTargetedHistogramFlag)
736        ->Add(num_missing_files);
737  }
738  base::FilePath dir_filepath = base::FilePath::FromUTF8Unsafe(dir);
739  for (std::set<base::FilePath>::iterator it = backups_only.begin();
740       it != backups_only.end();
741       ++it) {
742    base::FilePath restored_table_name =
743        RestoreFromBackup(dir_filepath.Append(*it));
744    result->push_back(FilePathToString(restored_table_name.BaseName()));
745  }
746}
747
748namespace {
749#if defined(OS_WIN)
750static base::PlatformFileError GetDirectoryEntries(
751    const base::FilePath& dir_param,
752    std::vector<base::FilePath>* result) {
753  result->clear();
754  base::FilePath dir_filepath = dir_param.Append(FILE_PATH_LITERAL("*"));
755  WIN32_FIND_DATA find_data;
756  HANDLE find_handle = FindFirstFile(dir_filepath.value().c_str(), &find_data);
757  if (find_handle == INVALID_HANDLE_VALUE) {
758    DWORD last_error = GetLastError();
759    if (last_error == ERROR_FILE_NOT_FOUND)
760      return base::PLATFORM_FILE_OK;
761    return base::LastErrorToPlatformFileError(last_error);
762  }
763  do {
764    base::FilePath filepath(find_data.cFileName);
765    base::FilePath::StringType basename = filepath.BaseName().value();
766    if (basename == FILE_PATH_LITERAL(".") ||
767        basename == FILE_PATH_LITERAL(".."))
768      continue;
769    result->push_back(filepath.BaseName());
770  } while (FindNextFile(find_handle, &find_data));
771  DWORD last_error = GetLastError();
772  base::PlatformFileError return_value = base::PLATFORM_FILE_OK;
773  if (last_error != ERROR_NO_MORE_FILES)
774    return_value = base::LastErrorToPlatformFileError(last_error);
775  FindClose(find_handle);
776  return return_value;
777}
778#else
779static base::PlatformFileError GetDirectoryEntries(
780    const base::FilePath& dir_filepath,
781    std::vector<base::FilePath>* result) {
782  const std::string dir_string = FilePathToString(dir_filepath);
783  result->clear();
784  DIR* dir = opendir(dir_string.c_str());
785  if (!dir)
786    return base::ErrnoToPlatformFileError(errno);
787  struct dirent dent_buf;
788  struct dirent* dent;
789  int readdir_result;
790  while ((readdir_result = readdir_r(dir, &dent_buf, &dent)) == 0 && dent) {
791    if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)
792      continue;
793    result->push_back(CreateFilePath(dent->d_name));
794  }
795  int saved_errno = errno;
796  closedir(dir);
797  if (readdir_result != 0)
798    return base::ErrnoToPlatformFileError(saved_errno);
799  return base::PLATFORM_FILE_OK;
800}
801#endif
802}
803
804Status ChromiumEnv::GetChildren(const std::string& dir_string,
805                                std::vector<std::string>* result) {
806  std::vector<base::FilePath> entries;
807  base::PlatformFileError error =
808      GetDirectoryEntries(CreateFilePath(dir_string), &entries);
809  if (error != base::PLATFORM_FILE_OK) {
810    RecordOSError(kGetChildren, error);
811    return MakeIOError(
812        dir_string, "Could not open/read directory", kGetChildren, error);
813  }
814  result->clear();
815  for (std::vector<base::FilePath>::iterator it = entries.begin();
816       it != entries.end();
817       ++it) {
818    result->push_back(FilePathToString(*it));
819  }
820
821  if (make_backup_)
822    RestoreIfNecessary(dir_string, result);
823  return Status::OK();
824}
825
826Status ChromiumEnv::DeleteFile(const std::string& fname) {
827  Status result;
828  base::FilePath fname_filepath = CreateFilePath(fname);
829  // TODO(jorlow): Should we assert this is a file?
830  if (!::base::DeleteFile(fname_filepath, false)) {
831    result = MakeIOError(fname, "Could not delete file.", kDeleteFile);
832    RecordErrorAt(kDeleteFile);
833  }
834  if (make_backup_ && fname_filepath.MatchesExtension(table_extension)) {
835    base::DeleteFile(fname_filepath.ReplaceExtension(backup_table_extension),
836                     false);
837  }
838  return result;
839}
840
841Status ChromiumEnv::CreateDir(const std::string& name) {
842  Status result;
843  base::PlatformFileError error = base::PLATFORM_FILE_OK;
844  Retrier retrier(kCreateDir, this);
845  do {
846    if (base::CreateDirectoryAndGetError(CreateFilePath(name), &error))
847      return result;
848  } while (retrier.ShouldKeepTrying(error));
849  result = MakeIOError(name, "Could not create directory.", kCreateDir, error);
850  RecordOSError(kCreateDir, error);
851  return result;
852}
853
854Status ChromiumEnv::DeleteDir(const std::string& name) {
855  Status result;
856  // TODO(jorlow): Should we assert this is a directory?
857  if (!::base::DeleteFile(CreateFilePath(name), false)) {
858    result = MakeIOError(name, "Could not delete directory.", kDeleteDir);
859    RecordErrorAt(kDeleteDir);
860  }
861  return result;
862}
863
864Status ChromiumEnv::GetFileSize(const std::string& fname, uint64_t* size) {
865  Status s;
866  int64_t signed_size;
867  if (!::base::GetFileSize(CreateFilePath(fname), &signed_size)) {
868    *size = 0;
869    s = MakeIOError(fname, "Could not determine file size.", kGetFileSize);
870    RecordErrorAt(kGetFileSize);
871  } else {
872    *size = static_cast<uint64_t>(signed_size);
873  }
874  return s;
875}
876
877Status ChromiumEnv::RenameFile(const std::string& src, const std::string& dst) {
878  Status result;
879  base::FilePath src_file_path = CreateFilePath(src);
880  if (!::base::PathExists(src_file_path))
881    return result;
882  base::FilePath destination = CreateFilePath(dst);
883
884  Retrier retrier(kRenameFile, this);
885  base::PlatformFileError error = base::PLATFORM_FILE_OK;
886  do {
887    if (base::ReplaceFile(src_file_path, destination, &error))
888      return result;
889  } while (retrier.ShouldKeepTrying(error));
890
891  DCHECK(error != base::PLATFORM_FILE_OK);
892  RecordOSError(kRenameFile, error);
893  char buf[100];
894  snprintf(buf,
895           sizeof(buf),
896           "Could not rename file: %s",
897           PlatformFileErrorString(error));
898  return MakeIOError(src, buf, kRenameFile, error);
899}
900
901Status ChromiumEnv::LockFile(const std::string& fname, FileLock** lock) {
902  *lock = NULL;
903  Status result;
904  int flags = ::base::PLATFORM_FILE_OPEN_ALWAYS |
905              ::base::PLATFORM_FILE_READ |
906              ::base::PLATFORM_FILE_WRITE;
907  bool created;
908  ::base::PlatformFileError error_code;
909  ::base::PlatformFile file;
910  Retrier retrier(kLockFile, this);
911  do {
912    file = ::base::CreatePlatformFile(
913        CreateFilePath(fname), flags, &created, &error_code);
914  } while (error_code != ::base::PLATFORM_FILE_OK &&
915           retrier.ShouldKeepTrying(error_code));
916
917  if (error_code == ::base::PLATFORM_FILE_ERROR_NOT_FOUND) {
918    ::base::FilePath parent = CreateFilePath(fname).DirName();
919    ::base::FilePath last_parent;
920    int num_missing_ancestors = 0;
921    do {
922      if (base::DirectoryExists(parent))
923        break;
924      ++num_missing_ancestors;
925      last_parent = parent;
926      parent = parent.DirName();
927    } while (parent != last_parent);
928    RecordLockFileAncestors(num_missing_ancestors);
929  }
930
931  if (error_code != ::base::PLATFORM_FILE_OK) {
932    result = MakeIOError(
933        fname, PlatformFileErrorString(error_code), kLockFile, error_code);
934    RecordOSError(kLockFile, error_code);
935    return result;
936  }
937
938  if (!locks_.Insert(fname)) {
939    result = MakeIOError(fname, "Lock file already locked.", kLockFile);
940    ::base::ClosePlatformFile(file);
941    return result;
942  }
943
944  Retrier lock_retrier = Retrier(kLockFile, this);
945  do {
946    error_code = ::base::LockPlatformFile(file);
947  } while (error_code != ::base::PLATFORM_FILE_OK &&
948           retrier.ShouldKeepTrying(error_code));
949
950  if (error_code != ::base::PLATFORM_FILE_OK) {
951    ::base::ClosePlatformFile(file);
952    locks_.Remove(fname);
953    result = MakeIOError(
954        fname, PlatformFileErrorString(error_code), kLockFile, error_code);
955    RecordOSError(kLockFile, error_code);
956    return result;
957  }
958
959  ChromiumFileLock* my_lock = new ChromiumFileLock;
960  my_lock->file_ = file;
961  my_lock->name_ = fname;
962  *lock = my_lock;
963  return result;
964}
965
966Status ChromiumEnv::UnlockFile(FileLock* lock) {
967  ChromiumFileLock* my_lock = reinterpret_cast<ChromiumFileLock*>(lock);
968  Status result;
969
970  ::base::PlatformFileError error_code =
971      ::base::UnlockPlatformFile(my_lock->file_);
972  if (error_code != ::base::PLATFORM_FILE_OK) {
973    result =
974        MakeIOError(my_lock->name_, "Could not unlock lock file.", kUnlockFile);
975    RecordOSError(kUnlockFile, error_code);
976    ::base::ClosePlatformFile(my_lock->file_);
977  } else if (!::base::ClosePlatformFile(my_lock->file_)) {
978    result =
979        MakeIOError(my_lock->name_, "Could not close lock file.", kUnlockFile);
980    RecordErrorAt(kUnlockFile);
981  }
982  bool removed = locks_.Remove(my_lock->name_);
983  DCHECK(removed);
984  delete my_lock;
985  return result;
986}
987
988Status ChromiumEnv::GetTestDirectory(std::string* path) {
989  mu_.Acquire();
990  if (test_directory_.empty()) {
991    if (!base::CreateNewTempDirectory(kLevelDBTestDirectoryPrefix,
992                                      &test_directory_)) {
993      mu_.Release();
994      RecordErrorAt(kGetTestDirectory);
995      return MakeIOError(
996          "Could not create temp directory.", "", kGetTestDirectory);
997    }
998  }
999  *path = FilePathToString(test_directory_);
1000  mu_.Release();
1001  return Status::OK();
1002}
1003
1004Status ChromiumEnv::NewLogger(const std::string& fname, Logger** result) {
1005  FILE* f = fopen_internal(fname.c_str(), "w");
1006  if (f == NULL) {
1007    *result = NULL;
1008    int saved_errno = errno;
1009    RecordOSError(kNewLogger, saved_errno);
1010    return MakeIOError(fname, strerror(saved_errno), kNewLogger, saved_errno);
1011  } else {
1012    *result = new ChromiumLogger(f);
1013    return Status::OK();
1014  }
1015}
1016
1017uint64_t ChromiumEnv::NowMicros() {
1018  return ::base::TimeTicks::Now().ToInternalValue();
1019}
1020
1021void ChromiumEnv::SleepForMicroseconds(int micros) {
1022  // Round up to the next millisecond.
1023  ::base::PlatformThread::Sleep(::base::TimeDelta::FromMicroseconds(micros));
1024}
1025
1026void ChromiumEnv::RecordErrorAt(MethodID method) const {
1027  GetMethodIOErrorHistogram()->Add(method);
1028}
1029
1030void ChromiumEnv::RecordLockFileAncestors(int num_missing_ancestors) const {
1031  GetLockFileAncestorHistogram()->Add(num_missing_ancestors);
1032}
1033
1034void ChromiumEnv::RecordOSError(MethodID method,
1035                                base::PlatformFileError error) const {
1036  DCHECK(error < 0);
1037  RecordErrorAt(method);
1038  GetOSErrorHistogram(method, -base::PLATFORM_FILE_ERROR_MAX)->Add(-error);
1039}
1040
1041void ChromiumEnv::RecordOSError(MethodID method, int error) const {
1042  DCHECK(error > 0);
1043  RecordErrorAt(method);
1044  GetOSErrorHistogram(method, ERANGE + 1)->Add(error);
1045}
1046
1047void ChromiumEnv::RecordBackupResult(bool result) const {
1048  std::string uma_name(name_);
1049  uma_name.append(".TableBackup");
1050  base::BooleanHistogram::FactoryGet(
1051      uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
1052}
1053
1054base::HistogramBase* ChromiumEnv::GetOSErrorHistogram(MethodID method,
1055                                                      int limit) const {
1056  std::string uma_name(name_);
1057  // TODO(dgrogan): This is probably not the best way to concatenate strings.
1058  uma_name.append(".IOError.").append(MethodIDToString(method));
1059  return base::LinearHistogram::FactoryGet(uma_name, 1, limit, limit + 1,
1060      base::Histogram::kUmaTargetedHistogramFlag);
1061}
1062
1063base::HistogramBase* ChromiumEnv::GetMethodIOErrorHistogram() const {
1064  std::string uma_name(name_);
1065  uma_name.append(".IOError");
1066  return base::LinearHistogram::FactoryGet(uma_name, 1, kNumEntries,
1067      kNumEntries + 1, base::Histogram::kUmaTargetedHistogramFlag);
1068}
1069
1070base::HistogramBase* ChromiumEnv::GetMaxFDHistogram(
1071    const std::string& type) const {
1072  std::string uma_name(name_);
1073  uma_name.append(".MaxFDs.").append(type);
1074  // These numbers make each bucket twice as large as the previous bucket.
1075  const int kFirstEntry = 1;
1076  const int kLastEntry = 65536;
1077  const int kNumBuckets = 18;
1078  return base::Histogram::FactoryGet(
1079      uma_name, kFirstEntry, kLastEntry, kNumBuckets,
1080      base::Histogram::kUmaTargetedHistogramFlag);
1081}
1082
1083base::HistogramBase* ChromiumEnv::GetLockFileAncestorHistogram() const {
1084  std::string uma_name(name_);
1085  uma_name.append(".LockFileAncestorsNotFound");
1086  const int kMin = 1;
1087  const int kMax = 10;
1088  const int kNumBuckets = 11;
1089  return base::LinearHistogram::FactoryGet(
1090      uma_name, kMin, kMax, kNumBuckets,
1091      base::Histogram::kUmaTargetedHistogramFlag);
1092}
1093
1094base::HistogramBase* ChromiumEnv::GetRetryTimeHistogram(MethodID method) const {
1095  std::string uma_name(name_);
1096  // TODO(dgrogan): This is probably not the best way to concatenate strings.
1097  uma_name.append(".TimeUntilSuccessFor").append(MethodIDToString(method));
1098
1099  const int kBucketSizeMillis = 25;
1100  // Add 2, 1 for each of the buckets <1 and >max.
1101  const int kNumBuckets = kMaxRetryTimeMillis / kBucketSizeMillis + 2;
1102  return base::Histogram::FactoryTimeGet(
1103      uma_name, base::TimeDelta::FromMilliseconds(1),
1104      base::TimeDelta::FromMilliseconds(kMaxRetryTimeMillis + 1),
1105      kNumBuckets,
1106      base::Histogram::kUmaTargetedHistogramFlag);
1107}
1108
1109base::HistogramBase* ChromiumEnv::GetRecoveredFromErrorHistogram(
1110    MethodID method) const {
1111  std::string uma_name(name_);
1112  uma_name.append(".RetryRecoveredFromErrorIn")
1113      .append(MethodIDToString(method));
1114  return base::LinearHistogram::FactoryGet(uma_name, 1, kNumEntries,
1115      kNumEntries + 1, base::Histogram::kUmaTargetedHistogramFlag);
1116}
1117
1118class Thread : public ::base::PlatformThread::Delegate {
1119 public:
1120  Thread(void (*function)(void* arg), void* arg)
1121      : function_(function), arg_(arg) {
1122    ::base::PlatformThreadHandle handle;
1123    bool success = ::base::PlatformThread::Create(0, this, &handle);
1124    DCHECK(success);
1125  }
1126  virtual ~Thread() {}
1127  virtual void ThreadMain() {
1128    (*function_)(arg_);
1129    delete this;
1130  }
1131
1132 private:
1133  void (*function_)(void* arg);
1134  void* arg_;
1135};
1136
1137void ChromiumEnv::Schedule(void (*function)(void*), void* arg) {
1138  mu_.Acquire();
1139
1140  // Start background thread if necessary
1141  if (!started_bgthread_) {
1142    started_bgthread_ = true;
1143    StartThread(&ChromiumEnv::BGThreadWrapper, this);
1144  }
1145
1146  // If the queue is currently empty, the background thread may currently be
1147  // waiting.
1148  if (queue_.empty()) {
1149    bgsignal_.Signal();
1150  }
1151
1152  // Add to priority queue
1153  queue_.push_back(BGItem());
1154  queue_.back().function = function;
1155  queue_.back().arg = arg;
1156
1157  mu_.Release();
1158}
1159
1160void ChromiumEnv::BGThread() {
1161  base::PlatformThread::SetName(name_.c_str());
1162
1163  while (true) {
1164    // Wait until there is an item that is ready to run
1165    mu_.Acquire();
1166    while (queue_.empty()) {
1167      bgsignal_.Wait();
1168    }
1169
1170    void (*function)(void*) = queue_.front().function;
1171    void* arg = queue_.front().arg;
1172    queue_.pop_front();
1173
1174    mu_.Release();
1175    TRACE_EVENT0("leveldb", "ChromiumEnv::BGThread-Task");
1176    (*function)(arg);
1177  }
1178}
1179
1180void ChromiumEnv::StartThread(void (*function)(void* arg), void* arg) {
1181  new Thread(function, arg); // Will self-delete.
1182}
1183
1184static std::string GetDirName(const std::string& filename) {
1185  base::FilePath file = base::FilePath::FromUTF8Unsafe(filename);
1186  return FilePathToString(file.DirName());
1187}
1188
1189void ChromiumEnv::DidCreateNewFile(const std::string& filename) {
1190  base::AutoLock auto_lock(map_lock_);
1191  needs_sync_map_[GetDirName(filename)] = true;
1192}
1193
1194bool ChromiumEnv::DoesDirNeedSync(const std::string& filename) {
1195  base::AutoLock auto_lock(map_lock_);
1196  return needs_sync_map_.find(GetDirName(filename)) != needs_sync_map_.end();
1197}
1198
1199void ChromiumEnv::DidSyncDir(const std::string& filename) {
1200  base::AutoLock auto_lock(map_lock_);
1201  needs_sync_map_.erase(GetDirName(filename));
1202}
1203
1204}  // namespace leveldb_env
1205
1206namespace leveldb {
1207
1208Env* IDBEnv() {
1209  return leveldb_env::idb_env.Pointer();
1210}
1211
1212Env* Env::Default() {
1213  return leveldb_env::default_env.Pointer();
1214}
1215
1216}  // namespace leveldb
1217
1218