1// Copyright (c) 2010 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/history/history_database.h"
6
7#include <algorithm>
8#include <set>
9#include <string>
10#include "app/sql/transaction.h"
11#include "base/command_line.h"
12#include "base/file_util.h"
13#include "base/metrics/histogram.h"
14#include "base/rand_util.h"
15#include "base/string_util.h"
16#include "chrome/browser/diagnostics/sqlite_diagnostics.h"
17
18#if defined(OS_MACOSX)
19#include "base/mac/mac_util.h"
20#endif
21
22namespace history {
23
24namespace {
25
26// Current version number. We write databases at the "current" version number,
27// but any previous version that can read the "compatible" one can make do with
28// or database without *too* many bad effects.
29static const int kCurrentVersionNumber = 20;
30static const int kCompatibleVersionNumber = 16;
31static const char kEarlyExpirationThresholdKey[] = "early_expiration_threshold";
32
33// Key in the meta table used to determine if we need to migrate thumbnails out
34// of history.
35static const char kNeedsThumbnailMigrationKey[] = "needs_thumbnail_migration";
36
37void ComputeDatabaseMetrics(const FilePath& history_name,
38                            sql::Connection& db) {
39  if (base::RandInt(1, 100) != 50)
40    return;  // Only do this computation sometimes since it can be expensive.
41
42  int64 file_size = 0;
43  if (!file_util::GetFileSize(history_name, &file_size))
44    return;
45  int file_mb = static_cast<int>(file_size / (1024 * 1024));
46  UMA_HISTOGRAM_MEMORY_MB("History.DatabaseFileMB", file_mb);
47
48  sql::Statement url_count(db.GetUniqueStatement("SELECT count(*) FROM urls"));
49  if (!url_count || !url_count.Step())
50    return;
51  UMA_HISTOGRAM_COUNTS("History.URLTableCount", url_count.ColumnInt(0));
52
53  sql::Statement visit_count(db.GetUniqueStatement(
54      "SELECT count(*) FROM visits"));
55  if (!visit_count || !visit_count.Step())
56    return;
57  UMA_HISTOGRAM_COUNTS("History.VisitTableCount", visit_count.ColumnInt(0));
58}
59
60}  // namespace
61
62HistoryDatabase::HistoryDatabase()
63    : needs_version_17_migration_(false) {
64}
65
66HistoryDatabase::~HistoryDatabase() {
67}
68
69sql::InitStatus HistoryDatabase::Init(const FilePath& history_name,
70                                      const FilePath& bookmarks_path) {
71  // Set the exceptional sqlite error handler.
72  db_.set_error_delegate(GetErrorHandlerForHistoryDb());
73
74  // Set the database page size to something a little larger to give us
75  // better performance (we're typically seek rather than bandwidth limited).
76  // This only has an effect before any tables have been created, otherwise
77  // this is a NOP. Must be a power of 2 and a max of 8192.
78  db_.set_page_size(4096);
79
80  // Increase the cache size. The page size, plus a little extra, times this
81  // value, tells us how much memory the cache will use maximum.
82  // 6000 * 4MB = 24MB
83  // TODO(brettw) scale this value to the amount of available memory.
84  db_.set_cache_size(6000);
85
86  // Note that we don't set exclusive locking here. That's done by
87  // BeginExclusiveMode below which is called later (we have to be in shared
88  // mode to start out for the in-memory backend to read the data).
89
90  if (!db_.Open(history_name))
91    return sql::INIT_FAILURE;
92
93  // Wrap the rest of init in a tranaction. This will prevent the database from
94  // getting corrupted if we crash in the middle of initialization or migration.
95  sql::Transaction committer(&db_);
96  if (!committer.Begin())
97    return sql::INIT_FAILURE;
98
99#if defined(OS_MACOSX)
100  // Exclude the history file and its journal from backups.
101  base::mac::SetFileBackupExclusion(history_name, true);
102  FilePath::StringType history_name_string(history_name.value());
103  history_name_string += "-journal";
104  FilePath history_journal_name(history_name_string);
105  base::mac::SetFileBackupExclusion(history_journal_name, true);
106#endif
107
108  // Prime the cache.
109  db_.Preload();
110
111  // Create the tables and indices.
112  // NOTE: If you add something here, also add it to
113  //       RecreateAllButStarAndURLTables.
114  if (!meta_table_.Init(&db_, GetCurrentVersion(), kCompatibleVersionNumber))
115    return sql::INIT_FAILURE;
116  if (!CreateURLTable(false) || !InitVisitTable() ||
117      !InitKeywordSearchTermsTable() || !InitDownloadTable() ||
118      !InitSegmentTables())
119    return sql::INIT_FAILURE;
120  CreateMainURLIndex();
121  CreateKeywordSearchTermsIndices();
122
123  // Version check.
124  sql::InitStatus version_status = EnsureCurrentVersion(bookmarks_path);
125  if (version_status != sql::INIT_OK)
126    return version_status;
127
128  ComputeDatabaseMetrics(history_name, db_);
129  return committer.Commit() ? sql::INIT_OK : sql::INIT_FAILURE;
130}
131
132void HistoryDatabase::BeginExclusiveMode() {
133  // We can't use set_exclusive_locking() since that only has an effect before
134  // the DB is opened.
135  db_.Execute("PRAGMA locking_mode=EXCLUSIVE");
136}
137
138// static
139int HistoryDatabase::GetCurrentVersion() {
140  return kCurrentVersionNumber;
141}
142
143void HistoryDatabase::BeginTransaction() {
144  db_.BeginTransaction();
145}
146
147void HistoryDatabase::CommitTransaction() {
148  db_.CommitTransaction();
149}
150
151bool HistoryDatabase::RecreateAllTablesButURL() {
152  if (!DropVisitTable())
153    return false;
154  if (!InitVisitTable())
155    return false;
156
157  if (!DropKeywordSearchTermsTable())
158    return false;
159  if (!InitKeywordSearchTermsTable())
160    return false;
161
162  if (!DropSegmentTables())
163    return false;
164  if (!InitSegmentTables())
165    return false;
166
167  // We also add the supplementary URL indices at this point. This index is
168  // over parts of the URL table that weren't automatically created when the
169  // temporary URL table was
170  CreateKeywordSearchTermsIndices();
171  return true;
172}
173
174void HistoryDatabase::Vacuum() {
175  DCHECK_EQ(0, db_.transaction_nesting()) <<
176      "Can not have a transaction when vacuuming.";
177  db_.Execute("VACUUM");
178}
179
180void HistoryDatabase::ThumbnailMigrationDone() {
181  meta_table_.SetValue(kNeedsThumbnailMigrationKey, 0);
182}
183
184bool HistoryDatabase::GetNeedsThumbnailMigration() {
185  int value = 0;
186  return (meta_table_.GetValue(kNeedsThumbnailMigrationKey, &value) &&
187          value != 0);
188}
189
190bool HistoryDatabase::SetSegmentID(VisitID visit_id, SegmentID segment_id) {
191  sql::Statement s(db_.GetCachedStatement(SQL_FROM_HERE,
192      "UPDATE visits SET segment_id = ? WHERE id = ?"));
193  if (!s) {
194    NOTREACHED() << db_.GetErrorMessage();
195    return false;
196  }
197  s.BindInt64(0, segment_id);
198  s.BindInt64(1, visit_id);
199  DCHECK(db_.GetLastChangeCount() == 1);
200  return s.Run();
201}
202
203SegmentID HistoryDatabase::GetSegmentID(VisitID visit_id) {
204  sql::Statement s(db_.GetCachedStatement(SQL_FROM_HERE,
205      "SELECT segment_id FROM visits WHERE id = ?"));
206  if (!s) {
207    NOTREACHED() << db_.GetErrorMessage();
208    return 0;
209  }
210
211  s.BindInt64(0, visit_id);
212  if (s.Step()) {
213    if (s.ColumnType(0) == sql::COLUMN_TYPE_NULL)
214      return 0;
215    else
216      return s.ColumnInt64(0);
217  }
218  return 0;
219}
220
221base::Time HistoryDatabase::GetEarlyExpirationThreshold() {
222  if (!cached_early_expiration_threshold_.is_null())
223    return cached_early_expiration_threshold_;
224
225  int64 threshold;
226  if (!meta_table_.GetValue(kEarlyExpirationThresholdKey, &threshold)) {
227    // Set to a very early non-zero time, so it's before all history, but not
228    // zero to avoid re-retrieval.
229    threshold = 1L;
230  }
231
232  cached_early_expiration_threshold_ = base::Time::FromInternalValue(threshold);
233  return cached_early_expiration_threshold_;
234}
235
236void HistoryDatabase::UpdateEarlyExpirationThreshold(base::Time threshold) {
237  meta_table_.SetValue(kEarlyExpirationThresholdKey,
238                       threshold.ToInternalValue());
239  cached_early_expiration_threshold_ = threshold;
240}
241
242sql::Connection& HistoryDatabase::GetDB() {
243  return db_;
244}
245
246// Migration -------------------------------------------------------------------
247
248sql::InitStatus HistoryDatabase::EnsureCurrentVersion(
249    const FilePath& tmp_bookmarks_path) {
250  // We can't read databases newer than we were designed for.
251  if (meta_table_.GetCompatibleVersionNumber() > kCurrentVersionNumber) {
252    LOG(WARNING) << "History database is too new.";
253    return sql::INIT_TOO_NEW;
254  }
255
256  // NOTICE: If you are changing structures for things shared with the archived
257  // history file like URLs, visits, or downloads, that will need migration as
258  // well. Instead of putting such migration code in this class, it should be
259  // in the corresponding file (url_database.cc, etc.) and called from here and
260  // from the archived_database.cc.
261
262  int cur_version = meta_table_.GetVersionNumber();
263
264  // Put migration code here
265
266  if (cur_version == 15) {
267    if (!MigrateBookmarksToFile(tmp_bookmarks_path) ||
268        !DropStarredIDFromURLs()) {
269      LOG(WARNING) << "Unable to update history database to version 16.";
270      return sql::INIT_FAILURE;
271    }
272    ++cur_version;
273    meta_table_.SetVersionNumber(cur_version);
274    meta_table_.SetCompatibleVersionNumber(
275        std::min(cur_version, kCompatibleVersionNumber));
276  }
277
278  if (cur_version == 16) {
279#if !defined(OS_WIN)
280    // In this version we bring the time format on Mac & Linux in sync with the
281    // Windows version so that profiles can be moved between computers.
282    MigrateTimeEpoch();
283#endif
284    // On all platforms we bump the version number, so on Windows this
285    // migration is a NOP. We keep the compatible version at 16 since things
286    // will basically still work, just history will be in the future if an
287    // old version reads it.
288    ++cur_version;
289    meta_table_.SetVersionNumber(cur_version);
290  }
291
292  if (cur_version == 17) {
293    // Version 17 was for thumbnails to top sites migration. We ended up
294    // disabling it though, so 17->18 does nothing.
295    ++cur_version;
296    meta_table_.SetVersionNumber(cur_version);
297  }
298
299  if (cur_version == 18) {
300    // This is the version prior to adding url_source column. We need to
301    // migrate the database.
302    cur_version = 19;
303    meta_table_.SetVersionNumber(cur_version);
304  }
305
306  if (cur_version == 19) {
307    cur_version++;
308    meta_table_.SetVersionNumber(cur_version);
309    // Set a key indicating we need to migrate thumbnails. When successfull the
310    // key is removed (ThumbnailMigrationDone).
311    meta_table_.SetValue(kNeedsThumbnailMigrationKey, 1);
312  }
313
314  // When the version is too old, we just try to continue anyway, there should
315  // not be a released product that makes a database too old for us to handle.
316  LOG_IF(WARNING, cur_version < GetCurrentVersion()) <<
317         "History database version " << cur_version << " is too old to handle.";
318
319  return sql::INIT_OK;
320}
321
322#if !defined(OS_WIN)
323void HistoryDatabase::MigrateTimeEpoch() {
324  // Update all the times in the URLs and visits table in the main database.
325  // For visits, clear the indexed flag since we'll delete the FTS databases in
326  // the next step.
327  db_.Execute(
328      "UPDATE urls "
329      "SET last_visit_time = last_visit_time + 11644473600000000 "
330      "WHERE id IN (SELECT id FROM urls WHERE last_visit_time > 0);");
331  db_.Execute(
332      "UPDATE visits "
333      "SET visit_time = visit_time + 11644473600000000, is_indexed = 0 "
334      "WHERE id IN (SELECT id FROM visits WHERE visit_time > 0);");
335  db_.Execute(
336      "UPDATE segment_usage "
337      "SET time_slot = time_slot + 11644473600000000 "
338      "WHERE id IN (SELECT id FROM segment_usage WHERE time_slot > 0);");
339
340  // Erase all the full text index files. These will take a while to update and
341  // are less important, so we just blow them away. Same with the archived
342  // database.
343  needs_version_17_migration_ = true;
344}
345#endif
346
347}  // namespace history
348