1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// A policy for storing activity log data to a database that performs
6// aggregation to reduce the size of the database.  The database layout is
7// nearly the same as FullStreamUIPolicy, which stores a complete log, with a
8// few changes:
9//   - a "count" column is added to track how many log records were merged
10//     together into this row
11//   - the "time" column measures the most recent time that the current row was
12//     updated
13// When writing a record, if a row already exists where all other columns
14// (extension_id, action_type, api_name, args, urls, etc.) all match, and the
15// previous time falls within today (the current time), then the count field on
16// the old row is incremented.  Otherwise, a new row is written.
17//
18// For many text columns, repeated strings are compressed by moving string
19// storage to a separate table ("string_ids") and storing only an identifier in
20// the logging table.  For example, if the api_name_x column contained the
21// value 4 and the string_ids table contained a row with primary key 4 and
22// value 'tabs.query', then the api_name field should be taken to have the
23// value 'tabs.query'.  Each column ending with "_x" is compressed in this way.
24// All lookups are to the string_ids table, except for the page_url_x and
25// arg_url_x columns, which are converted via the url_ids table (this
26// separation of URL values is to help simplify history clearing).
27//
28// The activitylog_uncompressed view allows for simpler reading of the activity
29// log contents with identifiers already translated to string values.
30
31#include "chrome/browser/extensions/activity_log/counting_policy.h"
32
33#include <map>
34#include <string>
35#include <vector>
36
37#include "base/callback.h"
38#include "base/files/file_path.h"
39#include "base/json/json_reader.h"
40#include "base/json/json_string_value_serializer.h"
41#include "base/strings/string_util.h"
42#include "base/strings/stringprintf.h"
43#include "chrome/common/chrome_constants.h"
44
45using content::BrowserThread;
46
47namespace {
48
49using extensions::Action;
50
51// Delay between cleaning passes (to delete old action records) through the
52// database.
53const int kCleaningDelayInHours = 12;
54
55// We should log the arguments to these API calls.  Be careful when
56// constructing this whitelist to not keep arguments that might compromise
57// privacy by logging too much data to the activity log.
58//
59// TODO(mvrable): The contents of this whitelist should be reviewed and
60// expanded as needed.
61struct ApiList {
62  Action::ActionType type;
63  const char* name;
64};
65
66const ApiList kAlwaysLog[] = {
67    {Action::ACTION_API_CALL, "bookmarks.create"},
68    {Action::ACTION_API_CALL, "bookmarks.update"},
69    {Action::ACTION_API_CALL, "cookies.get"},
70    {Action::ACTION_API_CALL, "cookies.getAll"},
71    {Action::ACTION_API_CALL, "extension.connect"},
72    {Action::ACTION_API_CALL, "extension.sendMessage"},
73    {Action::ACTION_API_CALL, "fileSystem.chooseEntry"},
74    {Action::ACTION_API_CALL, "socket.bind"},
75    {Action::ACTION_API_CALL, "socket.connect"},
76    {Action::ACTION_API_CALL, "socket.create"},
77    {Action::ACTION_API_CALL, "socket.listen"},
78    {Action::ACTION_API_CALL, "tabs.executeScript"},
79    {Action::ACTION_API_CALL, "tabs.insertCSS"},
80    {Action::ACTION_API_CALL, "types.ChromeSetting.clear"},
81    {Action::ACTION_API_CALL, "types.ChromeSetting.get"},
82    {Action::ACTION_API_CALL, "types.ChromeSetting.set"},
83    {Action::ACTION_CONTENT_SCRIPT, ""},
84    {Action::ACTION_DOM_ACCESS, "Document.createElement"},
85    {Action::ACTION_DOM_ACCESS, "Document.createElementNS"},
86};
87
88// Columns in the main database table.  See the file-level comment for a
89// discussion of how data is stored and the meanings of the _x columns.
90const char* kTableContentFields[] = {
91    "count", "extension_id_x", "time", "action_type", "api_name_x", "args_x",
92    "page_url_x", "page_title_x", "arg_url_x", "other_x"};
93const char* kTableFieldTypes[] = {
94    "INTEGER NOT NULL DEFAULT 1", "INTEGER NOT NULL", "INTEGER", "INTEGER",
95    "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",
96    "INTEGER"};
97
98// Miscellaneous SQL commands for initializing the database; these should be
99// idempotent.
100static const char kPolicyMiscSetup[] =
101    // The activitylog_uncompressed view performs string lookups for simpler
102    // access to the log data.
103    "DROP VIEW IF EXISTS activitylog_uncompressed;\n"
104    "CREATE VIEW activitylog_uncompressed AS\n"
105    "SELECT count,\n"
106    "    x1.value AS extension_id,\n"
107    "    time,\n"
108    "    action_type,\n"
109    "    x2.value AS api_name,\n"
110    "    x3.value AS args,\n"
111    "    x4.value AS page_url,\n"
112    "    x5.value AS page_title,\n"
113    "    x6.value AS arg_url,\n"
114    "    x7.value AS other,\n"
115    "    activitylog_compressed.rowid AS activity_id\n"
116    "FROM activitylog_compressed\n"
117    "    LEFT JOIN string_ids AS x1 ON (x1.id = extension_id_x)\n"
118    "    LEFT JOIN string_ids AS x2 ON (x2.id = api_name_x)\n"
119    "    LEFT JOIN string_ids AS x3 ON (x3.id = args_x)\n"
120    "    LEFT JOIN url_ids    AS x4 ON (x4.id = page_url_x)\n"
121    "    LEFT JOIN string_ids AS x5 ON (x5.id = page_title_x)\n"
122    "    LEFT JOIN url_ids    AS x6 ON (x6.id = arg_url_x)\n"
123    "    LEFT JOIN string_ids AS x7 ON (x7.id = other_x);\n"
124    // An index on all fields except count and time: all the fields that aren't
125    // changed when incrementing a count.  This should accelerate finding the
126    // rows to update (at worst several rows will need to be checked to find
127    // the one in the right time range).
128    "CREATE INDEX IF NOT EXISTS activitylog_compressed_index\n"
129    "ON activitylog_compressed(extension_id_x, action_type, api_name_x,\n"
130    "    args_x, page_url_x, page_title_x, arg_url_x, other_x)";
131
132// SQL statements to clean old, unused entries out of the string and URL id
133// tables.
134static const char kStringTableCleanup[] =
135    "DELETE FROM string_ids WHERE id NOT IN\n"
136    "(SELECT extension_id_x FROM activitylog_compressed\n"
137    "    WHERE extension_id_x IS NOT NULL\n"
138    " UNION SELECT api_name_x FROM activitylog_compressed\n"
139    "    WHERE api_name_x IS NOT NULL\n"
140    " UNION SELECT args_x FROM activitylog_compressed\n"
141    "    WHERE args_x IS NOT NULL\n"
142    " UNION SELECT page_title_x FROM activitylog_compressed\n"
143    "    WHERE page_title_x IS NOT NULL\n"
144    " UNION SELECT other_x FROM activitylog_compressed\n"
145    "    WHERE other_x IS NOT NULL)";
146static const char kUrlTableCleanup[] =
147    "DELETE FROM url_ids WHERE id NOT IN\n"
148    "(SELECT page_url_x FROM activitylog_compressed\n"
149    "    WHERE page_url_x IS NOT NULL\n"
150    " UNION SELECT arg_url_x FROM activitylog_compressed\n"
151    "    WHERE arg_url_x IS NOT NULL)";
152
153}  // namespace
154
155namespace extensions {
156
157const char* CountingPolicy::kTableName = "activitylog_compressed";
158const char* CountingPolicy::kReadViewName = "activitylog_uncompressed";
159
160CountingPolicy::CountingPolicy(Profile* profile)
161    : ActivityLogDatabasePolicy(
162          profile,
163          base::FilePath(chrome::kExtensionActivityLogFilename)),
164      string_table_("string_ids"),
165      url_table_("url_ids"),
166      retention_time_(base::TimeDelta::FromHours(60)) {
167  for (size_t i = 0; i < arraysize(kAlwaysLog); i++) {
168    api_arg_whitelist_.insert(
169        std::make_pair(kAlwaysLog[i].type, kAlwaysLog[i].name));
170  }
171}
172
173CountingPolicy::~CountingPolicy() {}
174
175bool CountingPolicy::InitDatabase(sql::Connection* db) {
176  if (!Util::DropObsoleteTables(db))
177    return false;
178
179  if (!string_table_.Initialize(db))
180    return false;
181  if (!url_table_.Initialize(db))
182    return false;
183
184  // Create the unified activity log entry table.
185  if (!ActivityDatabase::InitializeTable(db,
186                                         kTableName,
187                                         kTableContentFields,
188                                         kTableFieldTypes,
189                                         arraysize(kTableContentFields)))
190    return false;
191
192  // Create a view for easily accessing the uncompressed form of the data, and
193  // any necessary indexes if needed.
194  return db->Execute(kPolicyMiscSetup);
195}
196
197void CountingPolicy::ProcessAction(scoped_refptr<Action> action) {
198  ScheduleAndForget(this, &CountingPolicy::QueueAction, action);
199}
200
201void CountingPolicy::QueueAction(scoped_refptr<Action> action) {
202  if (activity_database()->is_db_valid()) {
203    action = action->Clone();
204    Util::StripPrivacySensitiveFields(action);
205    Util::StripArguments(api_arg_whitelist_, action);
206
207    // If the current action falls on a different date than the ones in the
208    // queue, flush the queue out now to prevent any false merging (actions
209    // from different days being merged).
210    base::Time new_date = action->time().LocalMidnight();
211    if (new_date != queued_actions_date_)
212      activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately);
213    queued_actions_date_ = new_date;
214
215    ActionQueue::iterator queued_entry = queued_actions_.find(action);
216    if (queued_entry == queued_actions_.end()) {
217      queued_actions_[action] = 1;
218    } else {
219      // Update the timestamp in the key to be the latest time seen.  Modifying
220      // the time is safe since that field is not involved in key comparisons
221      // in the map.
222      using std::max;
223      queued_entry->first->set_time(
224          max(queued_entry->first->time(), action->time()));
225      queued_entry->second++;
226    }
227    activity_database()->AdviseFlush(queued_actions_.size());
228  }
229}
230
231bool CountingPolicy::FlushDatabase(sql::Connection* db) {
232  // Columns that must match exactly for database rows to be coalesced.
233  static const char* matched_columns[] = {
234      "extension_id_x", "action_type", "api_name_x", "args_x", "page_url_x",
235      "page_title_x", "arg_url_x", "other_x"};
236  ActionQueue queue;
237  queue.swap(queued_actions_);
238
239  // Whether to clean old records out of the activity log database.  Do this
240  // much less frequently than database flushes since it is expensive, but
241  // always check on the first database flush (since there might be a large
242  // amount of data to clear).
243  bool clean_database = (last_database_cleaning_time_.is_null() ||
244                         Now() - last_database_cleaning_time_ >
245                             base::TimeDelta::FromHours(kCleaningDelayInHours));
246
247  if (queue.empty() && !clean_database)
248    return true;
249
250  sql::Transaction transaction(db);
251  if (!transaction.Begin())
252    return false;
253
254  // Adding an Action to the database is a two step process that depends on
255  // whether the count on an existing row can be incremented or a new row needs
256  // to be inserted.
257  //   1. Run the query in locate_str to search for a row which matches and can
258  //      have the count incremented.
259  //  2a. If found, increment the count using update_str and the rowid found in
260  //      step 1, or
261  //  2b. If not found, insert a new row using insert_str.
262  std::string locate_str =
263      "SELECT rowid FROM " + std::string(kTableName) +
264      " WHERE time >= ? AND time < ?";
265  std::string insert_str =
266      "INSERT INTO " + std::string(kTableName) + "(count, time";
267  std::string update_str =
268      "UPDATE " + std::string(kTableName) +
269      " SET count = count + ?, time = max(?, time)"
270      " WHERE rowid = ?";
271
272  for (size_t i = 0; i < arraysize(matched_columns); i++) {
273    locate_str = base::StringPrintf(
274        "%s AND %s IS ?", locate_str.c_str(), matched_columns[i]);
275    insert_str =
276        base::StringPrintf("%s, %s", insert_str.c_str(), matched_columns[i]);
277  }
278  insert_str += ") VALUES (?, ?";
279  for (size_t i = 0; i < arraysize(matched_columns); i++) {
280    insert_str += ", ?";
281  }
282  locate_str += " ORDER BY time DESC LIMIT 1";
283  insert_str += ")";
284
285  for (ActionQueue::iterator i = queue.begin(); i != queue.end(); ++i) {
286    const Action& action = *i->first.get();
287    int count = i->second;
288
289    base::Time day_start = action.time().LocalMidnight();
290    base::Time next_day = Util::AddDays(day_start, 1);
291
292    // The contents in values must match up with fields in matched_columns.  A
293    // value of -1 is used to encode a null database value.
294    int64 id;
295    std::vector<int64> matched_values;
296
297    if (!string_table_.StringToInt(db, action.extension_id(), &id))
298      return false;
299    matched_values.push_back(id);
300
301    matched_values.push_back(static_cast<int>(action.action_type()));
302
303    if (!string_table_.StringToInt(db, action.api_name(), &id))
304      return false;
305    matched_values.push_back(id);
306
307    if (action.args()) {
308      std::string args = Util::Serialize(action.args());
309      // TODO(mvrable): For now, truncate long argument lists.  This is a
310      // workaround for excessively-long values coming from DOM logging.  When
311      // the V8ValueConverter is fixed to return more reasonable values, we can
312      // drop the truncation.
313      if (args.length() > 10000) {
314        args = "[\"<too_large>\"]";
315      }
316      if (!string_table_.StringToInt(db, args, &id))
317        return false;
318      matched_values.push_back(id);
319    } else {
320      matched_values.push_back(-1);
321    }
322
323    std::string page_url_string = action.SerializePageUrl();
324    if (!page_url_string.empty()) {
325      if (!url_table_.StringToInt(db, page_url_string, &id))
326        return false;
327      matched_values.push_back(id);
328    } else {
329      matched_values.push_back(-1);
330    }
331
332    // TODO(mvrable): Create a title_table_?
333    if (!action.page_title().empty()) {
334      if (!string_table_.StringToInt(db, action.page_title(), &id))
335        return false;
336      matched_values.push_back(id);
337    } else {
338      matched_values.push_back(-1);
339    }
340
341    std::string arg_url_string = action.SerializeArgUrl();
342    if (!arg_url_string.empty()) {
343      if (!url_table_.StringToInt(db, arg_url_string, &id))
344        return false;
345      matched_values.push_back(id);
346    } else {
347      matched_values.push_back(-1);
348    }
349
350    if (action.other()) {
351      if (!string_table_.StringToInt(db, Util::Serialize(action.other()), &id))
352        return false;
353      matched_values.push_back(id);
354    } else {
355      matched_values.push_back(-1);
356    }
357
358    // Search for a matching row for this action whose count can be
359    // incremented.
360    sql::Statement locate_statement(db->GetCachedStatement(
361        sql::StatementID(SQL_FROM_HERE), locate_str.c_str()));
362    locate_statement.BindInt64(0, day_start.ToInternalValue());
363    locate_statement.BindInt64(1, next_day.ToInternalValue());
364    for (size_t j = 0; j < matched_values.size(); j++) {
365      // A call to BindNull when matched_values contains -1 is likely not
366      // necessary as parameters default to null before they are explicitly
367      // bound.  But to be completely clear, and in case a cached statement
368      // ever comes with some values already bound, we bind all parameters
369      // (even null ones) explicitly.
370      if (matched_values[j] == -1)
371        locate_statement.BindNull(j + 2);
372      else
373        locate_statement.BindInt64(j + 2, matched_values[j]);
374    }
375
376    if (locate_statement.Step()) {
377      // A matching row was found.  Update the count and time.
378      int64 rowid = locate_statement.ColumnInt64(0);
379      sql::Statement update_statement(db->GetCachedStatement(
380          sql::StatementID(SQL_FROM_HERE), update_str.c_str()));
381      update_statement.BindInt(0, count);
382      update_statement.BindInt64(1, action.time().ToInternalValue());
383      update_statement.BindInt64(2, rowid);
384      if (!update_statement.Run())
385        return false;
386    } else if (locate_statement.Succeeded()) {
387      // No matching row was found, so we need to insert one.
388      sql::Statement insert_statement(db->GetCachedStatement(
389          sql::StatementID(SQL_FROM_HERE), insert_str.c_str()));
390      insert_statement.BindInt(0, count);
391      insert_statement.BindInt64(1, action.time().ToInternalValue());
392      for (size_t j = 0; j < matched_values.size(); j++) {
393        if (matched_values[j] == -1)
394          insert_statement.BindNull(j + 2);
395        else
396          insert_statement.BindInt64(j + 2, matched_values[j]);
397      }
398      if (!insert_statement.Run())
399        return false;
400    } else {
401      // Database error.
402      return false;
403    }
404  }
405
406  if (clean_database) {
407    base::Time cutoff = (Now() - retention_time()).LocalMidnight();
408    if (!CleanOlderThan(db, cutoff))
409      return false;
410    last_database_cleaning_time_ = Now();
411  }
412
413  if (!transaction.Commit())
414    return false;
415
416  return true;
417}
418
419scoped_ptr<Action::ActionVector> CountingPolicy::DoReadFilteredData(
420    const std::string& extension_id,
421    const Action::ActionType type,
422    const std::string& api_name,
423    const std::string& page_url,
424    const std::string& arg_url,
425    const int days_ago) {
426  // Ensure data is flushed to the database first so that we query over all
427  // data.
428  activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately);
429  scoped_ptr<Action::ActionVector> actions(new Action::ActionVector());
430
431  sql::Connection* db = GetDatabaseConnection();
432  if (!db)
433    return actions.Pass();
434
435  // Build up the query based on which parameters were specified.
436  std::string where_str = "";
437  std::string where_next = "";
438  if (!extension_id.empty()) {
439    where_str += "extension_id=?";
440    where_next = " AND ";
441  }
442  if (!api_name.empty()) {
443    where_str += where_next + "api_name=?";
444    where_next = " AND ";
445  }
446  if (type != Action::ACTION_ANY) {
447    where_str += where_next + "action_type=?";
448    where_next = " AND ";
449  }
450  if (!page_url.empty()) {
451    where_str += where_next + "page_url LIKE ?";
452    where_next = " AND ";
453  }
454  if (!arg_url.empty()) {
455    where_str += where_next + "arg_url LIKE ?";
456    where_next = " AND ";
457  }
458  if (days_ago >= 0)
459    where_str += where_next + "time BETWEEN ? AND ?";
460
461  std::string query_str = base::StringPrintf(
462      "SELECT extension_id,time, action_type, api_name, args, page_url,"
463      "page_title, arg_url, other, count, activity_id FROM %s %s %s ORDER BY "
464      "count DESC, time DESC LIMIT 300",
465      kReadViewName,
466      where_str.empty() ? "" : "WHERE",
467      where_str.c_str());
468  sql::Statement query(db->GetUniqueStatement(query_str.c_str()));
469  int i = -1;
470  if (!extension_id.empty())
471    query.BindString(++i, extension_id);
472  if (!api_name.empty())
473    query.BindString(++i, api_name);
474  if (type != Action::ACTION_ANY)
475    query.BindInt(++i, static_cast<int>(type));
476  if (!page_url.empty())
477    query.BindString(++i, page_url + "%");
478  if (!arg_url.empty())
479    query.BindString(++i, arg_url + "%");
480  if (days_ago >= 0) {
481    int64 early_bound;
482    int64 late_bound;
483    Util::ComputeDatabaseTimeBounds(Now(), days_ago, &early_bound, &late_bound);
484    query.BindInt64(++i, early_bound);
485    query.BindInt64(++i, late_bound);
486  }
487
488  // Execute the query and get results.
489  while (query.is_valid() && query.Step()) {
490    scoped_refptr<Action> action =
491        new Action(query.ColumnString(0),
492                   base::Time::FromInternalValue(query.ColumnInt64(1)),
493                   static_cast<Action::ActionType>(query.ColumnInt(2)),
494                   query.ColumnString(3), query.ColumnInt64(10));
495
496    if (query.ColumnType(4) != sql::COLUMN_TYPE_NULL) {
497      scoped_ptr<base::Value> parsed_value(
498          base::JSONReader::Read(query.ColumnString(4)));
499      if (parsed_value && parsed_value->IsType(base::Value::TYPE_LIST)) {
500        action->set_args(make_scoped_ptr(
501            static_cast<base::ListValue*>(parsed_value.release())));
502      }
503    }
504
505    action->ParsePageUrl(query.ColumnString(5));
506    action->set_page_title(query.ColumnString(6));
507    action->ParseArgUrl(query.ColumnString(7));
508
509    if (query.ColumnType(8) != sql::COLUMN_TYPE_NULL) {
510      scoped_ptr<base::Value> parsed_value(
511          base::JSONReader::Read(query.ColumnString(8)));
512      if (parsed_value && parsed_value->IsType(base::Value::TYPE_DICTIONARY)) {
513        action->set_other(make_scoped_ptr(
514            static_cast<base::DictionaryValue*>(parsed_value.release())));
515      }
516    }
517    action->set_count(query.ColumnInt(9));
518    actions->push_back(action);
519  }
520
521  return actions.Pass();
522}
523
524void CountingPolicy::DoRemoveActions(const std::vector<int64>& action_ids) {
525  if (action_ids.empty())
526    return;
527
528  sql::Connection* db = GetDatabaseConnection();
529  if (!db) {
530    LOG(ERROR) << "Unable to connect to database";
531    return;
532  }
533
534  // Flush data first so the activity removal affects queued-up data as well.
535  activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately);
536
537  sql::Transaction transaction(db);
538  if (!transaction.Begin())
539    return;
540
541  std::string statement_str =
542      base::StringPrintf("DELETE FROM %s WHERE rowid = ?", kTableName);
543  sql::Statement statement(db->GetCachedStatement(
544      sql::StatementID(SQL_FROM_HERE), statement_str.c_str()));
545  for (size_t i = 0; i < action_ids.size(); i++) {
546    statement.Reset(true);
547    statement.BindInt64(0, action_ids[i]);
548    if (!statement.Run()) {
549      LOG(ERROR) << "Removing activities from database failed: "
550                 << statement.GetSQLStatement();
551      break;
552    }
553  }
554
555  CleanStringTables(db);
556
557  if (!transaction.Commit()) {
558    LOG(ERROR) << "Removing activities from database failed";
559  }
560}
561
562void CountingPolicy::DoRemoveURLs(const std::vector<GURL>& restrict_urls) {
563  sql::Connection* db = GetDatabaseConnection();
564  if (!db) {
565    LOG(ERROR) << "Unable to connect to database";
566    return;
567  }
568
569  // Flush data first so the URL clearing affects queued-up data as well.
570  activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately);
571
572  // If no restrictions then then all URLs need to be removed.
573  if (restrict_urls.empty()) {
574    std::string sql_str = base::StringPrintf(
575      "UPDATE %s SET page_url_x=NULL,page_title_x=NULL,arg_url_x=NULL",
576      kTableName);
577
578    sql::Statement statement;
579    statement.Assign(db->GetCachedStatement(
580        sql::StatementID(SQL_FROM_HERE), sql_str.c_str()));
581
582    if (!statement.Run()) {
583      LOG(ERROR) << "Removing all URLs from database failed: "
584                 << statement.GetSQLStatement();
585      return;
586    }
587  }
588
589  // If URLs are specified then restrict to only those URLs.
590  for (size_t i = 0; i < restrict_urls.size(); ++i) {
591    int64 url_id;
592    if (!restrict_urls[i].is_valid() ||
593        !url_table_.StringToInt(db, restrict_urls[i].spec(), &url_id)) {
594      continue;
595    }
596
597    // Remove any that match the page_url.
598    std::string sql_str = base::StringPrintf(
599      "UPDATE %s SET page_url_x=NULL,page_title_x=NULL WHERE page_url_x IS ?",
600      kTableName);
601
602    sql::Statement statement;
603    statement.Assign(db->GetCachedStatement(
604        sql::StatementID(SQL_FROM_HERE), sql_str.c_str()));
605    statement.BindInt64(0, url_id);
606
607    if (!statement.Run()) {
608      LOG(ERROR) << "Removing page URL from database failed: "
609                 << statement.GetSQLStatement();
610      return;
611    }
612
613    // Remove any that match the arg_url.
614    sql_str = base::StringPrintf(
615      "UPDATE %s SET arg_url_x=NULL WHERE arg_url_x IS ?", kTableName);
616
617    statement.Assign(db->GetCachedStatement(
618        sql::StatementID(SQL_FROM_HERE), sql_str.c_str()));
619    statement.BindInt64(0, url_id);
620
621    if (!statement.Run()) {
622      LOG(ERROR) << "Removing arg URL from database failed: "
623                 << statement.GetSQLStatement();
624      return;
625    }
626  }
627
628  // Clean up unused strings from the strings and urls table to really delete
629  // the urls and page titles. Should be called even if an error occured when
630  // removing a URL as there may some things to clean up.
631  CleanStringTables(db);
632}
633
634void CountingPolicy::DoRemoveExtensionData(const std::string& extension_id) {
635  if (extension_id.empty())
636    return;
637
638  sql::Connection* db = GetDatabaseConnection();
639  if (!db) {
640    LOG(ERROR) << "Unable to connect to database";
641    return;
642  }
643
644  // Make sure any queued in memory are sent to the database before cleaning.
645  activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately);
646
647  std::string sql_str = base::StringPrintf(
648      "DELETE FROM %s WHERE extension_id_x=?", kTableName);
649  sql::Statement statement(
650      db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE), sql_str.c_str()));
651  int64 id;
652  if (string_table_.StringToInt(db, extension_id, &id)) {
653    statement.BindInt64(0, id);
654  } else {
655    // If the string isn't listed, that means we never recorded anything about
656    // the extension so there's no deletion to do.
657    statement.Clear();
658    return;
659  }
660  if (!statement.Run()) {
661    LOG(ERROR) << "Removing URLs for extension "
662               << extension_id << "from database failed: "
663               << statement.GetSQLStatement();
664  }
665  CleanStringTables(db);
666}
667
668void CountingPolicy::DoDeleteDatabase() {
669  sql::Connection* db = GetDatabaseConnection();
670  if (!db) {
671    LOG(ERROR) << "Unable to connect to database";
672    return;
673  }
674
675  queued_actions_.clear();
676
677  // Not wrapped in a transaction because a late failure shouldn't undo a
678  // previous deletion.
679  std::string sql_str = base::StringPrintf("DELETE FROM %s", kTableName);
680  sql::Statement statement(db->GetCachedStatement(
681      sql::StatementID(SQL_FROM_HERE),
682      sql_str.c_str()));
683  if (!statement.Run()) {
684    LOG(ERROR) << "Deleting the database failed: "
685               << statement.GetSQLStatement();
686    return;
687  }
688  statement.Clear();
689  string_table_.ClearCache();
690  statement.Assign(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE),
691                                          "DELETE FROM string_ids"));
692  if (!statement.Run()) {
693    LOG(ERROR) << "Deleting the database failed: "
694               << statement.GetSQLStatement();
695    return;
696  }
697  statement.Clear();
698  url_table_.ClearCache();
699  statement.Assign(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE),
700                                          "DELETE FROM url_ids"));
701  if (!statement.Run()) {
702    LOG(ERROR) << "Deleting the database failed: "
703               << statement.GetSQLStatement();
704    return;
705  }
706  statement.Clear();
707  statement.Assign(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE),
708                                          "VACUUM"));
709  if (!statement.Run()) {
710    LOG(ERROR) << "Vacuuming the database failed: "
711               << statement.GetSQLStatement();
712  }
713}
714
715void CountingPolicy::ReadFilteredData(
716    const std::string& extension_id,
717    const Action::ActionType type,
718    const std::string& api_name,
719    const std::string& page_url,
720    const std::string& arg_url,
721    const int days_ago,
722    const base::Callback
723        <void(scoped_ptr<Action::ActionVector>)>& callback) {
724  BrowserThread::PostTaskAndReplyWithResult(
725      BrowserThread::DB,
726      FROM_HERE,
727      base::Bind(&CountingPolicy::DoReadFilteredData,
728                 base::Unretained(this),
729                 extension_id,
730                 type,
731                 api_name,
732                 page_url,
733                 arg_url,
734                 days_ago),
735      callback);
736}
737
738void CountingPolicy::RemoveActions(const std::vector<int64>& action_ids) {
739  ScheduleAndForget(this, &CountingPolicy::DoRemoveActions, action_ids);
740}
741
742void CountingPolicy::RemoveURLs(const std::vector<GURL>& restrict_urls) {
743  ScheduleAndForget(this, &CountingPolicy::DoRemoveURLs, restrict_urls);
744}
745
746void CountingPolicy::RemoveExtensionData(const std::string& extension_id) {
747  ScheduleAndForget(this, &CountingPolicy::DoRemoveExtensionData, extension_id);
748}
749
750void CountingPolicy::DeleteDatabase() {
751  ScheduleAndForget(this, &CountingPolicy::DoDeleteDatabase);
752}
753
754void CountingPolicy::OnDatabaseFailure() {
755  queued_actions_.clear();
756}
757
758void CountingPolicy::OnDatabaseClose() {
759  delete this;
760}
761
762// Cleans old records from the activity log database.
763bool CountingPolicy::CleanOlderThan(sql::Connection* db,
764                                    const base::Time& cutoff) {
765  std::string clean_statement =
766      "DELETE FROM " + std::string(kTableName) + " WHERE time < ?";
767  sql::Statement cleaner(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE),
768                                                clean_statement.c_str()));
769  cleaner.BindInt64(0, cutoff.ToInternalValue());
770  if (!cleaner.Run())
771    return false;
772  return CleanStringTables(db);
773}
774
775// Cleans unused interned strings from the database.  This should be run after
776// deleting rows from the main log table to clean out stale values.
777bool CountingPolicy::CleanStringTables(sql::Connection* db) {
778  sql::Statement cleaner1(db->GetCachedStatement(
779      sql::StatementID(SQL_FROM_HERE), kStringTableCleanup));
780  if (!cleaner1.Run())
781    return false;
782  if (db->GetLastChangeCount() > 0)
783    string_table_.ClearCache();
784
785  sql::Statement cleaner2(db->GetCachedStatement(
786      sql::StatementID(SQL_FROM_HERE), kUrlTableCleanup));
787  if (!cleaner2.Run())
788    return false;
789  if (db->GetLastChangeCount() > 0)
790    url_table_.ClearCache();
791
792  return true;
793}
794
795void CountingPolicy::Close() {
796  // The policy object should have never been created if there's no DB thread.
797  DCHECK(BrowserThread::IsMessageLoopValid(BrowserThread::DB));
798  ScheduleAndForget(activity_database(), &ActivityDatabase::Close);
799}
800
801}  // namespace extensions
802