1c3aae25116e66c177579b0b79182b09340b19753Chris Lattner// Copyright 2013 The Chromium Authors. All rights reserved. 2ea61c358720aa6c7a159d51658b34276316aa841Misha Brukman// Use of this source code is governed by a BSD-style license that can be 36fbcc26f1460eaee4e0eb8b426fc1ff0c7af11beJohn Criswell// found in the LICENSE file. 46fbcc26f1460eaee4e0eb8b426fc1ff0c7af11beJohn Criswell 57ed47a13356daed2a34cd2209a31f92552e3bdd8Chris Lattner// A policy for storing activity log data to a database that performs 67ed47a13356daed2a34cd2209a31f92552e3bdd8Chris Lattner// aggregation to reduce the size of the database. The database layout is 7ea61c358720aa6c7a159d51658b34276316aa841Misha Brukman// nearly the same as FullStreamUIPolicy, which stores a complete log, with a 86fbcc26f1460eaee4e0eb8b426fc1ff0c7af11beJohn Criswell// few changes: 9ea61c358720aa6c7a159d51658b34276316aa841Misha Brukman// - a "count" column is added to track how many log records were merged 10c3aae25116e66c177579b0b79182b09340b19753Chris Lattner// together into this row 11c3aae25116e66c177579b0b79182b09340b19753Chris Lattner// - the "time" column measures the most recent time that the current row was 12ea61c358720aa6c7a159d51658b34276316aa841Misha Brukman// updated 13cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner// When writing a record, if a row already exists where all other columns 14cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner// (extension_id, action_type, api_name, args, urls, etc.) all match, and the 15cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner// previous time falls within today (the current time), then the count field on 16cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner// the old row is incremented. Otherwise, a new row is written. 17cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner// 18c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene// For many text columns, repeated strings are compressed by moving string 194b84086e89d86fb16f562166d9fea8df37db6be7Dan Gohman// storage to a separate table ("string_ids") and storing only an identifier in 20255f89faee13dc491cb64fbeae3c763e7e2ea4e6Chandler Carruth// the logging table. For example, if the api_name_x column contained the 21444b4bf5c84c80833ff283244de0885124091a13Nadav Rotem// value 4 and the string_ids table contained a row with primary key 4 and 22583bd47f777fe3eb8305872fa0eadab31e833dffJim Laskey// value 'tabs.query', then the api_name field should be taken to have the 23c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman// value 'tabs.query'. Each column ending with "_x" is compressed in this way. 2498a366d547772010e94609e4584489b3e5ce0043Bill Wendling// All lookups are to the string_ids table, except for the page_url_x and 25acaf09dbe4a6781163857db1321bbd5795e7d410Dan Gohman// arg_url_x columns, which are converted via the url_ids table (this 26322812e603705e1c2037313633e72f689524b163Evan Cheng// separation of URL values is to help simplify history clearing). 27eb19e40efbd3cae80c908a30cdf4d33450733c45Chris Lattner// 28255f89faee13dc491cb64fbeae3c763e7e2ea4e6Chandler Carruth// The activitylog_uncompressed view allows for simpler reading of the activity 29d0fde30ce850b78371fd1386338350591f9ff494Brian Gaeke// log contents with identifiers already translated to string values. 30d0fde30ce850b78371fd1386338350591f9ff494Brian Gaeke 31fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman#include "chrome/browser/extensions/activity_log/counting_policy.h" 32fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman 33b4e6a5df5dada0cd919cc6e2717eb3118db9cc45Bill Wendling#include <map> 34b4e6a5df5dada0cd919cc6e2717eb3118db9cc45Bill Wendling#include <string> 3531441b7e95e0840e1ae144e5db6f791d6a36bc60Evan Cheng#include <vector> 36bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen 37b4e6a5df5dada0cd919cc6e2717eb3118db9cc45Bill Wendling#include "base/callback.h" 38ff7a562751604a9fe13efc75bd59622244b54d35Dan Gohman#include "base/files/file_path.h" 39e4b4edd72ae596ac8d1fdbc6e77f61e331f6775dChandler Carruth#include "base/json/json_reader.h" 40fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman#include "base/json/json_string_value_serializer.h" 418c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei#include "base/strings/string_util.h" 428c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei#include "base/strings/stringprintf.h" 438c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei#include "chrome/common/chrome_constants.h" 448c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei 458c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofeiusing content::BrowserThread; 468c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei 478c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofeinamespace { 488c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei 498c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofeiusing extensions::Action; 508c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei 518c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// Delay between cleaning passes (to delete old action records) through the 528c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// database. 538c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofeiconst int kCleaningDelayInHours = 12; 548c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei 558c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// We should log the arguments to these API calls. Be careful when 568c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// constructing this whitelist to not keep arguments that might compromise 578c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// privacy by logging too much data to the activity log. 588c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// 598c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// TODO(mvrable): The contents of this whitelist should be reviewed and 608c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei// expanded as needed. 618c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofeistruct ApiList { 628c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei Action::ActionType type; 638c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei const char* name; 648c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei}; 658c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei 668c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofeiconst ApiList kAlwaysLog[] = { 678c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "bookmarks.create"}, 688c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "bookmarks.update"}, 698c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "cookies.get"}, 708c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "cookies.getAll"}, 718c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "extension.connect"}, 728c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "extension.sendMessage"}, 738c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "fileSystem.chooseEntry"}, 748c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "socket.bind"}, 758c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "socket.connect"}, 768c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "socket.create"}, 778c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "socket.listen"}, 788c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "tabs.executeScript"}, 798c955ea858b0c99c856c7c10a3eee7576d13abd1Wan Xiaofei {Action::ACTION_API_CALL, "tabs.insertCSS"}, 808e4018e2de52c534405d7155c7009d0b35afb861Cedric Venet {Action::ACTION_API_CALL, "types.ChromeSetting.clear"}, 818e4018e2de52c534405d7155c7009d0b35afb861Cedric Venet {Action::ACTION_API_CALL, "types.ChromeSetting.get"}, 827309be6735666143bd9835b275dc8501617a2591Gabor Greif {Action::ACTION_API_CALL, "types.ChromeSetting.set"}, 83fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman {Action::ACTION_CONTENT_SCRIPT, ""}, 84fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman {Action::ACTION_DOM_ACCESS, "Document.createElement"}, 85c7f6b8c5d40e17bf43fd3a1549d7d89c9da735e1Gabor Greif {Action::ACTION_DOM_ACCESS, "Document.createElementNS"}, 86fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman}; 87fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman 88fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman// Columns in the main database table. See the file-level comment for a 89c23b8719ef9d6b1220e854b37d40e9e1c48a82bcGabor Greif// discussion of how data is stored and the meanings of the _x columns. 90c23b8719ef9d6b1220e854b37d40e9e1c48a82bcGabor Greifconst char* kTableContentFields[] = { 91f3841fcbd587c31aa9842b3f33bd57de40c9f443Gabor Greif "count", "extension_id_x", "time", "action_type", "api_name_x", "args_x", 92c23b8719ef9d6b1220e854b37d40e9e1c48a82bcGabor Greif "page_url_x", "page_title_x", "arg_url_x", "other_x"}; 93fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohmanconst char* kTableFieldTypes[] = { 9450bee42b54cd9aec5f49566307df2b0cf23afcf6Craig Topper "INTEGER NOT NULL DEFAULT 1", "INTEGER NOT NULL", "INTEGER", "INTEGER", 95fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", 96fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman "INTEGER"}; 97fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman 98fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman// Miscellaneous SQL commands for initializing the database; these should be 99c3aae25116e66c177579b0b79182b09340b19753Chris Lattner// idempotent. 100bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesenstatic const char kPolicyMiscSetup[] = 101bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen // The activitylog_uncompressed view performs string lookups for simpler 102bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng // access to the log data. 103bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng "DROP VIEW IF EXISTS activitylog_uncompressed;\n" 104bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng "CREATE VIEW activitylog_uncompressed AS\n" 105fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen "SELECT count,\n" 106fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " x1.value AS extension_id,\n" 107fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " time,\n" 108fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " action_type,\n" 109fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " x2.value AS api_name,\n" 110fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " x3.value AS args,\n" 111bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen " x4.value AS page_url,\n" 112bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng " x5.value AS page_title,\n" 113fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " x6.value AS arg_url,\n" 114f22fd3f7b557a967b1edc1fa9ae770006a39e97cCraig Topper " x7.value AS other,\n" 115f22fd3f7b557a967b1edc1fa9ae770006a39e97cCraig Topper " activitylog_compressed.rowid AS activity_id\n" 116bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen "FROM activitylog_compressed\n" 117001d3dc976d7cda8a3dd8c7fd4020b0b96033f4eCraig Topper " LEFT JOIN string_ids AS x1 ON (x1.id = extension_id_x)\n" 118001d3dc976d7cda8a3dd8c7fd4020b0b96033f4eCraig Topper " LEFT JOIN string_ids AS x2 ON (x2.id = api_name_x)\n" 119bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen " LEFT JOIN string_ids AS x3 ON (x3.id = args_x)\n" 120bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen " LEFT JOIN url_ids AS x4 ON (x4.id = page_url_x)\n" 121bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen " LEFT JOIN string_ids AS x5 ON (x5.id = page_title_x)\n" 122fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " LEFT JOIN url_ids AS x6 ON (x6.id = arg_url_x)\n" 123fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " LEFT JOIN string_ids AS x7 ON (x7.id = other_x);\n" 124fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen // An index on all fields except count and time: all the fields that aren't 125fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen // changed when incrementing a count. This should accelerate finding the 126bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng // rows to update (at worst several rows will need to be checked to find 127d4c6c3a7c3140ce487a805138b6f53f82ff6b783Devang Patel // the one in the right time range). 128bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen "CREATE INDEX IF NOT EXISTS activitylog_compressed_index\n" 129bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng "ON activitylog_compressed(extension_id_x, action_type, api_name_x,\n" 130bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen " args_x, page_url_x, page_title_x, arg_url_x, other_x)"; 131d4c6c3a7c3140ce487a805138b6f53f82ff6b783Devang Patel 132bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng// SQL statements to clean old, unused entries out of the string and URL id 133fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen// tables. 134bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Chengstatic const char kStringTableCleanup[] = 135bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng "DELETE FROM string_ids WHERE id NOT IN\n" 136bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng "(SELECT extension_id_x FROM activitylog_compressed\n" 137fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " WHERE extension_id_x IS NOT NULL\n" 138bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen " UNION SELECT api_name_x FROM activitylog_compressed\n" 139bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng " WHERE api_name_x IS NOT NULL\n" 14022a54c1cd711afccd4558374918d12a939e1cca5Benjamin Kramer " UNION SELECT args_x FROM activitylog_compressed\n" 141f22fd3f7b557a967b1edc1fa9ae770006a39e97cCraig Topper " WHERE args_x IS NOT NULL\n" 14222a54c1cd711afccd4558374918d12a939e1cca5Benjamin Kramer " UNION SELECT page_title_x FROM activitylog_compressed\n" 14322a54c1cd711afccd4558374918d12a939e1cca5Benjamin Kramer " WHERE page_title_x IS NOT NULL\n" 14422a54c1cd711afccd4558374918d12a939e1cca5Benjamin Kramer " UNION SELECT other_x FROM activitylog_compressed\n" 145bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen " WHERE other_x IS NOT NULL)"; 146bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Chengstatic const char kUrlTableCleanup[] = 147f22fd3f7b557a967b1edc1fa9ae770006a39e97cCraig Topper "DELETE FROM url_ids WHERE id NOT IN\n" 148bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng "(SELECT page_url_x FROM activitylog_compressed\n" 149bfcb3051899b7141a946d769fcf6e8a8453bc530Evan Cheng " WHERE page_url_x IS NOT NULL\n" 150fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " UNION SELECT arg_url_x FROM activitylog_compressed\n" 151fdb42fa5fe794cc2c89e2ed7f57a89ed24d9952aDale Johannesen " WHERE arg_url_x IS NOT NULL)"; 152bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen 153bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen} // namespace 154cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene 155cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greenenamespace extensions { 156cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene 157cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greeneconst char* CountingPolicy::kTableName = "activitylog_compressed"; 158c3aae25116e66c177579b0b79182b09340b19753Chris Lattnerconst char* CountingPolicy::kReadViewName = "activitylog_uncompressed"; 159c3aae25116e66c177579b0b79182b09340b19753Chris Lattner 160c3aae25116e66c177579b0b79182b09340b19753Chris LattnerCountingPolicy::CountingPolicy(Profile* profile) 161c3aae25116e66c177579b0b79182b09340b19753Chris Lattner : ActivityLogDatabasePolicy( 162c3aae25116e66c177579b0b79182b09340b19753Chris Lattner profile, 163c3aae25116e66c177579b0b79182b09340b19753Chris Lattner base::FilePath(chrome::kExtensionActivityLogFilename)), 164c3aae25116e66c177579b0b79182b09340b19753Chris Lattner string_table_("string_ids"), 165c3aae25116e66c177579b0b79182b09340b19753Chris Lattner url_table_("url_ids"), 166c3aae25116e66c177579b0b79182b09340b19753Chris Lattner retention_time_(base::TimeDelta::FromHours(60)) { 167c3aae25116e66c177579b0b79182b09340b19753Chris Lattner for (size_t i = 0; i < arraysize(kAlwaysLog); i++) { 168cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner api_arg_whitelist_.insert( 169cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner std::make_pair(kAlwaysLog[i].type, kAlwaysLog[i].name)); 17050d2b1ac029d63500ea9b9347561b1454fa6ed6aDan Gohman } 171ff7a562751604a9fe13efc75bd59622244b54d35Dan Gohman} 172e4b4edd72ae596ac8d1fdbc6e77f61e331f6775dChandler Carruth 1737c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan GohmanCountingPolicy::~CountingPolicy() {} 174512063dd0f91a76b9dd904dfff72a52b4d09e89fChris Lattner 1750508d047fefef36d4f943ee13c82c18cf3a943abDevang Patelbool CountingPolicy::InitDatabase(sql::Connection* db) { 176cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner if (!Util::DropObsoleteTables(db)) 177f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman return false; 178f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman 179f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman if (!string_table_.Initialize(db)) 180f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman return false; 181f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman if (!url_table_.Initialize(db)) 182cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner return false; 183213a16c637926bfc38ba373d3aba6778e181e3ecChris Lattner 184fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman // Create the unified activity log entry table. 185fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman if (!ActivityDatabase::InitializeTable(db, 186f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman kTableName, 187f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman kTableContentFields, 188f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman kTableFieldTypes, 189f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman arraysize(kTableContentFields))) 190f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman return false; 191f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman 192f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman // Create a view for easily accessing the uncompressed form of the data, and 193f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman // any necessary indexes if needed. 194691ef2ba066dda14ae4ac0ad645054fbc967785aAndrew Lenharth return db->Execute(kPolicyMiscSetup); 195213a16c637926bfc38ba373d3aba6778e181e3ecChris Lattner} 196d23b33435ae722ff5aa5ab97135a4f31041959e2Bob Wilson 197583bd47f777fe3eb8305872fa0eadab31e833dffJim Laskeyvoid CountingPolicy::ProcessAction(scoped_refptr<Action> action) { 198213a16c637926bfc38ba373d3aba6778e181e3ecChris Lattner ScheduleAndForget(this, &CountingPolicy::QueueAction, action); 199f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman} 200f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman 201f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohmanvoid CountingPolicy::QueueAction(scoped_refptr<Action> action) { 202e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman if (activity_database()->is_db_valid()) { 203e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman action = action->Clone(); 204e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman Util::StripPrivacySensitiveFields(action); 205e8be6c63915e0389f1eef6b53c64300d13b2ce99Dan Gohman Util::StripArguments(api_arg_whitelist_, action); 206bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen 207bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen // If the current action falls on a different date than the ones in the 208bfdf7f38523bd38ae0538861a2bfd8bdc46e5c33Dale Johannesen // queue, flush the queue out now to prevent any false merging (actions 209bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // from different days being merged). 210bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen base::Time new_date = action->time().LocalMidnight(); 211bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen if (new_date != queued_actions_date_) 212bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately); 213bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen queued_actions_date_ = new_date; 214bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen 215bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen ActionQueue::iterator queued_entry = queued_actions_.find(action); 216bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen if (queued_entry == queued_actions_.end()) { 217bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen queued_actions_[action] = 1; 218bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen } else { 219bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // Update the timestamp in the key to be the latest time seen. Modifying 220bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // the time is safe since that field is not involved in key comparisons 221bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // in the map. 222bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen using std::max; 223bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen queued_entry->first->set_time( 224bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen max(queued_entry->first->time(), action->time())); 225bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen queued_entry->second++; 226bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen } 227bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen activity_database()->AdviseFlush(queued_actions_.size()); 228bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen } 229bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen} 230bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen 231bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesenbool CountingPolicy::FlushDatabase(sql::Connection* db) { 232bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // Columns that must match exactly for database rows to be coalesced. 233bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen static const char* matched_columns[] = { 234bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen "extension_id_x", "action_type", "api_name_x", "args_x", "page_url_x", 235bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen "page_title_x", "arg_url_x", "other_x"}; 236bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen ActionQueue queue; 237bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen queue.swap(queued_actions_); 238bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen 239bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // Whether to clean old records out of the activity log database. Do this 240bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // much less frequently than database flushes since it is expensive, but 241bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // always check on the first database flush (since there might be a large 242bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen // amount of data to clear). 243bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen bool clean_database = (last_database_cleaning_time_.is_null() || 244bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen Now() - last_database_cleaning_time_ > 245bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen base::TimeDelta::FromHours(kCleaningDelayInHours)); 246bc7d448f242b1bbc1031fb87cd69c285ff9aaffaJakob Stoklund Olesen 247c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene if (queue.empty() && !clean_database) 248c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene return true; 249c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene 250c23b8719ef9d6b1220e854b37d40e9e1c48a82bcGabor Greif sql::Transaction transaction(db); 251c23b8719ef9d6b1220e854b37d40e9e1c48a82bcGabor Greif if (!transaction.Begin()) 252c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene return false; 253c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene 254001d3dc976d7cda8a3dd8c7fd4020b0b96033f4eCraig Topper // Adding an Action to the database is a two step process that depends on 255001d3dc976d7cda8a3dd8c7fd4020b0b96033f4eCraig Topper // whether the count on an existing row can be incremented or a new row needs 256819309efec6f11ba752bd7cbfe186495745f020bDaniel Dunbar // to be inserted. 257cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner // 1. Run the query in locate_str to search for a row which matches and can 2580508d047fefef36d4f943ee13c82c18cf3a943abDevang Patel // have the count incremented. 259cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner // 2a. If found, increment the count using update_str and the rowid found in 260cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner // step 1, or 2617c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan Gohman // 2b. If not found, insert a new row using insert_str. 2627c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan Gohman std::string locate_str = 2637c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan Gohman "SELECT rowid FROM " + std::string(kTableName) + 264e4b4edd72ae596ac8d1fdbc6e77f61e331f6775dChandler Carruth " WHERE time >= ? AND time < ?"; 2657c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan Gohman std::string insert_str = 2667c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan Gohman "INSERT INTO " + std::string(kTableName) + "(count, time"; 267f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman std::string update_str = 268f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman "UPDATE " + std::string(kTableName) + 2697c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan Gohman " SET count = count + ?, time = max(?, time)" 270f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman " WHERE rowid = ?"; 2717c3234c6be0dc0bdf4b5d6f848cd728a77f349d7Dan Gohman 27250d2b1ac029d63500ea9b9347561b1454fa6ed6aDan Gohman for (size_t i = 0; i < arraysize(matched_columns); i++) { 273ba54bca472a15d0657e1b88776f7069042b60b4eBill Wendling locate_str = base::StringPrintf( 274ba54bca472a15d0657e1b88776f7069042b60b4eBill Wendling "%s AND %s IS ?", locate_str.c_str(), matched_columns[i]); 275ba54bca472a15d0657e1b88776f7069042b60b4eBill Wendling insert_str = 276ff7a562751604a9fe13efc75bd59622244b54d35Dan Gohman base::StringPrintf("%s, %s", insert_str.c_str(), matched_columns[i]); 277e4b4edd72ae596ac8d1fdbc6e77f61e331f6775dChandler Carruth } 278d1474d09cbe5fdeec8ba0d6c6b52f316f3422532Owen Anderson insert_str += ") VALUES (?, ?"; 279cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner for (size_t i = 0; i < arraysize(matched_columns); i++) { 280ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey insert_str += ", ?"; 2811080b9ee534579c67f7c99364cc6fa11edbcd919Chris Lattner } 282462dc7f4960e5074ddf4769ec8b2ef1ba7a4d2c8Dan Gohman locate_str += " ORDER BY time DESC LIMIT 1"; 2830b12aef49087b57d276ed760a83525d1e2602144Dan Gohman insert_str += ")"; 284fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel 285ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey for (ActionQueue::iterator i = queue.begin(); i != queue.end(); ++i) { 286ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey const Action& action = *i->first.get(); 287ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey int count = i->second; 2881080b9ee534579c67f7c99364cc6fa11edbcd919Chris Lattner 289ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey base::Time day_start = action.time().LocalMidnight(); 290ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey base::Time next_day = Util::AddDays(day_start, 1); 291ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey 292fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel // The contents in values must match up with fields in matched_columns. A 293ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey // value of -1 is used to encode a null database value. 294ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey int64 id; 295ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey std::vector<int64> matched_values; 296fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel 297ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey if (!string_table_.StringToInt(db, action.extension_id(), &id)) 298ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey return false; 299ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey matched_values.push_back(id); 300fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel 301ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey matched_values.push_back(static_cast<int>(action.action_type())); 302ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey 303ec20402c90b605afeedbcf0e3aabe6f8054f23ddJim Laskey if (!string_table_.StringToInt(db, action.api_name(), &id)) 3041080b9ee534579c67f7c99364cc6fa11edbcd919Chris Lattner return false; 305c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene matched_values.push_back(id); 306c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene 307c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene if (action.args()) { 308c5e7e8d87d4a3b10edd5ac93ba1f3cdb4d1b449aDavid Greene std::string args = Util::Serialize(action.args()); 309fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman // TODO(mvrable): For now, truncate long argument lists. This is a 310b80e2be8894db9f843f32ebaffb9b7fd6b57d206Chris Lattner // workaround for excessively-long values coming from DOM logging. When 311b80e2be8894db9f843f32ebaffb9b7fd6b57d206Chris Lattner // the V8ValueConverter is fixed to return more reasonable values, we can 312fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman // drop the truncation. 313b80e2be8894db9f843f32ebaffb9b7fd6b57d206Chris Lattner if (args.length() > 10000) { 314b80e2be8894db9f843f32ebaffb9b7fd6b57d206Chris Lattner args = "[\"<too_large>\"]"; 315fed90b6d097d50881afb45e4d79f430db66dd741Dan Gohman } 3160e5f1306b059b62d7725f324e087efbc8e7a782dDan Gohman if (!string_table_.StringToInt(db, args, &id)) 3170e5f1306b059b62d7725f324e087efbc8e7a782dDan Gohman return false; 318fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel matched_values.push_back(id); 319c3aae25116e66c177579b0b79182b09340b19753Chris Lattner } else { 320cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner matched_values.push_back(-1); 321475871a144eb604ddaf37503397ba0941442e5fbDan Gohman } 322cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner 323c3aae25116e66c177579b0b79182b09340b19753Chris Lattner std::string page_url_string = action.SerializePageUrl(); 324c3aae25116e66c177579b0b79182b09340b19753Chris Lattner if (!page_url_string.empty()) { 325f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman if (!url_table_.StringToInt(db, page_url_string, &id)) 326f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman return false; 327f350b277f32d7d47f86c0e54f4aec4d470500618Dan Gohman matched_values.push_back(id); 328cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner } else { 329c3aae25116e66c177579b0b79182b09340b19753Chris Lattner matched_values.push_back(-1); 330cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner } 331475871a144eb604ddaf37503397ba0941442e5fbDan Gohman 332825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson // TODO(mvrable): Create a title_table_? 333acaf09dbe4a6781163857db1321bbd5795e7d410Dan Gohman if (!action.page_title().empty()) { 334cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene if (!string_table_.StringToInt(db, action.page_title(), &id)) 335cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene return false; 336cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene matched_values.push_back(id); 337cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene } else { 338cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene matched_values.push_back(-1); 339cf495bc2e505e52ad018da55bed11c7b8bc97db5David Greene } 340acaf09dbe4a6781163857db1321bbd5795e7d410Dan Gohman 341cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner std::string arg_url_string = action.SerializeArgUrl(); 3421d4d41411190dd9e62764e56713753d4155764ddNate Begeman if (!arg_url_string.empty()) { 34325cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands if (!url_table_.StringToInt(db, arg_url_string, &id)) 34425cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands return false; 34525cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands matched_values.push_back(id); 34698a366d547772010e94609e4584489b3e5ce0043Bill Wendling } else { 34798a366d547772010e94609e4584489b3e5ce0043Bill Wendling matched_values.push_back(-1); 34825cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands } 34901d029b82cb08367d81aa10cdc94d05360466649Chris Lattner 35025cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands if (action.other()) { 35125cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands if (!string_table_.StringToInt(db, Util::Serialize(action.other()), &id)) 35201d029b82cb08367d81aa10cdc94d05360466649Chris Lattner return false; 35301d029b82cb08367d81aa10cdc94d05360466649Chris Lattner matched_values.push_back(id); 35401d029b82cb08367d81aa10cdc94d05360466649Chris Lattner } else { 35525cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands matched_values.push_back(-1); 35625cf2275ff7de3de3bc0e508abaf457413d74725Duncan Sands } 357c3aae25116e66c177579b0b79182b09340b19753Chris Lattner 358c3aae25116e66c177579b0b79182b09340b19753Chris Lattner // Search for a matching row for this action whose count can be 359c3aae25116e66c177579b0b79182b09340b19753Chris Lattner // incremented. 360cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner sql::Statement locate_statement(db->GetCachedStatement( 361c3aae25116e66c177579b0b79182b09340b19753Chris Lattner sql::StatementID(SQL_FROM_HERE), locate_str.c_str())); 362c3aae25116e66c177579b0b79182b09340b19753Chris Lattner locate_statement.BindInt64(0, day_start.ToInternalValue()); 363975716af1b9a09274df6c2d92683449015bd8564Dan Gohman locate_statement.BindInt64(1, next_day.ToInternalValue()); 364c3aae25116e66c177579b0b79182b09340b19753Chris Lattner for (size_t j = 0; j < matched_values.size(); j++) { 3655c22c8074404797f1313b1334757254fb5c6487aEli Friedman // A call to BindNull when matched_values contains -1 is likely not 3665c22c8074404797f1313b1334757254fb5c6487aEli Friedman // necessary as parameters default to null before they are explicitly 3675c22c8074404797f1313b1334757254fb5c6487aEli Friedman // bound. But to be completely clear, and in case a cached statement 3685c22c8074404797f1313b1334757254fb5c6487aEli Friedman // ever comes with some values already bound, we bind all parameters 3695c22c8074404797f1313b1334757254fb5c6487aEli Friedman // (even null ones) explicitly. 3705c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (matched_values[j] == -1) 3715c22c8074404797f1313b1334757254fb5c6487aEli Friedman locate_statement.BindNull(j + 2); 3725c22c8074404797f1313b1334757254fb5c6487aEli Friedman else 3735c22c8074404797f1313b1334757254fb5c6487aEli Friedman locate_statement.BindInt64(j + 2, matched_values[j]); 3745c22c8074404797f1313b1334757254fb5c6487aEli Friedman } 3755c22c8074404797f1313b1334757254fb5c6487aEli Friedman 3765c22c8074404797f1313b1334757254fb5c6487aEli Friedman if (locate_statement.Step()) { 3775c22c8074404797f1313b1334757254fb5c6487aEli Friedman // A matching row was found. Update the count and time. 378d1fc96499b7619356c7542200d32da898b79f7c1Chris Lattner int64 rowid = locate_statement.ColumnInt64(0); 379190a418bf6b49a4ef1c1980229a2f0d516e8a2cdChris Lattner sql::Statement update_statement(db->GetCachedStatement( 380190a418bf6b49a4ef1c1980229a2f0d516e8a2cdChris Lattner sql::StatementID(SQL_FROM_HERE), update_str.c_str())); 381130a6471b90f66e99b1f9f42877fdf611c330ac6Evan Cheng update_statement.BindInt(0, count); 382130a6471b90f66e99b1f9f42877fdf611c330ac6Evan Cheng update_statement.BindInt64(1, action.time().ToInternalValue()); 383130a6471b90f66e99b1f9f42877fdf611c330ac6Evan Cheng update_statement.BindInt64(2, rowid); 384130a6471b90f66e99b1f9f42877fdf611c330ac6Evan Cheng if (!update_statement.Run()) 385130a6471b90f66e99b1f9f42877fdf611c330ac6Evan Cheng return false; 38670046e920fa37989a041af663ada2b2b646e258fChris Lattner } else if (locate_statement.Succeeded()) { 38770046e920fa37989a041af663ada2b2b646e258fChris Lattner // No matching row was found, so we need to insert one. 388e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson sql::Statement insert_statement(db->GetCachedStatement( 389e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson sql::StatementID(SQL_FROM_HERE), insert_str.c_str())); 390e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson insert_statement.BindInt(0, count); 391e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson insert_statement.BindInt64(1, action.time().ToInternalValue()); 392e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson for (size_t j = 0; j < matched_values.size(); j++) { 393fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel if (matched_values[j] == -1) 3941b1a49714ef26225a42199cf2930529f31868322Chris Lattner insert_statement.BindNull(j + 2); 39570046e920fa37989a041af663ada2b2b646e258fChris Lattner else 39670046e920fa37989a041af663ada2b2b646e258fChris Lattner insert_statement.BindInt64(j + 2, matched_values[j]); 397e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson } 398e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson if (!insert_statement.Run()) 399e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson return false; 400475871a144eb604ddaf37503397ba0941442e5fbDan Gohman } else { 401e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson // Database error. 402cbea3045ce0bdd061c494a831d0ce2d5834211ccChris Lattner return false; 403cbea3045ce0bdd061c494a831d0ce2d5834211ccChris Lattner } 404e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson } 4056394b099e836f56a937cdcc7332c9487b504ca68Dan Gohman 4066394b099e836f56a937cdcc7332c9487b504ca68Dan Gohman if (clean_database) { 407e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson base::Time cutoff = (Now() - retention_time()).LocalMidnight(); 4084fbd796a1251a27e6590765a0a34876f436a0af9Dan Gohman if (!CleanOlderThan(db, cutoff)) 4094fbd796a1251a27e6590765a0a34876f436a0af9Dan Gohman return false; 4100a406ae0d940d92c51ee145b48ff7a483a366849Dale Johannesen last_database_cleaning_time_ = Now(); 4110a406ae0d940d92c51ee145b48ff7a483a366849Dale Johannesen } 412e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson 413e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson if (!transaction.Commit()) 414e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson return false; 415e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson 416c9f8f416800784ca6453222b307bc44ad24739b0Chris Lattner return true; 417c9f8f416800784ca6453222b307bc44ad24739b0Chris Lattner} 418e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson 419f04afdbb48568ef09f11fd10ac03426101f2dbf8Dale Johannesenscoped_ptr<Action::ActionVector> CountingPolicy::DoReadFilteredData( 420f04afdbb48568ef09f11fd10ac03426101f2dbf8Dale Johannesen const std::string& extension_id, 421e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson const Action::ActionType type, 4224fbd796a1251a27e6590765a0a34876f436a0af9Dan Gohman const std::string& api_name, 4234fbd796a1251a27e6590765a0a34876f436a0af9Dan Gohman const std::string& page_url, 424ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick const std::string& arg_url, 4252a4ed82ce2ba4a02f2a02eb4ed8ce5186f3f93daChris Lattner const int days_ago) { 4262a4ed82ce2ba4a02f2a02eb4ed8ce5186f3f93daChris Lattner // Ensure data is flushed to the database first so that we query over all 427ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick // data. 4282a4ed82ce2ba4a02f2a02eb4ed8ce5186f3f93daChris Lattner activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately); 4292a4ed82ce2ba4a02f2a02eb4ed8ce5186f3f93daChris Lattner scoped_ptr<Action::ActionVector> actions(new Action::ActionVector()); 4300d881dabc1a4e1aefad6dd38de166d8358285638Devang Patel 431cbea3045ce0bdd061c494a831d0ce2d5834211ccChris Lattner sql::Connection* db = GetDatabaseConnection(); 432e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson if (!db) 433e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson return actions.Pass(); 434c9f8f416800784ca6453222b307bc44ad24739b0Chris Lattner 435c9f8f416800784ca6453222b307bc44ad24739b0Chris Lattner // Build up the query based on which parameters were specified. 436e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson std::string where_str = ""; 437f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner std::string where_next = ""; 438e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson if (!extension_id.empty()) { 439f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner where_str += "extension_id=?"; 440c9f8f416800784ca6453222b307bc44ad24739b0Chris Lattner where_next = " AND "; 44146510a73e977273ec67747eb34cbdb43f815e451Dan Gohman } 442f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner if (!api_name.empty()) { 443f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner where_str += where_next + "api_name=?"; 44446510a73e977273ec67747eb34cbdb43f815e451Dan Gohman where_next = " AND "; 445f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner } 446f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner if (type != Action::ACTION_ANY) { 447f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner where_str += where_next + "action_type=?"; 448c9f8f416800784ca6453222b307bc44ad24739b0Chris Lattner where_next = " AND "; 449e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson } 450f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner if (!page_url.empty()) { 451f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner where_str += where_next + "page_url LIKE ?"; 452475871a144eb604ddaf37503397ba0941442e5fbDan Gohman where_next = " AND "; 453e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson } 454f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner if (!arg_url.empty()) { 455f5a5546f435a261c93078defb41ed18d9cafd7d3Chris Lattner where_str += where_next + "arg_url LIKE ?"; 456d6594ae54cfde4db4d30272192645c0a45fb9902Evan Cheng where_next = " AND "; 45774500bdba3eae36a1a8a17d8bad0b971b9c212ecJakob Stoklund Olesen } 45874500bdba3eae36a1a8a17d8bad0b971b9c212ecJakob Stoklund Olesen if (days_ago >= 0) 4598ad9b43e690e8773cf836b30e8da26bc71e18844Dale Johannesen where_str += where_next + "time BETWEEN ? AND ?"; 4608ad9b43e690e8773cf836b30e8da26bc71e18844Dale Johannesen 461475871a144eb604ddaf37503397ba0941442e5fbDan Gohman std::string query_str = base::StringPrintf( 462ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick "SELECT extension_id,time, action_type, api_name, args, page_url," 463e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson "page_title, arg_url, other, count, activity_id FROM %s %s %s ORDER BY " 464ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick "count DESC, time DESC LIMIT 300", 465e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson kReadViewName, 4661af2231da64a14d638406d133c7912bfc1c8a9ceChris Lattner where_str.empty() ? "" : "WHERE", 467e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson where_str.c_str()); 468e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson sql::Statement query(db->GetUniqueStatement(query_str.c_str())); 4699cf37e8b48732fccd4c301ed51aafed7074bd84eJakob Stoklund Olesen int i = -1; 470ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (!extension_id.empty()) 47146510a73e977273ec67747eb34cbdb43f815e451Dan Gohman query.BindString(++i, extension_id); 4726c7ccaa3fd1d6e96d0bf922554b09d2b17c3b0e3Michael Liao if (!api_name.empty()) 4736c7ccaa3fd1d6e96d0bf922554b09d2b17c3b0e3Michael Liao query.BindString(++i, api_name); 4746c7ccaa3fd1d6e96d0bf922554b09d2b17c3b0e3Michael Liao if (type != Action::ACTION_ANY) 4756c7ccaa3fd1d6e96d0bf922554b09d2b17c3b0e3Michael Liao query.BindInt(++i, static_cast<int>(type)); 4766c7ccaa3fd1d6e96d0bf922554b09d2b17c3b0e3Michael Liao if (!page_url.empty()) 4776c7ccaa3fd1d6e96d0bf922554b09d2b17c3b0e3Michael Liao query.BindString(++i, page_url + "%"); 4786c7ccaa3fd1d6e96d0bf922554b09d2b17c3b0e3Michael Liao if (!arg_url.empty()) 479c3aae25116e66c177579b0b79182b09340b19753Chris Lattner query.BindString(++i, arg_url + "%"); 480ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (days_ago >= 0) { 481825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson int64 early_bound; 4820f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen int64 late_bound; 4830f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen Util::ComputeDatabaseTimeBounds(Now(), days_ago, &early_bound, &late_bound); 484cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner query.BindInt64(++i, early_bound); 485e3f1026683c38f6605ccaf698b7082f1b0a0f8c8Chris Lattner query.BindInt64(++i, late_bound); 48629d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner } 48729d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner 488ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick // Execute the query and get results. 48929d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner while (query.is_valid() && query.Step()) { 490f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner scoped_refptr<Action> action = 49129d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner new Action(query.ColumnString(0), 49229d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner base::Time::FromInternalValue(query.ColumnInt64(1)), 4930f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen static_cast<Action::ActionType>(query.ColumnInt(2)), 49466a48bbc3565b40ea0e6f2d58cf5e3a8e64802efEvan Cheng query.ColumnString(3), query.ColumnInt64(10)); 495475871a144eb604ddaf37503397ba0941442e5fbDan Gohman 496ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (query.ColumnType(4) != sql::COLUMN_TYPE_NULL) { 49729d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner scoped_ptr<base::Value> parsed_value( 498f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner base::JSONReader::Read(query.ColumnString(4))); 49929d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner if (parsed_value && parsed_value->IsType(base::Value::TYPE_LIST)) { 50029d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner action->set_args(make_scoped_ptr( 5010f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen static_cast<base::ListValue*>(parsed_value.release()))); 502fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel } 503ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick } 504825b72b0571821bf2d378749f69d6c4cfb52d2f9Owen Anderson 5050f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen action->ParsePageUrl(query.ColumnString(5)); 506fc1665793e62eb4f26d24b8a19eecf59cd872e2aDan Gohman action->set_page_title(query.ColumnString(6)); 5070f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen action->ParseArgUrl(query.ColumnString(7)); 508fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel 509e3f1026683c38f6605ccaf698b7082f1b0a0f8c8Chris Lattner if (query.ColumnType(8) != sql::COLUMN_TYPE_NULL) { 51029d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner scoped_ptr<base::Value> parsed_value( 51129d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner base::JSONReader::Read(query.ColumnString(8))); 512ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (parsed_value && parsed_value->IsType(base::Value::TYPE_DICTIONARY)) { 51329d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner action->set_other(make_scoped_ptr( 514f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner static_cast<base::DictionaryValue*>(parsed_value.release()))); 51529d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner } 51629d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner } 5170f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen action->set_count(query.ColumnInt(9)); 51818c2f13e0f9d0e5d6227cf6d1881e9ee3d1b6109Chris Lattner actions->push_back(action); 519475871a144eb604ddaf37503397ba0941442e5fbDan Gohman } 520cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner 52177cdf30742284a173fe818417eb482224cdee8d4Mon P Wang return actions.Pass(); 52277cdf30742284a173fe818417eb482224cdee8d4Mon P Wang} 523ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 5240f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesenvoid CountingPolicy::DoRemoveActions(const std::vector<int64>& action_ids) { 5250f502f6f44f2756f5cb7b17d8f1d8eae000d51b4Dale Johannesen if (action_ids.empty()) 5266154f6c9292179fab6346ae8336f2ad790b52028Owen Anderson return; 5279008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman 5289008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman sql::Connection* db = GetDatabaseConnection(); 5295a5ca1519e04310f585197c20e7ae584b7f2d11fNate Begeman if (!db) { 5309008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman LOG(ERROR) << "Unable to connect to database"; 531ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return; 5329008ca6b6b4f638cfafccb593cbc5b1d3f5ab877Nate Begeman } 53377cdf30742284a173fe818417eb482224cdee8d4Mon P Wang 534a3c42f3d4e5d14c8f4fb9bb123e7759c425d041bNadav Rotem // Flush data first so the activity removal affects queued-up data as well. 535a3c42f3d4e5d14c8f4fb9bb123e7759c425d041bNadav Rotem activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately); 536ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 537a3c42f3d4e5d14c8f4fb9bb123e7759c425d041bNadav Rotem sql::Transaction transaction(db); 5383a66a68b0cc81193abfc074b1d360a4950151d38Duncan Sands if (!transaction.Begin()) 5393a66a68b0cc81193abfc074b1d360a4950151d38Duncan Sands return; 540ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 5413a66a68b0cc81193abfc074b1d360a4950151d38Duncan Sands std::string statement_str = 5423a66a68b0cc81193abfc074b1d360a4950151d38Duncan Sands base::StringPrintf("DELETE FROM %s WHERE rowid = ?", kTableName); 5433a66a68b0cc81193abfc074b1d360a4950151d38Duncan Sands sql::Statement statement(db->GetCachedStatement( 544ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick sql::StatementID(SQL_FROM_HERE), statement_str.c_str())); 5453a66a68b0cc81193abfc074b1d360a4950151d38Duncan Sands for (size_t i = 0; i < action_ids.size(); i++) { 5461ccae666f596d5aeca5c9942995763600b622062Chris Lattner statement.Reset(true); 5471ccae666f596d5aeca5c9942995763600b622062Chris Lattner statement.BindInt64(0, action_ids[i]); 548ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (!statement.Run()) { 549fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel LOG(ERROR) << "Removing activities from database failed: " 5504c2454623841f05c6c665659b34c214950d12d7eBob Wilson << statement.GetSQLStatement(); 551ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick break; 5524c2454623841f05c6c665659b34c214950d12d7eBob Wilson } 5536a5b6d7633c96c72ca7d5f8ba0c855e4690ada04Chris Lattner } 55429d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner 555ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick CleanStringTables(db); 5566e0b2a0cb0d398f175a5294bf0ad5488c714e8c2Andrew Trick 557f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner if (!transaction.Commit()) { 558475871a144eb604ddaf37503397ba0941442e5fbDan Gohman LOG(ERROR) << "Removing activities from database failed"; 5596e0b2a0cb0d398f175a5294bf0ad5488c714e8c2Andrew Trick } 5606a5b6d7633c96c72ca7d5f8ba0c855e4690ada04Chris Lattner} 5611ccae666f596d5aeca5c9942995763600b622062Chris Lattner 5620f8d9c04d9feef86cee35cf5fecfb348a6b3de50Bill Wendlingvoid CountingPolicy::DoRemoveURLs(const std::vector<GURL>& restrict_urls) { 56329d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner sql::Connection* db = GetDatabaseConnection(); 564ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (!db) { 565475871a144eb604ddaf37503397ba0941442e5fbDan Gohman LOG(ERROR) << "Unable to connect to database"; 5666e0b2a0cb0d398f175a5294bf0ad5488c714e8c2Andrew Trick return; 567f1b4eafbfec976f939ec0ea3e8acf91cef5363e3Chris Lattner } 568475871a144eb604ddaf37503397ba0941442e5fbDan Gohman 5690f8d9c04d9feef86cee35cf5fecfb348a6b3de50Bill Wendling // Flush data first so the URL clearing affects queued-up data as well. 5700f8d9c04d9feef86cee35cf5fecfb348a6b3de50Bill Wendling activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately); 5710f8d9c04d9feef86cee35cf5fecfb348a6b3de50Bill Wendling 57229d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner // If no restrictions then then all URLs need to be removed. 5736e0b2a0cb0d398f175a5294bf0ad5488c714e8c2Andrew Trick if (restrict_urls.empty()) { 57429d8f0cae425f1bba583565227eaebf58f26ce73Chris Lattner std::string sql_str = base::StringPrintf( 5750f8d9c04d9feef86cee35cf5fecfb348a6b3de50Bill Wendling "UPDATE %s SET page_url_x=NULL,page_title_x=NULL,arg_url_x=NULL", 5760f8d9c04d9feef86cee35cf5fecfb348a6b3de50Bill Wendling kTableName); 577ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 578e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson sql::Statement statement; 579ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick statement.Assign(db->GetCachedStatement( 580e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen sql::StatementID(SQL_FROM_HERE), sql_str.c_str())); 581e8d7230f480654cdb8ff1c3d0a38e1e9ab0bd55fDale Johannesen 582b300d2aa3ef08b5074449e2c05804717f488f4e4Dale Johannesen if (!statement.Run()) { 583ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick LOG(ERROR) << "Removing all URLs from database failed: " 584e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson << statement.GetSQLStatement(); 585ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return; 586b300d2aa3ef08b5074449e2c05804717f488f4e4Dale Johannesen } 587b300d2aa3ef08b5074449e2c05804717f488f4e4Dale Johannesen } 588c3aae25116e66c177579b0b79182b09340b19753Chris Lattner 589cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner // If URLs are specified then restrict to only those URLs. 590ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick for (size_t i = 0; i < restrict_urls.size(); ++i) { 591ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick int64 url_id; 592ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (!restrict_urls[i].is_valid() || 593ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick !url_table_.StringToInt(db, restrict_urls[i].spec(), &url_id)) { 5947ade28cd62f5188951387e1056a46001388a21f9Bill Wendling continue; 595ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick } 5967ade28cd62f5188951387e1056a46001388a21f9Bill Wendling 597ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick // Remove any that match the page_url. 5987ade28cd62f5188951387e1056a46001388a21f9Bill Wendling std::string sql_str = base::StringPrintf( 5997ade28cd62f5188951387e1056a46001388a21f9Bill Wendling "UPDATE %s SET page_url_x=NULL,page_title_x=NULL WHERE page_url_x IS ?", 600ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick kTableName); 6017ade28cd62f5188951387e1056a46001388a21f9Bill Wendling 602ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick sql::Statement statement; 6037ade28cd62f5188951387e1056a46001388a21f9Bill Wendling statement.Assign(db->GetCachedStatement( 604ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick sql::StatementID(SQL_FROM_HERE), sql_str.c_str())); 6053853f74aba301ef08b699bac2fa8e53230714a58Benjamin Kramer statement.BindInt64(0, url_id); 6067ade28cd62f5188951387e1056a46001388a21f9Bill Wendling 607ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (!statement.Run()) { 6087ade28cd62f5188951387e1056a46001388a21f9Bill Wendling LOG(ERROR) << "Removing page URL from database failed: " 609ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick << statement.GetSQLStatement(); 6107ade28cd62f5188951387e1056a46001388a21f9Bill Wendling return; 611ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick } 612ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 613ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick // Remove any that match the arg_url. 6147ade28cd62f5188951387e1056a46001388a21f9Bill Wendling sql_str = base::StringPrintf( 615ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick "UPDATE %s SET arg_url_x=NULL WHERE arg_url_x IS ?", kTableName); 6167ade28cd62f5188951387e1056a46001388a21f9Bill Wendling 617ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick statement.Assign(db->GetCachedStatement( 6187ade28cd62f5188951387e1056a46001388a21f9Bill Wendling sql::StatementID(SQL_FROM_HERE), sql_str.c_str())); 619ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick statement.BindInt64(0, url_id); 6207ade28cd62f5188951387e1056a46001388a21f9Bill Wendling 6217ade28cd62f5188951387e1056a46001388a21f9Bill Wendling if (!statement.Run()) { 622475871a144eb604ddaf37503397ba0941442e5fbDan Gohman LOG(ERROR) << "Removing arg URL from database failed: " 62398ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman << statement.GetSQLStatement(); 62498ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman return; 62598ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman } 62698ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman } 62798ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman 62898ca4f2a325f72374a477f9deba7d09e8999c29bDan Gohman // Clean up unused strings from the strings and urls table to really delete 629ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick // the urls and page titles. Should be called even if an error occured when 63020adc9dc4650313f017b27d9818eb2176238113dMon P Wang // removing a URL as there may some things to clean up. 631e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattner CleanStringTables(db); 632e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattner} 6335c0d6ed325417baa5d119af9c2b6790231d8565fRafael Espindola 634ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trickvoid CountingPolicy::DoRemoveExtensionData(const std::string& extension_id) { 63520adc9dc4650313f017b27d9818eb2176238113dMon P Wang if (extension_id.empty()) 636e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattner return; 637e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattner 6385c0d6ed325417baa5d119af9c2b6790231d8565fRafael Espindola sql::Connection* db = GetDatabaseConnection(); 639ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (!db) { 64020adc9dc4650313f017b27d9818eb2176238113dMon P Wang LOG(ERROR) << "Unable to connect to database"; 641e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattner return; 6425c0d6ed325417baa5d119af9c2b6790231d8565fRafael Espindola } 6437cf7e3f33f25544d08492d47cc8a1cbba25dc8d7Chris Lattner 644475871a144eb604ddaf37503397ba0941442e5fbDan Gohman // Make sure any queued in memory are sent to the database before cleaning. 6457cf7e3f33f25544d08492d47cc8a1cbba25dc8d7Chris Lattner activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately); 646ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 647d69c3141ed6d237ad19fdfbfcef8901491b24c2eBill Wendling std::string sql_str = base::StringPrintf( 64828b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands "DELETE FROM %s WHERE extension_id_x=?", kTableName); 64928b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands sql::Statement statement( 65028b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE), sql_str.c_str())); 65128b77e968d2b01fc9da724762bd8ddcd80650e32Duncan Sands int64 id; 65285cc4d840256841ac231858c3c916e91fe469269Matt Arsenault if (string_table_.StringToInt(db, extension_id, &id)) { 65385cc4d840256841ac231858c3c916e91fe469269Matt Arsenault statement.BindInt64(0, id); 654d69c3141ed6d237ad19fdfbfcef8901491b24c2eBill Wendling } else { 6557cf7e3f33f25544d08492d47cc8a1cbba25dc8d7Chris Lattner // If the string isn't listed, that means we never recorded anything about 6569373a81e53ce5f9f2c06c4209b8b886605aece08Nate Begeman // the extension so there's no deletion to do. 65785cc4d840256841ac231858c3c916e91fe469269Matt Arsenault statement.Clear(); 65885cc4d840256841ac231858c3c916e91fe469269Matt Arsenault return; 65985cc4d840256841ac231858c3c916e91fe469269Matt Arsenault } 66085cc4d840256841ac231858c3c916e91fe469269Matt Arsenault if (!statement.Run()) { 66185cc4d840256841ac231858c3c916e91fe469269Matt Arsenault LOG(ERROR) << "Removing URLs for extension " 66285cc4d840256841ac231858c3c916e91fe469269Matt Arsenault << extension_id << "from database failed: " 66385cc4d840256841ac231858c3c916e91fe469269Matt Arsenault << statement.GetSQLStatement(); 66485cc4d840256841ac231858c3c916e91fe469269Matt Arsenault } 66585cc4d840256841ac231858c3c916e91fe469269Matt Arsenault CleanStringTables(db); 66685cc4d840256841ac231858c3c916e91fe469269Matt Arsenault} 66785cc4d840256841ac231858c3c916e91fe469269Matt Arsenault 66885cc4d840256841ac231858c3c916e91fe469269Matt Arsenaultvoid CountingPolicy::DoDeleteDatabase() { 6699373a81e53ce5f9f2c06c4209b8b886605aece08Nate Begeman sql::Connection* db = GetDatabaseConnection(); 670475871a144eb604ddaf37503397ba0941442e5fbDan Gohman if (!db) { 6719373a81e53ce5f9f2c06c4209b8b886605aece08Nate Begeman LOG(ERROR) << "Unable to connect to database"; 672ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return; 673d69c3141ed6d237ad19fdfbfcef8901491b24c2eBill Wendling } 674d69c3141ed6d237ad19fdfbfcef8901491b24c2eBill Wendling 675d69c3141ed6d237ad19fdfbfcef8901491b24c2eBill Wendling queued_actions_.clear(); 6769373a81e53ce5f9f2c06c4209b8b886605aece08Nate Begeman 677fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel // Not wrapped in a transaction because a late failure shouldn't undo a 678acc398c195a697795bff3245943d104eb19192b9Nate Begeman // previous deletion. 679acc398c195a697795bff3245943d104eb19192b9Nate Begeman std::string sql_str = base::StringPrintf("DELETE FROM %s", kTableName); 680ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick sql::Statement statement(db->GetCachedStatement( 681cbeeae23c31d32b833c9c7c3e8984e4cbcf22f45Rafael Espindola sql::StatementID(SQL_FROM_HERE), 6827cbd525ba85ebe440d15fa359ec940e404d14906Nate Begeman sql_str.c_str())); 683fdc40a0a696c658d550d894ea03772e5f8af2c94Scott Michel if (!statement.Run()) { 684c4d1021ead43cfa7da08a8f7ddc9a059a8ba14c5Mon P Wang LOG(ERROR) << "Deleting the database failed: " 685ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick << statement.GetSQLStatement(); 68660bddc8bcd787be645d2f3b64117fee884270e6aChris Lattner return; 68755ba816883842e793cdeb32fcb805c4e011b527fEli Friedman } 68855ba816883842e793cdeb32fcb805c4e011b527fEli Friedman statement.Clear(); 68955ba816883842e793cdeb32fcb805c4e011b527fEli Friedman string_table_.ClearCache(); 690ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick statement.Assign(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE), 691c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman "DELETE FROM string_ids")); 69255ba816883842e793cdeb32fcb805c4e011b527fEli Friedman if (!statement.Run()) { 69355ba816883842e793cdeb32fcb805c4e011b527fEli Friedman LOG(ERROR) << "Deleting the database failed: " 69455ba816883842e793cdeb32fcb805c4e011b527fEli Friedman << statement.GetSQLStatement(); 695ab0b949e0e9de452f3b052b11634ab761e008b23Andrew Lenharth return; 696327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman } 697327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman statement.Clear(); 698ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick url_table_.ClearCache(); 699e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesen statement.Assign(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE), 700327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman "DELETE FROM url_ids")); 701327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman if (!statement.Run()) { 702ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick LOG(ERROR) << "Deleting the database failed: " 703327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman << statement.GetSQLStatement(); 704327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman return; 705327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman } 706327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman statement.Clear(); 707327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman statement.Assign(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE), 708327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman "VACUUM")); 709ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (!statement.Run()) { 710327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedman LOG(ERROR) << "Vacuuming the database failed: " 71155ba816883842e793cdeb32fcb805c4e011b527fEli Friedman << statement.GetSQLStatement(); 71255ba816883842e793cdeb32fcb805c4e011b527fEli Friedman } 71355ba816883842e793cdeb32fcb805c4e011b527fEli Friedman} 714ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 715327236cd6c211e54fc6288b0ac2b413901cc0611Eli Friedmanvoid CountingPolicy::ReadFilteredData( 71655ba816883842e793cdeb32fcb805c4e011b527fEli Friedman const std::string& extension_id, 71755ba816883842e793cdeb32fcb805c4e011b527fEli Friedman const Action::ActionType type, 718ab0b949e0e9de452f3b052b11634ab761e008b23Andrew Lenharth const std::string& api_name, 719268c743a3ba44ada364938bc5ff9b1be219df54fAmara Emerson const std::string& page_url, 720268c743a3ba44ada364938bc5ff9b1be219df54fAmara Emerson const std::string& arg_url, 721268c743a3ba44ada364938bc5ff9b1be219df54fAmara Emerson const int days_ago, 722268c743a3ba44ada364938bc5ff9b1be219df54fAmara Emerson const base::Callback 723268c743a3ba44ada364938bc5ff9b1be219df54fAmara Emerson <void(scoped_ptr<Action::ActionVector>)>& callback) { 724268c743a3ba44ada364938bc5ff9b1be219df54fAmara Emerson BrowserThread::PostTaskAndReplyWithResult( 725268c743a3ba44ada364938bc5ff9b1be219df54fAmara Emerson BrowserThread::DB, 726c4d1021ead43cfa7da08a8f7ddc9a059a8ba14c5Mon P Wang FROM_HERE, 727c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman base::Bind(&CountingPolicy::DoReadFilteredData, 728c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman base::Unretained(this), 729c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman extension_id, 730ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick type, 731e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson api_name, 732e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesen page_url, 733e9ba5dd236f48708a00bd3bb0519148f943cc897Chris Lattner arg_url, 734e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesen days_ago), 735e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesen callback); 736c4d1021ead43cfa7da08a8f7ddc9a059a8ba14c5Mon P Wang} 737ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 738e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesenvoid CountingPolicy::RemoveActions(const std::vector<int64>& action_ids) { 739e9ba5dd236f48708a00bd3bb0519148f943cc897Chris Lattner ScheduleAndForget(this, &CountingPolicy::DoRemoveActions, action_ids); 740e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesen} 741e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesen 742c4d1021ead43cfa7da08a8f7ddc9a059a8ba14c5Mon P Wangvoid CountingPolicy::RemoveURLs(const std::vector<GURL>& restrict_urls) { 743ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick ScheduleAndForget(this, &CountingPolicy::DoRemoveURLs, restrict_urls); 744c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman} 745c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman 746c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohmanvoid CountingPolicy::RemoveExtensionData(const std::string& extension_id) { 7474bdcb61af33399d4e01fdf3c47ca1f1f5356e370Duncan Sands ScheduleAndForget(this, &CountingPolicy::DoRemoveExtensionData, extension_id); 748ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick} 749f9516208e57364ab1e7d8748af1f59a2ea5fb572Duncan Sands 750c3aae25116e66c177579b0b79182b09340b19753Chris Lattnervoid CountingPolicy::DeleteDatabase() { 751c3aae25116e66c177579b0b79182b09340b19753Chris Lattner ScheduleAndForget(this, &CountingPolicy::DoDeleteDatabase); 752cacf462915344c2af25eef1af1f3ee2c7280ff56Chris Lattner} 753ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 754e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattnervoid CountingPolicy::OnDatabaseFailure() { 755d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper queued_actions_.clear(); 75695d594cac3737ae1594a391276942a443cac426bRafael Espindola} 757ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 758e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattnervoid CountingPolicy::OnDatabaseClose() { 759e72f2027e9116c55a5b39ac72732df8d6c45d37cChris Lattner delete this; 760f96e4bd2a3b11928af75fb7472288930d16fec0bDan Gohman} 761f96e4bd2a3b11928af75fb7472288930d16fec0bDan Gohman 762ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick// Cleans old records from the activity log database. 763bcc8017c738e92d9c1af221b11c4916cb524184eEvan Chengbool CountingPolicy::CleanOlderThan(sql::Connection* db, 764bcc8017c738e92d9c1af221b11c4916cb524184eEvan Cheng const base::Time& cutoff) { 765ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick std::string clean_statement = 766bcc8017c738e92d9c1af221b11c4916cb524184eEvan Cheng "DELETE FROM " + std::string(kTableName) + " WHERE time < ?"; 7675c5cb2a1717f8e30b1849d7ec1cf269bc5d66877Chris Lattner sql::Statement cleaner(db->GetCachedStatement(sql::StatementID(SQL_FROM_HERE), 768d752e0f7e64585839cb3a458ef52456eaebbea3cPete Cooper clean_statement.c_str())); 76995d594cac3737ae1594a391276942a443cac426bRafael Espindola cleaner.BindInt64(0, cutoff.ToInternalValue()); 77095d594cac3737ae1594a391276942a443cac426bRafael Espindola if (!cleaner.Run()) 7715c5cb2a1717f8e30b1849d7ec1cf269bc5d66877Chris Lattner return false; 772ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick return CleanStringTables(db); 7735c5cb2a1717f8e30b1849d7ec1cf269bc5d66877Chris Lattner} 774c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman 7752d86ea21dd76647cb054fd5d27df9e49efc672b6Andrew Lenharth// Cleans unused interned strings from the database. This should be run after 776ad071e1cd1a4b880019f1b2e827ee81867815f82Evan Cheng// deleting rows from the main log table to clean out stale values. 777ad071e1cd1a4b880019f1b2e827ee81867815f82Evan Chengbool CountingPolicy::CleanStringTables(sql::Connection* db) { 778ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick sql::Statement cleaner1(db->GetCachedStatement( 7795c5cb2a1717f8e30b1849d7ec1cf269bc5d66877Chris Lattner sql::StatementID(SQL_FROM_HERE), kStringTableCleanup)); 780f96e4bd2a3b11928af75fb7472288930d16fec0bDan Gohman if (!cleaner1.Run()) 781f96e4bd2a3b11928af75fb7472288930d16fec0bDan Gohman return false; 782ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (db->GetLastChangeCount() > 0) 783c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman string_table_.ClearCache(); 784ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 7855c5cb2a1717f8e30b1849d7ec1cf269bc5d66877Chris Lattner sql::Statement cleaner2(db->GetCachedStatement( 7865c5cb2a1717f8e30b1849d7ec1cf269bc5d66877Chris Lattner sql::StatementID(SQL_FROM_HERE), kUrlTableCleanup)); 787f96e4bd2a3b11928af75fb7472288930d16fec0bDan Gohman if (!cleaner2.Run()) 788f96e4bd2a3b11928af75fb7472288930d16fec0bDan Gohman return false; 789ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick if (db->GetLastChangeCount() > 0) 790c76909abfec876c6b751d693ebd3df07df686aa0Dan Gohman url_table_.ClearCache(); 791ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick 792e8c17335c53f0f37262ee342f46b0d00ac0c1493Dale Johannesen return true; 793ad071e1cd1a4b880019f1b2e827ee81867815f82Evan Cheng} 794101a90f3d3c3ad9159676ba744b59b3a9d0b4af5Dan Gohman 795475871a144eb604ddaf37503397ba0941442e5fbDan Gohmanvoid CountingPolicy::Close() { 79669de1932b350d7cdfc0ed1f4198d6f78c7822a02Dan Gohman // The policy object should have never been created if there's no DB thread. 797decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner DCHECK(BrowserThread::IsMessageLoopValid(BrowserThread::DB)); 798decc2671516e6c52ee2f29f7746f8d02753845eaChris Lattner ScheduleAndForget(activity_database(), &ActivityDatabase::Close); 7996154f6c9292179fab6346ae8336f2ad790b52028Owen Anderson} 80092abc62399881ba9c525be80362c134ad836e2d9Duncan Sands 80192abc62399881ba9c525be80362c134ad836e2d9Duncan Sands} // namespace extensions 8026154f6c9292179fab6346ae8336f2ad790b52028Owen Anderson