1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "components/suggestions/suggestions_service.h"
6
7#include <sstream>
8#include <string>
9
10#include "base/memory/scoped_ptr.h"
11#include "base/message_loop/message_loop_proxy.h"
12#include "base/metrics/histogram.h"
13#include "base/metrics/sparse_histogram.h"
14#include "base/strings/string_number_conversions.h"
15#include "base/strings/string_util.h"
16#include "base/time/time.h"
17#include "components/pref_registry/pref_registry_syncable.h"
18#include "components/suggestions/blacklist_store.h"
19#include "components/suggestions/suggestions_store.h"
20#include "components/variations/variations_associated_data.h"
21#include "components/variations/variations_http_header_provider.h"
22#include "net/base/escape.h"
23#include "net/base/load_flags.h"
24#include "net/base/net_errors.h"
25#include "net/base/url_util.h"
26#include "net/http/http_response_headers.h"
27#include "net/http/http_status_code.h"
28#include "net/http/http_util.h"
29#include "net/url_request/url_fetcher.h"
30#include "net/url_request/url_request_status.h"
31#include "url/gurl.h"
32
33using base::CancelableClosure;
34
35namespace suggestions {
36
37namespace {
38
39// Used to UMA log the state of the last response from the server.
40enum SuggestionsResponseState {
41  RESPONSE_EMPTY,
42  RESPONSE_INVALID,
43  RESPONSE_VALID,
44  RESPONSE_STATE_SIZE
45};
46
47// Will log the supplied response |state|.
48void LogResponseState(SuggestionsResponseState state) {
49  UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
50                            RESPONSE_STATE_SIZE);
51}
52
53// Obtains the experiment parameter under the supplied |key|, or empty string
54// if the parameter does not exist.
55std::string GetExperimentParam(const std::string& key) {
56  return variations::GetVariationParamValue(kSuggestionsFieldTrialName, key);
57}
58
59GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
60                              const GURL& candidate_url) {
61  return GURL(blacklist_url_prefix +
62              net::EscapeQueryParamValue(candidate_url.spec(), true));
63}
64
65// Runs each callback in |requestors| on |suggestions|, then deallocates
66// |requestors|.
67void DispatchRequestsAndClear(
68    const SuggestionsProfile& suggestions,
69    std::vector<SuggestionsService::ResponseCallback>* requestors) {
70  std::vector<SuggestionsService::ResponseCallback>::iterator it;
71  for (it = requestors->begin(); it != requestors->end(); ++it) {
72    if (!it->is_null()) it->Run(suggestions);
73  }
74  std::vector<SuggestionsService::ResponseCallback>().swap(*requestors);
75}
76
77const int kDefaultRequestTimeoutMs = 200;
78
79// Default delay used when scheduling a blacklist request.
80const int kBlacklistDefaultDelaySec = 1;
81
82// Multiplier on the delay used when scheduling a blacklist request, in case the
83// last observed request was unsuccessful.
84const int kBlacklistBackoffMultiplier = 2;
85
86// Maximum valid delay for scheduling a request. Candidate delays larger than
87// this are rejected. This means the maximum backoff is at least 300 / 2, i.e.
88// 2.5 minutes.
89const int kBlacklistMaxDelaySec = 300;  // 5 minutes
90
91}  // namespace
92
93const char kSuggestionsFieldTrialName[] = "ChromeSuggestions";
94const char kSuggestionsFieldTrialURLParam[] = "url";
95const char kSuggestionsFieldTrialCommonParamsParam[] = "common_params";
96const char kSuggestionsFieldTrialBlacklistPathParam[] = "blacklist_path";
97const char kSuggestionsFieldTrialBlacklistUrlParam[] = "blacklist_url_param";
98const char kSuggestionsFieldTrialStateParam[] = "state";
99const char kSuggestionsFieldTrialControlParam[] = "control";
100const char kSuggestionsFieldTrialStateEnabled[] = "enabled";
101const char kSuggestionsFieldTrialTimeoutMs[] = "timeout_ms";
102
103// The default expiry timeout is 72 hours.
104const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
105
106namespace {
107
108std::string GetBlacklistUrlPrefix() {
109  std::stringstream blacklist_url_prefix_stream;
110  blacklist_url_prefix_stream
111      << GetExperimentParam(kSuggestionsFieldTrialURLParam)
112      << GetExperimentParam(kSuggestionsFieldTrialBlacklistPathParam) << "?"
113      << GetExperimentParam(kSuggestionsFieldTrialCommonParamsParam) << "&"
114      << GetExperimentParam(kSuggestionsFieldTrialBlacklistUrlParam) << "=";
115  return blacklist_url_prefix_stream.str();
116}
117
118}  // namespace
119
120SuggestionsService::SuggestionsService(
121    net::URLRequestContextGetter* url_request_context,
122    scoped_ptr<SuggestionsStore> suggestions_store,
123    scoped_ptr<ImageManager> thumbnail_manager,
124    scoped_ptr<BlacklistStore> blacklist_store)
125    : suggestions_store_(suggestions_store.Pass()),
126      blacklist_store_(blacklist_store.Pass()),
127      thumbnail_manager_(thumbnail_manager.Pass()),
128      url_request_context_(url_request_context),
129      blacklist_delay_sec_(kBlacklistDefaultDelaySec),
130      request_timeout_ms_(kDefaultRequestTimeoutMs),
131      weak_ptr_factory_(this) {
132  // Obtain various parameters from Variations.
133  suggestions_url_ =
134      GURL(GetExperimentParam(kSuggestionsFieldTrialURLParam) + "?" +
135           GetExperimentParam(kSuggestionsFieldTrialCommonParamsParam));
136  blacklist_url_prefix_ = GetBlacklistUrlPrefix();
137  std::string timeout = GetExperimentParam(kSuggestionsFieldTrialTimeoutMs);
138  int temp_timeout;
139  if (!timeout.empty() && base::StringToInt(timeout, &temp_timeout)) {
140    request_timeout_ms_ = temp_timeout;
141  }
142}
143
144SuggestionsService::~SuggestionsService() {}
145
146// static
147bool SuggestionsService::IsEnabled() {
148  return GetExperimentParam(kSuggestionsFieldTrialStateParam) ==
149         kSuggestionsFieldTrialStateEnabled;
150}
151
152// static
153bool SuggestionsService::IsControlGroup() {
154  return GetExperimentParam(kSuggestionsFieldTrialControlParam) ==
155         kSuggestionsFieldTrialStateEnabled;
156}
157
158void SuggestionsService::FetchSuggestionsData(
159    SyncState sync_state,
160    SuggestionsService::ResponseCallback callback) {
161  DCHECK(thread_checker_.CalledOnValidThread());
162  if (sync_state == NOT_INITIALIZED_ENABLED) {
163    // Sync is not initialized yet, but enabled. Serve previously cached
164    // suggestions if available.
165    waiting_requestors_.push_back(callback);
166    ServeFromCache();
167    return;
168  } else if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
169    // Cancel any ongoing request (and the timeout closure). We must no longer
170    // interact with the server.
171    pending_request_.reset(NULL);
172    pending_timeout_closure_.reset(NULL);
173    suggestions_store_->ClearSuggestions();
174    callback.Run(SuggestionsProfile());
175    DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
176    return;
177  }
178
179  FetchSuggestionsDataNoTimeout(callback);
180
181  // Post a task to serve the cached suggestions if the request hasn't completed
182  // after some time. Cancels the previous such task, if one existed.
183  pending_timeout_closure_.reset(new CancelableClosure(base::Bind(
184      &SuggestionsService::OnRequestTimeout, weak_ptr_factory_.GetWeakPtr())));
185  base::MessageLoopProxy::current()->PostDelayedTask(
186      FROM_HERE, pending_timeout_closure_->callback(),
187      base::TimeDelta::FromMilliseconds(request_timeout_ms_));
188}
189
190void SuggestionsService::GetPageThumbnail(
191    const GURL& url,
192    base::Callback<void(const GURL&, const SkBitmap*)> callback) {
193  thumbnail_manager_->GetImageForURL(url, callback);
194}
195
196void SuggestionsService::BlacklistURL(
197    const GURL& candidate_url,
198    const SuggestionsService::ResponseCallback& callback) {
199  DCHECK(thread_checker_.CalledOnValidThread());
200  waiting_requestors_.push_back(callback);
201
202  // Blacklist locally, for immediate effect.
203  if (!blacklist_store_->BlacklistUrl(candidate_url)) {
204    DVLOG(1) << "Failed blacklisting attempt.";
205    return;
206  }
207
208  // If there's an ongoing request, let it complete.
209  if (pending_request_.get()) return;
210  IssueRequest(BuildBlacklistRequestURL(blacklist_url_prefix_, candidate_url));
211}
212
213// static
214bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
215                                           GURL* url) {
216  bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
217                                              GetBlacklistUrlPrefix(), true);
218  if (!is_blacklist_request) return false;
219
220  // Extract the blacklisted URL from the blacklist request.
221  std::string blacklisted;
222  if (!net::GetValueForKeyInQuery(
223          request.GetOriginalURL(),
224          GetExperimentParam(kSuggestionsFieldTrialBlacklistUrlParam),
225          &blacklisted))
226    return false;
227
228  GURL blacklisted_url(blacklisted);
229  blacklisted_url.Swap(url);
230  return true;
231}
232
233// static
234void SuggestionsService::RegisterProfilePrefs(
235    user_prefs::PrefRegistrySyncable* registry) {
236  SuggestionsStore::RegisterProfilePrefs(registry);
237  BlacklistStore::RegisterProfilePrefs(registry);
238}
239
240void SuggestionsService::SetDefaultExpiryTimestamp(
241    SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
242  for (int i = 0; i < suggestions->suggestions_size(); ++i) {
243    ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
244    // Do not set expiry if the server has already provided a more specific
245    // expiry time for this suggestion.
246    if (!suggestion->has_expiry_ts()) {
247      suggestion->set_expiry_ts(default_timestamp_usec);
248    }
249  }
250}
251
252void SuggestionsService::FetchSuggestionsDataNoTimeout(
253    SuggestionsService::ResponseCallback callback) {
254  DCHECK(thread_checker_.CalledOnValidThread());
255  if (pending_request_.get()) {
256    // Request already exists, so just add requestor to queue.
257    waiting_requestors_.push_back(callback);
258    return;
259  }
260
261  // Form new request.
262  DCHECK(waiting_requestors_.empty());
263  waiting_requestors_.push_back(callback);
264  IssueRequest(suggestions_url_);
265}
266
267void SuggestionsService::IssueRequest(const GURL& url) {
268  pending_request_.reset(CreateSuggestionsRequest(url));
269  pending_request_->Start();
270  last_request_started_time_ = base::TimeTicks::Now();
271}
272
273net::URLFetcher* SuggestionsService::CreateSuggestionsRequest(const GURL& url) {
274  net::URLFetcher* request =
275      net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
276  request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
277  request->SetRequestContext(url_request_context_);
278  // Add Chrome experiment state to the request headers.
279  net::HttpRequestHeaders headers;
280  variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
281      request->GetOriginalURL(), false, false, &headers);
282  request->SetExtraRequestHeaders(headers.ToString());
283  return request;
284}
285
286void SuggestionsService::OnRequestTimeout() {
287  DCHECK(thread_checker_.CalledOnValidThread());
288  ServeFromCache();
289}
290
291void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
292  DCHECK(thread_checker_.CalledOnValidThread());
293  DCHECK_EQ(pending_request_.get(), source);
294  // We no longer need the timeout closure. Delete it whether or not it has run.
295  // If it hasn't, this cancels it.
296  pending_timeout_closure_.reset();
297
298  // The fetcher will be deleted when the request is handled.
299  scoped_ptr<const net::URLFetcher> request(pending_request_.release());
300  const net::URLRequestStatus& request_status = request->GetStatus();
301  if (request_status.status() != net::URLRequestStatus::SUCCESS) {
302    UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
303                                -request_status.error());
304    DVLOG(1) << "Suggestions server request failed with error: "
305             << request_status.error() << ": "
306             << net::ErrorToString(request_status.error());
307    // Dispatch the cached profile on error.
308    ServeFromCache();
309    ScheduleBlacklistUpload(false);
310    return;
311  }
312
313  // Log the response code.
314  const int response_code = request->GetResponseCode();
315  UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
316  if (response_code != net::HTTP_OK) {
317    // Aggressively clear the store.
318    suggestions_store_->ClearSuggestions();
319    DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
320    ScheduleBlacklistUpload(false);
321    return;
322  }
323
324  const base::TimeDelta latency =
325      base::TimeTicks::Now() - last_request_started_time_;
326  UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
327
328  // Handle a successful blacklisting.
329  GURL blacklisted_url;
330  if (GetBlacklistedUrl(*source, &blacklisted_url)) {
331    blacklist_store_->RemoveUrl(blacklisted_url);
332  }
333
334  std::string suggestions_data;
335  bool success = request->GetResponseAsString(&suggestions_data);
336  DCHECK(success);
337
338  // Compute suggestions, and dispatch them to requestors. On error still
339  // dispatch empty suggestions.
340  SuggestionsProfile suggestions;
341  if (suggestions_data.empty()) {
342    LogResponseState(RESPONSE_EMPTY);
343    suggestions_store_->ClearSuggestions();
344  } else if (suggestions.ParseFromString(suggestions_data)) {
345    LogResponseState(RESPONSE_VALID);
346    thumbnail_manager_->Initialize(suggestions);
347
348    int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
349        .ToInternalValue();
350    SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
351    suggestions_store_->StoreSuggestions(suggestions);
352  } else {
353    LogResponseState(RESPONSE_INVALID);
354    suggestions_store_->LoadSuggestions(&suggestions);
355    thumbnail_manager_->Initialize(suggestions);
356  }
357
358  FilterAndServe(&suggestions);
359  ScheduleBlacklistUpload(true);
360}
361
362void SuggestionsService::Shutdown() {
363  // Cancel pending request and timeout closure, then serve existing requestors
364  // from cache.
365  pending_request_.reset(NULL);
366  pending_timeout_closure_.reset(NULL);
367  ServeFromCache();
368}
369
370void SuggestionsService::ServeFromCache() {
371  SuggestionsProfile suggestions;
372  suggestions_store_->LoadSuggestions(&suggestions);
373  thumbnail_manager_->Initialize(suggestions);
374  FilterAndServe(&suggestions);
375}
376
377void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
378  blacklist_store_->FilterSuggestions(suggestions);
379  DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
380}
381
382void SuggestionsService::ScheduleBlacklistUpload(bool last_request_successful) {
383  DCHECK(thread_checker_.CalledOnValidThread());
384
385  UpdateBlacklistDelay(last_request_successful);
386
387  // Schedule a blacklist upload task.
388  GURL blacklist_url;
389  if (blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url)) {
390    base::Closure blacklist_cb =
391        base::Bind(&SuggestionsService::UploadOneFromBlacklist,
392                   weak_ptr_factory_.GetWeakPtr());
393    base::MessageLoopProxy::current()->PostDelayedTask(
394        FROM_HERE, blacklist_cb,
395        base::TimeDelta::FromSeconds(blacklist_delay_sec_));
396  }
397}
398
399void SuggestionsService::UploadOneFromBlacklist() {
400  DCHECK(thread_checker_.CalledOnValidThread());
401
402  // If there's an ongoing request, let it complete.
403  if (pending_request_.get()) return;
404
405  GURL blacklist_url;
406  if (!blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url))
407    return;  // Local blacklist is empty.
408
409  // Send blacklisting request.
410  IssueRequest(BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
411}
412
413void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
414  DCHECK(thread_checker_.CalledOnValidThread());
415
416  if (last_request_successful) {
417    blacklist_delay_sec_ = kBlacklistDefaultDelaySec;
418  } else {
419    int candidate_delay = blacklist_delay_sec_ * kBlacklistBackoffMultiplier;
420    if (candidate_delay < kBlacklistMaxDelaySec)
421      blacklist_delay_sec_ = candidate_delay;
422  }
423}
424
425}  // namespace suggestions
426