1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/metrics/user_metrics.h"
12#include "base/prefs/pref_service.h"
13#include "base/strings/string16.h"
14#include "base/strings/string_util.h"
15#include "base/strings/utf_string_conversions.h"
16#include "base/time/time.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier.h"
18#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
19#include "chrome/browser/autocomplete/chrome_autocomplete_provider_client.h"
20#include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
21#include "chrome/browser/autocomplete/history_url_provider.h"
22#include "chrome/browser/history/top_sites.h"
23#include "chrome/browser/profiles/profile.h"
24#include "chrome/common/pref_names.h"
25#include "components/history/core/browser/history_types.h"
26#include "components/metrics/proto/omnibox_input_type.pb.h"
27#include "components/omnibox/autocomplete_input.h"
28#include "components/omnibox/autocomplete_match.h"
29#include "components/omnibox/autocomplete_provider_listener.h"
30#include "components/omnibox/omnibox_field_trial.h"
31#include "components/omnibox/search_provider.h"
32#include "components/pref_registry/pref_registry_syncable.h"
33#include "components/search_engines/template_url_service.h"
34#include "components/variations/variations_http_header_provider.h"
35#include "net/base/escape.h"
36#include "net/base/load_flags.h"
37#include "net/base/net_util.h"
38#include "net/http/http_request_headers.h"
39#include "net/url_request/url_fetcher.h"
40#include "net/url_request/url_request_status.h"
41#include "url/gurl.h"
42
43namespace {
44
45// TODO(hfung): The histogram code was copied and modified from
46// search_provider.cc.  Refactor and consolidate the code.
47// We keep track in a histogram how many suggest requests we send, how
48// many suggest requests we invalidate (e.g., due to a user typing
49// another character), and how many replies we receive.
50// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
51//     (excluding the end-of-list enum value)
52// We do not want values of existing enums to change or else it screws
53// up the statistics.
54enum ZeroSuggestRequestsHistogramValue {
55  ZERO_SUGGEST_REQUEST_SENT = 1,
56  ZERO_SUGGEST_REQUEST_INVALIDATED,
57  ZERO_SUGGEST_REPLY_RECEIVED,
58  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
59};
60
61void LogOmniboxZeroSuggestRequest(
62    ZeroSuggestRequestsHistogramValue request_value) {
63  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
64                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
65}
66
67// The maximum relevance of the top match from this provider.
68const int kDefaultVerbatimZeroSuggestRelevance = 1300;
69
70// Relevance value to use if it was not set explicitly by the server.
71const int kDefaultZeroSuggestRelevance = 100;
72
73}  // namespace
74
75// static
76ZeroSuggestProvider* ZeroSuggestProvider::Create(
77    AutocompleteProviderListener* listener,
78    TemplateURLService* template_url_service,
79    Profile* profile) {
80  return new ZeroSuggestProvider(listener, template_url_service, profile);
81}
82
83// static
84void ZeroSuggestProvider::RegisterProfilePrefs(
85    user_prefs::PrefRegistrySyncable* registry) {
86  registry->RegisterStringPref(
87      prefs::kZeroSuggestCachedResults,
88      std::string(),
89      user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
90}
91
92void ZeroSuggestProvider::Start(const AutocompleteInput& input,
93                                bool minimal_changes) {
94  matches_.clear();
95  if (input.type() == metrics::OmniboxInputType::INVALID)
96    return;
97
98  Stop(true);
99  field_trial_triggered_ = false;
100  field_trial_triggered_in_session_ = false;
101  results_from_cache_ = false;
102  permanent_text_ = input.text();
103  current_query_ = input.current_url().spec();
104  current_page_classification_ = input.current_page_classification();
105  current_url_match_ = MatchForCurrentURL();
106
107  const TemplateURL* default_provider =
108     template_url_service_->GetDefaultSearchProvider();
109  if (default_provider == NULL)
110    return;
111
112  base::string16 prefix;
113  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
114  GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
115      search_term_args, template_url_service_->search_terms_data()));
116  if (!suggest_url.is_valid())
117    return;
118
119  // No need to send the current page URL in personalized suggest field trial.
120  if (CanSendURL(input.current_url(), suggest_url, default_provider,
121                 current_page_classification_,
122                 template_url_service_->search_terms_data(), client_.get()) &&
123      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
124    // Update suggest_url to include the current_page_url.
125    search_term_args.current_page_url = current_query_;
126    suggest_url = GURL(default_provider->suggestions_url_ref().
127                       ReplaceSearchTerms(
128                           search_term_args,
129                           template_url_service_->search_terms_data()));
130  } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
131                                                  input.current_url())) {
132    return;
133  }
134
135  done_ = false;
136  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
137  // These may be useful on the NTP or more relevant to the user than server
138  // suggestions, if based on local browsing history.
139  MaybeUseCachedSuggestions();
140  Run(suggest_url);
141}
142
143void ZeroSuggestProvider::Stop(bool clear_cached_results) {
144  if (fetcher_)
145    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
146  fetcher_.reset();
147  done_ = true;
148
149  if (clear_cached_results) {
150    // We do not call Clear() on |results_| to retain |verbatim_relevance|
151    // value in the |results_| object. |verbatim_relevance| is used at the
152    // beginning of the next StartZeroSuggest() call to determine the current
153    // url match relevance.
154    results_.suggest_results.clear();
155    results_.navigation_results.clear();
156    current_query_.clear();
157  }
158}
159
160void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
161  if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
162    // Remove the deleted match from the cache, so it is not shown to the user
163    // again. Since we cannot remove just one result, blow away the cache.
164    profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
165                                    std::string());
166  }
167  BaseSearchProvider::DeleteMatch(match);
168}
169
170void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
171  BaseSearchProvider::AddProviderInfo(provider_info);
172  if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
173    provider_info->back().set_times_returned_results_in_session(1);
174}
175
176void ZeroSuggestProvider::ResetSession() {
177  // The user has started editing in the omnibox, so leave
178  // |field_trial_triggered_in_session_| unchanged and set
179  // |field_trial_triggered_| to false since zero suggest is inactive now.
180  field_trial_triggered_ = false;
181}
182
183ZeroSuggestProvider::ZeroSuggestProvider(
184  AutocompleteProviderListener* listener,
185  TemplateURLService* template_url_service,
186  Profile* profile)
187    : BaseSearchProvider(template_url_service,
188                         scoped_ptr<AutocompleteProviderClient>(
189                             new ChromeAutocompleteProviderClient(profile)),
190                         AutocompleteProvider::TYPE_ZERO_SUGGEST),
191      listener_(listener),
192      profile_(profile),
193      results_from_cache_(false),
194      weak_ptr_factory_(this) {
195}
196
197ZeroSuggestProvider::~ZeroSuggestProvider() {
198}
199
200const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
201  // Zero suggest provider should not receive keyword results.
202  DCHECK(!is_keyword);
203  return template_url_service_->GetDefaultSearchProvider();
204}
205
206const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
207  return AutocompleteInput(
208      base::string16(), base::string16::npos, base::string16(),
209      GURL(current_query_), current_page_classification_, true, false, false,
210      true, ChromeAutocompleteSchemeClassifier(profile_));
211}
212
213bool ZeroSuggestProvider::ShouldAppendExtraParams(
214      const SearchSuggestionParser::SuggestResult& result) const {
215  // We always use the default provider for search, so append the params.
216  return true;
217}
218
219void ZeroSuggestProvider::RecordDeletionResult(bool success) {
220  if (success) {
221    base::RecordAction(
222        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
223  } else {
224    base::RecordAction(
225        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
226  }
227}
228
229void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
230  DCHECK(!done_);
231  DCHECK_EQ(fetcher_.get(), source);
232
233  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
234
235  bool results_updated = false;
236  if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
237    std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
238    scoped_ptr<base::Value> data(
239        SearchSuggestionParser::DeserializeJsonData(json_data));
240    if (data) {
241      if (StoreSuggestionResponse(json_data, *data))
242        return;
243      results_updated = ParseSuggestResults(
244          *data, kDefaultZeroSuggestRelevance, false, &results_);
245    }
246  }
247  fetcher_.reset();
248  done_ = true;
249  ConvertResultsToAutocompleteMatches();
250  listener_->OnProviderUpdate(results_updated);
251}
252
253bool ZeroSuggestProvider::StoreSuggestionResponse(
254    const std::string& json_data,
255    const base::Value& parsed_data) {
256  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
257      json_data.empty())
258    return false;
259  profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
260
261  // If we received an empty result list, we should update the display, as it
262  // may be showing cached results that should not be shown.
263  const base::ListValue* root_list = NULL;
264  const base::ListValue* results_list = NULL;
265  if (parsed_data.GetAsList(&root_list) &&
266      root_list->GetList(1, &results_list) &&
267      results_list->empty())
268    return false;
269
270  // We are finished with the request and want to bail early.
271  if (results_from_cache_)
272    done_ = true;
273
274  return results_from_cache_;
275}
276
277void ZeroSuggestProvider::AddSuggestResultsToMap(
278    const SearchSuggestionParser::SuggestResults& results,
279    MatchMap* map) {
280  for (size_t i = 0; i < results.size(); ++i)
281    AddMatchToMap(results[i], std::string(), i, false, false, map);
282}
283
284AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
285    const SearchSuggestionParser::NavigationResult& navigation) {
286  AutocompleteMatch match(this, navigation.relevance(), false,
287                          navigation.type());
288  match.destination_url = navigation.url();
289
290  // Zero suggest results should always omit protocols and never appear bold.
291  const std::string languages(
292      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
293  match.contents = net::FormatUrl(navigation.url(), languages,
294      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
295  match.fill_into_edit +=
296      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
297          match.contents, ChromeAutocompleteSchemeClassifier(profile_));
298
299  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
300      match.contents.length(), ACMatchClassification::URL,
301      &match.contents_class);
302
303  match.description =
304      AutocompleteMatch::SanitizeString(navigation.description());
305  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
306      match.description.length(), ACMatchClassification::NONE,
307      &match.description_class);
308  return match;
309}
310
311void ZeroSuggestProvider::Run(const GURL& suggest_url) {
312  const int kFetcherID = 1;
313  fetcher_.reset(
314      net::URLFetcher::Create(kFetcherID,
315          suggest_url,
316          net::URLFetcher::GET, this));
317  fetcher_->SetRequestContext(profile_->GetRequestContext());
318  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
319  // Add Chrome experiment state to the request headers.
320  net::HttpRequestHeaders headers;
321  variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
322      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
323  fetcher_->SetExtraRequestHeaders(headers.ToString());
324  fetcher_->Start();
325
326  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
327    most_visited_urls_.clear();
328    history::TopSites* ts = profile_->GetTopSites();
329    if (ts) {
330      ts->GetMostVisitedURLs(
331          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
332                     weak_ptr_factory_.GetWeakPtr()), false);
333    }
334  }
335  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
336}
337
338void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
339    const history::MostVisitedURLList& urls) {
340  most_visited_urls_ = urls;
341}
342
343void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
344  matches_.clear();
345
346  const TemplateURL* default_provider =
347      template_url_service_->GetDefaultSearchProvider();
348  // Fail if we can't set the clickthrough URL for query suggestions.
349  if (default_provider == NULL || !default_provider->SupportsReplacement(
350          template_url_service_->search_terms_data()))
351    return;
352
353  MatchMap map;
354  AddSuggestResultsToMap(results_.suggest_results, &map);
355
356  const int num_query_results = map.size();
357  const int num_nav_results = results_.navigation_results.size();
358  const int num_results = num_query_results + num_nav_results;
359  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
360  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
361  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
362
363  // Show Most Visited results after ZeroSuggest response is received.
364  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
365    if (!current_url_match_.destination_url.is_valid())
366      return;
367    matches_.push_back(current_url_match_);
368    int relevance = 600;
369    if (num_results > 0) {
370      UMA_HISTOGRAM_COUNTS(
371          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
372          most_visited_urls_.size());
373    }
374    const base::string16 current_query_string16(
375        base::ASCIIToUTF16(current_query_));
376    const std::string languages(
377        profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
378    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
379      const history::MostVisitedURL& url = most_visited_urls_[i];
380      SearchSuggestionParser::NavigationResult nav(
381          ChromeAutocompleteSchemeClassifier(profile_), url.url,
382          AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
383          relevance, true, current_query_string16, languages);
384      matches_.push_back(NavigationToMatch(nav));
385      --relevance;
386    }
387    return;
388  }
389
390  if (num_results == 0)
391    return;
392
393  // TODO(jered): Rip this out once the first match is decoupled from the
394  // current typing in the omnibox.
395  matches_.push_back(current_url_match_);
396
397  for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
398    matches_.push_back(it->second);
399
400  const SearchSuggestionParser::NavigationResults& nav_results(
401      results_.navigation_results);
402  for (SearchSuggestionParser::NavigationResults::const_iterator it(
403           nav_results.begin()); it != nav_results.end(); ++it)
404    matches_.push_back(NavigationToMatch(*it));
405}
406
407AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
408  AutocompleteMatch match;
409  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
410      permanent_text_, false, true, current_page_classification_, &match, NULL);
411  match.is_history_what_you_typed_match = false;
412  match.allowed_to_be_default_match = true;
413
414  // The placeholder suggestion for the current URL has high relevance so
415  // that it is in the first suggestion slot and inline autocompleted. It
416  // gets dropped as soon as the user types something.
417  match.relevance = GetVerbatimRelevance();
418
419  return match;
420}
421
422int ZeroSuggestProvider::GetVerbatimRelevance() const {
423  return results_.verbatim_relevance >= 0 ?
424      results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
425}
426
427bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
428    const GURL& suggest_url,
429    const GURL& current_page_url) const {
430  if (!ZeroSuggestEnabled(suggest_url,
431                          template_url_service_->GetDefaultSearchProvider(),
432                          current_page_classification_,
433                          template_url_service_->search_terms_data(),
434                          client_.get()))
435    return false;
436
437  // If we cannot send URLs, then only the MostVisited and Personalized
438  // variations can be shown.
439  if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
440      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
441    return false;
442
443  // Only show zero suggest for HTTP[S] pages.
444  // TODO(mariakhomenko): We may be able to expand this set to include pages
445  // with other schemes (e.g. chrome://). That may require improvements to
446  // the formatting of the verbatim result returned by MatchForCurrentURL().
447  if (!current_page_url.is_valid() ||
448      ((current_page_url.scheme() != url::kHttpScheme) &&
449      (current_page_url.scheme() != url::kHttpsScheme)))
450    return false;
451
452  return true;
453}
454
455void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
456  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
457    return;
458
459  std::string json_data = profile_->GetPrefs()->GetString(
460      prefs::kZeroSuggestCachedResults);
461  if (!json_data.empty()) {
462    scoped_ptr<base::Value> data(
463        SearchSuggestionParser::DeserializeJsonData(json_data));
464    if (data && ParseSuggestResults(
465            *data, kDefaultZeroSuggestRelevance, false, &results_)) {
466      ConvertResultsToAutocompleteMatches();
467      results_from_cache_ = !matches_.empty();
468    }
469  }
470}
471