zero_suggest_provider.cc revision 23730a6e56a168d1879203e4b3819bb36e3d8f1f 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_classifier.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
18#include "chrome/browser/autocomplete/autocomplete_input.h"
19#include "chrome/browser/autocomplete/autocomplete_match.h"
20#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
21#include "chrome/browser/autocomplete/history_url_provider.h"
22#include "chrome/browser/autocomplete/search_provider.h"
23#include "chrome/browser/autocomplete/url_prefix.h"
24#include "chrome/browser/history/history_types.h"
25#include "chrome/browser/history/top_sites.h"
26#include "chrome/browser/metrics/variations/variations_http_header_provider.h"
27#include "chrome/browser/omnibox/omnibox_field_trial.h"
28#include "chrome/browser/profiles/profile.h"
29#include "chrome/browser/search/search.h"
30#include "chrome/browser/search_engines/template_url_service.h"
31#include "chrome/browser/search_engines/template_url_service_factory.h"
32#include "chrome/common/net/url_fixer_upper.h"
33#include "chrome/common/pref_names.h"
34#include "chrome/common/url_constants.h"
35#include "content/public/browser/user_metrics.h"
36#include "net/base/escape.h"
37#include "net/base/load_flags.h"
38#include "net/base/net_util.h"
39#include "net/http/http_request_headers.h"
40#include "net/url_request/url_fetcher.h"
41#include "net/url_request/url_request_status.h"
42#include "url/gurl.h"
43
44namespace {
45
46// TODO(hfung): The histogram code was copied and modified from
47// search_provider.cc.  Refactor and consolidate the code.
48// We keep track in a histogram how many suggest requests we send, how
49// many suggest requests we invalidate (e.g., due to a user typing
50// another character), and how many replies we receive.
51// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
52//     (excluding the end-of-list enum value)
53// We do not want values of existing enums to change or else it screws
54// up the statistics.
55enum ZeroSuggestRequestsHistogramValue {
56  ZERO_SUGGEST_REQUEST_SENT = 1,
57  ZERO_SUGGEST_REQUEST_INVALIDATED,
58  ZERO_SUGGEST_REPLY_RECEIVED,
59  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
60};
61
62void LogOmniboxZeroSuggestRequest(
63    ZeroSuggestRequestsHistogramValue request_value) {
64  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
65                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
66}
67
68// The maximum relevance of the top match from this provider.
69const int kDefaultVerbatimZeroSuggestRelevance = 1300;
70
71// Relevance value to use if it was not set explicitly by the server.
72const int kDefaultZeroSuggestRelevance = 100;
73
74}  // namespace
75
76// static
77ZeroSuggestProvider* ZeroSuggestProvider::Create(
78    AutocompleteProviderListener* listener,
79    Profile* profile) {
80  return new ZeroSuggestProvider(listener, profile);
81}
82
83void ZeroSuggestProvider::Start(const AutocompleteInput& input,
84                                bool /*minimal_changes*/) {
85}
86
87void ZeroSuggestProvider::ResetSession() {
88  // The user has started editing in the omnibox, so leave
89  // |field_trial_triggered_in_session_| unchanged and set
90  // |field_trial_triggered_| to false since zero suggest is inactive now.
91  field_trial_triggered_ = false;
92
93  // This call clears out |matches_| so that they don't pollute prefix-based
94  // queries.
95  // TODO(mariakhomenko): Change the model to clear |matches_| on Start() like
96  // all the other providers.
97  Stop(true);
98}
99
100void ZeroSuggestProvider::StartZeroSuggest(
101    const GURL& current_page_url,
102    AutocompleteInput::PageClassification page_classification,
103    const base::string16& permanent_text) {
104  Stop(true);
105  field_trial_triggered_ = false;
106  field_trial_triggered_in_session_ = false;
107  permanent_text_ = permanent_text;
108  current_query_ = current_page_url.spec();
109  current_page_classification_ = page_classification;
110  current_url_match_ = MatchForCurrentURL();
111
112  const TemplateURL* default_provider =
113     template_url_service_->GetDefaultSearchProvider();
114  if (default_provider == NULL)
115    return;
116
117  base::string16 prefix;
118  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
119  GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
120      search_term_args));
121  if (!suggest_url.is_valid())
122    return;
123
124  // No need to send the current page URL in personalized suggest field trial.
125  if (CanSendURL(current_page_url, suggest_url, default_provider,
126                 current_page_classification_, profile_) &&
127      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
128    // Update suggest_url to include the current_page_url.
129    search_term_args.current_page_url = current_query_;
130    suggest_url = GURL(default_provider->suggestions_url_ref().
131                       ReplaceSearchTerms(search_term_args));
132  } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
133                                                  current_page_url)) {
134    return;
135  }
136
137  done_ = false;
138  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
139  // These may be useful on the NTP or more relevant to the user than server
140  // suggestions, if based on local browsing history.
141  Run(suggest_url);
142}
143
144ZeroSuggestProvider::ZeroSuggestProvider(
145  AutocompleteProviderListener* listener,
146  Profile* profile)
147    : BaseSearchProvider(listener, profile,
148                         AutocompleteProvider::TYPE_ZERO_SUGGEST),
149      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
150      weak_ptr_factory_(this) {
151}
152
153ZeroSuggestProvider::~ZeroSuggestProvider() {
154}
155
156const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
157  // Zero suggest provider should not receive keyword results.
158  DCHECK(!is_keyword);
159  return template_url_service_->GetDefaultSearchProvider();
160}
161
162const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
163  return AutocompleteInput(
164      base::string16(), base::string16::npos, base::string16(),
165      GURL(current_query_), current_page_classification_, true, false, false,
166      AutocompleteInput::ALL_MATCHES);
167}
168
169BaseSearchProvider::Results* ZeroSuggestProvider::GetResultsToFill(
170    bool is_keyword) {
171  DCHECK(!is_keyword);
172  return &results_;
173}
174
175bool ZeroSuggestProvider::ShouldAppendExtraParams(
176      const SuggestResult& result) const {
177  // We always use the default provider for search, so append the params.
178  return true;
179}
180
181void ZeroSuggestProvider::StopSuggest() {
182  if (suggest_results_pending_ > 0)
183    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
184  suggest_results_pending_ = 0;
185  fetcher_.reset();
186}
187
188void ZeroSuggestProvider::ClearAllResults() {
189  // We do not call Clear() on |results_| to retain |verbatim_relevance|
190  // value in the |results_| object. |verbatim_relevance| is used at the
191  // beginning of the next StartZeroSuggest() call to determine the current url
192  // match relevance.
193  results_.suggest_results.clear();
194  results_.navigation_results.clear();
195  current_query_.clear();
196  matches_.clear();
197}
198
199int ZeroSuggestProvider::GetDefaultResultRelevance() const {
200  return kDefaultZeroSuggestRelevance;
201}
202
203void ZeroSuggestProvider::RecordDeletionResult(bool success) {
204  if (success) {
205    content::RecordAction(
206        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
207  } else {
208    content::RecordAction(
209        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
210  }
211}
212
213void ZeroSuggestProvider::LogFetchComplete(bool success, bool is_keyword) {
214  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
215}
216
217bool ZeroSuggestProvider::IsKeywordFetcher(
218    const net::URLFetcher* fetcher) const {
219  // ZeroSuggestProvider does not have a keyword provider.
220  DCHECK_EQ(fetcher, fetcher_.get());
221  return false;
222}
223
224void ZeroSuggestProvider::UpdateMatches() {
225  done_ = true;
226  ConvertResultsToAutocompleteMatches();
227}
228
229void ZeroSuggestProvider::AddSuggestResultsToMap(
230    const SuggestResults& results,
231    MatchMap* map) {
232  for (size_t i = 0; i < results.size(); ++i)
233    AddMatchToMap(results[i], std::string(), i, false, map);
234}
235
236AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
237    const NavigationResult& navigation) {
238  AutocompleteMatch match(this, navigation.relevance(), false,
239                          AutocompleteMatchType::NAVSUGGEST);
240  match.destination_url = navigation.url();
241
242  // Zero suggest results should always omit protocols and never appear bold.
243  const std::string languages(
244      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
245  match.contents = net::FormatUrl(navigation.url(), languages,
246      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
247  match.fill_into_edit +=
248      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
249          match.contents);
250
251  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
252      match.contents.length(), ACMatchClassification::URL,
253      &match.contents_class);
254
255  match.description =
256      AutocompleteMatch::SanitizeString(navigation.description());
257  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
258      match.description.length(), ACMatchClassification::NONE,
259      &match.description_class);
260  return match;
261}
262
263void ZeroSuggestProvider::Run(const GURL& suggest_url) {
264  suggest_results_pending_ = 0;
265  const int kFetcherID = 1;
266  fetcher_.reset(
267      net::URLFetcher::Create(kFetcherID,
268          suggest_url,
269          net::URLFetcher::GET, this));
270  fetcher_->SetRequestContext(profile_->GetRequestContext());
271  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
272  // Add Chrome experiment state to the request headers.
273  net::HttpRequestHeaders headers;
274  chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
275      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
276  fetcher_->SetExtraRequestHeaders(headers.ToString());
277
278  fetcher_->Start();
279
280  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
281    most_visited_urls_.clear();
282    history::TopSites* ts = profile_->GetTopSites();
283    if (ts) {
284      ts->GetMostVisitedURLs(
285          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
286                     weak_ptr_factory_.GetWeakPtr()), false);
287    }
288  }
289  suggest_results_pending_ = 1;
290  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
291}
292
293void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
294    const history::MostVisitedURLList& urls) {
295  most_visited_urls_ = urls;
296}
297
298void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
299  matches_.clear();
300
301  const TemplateURL* default_provider =
302      template_url_service_->GetDefaultSearchProvider();
303  // Fail if we can't set the clickthrough URL for query suggestions.
304  if (default_provider == NULL || !default_provider->SupportsReplacement())
305    return;
306
307  MatchMap map;
308  AddSuggestResultsToMap(results_.suggest_results, &map);
309
310  const int num_query_results = map.size();
311  const int num_nav_results = results_.navigation_results.size();
312  const int num_results = num_query_results + num_nav_results;
313  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
314  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
315  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
316
317  // Show Most Visited results after ZeroSuggest response is received.
318  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
319    if (!current_url_match_.destination_url.is_valid())
320      return;
321    matches_.push_back(current_url_match_);
322    int relevance = 600;
323    if (num_results > 0) {
324      UMA_HISTOGRAM_COUNTS(
325          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
326          most_visited_urls_.size());
327    }
328    const base::string16 current_query_string16(
329        base::ASCIIToUTF16(current_query_));
330    const std::string languages(
331        profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
332    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
333      const history::MostVisitedURL& url = most_visited_urls_[i];
334      NavigationResult nav(*this, url.url, url.title, false, relevance, true,
335          current_query_string16, languages);
336      matches_.push_back(NavigationToMatch(nav));
337      --relevance;
338    }
339    return;
340  }
341
342  if (num_results == 0)
343    return;
344
345  // TODO(jered): Rip this out once the first match is decoupled from the
346  // current typing in the omnibox.
347  matches_.push_back(current_url_match_);
348
349  for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
350    matches_.push_back(it->second);
351
352  const NavigationResults& nav_results(results_.navigation_results);
353  for (NavigationResults::const_iterator it(nav_results.begin());
354       it != nav_results.end(); ++it)
355    matches_.push_back(NavigationToMatch(*it));
356}
357
358AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
359  AutocompleteInput input(permanent_text_, base::string16::npos, base::string16(),
360                          GURL(current_query_), current_page_classification_,
361                          false, false, true, AutocompleteInput::ALL_MATCHES);
362
363  AutocompleteMatch match;
364  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
365      permanent_text_, false, true, current_page_classification_, &match, NULL);
366  match.is_history_what_you_typed_match = false;
367  match.allowed_to_be_default_match = true;
368
369  // The placeholder suggestion for the current URL has high relevance so
370  // that it is in the first suggestion slot and inline autocompleted. It
371  // gets dropped as soon as the user types something.
372  match.relevance = GetVerbatimRelevance();
373
374  return match;
375}
376
377int ZeroSuggestProvider::GetVerbatimRelevance() const {
378  return results_.verbatim_relevance >= 0 ?
379      results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
380}
381
382bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
383    const GURL& suggest_url,
384    const GURL& current_page_url) const {
385  if (!ZeroSuggestEnabled(suggest_url,
386                          template_url_service_->GetDefaultSearchProvider(),
387                          current_page_classification_, profile_))
388    return false;
389
390  // If we cannot send URLs, then only the MostVisited and Personalized
391  // variations can be shown.
392  if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
393      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
394    return false;
395
396  // Only show zero suggest for HTTP[S] pages.
397  // TODO(mariakhomenko): We may be able to expand this set to include pages
398  // with other schemes (e.g. chrome://). That may require improvements to
399  // the formatting of the verbatim result returned by MatchForCurrentURL().
400  if (!current_page_url.is_valid() ||
401      ((current_page_url.scheme() != content::kHttpScheme) &&
402      (current_page_url.scheme() != content::kHttpsScheme)))
403    return false;
404
405  return true;
406}
407