zero_suggest_provider.cc revision a1401311d1ab56c4ed0a474bd38c108f75cb0cd9 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_classifier.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
18#include "chrome/browser/autocomplete/autocomplete_input.h"
19#include "chrome/browser/autocomplete/autocomplete_match.h"
20#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
21#include "chrome/browser/autocomplete/history_url_provider.h"
22#include "chrome/browser/autocomplete/search_provider.h"
23#include "chrome/browser/autocomplete/url_prefix.h"
24#include "chrome/browser/history/history_types.h"
25#include "chrome/browser/history/top_sites.h"
26#include "chrome/browser/metrics/variations/variations_http_header_provider.h"
27#include "chrome/browser/omnibox/omnibox_field_trial.h"
28#include "chrome/browser/profiles/profile.h"
29#include "chrome/browser/search/search.h"
30#include "chrome/browser/search_engines/template_url_service.h"
31#include "chrome/browser/search_engines/template_url_service_factory.h"
32#include "chrome/common/net/url_fixer_upper.h"
33#include "chrome/common/pref_names.h"
34#include "chrome/common/url_constants.h"
35#include "content/public/browser/user_metrics.h"
36#include "net/base/escape.h"
37#include "net/base/load_flags.h"
38#include "net/base/net_util.h"
39#include "net/http/http_request_headers.h"
40#include "net/http/http_response_headers.h"
41#include "net/url_request/url_fetcher.h"
42#include "net/url_request/url_request_status.h"
43#include "url/gurl.h"
44
45namespace {
46
47// TODO(hfung): The histogram code was copied and modified from
48// search_provider.cc.  Refactor and consolidate the code.
49// We keep track in a histogram how many suggest requests we send, how
50// many suggest requests we invalidate (e.g., due to a user typing
51// another character), and how many replies we receive.
52// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
53//     (excluding the end-of-list enum value)
54// We do not want values of existing enums to change or else it screws
55// up the statistics.
56enum ZeroSuggestRequestsHistogramValue {
57  ZERO_SUGGEST_REQUEST_SENT = 1,
58  ZERO_SUGGEST_REQUEST_INVALIDATED,
59  ZERO_SUGGEST_REPLY_RECEIVED,
60  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
61};
62
63void LogOmniboxZeroSuggestRequest(
64    ZeroSuggestRequestsHistogramValue request_value) {
65  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
66                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
67}
68
69// The maximum relevance of the top match from this provider.
70const int kDefaultVerbatimZeroSuggestRelevance = 1300;
71
72// Relevance value to use if it was not set explicitly by the server.
73const int kDefaultZeroSuggestRelevance = 100;
74
75}  // namespace
76
77// static
78ZeroSuggestProvider* ZeroSuggestProvider::Create(
79    AutocompleteProviderListener* listener,
80    Profile* profile) {
81  return new ZeroSuggestProvider(listener, profile);
82}
83
84void ZeroSuggestProvider::Start(const AutocompleteInput& input,
85                                bool /*minimal_changes*/) {
86}
87
88void ZeroSuggestProvider::ResetSession() {
89  // The user has started editing in the omnibox, so leave
90  // |field_trial_triggered_in_session_| unchanged and set
91  // |field_trial_triggered_| to false since zero suggest is inactive now.
92  field_trial_triggered_ = false;
93  Stop(true);
94}
95
96void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
97  have_pending_request_ = false;
98  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
99
100  std::string json_data;
101  source->GetResponseAsString(&json_data);
102  const bool request_succeeded =
103      source->GetStatus().is_success() && source->GetResponseCode() == 200;
104
105  if (request_succeeded) {
106    scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
107    if (data.get())
108      ParseSuggestResults(*data.get(), false, &results_);
109  }
110  done_ = true;
111
112  ConvertResultsToAutocompleteMatches();
113  if (!matches_.empty())
114    listener_->OnProviderUpdate(true);
115}
116
117void ZeroSuggestProvider::StartZeroSuggest(
118    const GURL& current_page_url,
119    AutocompleteInput::PageClassification page_classification,
120    const base::string16& permanent_text) {
121  Stop(true);
122  field_trial_triggered_ = false;
123  field_trial_triggered_in_session_ = false;
124  permanent_text_ = permanent_text;
125  current_query_ = current_page_url.spec();
126  current_page_classification_ = page_classification;
127  current_url_match_ = MatchForCurrentURL();
128
129  const TemplateURL* default_provider =
130     template_url_service_->GetDefaultSearchProvider();
131  if (default_provider == NULL)
132    return;
133  base::string16 prefix;
134  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
135  search_term_args.current_page_url = current_query_;
136  GURL suggest_url(default_provider->suggestions_url_ref().
137                   ReplaceSearchTerms(search_term_args));
138  if (!CanSendURL(current_page_url, suggest_url,
139          template_url_service_->GetDefaultSearchProvider(),
140          page_classification, profile_) ||
141      !OmniboxFieldTrial::InZeroSuggestFieldTrial())
142    return;
143  done_ = false;
144  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
145  // These may be useful on the NTP or more relevant to the user than server
146  // suggestions, if based on local browsing history.
147  Run(suggest_url);
148}
149
150ZeroSuggestProvider::ZeroSuggestProvider(
151  AutocompleteProviderListener* listener,
152  Profile* profile)
153    : BaseSearchProvider(listener, profile,
154                         AutocompleteProvider::TYPE_ZERO_SUGGEST),
155      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
156      have_pending_request_(false),
157      weak_ptr_factory_(this) {
158}
159
160ZeroSuggestProvider::~ZeroSuggestProvider() {
161}
162
163const TemplateURL* ZeroSuggestProvider::GetTemplateURL(
164    const SuggestResult& result) const {
165  // Zero suggest provider should not receive keyword results.
166  DCHECK(!result.from_keyword_provider());
167  return template_url_service_->GetDefaultSearchProvider();
168}
169
170const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
171  return AutocompleteInput(
172      base::string16(), base::string16::npos, base::string16(),
173      GURL(current_query_), current_page_classification_, true, false, false,
174      AutocompleteInput::ALL_MATCHES);
175}
176
177bool ZeroSuggestProvider::ShouldAppendExtraParams(
178      const SuggestResult& result) const {
179  // We always use the default provider for search, so append the params.
180  return true;
181}
182
183void ZeroSuggestProvider::StopSuggest() {
184  if (have_pending_request_)
185    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
186  have_pending_request_ = false;
187  fetcher_.reset();
188}
189
190void ZeroSuggestProvider::ClearAllResults() {
191  // We do not call Clear() on |results_| to retain |verbatim_relevance|
192  // value in the |results_| object. |verbatim_relevance| is used at the
193  // beginning of the next StartZeroSuggest() call to determine the current url
194  // match relevance.
195  results_.suggest_results.clear();
196  results_.navigation_results.clear();
197  current_query_.clear();
198  matches_.clear();
199}
200
201int ZeroSuggestProvider::GetDefaultResultRelevance() const {
202  return kDefaultZeroSuggestRelevance;
203}
204
205void ZeroSuggestProvider::RecordDeletionResult(bool success) {
206  if (success) {
207    content::RecordAction(
208        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
209  } else {
210    content::RecordAction(
211        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
212  }
213}
214
215void ZeroSuggestProvider::AddSuggestResultsToMap(
216    const SuggestResults& results,
217    MatchMap* map) {
218  for (size_t i = 0; i < results.size(); ++i)
219    AddMatchToMap(results[i], std::string(), i, false, map);
220}
221
222AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
223    const NavigationResult& navigation) {
224  AutocompleteMatch match(this, navigation.relevance(), false,
225                          AutocompleteMatchType::NAVSUGGEST);
226  match.destination_url = navigation.url();
227
228  // Zero suggest results should always omit protocols and never appear bold.
229  const std::string languages(
230      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
231  match.contents = net::FormatUrl(navigation.url(), languages,
232      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
233  match.fill_into_edit +=
234      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
235          match.contents);
236
237  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
238      match.contents.length(), ACMatchClassification::URL,
239      &match.contents_class);
240
241  match.description =
242      AutocompleteMatch::SanitizeString(navigation.description());
243  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
244      match.description.length(), ACMatchClassification::NONE,
245      &match.description_class);
246  return match;
247}
248
249void ZeroSuggestProvider::Run(const GURL& suggest_url) {
250  have_pending_request_ = false;
251  const int kFetcherID = 1;
252  fetcher_.reset(
253      net::URLFetcher::Create(kFetcherID,
254          suggest_url,
255          net::URLFetcher::GET, this));
256  fetcher_->SetRequestContext(profile_->GetRequestContext());
257  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
258  // Add Chrome experiment state to the request headers.
259  net::HttpRequestHeaders headers;
260  chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
261      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
262  fetcher_->SetExtraRequestHeaders(headers.ToString());
263
264  fetcher_->Start();
265
266  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
267    most_visited_urls_.clear();
268    history::TopSites* ts = profile_->GetTopSites();
269    if (ts) {
270      ts->GetMostVisitedURLs(
271          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
272                     weak_ptr_factory_.GetWeakPtr()), false);
273    }
274  }
275  have_pending_request_ = true;
276  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
277}
278
279void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
280    const history::MostVisitedURLList& urls) {
281  most_visited_urls_ = urls;
282}
283
284void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
285  matches_.clear();
286
287  const TemplateURL* default_provider =
288      template_url_service_->GetDefaultSearchProvider();
289  // Fail if we can't set the clickthrough URL for query suggestions.
290  if (default_provider == NULL || !default_provider->SupportsReplacement())
291    return;
292
293  MatchMap map;
294  AddSuggestResultsToMap(results_.suggest_results, &map);
295
296  const int num_query_results = map.size();
297  const int num_nav_results = results_.navigation_results.size();
298  const int num_results = num_query_results + num_nav_results;
299  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
300  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
301  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
302
303  // Show Most Visited results after ZeroSuggest response is received.
304  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
305    if (!current_url_match_.destination_url.is_valid())
306      return;
307    matches_.push_back(current_url_match_);
308    int relevance = 600;
309    if (num_results > 0) {
310      UMA_HISTOGRAM_COUNTS(
311          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
312          most_visited_urls_.size());
313    }
314    const base::string16 current_query_string16(
315        base::ASCIIToUTF16(current_query_));
316    const std::string languages(
317        profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
318    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
319      const history::MostVisitedURL& url = most_visited_urls_[i];
320      NavigationResult nav(*this, url.url, url.title, false, relevance, true,
321          current_query_string16, languages);
322      matches_.push_back(NavigationToMatch(nav));
323      --relevance;
324    }
325    return;
326  }
327
328  if (num_results == 0)
329    return;
330
331  // TODO(jered): Rip this out once the first match is decoupled from the
332  // current typing in the omnibox.
333  matches_.push_back(current_url_match_);
334
335  for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
336    matches_.push_back(it->second);
337
338  const NavigationResults& nav_results(results_.navigation_results);
339  for (NavigationResults::const_iterator it(nav_results.begin());
340       it != nav_results.end(); ++it)
341    matches_.push_back(NavigationToMatch(*it));
342}
343
344AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
345  AutocompleteInput input(permanent_text_, base::string16::npos, base::string16(),
346                          GURL(current_query_), current_page_classification_,
347                          false, false, true, AutocompleteInput::ALL_MATCHES);
348
349  AutocompleteMatch match;
350  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
351      permanent_text_, false, true, current_page_classification_, &match, NULL);
352  match.is_history_what_you_typed_match = false;
353  match.allowed_to_be_default_match = true;
354
355  // The placeholder suggestion for the current URL has high relevance so
356  // that it is in the first suggestion slot and inline autocompleted. It
357  // gets dropped as soon as the user types something.
358  match.relevance = GetVerbatimRelevance();
359
360  return match;
361}
362
363int ZeroSuggestProvider::GetVerbatimRelevance() const {
364  return results_.verbatim_relevance >= 0 ?
365      results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
366}
367