zero_suggest_provider.cc revision 116680a4aac90f2aa7413d9095a592090648e557 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_classifier.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
18#include "chrome/browser/autocomplete/autocomplete_match.h"
19#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
20#include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
21#include "chrome/browser/autocomplete/history_url_provider.h"
22#include "chrome/browser/autocomplete/search_provider.h"
23#include "chrome/browser/history/history_types.h"
24#include "chrome/browser/history/top_sites.h"
25#include "chrome/browser/omnibox/omnibox_field_trial.h"
26#include "chrome/browser/profiles/profile.h"
27#include "chrome/browser/search_engines/template_url_service_factory.h"
28#include "chrome/common/pref_names.h"
29#include "chrome/common/url_constants.h"
30#include "components/autocomplete/autocomplete_input.h"
31#include "components/metrics/proto/omnibox_input_type.pb.h"
32#include "components/pref_registry/pref_registry_syncable.h"
33#include "components/search_engines/template_url_service.h"
34#include "components/variations/variations_http_header_provider.h"
35#include "content/public/browser/user_metrics.h"
36#include "net/base/escape.h"
37#include "net/base/load_flags.h"
38#include "net/base/net_util.h"
39#include "net/http/http_request_headers.h"
40#include "net/url_request/url_fetcher.h"
41#include "net/url_request/url_request_status.h"
42#include "url/gurl.h"
43
44namespace {
45
46// TODO(hfung): The histogram code was copied and modified from
47// search_provider.cc.  Refactor and consolidate the code.
48// We keep track in a histogram how many suggest requests we send, how
49// many suggest requests we invalidate (e.g., due to a user typing
50// another character), and how many replies we receive.
51// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
52//     (excluding the end-of-list enum value)
53// We do not want values of existing enums to change or else it screws
54// up the statistics.
55enum ZeroSuggestRequestsHistogramValue {
56  ZERO_SUGGEST_REQUEST_SENT = 1,
57  ZERO_SUGGEST_REQUEST_INVALIDATED,
58  ZERO_SUGGEST_REPLY_RECEIVED,
59  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
60};
61
62void LogOmniboxZeroSuggestRequest(
63    ZeroSuggestRequestsHistogramValue request_value) {
64  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
65                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
66}
67
68// The maximum relevance of the top match from this provider.
69const int kDefaultVerbatimZeroSuggestRelevance = 1300;
70
71// Relevance value to use if it was not set explicitly by the server.
72const int kDefaultZeroSuggestRelevance = 100;
73
74}  // namespace
75
76// static
77ZeroSuggestProvider* ZeroSuggestProvider::Create(
78    AutocompleteProviderListener* listener,
79    Profile* profile) {
80  return new ZeroSuggestProvider(listener, profile);
81}
82
83// static
84void ZeroSuggestProvider::RegisterProfilePrefs(
85    user_prefs::PrefRegistrySyncable* registry) {
86  registry->RegisterStringPref(
87      prefs::kZeroSuggestCachedResults,
88      std::string(),
89      user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
90}
91
92void ZeroSuggestProvider::Start(const AutocompleteInput& input,
93                                bool minimal_changes) {
94  matches_.clear();
95  if (input.type() == metrics::OmniboxInputType::INVALID)
96    return;
97
98  Stop(true);
99  field_trial_triggered_ = false;
100  field_trial_triggered_in_session_ = false;
101  results_from_cache_ = false;
102  permanent_text_ = input.text();
103  current_query_ = input.current_url().spec();
104  current_page_classification_ = input.current_page_classification();
105  current_url_match_ = MatchForCurrentURL();
106
107  const TemplateURL* default_provider =
108     template_url_service_->GetDefaultSearchProvider();
109  if (default_provider == NULL)
110    return;
111
112  base::string16 prefix;
113  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
114  GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
115      search_term_args, template_url_service_->search_terms_data()));
116  if (!suggest_url.is_valid())
117    return;
118
119  // No need to send the current page URL in personalized suggest field trial.
120  if (CanSendURL(input.current_url(), suggest_url, default_provider,
121                 current_page_classification_, profile_) &&
122      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
123    // Update suggest_url to include the current_page_url.
124    search_term_args.current_page_url = current_query_;
125    suggest_url = GURL(default_provider->suggestions_url_ref().
126                       ReplaceSearchTerms(
127                           search_term_args,
128                           template_url_service_->search_terms_data()));
129  } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
130                                                  input.current_url())) {
131    return;
132  }
133
134  done_ = false;
135  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
136  // These may be useful on the NTP or more relevant to the user than server
137  // suggestions, if based on local browsing history.
138  MaybeUseCachedSuggestions();
139  Run(suggest_url);
140}
141
142void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
143  if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
144    // Remove the deleted match from the cache, so it is not shown to the user
145    // again. Since we cannot remove just one result, blow away the cache.
146    profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
147                                    std::string());
148  }
149  BaseSearchProvider::DeleteMatch(match);
150}
151
152void ZeroSuggestProvider::ResetSession() {
153  // The user has started editing in the omnibox, so leave
154  // |field_trial_triggered_in_session_| unchanged and set
155  // |field_trial_triggered_| to false since zero suggest is inactive now.
156  field_trial_triggered_ = false;
157}
158
159void ZeroSuggestProvider::ModifyProviderInfo(
160    metrics::OmniboxEventProto_ProviderInfo* provider_info) const {
161  if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
162    provider_info->set_times_returned_results_in_session(1);
163}
164
165ZeroSuggestProvider::ZeroSuggestProvider(
166  AutocompleteProviderListener* listener,
167  Profile* profile)
168    : BaseSearchProvider(listener, profile,
169                         AutocompleteProvider::TYPE_ZERO_SUGGEST),
170      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
171      results_from_cache_(false),
172      weak_ptr_factory_(this) {
173}
174
175ZeroSuggestProvider::~ZeroSuggestProvider() {
176}
177
178bool ZeroSuggestProvider::StoreSuggestionResponse(
179    const std::string& json_data,
180    const base::Value& parsed_data) {
181  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
182      json_data.empty())
183    return false;
184  profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
185
186  // If we received an empty result list, we should update the display, as it
187  // may be showing cached results that should not be shown.
188  const base::ListValue* root_list = NULL;
189  const base::ListValue* results_list = NULL;
190  if (parsed_data.GetAsList(&root_list) &&
191      root_list->GetList(1, &results_list) &&
192      results_list->empty())
193    return false;
194
195  // We are finished with the request and want to bail early.
196  if (results_from_cache_)
197    done_ = true;
198
199  return results_from_cache_;
200}
201
202const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
203  // Zero suggest provider should not receive keyword results.
204  DCHECK(!is_keyword);
205  return template_url_service_->GetDefaultSearchProvider();
206}
207
208const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
209  return AutocompleteInput(
210      base::string16(), base::string16::npos, base::string16(),
211      GURL(current_query_), current_page_classification_, true, false, false,
212      true, ChromeAutocompleteSchemeClassifier(profile_));
213}
214
215BaseSearchProvider::Results* ZeroSuggestProvider::GetResultsToFill(
216    bool is_keyword) {
217  DCHECK(!is_keyword);
218  return &results_;
219}
220
221bool ZeroSuggestProvider::ShouldAppendExtraParams(
222      const SuggestResult& result) const {
223  // We always use the default provider for search, so append the params.
224  return true;
225}
226
227void ZeroSuggestProvider::StopSuggest() {
228  if (suggest_results_pending_ > 0)
229    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
230  suggest_results_pending_ = 0;
231  fetcher_.reset();
232}
233
234void ZeroSuggestProvider::ClearAllResults() {
235  // We do not call Clear() on |results_| to retain |verbatim_relevance|
236  // value in the |results_| object. |verbatim_relevance| is used at the
237  // beginning of the next StartZeroSuggest() call to determine the current url
238  // match relevance.
239  results_.suggest_results.clear();
240  results_.navigation_results.clear();
241  current_query_.clear();
242}
243
244int ZeroSuggestProvider::GetDefaultResultRelevance() const {
245  return kDefaultZeroSuggestRelevance;
246}
247
248void ZeroSuggestProvider::RecordDeletionResult(bool success) {
249  if (success) {
250    content::RecordAction(
251        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
252  } else {
253    content::RecordAction(
254        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
255  }
256}
257
258void ZeroSuggestProvider::LogFetchComplete(bool success, bool is_keyword) {
259  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
260}
261
262bool ZeroSuggestProvider::IsKeywordFetcher(
263    const net::URLFetcher* fetcher) const {
264  // ZeroSuggestProvider does not have a keyword provider.
265  DCHECK_EQ(fetcher, fetcher_.get());
266  return false;
267}
268
269void ZeroSuggestProvider::UpdateMatches() {
270  done_ = true;
271  ConvertResultsToAutocompleteMatches();
272}
273
274void ZeroSuggestProvider::AddSuggestResultsToMap(
275    const SuggestResults& results,
276    MatchMap* map) {
277  for (size_t i = 0; i < results.size(); ++i)
278    AddMatchToMap(results[i], std::string(), i, false, map);
279}
280
281AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
282    const NavigationResult& navigation) {
283  AutocompleteMatch match(this, navigation.relevance(), false,
284                          navigation.type());
285  match.destination_url = navigation.url();
286
287  // Zero suggest results should always omit protocols and never appear bold.
288  const std::string languages(
289      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
290  match.contents = net::FormatUrl(navigation.url(), languages,
291      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
292  match.fill_into_edit +=
293      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
294          match.contents, ChromeAutocompleteSchemeClassifier(profile_));
295
296  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
297      match.contents.length(), ACMatchClassification::URL,
298      &match.contents_class);
299
300  match.description =
301      AutocompleteMatch::SanitizeString(navigation.description());
302  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
303      match.description.length(), ACMatchClassification::NONE,
304      &match.description_class);
305  return match;
306}
307
308void ZeroSuggestProvider::Run(const GURL& suggest_url) {
309  suggest_results_pending_ = 0;
310  const int kFetcherID = 1;
311  fetcher_.reset(
312      net::URLFetcher::Create(kFetcherID,
313          suggest_url,
314          net::URLFetcher::GET, this));
315  fetcher_->SetRequestContext(profile_->GetRequestContext());
316  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
317  // Add Chrome experiment state to the request headers.
318  net::HttpRequestHeaders headers;
319  variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
320      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
321  fetcher_->SetExtraRequestHeaders(headers.ToString());
322  fetcher_->Start();
323
324  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
325    most_visited_urls_.clear();
326    history::TopSites* ts = profile_->GetTopSites();
327    if (ts) {
328      ts->GetMostVisitedURLs(
329          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
330                     weak_ptr_factory_.GetWeakPtr()), false);
331    }
332  }
333  suggest_results_pending_ = 1;
334  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
335}
336
337void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
338    const history::MostVisitedURLList& urls) {
339  most_visited_urls_ = urls;
340}
341
342void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
343  matches_.clear();
344
345  const TemplateURL* default_provider =
346      template_url_service_->GetDefaultSearchProvider();
347  // Fail if we can't set the clickthrough URL for query suggestions.
348  if (default_provider == NULL || !default_provider->SupportsReplacement(
349          template_url_service_->search_terms_data()))
350    return;
351
352  MatchMap map;
353  AddSuggestResultsToMap(results_.suggest_results, &map);
354
355  const int num_query_results = map.size();
356  const int num_nav_results = results_.navigation_results.size();
357  const int num_results = num_query_results + num_nav_results;
358  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
359  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
360  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
361
362  // Show Most Visited results after ZeroSuggest response is received.
363  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
364    if (!current_url_match_.destination_url.is_valid())
365      return;
366    matches_.push_back(current_url_match_);
367    int relevance = 600;
368    if (num_results > 0) {
369      UMA_HISTOGRAM_COUNTS(
370          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
371          most_visited_urls_.size());
372    }
373    const base::string16 current_query_string16(
374        base::ASCIIToUTF16(current_query_));
375    const std::string languages(
376        profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
377    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
378      const history::MostVisitedURL& url = most_visited_urls_[i];
379      NavigationResult nav(
380          ChromeAutocompleteSchemeClassifier(profile_), url.url,
381          AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
382          relevance, true, current_query_string16, languages);
383      matches_.push_back(NavigationToMatch(nav));
384      --relevance;
385    }
386    return;
387  }
388
389  if (num_results == 0)
390    return;
391
392  // TODO(jered): Rip this out once the first match is decoupled from the
393  // current typing in the omnibox.
394  matches_.push_back(current_url_match_);
395
396  for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
397    matches_.push_back(it->second);
398
399  const NavigationResults& nav_results(results_.navigation_results);
400  for (NavigationResults::const_iterator it(nav_results.begin());
401       it != nav_results.end(); ++it)
402    matches_.push_back(NavigationToMatch(*it));
403}
404
405AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
406  AutocompleteMatch match;
407  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
408      permanent_text_, false, true, current_page_classification_, &match, NULL);
409  match.is_history_what_you_typed_match = false;
410  match.allowed_to_be_default_match = true;
411
412  // The placeholder suggestion for the current URL has high relevance so
413  // that it is in the first suggestion slot and inline autocompleted. It
414  // gets dropped as soon as the user types something.
415  match.relevance = GetVerbatimRelevance();
416
417  return match;
418}
419
420int ZeroSuggestProvider::GetVerbatimRelevance() const {
421  return results_.verbatim_relevance >= 0 ?
422      results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
423}
424
425bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
426    const GURL& suggest_url,
427    const GURL& current_page_url) const {
428  if (!ZeroSuggestEnabled(suggest_url,
429                          template_url_service_->GetDefaultSearchProvider(),
430                          current_page_classification_, profile_))
431    return false;
432
433  // If we cannot send URLs, then only the MostVisited and Personalized
434  // variations can be shown.
435  if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
436      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
437    return false;
438
439  // Only show zero suggest for HTTP[S] pages.
440  // TODO(mariakhomenko): We may be able to expand this set to include pages
441  // with other schemes (e.g. chrome://). That may require improvements to
442  // the formatting of the verbatim result returned by MatchForCurrentURL().
443  if (!current_page_url.is_valid() ||
444      ((current_page_url.scheme() != url::kHttpScheme) &&
445      (current_page_url.scheme() != url::kHttpsScheme)))
446    return false;
447
448  return true;
449}
450
451void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
452  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
453    return;
454
455  std::string json_data = profile_->GetPrefs()->GetString(
456      prefs::kZeroSuggestCachedResults);
457  if (!json_data.empty()) {
458    scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
459    if (data && ParseSuggestResults(*data.get(), false, &results_)) {
460      ConvertResultsToAutocompleteMatches();
461      results_from_cache_ = !matches_.empty();
462    }
463  }
464}
465