zero_suggest_provider.cc revision 5f1c94371a64b3196d4be9466099bb892df9b88e 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_classifier.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
18#include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
19#include "chrome/browser/autocomplete/history_url_provider.h"
20#include "chrome/browser/autocomplete/search_provider.h"
21#include "chrome/browser/history/history_types.h"
22#include "chrome/browser/history/top_sites.h"
23#include "chrome/browser/omnibox/omnibox_field_trial.h"
24#include "chrome/browser/profiles/profile.h"
25#include "chrome/common/pref_names.h"
26#include "components/metrics/proto/omnibox_input_type.pb.h"
27#include "components/omnibox/autocomplete_input.h"
28#include "components/omnibox/autocomplete_match.h"
29#include "components/omnibox/autocomplete_provider_listener.h"
30#include "components/pref_registry/pref_registry_syncable.h"
31#include "components/search_engines/template_url_service.h"
32#include "components/variations/variations_http_header_provider.h"
33#include "content/public/browser/user_metrics.h"
34#include "net/base/escape.h"
35#include "net/base/load_flags.h"
36#include "net/base/net_util.h"
37#include "net/http/http_request_headers.h"
38#include "net/url_request/url_fetcher.h"
39#include "net/url_request/url_request_status.h"
40#include "url/gurl.h"
41
42namespace {
43
44// TODO(hfung): The histogram code was copied and modified from
45// search_provider.cc.  Refactor and consolidate the code.
46// We keep track in a histogram how many suggest requests we send, how
47// many suggest requests we invalidate (e.g., due to a user typing
48// another character), and how many replies we receive.
49// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
50//     (excluding the end-of-list enum value)
51// We do not want values of existing enums to change or else it screws
52// up the statistics.
53enum ZeroSuggestRequestsHistogramValue {
54  ZERO_SUGGEST_REQUEST_SENT = 1,
55  ZERO_SUGGEST_REQUEST_INVALIDATED,
56  ZERO_SUGGEST_REPLY_RECEIVED,
57  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
58};
59
60void LogOmniboxZeroSuggestRequest(
61    ZeroSuggestRequestsHistogramValue request_value) {
62  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
63                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
64}
65
66// The maximum relevance of the top match from this provider.
67const int kDefaultVerbatimZeroSuggestRelevance = 1300;
68
69// Relevance value to use if it was not set explicitly by the server.
70const int kDefaultZeroSuggestRelevance = 100;
71
72}  // namespace
73
74// static
75ZeroSuggestProvider* ZeroSuggestProvider::Create(
76    AutocompleteProviderListener* listener,
77    TemplateURLService* template_url_service,
78    Profile* profile) {
79  return new ZeroSuggestProvider(listener, template_url_service, profile);
80}
81
82// static
83void ZeroSuggestProvider::RegisterProfilePrefs(
84    user_prefs::PrefRegistrySyncable* registry) {
85  registry->RegisterStringPref(
86      prefs::kZeroSuggestCachedResults,
87      std::string(),
88      user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
89}
90
91void ZeroSuggestProvider::Start(const AutocompleteInput& input,
92                                bool minimal_changes) {
93  matches_.clear();
94  if (input.type() == metrics::OmniboxInputType::INVALID)
95    return;
96
97  Stop(true);
98  field_trial_triggered_ = false;
99  field_trial_triggered_in_session_ = false;
100  results_from_cache_ = false;
101  permanent_text_ = input.text();
102  current_query_ = input.current_url().spec();
103  current_page_classification_ = input.current_page_classification();
104  current_url_match_ = MatchForCurrentURL();
105
106  const TemplateURL* default_provider =
107     template_url_service_->GetDefaultSearchProvider();
108  if (default_provider == NULL)
109    return;
110
111  base::string16 prefix;
112  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
113  GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
114      search_term_args, template_url_service_->search_terms_data()));
115  if (!suggest_url.is_valid())
116    return;
117
118  // No need to send the current page URL in personalized suggest field trial.
119  if (CanSendURL(input.current_url(), suggest_url, default_provider,
120                 current_page_classification_,
121                 template_url_service_->search_terms_data(), profile_) &&
122      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
123    // Update suggest_url to include the current_page_url.
124    search_term_args.current_page_url = current_query_;
125    suggest_url = GURL(default_provider->suggestions_url_ref().
126                       ReplaceSearchTerms(
127                           search_term_args,
128                           template_url_service_->search_terms_data()));
129  } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
130                                                  input.current_url())) {
131    return;
132  }
133
134  done_ = false;
135  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
136  // These may be useful on the NTP or more relevant to the user than server
137  // suggestions, if based on local browsing history.
138  MaybeUseCachedSuggestions();
139  Run(suggest_url);
140}
141
142void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
143  if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
144    // Remove the deleted match from the cache, so it is not shown to the user
145    // again. Since we cannot remove just one result, blow away the cache.
146    profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
147                                    std::string());
148  }
149  BaseSearchProvider::DeleteMatch(match);
150}
151
152void ZeroSuggestProvider::ResetSession() {
153  // The user has started editing in the omnibox, so leave
154  // |field_trial_triggered_in_session_| unchanged and set
155  // |field_trial_triggered_| to false since zero suggest is inactive now.
156  field_trial_triggered_ = false;
157}
158
159void ZeroSuggestProvider::ModifyProviderInfo(
160    metrics::OmniboxEventProto_ProviderInfo* provider_info) const {
161  if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
162    provider_info->set_times_returned_results_in_session(1);
163}
164
165ZeroSuggestProvider::ZeroSuggestProvider(
166  AutocompleteProviderListener* listener,
167  TemplateURLService* template_url_service,
168  Profile* profile)
169    : BaseSearchProvider(listener, template_url_service, profile,
170                         AutocompleteProvider::TYPE_ZERO_SUGGEST),
171      results_from_cache_(false),
172      weak_ptr_factory_(this) {
173}
174
175ZeroSuggestProvider::~ZeroSuggestProvider() {
176}
177
178bool ZeroSuggestProvider::StoreSuggestionResponse(
179    const std::string& json_data,
180    const base::Value& parsed_data) {
181  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
182      json_data.empty())
183    return false;
184  profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
185
186  // If we received an empty result list, we should update the display, as it
187  // may be showing cached results that should not be shown.
188  const base::ListValue* root_list = NULL;
189  const base::ListValue* results_list = NULL;
190  if (parsed_data.GetAsList(&root_list) &&
191      root_list->GetList(1, &results_list) &&
192      results_list->empty())
193    return false;
194
195  // We are finished with the request and want to bail early.
196  if (results_from_cache_)
197    done_ = true;
198
199  return results_from_cache_;
200}
201
202const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
203  // Zero suggest provider should not receive keyword results.
204  DCHECK(!is_keyword);
205  return template_url_service_->GetDefaultSearchProvider();
206}
207
208const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
209  return AutocompleteInput(
210      base::string16(), base::string16::npos, base::string16(),
211      GURL(current_query_), current_page_classification_, true, false, false,
212      true, ChromeAutocompleteSchemeClassifier(profile_));
213}
214
215SearchSuggestionParser::Results* ZeroSuggestProvider::GetResultsToFill(
216    bool is_keyword) {
217  DCHECK(!is_keyword);
218  return &results_;
219}
220
221bool ZeroSuggestProvider::ShouldAppendExtraParams(
222      const SearchSuggestionParser::SuggestResult& result) const {
223  // We always use the default provider for search, so append the params.
224  return true;
225}
226
227void ZeroSuggestProvider::StopSuggest() {
228  if (suggest_results_pending_ > 0)
229    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
230  suggest_results_pending_ = 0;
231  fetcher_.reset();
232}
233
234void ZeroSuggestProvider::ClearAllResults() {
235  // We do not call Clear() on |results_| to retain |verbatim_relevance|
236  // value in the |results_| object. |verbatim_relevance| is used at the
237  // beginning of the next StartZeroSuggest() call to determine the current url
238  // match relevance.
239  results_.suggest_results.clear();
240  results_.navigation_results.clear();
241  current_query_.clear();
242}
243
244int ZeroSuggestProvider::GetDefaultResultRelevance() const {
245  return kDefaultZeroSuggestRelevance;
246}
247
248void ZeroSuggestProvider::RecordDeletionResult(bool success) {
249  if (success) {
250    content::RecordAction(
251        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
252  } else {
253    content::RecordAction(
254        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
255  }
256}
257
258void ZeroSuggestProvider::LogFetchComplete(bool success, bool is_keyword) {
259  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
260}
261
262bool ZeroSuggestProvider::IsKeywordFetcher(
263    const net::URLFetcher* fetcher) const {
264  // ZeroSuggestProvider does not have a keyword provider.
265  DCHECK_EQ(fetcher, fetcher_.get());
266  return false;
267}
268
269void ZeroSuggestProvider::UpdateMatches() {
270  done_ = true;
271  ConvertResultsToAutocompleteMatches();
272}
273
274void ZeroSuggestProvider::AddSuggestResultsToMap(
275    const SearchSuggestionParser::SuggestResults& results,
276    MatchMap* map) {
277  for (size_t i = 0; i < results.size(); ++i)
278    AddMatchToMap(results[i], std::string(), i, false, map);
279}
280
281AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
282    const SearchSuggestionParser::NavigationResult& navigation) {
283  AutocompleteMatch match(this, navigation.relevance(), false,
284                          navigation.type());
285  match.destination_url = navigation.url();
286
287  // Zero suggest results should always omit protocols and never appear bold.
288  const std::string languages(
289      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
290  match.contents = net::FormatUrl(navigation.url(), languages,
291      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
292  match.fill_into_edit +=
293      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
294          match.contents, ChromeAutocompleteSchemeClassifier(profile_));
295
296  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
297      match.contents.length(), ACMatchClassification::URL,
298      &match.contents_class);
299
300  match.description =
301      AutocompleteMatch::SanitizeString(navigation.description());
302  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
303      match.description.length(), ACMatchClassification::NONE,
304      &match.description_class);
305  return match;
306}
307
308void ZeroSuggestProvider::Run(const GURL& suggest_url) {
309  suggest_results_pending_ = 0;
310  const int kFetcherID = 1;
311  fetcher_.reset(
312      net::URLFetcher::Create(kFetcherID,
313          suggest_url,
314          net::URLFetcher::GET, this));
315  fetcher_->SetRequestContext(profile_->GetRequestContext());
316  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
317  // Add Chrome experiment state to the request headers.
318  net::HttpRequestHeaders headers;
319  variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
320      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
321  fetcher_->SetExtraRequestHeaders(headers.ToString());
322  fetcher_->Start();
323
324  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
325    most_visited_urls_.clear();
326    history::TopSites* ts = profile_->GetTopSites();
327    if (ts) {
328      ts->GetMostVisitedURLs(
329          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
330                     weak_ptr_factory_.GetWeakPtr()), false);
331    }
332  }
333  suggest_results_pending_ = 1;
334  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
335}
336
337void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
338    const history::MostVisitedURLList& urls) {
339  most_visited_urls_ = urls;
340}
341
342void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
343  matches_.clear();
344
345  const TemplateURL* default_provider =
346      template_url_service_->GetDefaultSearchProvider();
347  // Fail if we can't set the clickthrough URL for query suggestions.
348  if (default_provider == NULL || !default_provider->SupportsReplacement(
349          template_url_service_->search_terms_data()))
350    return;
351
352  MatchMap map;
353  AddSuggestResultsToMap(results_.suggest_results, &map);
354
355  const int num_query_results = map.size();
356  const int num_nav_results = results_.navigation_results.size();
357  const int num_results = num_query_results + num_nav_results;
358  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
359  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
360  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
361
362  // Show Most Visited results after ZeroSuggest response is received.
363  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
364    if (!current_url_match_.destination_url.is_valid())
365      return;
366    matches_.push_back(current_url_match_);
367    int relevance = 600;
368    if (num_results > 0) {
369      UMA_HISTOGRAM_COUNTS(
370          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
371          most_visited_urls_.size());
372    }
373    const base::string16 current_query_string16(
374        base::ASCIIToUTF16(current_query_));
375    const std::string languages(
376        profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
377    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
378      const history::MostVisitedURL& url = most_visited_urls_[i];
379      SearchSuggestionParser::NavigationResult nav(
380          ChromeAutocompleteSchemeClassifier(profile_), url.url,
381          AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
382          relevance, true, current_query_string16, languages);
383      matches_.push_back(NavigationToMatch(nav));
384      --relevance;
385    }
386    return;
387  }
388
389  if (num_results == 0)
390    return;
391
392  // TODO(jered): Rip this out once the first match is decoupled from the
393  // current typing in the omnibox.
394  matches_.push_back(current_url_match_);
395
396  for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
397    matches_.push_back(it->second);
398
399  const SearchSuggestionParser::NavigationResults& nav_results(
400      results_.navigation_results);
401  for (SearchSuggestionParser::NavigationResults::const_iterator it(
402           nav_results.begin()); it != nav_results.end(); ++it)
403    matches_.push_back(NavigationToMatch(*it));
404}
405
406AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
407  AutocompleteMatch match;
408  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
409      permanent_text_, false, true, current_page_classification_, &match, NULL);
410  match.is_history_what_you_typed_match = false;
411  match.allowed_to_be_default_match = true;
412
413  // The placeholder suggestion for the current URL has high relevance so
414  // that it is in the first suggestion slot and inline autocompleted. It
415  // gets dropped as soon as the user types something.
416  match.relevance = GetVerbatimRelevance();
417
418  return match;
419}
420
421int ZeroSuggestProvider::GetVerbatimRelevance() const {
422  return results_.verbatim_relevance >= 0 ?
423      results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
424}
425
426bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
427    const GURL& suggest_url,
428    const GURL& current_page_url) const {
429  if (!ZeroSuggestEnabled(suggest_url,
430                          template_url_service_->GetDefaultSearchProvider(),
431                          current_page_classification_,
432                          template_url_service_->search_terms_data(), profile_))
433    return false;
434
435  // If we cannot send URLs, then only the MostVisited and Personalized
436  // variations can be shown.
437  if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
438      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
439    return false;
440
441  // Only show zero suggest for HTTP[S] pages.
442  // TODO(mariakhomenko): We may be able to expand this set to include pages
443  // with other schemes (e.g. chrome://). That may require improvements to
444  // the formatting of the verbatim result returned by MatchForCurrentURL().
445  if (!current_page_url.is_valid() ||
446      ((current_page_url.scheme() != url::kHttpScheme) &&
447      (current_page_url.scheme() != url::kHttpsScheme)))
448    return false;
449
450  return true;
451}
452
453void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
454  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
455    return;
456
457  std::string json_data = profile_->GetPrefs()->GetString(
458      prefs::kZeroSuggestCachedResults);
459  if (!json_data.empty()) {
460    scoped_ptr<base::Value> data(
461        SearchSuggestionParser::DeserializeJsonData(json_data));
462    if (data && ParseSuggestResults(*data.get(), false, &results_)) {
463      ConvertResultsToAutocompleteMatches();
464      results_from_cache_ = !matches_.empty();
465    }
466  }
467}
468