zero_suggest_provider.cc revision 03b57e008b61dfcb1fbad3aea950ae0e001748b0 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/metrics/user_metrics.h"
12#include "base/prefs/pref_service.h"
13#include "base/strings/string16.h"
14#include "base/strings/string_util.h"
15#include "base/strings/utf_string_conversions.h"
16#include "base/time/time.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier.h"
18#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
19#include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
20#include "chrome/browser/autocomplete/history_url_provider.h"
21#include "chrome/browser/autocomplete/search_provider.h"
22#include "chrome/browser/history/history_types.h"
23#include "chrome/browser/history/top_sites.h"
24#include "chrome/browser/profiles/profile.h"
25#include "chrome/common/pref_names.h"
26#include "components/metrics/proto/omnibox_input_type.pb.h"
27#include "components/omnibox/autocomplete_input.h"
28#include "components/omnibox/autocomplete_match.h"
29#include "components/omnibox/autocomplete_provider_listener.h"
30#include "components/omnibox/omnibox_field_trial.h"
31#include "components/pref_registry/pref_registry_syncable.h"
32#include "components/search_engines/template_url_service.h"
33#include "components/variations/variations_http_header_provider.h"
34#include "net/base/escape.h"
35#include "net/base/load_flags.h"
36#include "net/base/net_util.h"
37#include "net/http/http_request_headers.h"
38#include "net/url_request/url_fetcher.h"
39#include "net/url_request/url_request_status.h"
40#include "url/gurl.h"
41
42namespace {
43
44// TODO(hfung): The histogram code was copied and modified from
45// search_provider.cc.  Refactor and consolidate the code.
46// We keep track in a histogram how many suggest requests we send, how
47// many suggest requests we invalidate (e.g., due to a user typing
48// another character), and how many replies we receive.
49// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
50//     (excluding the end-of-list enum value)
51// We do not want values of existing enums to change or else it screws
52// up the statistics.
53enum ZeroSuggestRequestsHistogramValue {
54  ZERO_SUGGEST_REQUEST_SENT = 1,
55  ZERO_SUGGEST_REQUEST_INVALIDATED,
56  ZERO_SUGGEST_REPLY_RECEIVED,
57  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
58};
59
60void LogOmniboxZeroSuggestRequest(
61    ZeroSuggestRequestsHistogramValue request_value) {
62  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
63                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
64}
65
66// The maximum relevance of the top match from this provider.
67const int kDefaultVerbatimZeroSuggestRelevance = 1300;
68
69// Relevance value to use if it was not set explicitly by the server.
70const int kDefaultZeroSuggestRelevance = 100;
71
72}  // namespace
73
74// static
75ZeroSuggestProvider* ZeroSuggestProvider::Create(
76    AutocompleteProviderListener* listener,
77    TemplateURLService* template_url_service,
78    Profile* profile) {
79  return new ZeroSuggestProvider(listener, template_url_service, profile);
80}
81
82// static
83void ZeroSuggestProvider::RegisterProfilePrefs(
84    user_prefs::PrefRegistrySyncable* registry) {
85  registry->RegisterStringPref(
86      prefs::kZeroSuggestCachedResults,
87      std::string(),
88      user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
89}
90
91void ZeroSuggestProvider::Start(const AutocompleteInput& input,
92                                bool minimal_changes) {
93  matches_.clear();
94  if (input.type() == metrics::OmniboxInputType::INVALID)
95    return;
96
97  Stop(true);
98  field_trial_triggered_ = false;
99  field_trial_triggered_in_session_ = false;
100  results_from_cache_ = false;
101  permanent_text_ = input.text();
102  current_query_ = input.current_url().spec();
103  current_page_classification_ = input.current_page_classification();
104  current_url_match_ = MatchForCurrentURL();
105
106  const TemplateURL* default_provider =
107     template_url_service_->GetDefaultSearchProvider();
108  if (default_provider == NULL)
109    return;
110
111  base::string16 prefix;
112  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
113  GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
114      search_term_args, template_url_service_->search_terms_data()));
115  if (!suggest_url.is_valid())
116    return;
117
118  // No need to send the current page URL in personalized suggest field trial.
119  if (CanSendURL(input.current_url(), suggest_url, default_provider,
120                 current_page_classification_,
121                 template_url_service_->search_terms_data(), profile_) &&
122      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
123    // Update suggest_url to include the current_page_url.
124    search_term_args.current_page_url = current_query_;
125    suggest_url = GURL(default_provider->suggestions_url_ref().
126                       ReplaceSearchTerms(
127                           search_term_args,
128                           template_url_service_->search_terms_data()));
129  } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
130                                                  input.current_url())) {
131    return;
132  }
133
134  done_ = false;
135  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
136  // These may be useful on the NTP or more relevant to the user than server
137  // suggestions, if based on local browsing history.
138  MaybeUseCachedSuggestions();
139  Run(suggest_url);
140}
141
142void ZeroSuggestProvider::Stop(bool clear_cached_results) {
143  if (fetcher_)
144    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
145  fetcher_.reset();
146  done_ = true;
147
148  if (clear_cached_results) {
149    // We do not call Clear() on |results_| to retain |verbatim_relevance|
150    // value in the |results_| object. |verbatim_relevance| is used at the
151    // beginning of the next StartZeroSuggest() call to determine the current
152    // url match relevance.
153    results_.suggest_results.clear();
154    results_.navigation_results.clear();
155    current_query_.clear();
156  }
157}
158
159void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
160  if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
161    // Remove the deleted match from the cache, so it is not shown to the user
162    // again. Since we cannot remove just one result, blow away the cache.
163    profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
164                                    std::string());
165  }
166  BaseSearchProvider::DeleteMatch(match);
167}
168
169void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
170  BaseSearchProvider::AddProviderInfo(provider_info);
171  if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
172    provider_info->back().set_times_returned_results_in_session(1);
173}
174
175void ZeroSuggestProvider::ResetSession() {
176  // The user has started editing in the omnibox, so leave
177  // |field_trial_triggered_in_session_| unchanged and set
178  // |field_trial_triggered_| to false since zero suggest is inactive now.
179  field_trial_triggered_ = false;
180}
181
182ZeroSuggestProvider::ZeroSuggestProvider(
183  AutocompleteProviderListener* listener,
184  TemplateURLService* template_url_service,
185  Profile* profile)
186    : BaseSearchProvider(template_url_service, profile,
187                         AutocompleteProvider::TYPE_ZERO_SUGGEST),
188      listener_(listener),
189      results_from_cache_(false),
190      weak_ptr_factory_(this) {
191}
192
193ZeroSuggestProvider::~ZeroSuggestProvider() {
194}
195
196const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
197  // Zero suggest provider should not receive keyword results.
198  DCHECK(!is_keyword);
199  return template_url_service_->GetDefaultSearchProvider();
200}
201
202const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
203  return AutocompleteInput(
204      base::string16(), base::string16::npos, base::string16(),
205      GURL(current_query_), current_page_classification_, true, false, false,
206      true, ChromeAutocompleteSchemeClassifier(profile_));
207}
208
209bool ZeroSuggestProvider::ShouldAppendExtraParams(
210      const SearchSuggestionParser::SuggestResult& result) const {
211  // We always use the default provider for search, so append the params.
212  return true;
213}
214
215void ZeroSuggestProvider::RecordDeletionResult(bool success) {
216  if (success) {
217    base::RecordAction(
218        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
219  } else {
220    base::RecordAction(
221        base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
222  }
223}
224
225void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
226  DCHECK(!done_);
227  DCHECK_EQ(fetcher_.get(), source);
228
229  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
230
231  bool results_updated = false;
232  if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
233    std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
234    scoped_ptr<base::Value> data(
235        SearchSuggestionParser::DeserializeJsonData(json_data));
236    if (data) {
237      if (StoreSuggestionResponse(json_data, *data))
238        return;
239      results_updated = ParseSuggestResults(
240          *data, kDefaultZeroSuggestRelevance, false, &results_);
241    }
242  }
243  fetcher_.reset();
244  done_ = true;
245  ConvertResultsToAutocompleteMatches();
246  listener_->OnProviderUpdate(results_updated);
247}
248
249bool ZeroSuggestProvider::StoreSuggestionResponse(
250    const std::string& json_data,
251    const base::Value& parsed_data) {
252  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
253      json_data.empty())
254    return false;
255  profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
256
257  // If we received an empty result list, we should update the display, as it
258  // may be showing cached results that should not be shown.
259  const base::ListValue* root_list = NULL;
260  const base::ListValue* results_list = NULL;
261  if (parsed_data.GetAsList(&root_list) &&
262      root_list->GetList(1, &results_list) &&
263      results_list->empty())
264    return false;
265
266  // We are finished with the request and want to bail early.
267  if (results_from_cache_)
268    done_ = true;
269
270  return results_from_cache_;
271}
272
273void ZeroSuggestProvider::AddSuggestResultsToMap(
274    const SearchSuggestionParser::SuggestResults& results,
275    MatchMap* map) {
276  for (size_t i = 0; i < results.size(); ++i)
277    AddMatchToMap(results[i], std::string(), i, false, false, map);
278}
279
280AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
281    const SearchSuggestionParser::NavigationResult& navigation) {
282  AutocompleteMatch match(this, navigation.relevance(), false,
283                          navigation.type());
284  match.destination_url = navigation.url();
285
286  // Zero suggest results should always omit protocols and never appear bold.
287  const std::string languages(
288      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
289  match.contents = net::FormatUrl(navigation.url(), languages,
290      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
291  match.fill_into_edit +=
292      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
293          match.contents, ChromeAutocompleteSchemeClassifier(profile_));
294
295  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
296      match.contents.length(), ACMatchClassification::URL,
297      &match.contents_class);
298
299  match.description =
300      AutocompleteMatch::SanitizeString(navigation.description());
301  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
302      match.description.length(), ACMatchClassification::NONE,
303      &match.description_class);
304  return match;
305}
306
307void ZeroSuggestProvider::Run(const GURL& suggest_url) {
308  const int kFetcherID = 1;
309  fetcher_.reset(
310      net::URLFetcher::Create(kFetcherID,
311          suggest_url,
312          net::URLFetcher::GET, this));
313  fetcher_->SetRequestContext(profile_->GetRequestContext());
314  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
315  // Add Chrome experiment state to the request headers.
316  net::HttpRequestHeaders headers;
317  variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
318      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
319  fetcher_->SetExtraRequestHeaders(headers.ToString());
320  fetcher_->Start();
321
322  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
323    most_visited_urls_.clear();
324    history::TopSites* ts = profile_->GetTopSites();
325    if (ts) {
326      ts->GetMostVisitedURLs(
327          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
328                     weak_ptr_factory_.GetWeakPtr()), false);
329    }
330  }
331  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
332}
333
334void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
335    const history::MostVisitedURLList& urls) {
336  most_visited_urls_ = urls;
337}
338
339void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
340  matches_.clear();
341
342  const TemplateURL* default_provider =
343      template_url_service_->GetDefaultSearchProvider();
344  // Fail if we can't set the clickthrough URL for query suggestions.
345  if (default_provider == NULL || !default_provider->SupportsReplacement(
346          template_url_service_->search_terms_data()))
347    return;
348
349  MatchMap map;
350  AddSuggestResultsToMap(results_.suggest_results, &map);
351
352  const int num_query_results = map.size();
353  const int num_nav_results = results_.navigation_results.size();
354  const int num_results = num_query_results + num_nav_results;
355  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
356  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
357  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
358
359  // Show Most Visited results after ZeroSuggest response is received.
360  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
361    if (!current_url_match_.destination_url.is_valid())
362      return;
363    matches_.push_back(current_url_match_);
364    int relevance = 600;
365    if (num_results > 0) {
366      UMA_HISTOGRAM_COUNTS(
367          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
368          most_visited_urls_.size());
369    }
370    const base::string16 current_query_string16(
371        base::ASCIIToUTF16(current_query_));
372    const std::string languages(
373        profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
374    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
375      const history::MostVisitedURL& url = most_visited_urls_[i];
376      SearchSuggestionParser::NavigationResult nav(
377          ChromeAutocompleteSchemeClassifier(profile_), url.url,
378          AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
379          relevance, true, current_query_string16, languages);
380      matches_.push_back(NavigationToMatch(nav));
381      --relevance;
382    }
383    return;
384  }
385
386  if (num_results == 0)
387    return;
388
389  // TODO(jered): Rip this out once the first match is decoupled from the
390  // current typing in the omnibox.
391  matches_.push_back(current_url_match_);
392
393  for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
394    matches_.push_back(it->second);
395
396  const SearchSuggestionParser::NavigationResults& nav_results(
397      results_.navigation_results);
398  for (SearchSuggestionParser::NavigationResults::const_iterator it(
399           nav_results.begin()); it != nav_results.end(); ++it)
400    matches_.push_back(NavigationToMatch(*it));
401}
402
403AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
404  AutocompleteMatch match;
405  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
406      permanent_text_, false, true, current_page_classification_, &match, NULL);
407  match.is_history_what_you_typed_match = false;
408  match.allowed_to_be_default_match = true;
409
410  // The placeholder suggestion for the current URL has high relevance so
411  // that it is in the first suggestion slot and inline autocompleted. It
412  // gets dropped as soon as the user types something.
413  match.relevance = GetVerbatimRelevance();
414
415  return match;
416}
417
418int ZeroSuggestProvider::GetVerbatimRelevance() const {
419  return results_.verbatim_relevance >= 0 ?
420      results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
421}
422
423bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
424    const GURL& suggest_url,
425    const GURL& current_page_url) const {
426  if (!ZeroSuggestEnabled(suggest_url,
427                          template_url_service_->GetDefaultSearchProvider(),
428                          current_page_classification_,
429                          template_url_service_->search_terms_data(), profile_))
430    return false;
431
432  // If we cannot send URLs, then only the MostVisited and Personalized
433  // variations can be shown.
434  if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
435      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
436    return false;
437
438  // Only show zero suggest for HTTP[S] pages.
439  // TODO(mariakhomenko): We may be able to expand this set to include pages
440  // with other schemes (e.g. chrome://). That may require improvements to
441  // the formatting of the verbatim result returned by MatchForCurrentURL().
442  if (!current_page_url.is_valid() ||
443      ((current_page_url.scheme() != url::kHttpScheme) &&
444      (current_page_url.scheme() != url::kHttpsScheme)))
445    return false;
446
447  return true;
448}
449
450void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
451  if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
452    return;
453
454  std::string json_data = profile_->GetPrefs()->GetString(
455      prefs::kZeroSuggestCachedResults);
456  if (!json_data.empty()) {
457    scoped_ptr<base::Value> data(
458        SearchSuggestionParser::DeserializeJsonData(json_data));
459    if (data && ParseSuggestResults(
460            *data, kDefaultZeroSuggestRelevance, false, &results_)) {
461      ConvertResultsToAutocompleteMatches();
462      results_from_cache_ = !matches_.empty();
463    }
464  }
465}
466