zero_suggest_provider.cc revision 1320f92c476a1ad9d19dba2a48c72b75566198e9
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "chrome/browser/autocomplete/zero_suggest_provider.h" 6 7#include "base/callback.h" 8#include "base/i18n/case_conversion.h" 9#include "base/json/json_string_value_serializer.h" 10#include "base/metrics/histogram.h" 11#include "base/metrics/user_metrics.h" 12#include "base/prefs/pref_service.h" 13#include "base/strings/string16.h" 14#include "base/strings/string_util.h" 15#include "base/strings/utf_string_conversions.h" 16#include "base/time/time.h" 17#include "chrome/browser/autocomplete/autocomplete_classifier.h" 18#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h" 19#include "chrome/browser/autocomplete/chrome_autocomplete_provider_client.h" 20#include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h" 21#include "chrome/browser/autocomplete/history_url_provider.h" 22#include "chrome/browser/history/top_sites.h" 23#include "chrome/browser/profiles/profile.h" 24#include "chrome/common/pref_names.h" 25#include "components/history/core/browser/history_types.h" 26#include "components/metrics/proto/omnibox_input_type.pb.h" 27#include "components/omnibox/autocomplete_input.h" 28#include "components/omnibox/autocomplete_match.h" 29#include "components/omnibox/autocomplete_provider_listener.h" 30#include "components/omnibox/omnibox_field_trial.h" 31#include "components/omnibox/search_provider.h" 32#include "components/pref_registry/pref_registry_syncable.h" 33#include "components/search_engines/template_url_service.h" 34#include "components/variations/variations_http_header_provider.h" 35#include "net/base/escape.h" 36#include "net/base/load_flags.h" 37#include "net/base/net_util.h" 38#include "net/http/http_request_headers.h" 39#include "net/url_request/url_fetcher.h" 40#include "net/url_request/url_request_status.h" 41#include "url/gurl.h" 42 43namespace { 44 45// TODO(hfung): The histogram code was copied and modified from 46// search_provider.cc. Refactor and consolidate the code. 47// We keep track in a histogram how many suggest requests we send, how 48// many suggest requests we invalidate (e.g., due to a user typing 49// another character), and how many replies we receive. 50// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! *** 51// (excluding the end-of-list enum value) 52// We do not want values of existing enums to change or else it screws 53// up the statistics. 54enum ZeroSuggestRequestsHistogramValue { 55 ZERO_SUGGEST_REQUEST_SENT = 1, 56 ZERO_SUGGEST_REQUEST_INVALIDATED, 57 ZERO_SUGGEST_REPLY_RECEIVED, 58 ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE 59}; 60 61void LogOmniboxZeroSuggestRequest( 62 ZeroSuggestRequestsHistogramValue request_value) { 63 UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value, 64 ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE); 65} 66 67// The maximum relevance of the top match from this provider. 68const int kDefaultVerbatimZeroSuggestRelevance = 1300; 69 70// Relevance value to use if it was not set explicitly by the server. 71const int kDefaultZeroSuggestRelevance = 100; 72 73} // namespace 74 75// static 76ZeroSuggestProvider* ZeroSuggestProvider::Create( 77 AutocompleteProviderListener* listener, 78 TemplateURLService* template_url_service, 79 Profile* profile) { 80 return new ZeroSuggestProvider(listener, template_url_service, profile); 81} 82 83// static 84void ZeroSuggestProvider::RegisterProfilePrefs( 85 user_prefs::PrefRegistrySyncable* registry) { 86 registry->RegisterStringPref( 87 prefs::kZeroSuggestCachedResults, 88 std::string(), 89 user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF); 90} 91 92void ZeroSuggestProvider::Start(const AutocompleteInput& input, 93 bool minimal_changes) { 94 matches_.clear(); 95 if (input.type() == metrics::OmniboxInputType::INVALID) 96 return; 97 98 Stop(true); 99 field_trial_triggered_ = false; 100 field_trial_triggered_in_session_ = false; 101 results_from_cache_ = false; 102 permanent_text_ = input.text(); 103 current_query_ = input.current_url().spec(); 104 current_page_classification_ = input.current_page_classification(); 105 current_url_match_ = MatchForCurrentURL(); 106 107 const TemplateURL* default_provider = 108 template_url_service_->GetDefaultSearchProvider(); 109 if (default_provider == NULL) 110 return; 111 112 base::string16 prefix; 113 TemplateURLRef::SearchTermsArgs search_term_args(prefix); 114 GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms( 115 search_term_args, template_url_service_->search_terms_data())); 116 if (!suggest_url.is_valid()) 117 return; 118 119 // No need to send the current page URL in personalized suggest field trial. 120 if (CanSendURL(input.current_url(), suggest_url, default_provider, 121 current_page_classification_, 122 template_url_service_->search_terms_data(), client_.get()) && 123 !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) { 124 // Update suggest_url to include the current_page_url. 125 search_term_args.current_page_url = current_query_; 126 suggest_url = GURL(default_provider->suggestions_url_ref(). 127 ReplaceSearchTerms( 128 search_term_args, 129 template_url_service_->search_terms_data())); 130 } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url, 131 input.current_url())) { 132 return; 133 } 134 135 done_ = false; 136 // TODO(jered): Consider adding locally-sourced zero-suggestions here too. 137 // These may be useful on the NTP or more relevant to the user than server 138 // suggestions, if based on local browsing history. 139 MaybeUseCachedSuggestions(); 140 Run(suggest_url); 141} 142 143void ZeroSuggestProvider::Stop(bool clear_cached_results) { 144 if (fetcher_) 145 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED); 146 fetcher_.reset(); 147 done_ = true; 148 149 if (clear_cached_results) { 150 // We do not call Clear() on |results_| to retain |verbatim_relevance| 151 // value in the |results_| object. |verbatim_relevance| is used at the 152 // beginning of the next StartZeroSuggest() call to determine the current 153 // url match relevance. 154 results_.suggest_results.clear(); 155 results_.navigation_results.clear(); 156 current_query_.clear(); 157 } 158} 159 160void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) { 161 if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) { 162 // Remove the deleted match from the cache, so it is not shown to the user 163 // again. Since we cannot remove just one result, blow away the cache. 164 profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, 165 std::string()); 166 } 167 BaseSearchProvider::DeleteMatch(match); 168} 169 170void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const { 171 BaseSearchProvider::AddProviderInfo(provider_info); 172 if (!results_.suggest_results.empty() || !results_.navigation_results.empty()) 173 provider_info->back().set_times_returned_results_in_session(1); 174} 175 176void ZeroSuggestProvider::ResetSession() { 177 // The user has started editing in the omnibox, so leave 178 // |field_trial_triggered_in_session_| unchanged and set 179 // |field_trial_triggered_| to false since zero suggest is inactive now. 180 field_trial_triggered_ = false; 181} 182 183ZeroSuggestProvider::ZeroSuggestProvider( 184 AutocompleteProviderListener* listener, 185 TemplateURLService* template_url_service, 186 Profile* profile) 187 : BaseSearchProvider(template_url_service, 188 scoped_ptr<AutocompleteProviderClient>( 189 new ChromeAutocompleteProviderClient(profile)), 190 AutocompleteProvider::TYPE_ZERO_SUGGEST), 191 listener_(listener), 192 profile_(profile), 193 results_from_cache_(false), 194 weak_ptr_factory_(this) { 195} 196 197ZeroSuggestProvider::~ZeroSuggestProvider() { 198} 199 200const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const { 201 // Zero suggest provider should not receive keyword results. 202 DCHECK(!is_keyword); 203 return template_url_service_->GetDefaultSearchProvider(); 204} 205 206const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const { 207 return AutocompleteInput( 208 base::string16(), base::string16::npos, base::string16(), 209 GURL(current_query_), current_page_classification_, true, false, false, 210 true, ChromeAutocompleteSchemeClassifier(profile_)); 211} 212 213bool ZeroSuggestProvider::ShouldAppendExtraParams( 214 const SearchSuggestionParser::SuggestResult& result) const { 215 // We always use the default provider for search, so append the params. 216 return true; 217} 218 219void ZeroSuggestProvider::RecordDeletionResult(bool success) { 220 if (success) { 221 base::RecordAction( 222 base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success")); 223 } else { 224 base::RecordAction( 225 base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure")); 226 } 227} 228 229void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) { 230 DCHECK(!done_); 231 DCHECK_EQ(fetcher_.get(), source); 232 233 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED); 234 235 bool results_updated = false; 236 if (source->GetStatus().is_success() && source->GetResponseCode() == 200) { 237 std::string json_data = SearchSuggestionParser::ExtractJsonData(source); 238 scoped_ptr<base::Value> data( 239 SearchSuggestionParser::DeserializeJsonData(json_data)); 240 if (data) { 241 if (StoreSuggestionResponse(json_data, *data)) 242 return; 243 results_updated = ParseSuggestResults( 244 *data, kDefaultZeroSuggestRelevance, false, &results_); 245 } 246 } 247 fetcher_.reset(); 248 done_ = true; 249 ConvertResultsToAutocompleteMatches(); 250 listener_->OnProviderUpdate(results_updated); 251} 252 253bool ZeroSuggestProvider::StoreSuggestionResponse( 254 const std::string& json_data, 255 const base::Value& parsed_data) { 256 if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() || 257 json_data.empty()) 258 return false; 259 profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data); 260 261 // If we received an empty result list, we should update the display, as it 262 // may be showing cached results that should not be shown. 263 const base::ListValue* root_list = NULL; 264 const base::ListValue* results_list = NULL; 265 if (parsed_data.GetAsList(&root_list) && 266 root_list->GetList(1, &results_list) && 267 results_list->empty()) 268 return false; 269 270 // We are finished with the request and want to bail early. 271 if (results_from_cache_) 272 done_ = true; 273 274 return results_from_cache_; 275} 276 277void ZeroSuggestProvider::AddSuggestResultsToMap( 278 const SearchSuggestionParser::SuggestResults& results, 279 MatchMap* map) { 280 for (size_t i = 0; i < results.size(); ++i) 281 AddMatchToMap(results[i], std::string(), i, false, false, map); 282} 283 284AutocompleteMatch ZeroSuggestProvider::NavigationToMatch( 285 const SearchSuggestionParser::NavigationResult& navigation) { 286 AutocompleteMatch match(this, navigation.relevance(), false, 287 navigation.type()); 288 match.destination_url = navigation.url(); 289 290 // Zero suggest results should always omit protocols and never appear bold. 291 const std::string languages( 292 profile_->GetPrefs()->GetString(prefs::kAcceptLanguages)); 293 match.contents = net::FormatUrl(navigation.url(), languages, 294 net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL); 295 match.fill_into_edit += 296 AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(), 297 match.contents, ChromeAutocompleteSchemeClassifier(profile_)); 298 299 AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0, 300 match.contents.length(), ACMatchClassification::URL, 301 &match.contents_class); 302 303 match.description = 304 AutocompleteMatch::SanitizeString(navigation.description()); 305 AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0, 306 match.description.length(), ACMatchClassification::NONE, 307 &match.description_class); 308 return match; 309} 310 311void ZeroSuggestProvider::Run(const GURL& suggest_url) { 312 const int kFetcherID = 1; 313 fetcher_.reset( 314 net::URLFetcher::Create(kFetcherID, 315 suggest_url, 316 net::URLFetcher::GET, this)); 317 fetcher_->SetRequestContext(profile_->GetRequestContext()); 318 fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES); 319 // Add Chrome experiment state to the request headers. 320 net::HttpRequestHeaders headers; 321 variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders( 322 fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers); 323 fetcher_->SetExtraRequestHeaders(headers.ToString()); 324 fetcher_->Start(); 325 326 if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) { 327 most_visited_urls_.clear(); 328 history::TopSites* ts = profile_->GetTopSites(); 329 if (ts) { 330 ts->GetMostVisitedURLs( 331 base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable, 332 weak_ptr_factory_.GetWeakPtr()), false); 333 } 334 } 335 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT); 336} 337 338void ZeroSuggestProvider::OnMostVisitedUrlsAvailable( 339 const history::MostVisitedURLList& urls) { 340 most_visited_urls_ = urls; 341} 342 343void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() { 344 matches_.clear(); 345 346 const TemplateURL* default_provider = 347 template_url_service_->GetDefaultSearchProvider(); 348 // Fail if we can't set the clickthrough URL for query suggestions. 349 if (default_provider == NULL || !default_provider->SupportsReplacement( 350 template_url_service_->search_terms_data())) 351 return; 352 353 MatchMap map; 354 AddSuggestResultsToMap(results_.suggest_results, &map); 355 356 const int num_query_results = map.size(); 357 const int num_nav_results = results_.navigation_results.size(); 358 const int num_results = num_query_results + num_nav_results; 359 UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results); 360 UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results); 361 UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results); 362 363 // Show Most Visited results after ZeroSuggest response is received. 364 if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) { 365 if (!current_url_match_.destination_url.is_valid()) 366 return; 367 matches_.push_back(current_url_match_); 368 int relevance = 600; 369 if (num_results > 0) { 370 UMA_HISTOGRAM_COUNTS( 371 "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual", 372 most_visited_urls_.size()); 373 } 374 const base::string16 current_query_string16( 375 base::ASCIIToUTF16(current_query_)); 376 const std::string languages( 377 profile_->GetPrefs()->GetString(prefs::kAcceptLanguages)); 378 for (size_t i = 0; i < most_visited_urls_.size(); i++) { 379 const history::MostVisitedURL& url = most_visited_urls_[i]; 380 SearchSuggestionParser::NavigationResult nav( 381 ChromeAutocompleteSchemeClassifier(profile_), url.url, 382 AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false, 383 relevance, true, current_query_string16, languages); 384 matches_.push_back(NavigationToMatch(nav)); 385 --relevance; 386 } 387 return; 388 } 389 390 if (num_results == 0) 391 return; 392 393 // TODO(jered): Rip this out once the first match is decoupled from the 394 // current typing in the omnibox. 395 matches_.push_back(current_url_match_); 396 397 for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it) 398 matches_.push_back(it->second); 399 400 const SearchSuggestionParser::NavigationResults& nav_results( 401 results_.navigation_results); 402 for (SearchSuggestionParser::NavigationResults::const_iterator it( 403 nav_results.begin()); it != nav_results.end(); ++it) 404 matches_.push_back(NavigationToMatch(*it)); 405} 406 407AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() { 408 AutocompleteMatch match; 409 AutocompleteClassifierFactory::GetForProfile(profile_)->Classify( 410 permanent_text_, false, true, current_page_classification_, &match, NULL); 411 match.is_history_what_you_typed_match = false; 412 match.allowed_to_be_default_match = true; 413 414 // The placeholder suggestion for the current URL has high relevance so 415 // that it is in the first suggestion slot and inline autocompleted. It 416 // gets dropped as soon as the user types something. 417 match.relevance = GetVerbatimRelevance(); 418 419 return match; 420} 421 422int ZeroSuggestProvider::GetVerbatimRelevance() const { 423 return results_.verbatim_relevance >= 0 ? 424 results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance; 425} 426 427bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL( 428 const GURL& suggest_url, 429 const GURL& current_page_url) const { 430 if (!ZeroSuggestEnabled(suggest_url, 431 template_url_service_->GetDefaultSearchProvider(), 432 current_page_classification_, 433 template_url_service_->search_terms_data(), 434 client_.get())) 435 return false; 436 437 // If we cannot send URLs, then only the MostVisited and Personalized 438 // variations can be shown. 439 if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() && 440 !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) 441 return false; 442 443 // Only show zero suggest for HTTP[S] pages. 444 // TODO(mariakhomenko): We may be able to expand this set to include pages 445 // with other schemes (e.g. chrome://). That may require improvements to 446 // the formatting of the verbatim result returned by MatchForCurrentURL(). 447 if (!current_page_url.is_valid() || 448 ((current_page_url.scheme() != url::kHttpScheme) && 449 (current_page_url.scheme() != url::kHttpsScheme))) 450 return false; 451 452 return true; 453} 454 455void ZeroSuggestProvider::MaybeUseCachedSuggestions() { 456 if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) 457 return; 458 459 std::string json_data = profile_->GetPrefs()->GetString( 460 prefs::kZeroSuggestCachedResults); 461 if (!json_data.empty()) { 462 scoped_ptr<base::Value> data( 463 SearchSuggestionParser::DeserializeJsonData(json_data)); 464 if (data && ParseSuggestResults( 465 *data, kDefaultZeroSuggestRelevance, false, &results_)) { 466 ConvertResultsToAutocompleteMatches(); 467 results_from_cache_ = !matches_.empty(); 468 } 469 } 470} 471