zero_suggest_provider.cc revision 424c4d7b64af9d0d8fd9624f381f469654d5e3d2 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_classifier.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
18#include "chrome/browser/autocomplete/autocomplete_input.h"
19#include "chrome/browser/autocomplete/autocomplete_match.h"
20#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
21#include "chrome/browser/autocomplete/history_url_provider.h"
22#include "chrome/browser/autocomplete/search_provider.h"
23#include "chrome/browser/autocomplete/url_prefix.h"
24#include "chrome/browser/google/google_util.h"
25#include "chrome/browser/metrics/variations/variations_http_header_provider.h"
26#include "chrome/browser/omnibox/omnibox_field_trial.h"
27#include "chrome/browser/profiles/profile.h"
28#include "chrome/browser/search/search.h"
29#include "chrome/browser/search_engines/template_url_service.h"
30#include "chrome/browser/search_engines/template_url_service_factory.h"
31#include "chrome/browser/sync/profile_sync_service.h"
32#include "chrome/browser/sync/profile_sync_service_factory.h"
33#include "chrome/common/net/url_fixer_upper.h"
34#include "chrome/common/pref_names.h"
35#include "chrome/common/url_constants.h"
36#include "net/base/escape.h"
37#include "net/base/load_flags.h"
38#include "net/base/net_util.h"
39#include "net/http/http_request_headers.h"
40#include "net/http/http_response_headers.h"
41#include "net/url_request/url_fetcher.h"
42#include "net/url_request/url_request_status.h"
43#include "url/gurl.h"
44
45namespace {
46
47// TODO(hfung): The histogram code was copied and modified from
48// search_provider.cc.  Refactor and consolidate the code.
49// We keep track in a histogram how many suggest requests we send, how
50// many suggest requests we invalidate (e.g., due to a user typing
51// another character), and how many replies we receive.
52// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
53//     (excluding the end-of-list enum value)
54// We do not want values of existing enums to change or else it screws
55// up the statistics.
56enum ZeroSuggestRequestsHistogramValue {
57  ZERO_SUGGEST_REQUEST_SENT = 1,
58  ZERO_SUGGEST_REQUEST_INVALIDATED,
59  ZERO_SUGGEST_REPLY_RECEIVED,
60  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
61};
62
63void LogOmniboxZeroSuggestRequest(
64    ZeroSuggestRequestsHistogramValue request_value) {
65  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
66                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
67}
68
69// The maximum relevance of the top match from this provider.
70const int kDefaultVerbatimZeroSuggestRelevance = 1300;
71
72// Relevance value to use if it was not set explicitly by the server.
73const int kDefaultZeroSuggestRelevance = 100;
74
75}  // namespace
76
77// static
78ZeroSuggestProvider* ZeroSuggestProvider::Create(
79    AutocompleteProviderListener* listener,
80    Profile* profile) {
81  return new ZeroSuggestProvider(listener, profile);
82}
83
84void ZeroSuggestProvider::Start(const AutocompleteInput& input,
85                                bool /*minimal_changes*/) {
86}
87
88void ZeroSuggestProvider::Stop(bool clear_cached_results) {
89  if (have_pending_request_)
90    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
91  have_pending_request_ = false;
92  fetcher_.reset();
93  done_ = true;
94  if (clear_cached_results) {
95    query_matches_map_.clear();
96    navigation_results_.clear();
97    current_query_.clear();
98    matches_.clear();
99  }
100}
101
102void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
103  provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
104  metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
105  new_entry.set_provider(AsOmniboxEventProviderType());
106  new_entry.set_provider_done(done_);
107  std::vector<uint32> field_trial_hashes;
108  OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
109  for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
110    if (field_trial_triggered_)
111      new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
112    if (field_trial_triggered_in_session_) {
113      new_entry.mutable_field_trial_triggered_in_session()->Add(
114          field_trial_hashes[i]);
115     }
116  }
117}
118
119void ZeroSuggestProvider::ResetSession() {
120  // The user has started editing in the omnibox, so leave
121  // |field_trial_triggered_in_session_| unchanged and set
122  // |field_trial_triggered_| to false since zero suggest is inactive now.
123  field_trial_triggered_ = false;
124  Stop(true);
125}
126
127void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
128  have_pending_request_ = false;
129  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
130
131  std::string json_data;
132  source->GetResponseAsString(&json_data);
133  const bool request_succeeded =
134      source->GetStatus().is_success() && source->GetResponseCode() == 200;
135
136  bool have_results = false;
137  if (request_succeeded) {
138    JSONStringValueSerializer deserializer(json_data);
139    deserializer.set_allow_trailing_comma(true);
140    scoped_ptr<Value> data(deserializer.Deserialize(NULL, NULL));
141    if (data.get()) {
142      ParseSuggestResults(*data.get());
143      have_results = !query_matches_map_.empty() ||
144          !navigation_results_.empty();
145    }
146  }
147  done_ = true;
148
149  if (have_results) {
150    ConvertResultsToAutocompleteMatches();
151    listener_->OnProviderUpdate(true);
152  }
153}
154
155void ZeroSuggestProvider::StartZeroSuggest(
156    const GURL& url,
157    AutocompleteInput::PageClassification page_classification,
158    const string16& permanent_text) {
159  Stop(true);
160  field_trial_triggered_ = false;
161  field_trial_triggered_in_session_ = false;
162  if (!ShouldRunZeroSuggest(url))
163    return;
164  verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance;
165  done_ = false;
166  permanent_text_ = permanent_text;
167  current_query_ = url.spec();
168  current_page_classification_ = page_classification;
169  current_url_match_ = MatchForCurrentURL();
170  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
171  // These may be useful on the NTP or more relevant to the user than server
172  // suggestions, if based on local browsing history.
173  Run();
174}
175
176ZeroSuggestProvider::ZeroSuggestProvider(
177  AutocompleteProviderListener* listener,
178  Profile* profile)
179    : AutocompleteProvider(listener, profile,
180          AutocompleteProvider::TYPE_ZERO_SUGGEST),
181      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
182      have_pending_request_(false),
183      verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance),
184      field_trial_triggered_(false),
185      field_trial_triggered_in_session_(false) {
186}
187
188ZeroSuggestProvider::~ZeroSuggestProvider() {
189}
190
191bool ZeroSuggestProvider::ShouldRunZeroSuggest(const GURL& url) const {
192  if (!ShouldSendURL(url))
193    return false;
194
195  // Don't run if there's no profile or in incognito mode.
196  if (profile_ == NULL || profile_->IsOffTheRecord())
197    return false;
198
199  // Don't run if we can't get preferences or search suggest is not enabled.
200  PrefService* prefs = profile_->GetPrefs();
201  if (prefs == NULL || !prefs->GetBoolean(prefs::kSearchSuggestEnabled))
202    return false;
203
204  ProfileSyncService* service =
205      ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile_);
206  browser_sync::SyncPrefs sync_prefs(prefs);
207  // The user has needs to have Chrome Sync enabled (for permissions to
208  // transmit their current URL) and be in the field trial.
209  if (!OmniboxFieldTrial::InZeroSuggestFieldTrial() ||
210      service == NULL ||
211      !service->IsSyncEnabledAndLoggedIn() ||
212      !sync_prefs.HasKeepEverythingSynced()) {
213    return false;
214  }
215  return true;
216}
217
218bool ZeroSuggestProvider::ShouldSendURL(const GURL& url) const {
219  if (!url.is_valid())
220    return false;
221
222  // Only allow HTTP URLs or Google HTTPS URLs (including Google search
223  // result pages).  For the latter case, Google was already sent the HTTPS
224  // URLs when requesting the page, so the information is just re-sent.
225  return (url.scheme() == chrome::kHttpScheme) ||
226      google_util::IsGoogleDomainUrl(url, google_util::ALLOW_SUBDOMAIN,
227                                     google_util::ALLOW_NON_STANDARD_PORTS);
228}
229
230void ZeroSuggestProvider::FillResults(
231    const Value& root_val,
232    int* verbatim_relevance,
233    SearchProvider::SuggestResults* suggest_results,
234    SearchProvider::NavigationResults* navigation_results) {
235  string16 query;
236  const ListValue* root_list = NULL;
237  const ListValue* results = NULL;
238  const ListValue* relevances = NULL;
239  // The response includes the query, which should be empty for ZeroSuggest
240  // responses.
241  if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
242      (!query.empty()) || !root_list->GetList(1, &results))
243    return;
244
245  // 3rd element: Description list.
246  const ListValue* descriptions = NULL;
247  root_list->GetList(2, &descriptions);
248
249  // 4th element: Disregard the query URL list for now.
250
251  // Reset suggested relevance information from the provider.
252  *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance;
253
254  // 5th element: Optional key-value pairs from the Suggest server.
255  const ListValue* types = NULL;
256  const DictionaryValue* extras = NULL;
257  if (root_list->GetDictionary(4, &extras)) {
258    extras->GetList("google:suggesttype", &types);
259
260    // Discard this list if its size does not match that of the suggestions.
261    if (extras->GetList("google:suggestrelevance", &relevances) &&
262        relevances->GetSize() != results->GetSize())
263      relevances = NULL;
264    extras->GetInteger("google:verbatimrelevance", verbatim_relevance);
265
266    // Check if the active suggest field trial (if any) has triggered.
267    bool triggered = false;
268    extras->GetBoolean("google:fieldtrialtriggered", &triggered);
269    field_trial_triggered_ |= triggered;
270    field_trial_triggered_in_session_ |= triggered;
271  }
272
273  // Clear the previous results now that new results are available.
274  suggest_results->clear();
275  navigation_results->clear();
276
277  string16 result, title;
278  std::string type;
279  for (size_t index = 0; results->GetString(index, &result); ++index) {
280    // Google search may return empty suggestions for weird input characters,
281    // they make no sense at all and can cause problems in our code.
282    if (result.empty())
283      continue;
284
285    int relevance = kDefaultZeroSuggestRelevance;
286
287    // Apply valid suggested relevance scores; discard invalid lists.
288    if (relevances != NULL && !relevances->GetInteger(index, &relevance))
289      relevances = NULL;
290    if (types && types->GetString(index, &type) && (type == "NAVIGATION")) {
291      // Do not blindly trust the URL coming from the server to be valid.
292      GURL url(URLFixerUpper::FixupURL(UTF16ToUTF8(result), std::string()));
293      if (url.is_valid()) {
294        if (descriptions != NULL)
295          descriptions->GetString(index, &title);
296        navigation_results->push_back(SearchProvider::NavigationResult(
297            *this, url, title, false, relevance, relevances != NULL));
298      }
299    } else {
300      suggest_results->push_back(SearchProvider::SuggestResult(
301          result, false, relevance, relevances != NULL, false));
302    }
303  }
304}
305
306void ZeroSuggestProvider::AddSuggestResultsToMap(
307    const SearchProvider::SuggestResults& results,
308    const TemplateURL* template_url,
309    SearchProvider::MatchMap* map) {
310  for (size_t i = 0; i < results.size(); ++i) {
311    AddMatchToMap(results[i].relevance(), AutocompleteMatchType::SEARCH_SUGGEST,
312                  template_url, results[i].suggestion(), i, map);
313  }
314}
315
316void ZeroSuggestProvider::AddMatchToMap(int relevance,
317                                        AutocompleteMatch::Type type,
318                                        const TemplateURL* template_url,
319                                        const string16& query_string,
320                                        int accepted_suggestion,
321                                        SearchProvider::MatchMap* map) {
322  // Pass in query_string as the input_text since we don't want any bolding.
323  // TODO(samarth|melevin): use the actual omnibox margin here as well instead
324  // of passing in -1.
325  AutocompleteMatch match = SearchProvider::CreateSearchSuggestion(
326      this, relevance, type, template_url, query_string, query_string,
327      AutocompleteInput(), false, accepted_suggestion, -1, true);
328  if (!match.destination_url.is_valid())
329    return;
330
331  // Try to add |match| to |map|.  If a match for |query_string| is already in
332  // |map|, replace it if |match| is more relevant.
333  // NOTE: Keep this ToLower() call in sync with url_database.cc.
334  const std::pair<SearchProvider::MatchMap::iterator, bool> i(map->insert(
335      std::make_pair(base::i18n::ToLower(query_string), match)));
336  // NOTE: We purposefully do a direct relevance comparison here instead of
337  // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items added
338  // first" rather than "items alphabetically first" when the scores are equal.
339  // The only case this matters is when a user has results with the same score
340  // that differ only by capitalization; because the history system returns
341  // results sorted by recency, this means we'll pick the most recent such
342  // result even if the precision of our relevance score is too low to
343  // distinguish the two.
344  if (!i.second && (match.relevance > i.first->second.relevance))
345    i.first->second = match;
346}
347
348AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
349    const SearchProvider::NavigationResult& navigation) {
350  AutocompleteMatch match(this, navigation.relevance(), false,
351                          AutocompleteMatchType::NAVSUGGEST);
352  match.destination_url = navigation.url();
353
354  const std::string languages(
355      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
356  match.contents = net::FormatUrl(navigation.url(), languages,
357      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
358  match.fill_into_edit +=
359      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
360          match.contents);
361
362  AutocompleteMatch::ClassifyLocationInString(string16::npos, 0,
363      match.contents.length(), ACMatchClassification::URL,
364      &match.contents_class);
365
366  match.description =
367      AutocompleteMatch::SanitizeString(navigation.description());
368  AutocompleteMatch::ClassifyLocationInString(string16::npos, 0,
369      match.description.length(), ACMatchClassification::NONE,
370      &match.description_class);
371  return match;
372}
373
374void ZeroSuggestProvider::Run() {
375  have_pending_request_ = false;
376  const int kFetcherID = 1;
377
378  const TemplateURL* default_provider =
379     template_url_service_->GetDefaultSearchProvider();
380  // TODO(hfung): Generalize if the default provider supports zero suggest.
381  // Only make the request if we know that the provider supports zero suggest
382  // (currently only the prepopulated Google provider).
383  if (default_provider == NULL || !default_provider->SupportsReplacement() ||
384      default_provider->prepopulate_id() != 1) {
385    Stop(true);
386    return;
387  }
388  string16 prefix;
389  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
390  search_term_args.zero_prefix_url = current_query_;
391  std::string req_url = default_provider->suggestions_url_ref().
392      ReplaceSearchTerms(search_term_args);
393  GURL suggest_url(req_url);
394  // Make sure we are sending the suggest request through HTTPS.
395  if (!suggest_url.SchemeIs(content::kHttpsScheme)) {
396    Stop(true);
397    return;
398  }
399
400  fetcher_.reset(
401      net::URLFetcher::Create(kFetcherID,
402          suggest_url,
403          net::URLFetcher::GET, this));
404  fetcher_->SetRequestContext(profile_->GetRequestContext());
405  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
406  // Add Chrome experiment state to the request headers.
407  net::HttpRequestHeaders headers;
408  chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
409      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
410  fetcher_->SetExtraRequestHeaders(headers.ToString());
411
412  fetcher_->Start();
413  have_pending_request_ = true;
414  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
415}
416
417void ZeroSuggestProvider::ParseSuggestResults(const Value& root_val) {
418  SearchProvider::SuggestResults suggest_results;
419  FillResults(root_val, &verbatim_relevance_,
420              &suggest_results, &navigation_results_);
421
422  query_matches_map_.clear();
423  AddSuggestResultsToMap(suggest_results,
424                         template_url_service_->GetDefaultSearchProvider(),
425                         &query_matches_map_);
426}
427
428void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
429  matches_.clear();
430
431  const TemplateURL* default_provider =
432      template_url_service_->GetDefaultSearchProvider();
433  // Fail if we can't set the clickthrough URL for query suggestions.
434  if (default_provider == NULL || !default_provider->SupportsReplacement())
435    return;
436
437  const int num_query_results = query_matches_map_.size();
438  const int num_nav_results = navigation_results_.size();
439  const int num_results = num_query_results + num_nav_results;
440  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
441  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
442  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
443
444  if (num_results == 0)
445    return;
446
447  // TODO(jered): Rip this out once the first match is decoupled from the
448  // current typing in the omnibox.
449  matches_.push_back(current_url_match_);
450
451  for (SearchProvider::MatchMap::const_iterator it(query_matches_map_.begin());
452       it != query_matches_map_.end(); ++it)
453    matches_.push_back(it->second);
454
455  for (SearchProvider::NavigationResults::const_iterator it(
456       navigation_results_.begin()); it != navigation_results_.end(); ++it)
457    matches_.push_back(NavigationToMatch(*it));
458}
459
460AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
461  AutocompleteInput input(permanent_text_, string16::npos, string16(),
462                          GURL(current_query_), current_page_classification_,
463                          false, false, true, AutocompleteInput::ALL_MATCHES);
464
465  AutocompleteMatch match;
466  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
467      permanent_text_, false, true, &match, NULL);
468  match.is_history_what_you_typed_match = false;
469  match.allowed_to_be_default_match = true;
470
471  // The placeholder suggestion for the current URL has high relevance so
472  // that it is in the first suggestion slot and inline autocompleted. It
473  // gets dropped as soon as the user types something.
474  match.relevance = verbatim_relevance_;
475
476  return match;
477}
478