zero_suggest_provider.cc revision bb1529ce867d8845a77ec7cdf3e3003ef1771a40 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_input.h"
17#include "chrome/browser/autocomplete/autocomplete_match.h"
18#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
19#include "chrome/browser/autocomplete/history_url_provider.h"
20#include "chrome/browser/autocomplete/search_provider.h"
21#include "chrome/browser/autocomplete/url_prefix.h"
22#include "chrome/browser/google/google_util.h"
23#include "chrome/browser/metrics/variations/variations_http_header_provider.h"
24#include "chrome/browser/omnibox/omnibox_field_trial.h"
25#include "chrome/browser/profiles/profile.h"
26#include "chrome/browser/search/search.h"
27#include "chrome/browser/search_engines/template_url_service.h"
28#include "chrome/browser/search_engines/template_url_service_factory.h"
29#include "chrome/browser/sync/profile_sync_service.h"
30#include "chrome/browser/sync/profile_sync_service_factory.h"
31#include "chrome/common/net/url_fixer_upper.h"
32#include "chrome/common/pref_names.h"
33#include "chrome/common/url_constants.h"
34#include "net/base/escape.h"
35#include "net/base/load_flags.h"
36#include "net/base/net_util.h"
37#include "net/http/http_request_headers.h"
38#include "net/http/http_response_headers.h"
39#include "net/url_request/url_fetcher.h"
40#include "net/url_request/url_request_status.h"
41#include "url/gurl.h"
42
43namespace {
44
45// TODO(hfung): The histogram code was copied and modified from
46// search_provider.cc.  Refactor and consolidate the code.
47// We keep track in a histogram how many suggest requests we send, how
48// many suggest requests we invalidate (e.g., due to a user typing
49// another character), and how many replies we receive.
50// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
51//     (excluding the end-of-list enum value)
52// We do not want values of existing enums to change or else it screws
53// up the statistics.
54enum ZeroSuggestRequestsHistogramValue {
55  ZERO_SUGGEST_REQUEST_SENT = 1,
56  ZERO_SUGGEST_REQUEST_INVALIDATED,
57  ZERO_SUGGEST_REPLY_RECEIVED,
58  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
59};
60
61void LogOmniboxZeroSuggestRequest(
62    ZeroSuggestRequestsHistogramValue request_value) {
63  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
64                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
65}
66
67// The maximum relevance of the top match from this provider.
68const int kDefaultVerbatimZeroSuggestRelevance = 1300;
69
70// Relevance value to use if it was not set explicitly by the server.
71const int kDefaultZeroSuggestRelevance = 100;
72
73}  // namespace
74
75// static
76ZeroSuggestProvider* ZeroSuggestProvider::Create(
77    AutocompleteProviderListener* listener,
78    Profile* profile) {
79  return new ZeroSuggestProvider(listener, profile);
80}
81
82void ZeroSuggestProvider::Start(const AutocompleteInput& input,
83                                bool /*minimal_changes*/) {
84}
85
86void ZeroSuggestProvider::Stop(bool clear_cached_results) {
87  if (have_pending_request_)
88    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
89  have_pending_request_ = false;
90  fetcher_.reset();
91  done_ = true;
92  if (clear_cached_results) {
93    query_matches_map_.clear();
94    navigation_results_.clear();
95    current_query_.clear();
96    matches_.clear();
97  }
98}
99
100void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
101  provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
102  metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
103  new_entry.set_provider(AsOmniboxEventProviderType());
104  new_entry.set_provider_done(done_);
105  std::vector<uint32> field_trial_hashes;
106  OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
107  for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
108    if (field_trial_triggered_)
109      new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
110    if (field_trial_triggered_in_session_) {
111      new_entry.mutable_field_trial_triggered_in_session()->Add(
112          field_trial_hashes[i]);
113     }
114  }
115}
116
117void ZeroSuggestProvider::ResetSession() {
118  // The user has started editing in the omnibox, so leave
119  // |field_trial_triggered_in_session_| unchanged and set
120  // |field_trial_triggered_| to false since zero suggest is inactive now.
121  field_trial_triggered_ = false;
122  Stop(true);
123}
124
125void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
126  have_pending_request_ = false;
127  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
128
129  std::string json_data;
130  source->GetResponseAsString(&json_data);
131  const bool request_succeeded =
132      source->GetStatus().is_success() && source->GetResponseCode() == 200;
133
134  bool have_results = false;
135  if (request_succeeded) {
136    JSONStringValueSerializer deserializer(json_data);
137    deserializer.set_allow_trailing_comma(true);
138    scoped_ptr<Value> data(deserializer.Deserialize(NULL, NULL));
139    if (data.get()) {
140      ParseSuggestResults(*data.get());
141      have_results = !query_matches_map_.empty() ||
142          !navigation_results_.empty();
143    }
144  }
145  done_ = true;
146
147  if (have_results) {
148    ConvertResultsToAutocompleteMatches();
149    listener_->OnProviderUpdate(true);
150  }
151}
152
153void ZeroSuggestProvider::StartZeroSuggest(
154    const GURL& url,
155    AutocompleteInput::PageClassification page_classification,
156    const string16& permanent_text) {
157  Stop(true);
158  field_trial_triggered_ = false;
159  field_trial_triggered_in_session_ = false;
160  if (!ShouldRunZeroSuggest(url))
161    return;
162  verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance;
163  done_ = false;
164  permanent_text_ = permanent_text;
165  current_query_ = url.spec();
166  current_page_classification_ = page_classification;
167  current_url_match_ = MatchForCurrentURL();
168  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
169  // These may be useful on the NTP or more relevant to the user than server
170  // suggestions, if based on local browsing history.
171  Run();
172}
173
174ZeroSuggestProvider::ZeroSuggestProvider(
175  AutocompleteProviderListener* listener,
176  Profile* profile)
177    : AutocompleteProvider(listener, profile,
178          AutocompleteProvider::TYPE_ZERO_SUGGEST),
179      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
180      have_pending_request_(false),
181      verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance),
182      field_trial_triggered_(false),
183      field_trial_triggered_in_session_(false) {
184}
185
186ZeroSuggestProvider::~ZeroSuggestProvider() {
187}
188
189bool ZeroSuggestProvider::ShouldRunZeroSuggest(const GURL& url) const {
190  if (!ShouldSendURL(url))
191    return false;
192
193  // Don't run if there's no profile or in incognito mode.
194  if (profile_ == NULL || profile_->IsOffTheRecord())
195    return false;
196
197  // Don't run if we can't get preferences or search suggest is not enabled.
198  PrefService* prefs = profile_->GetPrefs();
199  if (prefs == NULL || !prefs->GetBoolean(prefs::kSearchSuggestEnabled))
200    return false;
201
202  ProfileSyncService* service =
203      ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile_);
204  browser_sync::SyncPrefs sync_prefs(prefs);
205  // The user has needs to have Chrome Sync enabled (for permissions to
206  // transmit their current URL) and be in the field trial.
207  if (!OmniboxFieldTrial::InZeroSuggestFieldTrial() ||
208      service == NULL ||
209      !service->IsSyncEnabledAndLoggedIn() ||
210      !sync_prefs.HasKeepEverythingSynced()) {
211    return false;
212  }
213  return true;
214}
215
216bool ZeroSuggestProvider::ShouldSendURL(const GURL& url) const {
217  if (!url.is_valid())
218    return false;
219
220  // Only allow HTTP URLs or Google HTTPS URLs (including Google search
221  // result pages).  For the latter case, Google was already sent the HTTPS
222  // URLs when requesting the page, so the information is just re-sent.
223  return (url.scheme() == chrome::kHttpScheme) ||
224      google_util::IsGoogleDomainUrl(url, google_util::ALLOW_SUBDOMAIN,
225                                     google_util::ALLOW_NON_STANDARD_PORTS);
226}
227
228void ZeroSuggestProvider::FillResults(
229    const Value& root_val,
230    int* verbatim_relevance,
231    SearchProvider::SuggestResults* suggest_results,
232    SearchProvider::NavigationResults* navigation_results) {
233  string16 query;
234  const ListValue* root_list = NULL;
235  const ListValue* results = NULL;
236  const ListValue* relevances = NULL;
237  // The response includes the query, which should be empty for ZeroSuggest
238  // responses.
239  if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
240      (!query.empty()) || !root_list->GetList(1, &results))
241    return;
242
243  // 3rd element: Description list.
244  const ListValue* descriptions = NULL;
245  root_list->GetList(2, &descriptions);
246
247  // 4th element: Disregard the query URL list for now.
248
249  // Reset suggested relevance information from the provider.
250  *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance;
251
252  // 5th element: Optional key-value pairs from the Suggest server.
253  const ListValue* types = NULL;
254  const DictionaryValue* extras = NULL;
255  if (root_list->GetDictionary(4, &extras)) {
256    extras->GetList("google:suggesttype", &types);
257
258    // Discard this list if its size does not match that of the suggestions.
259    if (extras->GetList("google:suggestrelevance", &relevances) &&
260        relevances->GetSize() != results->GetSize())
261      relevances = NULL;
262    extras->GetInteger("google:verbatimrelevance", verbatim_relevance);
263
264    // Check if the active suggest field trial (if any) has triggered.
265    bool triggered = false;
266    extras->GetBoolean("google:fieldtrialtriggered", &triggered);
267    field_trial_triggered_ |= triggered;
268    field_trial_triggered_in_session_ |= triggered;
269  }
270
271  // Clear the previous results now that new results are available.
272  suggest_results->clear();
273  navigation_results->clear();
274
275  string16 result, title;
276  std::string type;
277  for (size_t index = 0; results->GetString(index, &result); ++index) {
278    // Google search may return empty suggestions for weird input characters,
279    // they make no sense at all and can cause problems in our code.
280    if (result.empty())
281      continue;
282
283    int relevance = kDefaultZeroSuggestRelevance;
284
285    // Apply valid suggested relevance scores; discard invalid lists.
286    if (relevances != NULL && !relevances->GetInteger(index, &relevance))
287      relevances = NULL;
288    if (types && types->GetString(index, &type) && (type == "NAVIGATION")) {
289      // Do not blindly trust the URL coming from the server to be valid.
290      GURL url(URLFixerUpper::FixupURL(UTF16ToUTF8(result), std::string()));
291      if (url.is_valid()) {
292        if (descriptions != NULL)
293          descriptions->GetString(index, &title);
294        navigation_results->push_back(SearchProvider::NavigationResult(
295            *this, url, title, false, relevance, relevances != NULL));
296      }
297    } else {
298      suggest_results->push_back(SearchProvider::SuggestResult(
299          result, false, relevance, relevances != NULL));
300    }
301  }
302}
303
304void ZeroSuggestProvider::AddSuggestResultsToMap(
305    const SearchProvider::SuggestResults& results,
306    const TemplateURL* template_url,
307    SearchProvider::MatchMap* map) {
308  for (size_t i = 0; i < results.size(); ++i) {
309    AddMatchToMap(results[i].relevance(), AutocompleteMatchType::SEARCH_SUGGEST,
310                  template_url, results[i].suggestion(), i, map);
311  }
312}
313
314void ZeroSuggestProvider::AddMatchToMap(int relevance,
315                                        AutocompleteMatch::Type type,
316                                        const TemplateURL* template_url,
317                                        const string16& query_string,
318                                        int accepted_suggestion,
319                                        SearchProvider::MatchMap* map) {
320  // Pass in query_string as the input_text since we don't want any bolding.
321  // TODO(samarth|melevin): use the actual omnibox margin here as well instead
322  // of passing in -1.
323  AutocompleteMatch match = SearchProvider::CreateSearchSuggestion(
324      this, relevance, type, template_url, query_string, query_string,
325      AutocompleteInput(), false, accepted_suggestion, -1, true);
326  if (!match.destination_url.is_valid())
327    return;
328
329  // Try to add |match| to |map|.  If a match for |query_string| is already in
330  // |map|, replace it if |match| is more relevant.
331  // NOTE: Keep this ToLower() call in sync with url_database.cc.
332  const std::pair<SearchProvider::MatchMap::iterator, bool> i(map->insert(
333      std::make_pair(base::i18n::ToLower(query_string), match)));
334  // NOTE: We purposefully do a direct relevance comparison here instead of
335  // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items added
336  // first" rather than "items alphabetically first" when the scores are equal.
337  // The only case this matters is when a user has results with the same score
338  // that differ only by capitalization; because the history system returns
339  // results sorted by recency, this means we'll pick the most recent such
340  // result even if the precision of our relevance score is too low to
341  // distinguish the two.
342  if (!i.second && (match.relevance > i.first->second.relevance))
343    i.first->second = match;
344}
345
346AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
347    const SearchProvider::NavigationResult& navigation) {
348  AutocompleteMatch match(this, navigation.relevance(), false,
349                          AutocompleteMatchType::NAVSUGGEST);
350  match.destination_url = navigation.url();
351
352  const std::string languages(
353      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
354  match.contents = net::FormatUrl(navigation.url(), languages,
355      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
356  match.fill_into_edit +=
357      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
358          match.contents);
359
360  AutocompleteMatch::ClassifyLocationInString(string16::npos, 0,
361      match.contents.length(), ACMatchClassification::URL,
362      &match.contents_class);
363
364  match.description =
365      AutocompleteMatch::SanitizeString(navigation.description());
366  AutocompleteMatch::ClassifyLocationInString(string16::npos, 0,
367      match.description.length(), ACMatchClassification::NONE,
368      &match.description_class);
369  return match;
370}
371
372void ZeroSuggestProvider::Run() {
373  have_pending_request_ = false;
374  const int kFetcherID = 1;
375
376  const TemplateURL* default_provider =
377     template_url_service_->GetDefaultSearchProvider();
378  // TODO(hfung): Generalize if the default provider supports zero suggest.
379  // Only make the request if we know that the provider supports zero suggest
380  // (currently only the prepopulated Google provider).
381  if (default_provider == NULL || !default_provider->SupportsReplacement() ||
382      default_provider->prepopulate_id() != 1) {
383    Stop(true);
384    return;
385  }
386  string16 prefix;
387  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
388  search_term_args.zero_prefix_url = current_query_;
389  std::string req_url = default_provider->suggestions_url_ref().
390      ReplaceSearchTerms(search_term_args);
391  GURL suggest_url(req_url);
392  // Make sure we are sending the suggest request through HTTPS.
393  if (!suggest_url.SchemeIs(chrome::kHttpsScheme)) {
394    Stop(true);
395    return;
396  }
397
398  fetcher_.reset(
399      net::URLFetcher::Create(kFetcherID,
400          suggest_url,
401          net::URLFetcher::GET, this));
402  fetcher_->SetRequestContext(profile_->GetRequestContext());
403  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
404  // Add Chrome experiment state to the request headers.
405  net::HttpRequestHeaders headers;
406  chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
407      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
408  fetcher_->SetExtraRequestHeaders(headers.ToString());
409
410  fetcher_->Start();
411  have_pending_request_ = true;
412  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
413}
414
415void ZeroSuggestProvider::ParseSuggestResults(const Value& root_val) {
416  SearchProvider::SuggestResults suggest_results;
417  FillResults(root_val, &verbatim_relevance_,
418              &suggest_results, &navigation_results_);
419
420  query_matches_map_.clear();
421  AddSuggestResultsToMap(suggest_results,
422                         template_url_service_->GetDefaultSearchProvider(),
423                         &query_matches_map_);
424}
425
426void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
427  matches_.clear();
428
429  const TemplateURL* default_provider =
430      template_url_service_->GetDefaultSearchProvider();
431  // Fail if we can't set the clickthrough URL for query suggestions.
432  if (default_provider == NULL || !default_provider->SupportsReplacement())
433    return;
434
435  const int num_query_results = query_matches_map_.size();
436  const int num_nav_results = navigation_results_.size();
437  const int num_results = num_query_results + num_nav_results;
438  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
439  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
440  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
441
442  if (num_results == 0)
443    return;
444
445  // TODO(jered): Rip this out once the first match is decoupled from the
446  // current typing in the omnibox.
447  matches_.push_back(current_url_match_);
448
449  for (SearchProvider::MatchMap::const_iterator it(query_matches_map_.begin());
450       it != query_matches_map_.end(); ++it)
451    matches_.push_back(it->second);
452
453  for (SearchProvider::NavigationResults::const_iterator it(
454       navigation_results_.begin()); it != navigation_results_.end(); ++it)
455    matches_.push_back(NavigationToMatch(*it));
456}
457
458AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
459  AutocompleteInput input(permanent_text_, string16::npos, string16(),
460                          GURL(current_query_), current_page_classification_,
461                          false, false, true, AutocompleteInput::ALL_MATCHES);
462
463  AutocompleteMatch match(
464      HistoryURLProvider::SuggestExactInput(this, input,
465                                            !HasHTTPScheme(input.text())));
466  match.is_history_what_you_typed_match = false;
467
468  // The placeholder suggestion for the current URL has high relevance so
469  // that it is in the first suggestion slot and inline autocompleted. It
470  // gets dropped as soon as the user types something.
471  match.relevance = verbatim_relevance_;
472
473  return match;
474}
475