zero_suggest_provider.cc revision 7d4cd473f85ac64c3747c96c277f9e506a0d2246 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time.h"
16#include "chrome/browser/autocomplete/autocomplete_input.h"
17#include "chrome/browser/autocomplete/autocomplete_match.h"
18#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
19#include "chrome/browser/autocomplete/history_url_provider.h"
20#include "chrome/browser/autocomplete/search_provider.h"
21#include "chrome/browser/autocomplete/url_prefix.h"
22#include "chrome/browser/metrics/variations/variations_http_header_provider.h"
23#include "chrome/browser/net/url_fixer_upper.h"
24#include "chrome/browser/omnibox/omnibox_field_trial.h"
25#include "chrome/browser/profiles/profile.h"
26#include "chrome/browser/search/search.h"
27#include "chrome/browser/search_engines/template_url_service.h"
28#include "chrome/browser/search_engines/template_url_service_factory.h"
29#include "chrome/browser/sync/profile_sync_service.h"
30#include "chrome/browser/sync/profile_sync_service_factory.h"
31#include "chrome/common/pref_names.h"
32#include "chrome/common/url_constants.h"
33#include "googleurl/src/gurl.h"
34#include "net/base/escape.h"
35#include "net/base/load_flags.h"
36#include "net/base/net_util.h"
37#include "net/http/http_request_headers.h"
38#include "net/http/http_response_headers.h"
39#include "net/url_request/url_fetcher.h"
40#include "net/url_request/url_request_status.h"
41
42namespace {
43
44// TODO(hfung): The histogram code was copied and modified from
45// search_provider.cc.  Refactor and consolidate the code.
46// We keep track in a histogram how many suggest requests we send, how
47// many suggest requests we invalidate (e.g., due to a user typing
48// another character), and how many replies we receive.
49// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
50//     (excluding the end-of-list enum value)
51// We do not want values of existing enums to change or else it screws
52// up the statistics.
53enum ZeroSuggestRequestsHistogramValue {
54  ZERO_SUGGEST_REQUEST_SENT = 1,
55  ZERO_SUGGEST_REQUEST_INVALIDATED,
56  ZERO_SUGGEST_REPLY_RECEIVED,
57  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
58};
59
60void LogOmniboxZeroSuggestRequest(
61    ZeroSuggestRequestsHistogramValue request_value) {
62  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
63                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
64}
65
66// The maximum relevance of the top match from this provider.
67const int kDefaultVerbatimZeroSuggestRelevance = 1300;
68
69// Relevance value to use if it was not set explicitly by the server.
70const int kDefaultZeroSuggestRelevance = 100;
71
72}  // namespace
73
74// static
75ZeroSuggestProvider* ZeroSuggestProvider::Create(
76    AutocompleteProviderListener* listener,
77    Profile* profile) {
78  return new ZeroSuggestProvider(listener, profile);
79}
80
81void ZeroSuggestProvider::Start(const AutocompleteInput& input,
82                                bool /*minimal_changes*/) {
83  CheckIfTextModfied(input.text());
84  // Clear results only if the user text was modified.
85  Stop(user_text_modified_);
86  ConvertResultsToAutocompleteMatches(input.text(), false);
87  // listener_->OnProviderUpdate() does not need to be called because this
88  // function is only called in the synchronous pass when a user has performed
89  // an action (such as typing a character in the omnobox).
90}
91
92void ZeroSuggestProvider::Stop(bool clear_cached_results) {
93  if (have_pending_request_)
94    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
95  have_pending_request_ = false;
96  fetcher_.reset();
97  done_ = true;
98  if (clear_cached_results) {
99    query_matches_map_.clear();
100    navigation_results_.clear();
101    current_query_.clear();
102    matches_.clear();
103  }
104}
105
106void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
107  provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
108  metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
109  new_entry.set_provider(AsOmniboxEventProviderType());
110  new_entry.set_provider_done(done_);
111  std::vector<uint32> field_trial_hashes;
112  OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
113  for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
114    if (field_trial_triggered_)
115      new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
116    if (field_trial_triggered_in_session_) {
117      new_entry.mutable_field_trial_triggered_in_session()->Add(
118          field_trial_hashes[i]);
119     }
120  }
121}
122
123void ZeroSuggestProvider::ResetSession() {
124  // The user has started editing in the omnibox, so leave
125  // |field_trial_triggered_in_session_| unchanged and set
126  // |field_trial_triggered_| to false since zero suggest is inactive now.
127  field_trial_triggered_ = false;
128}
129
130void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
131  have_pending_request_ = false;
132  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
133
134  std::string json_data;
135  source->GetResponseAsString(&json_data);
136  const bool request_succeeded =
137      source->GetStatus().is_success() && source->GetResponseCode() == 200;
138
139  bool have_results = false;
140  if (request_succeeded) {
141    JSONStringValueSerializer deserializer(json_data);
142    deserializer.set_allow_trailing_comma(true);
143    scoped_ptr<Value> data(deserializer.Deserialize(NULL, NULL));
144    if (data.get()) {
145      ParseSuggestResults(*data.get());
146      have_results = !query_matches_map_.empty() ||
147          !navigation_results_.empty();
148    }
149  }
150  done_ = true;
151
152  if (have_results) {
153    ConvertResultsToAutocompleteMatches(original_user_text_, true);
154    listener_->OnProviderUpdate(true);
155  }
156}
157
158void ZeroSuggestProvider::StartZeroSuggest(const GURL& url,
159                                           const string16& user_text,
160                                           const string16& permanent_text) {
161  Stop(true);
162  field_trial_triggered_ = false;
163  field_trial_triggered_in_session_ = false;
164  if (!ShouldRunZeroSuggest(url))
165    return;
166  verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance;
167  done_ = false;
168  original_user_text_ = user_text;
169  permanent_text_ = permanent_text;
170  current_query_ = url.spec();
171  current_url_match_ = MatchForCurrentURL();
172  user_text_modified_ = false;
173  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
174  // These may be useful on the NTP or more relevant to the user than server
175  // suggestions, if based on local browsing history.
176  Run();
177}
178
179ZeroSuggestProvider::ZeroSuggestProvider(
180  AutocompleteProviderListener* listener,
181  Profile* profile)
182    : AutocompleteProvider(listener, profile,
183          AutocompleteProvider::TYPE_ZERO_SUGGEST),
184      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
185      user_text_modified_(false),
186      have_pending_request_(false),
187      verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance),
188      field_trial_triggered_(false),
189      field_trial_triggered_in_session_(false) {
190}
191
192ZeroSuggestProvider::~ZeroSuggestProvider() {
193}
194
195bool ZeroSuggestProvider::ShouldRunZeroSuggest(const GURL& url) const {
196  if (!url.is_valid())
197    return false;
198
199  // Do not query non-http URLs. There will be no useful suggestions for https
200  // or chrome URLs.
201  if (url.scheme() != chrome::kHttpScheme)
202    return false;
203
204  // Don't enable ZeroSuggest until InstantExtended works with ZeroSuggest.
205  if (chrome::IsInstantExtendedAPIEnabled())
206    return false;
207
208  // Don't run if there's no profile or in incognito mode.
209  if (profile_ == NULL || profile_->IsOffTheRecord())
210    return false;
211
212  // Don't run if we can't get preferences or search suggest is not enabled.
213  PrefService* prefs = profile_->GetPrefs();
214  if (prefs == NULL || !prefs->GetBoolean(prefs::kSearchSuggestEnabled))
215    return false;
216
217  ProfileSyncService* service =
218      ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile_);
219  browser_sync::SyncPrefs sync_prefs(prefs);
220  // The user has needs to have Chrome Sync enabled (for permissions to
221  // transmit their current URL) and be in the field trial.
222  if (!OmniboxFieldTrial::InZeroSuggestFieldTrial() ||
223      service == NULL ||
224      !service->IsSyncEnabledAndLoggedIn() ||
225      !sync_prefs.HasKeepEverythingSynced()) {
226    return false;
227  }
228  return true;
229}
230
231void ZeroSuggestProvider::FillResults(
232    const Value& root_val,
233    int* verbatim_relevance,
234    SearchProvider::SuggestResults* suggest_results,
235    SearchProvider::NavigationResults* navigation_results) {
236  string16 query;
237  const ListValue* root_list = NULL;
238  const ListValue* results = NULL;
239  const ListValue* relevances = NULL;
240  // The response includes the query, which should be empty for ZeroSuggest
241  // responses.
242  if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
243      (!query.empty()) || !root_list->GetList(1, &results))
244    return;
245
246  // 3rd element: Description list.
247  const ListValue* descriptions = NULL;
248  root_list->GetList(2, &descriptions);
249
250  // 4th element: Disregard the query URL list for now.
251
252  // Reset suggested relevance information from the provider.
253  *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance;
254
255  // 5th element: Optional key-value pairs from the Suggest server.
256  const ListValue* types = NULL;
257  const DictionaryValue* extras = NULL;
258  if (root_list->GetDictionary(4, &extras)) {
259    extras->GetList("google:suggesttype", &types);
260
261    // Discard this list if its size does not match that of the suggestions.
262    if (extras->GetList("google:suggestrelevance", &relevances) &&
263        relevances->GetSize() != results->GetSize())
264      relevances = NULL;
265    extras->GetInteger("google:verbatimrelevance", verbatim_relevance);
266
267    // Check if the active suggest field trial (if any) has triggered.
268    bool triggered = false;
269    extras->GetBoolean("google:fieldtrialtriggered", &triggered);
270    field_trial_triggered_ |= triggered;
271    field_trial_triggered_in_session_ |= triggered;
272  }
273
274  // Clear the previous results now that new results are available.
275  suggest_results->clear();
276  navigation_results->clear();
277
278  string16 result, title;
279  std::string type;
280  for (size_t index = 0; results->GetString(index, &result); ++index) {
281    // Google search may return empty suggestions for weird input characters,
282    // they make no sense at all and can cause problems in our code.
283    if (result.empty())
284      continue;
285
286    int relevance = kDefaultZeroSuggestRelevance;
287
288    // Apply valid suggested relevance scores; discard invalid lists.
289    if (relevances != NULL && !relevances->GetInteger(index, &relevance))
290      relevances = NULL;
291    if (types && types->GetString(index, &type) && (type == "NAVIGATION")) {
292      // Do not blindly trust the URL coming from the server to be valid.
293      GURL url(URLFixerUpper::FixupURL(UTF16ToUTF8(result), std::string()));
294      if (url.is_valid()) {
295        if (descriptions != NULL)
296          descriptions->GetString(index, &title);
297        navigation_results->push_back(SearchProvider::NavigationResult(
298            *this, url, title, false, relevance));
299      }
300    } else {
301      suggest_results->push_back(SearchProvider::SuggestResult(
302          result, false, relevance));
303    }
304  }
305}
306
307void ZeroSuggestProvider::AddSuggestResultsToMap(
308    const SearchProvider::SuggestResults& results,
309    const string16& provider_keyword,
310    SearchProvider::MatchMap* map) {
311  for (size_t i = 0; i < results.size(); ++i) {
312    AddMatchToMap(results[i].suggestion(),
313                  provider_keyword,
314                  results[i].relevance(),
315                  AutocompleteMatchType::SEARCH_SUGGEST, i, map);
316  }
317}
318
319void ZeroSuggestProvider::AddMatchToMap(const string16& query_string,
320                                        const string16& provider_keyword,
321                                        int relevance,
322                                        AutocompleteMatch::Type type,
323                                        int accepted_suggestion,
324                                        SearchProvider::MatchMap* map) {
325  // Pass in query_string as the input_text since we don't want any bolding.
326  // TODO(samarth|melevin): use the actual omnibox margin here as well instead
327  // of passing in -1.
328  AutocompleteMatch match = SearchProvider::CreateSearchSuggestion(
329      profile_, this, AutocompleteInput(),
330      query_string, query_string, relevance, type, accepted_suggestion,
331      false, provider_keyword, -1);
332  if (!match.destination_url.is_valid())
333    return;
334
335  // Try to add |match| to |map|.  If a match for |query_string| is already in
336  // |map|, replace it if |match| is more relevant.
337  // NOTE: Keep this ToLower() call in sync with url_database.cc.
338  const std::pair<SearchProvider::MatchMap::iterator, bool> i = map->insert(
339      std::pair<string16, AutocompleteMatch>(
340          base::i18n::ToLower(query_string), match));
341  // NOTE: We purposefully do a direct relevance comparison here instead of
342  // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items added
343  // first" rather than "items alphabetically first" when the scores are equal.
344  // The only case this matters is when a user has results with the same score
345  // that differ only by capitalization; because the history system returns
346  // results sorted by recency, this means we'll pick the most recent such
347  // result even if the precision of our relevance score is too low to
348  // distinguish the two.
349  if (!i.second && (match.relevance > i.first->second.relevance))
350    i.first->second = match;
351}
352
353AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
354    const SearchProvider::NavigationResult& navigation) {
355  AutocompleteMatch match(this, navigation.relevance(), false,
356                          AutocompleteMatchType::NAVSUGGEST);
357  match.destination_url = navigation.url();
358
359  const std::string languages(
360      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
361  match.contents = net::FormatUrl(navigation.url(), languages,
362      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
363  match.fill_into_edit +=
364      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
365          match.contents);
366  match.inline_autocomplete_offset = string16::npos;
367
368  AutocompleteMatch::ClassifyLocationInString(string16::npos, 0,
369      match.contents.length(), ACMatchClassification::URL,
370      &match.contents_class);
371  match.description = navigation.description();
372  return match;
373}
374
375void ZeroSuggestProvider::Run() {
376  have_pending_request_ = false;
377  const int kFetcherID = 1;
378
379  const TemplateURL* default_provider =
380     template_url_service_->GetDefaultSearchProvider();
381  // TODO(hfung): Generalize if the default provider supports zero suggest.
382  // Only make the request if we know that the provider supports zero suggest
383  // (currently only the prepopulated Google provider).
384  if (default_provider == NULL || !default_provider->SupportsReplacement() ||
385      default_provider->prepopulate_id() != 1) {
386    Stop(true);
387    return;
388  }
389  string16 prefix;
390  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
391  search_term_args.zero_prefix_url = current_query_;
392  std::string req_url = default_provider->suggestions_url_ref().
393      ReplaceSearchTerms(search_term_args);
394  GURL suggest_url(req_url);
395  // Make sure we are sending the suggest request through HTTPS.
396  if (!suggest_url.SchemeIs(chrome::kHttpsScheme)) {
397    Stop(true);
398    return;
399  }
400
401  fetcher_.reset(
402      net::URLFetcher::Create(kFetcherID,
403          suggest_url,
404          net::URLFetcher::GET, this));
405  fetcher_->SetRequestContext(profile_->GetRequestContext());
406  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
407  // Add Chrome experiment state to the request headers.
408  net::HttpRequestHeaders headers;
409  chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
410      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
411  fetcher_->SetExtraRequestHeaders(headers.ToString());
412
413  fetcher_->Start();
414  have_pending_request_ = true;
415  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
416}
417
418void ZeroSuggestProvider::CheckIfTextModfied(const string16& user_text) {
419  if (!user_text.empty() && user_text != permanent_text_)
420    user_text_modified_ = true;
421}
422
423void ZeroSuggestProvider::ParseSuggestResults(const Value& root_val) {
424  SearchProvider::SuggestResults suggest_results;
425  FillResults(root_val, &verbatim_relevance_,
426              &suggest_results, &navigation_results_);
427
428  query_matches_map_.clear();
429  const TemplateURL* default_provider =
430     template_url_service_->GetDefaultSearchProvider();
431  AddSuggestResultsToMap(suggest_results, default_provider->keyword(),
432                         &query_matches_map_);
433}
434
435void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches(
436    string16 user_text, bool update_histograms) {
437  matches_.clear();
438
439  const TemplateURL* default_provider =
440      template_url_service_->GetDefaultSearchProvider();
441  // Fail if we can't set the clickthrough URL for query suggestions.
442  if (default_provider == NULL || !default_provider->SupportsReplacement())
443    return;
444
445  const int num_query_results = query_matches_map_.size();
446  const int num_nav_results = navigation_results_.size();
447  const int num_results = num_query_results + num_nav_results;
448  if (update_histograms) {
449    UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
450    UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
451    UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
452  }
453
454  if (num_results == 0 || user_text_modified_)
455    return;
456
457  // TODO(jered): Rip this out once the first match is decoupled from the
458  // current typing in the omnibox.
459  // If the user text is empty, we can autocomplete to the URL.  Otherwise,
460  // don't modify the omnibox text.
461  current_url_match_.inline_autocomplete_offset = user_text.empty() ?
462      0 : string16::npos;
463  matches_.push_back(current_url_match_);
464
465  for (SearchProvider::MatchMap::const_iterator it = query_matches_map_.begin();
466       it != query_matches_map_.end(); ++it) {
467    matches_.push_back(it->second);
468  }
469
470  for (SearchProvider::NavigationResults::const_iterator it =
471           navigation_results_.begin();
472       it != navigation_results_.end(); ++it) {
473    matches_.push_back(NavigationToMatch(*it));
474  }
475}
476
477AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
478  AutocompleteInput input(permanent_text_, string16::npos,
479                          string16(), GURL(current_query_),
480                          false, false, true, AutocompleteInput::ALL_MATCHES);
481
482  AutocompleteMatch match(
483      HistoryURLProvider::SuggestExactInput(this, input,
484                                            !HasHTTPScheme(input.text())));
485  match.is_history_what_you_typed_match = false;
486
487  // The placeholder suggestion for the current URL has high relevance so
488  // that it is in the first suggestion slot and inline autocompleted. It
489  // gets dropped as soon as the user types something.
490  match.relevance = verbatim_relevance_;
491
492  return match;
493}
494