zero_suggest_provider.cc revision f2477e01787aa58f445919b809d89e252beef54f 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_classifier.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
18#include "chrome/browser/autocomplete/autocomplete_input.h"
19#include "chrome/browser/autocomplete/autocomplete_match.h"
20#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
21#include "chrome/browser/autocomplete/history_url_provider.h"
22#include "chrome/browser/autocomplete/search_provider.h"
23#include "chrome/browser/autocomplete/url_prefix.h"
24#include "chrome/browser/history/history_types.h"
25#include "chrome/browser/history/top_sites.h"
26#include "chrome/browser/metrics/variations/variations_http_header_provider.h"
27#include "chrome/browser/omnibox/omnibox_field_trial.h"
28#include "chrome/browser/profiles/profile.h"
29#include "chrome/browser/search/search.h"
30#include "chrome/browser/search_engines/template_url_service.h"
31#include "chrome/browser/search_engines/template_url_service_factory.h"
32#include "chrome/common/net/url_fixer_upper.h"
33#include "chrome/common/pref_names.h"
34#include "chrome/common/url_constants.h"
35#include "net/base/escape.h"
36#include "net/base/load_flags.h"
37#include "net/base/net_util.h"
38#include "net/http/http_request_headers.h"
39#include "net/http/http_response_headers.h"
40#include "net/url_request/url_fetcher.h"
41#include "net/url_request/url_request_status.h"
42#include "url/gurl.h"
43
44namespace {
45
46// TODO(hfung): The histogram code was copied and modified from
47// search_provider.cc.  Refactor and consolidate the code.
48// We keep track in a histogram how many suggest requests we send, how
49// many suggest requests we invalidate (e.g., due to a user typing
50// another character), and how many replies we receive.
51// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
52//     (excluding the end-of-list enum value)
53// We do not want values of existing enums to change or else it screws
54// up the statistics.
55enum ZeroSuggestRequestsHistogramValue {
56  ZERO_SUGGEST_REQUEST_SENT = 1,
57  ZERO_SUGGEST_REQUEST_INVALIDATED,
58  ZERO_SUGGEST_REPLY_RECEIVED,
59  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
60};
61
62void LogOmniboxZeroSuggestRequest(
63    ZeroSuggestRequestsHistogramValue request_value) {
64  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
65                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
66}
67
68// The maximum relevance of the top match from this provider.
69const int kDefaultVerbatimZeroSuggestRelevance = 1300;
70
71// Relevance value to use if it was not set explicitly by the server.
72const int kDefaultZeroSuggestRelevance = 100;
73
74}  // namespace
75
76// static
77ZeroSuggestProvider* ZeroSuggestProvider::Create(
78    AutocompleteProviderListener* listener,
79    Profile* profile) {
80  return new ZeroSuggestProvider(listener, profile);
81}
82
83void ZeroSuggestProvider::Start(const AutocompleteInput& input,
84                                bool /*minimal_changes*/) {
85}
86
87void ZeroSuggestProvider::Stop(bool clear_cached_results) {
88  if (have_pending_request_)
89    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
90  have_pending_request_ = false;
91  fetcher_.reset();
92  done_ = true;
93  if (clear_cached_results) {
94    query_matches_map_.clear();
95    navigation_results_.clear();
96    current_query_.clear();
97    matches_.clear();
98  }
99}
100
101void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
102  provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
103  metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
104  new_entry.set_provider(AsOmniboxEventProviderType());
105  new_entry.set_provider_done(done_);
106  std::vector<uint32> field_trial_hashes;
107  OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
108  for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
109    if (field_trial_triggered_)
110      new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
111    if (field_trial_triggered_in_session_) {
112      new_entry.mutable_field_trial_triggered_in_session()->Add(
113          field_trial_hashes[i]);
114     }
115  }
116}
117
118void ZeroSuggestProvider::ResetSession() {
119  // The user has started editing in the omnibox, so leave
120  // |field_trial_triggered_in_session_| unchanged and set
121  // |field_trial_triggered_| to false since zero suggest is inactive now.
122  field_trial_triggered_ = false;
123  Stop(true);
124}
125
126void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
127  have_pending_request_ = false;
128  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
129
130  std::string json_data;
131  source->GetResponseAsString(&json_data);
132  const bool request_succeeded =
133      source->GetStatus().is_success() && source->GetResponseCode() == 200;
134
135  if (request_succeeded) {
136    JSONStringValueSerializer deserializer(json_data);
137    deserializer.set_allow_trailing_comma(true);
138    scoped_ptr<Value> data(deserializer.Deserialize(NULL, NULL));
139    if (data.get())
140      ParseSuggestResults(*data.get());
141  }
142  done_ = true;
143
144  ConvertResultsToAutocompleteMatches();
145  if (!matches_.empty())
146    listener_->OnProviderUpdate(true);
147}
148
149void ZeroSuggestProvider::StartZeroSuggest(
150    const GURL& current_page_url,
151    AutocompleteInput::PageClassification page_classification,
152    const string16& permanent_text) {
153  Stop(true);
154  field_trial_triggered_ = false;
155  field_trial_triggered_in_session_ = false;
156  permanent_text_ = permanent_text;
157  current_query_ = current_page_url.spec();
158  current_page_classification_ = page_classification;
159  current_url_match_ = MatchForCurrentURL();
160
161  const TemplateURL* default_provider =
162     template_url_service_->GetDefaultSearchProvider();
163  if (default_provider == NULL)
164    return;
165  string16 prefix;
166  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
167  search_term_args.current_page_url = current_query_;
168  GURL suggest_url(default_provider->suggestions_url_ref().
169                   ReplaceSearchTerms(search_term_args));
170  if (!SearchProvider::CanSendURL(
171          current_page_url, suggest_url,
172          template_url_service_->GetDefaultSearchProvider(),
173          page_classification, profile_) ||
174      !OmniboxFieldTrial::InZeroSuggestFieldTrial())
175    return;
176  verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance;
177  done_ = false;
178  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
179  // These may be useful on the NTP or more relevant to the user than server
180  // suggestions, if based on local browsing history.
181  Run(suggest_url);
182}
183
184ZeroSuggestProvider::ZeroSuggestProvider(
185  AutocompleteProviderListener* listener,
186  Profile* profile)
187    : AutocompleteProvider(listener, profile,
188          AutocompleteProvider::TYPE_ZERO_SUGGEST),
189      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
190      have_pending_request_(false),
191      verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance),
192      field_trial_triggered_(false),
193      field_trial_triggered_in_session_(false),
194      weak_ptr_factory_(this) {
195}
196
197ZeroSuggestProvider::~ZeroSuggestProvider() {
198}
199
200void ZeroSuggestProvider::FillResults(
201    const Value& root_val,
202    int* verbatim_relevance,
203    SearchProvider::SuggestResults* suggest_results,
204    SearchProvider::NavigationResults* navigation_results) {
205  string16 query;
206  const ListValue* root_list = NULL;
207  const ListValue* results = NULL;
208  const ListValue* relevances = NULL;
209  // The response includes the query, which should be empty for ZeroSuggest
210  // responses.
211  if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
212      (!query.empty()) || !root_list->GetList(1, &results))
213    return;
214
215  // 3rd element: Description list.
216  const ListValue* descriptions = NULL;
217  root_list->GetList(2, &descriptions);
218
219  // 4th element: Disregard the query URL list for now.
220
221  // Reset suggested relevance information from the provider.
222  *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance;
223
224  // 5th element: Optional key-value pairs from the Suggest server.
225  const ListValue* types = NULL;
226  const DictionaryValue* extras = NULL;
227  if (root_list->GetDictionary(4, &extras)) {
228    extras->GetList("google:suggesttype", &types);
229
230    // Discard this list if its size does not match that of the suggestions.
231    if (extras->GetList("google:suggestrelevance", &relevances) &&
232        relevances->GetSize() != results->GetSize())
233      relevances = NULL;
234    extras->GetInteger("google:verbatimrelevance", verbatim_relevance);
235
236    // Check if the active suggest field trial (if any) has triggered.
237    bool triggered = false;
238    extras->GetBoolean("google:fieldtrialtriggered", &triggered);
239    field_trial_triggered_ |= triggered;
240    field_trial_triggered_in_session_ |= triggered;
241  }
242
243  // Clear the previous results now that new results are available.
244  suggest_results->clear();
245  navigation_results->clear();
246
247  string16 result, title;
248  std::string type;
249  for (size_t index = 0; results->GetString(index, &result); ++index) {
250    // Google search may return empty suggestions for weird input characters,
251    // they make no sense at all and can cause problems in our code.
252    if (result.empty())
253      continue;
254
255    int relevance = kDefaultZeroSuggestRelevance;
256
257    // Apply valid suggested relevance scores; discard invalid lists.
258    if (relevances != NULL && !relevances->GetInteger(index, &relevance))
259      relevances = NULL;
260    if (types && types->GetString(index, &type) && (type == "NAVIGATION")) {
261      // Do not blindly trust the URL coming from the server to be valid.
262      GURL url(URLFixerUpper::FixupURL(UTF16ToUTF8(result), std::string()));
263      if (url.is_valid()) {
264        if (descriptions != NULL)
265          descriptions->GetString(index, &title);
266        navigation_results->push_back(SearchProvider::NavigationResult(
267            *this, url, title, false, relevance, relevances != NULL));
268      }
269    } else {
270      suggest_results->push_back(SearchProvider::SuggestResult(
271          result, result, string16(), std::string(), false, relevance,
272          relevances != NULL, false));
273    }
274  }
275}
276
277void ZeroSuggestProvider::AddSuggestResultsToMap(
278    const SearchProvider::SuggestResults& results,
279    const TemplateURL* template_url,
280    SearchProvider::MatchMap* map) {
281  for (size_t i = 0; i < results.size(); ++i) {
282    AddMatchToMap(results[i].relevance(), AutocompleteMatchType::SEARCH_SUGGEST,
283                  template_url, results[i].suggestion(), i, map);
284  }
285}
286
287void ZeroSuggestProvider::AddMatchToMap(int relevance,
288                                        AutocompleteMatch::Type type,
289                                        const TemplateURL* template_url,
290                                        const string16& query_string,
291                                        int accepted_suggestion,
292                                        SearchProvider::MatchMap* map) {
293  // Pass in query_string as the input_text since we don't want any bolding.
294  // TODO(samarth|melevin): use the actual omnibox margin here as well instead
295  // of passing in -1.
296  AutocompleteMatch match = SearchProvider::CreateSearchSuggestion(
297      this, AutocompleteInput(), query_string, relevance, type, false,
298      query_string, string16(), template_url, query_string, std::string(),
299      accepted_suggestion, -1, true);
300  if (!match.destination_url.is_valid())
301    return;
302
303  // Try to add |match| to |map|.  If a match for |query_string| is already in
304  // |map|, replace it if |match| is more relevant.
305  // NOTE: Keep this ToLower() call in sync with url_database.cc.
306  SearchProvider::MatchKey match_key(
307      std::make_pair(base::i18n::ToLower(query_string), std::string()));
308  const std::pair<SearchProvider::MatchMap::iterator, bool> i(map->insert(
309      std::make_pair(match_key, match)));
310  // NOTE: We purposefully do a direct relevance comparison here instead of
311  // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items added
312  // first" rather than "items alphabetically first" when the scores are equal.
313  // The only case this matters is when a user has results with the same score
314  // that differ only by capitalization; because the history system returns
315  // results sorted by recency, this means we'll pick the most recent such
316  // result even if the precision of our relevance score is too low to
317  // distinguish the two.
318  if (!i.second && (match.relevance > i.first->second.relevance))
319    i.first->second = match;
320}
321
322AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
323    const SearchProvider::NavigationResult& navigation) {
324  AutocompleteMatch match(this, navigation.relevance(), false,
325                          AutocompleteMatchType::NAVSUGGEST);
326  match.destination_url = navigation.url();
327
328  const std::string languages(
329      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
330  match.contents = net::FormatUrl(navigation.url(), languages,
331      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
332  match.fill_into_edit +=
333      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
334          match.contents);
335
336  AutocompleteMatch::ClassifyLocationInString(string16::npos, 0,
337      match.contents.length(), ACMatchClassification::URL,
338      &match.contents_class);
339
340  match.description =
341      AutocompleteMatch::SanitizeString(navigation.description());
342  AutocompleteMatch::ClassifyLocationInString(string16::npos, 0,
343      match.description.length(), ACMatchClassification::NONE,
344      &match.description_class);
345  return match;
346}
347
348void ZeroSuggestProvider::Run(const GURL& suggest_url) {
349  have_pending_request_ = false;
350  const int kFetcherID = 1;
351  fetcher_.reset(
352      net::URLFetcher::Create(kFetcherID,
353          suggest_url,
354          net::URLFetcher::GET, this));
355  fetcher_->SetRequestContext(profile_->GetRequestContext());
356  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
357  // Add Chrome experiment state to the request headers.
358  net::HttpRequestHeaders headers;
359  chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
360      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
361  fetcher_->SetExtraRequestHeaders(headers.ToString());
362
363  fetcher_->Start();
364
365  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
366    most_visited_urls_.clear();
367    history::TopSites* ts = profile_->GetTopSites();
368    if (ts) {
369      ts->GetMostVisitedURLs(
370          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
371                     weak_ptr_factory_.GetWeakPtr()), false);
372    }
373  }
374  have_pending_request_ = true;
375  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
376}
377
378void ZeroSuggestProvider::ParseSuggestResults(const Value& root_val) {
379  SearchProvider::SuggestResults suggest_results;
380  FillResults(root_val, &verbatim_relevance_,
381              &suggest_results, &navigation_results_);
382
383  query_matches_map_.clear();
384  AddSuggestResultsToMap(suggest_results,
385                         template_url_service_->GetDefaultSearchProvider(),
386                         &query_matches_map_);
387}
388
389void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
390    const history::MostVisitedURLList& urls) {
391  most_visited_urls_ = urls;
392}
393
394void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
395  matches_.clear();
396
397  const TemplateURL* default_provider =
398      template_url_service_->GetDefaultSearchProvider();
399  // Fail if we can't set the clickthrough URL for query suggestions.
400  if (default_provider == NULL || !default_provider->SupportsReplacement())
401    return;
402
403  const int num_query_results = query_matches_map_.size();
404  const int num_nav_results = navigation_results_.size();
405  const int num_results = num_query_results + num_nav_results;
406  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
407  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
408  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
409
410  // Show Most Visited results after ZeroSuggest response is received.
411  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
412    if (!current_url_match_.destination_url.is_valid())
413      return;
414    matches_.push_back(current_url_match_);
415    int relevance = 600;
416    if (num_results > 0) {
417      UMA_HISTOGRAM_COUNTS(
418          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
419          most_visited_urls_.size());
420    }
421    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
422      const history::MostVisitedURL& url = most_visited_urls_[i];
423      SearchProvider::NavigationResult nav(*this, url.url, url.title, false,
424                                           relevance, true);
425      matches_.push_back(NavigationToMatch(nav));
426      --relevance;
427    }
428    return;
429  }
430
431  if (num_results == 0)
432    return;
433
434  // TODO(jered): Rip this out once the first match is decoupled from the
435  // current typing in the omnibox.
436  matches_.push_back(current_url_match_);
437
438  for (SearchProvider::MatchMap::const_iterator it(query_matches_map_.begin());
439       it != query_matches_map_.end(); ++it)
440    matches_.push_back(it->second);
441
442  for (SearchProvider::NavigationResults::const_iterator it(
443       navigation_results_.begin()); it != navigation_results_.end(); ++it)
444    matches_.push_back(NavigationToMatch(*it));
445}
446
447AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
448  AutocompleteInput input(permanent_text_, string16::npos, string16(),
449                          GURL(current_query_), current_page_classification_,
450                          false, false, true, AutocompleteInput::ALL_MATCHES);
451
452  AutocompleteMatch match;
453  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
454      permanent_text_, false, true, &match, NULL);
455  match.is_history_what_you_typed_match = false;
456  match.allowed_to_be_default_match = true;
457
458  // The placeholder suggestion for the current URL has high relevance so
459  // that it is in the first suggestion slot and inline autocompleted. It
460  // gets dropped as soon as the user types something.
461  match.relevance = verbatim_relevance_;
462
463  return match;
464}
465