zero_suggest_provider.cc revision 5d1f7b1de12d16ceb2c938c56701a3e8bfa558f7 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/autocomplete/zero_suggest_provider.h"
6
7#include "base/callback.h"
8#include "base/i18n/case_conversion.h"
9#include "base/json/json_string_value_serializer.h"
10#include "base/metrics/histogram.h"
11#include "base/prefs/pref_service.h"
12#include "base/strings/string16.h"
13#include "base/strings/string_util.h"
14#include "base/strings/utf_string_conversions.h"
15#include "base/time/time.h"
16#include "chrome/browser/autocomplete/autocomplete_classifier.h"
17#include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
18#include "chrome/browser/autocomplete/autocomplete_input.h"
19#include "chrome/browser/autocomplete/autocomplete_match.h"
20#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
21#include "chrome/browser/autocomplete/history_url_provider.h"
22#include "chrome/browser/autocomplete/search_provider.h"
23#include "chrome/browser/autocomplete/url_prefix.h"
24#include "chrome/browser/history/history_types.h"
25#include "chrome/browser/history/top_sites.h"
26#include "chrome/browser/metrics/variations/variations_http_header_provider.h"
27#include "chrome/browser/omnibox/omnibox_field_trial.h"
28#include "chrome/browser/profiles/profile.h"
29#include "chrome/browser/search/search.h"
30#include "chrome/browser/search_engines/template_url_service.h"
31#include "chrome/browser/search_engines/template_url_service_factory.h"
32#include "chrome/common/net/url_fixer_upper.h"
33#include "chrome/common/pref_names.h"
34#include "chrome/common/url_constants.h"
35#include "net/base/escape.h"
36#include "net/base/load_flags.h"
37#include "net/base/net_util.h"
38#include "net/http/http_request_headers.h"
39#include "net/http/http_response_headers.h"
40#include "net/url_request/url_fetcher.h"
41#include "net/url_request/url_request_status.h"
42#include "url/gurl.h"
43
44namespace {
45
46// TODO(hfung): The histogram code was copied and modified from
47// search_provider.cc.  Refactor and consolidate the code.
48// We keep track in a histogram how many suggest requests we send, how
49// many suggest requests we invalidate (e.g., due to a user typing
50// another character), and how many replies we receive.
51// *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
52//     (excluding the end-of-list enum value)
53// We do not want values of existing enums to change or else it screws
54// up the statistics.
55enum ZeroSuggestRequestsHistogramValue {
56  ZERO_SUGGEST_REQUEST_SENT = 1,
57  ZERO_SUGGEST_REQUEST_INVALIDATED,
58  ZERO_SUGGEST_REPLY_RECEIVED,
59  ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
60};
61
62void LogOmniboxZeroSuggestRequest(
63    ZeroSuggestRequestsHistogramValue request_value) {
64  UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
65                            ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
66}
67
68// The maximum relevance of the top match from this provider.
69const int kDefaultVerbatimZeroSuggestRelevance = 1300;
70
71// Relevance value to use if it was not set explicitly by the server.
72const int kDefaultZeroSuggestRelevance = 100;
73
74}  // namespace
75
76// static
77ZeroSuggestProvider* ZeroSuggestProvider::Create(
78    AutocompleteProviderListener* listener,
79    Profile* profile) {
80  return new ZeroSuggestProvider(listener, profile);
81}
82
83void ZeroSuggestProvider::Start(const AutocompleteInput& input,
84                                bool /*minimal_changes*/) {
85}
86
87void ZeroSuggestProvider::ResetSession() {
88  // The user has started editing in the omnibox, so leave
89  // |field_trial_triggered_in_session_| unchanged and set
90  // |field_trial_triggered_| to false since zero suggest is inactive now.
91  field_trial_triggered_ = false;
92  Stop(true);
93}
94
95void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
96  have_pending_request_ = false;
97  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
98
99  std::string json_data;
100  source->GetResponseAsString(&json_data);
101  const bool request_succeeded =
102      source->GetStatus().is_success() && source->GetResponseCode() == 200;
103
104  if (request_succeeded) {
105    scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
106    if (data.get())
107      ParseSuggestResults(*data.get());
108  }
109  done_ = true;
110
111  ConvertResultsToAutocompleteMatches();
112  if (!matches_.empty())
113    listener_->OnProviderUpdate(true);
114}
115
116void ZeroSuggestProvider::StartZeroSuggest(
117    const GURL& current_page_url,
118    AutocompleteInput::PageClassification page_classification,
119    const base::string16& permanent_text) {
120  Stop(true);
121  field_trial_triggered_ = false;
122  field_trial_triggered_in_session_ = false;
123  permanent_text_ = permanent_text;
124  current_query_ = current_page_url.spec();
125  current_page_classification_ = page_classification;
126  current_url_match_ = MatchForCurrentURL();
127
128  const TemplateURL* default_provider =
129     template_url_service_->GetDefaultSearchProvider();
130  if (default_provider == NULL)
131    return;
132  base::string16 prefix;
133  TemplateURLRef::SearchTermsArgs search_term_args(prefix);
134  search_term_args.current_page_url = current_query_;
135  GURL suggest_url(default_provider->suggestions_url_ref().
136                   ReplaceSearchTerms(search_term_args));
137  if (!CanSendURL(current_page_url, suggest_url,
138          template_url_service_->GetDefaultSearchProvider(),
139          page_classification, profile_) ||
140      !OmniboxFieldTrial::InZeroSuggestFieldTrial())
141    return;
142  verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance;
143  done_ = false;
144  // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
145  // These may be useful on the NTP or more relevant to the user than server
146  // suggestions, if based on local browsing history.
147  Run(suggest_url);
148}
149
150ZeroSuggestProvider::ZeroSuggestProvider(
151  AutocompleteProviderListener* listener,
152  Profile* profile)
153    : BaseSearchProvider(listener, profile,
154                         AutocompleteProvider::TYPE_ZERO_SUGGEST),
155      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
156      have_pending_request_(false),
157      verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance),
158      weak_ptr_factory_(this) {
159}
160
161ZeroSuggestProvider::~ZeroSuggestProvider() {
162}
163
164const TemplateURL* ZeroSuggestProvider::GetTemplateURL(
165    const SuggestResult& result) const {
166  // Zero suggest provider should not receive keyword results.
167  DCHECK(!result.from_keyword_provider());
168  return template_url_service_->GetDefaultSearchProvider();
169}
170
171const AutocompleteInput ZeroSuggestProvider::GetInput(
172    const SuggestResult& result) const {
173  AutocompleteInput input;
174  // Set |input|'s text to be |query_string| to avoid bolding.
175  input.UpdateText(result.suggestion(), base::string16::npos, input.parts());
176  return input;
177}
178
179bool ZeroSuggestProvider::ShouldAppendExtraParams(
180      const SuggestResult& result) const {
181  // We always use the default provider for search, so append the params.
182  return true;
183}
184
185void ZeroSuggestProvider::StopSuggest() {
186  if (have_pending_request_)
187    LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
188  have_pending_request_ = false;
189  fetcher_.reset();
190}
191
192void ZeroSuggestProvider::ClearAllResults() {
193  query_matches_map_.clear();
194  navigation_results_.clear();
195  current_query_.clear();
196  matches_.clear();
197}
198
199void ZeroSuggestProvider::FillResults(const base::Value& root_val,
200                                      int* verbatim_relevance,
201                                      SuggestResults* suggest_results,
202                                      NavigationResults* navigation_results) {
203  base::string16 query;
204  const base::ListValue* root_list = NULL;
205  const base::ListValue* results = NULL;
206  const base::ListValue* relevances = NULL;
207  // The response includes the query, which should be empty for ZeroSuggest
208  // responses.
209  if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
210      (!query.empty()) || !root_list->GetList(1, &results))
211    return;
212
213  // 3rd element: Description list.
214  const base::ListValue* descriptions = NULL;
215  root_list->GetList(2, &descriptions);
216
217  // 4th element: Disregard the query URL list for now.
218
219  // Reset suggested relevance information from the provider.
220  *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance;
221
222  // 5th element: Optional key-value pairs from the Suggest server.
223  const base::ListValue* types = NULL;
224  const base::DictionaryValue* extras = NULL;
225  if (root_list->GetDictionary(4, &extras)) {
226    extras->GetList("google:suggesttype", &types);
227
228    // Discard this list if its size does not match that of the suggestions.
229    if (extras->GetList("google:suggestrelevance", &relevances) &&
230        relevances->GetSize() != results->GetSize())
231      relevances = NULL;
232    extras->GetInteger("google:verbatimrelevance", verbatim_relevance);
233
234    // Check if the active suggest field trial (if any) has triggered.
235    bool triggered = false;
236    extras->GetBoolean("google:fieldtrialtriggered", &triggered);
237    field_trial_triggered_ |= triggered;
238    field_trial_triggered_in_session_ |= triggered;
239  }
240
241  // Clear the previous results now that new results are available.
242  suggest_results->clear();
243  navigation_results->clear();
244
245  base::string16 result, title;
246  std::string type;
247  const base::string16 current_query_string16 =
248      base::ASCIIToUTF16(current_query_);
249  const std::string languages(
250      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
251  for (size_t index = 0; results->GetString(index, &result); ++index) {
252    // Google search may return empty suggestions for weird input characters,
253    // they make no sense at all and can cause problems in our code.
254    if (result.empty())
255      continue;
256
257    int relevance = kDefaultZeroSuggestRelevance;
258
259    // Apply valid suggested relevance scores; discard invalid lists.
260    if (relevances != NULL && !relevances->GetInteger(index, &relevance))
261      relevances = NULL;
262    if (types && types->GetString(index, &type) && (type == "NAVIGATION")) {
263      // Do not blindly trust the URL coming from the server to be valid.
264      GURL url(URLFixerUpper::FixupURL(
265          base::UTF16ToUTF8(result), std::string()));
266      if (url.is_valid()) {
267        if (descriptions != NULL)
268          descriptions->GetString(index, &title);
269        navigation_results->push_back(NavigationResult(
270            *this, url, title, false, relevance, relevances != NULL,
271            current_query_string16, languages));
272      }
273    } else {
274      suggest_results->push_back(SuggestResult(
275          result, AutocompleteMatchType::SEARCH_SUGGEST, result,
276          base::string16(), std::string(), std::string(), false, relevance,
277          relevances != NULL, false, current_query_string16));
278    }
279  }
280}
281
282void ZeroSuggestProvider::AddSuggestResultsToMap(
283    const SuggestResults& results,
284    MatchMap* map) {
285  for (size_t i = 0; i < results.size(); ++i) {
286    const base::string16& query_string(results[i].suggestion());
287    // TODO(mariakhomenko): Do not reconstruct SuggestResult objects with
288    // a different query -- create correct objects to begin with.
289    const SuggestResult suggestion(
290        query_string, AutocompleteMatchType::SEARCH_SUGGEST, query_string,
291        base::string16(), std::string(), std::string(), false,
292        results[i].relevance(), true, false, query_string);
293    AddMatchToMap(suggestion, std::string(), i, map);
294  }
295}
296
297AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
298    const NavigationResult& navigation) {
299  AutocompleteMatch match(this, navigation.relevance(), false,
300                          AutocompleteMatchType::NAVSUGGEST);
301  match.destination_url = navigation.url();
302
303  // Zero suggest results should always omit protocols and never appear bold.
304  const std::string languages(
305      profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
306  match.contents = net::FormatUrl(navigation.url(), languages,
307      net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
308  match.fill_into_edit +=
309      AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
310          match.contents);
311
312  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
313      match.contents.length(), ACMatchClassification::URL,
314      &match.contents_class);
315
316  match.description =
317      AutocompleteMatch::SanitizeString(navigation.description());
318  AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
319      match.description.length(), ACMatchClassification::NONE,
320      &match.description_class);
321  return match;
322}
323
324void ZeroSuggestProvider::Run(const GURL& suggest_url) {
325  have_pending_request_ = false;
326  const int kFetcherID = 1;
327  fetcher_.reset(
328      net::URLFetcher::Create(kFetcherID,
329          suggest_url,
330          net::URLFetcher::GET, this));
331  fetcher_->SetRequestContext(profile_->GetRequestContext());
332  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
333  // Add Chrome experiment state to the request headers.
334  net::HttpRequestHeaders headers;
335  chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
336      fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
337  fetcher_->SetExtraRequestHeaders(headers.ToString());
338
339  fetcher_->Start();
340
341  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
342    most_visited_urls_.clear();
343    history::TopSites* ts = profile_->GetTopSites();
344    if (ts) {
345      ts->GetMostVisitedURLs(
346          base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
347                     weak_ptr_factory_.GetWeakPtr()), false);
348    }
349  }
350  have_pending_request_ = true;
351  LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
352}
353
354void ZeroSuggestProvider::ParseSuggestResults(const base::Value& root_val) {
355  SuggestResults suggest_results;
356  FillResults(root_val, &verbatim_relevance_,
357              &suggest_results, &navigation_results_);
358
359  query_matches_map_.clear();
360  AddSuggestResultsToMap(suggest_results, &query_matches_map_);
361}
362
363void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
364    const history::MostVisitedURLList& urls) {
365  most_visited_urls_ = urls;
366}
367
368void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
369  matches_.clear();
370
371  const TemplateURL* default_provider =
372      template_url_service_->GetDefaultSearchProvider();
373  // Fail if we can't set the clickthrough URL for query suggestions.
374  if (default_provider == NULL || !default_provider->SupportsReplacement())
375    return;
376
377  const int num_query_results = query_matches_map_.size();
378  const int num_nav_results = navigation_results_.size();
379  const int num_results = num_query_results + num_nav_results;
380  UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
381  UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
382  UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
383
384  // Show Most Visited results after ZeroSuggest response is received.
385  if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
386    if (!current_url_match_.destination_url.is_valid())
387      return;
388    matches_.push_back(current_url_match_);
389    int relevance = 600;
390    if (num_results > 0) {
391      UMA_HISTOGRAM_COUNTS(
392          "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
393          most_visited_urls_.size());
394    }
395    const base::string16 current_query_string16(
396        base::ASCIIToUTF16(current_query_));
397    const std::string languages(
398        profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
399    for (size_t i = 0; i < most_visited_urls_.size(); i++) {
400      const history::MostVisitedURL& url = most_visited_urls_[i];
401      NavigationResult nav(*this, url.url, url.title, false, relevance, true,
402          current_query_string16, languages);
403      matches_.push_back(NavigationToMatch(nav));
404      --relevance;
405    }
406    return;
407  }
408
409  if (num_results == 0)
410    return;
411
412  // TODO(jered): Rip this out once the first match is decoupled from the
413  // current typing in the omnibox.
414  matches_.push_back(current_url_match_);
415
416  for (MatchMap::const_iterator it(query_matches_map_.begin());
417       it != query_matches_map_.end(); ++it)
418    matches_.push_back(it->second);
419
420  for (NavigationResults::const_iterator it(navigation_results_.begin());
421       it != navigation_results_.end(); ++it)
422    matches_.push_back(NavigationToMatch(*it));
423}
424
425AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
426  AutocompleteInput input(permanent_text_, base::string16::npos, base::string16(),
427                          GURL(current_query_), current_page_classification_,
428                          false, false, true, AutocompleteInput::ALL_MATCHES);
429
430  AutocompleteMatch match;
431  AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
432      permanent_text_, false, true, current_page_classification_, &match, NULL);
433  match.is_history_what_you_typed_match = false;
434  match.allowed_to_be_default_match = true;
435
436  // The placeholder suggestion for the current URL has high relevance so
437  // that it is in the first suggestion slot and inline autocompleted. It
438  // gets dropped as soon as the user types something.
439  match.relevance = verbatim_relevance_;
440
441  return match;
442}
443