base_search_provider.cc revision 1320f92c476a1ad9d19dba2a48c72b75566198e9 
1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "components/omnibox/base_search_provider.h"
6
7#include "base/i18n/case_conversion.h"
8#include "base/strings/string_util.h"
9#include "base/strings/utf_string_conversions.h"
10#include "components/metrics/proto/omnibox_event.pb.h"
11#include "components/metrics/proto/omnibox_input_type.pb.h"
12#include "components/omnibox/autocomplete_provider_client.h"
13#include "components/omnibox/autocomplete_provider_listener.h"
14#include "components/omnibox/omnibox_field_trial.h"
15#include "components/search_engines/template_url.h"
16#include "components/search_engines/template_url_prepopulate_data.h"
17#include "components/search_engines/template_url_service.h"
18#include "net/base/registry_controlled_domains/registry_controlled_domain.h"
19#include "net/url_request/url_fetcher.h"
20#include "net/url_request/url_fetcher_delegate.h"
21#include "url/gurl.h"
22
23using metrics::OmniboxEventProto;
24
25// SuggestionDeletionHandler -------------------------------------------------
26
27// This class handles making requests to the server in order to delete
28// personalized suggestions.
29class SuggestionDeletionHandler : public net::URLFetcherDelegate {
30 public:
31  typedef base::Callback<void(bool, SuggestionDeletionHandler*)>
32      DeletionCompletedCallback;
33
34  SuggestionDeletionHandler(
35      const std::string& deletion_url,
36      net::URLRequestContextGetter* request_context,
37      const DeletionCompletedCallback& callback);
38
39  virtual ~SuggestionDeletionHandler();
40
41 private:
42  // net::URLFetcherDelegate:
43  virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
44
45  scoped_ptr<net::URLFetcher> deletion_fetcher_;
46  DeletionCompletedCallback callback_;
47
48  DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
49};
50
51SuggestionDeletionHandler::SuggestionDeletionHandler(
52    const std::string& deletion_url,
53    net::URLRequestContextGetter* request_context,
54    const DeletionCompletedCallback& callback) : callback_(callback) {
55  GURL url(deletion_url);
56  DCHECK(url.is_valid());
57
58  deletion_fetcher_.reset(net::URLFetcher::Create(
59      BaseSearchProvider::kDeletionURLFetcherID,
60      url,
61      net::URLFetcher::GET,
62      this));
63  deletion_fetcher_->SetRequestContext(request_context);
64  deletion_fetcher_->Start();
65}
66
67SuggestionDeletionHandler::~SuggestionDeletionHandler() {
68}
69
70void SuggestionDeletionHandler::OnURLFetchComplete(
71    const net::URLFetcher* source) {
72  DCHECK(source == deletion_fetcher_.get());
73  callback_.Run(
74      source->GetStatus().is_success() && (source->GetResponseCode() == 200),
75      this);
76}
77
78// BaseSearchProvider ---------------------------------------------------------
79
80// static
81const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1;
82const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2;
83const int BaseSearchProvider::kDeletionURLFetcherID = 3;
84
85BaseSearchProvider::BaseSearchProvider(
86    TemplateURLService* template_url_service,
87    scoped_ptr<AutocompleteProviderClient> client,
88    AutocompleteProvider::Type type)
89    : AutocompleteProvider(type),
90      template_url_service_(template_url_service),
91      client_(client.Pass()),
92      field_trial_triggered_(false),
93      field_trial_triggered_in_session_(false) {
94}
95
96// static
97bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
98  return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
99}
100
101// static
102AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
103    const base::string16& suggestion,
104    AutocompleteMatchType::Type type,
105    bool from_keyword_provider,
106    const TemplateURL* template_url,
107    const SearchTermsData& search_terms_data) {
108  // These calls use a number of default values.  For instance, they assume
109  // that if this match is from a keyword provider, then the user is in keyword
110  // mode.  They also assume the caller knows what it's doing and we set
111  // this match to look as if it was received/created synchronously.
112  SearchSuggestionParser::SuggestResult suggest_result(
113      suggestion, type, suggestion, base::string16(), base::string16(),
114      base::string16(), base::string16(), std::string(), std::string(),
115      from_keyword_provider, 0, false, false, base::string16());
116  suggest_result.set_received_after_last_keystroke(false);
117  return CreateSearchSuggestion(
118      NULL, AutocompleteInput(), from_keyword_provider, suggest_result,
119      template_url, search_terms_data, 0, false);
120}
121
122void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) {
123  DCHECK(match.deletable);
124  if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) {
125    deletion_handlers_.push_back(new SuggestionDeletionHandler(
126        match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey),
127        client_->RequestContext(),
128        base::Bind(&BaseSearchProvider::OnDeletionComplete,
129                   base::Unretained(this))));
130  }
131
132  TemplateURL* template_url =
133      match.GetTemplateURL(template_url_service_, false);
134  // This may be NULL if the template corresponding to the keyword has been
135  // deleted or there is no keyword set.
136  if (template_url != NULL) {
137    client_->DeleteMatchingURLsForKeywordFromHistory(template_url->id(),
138                                                     match.contents);
139  }
140
141  // Immediately update the list of matches to show the match was deleted,
142  // regardless of whether the server request actually succeeds.
143  DeleteMatchFromMatches(match);
144}
145
146void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
147  provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
148  metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
149  new_entry.set_provider(AsOmniboxEventProviderType());
150  new_entry.set_provider_done(done_);
151  std::vector<uint32> field_trial_hashes;
152  OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
153  for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
154    if (field_trial_triggered_)
155      new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
156    if (field_trial_triggered_in_session_) {
157      new_entry.mutable_field_trial_triggered_in_session()->Add(
158          field_trial_hashes[i]);
159    }
160  }
161}
162
163// static
164const char BaseSearchProvider::kRelevanceFromServerKey[] =
165    "relevance_from_server";
166const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
167const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
168const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
169const char BaseSearchProvider::kTrue[] = "true";
170const char BaseSearchProvider::kFalse[] = "false";
171
172BaseSearchProvider::~BaseSearchProvider() {}
173
174void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url,
175                                        AutocompleteMatch* match) {
176  if (deletion_url.empty())
177    return;
178  if (!template_url_service_)
179    return;
180  GURL url =
181      template_url_service_->GetDefaultSearchProvider()->GenerateSearchURL(
182          template_url_service_->search_terms_data());
183  url = url.GetOrigin().Resolve(deletion_url);
184  if (url.is_valid()) {
185    match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey,
186        url.spec());
187    match->deletable = true;
188  }
189}
190
191// static
192AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
193    AutocompleteProvider* autocomplete_provider,
194    const AutocompleteInput& input,
195    const bool in_keyword_mode,
196    const SearchSuggestionParser::SuggestResult& suggestion,
197    const TemplateURL* template_url,
198    const SearchTermsData& search_terms_data,
199    int accepted_suggestion,
200    bool append_extra_query_params) {
201  AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
202                          suggestion.type());
203
204  if (!template_url)
205    return match;
206  match.keyword = template_url->keyword();
207  match.contents = suggestion.match_contents();
208  match.contents_class = suggestion.match_contents_class();
209  match.answer_contents = suggestion.answer_contents();
210  match.answer_type = suggestion.answer_type();
211  if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
212    match.RecordAdditionalInfo(
213        kACMatchPropertyInputText, base::UTF16ToUTF8(input.text()));
214    match.RecordAdditionalInfo(
215        kACMatchPropertyContentsPrefix,
216        base::UTF16ToUTF8(suggestion.match_contents_prefix()));
217    match.RecordAdditionalInfo(
218        kACMatchPropertyContentsStartIndex,
219        static_cast<int>(
220            suggestion.suggestion().length() - match.contents.length()));
221  }
222
223  if (!suggestion.annotation().empty())
224    match.description = suggestion.annotation();
225
226  // suggestion.match_contents() should have already been collapsed.
227  match.allowed_to_be_default_match =
228      (!in_keyword_mode || suggestion.from_keyword_provider()) &&
229      (base::CollapseWhitespace(input.text(), false) ==
230       suggestion.match_contents());
231
232  // When the user forced a query, we need to make sure all the fill_into_edit
233  // values preserve that property.  Otherwise, if the user starts editing a
234  // suggestion, non-Search results will suddenly appear.
235  if (input.type() == metrics::OmniboxInputType::FORCED_QUERY)
236    match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
237  if (suggestion.from_keyword_provider())
238    match.fill_into_edit.append(match.keyword + base::char16(' '));
239  // We only allow inlinable navsuggestions that were received before the
240  // last keystroke because we don't want asynchronous inline autocompletions.
241  if (!input.prevent_inline_autocomplete() &&
242      !suggestion.received_after_last_keystroke() &&
243      (!in_keyword_mode || suggestion.from_keyword_provider()) &&
244      StartsWith(suggestion.suggestion(), input.text(), false)) {
245    match.inline_autocompletion =
246        suggestion.suggestion().substr(input.text().length());
247    match.allowed_to_be_default_match = true;
248  }
249  match.fill_into_edit.append(suggestion.suggestion());
250
251  const TemplateURLRef& search_url = template_url->url_ref();
252  DCHECK(search_url.SupportsReplacement(search_terms_data));
253  match.search_terms_args.reset(
254      new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
255  match.search_terms_args->original_query = input.text();
256  match.search_terms_args->accepted_suggestion = accepted_suggestion;
257  match.search_terms_args->enable_omnibox_start_margin = true;
258  match.search_terms_args->suggest_query_params =
259      suggestion.suggest_query_params();
260  match.search_terms_args->append_extra_query_params =
261      append_extra_query_params;
262  // This is the destination URL sans assisted query stats.  This must be set
263  // so the AutocompleteController can properly de-dupe; the controller will
264  // eventually overwrite it before it reaches the user.
265  match.destination_url =
266      GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(),
267                                         search_terms_data));
268
269  // Search results don't look like URLs.
270  match.transition = suggestion.from_keyword_provider() ?
271      ui::PAGE_TRANSITION_KEYWORD : ui::PAGE_TRANSITION_GENERATED;
272
273  return match;
274}
275
276// static
277bool BaseSearchProvider::ZeroSuggestEnabled(
278    const GURL& suggest_url,
279    const TemplateURL* template_url,
280    OmniboxEventProto::PageClassification page_classification,
281    const SearchTermsData& search_terms_data,
282    AutocompleteProviderClient* client) {
283  if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
284    return false;
285
286  // Make sure we are sending the suggest request through HTTPS to prevent
287  // exposing the current page URL or personalized results without encryption.
288  if (!suggest_url.SchemeIs(url::kHttpsScheme))
289    return false;
290
291  // Don't show zero suggest on the NTP.
292  // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
293  // under the conditions described in crbug.com/305366.
294  if ((page_classification ==
295       OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
296      (page_classification ==
297       OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
298    return false;
299
300  // Don't run if in incognito mode.
301  if (client->IsOffTheRecord())
302    return false;
303
304  // Don't run if we can't get preferences or search suggest is not enabled.
305  if (!client->SearchSuggestEnabled())
306    return false;
307
308  // Only make the request if we know that the provider supports zero suggest
309  // (currently only the prepopulated Google provider).
310  if (template_url == NULL ||
311      !template_url->SupportsReplacement(search_terms_data) ||
312      TemplateURLPrepopulateData::GetEngineType(
313          *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE)
314    return false;
315
316  return true;
317}
318
319// static
320bool BaseSearchProvider::CanSendURL(
321    const GURL& current_page_url,
322    const GURL& suggest_url,
323    const TemplateURL* template_url,
324    OmniboxEventProto::PageClassification page_classification,
325    const SearchTermsData& search_terms_data,
326    AutocompleteProviderClient* client) {
327  if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
328                          search_terms_data, client))
329    return false;
330
331  if (!current_page_url.is_valid())
332    return false;
333
334  // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
335  // provider.
336  if ((current_page_url.scheme() != url::kHttpScheme) &&
337      ((current_page_url.scheme() != url::kHttpsScheme) ||
338       !net::registry_controlled_domains::SameDomainOrHost(
339           current_page_url, suggest_url,
340           net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
341    return false;
342
343  if (!client->TabSyncEnabledAndUnencrypted())
344    return false;
345
346  return true;
347}
348
349void BaseSearchProvider::AddMatchToMap(
350    const SearchSuggestionParser::SuggestResult& result,
351    const std::string& metadata,
352    int accepted_suggestion,
353    bool mark_as_deletable,
354    bool in_keyword_mode,
355    MatchMap* map) {
356  AutocompleteMatch match = CreateSearchSuggestion(
357      this, GetInput(result.from_keyword_provider()), in_keyword_mode, result,
358      GetTemplateURL(result.from_keyword_provider()),
359      template_url_service_->search_terms_data(), accepted_suggestion,
360      ShouldAppendExtraParams(result));
361  if (!match.destination_url.is_valid())
362    return;
363  match.search_terms_args->bookmark_bar_pinned = client_->ShowBookmarkBar();
364  match.RecordAdditionalInfo(kRelevanceFromServerKey,
365                             result.relevance_from_server() ? kTrue : kFalse);
366  match.RecordAdditionalInfo(kShouldPrefetchKey,
367                             result.should_prefetch() ? kTrue : kFalse);
368  SetDeletionURL(result.deletion_url(), &match);
369  if (mark_as_deletable)
370    match.deletable = true;
371  // Metadata is needed only for prefetching queries.
372  if (result.should_prefetch())
373    match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
374
375  // Try to add |match| to |map|.  If a match for this suggestion is
376  // already in |map|, replace it if |match| is more relevant.
377  // NOTE: Keep this ToLower() call in sync with url_database.cc.
378  MatchKey match_key(
379      std::make_pair(base::i18n::ToLower(result.suggestion()),
380                     match.search_terms_args->suggest_query_params));
381  const std::pair<MatchMap::iterator, bool> i(
382       map->insert(std::make_pair(match_key, match)));
383
384  bool should_prefetch = result.should_prefetch();
385  if (!i.second) {
386    // NOTE: We purposefully do a direct relevance comparison here instead of
387    // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
388    // added first" rather than "items alphabetically first" when the scores
389    // are equal. The only case this matters is when a user has results with
390    // the same score that differ only by capitalization; because the history
391    // system returns results sorted by recency, this means we'll pick the most
392    // recent such result even if the precision of our relevance score is too
393    // low to distinguish the two.
394    if (match.relevance > i.first->second.relevance) {
395      match.duplicate_matches.insert(match.duplicate_matches.end(),
396                                     i.first->second.duplicate_matches.begin(),
397                                     i.first->second.duplicate_matches.end());
398      i.first->second.duplicate_matches.clear();
399      match.duplicate_matches.push_back(i.first->second);
400      i.first->second = match;
401    } else {
402      i.first->second.duplicate_matches.push_back(match);
403      if (match.keyword == i.first->second.keyword) {
404        // Old and new matches are from the same search provider. It is okay to
405        // record one match's prefetch data onto a different match (for the same
406        // query string) for the following reasons:
407        // 1. Because the suggest server only sends down a query string from
408        // which we construct a URL, rather than sending a full URL, and because
409        // we construct URLs from query strings in the same way every time, the
410        // URLs for the two matches will be the same. Therefore, we won't end up
411        // prefetching something the server didn't intend.
412        // 2. Presumably the server sets the prefetch bit on a match it things
413        // is sufficiently relevant that the user is likely to choose it.
414        // Surely setting the prefetch bit on a match of even higher relevance
415        // won't violate this assumption.
416        should_prefetch |= ShouldPrefetch(i.first->second);
417        i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
418                                             should_prefetch ? kTrue : kFalse);
419        if (should_prefetch)
420          i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
421      }
422    }
423    // Copy over answer data from lower-ranking item, if necessary.
424    // This depends on the lower-ranking item always being added last - see
425    // use of push_back above.
426    AutocompleteMatch& more_relevant_match = i.first->second;
427    const AutocompleteMatch& less_relevant_match =
428        more_relevant_match.duplicate_matches.back();
429    if (!less_relevant_match.answer_type.empty() &&
430        more_relevant_match.answer_type.empty()) {
431      more_relevant_match.answer_type = less_relevant_match.answer_type;
432      more_relevant_match.answer_contents = less_relevant_match.answer_contents;
433    }
434  }
435}
436
437bool BaseSearchProvider::ParseSuggestResults(
438    const base::Value& root_val,
439    int default_result_relevance,
440    bool is_keyword_result,
441    SearchSuggestionParser::Results* results) {
442  if (!SearchSuggestionParser::ParseSuggestResults(
443      root_val, GetInput(is_keyword_result),
444      client_->SchemeClassifier(), default_result_relevance,
445      client_->AcceptLanguages(), is_keyword_result, results))
446    return false;
447
448  for (std::vector<GURL>::const_iterator it =
449           results->answers_image_urls.begin();
450       it != results->answers_image_urls.end(); ++it)
451    client_->PrefetchImage(*it);
452
453  field_trial_triggered_ |= results->field_trial_triggered;
454  field_trial_triggered_in_session_ |= results->field_trial_triggered;
455  return true;
456}
457
458void BaseSearchProvider::DeleteMatchFromMatches(
459    const AutocompleteMatch& match) {
460  for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) {
461    // Find the desired match to delete by checking the type and contents.
462    // We can't check the destination URL, because the autocomplete controller
463    // may have reformulated that. Not that while checking for matching
464    // contents works for personalized suggestions, if more match types gain
465    // deletion support, this algorithm may need to be re-examined.
466    if (i->contents == match.contents && i->type == match.type) {
467      matches_.erase(i);
468      break;
469    }
470  }
471}
472
473void BaseSearchProvider::OnDeletionComplete(
474    bool success, SuggestionDeletionHandler* handler) {
475  RecordDeletionResult(success);
476  SuggestionDeletionHandlers::iterator it = std::find(
477      deletion_handlers_.begin(), deletion_handlers_.end(), handler);
478  DCHECK(it != deletion_handlers_.end());
479  deletion_handlers_.erase(it);
480}
481