template_url_model.cc revision c407dc5cd9bdc5668497f21b26b09d988ab439de 
1// Copyright (c) 2010 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/search_engines/template_url_model.h"
6
7#include "app/l10n_util.h"
8#include "base/stl_util-inl.h"
9#include "base/utf_string_conversions.h"
10#include "chrome/browser/extensions/extensions_service.h"
11#include "chrome/browser/google_url_tracker.h"
12#include "chrome/browser/history/history.h"
13#include "chrome/browser/history/history_notifications.h"
14#include "chrome/browser/net/url_fixer_upper.h"
15#include "chrome/browser/pref_service.h"
16#include "chrome/browser/profile.h"
17#include "chrome/browser/rlz/rlz.h"
18#include "chrome/browser/search_engines/template_url_prepopulate_data.h"
19#include "chrome/common/extensions/extension.h"
20#include "chrome/common/notification_service.h"
21#include "chrome/common/pref_names.h"
22#include "chrome/common/url_constants.h"
23#include "net/base/net_util.h"
24
25using base::Time;
26
27// String in the URL that is replaced by the search term.
28static const char kSearchTermParameter[] = "{searchTerms}";
29
30// String in Initializer that is replaced with kSearchTermParameter.
31static const char kTemplateParameter[] = "%s";
32
33// Term used when generating a search url. Use something obscure so that on
34// the rare case the term replaces the URL it's unlikely another keyword would
35// have the same url.
36static const wchar_t kReplacementTerm[] = L"blah.blah.blah.blah.blah";
37
38class TemplateURLModel::LessWithPrefix {
39 public:
40  // We want to find the set of keywords that begin with a prefix.  The STL
41  // algorithms will return the set of elements that are "equal to" the
42  // prefix, where "equal(x, y)" means "!(cmp(x, y) || cmp(y, x))".  When
43  // cmp() is the typical std::less<>, this results in lexicographic equality;
44  // we need to extend this to mark a prefix as "not less than" a keyword it
45  // begins, which will cause the desired elements to be considered "equal to"
46  // the prefix.  Note: this is still a strict weak ordering, as required by
47  // equal_range() (though I will not prove that here).
48  //
49  // Unfortunately the calling convention is not "prefix and element" but
50  // rather "two elements", so we pass the prefix as a fake "element" which has
51  // a NULL KeywordDataElement pointer.
52  bool operator()(const KeywordToTemplateMap::value_type& elem1,
53                  const KeywordToTemplateMap::value_type& elem2) const {
54    return (elem1.second == NULL) ?
55        (elem2.first.compare(0, elem1.first.length(), elem1.first) > 0) :
56        (elem1.first < elem2.first);
57  }
58};
59
60TemplateURLModel::TemplateURLModel(Profile* profile)
61    : profile_(profile),
62      loaded_(false),
63      load_failed_(false),
64      load_handle_(0),
65      default_search_provider_(NULL),
66      next_id_(1) {
67  DCHECK(profile_);
68  Init(NULL, 0);
69}
70
71TemplateURLModel::TemplateURLModel(const Initializer* initializers,
72                                   const int count)
73    : profile_(NULL),
74      loaded_(true),
75      load_failed_(false),
76      load_handle_(0),
77      service_(NULL),
78      default_search_provider_(NULL),
79      next_id_(1) {
80  Init(initializers, count);
81}
82
83TemplateURLModel::~TemplateURLModel() {
84  if (load_handle_) {
85    DCHECK(service_.get());
86    service_->CancelRequest(load_handle_);
87  }
88
89  STLDeleteElements(&template_urls_);
90}
91
92void TemplateURLModel::Init(const Initializer* initializers,
93                            int num_initializers) {
94  // Register for notifications.
95  if (profile_) {
96    // TODO(sky): bug 1166191. The keywords should be moved into the history
97    // db, which will mean we no longer need this notification and the history
98    // backend can handle automatically adding the search terms as the user
99    // navigates.
100    registrar_.Add(this, NotificationType::HISTORY_URL_VISITED,
101                   Source<Profile>(profile_->GetOriginalProfile()));
102  }
103  registrar_.Add(this, NotificationType::GOOGLE_URL_UPDATED,
104                 NotificationService::AllSources());
105
106  // Add specific initializers, if any.
107  for (int i(0); i < num_initializers; ++i) {
108    DCHECK(initializers[i].keyword);
109    DCHECK(initializers[i].url);
110    DCHECK(initializers[i].content);
111
112    size_t template_position =
113        std::string(initializers[i].url).find(kTemplateParameter);
114    DCHECK(template_position != std::wstring::npos);
115    std::string osd_url(initializers[i].url);
116    osd_url.replace(template_position, arraysize(kTemplateParameter) - 1,
117                    kSearchTermParameter);
118
119    // TemplateURLModel ends up owning the TemplateURL, don't try and free it.
120    TemplateURL* template_url = new TemplateURL();
121    template_url->set_keyword(initializers[i].keyword);
122    template_url->set_short_name(initializers[i].content);
123    template_url->SetURL(osd_url, 0, 0);
124    Add(template_url);
125  }
126
127  // Request a server check for the correct Google URL if Google is the default
128  // search engine.
129  const TemplateURL* default_provider = GetDefaultSearchProvider();
130  if (default_provider) {
131    const TemplateURLRef* default_provider_ref = default_provider->url();
132    if (default_provider_ref && default_provider_ref->HasGoogleBaseURLs())
133      GoogleURLTracker::RequestServerCheck();
134  }
135}
136
137// static
138std::wstring TemplateURLModel::GenerateKeyword(const GURL& url,
139                                               bool autodetected) {
140  // Don't autogenerate keywords for referrers that are the result of a form
141  // submission (TODO: right now we approximate this by checking for the URL
142  // having a query, but we should replace this with a call to WebCore to see if
143  // the originating page was actually a form submission), anything other than
144  // http, or referrers with a path.
145  //
146  // If we relax the path constraint, we need to be sure to sanitize the path
147  // elements and update AutocompletePopup to look for keywords using the path.
148  // See http://b/issue?id=863583.
149  if (!url.is_valid() ||
150      (autodetected && (url.has_query() || !url.SchemeIs(chrome::kHttpScheme) ||
151                        ((url.path() != "") && (url.path() != "/")))))
152    return std::wstring();
153
154  // Strip "www." off the front of the keyword; otherwise the keyword won't work
155  // properly.  See http://code.google.com/p/chromium/issues/detail?id=6984 .
156  return net::StripWWW(UTF8ToWide(url.host()));
157}
158
159// static
160std::wstring TemplateURLModel::CleanUserInputKeyword(
161    const std::wstring& keyword) {
162  // Remove the scheme.
163  std::wstring result(l10n_util::ToLower(keyword));
164  url_parse::Component scheme_component;
165  if (url_parse::ExtractScheme(WideToUTF8(keyword).c_str(),
166                               static_cast<int>(keyword.length()),
167                               &scheme_component)) {
168    // Include trailing ':'.
169    result.erase(0, scheme_component.end() + 1);
170    // Many schemes usually have "//" after them, so strip it too.
171    const std::wstring after_scheme(L"//");
172    if (result.compare(0, after_scheme.length(), after_scheme) == 0)
173      result.erase(0, after_scheme.length());
174  }
175
176  // Remove leading "www.".
177  result = net::StripWWW(result);
178
179  // Remove trailing "/".
180  return (result.length() > 0 && result[result.length() - 1] == L'/') ?
181      result.substr(0, result.length() - 1) : result;
182}
183
184// static
185GURL TemplateURLModel::GenerateSearchURL(const TemplateURL* t_url) {
186  DCHECK(t_url);
187  const TemplateURLRef* search_ref = t_url->url();
188  // Extension keywords don't have host-based search URLs.
189  if (!search_ref || !search_ref->IsValid() || t_url->IsExtensionKeyword())
190    return GURL();
191
192  if (!search_ref->SupportsReplacement())
193    return GURL(search_ref->url());
194
195  return GURL(search_ref->ReplaceSearchTerms(
196      *t_url, kReplacementTerm, TemplateURLRef::NO_SUGGESTIONS_AVAILABLE,
197      std::wstring()));
198}
199
200bool TemplateURLModel::CanReplaceKeyword(
201    const std::wstring& keyword,
202    const GURL& url,
203    const TemplateURL** template_url_to_replace) {
204  DCHECK(!keyword.empty()); // This should only be called for non-empty
205                            // keywords. If we need to support empty kewords
206                            // the code needs to change slightly.
207  const TemplateURL* existing_url = GetTemplateURLForKeyword(keyword);
208  if (existing_url) {
209    // We already have a TemplateURL for this keyword. Only allow it to be
210    // replaced if the TemplateURL can be replaced.
211    if (template_url_to_replace)
212      *template_url_to_replace = existing_url;
213    return CanReplace(existing_url);
214  }
215
216  // We don't have a TemplateURL with keyword. Only allow a new one if there
217  // isn't a TemplateURL for the specified host, or there is one but it can
218  // be replaced. We do this to ensure that if the user assigns a different
219  // keyword to a generated TemplateURL, we won't regenerate another keyword for
220  // the same host.
221  if (url.is_valid() && !url.host().empty())
222    return CanReplaceKeywordForHost(url.host(), template_url_to_replace);
223  return true;
224}
225
226void TemplateURLModel::FindMatchingKeywords(
227    const std::wstring& prefix,
228    bool support_replacement_only,
229    std::vector<std::wstring>* matches) const {
230  // Sanity check args.
231  if (prefix.empty())
232    return;
233  DCHECK(matches != NULL);
234  DCHECK(matches->empty());  // The code for exact matches assumes this.
235
236  // Find matching keyword range.  Searches the element map for keywords
237  // beginning with |prefix| and stores the endpoints of the resulting set in
238  // |match_range|.
239  const std::pair<KeywordToTemplateMap::const_iterator,
240                  KeywordToTemplateMap::const_iterator> match_range(
241      std::equal_range(
242          keyword_to_template_map_.begin(), keyword_to_template_map_.end(),
243          KeywordToTemplateMap::value_type(prefix, NULL), LessWithPrefix()));
244
245  // Return vector of matching keywords.
246  for (KeywordToTemplateMap::const_iterator i(match_range.first);
247       i != match_range.second; ++i) {
248    DCHECK(i->second->url());
249    if (!support_replacement_only || i->second->url()->SupportsReplacement())
250      matches->push_back(i->first);
251  }
252}
253
254const TemplateURL* TemplateURLModel::GetTemplateURLForKeyword(
255                                     const std::wstring& keyword) const {
256  KeywordToTemplateMap::const_iterator elem(
257      keyword_to_template_map_.find(keyword));
258  return (elem == keyword_to_template_map_.end()) ? NULL : elem->second;
259}
260
261const TemplateURL* TemplateURLModel::GetTemplateURLForHost(
262    const std::string& host) const {
263  HostToURLsMap::const_iterator iter = host_to_urls_map_.find(host);
264  if (iter == host_to_urls_map_.end() || iter->second.empty())
265    return NULL;
266  return *(iter->second.begin());  // Return the 1st element.
267}
268
269void TemplateURLModel::Add(TemplateURL* template_url) {
270  DCHECK(template_url);
271  DCHECK(template_url->id() == 0);
272  DCHECK(find(template_urls_.begin(), template_urls_.end(), template_url) ==
273         template_urls_.end());
274  template_url->set_id(++next_id_);
275  template_urls_.push_back(template_url);
276  AddToMaps(template_url);
277
278  if (service_.get())
279    service_->AddKeyword(*template_url);
280
281  if (loaded_) {
282    FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
283                      OnTemplateURLModelChanged());
284  }
285}
286
287void TemplateURLModel::AddToMaps(const TemplateURL* template_url) {
288  if (!template_url->keyword().empty())
289    keyword_to_template_map_[template_url->keyword()] = template_url;
290
291  const GURL url(GenerateSearchURL(template_url));
292  if (url.is_valid() && url.has_host())
293    host_to_urls_map_[url.host()].insert(template_url);
294}
295
296void TemplateURLModel::Remove(const TemplateURL* template_url) {
297  TemplateURLVector::iterator i = find(template_urls_.begin(),
298                                       template_urls_.end(),
299                                       template_url);
300  if (i == template_urls_.end())
301    return;
302
303  if (template_url == default_search_provider_) {
304    // Should never delete the default search provider.
305    NOTREACHED();
306    return;
307  }
308
309  RemoveFromMaps(template_url);
310
311  // Remove it from the vector containing all TemplateURLs.
312  template_urls_.erase(i);
313
314  if (loaded_) {
315    FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
316                      OnTemplateURLModelChanged());
317  }
318
319  if (service_.get())
320    service_->RemoveKeyword(*template_url);
321
322  if (profile_) {
323    HistoryService* history =
324        profile_->GetHistoryService(Profile::EXPLICIT_ACCESS);
325    if (history)
326      history->DeleteAllSearchTermsForKeyword(template_url->id());
327  }
328
329  // We own the TemplateURL and need to delete it.
330  delete template_url;
331}
332
333void TemplateURLModel::Replace(const TemplateURL* existing_turl,
334                               TemplateURL* new_turl) {
335  DCHECK(existing_turl && new_turl);
336
337  TemplateURLVector::iterator i = find(template_urls_.begin(),
338                                       template_urls_.end(),
339                                       existing_turl);
340  DCHECK(i != template_urls_.end());
341  RemoveFromMaps(existing_turl);
342  template_urls_.erase(i);
343
344  new_turl->set_id(existing_turl->id());
345
346  template_urls_.push_back(new_turl);
347  AddToMaps(new_turl);
348
349  if (service_.get())
350    service_->UpdateKeyword(*new_turl);
351
352  if (default_search_provider_ == existing_turl)
353    SetDefaultSearchProvider(new_turl);
354
355  if (loaded_) {
356    FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
357                      OnTemplateURLModelChanged());
358  }
359
360  delete existing_turl;
361}
362
363void TemplateURLModel::RemoveAutoGeneratedBetween(Time created_after,
364                                                  Time created_before) {
365  for (size_t i = 0; i < template_urls_.size();) {
366    if (template_urls_[i]->date_created() >= created_after &&
367        (created_before.is_null() ||
368         template_urls_[i]->date_created() < created_before) &&
369        CanReplace(template_urls_[i])) {
370      Remove(template_urls_[i]);
371    } else {
372      ++i;
373    }
374  }
375}
376
377void TemplateURLModel::RemoveAutoGeneratedSince(Time created_after) {
378  RemoveAutoGeneratedBetween(created_after, Time());
379}
380
381void TemplateURLModel::SetKeywordSearchTermsForURL(const TemplateURL* t_url,
382                                                   const GURL& url,
383                                                   const std::wstring& term) {
384  HistoryService* history = profile_  ?
385      profile_->GetHistoryService(Profile::EXPLICIT_ACCESS) : NULL;
386  if (!history)
387    return;
388  history->SetKeywordSearchTermsForURL(url, t_url->id(),
389                                       WideToUTF16Hack(term));
390}
391
392void TemplateURLModel::RemoveFromMaps(const TemplateURL* template_url) {
393  if (!template_url->keyword().empty()) {
394    keyword_to_template_map_.erase(template_url->keyword());
395  }
396
397  const GURL url(GenerateSearchURL(template_url));
398  if (url.is_valid() && url.has_host()) {
399    const std::string host(url.host());
400    DCHECK(host_to_urls_map_.find(host) != host_to_urls_map_.end());
401    TemplateURLSet& urls = host_to_urls_map_[host];
402    DCHECK(urls.find(template_url) != urls.end());
403    urls.erase(urls.find(template_url));
404    if (urls.empty())
405      host_to_urls_map_.erase(host_to_urls_map_.find(host));
406  }
407}
408
409void TemplateURLModel::RemoveFromMapsByPointer(
410    const TemplateURL* template_url) {
411  DCHECK(template_url);
412  for (KeywordToTemplateMap::iterator i = keyword_to_template_map_.begin();
413       i != keyword_to_template_map_.end(); ++i) {
414    if (i->second == template_url) {
415      keyword_to_template_map_.erase(i);
416      // A given TemplateURL only occurs once in the map. As soon as we find the
417      // entry, stop.
418      break;
419    }
420  }
421
422  for (HostToURLsMap::iterator i = host_to_urls_map_.begin();
423       i != host_to_urls_map_.end(); ++i) {
424    TemplateURLSet::iterator url_set_iterator = i->second.find(template_url);
425    if (url_set_iterator != i->second.end()) {
426      i->second.erase(url_set_iterator);
427      if (i->second.empty())
428        host_to_urls_map_.erase(i);
429      // A given TemplateURL only occurs once in the map. As soon as we find the
430      // entry, stop.
431      return;
432    }
433  }
434}
435
436void TemplateURLModel::SetTemplateURLs(
437      const std::vector<const TemplateURL*>& urls) {
438  // Add mappings for the new items.
439  for (TemplateURLVector::const_iterator i = urls.begin(); i != urls.end();
440       ++i) {
441    next_id_ = std::max(next_id_, (*i)->id());
442    AddToMaps(*i);
443    template_urls_.push_back(*i);
444  }
445}
446
447std::vector<const TemplateURL*> TemplateURLModel::GetTemplateURLs() const {
448  return template_urls_;
449}
450
451void TemplateURLModel::IncrementUsageCount(const TemplateURL* url) {
452  DCHECK(url && find(template_urls_.begin(), template_urls_.end(), url) !=
453         template_urls_.end());
454  const_cast<TemplateURL*>(url)->set_usage_count(url->usage_count() + 1);
455  if (service_.get())
456    service_.get()->UpdateKeyword(*url);
457}
458
459void TemplateURLModel::ResetTemplateURL(const TemplateURL* url,
460                                        const std::wstring& title,
461                                        const std::wstring& keyword,
462                                        const std::string& search_url) {
463  DCHECK(url && find(template_urls_.begin(), template_urls_.end(), url) !=
464         template_urls_.end());
465  RemoveFromMaps(url);
466  TemplateURL* modifiable_url = const_cast<TemplateURL*>(url);
467  modifiable_url->set_short_name(title);
468  modifiable_url->set_keyword(keyword);
469  if ((modifiable_url->url() && search_url.empty()) ||
470      (!modifiable_url->url() && !search_url.empty()) ||
471      (modifiable_url->url() && modifiable_url->url()->url() != search_url)) {
472    // The urls have changed, reset the favicon url.
473    modifiable_url->SetFavIconURL(GURL());
474    modifiable_url->SetURL(search_url, 0, 0);
475  }
476  modifiable_url->set_safe_for_autoreplace(false);
477  AddToMaps(url);
478  if (service_.get())
479    service_.get()->UpdateKeyword(*url);
480
481  FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
482                    OnTemplateURLModelChanged());
483}
484
485void TemplateURLModel::SetDefaultSearchProvider(const TemplateURL* url) {
486  if (default_search_provider_ == url)
487    return;
488
489  DCHECK(!url || find(template_urls_.begin(), template_urls_.end(), url) !=
490         template_urls_.end());
491  default_search_provider_ = url;
492
493  if (url) {
494    TemplateURL* modifiable_url = const_cast<TemplateURL*>(url);
495    // Don't mark the url as edited, otherwise we won't be able to rev the
496    // templateurls we ship with.
497    modifiable_url->set_show_in_default_list(true);
498    if (service_.get())
499      service_.get()->UpdateKeyword(*url);
500
501    const TemplateURLRef* url_ref = url->url();
502    if (url_ref && url_ref->HasGoogleBaseURLs()) {
503      GoogleURLTracker::RequestServerCheck();
504#if defined(OS_WIN)
505      RLZTracker::RecordProductEvent(rlz_lib::CHROME,
506                                     rlz_lib::CHROME_OMNIBOX,
507                                     rlz_lib::SET_TO_GOOGLE);
508#endif
509    }
510  }
511
512  SaveDefaultSearchProviderToPrefs(url);
513
514  if (service_.get())
515    service_->SetDefaultSearchProvider(url);
516
517  if (loaded_) {
518    FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
519                      OnTemplateURLModelChanged());
520  }
521}
522
523const TemplateURL* TemplateURLModel::GetDefaultSearchProvider() {
524  if (loaded_ && !load_failed_)
525    return default_search_provider_;
526
527  if (!prefs_default_search_provider_.get()) {
528    TemplateURL* default_from_prefs;
529    if (LoadDefaultSearchProviderFromPrefs(&default_from_prefs)) {
530      prefs_default_search_provider_.reset(default_from_prefs);
531    } else {
532      std::vector<TemplateURL*> loaded_urls;
533      size_t default_search_index;
534      TemplateURLPrepopulateData::GetPrepopulatedEngines(GetPrefs(),
535                                                         &loaded_urls,
536                                                         &default_search_index);
537      if (default_search_index < loaded_urls.size()) {
538        prefs_default_search_provider_.reset(loaded_urls[default_search_index]);
539        loaded_urls.erase(loaded_urls.begin() + default_search_index);
540      }
541      STLDeleteElements(&loaded_urls);
542    }
543  }
544
545  return prefs_default_search_provider_.get();
546}
547
548void TemplateURLModel::AddObserver(TemplateURLModelObserver* observer) {
549  model_observers_.AddObserver(observer);
550}
551
552void TemplateURLModel::RemoveObserver(TemplateURLModelObserver* observer) {
553  model_observers_.RemoveObserver(observer);
554}
555
556void TemplateURLModel::Load() {
557  if (loaded_ || load_handle_)
558    return;
559
560  if (!service_.get())
561    service_ = profile_->GetWebDataService(Profile::EXPLICIT_ACCESS);
562
563  if (service_.get()) {
564    load_handle_ = service_->GetKeywords(this);
565  } else {
566    loaded_ = true;
567    NotifyLoaded();
568  }
569}
570
571void TemplateURLModel::OnWebDataServiceRequestDone(
572                       WebDataService::Handle h,
573                       const WDTypedResult* result) {
574  // Reset the load_handle so that we don't try and cancel the load in
575  // the destructor.
576  load_handle_ = 0;
577
578  if (!result) {
579    // Results are null if the database went away or (most likely) wasn't
580    // loaded.
581    loaded_ = true;
582    load_failed_ = true;
583    NotifyLoaded();
584    return;
585  }
586
587  DCHECK(result->GetType() == KEYWORDS_RESULT);
588
589  WDKeywordsResult keyword_result = reinterpret_cast<
590      const WDResult<WDKeywordsResult>*>(result)->GetValue();
591
592  // prefs_default_search_provider_ is only needed before we've finished
593  // loading. Now that we've loaded we can nuke it.
594  prefs_default_search_provider_.reset();
595
596  // Compiler won't implicitly convert std::vector<TemplateURL*> to
597  // std::vector<const TemplateURL*>, and reinterpret_cast is unsafe,
598  // so we just copy it.
599  std::vector<const TemplateURL*> template_urls(keyword_result.keywords.begin(),
600                                                keyword_result.keywords.end());
601
602  const int resource_keyword_version =
603      TemplateURLPrepopulateData::GetDataVersion(GetPrefs());
604  if (keyword_result.builtin_keyword_version != resource_keyword_version) {
605    // There should never be duplicate TemplateURLs. We had a bug such that
606    // duplicate TemplateURLs existed for one locale. As such we invoke
607    // RemoveDuplicatePrepopulateIDs to nuke the duplicates.
608    RemoveDuplicatePrepopulateIDs(&template_urls);
609  }
610  SetTemplateURLs(template_urls);
611
612  if (keyword_result.default_search_provider_id) {
613    // See if we can find the default search provider.
614    for (TemplateURLVector::iterator i = template_urls_.begin();
615         i != template_urls_.end(); ++i) {
616      if ((*i)->id() == keyword_result.default_search_provider_id) {
617        default_search_provider_ = *i;
618        break;
619      }
620    }
621  }
622
623  if (keyword_result.builtin_keyword_version != resource_keyword_version) {
624    MergeEnginesFromPrepopulateData();
625    service_->SetBuiltinKeywordVersion(resource_keyword_version);
626  }
627
628  // Always save the default search provider to prefs. That way we don't have to
629  // worry about it being out of sync.
630  if (default_search_provider_)
631    SaveDefaultSearchProviderToPrefs(default_search_provider_);
632
633  // Delete any hosts that were deleted before we finished loading.
634  for (std::vector<std::wstring>::iterator i = hosts_to_delete_.begin();
635       i != hosts_to_delete_.end(); ++i) {
636    DeleteGeneratedKeywordsMatchingHost(*i);
637  }
638  hosts_to_delete_.clear();
639
640  // Index any visits that occurred before we finished loading.
641  for (size_t i = 0; i < visits_to_add_.size(); ++i)
642    UpdateKeywordSearchTermsForURL(visits_to_add_[i]);
643  visits_to_add_.clear();
644
645  loaded_ = true;
646
647  FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
648                    OnTemplateURLModelChanged());
649
650  NotifyLoaded();
651}
652
653void TemplateURLModel::RemoveDuplicatePrepopulateIDs(
654    std::vector<const TemplateURL*>* urls) {
655  std::set<int> ids;
656  for (std::vector<const TemplateURL*>::iterator i = urls->begin();
657       i != urls->end(); ) {
658    int prepopulate_id = (*i)->prepopulate_id();
659    if (prepopulate_id) {
660      if (ids.find(prepopulate_id) != ids.end()) {
661        if (service_.get())
662          service_->RemoveKeyword(**i);
663        delete *i;
664        i = urls->erase(i);
665      } else {
666        ids.insert(prepopulate_id);
667        ++i;
668      }
669    } else {
670      ++i;
671    }
672  }
673}
674
675std::wstring TemplateURLModel::GetKeywordShortName(const std::wstring& keyword,
676                                                   bool* is_extension_keyword) {
677  const TemplateURL* template_url = GetTemplateURLForKeyword(keyword);
678
679  // TODO(sky): Once LocationBarView adds a listener to the TemplateURLModel
680  // to track changes to the model, this should become a DCHECK.
681  if (template_url) {
682    *is_extension_keyword = template_url->IsExtensionKeyword();
683    return template_url->AdjustedShortNameForLocaleDirection();
684  }
685  *is_extension_keyword = false;
686  return std::wstring();
687}
688
689void TemplateURLModel::Observe(NotificationType type,
690                               const NotificationSource& source,
691                               const NotificationDetails& details) {
692  if (type == NotificationType::HISTORY_URL_VISITED) {
693    Details<history::URLVisitedDetails> visit_details(details);
694
695    if (!loaded())
696      visits_to_add_.push_back(*visit_details.ptr());
697    else
698      UpdateKeywordSearchTermsForURL(*visit_details.ptr());
699  } else if (type == NotificationType::GOOGLE_URL_UPDATED) {
700    if (loaded_)
701      GoogleBaseURLChanged();
702  } else {
703    NOTREACHED();
704  }
705}
706
707void TemplateURLModel::DeleteGeneratedKeywordsMatchingHost(
708    const std::wstring& host) {
709  const std::wstring host_slash = host + L"/";
710  // Iterate backwards as we may end up removing multiple entries.
711  for (int i = static_cast<int>(template_urls_.size()) - 1; i >= 0; --i) {
712    if (CanReplace(template_urls_[i]) &&
713        (template_urls_[i]->keyword() == host ||
714         template_urls_[i]->keyword().compare(0, host_slash.length(),
715                                              host_slash) == 0)) {
716      Remove(template_urls_[i]);
717    }
718  }
719}
720
721void TemplateURLModel::NotifyLoaded() {
722  NotificationService::current()->Notify(
723      NotificationType::TEMPLATE_URL_MODEL_LOADED,
724      Source<TemplateURLModel>(this),
725      NotificationService::NoDetails());
726
727  for (size_t i = 0; i < pending_extension_ids_.size(); ++i) {
728    Extension* extension = profile_->GetExtensionsService()->
729        GetExtensionById(pending_extension_ids_[i], true);
730    if (extension)
731      RegisterExtensionKeyword(extension);
732  }
733  pending_extension_ids_.clear();
734}
735
736void TemplateURLModel::MergeEnginesFromPrepopulateData() {
737  // Build a map from prepopulate id to TemplateURL of existing urls.
738  typedef std::map<int, const TemplateURL*> IDMap;
739  IDMap id_to_turl;
740  for (TemplateURLVector::const_iterator i(template_urls_.begin());
741       i != template_urls_.end(); ++i) {
742    int prepopulate_id = (*i)->prepopulate_id();
743    if (prepopulate_id > 0)
744      id_to_turl[prepopulate_id] = *i;
745  }
746
747  std::vector<TemplateURL*> loaded_urls;
748  size_t default_search_index;
749  TemplateURLPrepopulateData::GetPrepopulatedEngines(GetPrefs(),
750                                                     &loaded_urls,
751                                                     &default_search_index);
752
753  std::set<int> updated_ids;
754  for (size_t i = 0; i < loaded_urls.size(); ++i) {
755    // We take ownership of |t_url|.
756    scoped_ptr<TemplateURL> t_url(loaded_urls[i]);
757    int t_url_id = t_url->prepopulate_id();
758    if (!t_url_id || updated_ids.count(t_url_id)) {
759      // Prepopulate engines need a unique id.
760      NOTREACHED();
761      continue;
762    }
763
764    IDMap::iterator existing_url_iter(id_to_turl.find(t_url_id));
765    if (existing_url_iter != id_to_turl.end()) {
766      const TemplateURL* existing_url = existing_url_iter->second;
767      if (!existing_url->safe_for_autoreplace()) {
768        // User edited the entry, preserve the keyword and description.
769        t_url->set_safe_for_autoreplace(false);
770        t_url->set_keyword(existing_url->keyword());
771        t_url->set_autogenerate_keyword(
772            existing_url->autogenerate_keyword());
773        t_url->set_short_name(existing_url->short_name());
774      }
775      Replace(existing_url, t_url.release());
776      id_to_turl.erase(existing_url_iter);
777    } else {
778      Add(t_url.release());
779    }
780    if (i == default_search_index && !default_search_provider_)
781      SetDefaultSearchProvider(loaded_urls[i]);
782
783    updated_ids.insert(t_url_id);
784  }
785
786  // Remove any prepopulated engines which are no longer in the master list, as
787  // long as the user hasn't modified them or made them the default engine.
788  for (IDMap::iterator i(id_to_turl.begin()); i != id_to_turl.end(); ++i) {
789    const TemplateURL* template_url = i->second;
790    // We use default_search_provider_ instead of GetDefaultSearchProvider()
791    // because we're running before |loaded_| is set, and calling
792    // GetDefaultSearchProvider() will erroneously try to read the prefs.
793    if ((template_url->safe_for_autoreplace()) &&
794        (template_url != default_search_provider_))
795      Remove(template_url);
796  }
797}
798
799void TemplateURLModel::SaveDefaultSearchProviderToPrefs(
800    const TemplateURL* t_url) {
801  PrefService* prefs = GetPrefs();
802  if (!prefs)
803    return;
804
805  RegisterPrefs(prefs);
806
807  const std::string search_url =
808      (t_url && t_url->url()) ? t_url->url()->url() : std::string();
809  prefs->SetString(prefs::kDefaultSearchProviderSearchURL, search_url);
810
811  const std::string suggest_url =
812      (t_url && t_url->suggestions_url()) ? t_url->suggestions_url()->url() :
813                                            std::string();
814  prefs->SetString(prefs::kDefaultSearchProviderSuggestURL, suggest_url);
815
816  const std::string name =
817      t_url ? WideToUTF8(t_url->short_name()) : std::string();
818  prefs->SetString(prefs::kDefaultSearchProviderName, name);
819
820  const std::string id_string =
821      t_url ? Int64ToString(t_url->id()) : std::string();
822  prefs->SetString(prefs::kDefaultSearchProviderID, id_string);
823
824  const std::string prepopulate_id =
825      t_url ? Int64ToString(t_url->prepopulate_id()) : std::string();
826  prefs->SetString(prefs::kDefaultSearchProviderPrepopulateID, prepopulate_id);
827
828  prefs->ScheduleSavePersistentPrefs();
829}
830
831bool TemplateURLModel::LoadDefaultSearchProviderFromPrefs(
832    TemplateURL** default_provider) {
833  PrefService* prefs = GetPrefs();
834  if (!prefs || !prefs->HasPrefPath(prefs::kDefaultSearchProviderSearchURL) ||
835      !prefs->HasPrefPath(prefs::kDefaultSearchProviderSuggestURL) ||
836      !prefs->HasPrefPath(prefs::kDefaultSearchProviderName) ||
837      !prefs->HasPrefPath(prefs::kDefaultSearchProviderID)) {
838    return false;
839  }
840  RegisterPrefs(prefs);
841
842  std::string suggest_url =
843      prefs->GetString(prefs::kDefaultSearchProviderSuggestURL);
844  std::string search_url =
845      prefs->GetString(prefs::kDefaultSearchProviderSearchURL);
846
847  if (suggest_url.empty() && search_url.empty()) {
848    // The user doesn't want a default search provider.
849    *default_provider = NULL;
850    return true;
851  }
852
853  std::wstring name =
854      UTF8ToWide(prefs->GetString(prefs::kDefaultSearchProviderName));
855
856  std::string id_string = prefs->GetString(prefs::kDefaultSearchProviderID);
857
858  std::string prepopulate_id =
859      prefs->GetString(prefs::kDefaultSearchProviderPrepopulateID);
860
861  *default_provider = new TemplateURL();
862  (*default_provider)->set_short_name(name);
863  (*default_provider)->SetURL(search_url, 0, 0);
864  (*default_provider)->SetSuggestionsURL(suggest_url, 0, 0);
865  if (!id_string.empty())
866    (*default_provider)->set_id(StringToInt64(id_string));
867  if (!prepopulate_id.empty())
868    (*default_provider)->set_prepopulate_id(StringToInt(prepopulate_id));
869  return true;
870}
871
872void TemplateURLModel::RegisterPrefs(PrefService* prefs) {
873  if (prefs->FindPreference(prefs::kDefaultSearchProviderName))
874    return;
875  prefs->RegisterStringPref(
876      prefs::kDefaultSearchProviderName, std::string());
877  prefs->RegisterStringPref(
878      prefs::kDefaultSearchProviderID, std::string());
879  prefs->RegisterStringPref(
880      prefs::kDefaultSearchProviderPrepopulateID, std::string());
881  prefs->RegisterStringPref(
882      prefs::kDefaultSearchProviderSuggestURL, std::string());
883  prefs->RegisterStringPref(
884      prefs::kDefaultSearchProviderSearchURL, std::string());
885}
886
887bool TemplateURLModel::CanReplaceKeywordForHost(
888    const std::string& host,
889    const TemplateURL** to_replace) {
890  const HostToURLsMap::iterator matching_urls = host_to_urls_map_.find(host);
891  const bool have_matching_urls = (matching_urls != host_to_urls_map_.end());
892  if (have_matching_urls) {
893    TemplateURLSet& urls = matching_urls->second;
894    for (TemplateURLSet::iterator i = urls.begin(); i != urls.end(); ++i) {
895      const TemplateURL* url = *i;
896      if (CanReplace(url)) {
897        if (to_replace)
898          *to_replace = url;
899        return true;
900      }
901    }
902  }
903
904  if (to_replace)
905    *to_replace = NULL;
906  return !have_matching_urls;
907}
908
909bool TemplateURLModel::CanReplace(const TemplateURL* t_url) {
910  return (t_url != default_search_provider_ && !t_url->show_in_default_list() &&
911          t_url->safe_for_autoreplace());
912}
913
914PrefService* TemplateURLModel::GetPrefs() {
915  return profile_ ? profile_->GetPrefs() : NULL;
916}
917
918void TemplateURLModel::UpdateKeywordSearchTermsForURL(
919    const history::URLVisitedDetails& details) {
920  const history::URLRow& row = details.row;
921  if (!row.url().is_valid() ||
922      !row.url().parsed_for_possibly_invalid_spec().query.is_nonempty()) {
923    return;
924  }
925
926  HostToURLsMap::const_iterator t_urls_for_host_iterator =
927      host_to_urls_map_.find(row.url().host());
928  if (t_urls_for_host_iterator == host_to_urls_map_.end() ||
929      t_urls_for_host_iterator->second.empty()) {
930    return;
931  }
932
933  const TemplateURLSet& urls_for_host = t_urls_for_host_iterator->second;
934  QueryTerms query_terms;
935  bool built_terms = false;  // Most URLs won't match a TemplateURLs host;
936                             // so we lazily build the query_terms.
937  const std::string path = row.url().path();
938
939  for (TemplateURLSet::const_iterator i = urls_for_host.begin();
940       i != urls_for_host.end(); ++i) {
941    const TemplateURLRef* search_ref = (*i)->url();
942
943    // Count the URL against a TemplateURL if the host and path of the
944    // visited URL match that of the TemplateURL as well as the search term's
945    // key of the TemplateURL occurring in the visited url.
946    //
947    // NOTE: Even though we're iterating over TemplateURLs indexed by the host
948    // of the URL we still need to call GetHost on the search_ref. In
949    // particular, GetHost returns an empty string if search_ref doesn't support
950    // replacement or isn't valid for use in keyword search terms.
951
952    if (search_ref && search_ref->GetHost() == row.url().host() &&
953        search_ref->GetPath() == path) {
954      if (!built_terms && !BuildQueryTerms(row.url(), &query_terms)) {
955        // No query terms. No need to continue with the rest of the
956        // TemplateURLs.
957        return;
958      }
959      built_terms = true;
960
961      if (PageTransition::StripQualifier(details.transition) ==
962          PageTransition::KEYWORD) {
963        // The visit is the result of the user entering a keyword, generate a
964        // KEYWORD_GENERATED visit for the KEYWORD so that the keyword typed
965        // count is boosted.
966        AddTabToSearchVisit(**i);
967      }
968
969      QueryTerms::iterator terms_iterator =
970          query_terms.find(search_ref->GetSearchTermKey());
971      if (terms_iterator != query_terms.end() &&
972          !terms_iterator->second.empty()) {
973        SetKeywordSearchTermsForURL(
974            *i, row.url(), search_ref->SearchTermToWide(*(*i),
975            terms_iterator->second));
976      }
977    }
978  }
979}
980
981void TemplateURLModel::AddTabToSearchVisit(const TemplateURL& t_url) {
982  // Only add visits for entries the user hasn't modified. If the user modified
983  // the entry the keyword may no longer correspond to the host name. It may be
984  // possible to do something more sophisticated here, but it's so rare as to
985  // not be worth it.
986  if (!t_url.safe_for_autoreplace())
987    return;
988
989  if (!profile_)
990    return;
991
992  HistoryService* history =
993      profile_->GetHistoryService(Profile::EXPLICIT_ACCESS);
994  if (!history)
995    return;
996
997  GURL url(URLFixerUpper::FixupURL(WideToUTF8(t_url.keyword()), std::string()));
998  if (!url.is_valid())
999    return;
1000
1001  // Synthesize a visit for the keyword. This ensures the url for the keyword is
1002  // autocompleted even if the user doesn't type the url in directly.
1003  history->AddPage(url, NULL, 0, GURL(),
1004                   PageTransition::KEYWORD_GENERATED,
1005                   history::RedirectList(), false);
1006}
1007
1008// static
1009bool TemplateURLModel::BuildQueryTerms(const GURL& url,
1010                                       QueryTerms* query_terms) {
1011  url_parse::Component query = url.parsed_for_possibly_invalid_spec().query;
1012  url_parse::Component key, value;
1013  size_t valid_term_count = 0;
1014  while (url_parse::ExtractQueryKeyValue(url.spec().c_str(), &query, &key,
1015                                         &value)) {
1016    if (key.is_nonempty() && value.is_nonempty()) {
1017      std::string key_string = url.spec().substr(key.begin, key.len);
1018      std::string value_string = url.spec().substr(value.begin, value.len);
1019      QueryTerms::iterator query_terms_iterator =
1020          query_terms->find(key_string);
1021      if (query_terms_iterator != query_terms->end()) {
1022        if (!query_terms_iterator->second.empty() &&
1023            query_terms_iterator->second != value_string) {
1024          // The term occurs in multiple places with different values. Treat
1025          // this as if the term doesn't occur by setting the value to an empty
1026          // string.
1027          (*query_terms)[key_string] = std::string();
1028          DCHECK(valid_term_count > 0);
1029          valid_term_count--;
1030        }
1031      } else {
1032        valid_term_count++;
1033        (*query_terms)[key_string] = value_string;
1034      }
1035    }
1036  }
1037  return (valid_term_count > 0);
1038}
1039
1040void TemplateURLModel::GoogleBaseURLChanged() {
1041  bool something_changed = false;
1042  for (size_t i = 0; i < template_urls_.size(); ++i) {
1043    const TemplateURL* t_url = template_urls_[i];
1044    if ((t_url->url() && t_url->url()->HasGoogleBaseURLs()) ||
1045        (t_url->suggestions_url() &&
1046         t_url->suggestions_url()->HasGoogleBaseURLs())) {
1047      RemoveFromMapsByPointer(t_url);
1048      t_url->InvalidateCachedValues();
1049      AddToMaps(t_url);
1050      something_changed = true;
1051    }
1052  }
1053
1054  if (something_changed && loaded_) {
1055    FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
1056                      OnTemplateURLModelChanged());
1057  }
1058}
1059
1060void TemplateURLModel::RegisterExtensionKeyword(Extension* extension) {
1061  // TODO(mpcomplete): disable the keyword when the extension is disabled.
1062  if (extension->omnibox_keyword().empty())
1063    return;
1064
1065  Load();
1066  if (!loaded_) {
1067    pending_extension_ids_.push_back(extension->id());
1068    return;
1069  }
1070
1071  const TemplateURL* existing_url = GetTemplateURLForExtension(extension);
1072  std::wstring keyword = UTF8ToWide(extension->omnibox_keyword());
1073
1074  TemplateURL* template_url = new TemplateURL;
1075  template_url->set_short_name(UTF8ToWide(extension->name()));
1076  template_url->set_keyword(keyword);
1077  // This URL is not actually used for navigation. It holds the extension's
1078  // ID, as well as forcing the TemplateURL to be treated as a search keyword.
1079  template_url->SetURL(
1080      std::string(chrome::kExtensionScheme) + "://" +
1081      extension->id() + "/?q={searchTerms}", 0, 0);
1082  template_url->set_safe_for_autoreplace(false);
1083
1084  if (existing_url) {
1085    // TODO(mpcomplete): only replace if the user hasn't changed the keyword.
1086    // (We don't have UI for that yet).
1087    Replace(existing_url, template_url);
1088  } else {
1089    Add(template_url);
1090  }
1091}
1092
1093void TemplateURLModel::UnregisterExtensionKeyword(Extension* extension) {
1094  const TemplateURL* url = GetTemplateURLForExtension(extension);
1095  if (url)
1096    Remove(url);
1097}
1098
1099const TemplateURL* TemplateURLModel::GetTemplateURLForExtension(
1100    Extension* extension) const {
1101  for (TemplateURLVector::const_iterator i = template_urls_.begin();
1102       i != template_urls_.end(); ++i) {
1103    if ((*i)->IsExtensionKeyword() && (*i)->url()->GetHost() == extension->id())
1104      return *i;
1105  }
1106
1107  return NULL;
1108}
1109