1// Copyright (c) 2011 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/importer/firefox3_importer.h"
6
7#include <set>
8
9#include "app/sql/connection.h"
10#include "app/sql/statement.h"
11#include "base/file_util.h"
12#include "base/memory/scoped_ptr.h"
13#include "base/message_loop.h"
14#include "base/stl_util-inl.h"
15#include "base/string_util.h"
16#include "base/utf_string_conversions.h"
17#include "chrome/browser/browser_process.h"
18#include "chrome/browser/history/history_types.h"
19#include "chrome/browser/importer/firefox2_importer.h"
20#include "chrome/browser/importer/firefox_importer_utils.h"
21#include "chrome/browser/importer/importer_bridge.h"
22#include "chrome/browser/importer/nss_decryptor.h"
23#include "chrome/browser/search_engines/template_url.h"
24#include "chrome/common/time_format.h"
25#include "content/browser/browser_thread.h"
26#include "googleurl/src/gurl.h"
27#include "grit/generated_resources.h"
28#include "webkit/glue/password_form.h"
29
30namespace {
31
32// Original definition is in http://mxr.mozilla.org/firefox/source/toolkit/
33//  components/places/public/nsINavBookmarksService.idl
34enum BookmarkItemType {
35  TYPE_BOOKMARK = 1,
36  TYPE_FOLDER = 2,
37  TYPE_SEPARATOR = 3,
38  TYPE_DYNAMIC_CONTAINER = 4
39};
40
41}  // namespace
42
43struct Firefox3Importer::BookmarkItem {
44  int parent;
45  int id;
46  GURL url;
47  string16 title;
48  BookmarkItemType type;
49  std::string keyword;
50  base::Time date_added;
51  int64 favicon;
52  bool empty_folder;
53};
54
55Firefox3Importer::Firefox3Importer() {
56#if defined(OS_LINUX)
57  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
58  locale_ = g_browser_process->GetApplicationLocale();
59#endif
60}
61
62Firefox3Importer::~Firefox3Importer() {
63}
64
65void Firefox3Importer::StartImport(
66    const importer::SourceProfile& source_profile,
67    uint16 items,
68    ImporterBridge* bridge) {
69#if defined(OS_LINUX)
70  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
71#endif
72  bridge_ = bridge;
73  source_path_ = source_profile.source_path;
74  app_path_ = source_profile.app_path;
75
76  // The order here is important!
77  bridge_->NotifyStarted();
78  if ((items & importer::HOME_PAGE) && !cancelled())
79    ImportHomepage();  // Doesn't have a UI item.
80
81  // Note history should be imported before bookmarks because bookmark import
82  // will also import favicons and we store favicon for a URL only if the URL
83  // exist in history or bookmarks.
84  if ((items & importer::HISTORY) && !cancelled()) {
85    bridge_->NotifyItemStarted(importer::HISTORY);
86    ImportHistory();
87    bridge_->NotifyItemEnded(importer::HISTORY);
88  }
89
90  if ((items & importer::FAVORITES) && !cancelled()) {
91    bridge_->NotifyItemStarted(importer::FAVORITES);
92    ImportBookmarks();
93    bridge_->NotifyItemEnded(importer::FAVORITES);
94  }
95  if ((items & importer::SEARCH_ENGINES) && !cancelled()) {
96    bridge_->NotifyItemStarted(importer::SEARCH_ENGINES);
97    ImportSearchEngines();
98    bridge_->NotifyItemEnded(importer::SEARCH_ENGINES);
99  }
100  if ((items & importer::PASSWORDS) && !cancelled()) {
101    bridge_->NotifyItemStarted(importer::PASSWORDS);
102    ImportPasswords();
103    bridge_->NotifyItemEnded(importer::PASSWORDS);
104  }
105  bridge_->NotifyEnded();
106}
107
108void Firefox3Importer::ImportHistory() {
109  FilePath file = source_path_.AppendASCII("places.sqlite");
110  if (!file_util::PathExists(file))
111    return;
112
113  sql::Connection db;
114  if (!db.Open(file))
115    return;
116
117  // |visit_type| represent the transition type of URLs (typed, click,
118  // redirect, bookmark, etc.) We eliminate some URLs like sub-frames and
119  // redirects, since we don't want them to appear in history.
120  // Firefox transition types are defined in:
121  //   toolkit/components/places/public/nsINavHistoryService.idl
122  const char* query = "SELECT h.url, h.title, h.visit_count, "
123                      "h.hidden, h.typed, v.visit_date "
124                      "FROM moz_places h JOIN moz_historyvisits v "
125                      "ON h.id = v.place_id "
126                      "WHERE v.visit_type <= 3";
127
128  sql::Statement s(db.GetUniqueStatement(query));
129  if (!s)
130    return;
131
132  std::vector<history::URLRow> rows;
133  while (s.Step() && !cancelled()) {
134    GURL url(s.ColumnString(0));
135
136    // Filter out unwanted URLs.
137    if (!CanImportURL(url))
138      continue;
139
140    history::URLRow row(url);
141    row.set_title(s.ColumnString16(1));
142    row.set_visit_count(s.ColumnInt(2));
143    row.set_hidden(s.ColumnInt(3) == 1);
144    row.set_typed_count(s.ColumnInt(4));
145    row.set_last_visit(base::Time::FromTimeT(s.ColumnInt64(5)/1000000));
146
147    rows.push_back(row);
148  }
149
150  if (!rows.empty() && !cancelled())
151    bridge_->SetHistoryItems(rows, history::SOURCE_FIREFOX_IMPORTED);
152}
153
154void Firefox3Importer::ImportBookmarks() {
155  FilePath file = source_path_.AppendASCII("places.sqlite");
156  if (!file_util::PathExists(file))
157    return;
158
159  sql::Connection db;
160  if (!db.Open(file))
161    return;
162
163  // Get the bookmark folders that we are interested in.
164  int toolbar_folder_id = -1;
165  int menu_folder_id = -1;
166  int unsorted_folder_id = -1;
167  LoadRootNodeID(&db, &toolbar_folder_id, &menu_folder_id, &unsorted_folder_id);
168
169  // Load livemark IDs.
170  std::set<int> livemark_id;
171  LoadLivemarkIDs(&db, &livemark_id);
172
173  // Load the default bookmarks. Its storage is the same as Firefox 2.
174  std::set<GURL> default_urls;
175  Firefox2Importer::LoadDefaultBookmarks(app_path_, &default_urls);
176
177  BookmarkList list;
178  GetTopBookmarkFolder(&db, toolbar_folder_id, &list);
179  GetTopBookmarkFolder(&db, menu_folder_id, &list);
180  GetTopBookmarkFolder(&db, unsorted_folder_id, &list);
181  size_t count = list.size();
182  for (size_t i = 0; i < count; ++i)
183    GetWholeBookmarkFolder(&db, &list, i, NULL);
184
185  std::vector<ProfileWriter::BookmarkEntry> bookmarks;
186  std::vector<TemplateURL*> template_urls;
187  FaviconMap favicon_map;
188
189  // TODO(jcampan): http://b/issue?id=1196285 we do not support POST based
190  //                keywords yet.  We won't include them in the list.
191  std::set<int> post_keyword_ids;
192  const char* query = "SELECT b.id FROM moz_bookmarks b "
193      "INNER JOIN moz_items_annos ia ON ia.item_id = b.id "
194      "INNER JOIN moz_anno_attributes aa ON ia.anno_attribute_id = aa.id "
195      "WHERE aa.name = 'bookmarkProperties/POSTData'";
196  sql::Statement s(db.GetUniqueStatement(query));
197  if (s) {
198    while (s.Step() && !cancelled())
199      post_keyword_ids.insert(s.ColumnInt(0));
200  } else {
201    NOTREACHED();
202    return;
203  }
204
205  string16 firefox_folder =
206      bridge_->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX);
207  for (size_t i = 0; i < list.size(); ++i) {
208    BookmarkItem* item = list[i];
209
210    if (item->type == TYPE_FOLDER) {
211      // Folders are added implicitly on adding children,
212      // so now we pass only empty folders to add them explicitly.
213      if (!item->empty_folder)
214        continue;
215    } else if (item->type == TYPE_BOOKMARK) {
216      // Import only valid bookmarks
217      if (!CanImportURL(item->url))
218        continue;
219    } else {
220      continue;
221    }
222
223    // Skip the default bookmarks and unwanted URLs.
224    if (default_urls.find(item->url) != default_urls.end() ||
225        post_keyword_ids.find(item->id) != post_keyword_ids.end())
226      continue;
227
228    // Find the bookmark path by tracing their links to parent folders.
229    std::vector<string16> path;
230    BookmarkItem* child = item;
231    bool found_path = false;
232    bool is_in_toolbar = false;
233    while (child->parent >= 0) {
234      BookmarkItem* parent = list[child->parent];
235      if (parent->id == toolbar_folder_id) {
236        // This bookmark entry should be put in the bookmark bar.
237        // But we put it in the Firefox group after first run, so
238        // that do not mess up the bookmark bar.
239        if (import_to_bookmark_bar()) {
240          is_in_toolbar = true;
241        } else {
242          path.insert(path.begin(), parent->title);
243          path.insert(path.begin(), firefox_folder);
244        }
245        found_path = true;
246        break;
247      } else if (parent->id == menu_folder_id ||
248                 parent->id == unsorted_folder_id) {
249        // After the first run, the item will be placed in a folder in
250        // the "Other bookmarks".
251        if (!import_to_bookmark_bar())
252          path.insert(path.begin(), firefox_folder);
253        found_path = true;
254        break;
255      } else if (livemark_id.find(parent->id) != livemark_id.end()) {
256        // If the entry is under a livemark folder, we don't import it.
257        break;
258      }
259      path.insert(path.begin(), parent->title);
260      child = parent;
261    }
262
263    if (!found_path)
264      continue;
265
266    ProfileWriter::BookmarkEntry entry;
267    entry.creation_time = item->date_added;
268    entry.title = item->title;
269    entry.url = item->url;
270    entry.path = path;
271    entry.in_toolbar = is_in_toolbar;
272    entry.is_folder = item->type == TYPE_FOLDER;
273
274    bookmarks.push_back(entry);
275
276    if (item->type == TYPE_BOOKMARK) {
277      if (item->favicon)
278        favicon_map[item->favicon].insert(item->url);
279
280      // This bookmark has a keyword, we import it to our TemplateURL model.
281      TemplateURL* t_url = Firefox2Importer::CreateTemplateURL(
282          item->title, UTF8ToUTF16(item->keyword), item->url);
283      if (t_url)
284        template_urls.push_back(t_url);
285    }
286  }
287
288  STLDeleteContainerPointers(list.begin(), list.end());
289
290  // Write into profile.
291  if (!bookmarks.empty() && !cancelled()) {
292    const string16& first_folder_name =
293        bridge_->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX);
294    int options = 0;
295    if (import_to_bookmark_bar())
296      options = ProfileWriter::IMPORT_TO_BOOKMARK_BAR;
297    bridge_->AddBookmarkEntries(bookmarks, first_folder_name, options);
298  }
299  if (!template_urls.empty() && !cancelled()) {
300    bridge_->SetKeywords(template_urls, -1, false);
301  } else {
302    STLDeleteContainerPointers(template_urls.begin(), template_urls.end());
303  }
304  if (!favicon_map.empty() && !cancelled()) {
305    std::vector<history::ImportedFaviconUsage> favicons;
306    LoadFavicons(&db, favicon_map, &favicons);
307    bridge_->SetFavicons(favicons);
308  }
309}
310
311void Firefox3Importer::ImportPasswords() {
312  // Initializes NSS3.
313  NSSDecryptor decryptor;
314  if (!decryptor.Init(source_path_, source_path_) &&
315      !decryptor.Init(app_path_, source_path_)) {
316    return;
317  }
318
319  std::vector<webkit_glue::PasswordForm> forms;
320  FilePath source_path = source_path_;
321  FilePath file = source_path.AppendASCII("signons.sqlite");
322  if (file_util::PathExists(file)) {
323    // Since Firefox 3.1, passwords are in signons.sqlite db.
324    decryptor.ReadAndParseSignons(file, &forms);
325  } else {
326    // Firefox 3.0 uses signons3.txt to store the passwords.
327    file = source_path.AppendASCII("signons3.txt");
328    if (!file_util::PathExists(file))
329      file = source_path.AppendASCII("signons2.txt");
330
331    std::string content;
332    file_util::ReadFileToString(file, &content);
333    decryptor.ParseSignons(content, &forms);
334  }
335
336  if (!cancelled()) {
337    for (size_t i = 0; i < forms.size(); ++i) {
338      bridge_->SetPasswordForm(forms[i]);
339    }
340  }
341}
342
343void Firefox3Importer::ImportSearchEngines() {
344  std::vector<FilePath> files;
345  GetSearchEnginesXMLFiles(&files);
346
347  std::vector<TemplateURL*> search_engines;
348  ParseSearchEnginesFromXMLFiles(files, &search_engines);
349  int default_index =
350      GetFirefoxDefaultSearchEngineIndex(search_engines, source_path_);
351  bridge_->SetKeywords(search_engines, default_index, true);
352}
353
354void Firefox3Importer::ImportHomepage() {
355  GURL home_page = GetHomepage(source_path_);
356  if (home_page.is_valid() && !IsDefaultHomepage(home_page, app_path_)) {
357    bridge_->AddHomePage(home_page);
358  }
359}
360
361void Firefox3Importer::GetSearchEnginesXMLFiles(
362    std::vector<FilePath>* files) {
363  FilePath file = source_path_.AppendASCII("search.sqlite");
364  if (!file_util::PathExists(file))
365    return;
366
367  sql::Connection db;
368  if (!db.Open(file))
369    return;
370
371  const char* query = "SELECT engineid FROM engine_data "
372                      "WHERE engineid NOT IN "
373                      "(SELECT engineid FROM engine_data "
374                      "WHERE name='hidden') "
375                      "ORDER BY value ASC";
376
377  sql::Statement s(db.GetUniqueStatement(query));
378  if (!s)
379    return;
380
381  FilePath app_path = app_path_.AppendASCII("searchplugins");
382  FilePath profile_path = source_path_.AppendASCII("searchplugins");
383
384  // Firefox doesn't store a search engine in its sqlite database unless the
385  // user has added a engine. So we get search engines from sqlite db as well
386  // as from the file system.
387  if (s.Step()) {
388    const std::wstring kAppPrefix = L"[app]/";
389    const std::wstring kProfilePrefix = L"[profile]/";
390    do {
391      FilePath file;
392      std::wstring engine = UTF8ToWide(s.ColumnString(0));
393
394      // The string contains [app]/<name>.xml or [profile]/<name>.xml where
395      // the [app] and [profile] need to be replaced with the actual app or
396      // profile path.
397      size_t index = engine.find(kAppPrefix);
398      if (index != std::wstring::npos) {
399        // Remove '[app]/'.
400        file = app_path.Append(FilePath::FromWStringHack(
401                                   engine.substr(index + kAppPrefix.length())));
402      } else if ((index = engine.find(kProfilePrefix)) != std::wstring::npos) {
403        // Remove '[profile]/'.
404          file = profile_path.Append(
405              FilePath::FromWStringHack(
406                  engine.substr(index + kProfilePrefix.length())));
407      } else {
408        // Looks like absolute path to the file.
409        file = FilePath::FromWStringHack(engine);
410      }
411      files->push_back(file);
412    } while (s.Step() && !cancelled());
413  }
414
415#if defined(OS_LINUX)
416  // Ubuntu-flavored Firefox3 supports locale-specific search engines via
417  // locale-named subdirectories. They fall back to en-US.
418  // See http://crbug.com/53899
419  // TODO(jshin): we need to make sure our locale code matches that of
420  // Firefox.
421  FilePath locale_app_path = app_path.AppendASCII(locale_);
422  FilePath default_locale_app_path = app_path.AppendASCII("en-US");
423  if (file_util::DirectoryExists(locale_app_path))
424    app_path = locale_app_path;
425  else if (file_util::DirectoryExists(default_locale_app_path))
426    app_path = default_locale_app_path;
427#endif
428
429  // Get search engine definition from file system.
430  file_util::FileEnumerator engines(app_path, false,
431                                    file_util::FileEnumerator::FILES);
432  for (FilePath engine_path = engines.Next(); !engine_path.value().empty();
433       engine_path = engines.Next()) {
434    files->push_back(engine_path);
435  }
436}
437
438void Firefox3Importer::LoadRootNodeID(sql::Connection* db,
439                                      int* toolbar_folder_id,
440                                      int* menu_folder_id,
441                                      int* unsorted_folder_id) {
442  static const char* kToolbarFolderName = "toolbar";
443  static const char* kMenuFolderName = "menu";
444  static const char* kUnsortedFolderName = "unfiled";
445
446  const char* query = "SELECT root_name, folder_id FROM moz_bookmarks_roots";
447  sql::Statement s(db->GetUniqueStatement(query));
448  if (!s)
449    return;
450
451  while (s.Step()) {
452    std::string folder = s.ColumnString(0);
453    int id = s.ColumnInt(1);
454    if (folder == kToolbarFolderName)
455      *toolbar_folder_id = id;
456    else if (folder == kMenuFolderName)
457      *menu_folder_id = id;
458    else if (folder == kUnsortedFolderName)
459      *unsorted_folder_id = id;
460  }
461}
462
463void Firefox3Importer::LoadLivemarkIDs(sql::Connection* db,
464                                       std::set<int>* livemark) {
465  static const char* kFeedAnnotation = "livemark/feedURI";
466  livemark->clear();
467
468  const char* query = "SELECT b.item_id "
469                      "FROM moz_anno_attributes a "
470                      "JOIN moz_items_annos b ON a.id = b.anno_attribute_id "
471                      "WHERE a.name = ? ";
472  sql::Statement s(db->GetUniqueStatement(query));
473  if (!s)
474    return;
475
476  s.BindString(0, kFeedAnnotation);
477  while (s.Step() && !cancelled())
478    livemark->insert(s.ColumnInt(0));
479}
480
481void Firefox3Importer::GetTopBookmarkFolder(sql::Connection* db,
482                                            int folder_id,
483                                            BookmarkList* list) {
484  const char* query = "SELECT b.title "
485                     "FROM moz_bookmarks b "
486                     "WHERE b.type = 2 AND b.id = ? "
487                     "ORDER BY b.position";
488  sql::Statement s(db->GetUniqueStatement(query));
489  if (!s)
490    return;
491
492  s.BindInt(0, folder_id);
493  if (s.Step()) {
494    BookmarkItem* item = new BookmarkItem;
495    item->parent = -1;  // The top level folder has no parent.
496    item->id = folder_id;
497    item->title = s.ColumnString16(0);
498    item->type = TYPE_FOLDER;
499    item->favicon = 0;
500    item->empty_folder = true;
501    list->push_back(item);
502  }
503}
504
505void Firefox3Importer::GetWholeBookmarkFolder(sql::Connection* db,
506                                              BookmarkList* list,
507                                              size_t position,
508                                              bool* empty_folder) {
509  if (position >= list->size()) {
510    NOTREACHED();
511    return;
512  }
513
514  const char* query = "SELECT b.id, h.url, COALESCE(b.title, h.title), "
515         "b.type, k.keyword, b.dateAdded, h.favicon_id "
516         "FROM moz_bookmarks b "
517         "LEFT JOIN moz_places h ON b.fk = h.id "
518         "LEFT JOIN moz_keywords k ON k.id = b.keyword_id "
519         "WHERE b.type IN (1,2) AND b.parent = ? "
520         "ORDER BY b.position";
521  sql::Statement s(db->GetUniqueStatement(query));
522  if (!s)
523    return;
524
525  s.BindInt(0, (*list)[position]->id);
526  BookmarkList temp_list;
527  while (s.Step()) {
528    BookmarkItem* item = new BookmarkItem;
529    item->parent = static_cast<int>(position);
530    item->id = s.ColumnInt(0);
531    item->url = GURL(s.ColumnString(1));
532    item->title = s.ColumnString16(2);
533    item->type = static_cast<BookmarkItemType>(s.ColumnInt(3));
534    item->keyword = s.ColumnString(4);
535    item->date_added = base::Time::FromTimeT(s.ColumnInt64(5)/1000000);
536    item->favicon = s.ColumnInt64(6);
537    item->empty_folder = true;
538
539    temp_list.push_back(item);
540    if (empty_folder != NULL)
541      *empty_folder = false;
542  }
543
544  // Appends all items to the list.
545  for (BookmarkList::iterator i = temp_list.begin();
546       i != temp_list.end(); ++i) {
547    list->push_back(*i);
548    // Recursive add bookmarks in sub-folders.
549    if ((*i)->type == TYPE_FOLDER)
550      GetWholeBookmarkFolder(db, list, list->size() - 1, &(*i)->empty_folder);
551  }
552}
553
554void Firefox3Importer::LoadFavicons(
555    sql::Connection* db,
556    const FaviconMap& favicon_map,
557    std::vector<history::ImportedFaviconUsage>* favicons) {
558  const char* query = "SELECT url, data FROM moz_favicons WHERE id=?";
559  sql::Statement s(db->GetUniqueStatement(query));
560  if (!s)
561    return;
562
563  for (FaviconMap::const_iterator i = favicon_map.begin();
564       i != favicon_map.end(); ++i) {
565    s.BindInt64(0, i->first);
566    if (s.Step()) {
567      history::ImportedFaviconUsage usage;
568
569      usage.favicon_url = GURL(s.ColumnString(0));
570      if (!usage.favicon_url.is_valid())
571        continue;  // Don't bother importing favicons with invalid URLs.
572
573      std::vector<unsigned char> data;
574      s.ColumnBlobAsVector(1, &data);
575      if (data.empty())
576        continue;  // Data definitely invalid.
577
578      if (!ReencodeFavicon(&data[0], data.size(), &usage.png_data))
579        continue;  // Unable to decode.
580
581      usage.urls = i->second;
582      favicons->push_back(usage);
583    }
584    s.Reset();
585  }
586}
587