1// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// This tool converts Hunspell .aff/.dic pairs to a combined binary dictionary
6// format (.bdic). This format is more compact, and can be more efficiently
7// read by the client application.
8//
9// We do this conversion manually before publishing dictionary files. It is not
10// part of any build process.
11//
12// See PrintHelp() below for usage.
13
14#include <stdio.h>
15
16#include "base/at_exit.h"
17#include "base/files/file_path.h"
18#include "base/files/file_util.h"
19#include "base/i18n/icu_util.h"
20#include "base/logging.h"
21#include "base/process/memory.h"
22#include "base/strings/string_util.h"
23#include "chrome/tools/convert_dict/aff_reader.h"
24#include "chrome/tools/convert_dict/dic_reader.h"
25#include "third_party/hunspell/google/bdict_reader.h"
26#include "third_party/hunspell/google/bdict_writer.h"
27
28namespace {
29
30// Compares the given word list with the serialized trie to make sure they
31// are the same.
32bool VerifyWords(const convert_dict::DicReader::WordList& org_words,
33                 const std::string& serialized) {
34  hunspell::BDictReader reader;
35  if (!reader.Init(reinterpret_cast<const unsigned char*>(serialized.data()),
36                   serialized.size())) {
37    printf("BDict is invalid\n");
38    return false;
39  }
40  hunspell::WordIterator iter = reader.GetAllWordIterator();
41
42  int affix_ids[hunspell::BDict::MAX_AFFIXES_PER_WORD];
43
44  static const int buf_size = 128;
45  char buf[buf_size];
46  for (size_t i = 0; i < org_words.size(); i++) {
47    int affix_matches = iter.Advance(buf, buf_size, affix_ids);
48    if (affix_matches == 0) {
49      printf("Found the end before we expected\n");
50      return false;
51    }
52
53    if (org_words[i].first != buf) {
54      printf("Word doesn't match, word #%s\n", buf);
55      return false;
56    }
57
58    if (affix_matches != static_cast<int>(org_words[i].second.size())) {
59      printf("Different number of affix indices, word #%s\n", buf);
60      return false;
61    }
62
63    // Check the individual affix indices.
64    for (size_t affix_index = 0; affix_index < org_words[i].second.size();
65         affix_index++) {
66      if (affix_ids[affix_index] != org_words[i].second[affix_index]) {
67        printf("Index doesn't match, word #%s\n", buf);
68        return false;
69      }
70    }
71  }
72
73  return true;
74}
75
76int PrintHelp() {
77  printf("Usage: convert_dict <dicfile base name>\n\n");
78  printf("Example:\n");
79  printf("  convert_dict en-US\nwill read en-US.dic, en-US.dic_delta, and "
80         "en-US.aff from the current directory and generate en-US.bdic\n\n");
81  return 1;
82}
83
84}  // namespace
85
86#if defined(OS_WIN)
87int wmain(int argc, wchar_t* argv[]) {
88#else
89int main(int argc, char* argv[]) {
90#endif
91  base::EnableTerminationOnHeapCorruption();
92  if (argc != 2)
93    return PrintHelp();
94
95  base::AtExitManager exit_manager;
96  base::i18n::InitializeICU();
97
98  base::FilePath file_base = base::FilePath(argv[1]);
99
100  base::FilePath aff_path =
101      file_base.ReplaceExtension(FILE_PATH_LITERAL(".aff"));
102  printf("Reading %" PRFilePath " ...\n", aff_path.value().c_str());
103  convert_dict::AffReader aff_reader(aff_path);
104  if (!aff_reader.Read()) {
105    printf("Unable to read the aff file.\n");
106    return 1;
107  }
108
109  base::FilePath dic_path =
110      file_base.ReplaceExtension(FILE_PATH_LITERAL(".dic"));
111  printf("Reading %" PRFilePath " ...\n", dic_path.value().c_str());
112  // DicReader will also read the .dic_delta file.
113  convert_dict::DicReader dic_reader(dic_path);
114  if (!dic_reader.Read(&aff_reader)) {
115    printf("Unable to read the dic file.\n");
116    return 1;
117  }
118
119  hunspell::BDictWriter writer;
120  writer.SetComment(aff_reader.comments());
121  writer.SetAffixRules(aff_reader.affix_rules());
122  writer.SetAffixGroups(aff_reader.GetAffixGroups());
123  writer.SetReplacements(aff_reader.replacements());
124  writer.SetOtherCommands(aff_reader.other_commands());
125  writer.SetWords(dic_reader.words());
126
127  printf("Serializing...\n");
128  std::string serialized = writer.GetBDict();
129
130  printf("Verifying...\n");
131  if (!VerifyWords(dic_reader.words(), serialized)) {
132    printf("ERROR converting, the dictionary does not check out OK.");
133    return 1;
134  }
135
136  base::FilePath out_path =
137      file_base.ReplaceExtension(FILE_PATH_LITERAL(".bdic"));
138  printf("Writing %" PRFilePath " ...\n", out_path.value().c_str());
139  FILE* out_file = base::OpenFile(out_path, "wb");
140  if (!out_file) {
141    printf("ERROR writing file\n");
142    return 1;
143  }
144  size_t written = fwrite(&serialized[0], 1, serialized.size(), out_file);
145  CHECK(written == serialized.size());
146  base::CloseFile(out_file);
147
148  return 0;
149}
150