14a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// symbol-table.h 24a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 34a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// Licensed under the Apache License, Version 2.0 (the "License"); 44a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// you may not use this file except in compliance with the License. 54a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// You may obtain a copy of the License at 64a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 74a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// http://www.apache.org/licenses/LICENSE-2.0 84a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 94a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// Unless required by applicable law or agreed to in writing, software 104a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// distributed under the License is distributed on an "AS IS" BASIS, 114a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 124a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// See the License for the specific language governing permissions and 134a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// limitations under the License. 144a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 154a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 164a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// \file 174a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// Classes to provide symbol-to-integer and integer-to-symbol mappings. 184a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 194a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#ifndef FST_LIB_SYMBOL_TABLE_H__ 204a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#define FST_LIB_SYMBOL_TABLE_H__ 214a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 224a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#include <fstream> 234a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#include <iostream> 244a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#include <string> 2573018b4a1d088cdda0e7bd059fddf1f308a8195aIan Rogers#include <unordered_map> 264a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#include <vector> 274a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 284a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#include "fst/lib/compat.h" 294a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 304a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 314a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 324a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source ProjectDECLARE_bool(fst_compat_symbols); 334a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 344a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Projectnamespace fst { 354a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 364a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Projectclass SymbolTableImpl { 374a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project friend class SymbolTableIterator; 384a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project public: 394a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTableImpl(const string &name) 404a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project : name_(name), available_key_(0), ref_count_(1), 414a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project check_sum_finalized_(false) {} 424a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project ~SymbolTableImpl() { 434a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project for (size_t i = 0; i < symbols_.size(); ++i) 444a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project delete[] symbols_[i]; 454a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 464a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 474a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 AddSymbol(const string& symbol, int64 key); 484a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 494a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 AddSymbol(const string& symbol) { 504a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 key = Find(symbol); 514a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return (key == -1) ? AddSymbol(symbol, available_key_++) : key; 524a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 534a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 544a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project void AddTable(SymbolTableImpl* table) { 554a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project for (size_t i = 0; i < table->symbols_.size(); ++i) { 564a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project AddSymbol(table->symbols_[i]); 574a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 584a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 594a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 604a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project static SymbolTableImpl* ReadText(const string& filename); 614a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 624a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project static SymbolTableImpl* Read(istream &strm, const string& source); 634a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 644a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project bool Write(ostream &strm) const; 654a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 664a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project bool WriteText(ostream &strm) const; 674a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 684a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // 694a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Return the string associated with the key. If the key is out of 704a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // range (<0, >max), return an empty string. 714a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project string Find(int64 key) const { 7273018b4a1d088cdda0e7bd059fddf1f308a8195aIan Rogers std::unordered_map<int64, string>::const_iterator it = 734a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project key_map_.find(key); 744a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (it == key_map_.end()) { 754a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return ""; 764a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 774a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return it->second; 784a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 794a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 804a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // 814a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Return the key associated with the symbol. If the symbol 824a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // does not exists, return -1. 834a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 Find(const string& symbol) const { 844a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return Find(symbol.c_str()); 854a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 864a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 874a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // 884a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Return the key associated with the symbol. If the symbol 894a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // does not exists, return -1. 904a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 Find(const char* symbol) const { 9173018b4a1d088cdda0e7bd059fddf1f308a8195aIan Rogers unordered_map<string, int64>::const_iterator it = 924a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project symbol_map_.find(symbol); 934a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (it == symbol_map_.end()) { 944a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return -1; 954a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 964a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return it->second; 974a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 984a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 994a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project const string& Name() const { return name_; } 1004a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1014a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int IncrRefCount() const { 1024a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return ++ref_count_; 1034a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1044a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int DecrRefCount() const { 1054a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return --ref_count_; 1064a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1074a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1084a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project string CheckSum() const { 1094a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!check_sum_finalized_) { 1104a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project RecomputeCheckSum(); 1114a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project check_sum_string_ = check_sum_.Digest(); 1124a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1134a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return check_sum_string_; 1144a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1154a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1164a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 AvailableKey() const { 1174a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return available_key_; 1184a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1194a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1204a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // private support methods 1214a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project private: 1224a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project void RecomputeCheckSum() const; 1234a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project static SymbolTableImpl* Read1(istream &, const string &); 1244a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1254a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project string name_; 1264a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 available_key_; 1274a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project vector<const char *> symbols_; 12873018b4a1d088cdda0e7bd059fddf1f308a8195aIan Rogers std::unordered_map<int64, string> key_map_; 12973018b4a1d088cdda0e7bd059fddf1f308a8195aIan Rogers std::unordered_map<string, int64> symbol_map_; 1304a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1314a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project mutable int ref_count_; 1324a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project mutable bool check_sum_finalized_; 1334a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project mutable MD5 check_sum_; 1344a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project mutable string check_sum_string_; 1354a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1364a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project DISALLOW_EVIL_CONSTRUCTORS(SymbolTableImpl); 1374a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project}; 1384a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1394a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1404a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Projectclass SymbolTableIterator; 1414a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1424a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 1434a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// \class SymbolTable 1444a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// \brief Symbol (string) to int and reverse mapping 1454a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 1464a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// The SymbolTable implements the mappings of labels to strings and reverse. 1474a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// SymbolTables are used to describe the alphabet of the input and output 1484a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// labels for arcs in a Finite State Transducer. 1494a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 1504a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// SymbolTables are reference counted and can therefore be shared across 1514a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// multiple machines. For example a language model grammar G, with a 1524a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// SymbolTable for the words in the language model can share this symbol 1534a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// table with the lexical representation L o G. 1544a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 1554a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Projectclass SymbolTable { 1564a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project friend class SymbolTableIterator; 1574a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project public: 1584a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project static const int64 kNoSymbol = -1; 1594a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1604a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Construct symbol table with a unique name. 1614a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTable(const string& name) : impl_(new SymbolTableImpl(name)) {} 1624a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1634a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Create a reference counted copy. 1644a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTable(const SymbolTable& table) : impl_(table.impl_) { 1654a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project impl_->IncrRefCount(); 1664a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1674a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1684a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Derefence implentation object. When reference count hits 0, delete 1694a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // implementation. 1704a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project ~SymbolTable() { 1714a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!impl_->DecrRefCount()) delete impl_; 1724a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1734a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1744a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // create a reference counted copy 1754a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTable* Copy() const { 1764a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return new SymbolTable(*this); 1774a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1784a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1794a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Add a symbol with given key to table. A symbol table also 1804a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // keeps track of the last available key (highest key value in 1814a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // the symbol table). 1824a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // 1834a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // \param symbol string symbol to add 1844a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // \param key associated key for string symbol 1854a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // \return the key created by the symbol table. Symbols allready added to 1864a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // the symbol table will not get a different key. 1874a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 AddSymbol(const string& symbol, int64 key) { 1884a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->AddSymbol(symbol, key); 1894a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1904a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 1914a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Add a symbol to the table. The associated value key is automatically 1924a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // assigned by the symbol table. 1934a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // 1944a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // \param symbol string to add to the table 1954a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // \return the value key assigned to the associated string symbol 1964a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 AddSymbol(const string& symbol) { 1974a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->AddSymbol(symbol); 1984a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 1994a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2004a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Add another symbol table to this table. All key values will be offset 2014a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // by the current available key (highest key value in the symbol table). 2024a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Note string symbols with the same key value with still have the same 2034a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // key value after the symbol table has been merged, but a different 2044a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // value. Adding symbol tables do not result in changes in the base table. 2054a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // 2064a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Merging N symbol tables is often useful when combining the various 2074a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // name spaces of transducers to a unified representation. 2084a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // 2094a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // \param table the symbol table to add to this table 2104a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project void AddTable(const SymbolTable& table) { 2114a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->AddTable(table.impl_); 2124a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2134a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2144a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // return the name of the symbol table 2154a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project const string& Name() const { 2164a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->Name(); 2174a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2184a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2194a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // return the MD5 check-sum for this table. All new symbols added to 2204a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // the table will result in an updated checksum. 2214a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project string CheckSum() const { 2224a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->CheckSum(); 2234a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2244a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2254a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // read an ascii representation of the symbol table 2264a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project static SymbolTable* ReadText(const string& filename) { 2274a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTableImpl* impl = SymbolTableImpl::ReadText(filename); 2284a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!impl) 2294a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return 0; 2304a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project else 2314a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return new SymbolTable(impl); 2324a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2334a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2344a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // read a binary dump of the symbol table 2354a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project static SymbolTable* Read(istream &strm, const string& source) { 2364a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTableImpl* impl = SymbolTableImpl::Read(strm, source); 2374a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!impl) 2384a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return 0; 2394a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project else 2404a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return new SymbolTable(impl); 2414a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2424a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2434a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // read a binary dump of the symbol table 2444a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project static SymbolTable* Read(const string& filename) { 2454a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project ifstream strm(filename.c_str()); 2464a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!strm) { 2474a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project LOG(ERROR) << "SymbolTable::Read: Can't open file " << filename; 2484a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return 0; 2494a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2504a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return Read(strm, filename); 2514a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2524a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2534a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project bool Write(ostream &strm) const { 2544a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->Write(strm); 2554a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2564a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2574a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project bool Write(const string& filename) const { 2584a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project ofstream strm(filename.c_str()); 2594a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!strm) { 2604a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project LOG(ERROR) << "SymbolTable::Write: Can't open file " << filename; 2614a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return false; 2624a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2634a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return Write(strm); 2644a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2654a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2664a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Dump an ascii text representation of the symbol table 2674a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project bool WriteText(ostream &strm) const { 2684a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->WriteText(strm); 2694a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2704a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2714a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Dump an ascii text representation of the symbol table 2724a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project bool WriteText(const string& filename) const { 2734a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project ofstream strm(filename.c_str()); 2744a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!strm) { 2754a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project LOG(ERROR) << "SymbolTable::WriteText: Can't open file " << filename; 2764a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return false; 2774a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2784a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return WriteText(strm); 2794a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2804a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2814a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Return the string associated with the key. If the key is out of 2824a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // range (<0, >max), log error and return an empty string. 2834a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project string Find(int64 key) const { 2844a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->Find(key); 2854a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2864a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2874a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Return the key associated with the symbol. If the symbol 2884a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // does not exists, log error and return -1 2894a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 Find(const string& symbol) const { 2904a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->Find(symbol); 2914a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2924a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2934a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Return the key associated with the symbol. If the symbol 2944a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // does not exists, log error and return -1 2954a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 Find(const char* symbol) const { 2964a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->Find(symbol); 2974a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 2984a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 2994a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // return the current available key (i.e highest key number) in 3004a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // the symbol table 3014a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 AvailableKey(void) const { 3024a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->AvailableKey(); 3034a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3044a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3054a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project protected: 3064a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project explicit SymbolTable(SymbolTableImpl* impl) : impl_(impl) {} 3074a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3084a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project const SymbolTableImpl* Impl() const { 3094a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_; 3104a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3114a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3124a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project private: 3134a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTableImpl* impl_; 3144a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3154a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3164a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project void operator=(const SymbolTable &table); // disallow 3174a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project}; 3184a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3194a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3204a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// 3214a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// \class SymbolTableIterator 3224a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// \brief Iterator class for symbols in a symbol table 3234a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Projectclass SymbolTableIterator { 3244a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project public: 3254a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // Constructor creates a refcounted copy of underlying implementation 3264a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project SymbolTableIterator(const SymbolTable& symbol_table) { 3274a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project impl_ = symbol_table.Impl(); 3284a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project impl_->IncrRefCount(); 3294a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project pos_ = 0; 3304a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project size_ = impl_->symbols_.size(); 3314a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3324a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3334a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // decrement implementation refcount, and delete if 0 3344a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project ~SymbolTableIterator() { 3354a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!impl_->DecrRefCount()) delete impl_; 3364a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3374a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3384a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // is iterator done 3394a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project bool Done(void) { 3404a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return (pos_ == size_); 3414a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3424a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3434a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // return the Value() of the current symbol (in64 key) 3444a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project int64 Value(void) { 3454a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->Find(impl_->symbols_[pos_]); 3464a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3474a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3484a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // return the string of the current symbol 3494a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project const char* Symbol(void) { 3504a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return impl_->symbols_[pos_]; 3514a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3524a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3534a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // advance iterator forward 3544a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project void Next(void) { 3554a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (Done()) return; 3564a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project ++pos_; 3574a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3584a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3594a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project // reset iterator 3604a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project void Reset(void) { 3614a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project pos_ = 0; 3624a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project } 3634a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3644a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project private: 3654a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project const SymbolTableImpl* impl_; 3664a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project size_t pos_; 3674a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project size_t size_; 3684a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project}; 3694a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3704a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3714a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project// Tests compatibilty between two sets of symbol tables 3724a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Projectinline bool CompatSymbols(const SymbolTable *syms1, 3734a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project const SymbolTable *syms2) { 3744a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project if (!FLAGS_fst_compat_symbols) 3754a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return true; 3764a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project else if (!syms1 && !syms2) 3774a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return true; 37873018b4a1d088cdda0e7bd059fddf1f308a8195aIan Rogers else if ((syms1 && !syms2) || (!syms1 && syms2)) 3794a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return false; 3804a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project else 3814a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project return syms1->CheckSum() == syms2->CheckSum(); 3824a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project} 3834a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3844a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project} // namespace fst 3854a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project 3864a68b3365c8c50aa93505e99ead2565ab73dcdb0The Android Open Source Project#endif // FST_LIB_SYMBOL_TABLE_H__ 387