1f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// far.h 2f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 3f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// Licensed under the Apache License, Version 2.0 (the "License"); 4f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// you may not use this file except in compliance with the License. 5f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// You may obtain a copy of the License at 6f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// 7f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// http://www.apache.org/licenses/LICENSE-2.0 8f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// 9f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// Unless required by applicable law or agreed to in writing, software 10f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// distributed under the License is distributed on an "AS IS" BASIS, 11f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// See the License for the specific language governing permissions and 13f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// limitations under the License. 14f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// 15f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// Copyright 2005-2010 Google, Inc. 16f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// Author: riley@google.com (Michael Riley) 17f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// 18f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// \file 19f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// Finite-State Transducer (FST) archive classes. 20f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// 21f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 22f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson#ifndef FST_EXTENSIONS_FAR_FAR_H__ 23f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson#define FST_EXTENSIONS_FAR_FAR_H__ 24f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 25f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson#include <fst/extensions/far/stlist.h> 26f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson#include <fst/extensions/far/sttable.h> 27f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson#include <fst/fst.h> 28f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson#include <fst/vector-fst.h> 29f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 30f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonnamespace fst { 31f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 32f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonenum FarEntryType { FET_LINE, FET_FILE }; 33f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonenum FarTokenType { FTT_SYMBOL, FTT_BYTE, FTT_UTF8 }; 34f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 35dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkininline bool IsFst(const string &filename) { 36dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ifstream strm(filename.c_str()); 37dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (!strm) 38dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return false; 39dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return IsFstHeader(strm, filename); 40dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin} 41dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 42f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// FST archive header class 43f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass FarHeader { 44f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 45f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson const string &FarType() const { return fartype_; } 46f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson const string &ArcType() const { return arctype_; } 47f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 48f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Read(const string &filename) { 49f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson FstHeader fsthdr; 50dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (filename.empty()) { 51dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin // Header reading unsupported on stdin. Assumes STList and StdArc. 52dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin fartype_ = "stlist"; 53dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin arctype_ = "standard"; 54dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return true; 55f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } else if (IsSTTable(filename)) { // Check if STTable 56f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson ReadSTTableHeader(filename, &fsthdr); 57f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson fartype_ = "sttable"; 58f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson arctype_ = fsthdr.ArcType().empty() ? "unknown" : fsthdr.ArcType(); 59f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return true; 60f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } else if (IsSTList(filename)) { // Check if STList 61f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson ReadSTListHeader(filename, &fsthdr); 62f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson fartype_ = "sttable"; 63f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson arctype_ = fsthdr.ArcType().empty() ? "unknown" : fsthdr.ArcType(); 64f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return true; 65dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } else if (IsFst(filename)) { // Check if Fst 66dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ifstream istrm(filename.c_str()); 67dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin fsthdr.Read(istrm, filename); 68dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin fartype_ = "fst"; 69dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin arctype_ = fsthdr.ArcType().empty() ? "unknown" : fsthdr.ArcType(); 70dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return true; 71f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 72f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return false; 73f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 74f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 75f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 76f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson string fartype_; 77f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson string arctype_; 78f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 79f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 80dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkinenum FarType { 81dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FAR_DEFAULT = 0, 82dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FAR_STTABLE = 1, 83dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FAR_STLIST = 2, 84dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FAR_FST = 3, 85dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin}; 86f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 87f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// This class creates an archive of FSTs. 88f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 89f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass FarWriter { 90f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 91f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson typedef A Arc; 92f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 93f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Creates a new (empty) FST archive; returns NULL on error. 94f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson static FarWriter *Create(const string &filename, FarType type = FAR_DEFAULT); 95f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 96f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Adds an FST to the end of an archive. Keys must be non-empty and 97f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // in lexicographic order. FSTs must have a suitable write method. 98f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual void Add(const string &key, const Fst<A> &fst) = 0; 99f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 100f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual FarType Type() const = 0; 101f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 102f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual bool Error() const = 0; 103f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 104f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual ~FarWriter() {} 105f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 106f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson protected: 107f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson FarWriter() {} 108f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 109f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 110f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson DISALLOW_COPY_AND_ASSIGN(FarWriter); 111f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 112f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 113f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 114f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson// This class iterates through an existing archive of FSTs. 115f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 116f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass FarReader { 117f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 118f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson typedef A Arc; 119f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 120f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Opens an existing FST archive in a single file; returns NULL on error. 121f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Sets current position to the beginning of the achive. 122f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson static FarReader *Open(const string &filename); 123f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 124f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Opens an existing FST archive in multiple files; returns NULL on error. 125f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Sets current position to the beginning of the achive. 126f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson static FarReader *Open(const vector<string> &filenames); 127f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 128f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Resets current posision to beginning of archive. 129f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual void Reset() = 0; 130f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 131f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Sets current position to first entry >= key. Returns true if a match. 132f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual bool Find(const string &key) = 0; 133f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 134f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Current position at end of archive? 135f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual bool Done() const = 0; 136f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 137f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Move current position to next FST. 138f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual void Next() = 0; 139f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 140f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Returns key at the current position. This reference is invalidated if 141f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // the current position in the archive is changed. 142f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual const string &GetKey() const = 0; 143f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 144f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // Returns FST at the current position. This reference is invalidated if 145f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // the current position in the archive is changed. 146f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual const Fst<A> &GetFst() const = 0; 147f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 148f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual FarType Type() const = 0; 149f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 150f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual bool Error() const = 0; 151f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 152f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson virtual ~FarReader() {} 153f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 154f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson protected: 155f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson FarReader() {} 156f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 157f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 158f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson DISALLOW_COPY_AND_ASSIGN(FarReader); 159f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 160f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 161f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 162f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 163f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass FstWriter { 164f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 165f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson void operator()(ostream &strm, const Fst<A> &fst) const { 166f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson fst.Write(strm, FstWriteOptions()); 167f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 168f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 169f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 170f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 171f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 172f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass STTableFarWriter : public FarWriter<A> { 173f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 174f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson typedef A Arc; 175f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 176dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin static STTableFarWriter *Create(const string &filename) { 177f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableWriter<Fst<A>, FstWriter<A> > *writer = 178f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableWriter<Fst<A>, FstWriter<A> >::Create(filename); 179f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return new STTableFarWriter(writer); 180f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 181f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 182f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson void Add(const string &key, const Fst<A> &fst) { writer_->Add(key, fst); } 183f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 184f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson FarType Type() const { return FAR_STTABLE; } 185f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 186f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Error() const { return writer_->Error(); } 187f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 188f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson ~STTableFarWriter() { delete writer_; } 189f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 190f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 191f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson explicit STTableFarWriter(STTableWriter<Fst<A>, FstWriter<A> > *writer) 192f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson : writer_(writer) {} 193f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 194f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 195f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableWriter<Fst<A>, FstWriter<A> > *writer_; 196f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 197f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson DISALLOW_COPY_AND_ASSIGN(STTableFarWriter); 198f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 199f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 200f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 201f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 202f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass STListFarWriter : public FarWriter<A> { 203f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 204f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson typedef A Arc; 205f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 206dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin static STListFarWriter *Create(const string &filename) { 207f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListWriter<Fst<A>, FstWriter<A> > *writer = 208f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListWriter<Fst<A>, FstWriter<A> >::Create(filename); 209f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return new STListFarWriter(writer); 210f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 211f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 212f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson void Add(const string &key, const Fst<A> &fst) { writer_->Add(key, fst); } 213f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 214f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson FarType Type() const { return FAR_STLIST; } 215f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 216f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Error() const { return writer_->Error(); } 217f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 218f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson ~STListFarWriter() { delete writer_; } 219f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 220f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 221f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson explicit STListFarWriter(STListWriter<Fst<A>, FstWriter<A> > *writer) 222f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson : writer_(writer) {} 223f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 224f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 225f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListWriter<Fst<A>, FstWriter<A> > *writer_; 226f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 227f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson DISALLOW_COPY_AND_ASSIGN(STListFarWriter); 228f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 229f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 230f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 231f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 232dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkinclass FstFarWriter : public FarWriter<A> { 233dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin public: 234dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin typedef A Arc; 235dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 236dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin explicit FstFarWriter(const string &filename) 237dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin : filename_(filename), error_(false), written_(false) {} 238dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 239dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin static FstFarWriter *Create(const string &filename) { 240dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return new FstFarWriter(filename); 241dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 242dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 243dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin void Add(const string &key, const Fst<A> &fst) { 244dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (written_) { 245dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin LOG(WARNING) << "FstFarWriter::Add: only one Fst supported," 246dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin << " subsequent entries discarded."; 247dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } else { 248dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin error_ = !fst.Write(filename_); 249dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin written_ = true; 250dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 251dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 252dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 253dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FarType Type() const { return FAR_FST; } 254dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 255dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin bool Error() const { return error_; } 256dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 257dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ~FstFarWriter() {} 258dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 259dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin private: 260dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin string filename_; 261dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin bool error_; 262dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin bool written_; 263dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 264dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin DISALLOW_COPY_AND_ASSIGN(FstFarWriter); 265dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin}; 266dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 267dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 268dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkintemplate <class A> 269f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian HodsonFarWriter<A> *FarWriter<A>::Create(const string &filename, FarType type) { 270f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson switch(type) { 271f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson case FAR_DEFAULT: 272f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson if (filename.empty()) 273f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STListFarWriter<A>::Create(filename); 274f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson case FAR_STTABLE: 275f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STTableFarWriter<A>::Create(filename); 276f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson case FAR_STLIST: 277f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STListFarWriter<A>::Create(filename); 278dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin case FAR_FST: 279dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return FstFarWriter<A>::Create(filename); 280f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson default: 281f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson LOG(ERROR) << "FarWriter::Create: unknown far type"; 282f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return 0; 283f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 284f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson} 285f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 286f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 287f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 288f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass FstReader { 289f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 290f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson Fst<A> *operator()(istream &strm) const { 291f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return Fst<A>::Read(strm, FstReadOptions()); 292f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 293f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 294f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 295f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 296f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 297f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass STTableFarReader : public FarReader<A> { 298f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 299f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson typedef A Arc; 300f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 301f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson static STTableFarReader *Open(const string &filename) { 302f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableReader<Fst<A>, FstReader<A> > *reader = 303f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableReader<Fst<A>, FstReader<A> >::Open(filename); 304f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // TODO: error check 305f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return new STTableFarReader(reader); 306f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 307f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 308f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson static STTableFarReader *Open(const vector<string> &filenames) { 309f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableReader<Fst<A>, FstReader<A> > *reader = 310f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableReader<Fst<A>, FstReader<A> >::Open(filenames); 311f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // TODO: error check 312f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return new STTableFarReader(reader); 313f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 314f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 315f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson void Reset() { reader_->Reset(); } 316f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 317f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Find(const string &key) { return reader_->Find(key); } 318f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 319f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Done() const { return reader_->Done(); } 320f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 321f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson void Next() { return reader_->Next(); } 322f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 323f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson const string &GetKey() const { return reader_->GetKey(); } 324f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 325f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson const Fst<A> &GetFst() const { return reader_->GetEntry(); } 326f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 327f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson FarType Type() const { return FAR_STTABLE; } 328f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 329f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Error() const { return reader_->Error(); } 330f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 331f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson ~STTableFarReader() { delete reader_; } 332f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 333f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 334f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson explicit STTableFarReader(STTableReader<Fst<A>, FstReader<A> > *reader) 335f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson : reader_(reader) {} 336f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 337f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 338f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STTableReader<Fst<A>, FstReader<A> > *reader_; 339f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 340f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson DISALLOW_COPY_AND_ASSIGN(STTableFarReader); 341f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 342f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 343f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 344f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 345f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsonclass STListFarReader : public FarReader<A> { 346f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson public: 347f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson typedef A Arc; 348f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 349f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson static STListFarReader *Open(const string &filename) { 350f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListReader<Fst<A>, FstReader<A> > *reader = 351f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListReader<Fst<A>, FstReader<A> >::Open(filename); 352f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // TODO: error check 353f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return new STListFarReader(reader); 354f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 355f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 356f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson static STListFarReader *Open(const vector<string> &filenames) { 357f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListReader<Fst<A>, FstReader<A> > *reader = 358f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListReader<Fst<A>, FstReader<A> >::Open(filenames); 359f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson // TODO: error check 360f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return new STListFarReader(reader); 361f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson } 362f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 363f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson void Reset() { reader_->Reset(); } 364f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 365f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Find(const string &key) { return reader_->Find(key); } 366f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 367f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Done() const { return reader_->Done(); } 368f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 369f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson void Next() { return reader_->Next(); } 370f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 371f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson const string &GetKey() const { return reader_->GetKey(); } 372f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 373f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson const Fst<A> &GetFst() const { return reader_->GetEntry(); } 374f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 375f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson FarType Type() const { return FAR_STLIST; } 376f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 377f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson bool Error() const { return reader_->Error(); } 378f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 379f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson ~STListFarReader() { delete reader_; } 380f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 381f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 382f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson explicit STListFarReader(STListReader<Fst<A>, FstReader<A> > *reader) 383f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson : reader_(reader) {} 384f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 385f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson private: 386f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson STListReader<Fst<A>, FstReader<A> > *reader_; 387f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 388f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson DISALLOW_COPY_AND_ASSIGN(STListFarReader); 389f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson}; 390f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 391dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkintemplate <class A> 392dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkinclass FstFarReader : public FarReader<A> { 393dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin public: 394dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin typedef A Arc; 395dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 396dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin static FstFarReader *Open(const string &filename) { 397dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin vector<string> filenames; 398dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin filenames.push_back(filename); 399dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return new FstFarReader<A>(filenames); 400dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 401dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 402dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin static FstFarReader *Open(const vector<string> &filenames) { 403dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return new FstFarReader<A>(filenames); 404dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 405dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 406dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FstFarReader(const vector<string> &filenames) 407dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin : keys_(filenames), has_stdin_(false), pos_(0), fst_(0), error_(false) { 408dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin sort(keys_.begin(), keys_.end()); 409dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin streams_.resize(keys_.size(), 0); 410dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin for (size_t i = 0; i < keys_.size(); ++i) { 411dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (keys_[i].empty()) { 412dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (!has_stdin_) { 413dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin streams_[i] = &cin; 414dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin //sources_[i] = "stdin"; 415dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin has_stdin_ = true; 416dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } else { 417dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FSTERROR() << "FstFarReader::FstFarReader: stdin should only " 418dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin << "appear once in the input file list."; 419dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin error_ = true; 420dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return; 421dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 422dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } else { 423dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin streams_[i] = new ifstream( 424dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin keys_[i].c_str(), ifstream::in | ifstream::binary); 425dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 426dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 427dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (pos_ >= keys_.size()) return; 428dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ReadFst(); 429dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 430dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 431dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin void Reset() { 432dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (has_stdin_) { 433dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FSTERROR() << "FstFarReader::Reset: operation not supported on stdin"; 434dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin error_ = true; 435dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return; 436dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 437dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin pos_ = 0; 438dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ReadFst(); 439dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 440dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 441dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin bool Find(const string &key) { 442dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (has_stdin_) { 443dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FSTERROR() << "FstFarReader::Find: operation not supported on stdin"; 444dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin error_ = true; 445dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return false; 446dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 447dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin pos_ = 0;//TODO 448dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ReadFst(); 449dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return true; 450dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 451dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 452dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin bool Done() const { return error_ || pos_ >= keys_.size(); } 453dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 454dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin void Next() { 455dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ++pos_; 456dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ReadFst(); 457dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 458dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 459dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin const string &GetKey() const { 460dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return keys_[pos_]; 461dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 462dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 463dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin const Fst<A> &GetFst() const { 464dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return *fst_; 465dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 466dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 467dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FarType Type() const { return FAR_FST; } 468dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 469dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin bool Error() const { return error_; } 470dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 471dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin ~FstFarReader() { 472dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (fst_) delete fst_; 473dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin for (size_t i = 0; i < keys_.size(); ++i) 474dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin delete streams_[i]; 475dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 476dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 477dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin private: 478dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin void ReadFst() { 479dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (fst_) delete fst_; 480dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (pos_ >= keys_.size()) return; 481dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin streams_[pos_]->seekg(0); 482dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin fst_ = Fst<A>::Read(*streams_[pos_], FstReadOptions()); 483dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin if (!fst_) { 484dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin FSTERROR() << "FstFarReader: error reading Fst from: " << keys_[pos_]; 485dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin error_ = true; 486dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 487dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin } 488dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 489dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin private: 490dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin vector<string> keys_; 491dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin vector<istream*> streams_; 492dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin bool has_stdin_; 493dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin size_t pos_; 494dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin mutable Fst<A> *fst_; 495dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin mutable bool error_; 496dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin 497dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin DISALLOW_COPY_AND_ASSIGN(FstFarReader); 498dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin}; 499f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 500f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 501f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian HodsonFarReader<A> *FarReader<A>::Open(const string &filename) { 502f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson if (filename.empty()) 503f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STListFarReader<A>::Open(filename); 504f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson else if (IsSTTable(filename)) 505f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STTableFarReader<A>::Open(filename); 506f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson else if (IsSTList(filename)) 507f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STListFarReader<A>::Open(filename); 508dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin else if (IsFst(filename)) 509dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return FstFarReader<A>::Open(filename); 510f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return 0; 511f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson} 512f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 513f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 514f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodsontemplate <class A> 515f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian HodsonFarReader<A> *FarReader<A>::Open(const vector<string> &filenames) { 516f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson if (!filenames.empty() && filenames[0].empty()) 517f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STListFarReader<A>::Open(filenames); 518f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson else if (!filenames.empty() && IsSTTable(filenames[0])) 519f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STTableFarReader<A>::Open(filenames); 520f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson else if (!filenames.empty() && IsSTList(filenames[0])) 521f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return STListFarReader<A>::Open(filenames); 522dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin else if (!filenames.empty() && IsFst(filenames[0])) 523dfd8b8327b93660601d016cdc6f29f433b45a8d8Alexander Gutkin return FstFarReader<A>::Open(filenames); 524f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson return 0; 525f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson} 526f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 527f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson} // namespace fst 528f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson 529f4c12fce1ee58e670f9c3fce46c40296ba9ee8a2Ian Hodson#endif // FST_EXTENSIONS_FAR_FAR_H__ 530