fst.cpp revision 31124d583cfcd1ea9407856747c178d0125c1070
15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// fst.cc 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Licensed under the Apache License, Version 2.0 (the "License"); 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// you may not use this file except in compliance with the License. 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// You may obtain a copy of the License at 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// http://www.apache.org/licenses/LICENSE-2.0 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Unless required by applicable law or agreed to in writing, software 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// distributed under the License is distributed on an "AS IS" BASIS, 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// See the License for the specific language governing permissions and 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// limitations under the License. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// \file 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// FST definitions. 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fst/lib/fst.h" 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Include these so they are registered 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fst/lib/const-fst.h" 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fst/lib/vector-fst.h" 2490dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// FST flag definitions 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_bool(fst_verify_properties, false, 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Verify fst properties queried by TestProperties"); 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_string(fst_product_separator, ",", 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Character separator between printed weights" 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) " in a product semiring"); 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_bool(fst_default_cache_gc, true, "Enable garbage collection of cache"); 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_int64(fst_default_cache_gc_limit, 1<<20LL, 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Cache byte size that triggers garbage collection"); 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace fst { 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Register VectorFst and ConstFst for common arcs types 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(VectorFst, StdArc); 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(VectorFst, LogArc); 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(ConstFst, StdArc); 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(ConstFst, LogArc); 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Identifies stream data as an FST (and its endianity) 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const int32 kFstMagicNumber = 2125659606; 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Check Fst magic number and read in Fst header. 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool FstHeader::Read(istream &strm, const string &source) { 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int32 magic_number = 0; 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReadType(strm, &magic_number); 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (magic_number != kFstMagicNumber) { 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LOG(ERROR) << "FstHeader::Read: Bad FST header: " << source; 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReadType(strm, &fsttype_); 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReadType(strm, &arctype_); 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReadType(strm, &version_); 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReadType(strm, &flags_); 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ReadType(strm, &properties_); 6590dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) ReadType(strm, &start_); 6690dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) ReadType(strm, &numstates_); 6790dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) ReadType(strm, &numarcs_); 6890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) if (!strm) { 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LOG(ERROR) << "FstHeader::Read: read failed: " << source; 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return false; 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Write Fst magic number and Fst header. 7690dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)bool FstHeader::Write(ostream &strm, const string &source) const { 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, kFstMagicNumber); 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, fsttype_); 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, arctype_); 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, version_); 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, flags_); 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, properties_); 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, start_); 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, numstates_); 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) WriteType(strm, numarcs_); 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (!strm) { 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LOG(ERROR) << "FstHeader::Write: write failed: " << source; 8890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) return false; 8990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) } 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return true; 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)