fst.cpp revision 31124d583cfcd1ea9407856747c178d0125c1070
15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// fst.cc
25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Licensed under the Apache License, Version 2.0 (the "License");
45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// you may not use this file except in compliance with the License.
55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// You may obtain a copy of the License at
65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//      http://www.apache.org/licenses/LICENSE-2.0
85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Unless required by applicable law or agreed to in writing, software
105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// distributed under the License is distributed on an "AS IS" BASIS,
115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// See the License for the specific language governing permissions and
135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// limitations under the License.
145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)//
165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// \file
175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// FST definitions.
185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fst/lib/fst.h"
205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Include these so they are registered
225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fst/lib/const-fst.h"
235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fst/lib/vector-fst.h"
2490dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)
255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// FST flag definitions
265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_bool(fst_verify_properties, false,
285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)            "Verify fst properties queried by TestProperties");
295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_string(fst_product_separator, ",",
315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)              "Character separator between printed weights"
325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)              " in a product semiring");
335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_bool(fst_default_cache_gc, true, "Enable garbage collection of cache");
355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFINE_int64(fst_default_cache_gc_limit, 1<<20LL,
375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)             "Cache byte size that triggers garbage collection");
385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace fst {
405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Register VectorFst and ConstFst for common arcs types
425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(VectorFst, StdArc);
445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(VectorFst, LogArc);
455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(ConstFst, StdArc);
465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)REGISTER_FST(ConstFst, LogArc);
475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Identifies stream data as an FST (and its endianity)
495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const int32 kFstMagicNumber = 2125659606;
505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Check Fst magic number and read in Fst header.
525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)bool FstHeader::Read(istream &strm, const string &source) {
535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  int32 magic_number = 0;
545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ReadType(strm, &magic_number);
555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (magic_number != kFstMagicNumber) {
565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    LOG(ERROR) << "FstHeader::Read: Bad FST header: " << source;
575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return false;
585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ReadType(strm, &fsttype_);
615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ReadType(strm, &arctype_);
625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ReadType(strm, &version_);
635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ReadType(strm, &flags_);
645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  ReadType(strm, &properties_);
6590dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  ReadType(strm, &start_);
6690dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  ReadType(strm, &numstates_);
6790dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  ReadType(strm, &numarcs_);
6890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  if (!strm) {
695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    LOG(ERROR) << "FstHeader::Read: read failed: " << source;
705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    return false;
715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  }
725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return true;
735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Write Fst magic number and Fst header.
7690dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)bool FstHeader::Write(ostream &strm, const string &source) const {
775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, kFstMagicNumber);
785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, fsttype_);
795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, arctype_);
805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, version_);
815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, flags_);
825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, properties_);
835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, start_);
845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, numstates_);
855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  WriteType(strm, numarcs_);
865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if (!strm) {
875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    LOG(ERROR) << "FstHeader::Write: write failed: " << source;
8890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)    return false;
8990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)  }
905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return true;
915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}
945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)