1/*---------------------------------------------------------------------------*
2 *  srec_context.h  *
3 *                                                                           *
4 *  Copyright 2007, 2008 Nuance Communciations, Inc.                               *
5 *                                                                           *
6 *  Licensed under the Apache License, Version 2.0 (the 'License');          *
7 *  you may not use this file except in compliance with the License.         *
8 *                                                                           *
9 *  You may obtain a copy of the License at                                  *
10 *      http://www.apache.org/licenses/LICENSE-2.0                           *
11 *                                                                           *
12 *  Unless required by applicable law or agreed to in writing, software      *
13 *  distributed under the License is distributed on an 'AS IS' BASIS,        *
14 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
15 *  See the License for the specific language governing permissions and      *
16 *  limitations under the License.                                           *
17 *                                                                           *
18 *---------------------------------------------------------------------------*/
19
20/* this file contains defines needed by the srec search component*/
21
22#ifndef _h_srec_context_
23#define _h_srec_context_
24
25#include "srec_arb.h"
26#include "astar.h"
27
28#include "portable.h"
29
30#include "phashtable.h"
31
32#define CONTEXT_FILE_FORMAT_VERSION1_ID 10001
33#define IMAGE_FORMAT_V1   32432
34#define IMAGE_FORMAT_V2   32439
35#define USE_HMM_BASED_ENROLLMENT 0
36
37/*********************************************************************
38 *                                                                   *
39 * WordMap                                                           *
40 *                                                                   *
41 *********************************************************************/
42
43/* todo: for dynamic vocabs, a word should eventually be replaced
44   not so much "char* word" but "wordblock* head", Jean to merge
45   in that code later. */
46
47typedef struct
48{
49  wordID num_words;
50  wordID num_slots;
51  wordID max_words;
52  wordID num_base_words;      /* before any additions */
53  /* ptr32* words; c55 ?? */
54  char** words;               /* size max_words */
55
56  char* chars;                /* FOUR_BYTE_PTR(char*, chars, dummy1); */
57  asr_int32_t max_chars;
58  char* next_chars;  /* FOUR_BYTE_PTR(char*, next_chars, dummy2); */
59  char* next_base_chars;      /* before any additions */
60  PHashTable *wordIDForWord;
61}
62wordmap;
63
64/*********************************************************************
65 *                                                                   *
66 * FST                                                               *
67 *                                                                   *
68 *********************************************************************/
69
70typedef struct srec_fsm_entry_point_t
71{
72  nodeID node_index;
73  nodeID* node_for_lpcp;        /* size num_lpcps */
74}
75srec_fsm_entry_point;
76
77/**
78 * srec_fsm_exit_point_t holds information about a particular slot within
79 * the fst, so that we don't need to recalculate it each time.
80 */
81typedef struct srec_fsm_exit_point_t
82{
83  nodeID from_node_index;  /* from node, there can be multiple arcs leaving here */
84  arcID arc_index;         /* arc on which the "class" rests */
85  nodeID wbto_node_index;  /* node index after the .wb ilabel */
86}
87srec_fsm_exit_point;
88#define MAX_NUM_SLOTS 12    /* SLOTS */
89#define IMPORTED_RULES_DELIM '.' /* SLOT MARKER */
90
91typedef char FSMnode_info;
92#define NODE_INFO_UNKNOWN 0
93#define NODE_INFO_ENDNODE 1
94#define NODE_INFO_OPTENDN 2
95#define NODE_INFO_REGULAR 3
96#define NODE_INFO_NUMS    4
97
98typedef struct srec_context
99{
100  asr_uint32_t modelid;  /* modelid at compilation time, or 0 for unknown */
101  int grmtyp;            /* GrammarType */
102
103  FSMarc* FSMarc_list;   /* allocation base */
104  arcID num_arcs;        /* number of arcs actually used */
105  arcID FSMarc_list_len; /* number of arcs allocated     */
106  arcID num_base_arcs;   /* number of arcs before additions */
107  arcID FSMarc_freelist; /* head of the free list */
108
109  FSMnode* FSMnode_list;
110  nodeID num_nodes;
111  nodeID FSMnode_list_len;
112  nodeID num_base_nodes;
113  nodeID FSMnode_freelist;
114  FSMnode_info* FSMnode_info_list; /* todo: change this to an ary of 2bit els*/
115
116  costdata wrapup_cost;        /* cost of going from optend nodes to endnode */
117  costdata wtw_average;        /* cost of going from optend nodes to endnode */
118
119  nodeID start_node;
120  nodeID end_node;
121
122  asr_int16_t num_fsm_exit_points;  /* one per rule import */
123  srec_fsm_exit_point fsm_exit_points[MAX_NUM_SLOTS];
124  /* caching for add word, because FST_AddWordToSlot() is often sequentially
125	 on the same slot */
126  wordID addWordCaching_lastslot_num;
127  LCHAR* addWordCaching_lastslot_name;
128  ESR_BOOL addWordCaching_lastslot_needs_post_silence;
129  wordID addWordCaching_lastslot_ifsm_exit_point;
130
131  wordID beg_silence_word;
132  wordID end_silence_word;
133  wordID hack_silence_word;
134
135  /* aux */
136  wordmap *ilabels;           /* input arc labels */
137  wordmap *olabels;           /* word labels */
138  srec_arbdata *allotree;          /* for addword, knows hmm to state conversion */
139
140  /* word graph, for a-star */
141  arc_token* arc_token_list;
142  arcID arc_token_list_len;
143  arc_token* arc_token_freelist;
144  arc_token* arc_token_insert_start;
145
146  /* search capabilities, return error if adding words beyond this! */
147  nodeID max_searchable_nodes;
148  arcID max_searchable_arcs;
149
150  /* these are pointers to data owned by others, made part of this
151     structure for completeness of information needed by an active
152     search, for banked memory models, we may want these to be a "copy" */
153  asr_int16_t hmm_ilabel_offset;        /* offset for ilabels to hmm */
154  HMMInfo* hmm_info_for_ilabel;   /* ilabel to state conversion */
155  featdata* _unused_avg_state_durations;  /* average durations */
156
157  /* says whether a grammar has been prepared FST_Prepare()
158     a Grammar must be prepared before it is used in a recognition */
159  asr_int16_t whether_prepared;
160}
161srec_context;
162
163
164/*********************************************************************
165 *                                                                   *
166 * Functions                                                         *
167 *                                                                   *
168 *********************************************************************/
169
170#define FST_SUCCESS_ON_OLD_WORD 2
171#define FST_CONTINUE   1
172#define FST_SUCCESS 0
173#define FST_FAILED_ON_INVALID_ARGS -2
174#define FST_FAILED_ON_MEMORY -3
175#define FST_FAILED_ON_HOMONYM -4
176#define FST_FAILED_ON_HOMOGRAPH -5
177#define FST_FAILED_INTERNAL -6
178/* #define FST_NEWWORD    2 // implies success */
179
180#define NUM_ITEMLIST_HDRWDS    4
181enum GrammarType { GrammarTypeUnknown = 0, GrammarTypeBNF = 1, GrammarTypeItemList = 2 };
182
183#ifdef __cplusplus
184extern "C"
185{
186#endif
187
188  /* FST type functions */
189  int FST_AttachArbdata(srec_context* fst, srec_arbdata* allophone_tree);
190  int FST_DumpGraph(srec_context* fst, PFile* fp);
191  int FST_DumpWordMap(PFile* fp, wordmap* wmap);
192  int FST_DumpReverseWordGraph(srec_context* context, PFile* fp);
193
194  int FST_AddWordToGrammar(srec_context* fst,
195                           const char* slot,
196                           const char* word,
197                           const char* pron, const int cost);
198  int FST_ResetGrammar(srec_context* fst);
199
200  int FST_PrepareContext(srec_context* fst);
201  int FST_IsVoiceEnrollment(srec_context* context);
202  int FST_LoadContext(const char* synbase, srec_context** pcontext, int num_words_to_add);
203  void FST_UnloadContext(srec_context* context);
204
205  int FST_LoadWordMap(wordmap** pwmap, int num_words_to_add, PFile* fp);
206  int FST_UnloadWordMap(wordmap** pwmap);
207  int FST_LoadGraph(srec_context* pfst, wordmap* imap, wordmap* omap,
208                    int num_words_to_add, PFile* fp);
209  int FST_UnloadGraph(srec_context* pfst);
210
211#if defined(DO_ALLOW_V1_G2G_FILES)
212  int FST_DumpContextAsImageV1(srec_context* context, PFile* fp);
213#endif
214  int FST_DumpContextAsImageV2(srec_context* context, PFile* fp);
215  int FST_LoadContextFromImage(srec_context** pcontext, PFile* fp);
216
217  int FST_CheckPath(srec_context* context, const char* transcription,
218                    char* literal, size_t max_literal_len);
219#define FST_GetNodeInfo(cn,nd) (cn->FSMnode_info_list[nd])
220
221  /* wordmap functions */
222  int wordmap_whether_in_rule(wordmap* wmap, wordID word, wordID rule);
223  wordID wordmap_find_index(wordmap* wmap, const char* word);
224  wordID wordmap_find_index_in_rule(wordmap* wmap, const char* word, wordID rule);
225  wordID wordmap_find_rule_index(wordmap* wmap, const char* rule);
226  int wordmap_create(wordmap** pwmap, int num_chars, int num_words, int num_words_to_add);
227  int wordmap_destroy(wordmap** pwmap);
228  wordID wordmap_add_word(wordmap* wmap, const char* word);
229  void wordmap_reset(wordmap* wmap);
230  void wordmap_setbase(wordmap* wmap);
231  void wordmap_ceiling(wordmap* wmap);
232  wordID wordmap_add_word_in_rule(wordmap* wmap, const char* word, wordID rule);
233
234  /* utils */
235  asr_int32_t atoi_with_check(const char* buf, asr_int32_t mymax);
236  arc_token_lnk get_first_arc_leaving_node(arc_token* arc_token_list, arcID num_arcs, nodeID node);
237  ESR_ReturnCode deserializeWordMapV2(wordmap **pwordmap, PFile* fp);
238  ESR_ReturnCode serializeWordMapV2(wordmap *wordmap, PFile* fp);
239  int FST_GetGrammarType(srec_context* context);
240
241#ifdef __cplusplus
242}
243#endif
244
245
246#endif
247