1/*
2 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16/**
17 * @file picopr.c
18 *
19 * text preprocessor
20 *
21 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
22 * All rights reserved.
23 *
24 * History:
25 * - 2009-04-20 -- initial version
26 *
27 */
28
29#include "picodefs.h"
30#include "picoos.h"
31#include "picobase.h"
32#include "picodbg.h"
33#include "picodata.h"
34#include "picokpr.h"
35#include "picopr.h"
36#include "picoktab.h"
37
38#ifdef __cplusplus
39extern "C" {
40#endif
41#if 0
42}
43#endif
44
45
46/* *****************************************************************************/
47/* constants */
48/* *****************************************************************************/
49
50#define PR_TRACE_MEM      FALSE
51#define PR_TRACE_MAX_MEM  FALSE
52#define PR_TRACE_PATHCOST TRUE
53
54#define PR_WORK_MEM_SIZE  10000
55#define PR_DYN_MEM_SIZE   7000
56
57#define PR_ENABLED TRUE
58
59#define PR_MAX_NR_ITERATIONS 1000;
60
61#define SPEC_CHAR           "\\/"
62
63#define PICO_ERR_CONTEXT_NOT_FOUND            PICO_ERR_OTHER
64#define PICO_ERR_MAX_PREPROC_PATH_LEN_REACHED PICO_ERR_OTHER
65
66#define IN_BUF_SIZE   255
67#define OUT_BUF_SIZE  IN_BUF_SIZE + 3 * PICODATA_ITEM_HEADSIZE + 3
68
69#define PR_MAX_NR_PREPROC   (1 + PICOKNOW_MAX_NUM_UTPP)
70
71#define PR_MAX_PATH_LEN     130
72#define PR_MAX_DATA_LEN     IN_BUF_SIZE
73#define PR_MAX_DATA_LEN_Z   PR_MAX_DATA_LEN + 1      /* all strings in picopr should use this constant
74                                                        to ensure zero termination */
75#define PR_COST_INIT        100000
76#define PR_COST             10
77#define PR_EOL              '\n'
78
79/* Bit mask constants for token sets with parameters */
80#define PR_TSE_MASK_OUT      (1<<PR_TSEOut)
81#define PR_TSE_MASK_MIN      (1<<PR_TSEMin)
82#define PR_TSE_MASK_MAX      (1<<PR_TSEMax)
83#define PR_TSE_MASK_LEN      (1<<PR_TSELen)
84#define PR_TSE_MASK_VAL      (1<<PR_TSEVal)
85#define PR_TSE_MASK_STR      (1<<PR_TSEStr)
86#define PR_TSE_MASK_HEAD     (1<<PR_TSEHead)
87#define PR_TSE_MASK_MID      (1<<PR_TSEMid)
88#define PR_TSE_MASK_TAIL     (1<<PR_TSETail)
89#define PR_TSE_MASK_PROD     (1<<PR_TSEProd)
90#define PR_TSE_MASK_PRODEXT  (1<<PR_TSEProdExt)
91#define PR_TSE_MASK_VAR      (1<<PR_TSEVar)
92#define PR_TSE_MASK_LEX      (1<<PR_TSELex)
93#define PR_TSE_MASK_COST     (1<<PR_TSECost)
94#define PR_TSE_MASK_ID       (1<<PR_TSEID)
95#define PR_TSE_MASK_DUMMY1   (1<<PR_TSEDummy1)
96#define PR_TSE_MASK_DUMMY2   (1<<PR_TSEDummy2)
97#define PR_TSE_MASK_DUMMY3   (1<<PR_TSEDummy3)
98
99/* Bit mask constants for token sets without parameters */
100#define PR_TSE_MASK_BEGIN      (1<<PR_TSEBegin)
101#define PR_TSE_MASK_END        (1<<PR_TSEEnd)
102#define PR_TSE_MASK_SPACE      (1<<PR_TSESpace)
103#define PR_TSE_MASK_DIGIT      (1<<PR_TSEDigit)
104#define PR_TSE_MASK_LETTER     (1<<PR_TSELetter)
105#define PR_TSE_MASK_CHAR       (1<<PR_TSEChar)
106#define PR_TSE_MASK_SEQ        (1<<PR_TSESeq)
107#define PR_TSE_MASK_CMPR       (1<<PR_TSECmpr)
108#define PR_TSE_MASK_NLZ        (1<<PR_TSENLZ)
109#define PR_TSE_MASK_ROMAN      (1<<PR_TSERoman)
110#define PR_TSE_MASK_CI         (1<<PR_TSECI)
111#define PR_TSE_MASK_CIS        (1<<PR_TSECIS)
112#define PR_TSE_MASK_AUC        (1<<PR_TSEAUC)
113#define PR_TSE_MASK_ALC        (1<<PR_TSEALC)
114#define PR_TSE_MASK_SUC        (1<<PR_TSESUC)
115#define PR_TSE_MASK_ACCEPT     (1<<PR_TSEAccept)
116#define PR_TSE_MASK_NEXT       (1<<PR_TSENext)
117#define PR_TSE_MASK_ALTL       (1<<PR_TSEAltL)
118#define PR_TSE_MASK_ALTR       (1<<PR_TSEAltR)
119
120#define PR_FIRST_TSE_WP PR_TSEOut
121
122#define PR_SMALLER 1
123#define PR_EQUAL   0
124#define PR_LARGER  2
125
126#define PR_SPELL_WITH_SENTENCE_BREAK  -2
127#define PR_SPELL_WITH_PHRASE_BREAK    -1
128#define PR_SPELL                       0
129
130#define PICO_SPEED_MIN           20
131#define PICO_SPEED_MAX          500
132#define PICO_SPEED_DEFAULT      100
133#define PICO_SPEED_FACTOR_MIN   500
134#define PICO_SPEED_FACTOR_MAX  2000
135
136#define PICO_PITCH_MIN           50
137#define PICO_PITCH_MAX          200
138#define PICO_PITCH_DEFAULT      100
139#define PICO_PITCH_FACTOR_MIN   500
140#define PICO_PITCH_FACTOR_MAX  2000
141#define PICO_PITCH_ADD_MIN     -100
142#define PICO_PITCH_ADD_MAX      100
143#define PICO_PITCH_ADD_DEFAULT    0
144
145#define PICO_VOLUME_MIN           0
146#define PICO_VOLUME_MAX         500
147#define PICO_VOLUME_DEFAULT     100
148#define PICO_VOLUME_FACTOR_MIN  500
149#define PICO_VOLUME_FACTOR_MAX 2000
150
151#define PICO_CONTEXT_DEFAULT   "DEFAULT"
152
153#define PICO_PARAGRAPH_PAUSE_DUR 500
154
155
156/* *****************************************************************************/
157/* types */
158/* *****************************************************************************/
159
160typedef enum {PR_OStr, PR_OVar, PR_OItem, PR_OSpell, PR_ORomanToCard, PR_OVal,
161              PR_OLeft, PR_ORight, PR_ORLZ, PR_OIgnore, PR_OPitch, PR_OSpeed,
162              PR_OVolume, PR_OVoice, PR_OContext, PR_OPhonSVOXPA, PR_OPhonSAMPA,
163              PR_OPlay, PR_OUseSig, PR_OGenFile, PR_OAudioEdit, PR_OPara,
164              PR_OSent, PR_OBreak, PR_OMark, PR_OConcat, PR_OLast}  pr_OutType;
165
166typedef enum {PR_TSEBegin, PR_TSEEnd, PR_TSESpace, PR_TSEDigit, PR_TSELetter, PR_TSEChar, PR_TSESeq,
167              PR_TSECmpr, PR_TSENLZ, PR_TSERoman, PR_TSECI, PR_TSECIS, PR_TSEAUC, PR_TSEALC, PR_TSESUC,
168              PR_TSEAccept, PR_TSENext, PR_TSEAltL, PR_TSEAltR}  pr_TokSetEleNP;
169
170typedef enum {PR_TSEOut, PR_TSEMin, PR_TSEMax, PR_TSELen, PR_TSEVal, PR_TSEStr, PR_TSEHead, PR_TSEMid,
171              PR_TSETail, PR_TSEProd, PR_TSEProdExt, PR_TSEVar, PR_TSELex, PR_TSECost, PR_TSEID,
172              PR_TSEDummy1, PR_TSEDummy2, PR_TSEDummy3}  pr_TokSetEleWP;
173
174typedef enum {PR_GSNoPreproc, PR_GS_START, PR_GSContinue, PR_GSNeedToken, PR_GSNotFound, PR_GSFound}  pr_GlobalState;
175
176typedef enum {PR_LSError, PR_LSInit, PR_LSGetToken, PR_LSGetToken2, PR_LSMatch, PR_LSGoBack,
177              PR_LSGetProdToken, PR_LSInProd, PR_LSGetProdContToken, PR_LSInProdCont, PR_LSGetNextToken,
178              PR_LSGetAltToken}  pr_LocalState;
179
180typedef enum {PR_MSNotMatched, PR_MSMatched, PR_MSMatchedContinue, PR_MSMatchedMulti}  pr_MatchState;
181
182typedef struct pr_Prod * pr_ProdList;
183typedef struct pr_Prod {
184    picokpr_Preproc rNetwork;
185    picokpr_ProdArrOffset rProdOfs;
186    pr_ProdList rNext;
187} pr_Prod;
188
189typedef struct pr_Context * pr_ContextList;
190typedef struct pr_Context {
191    picoos_uchar * rContextName;
192    pr_ProdList rProdList;
193    pr_ContextList rNext;
194} pr_Context;
195
196/* *****************************************************************************/
197/* used, but to be checked */
198
199#define MaxNrShortStrParams 2
200#define MaxPhoneLen 14
201#define ShortStrParamLen (2 * MaxPhoneLen)
202typedef picoos_uchar ShortStrParam[ShortStrParamLen];
203
204
205typedef struct pr_ioItem * pr_ioItemPtr;
206typedef struct pr_ioItem {
207    pr_ioItemPtr next;
208    picoos_int32 val;
209    struct picodata_itemhead head;
210    picoos_uchar * strci;
211    picoos_uchar * strcis;
212    picoos_bool alc;
213    picoos_bool auc;
214    picoos_bool suc;
215    picoos_uchar data[PR_MAX_DATA_LEN_Z];
216} pr_ioItem;
217
218typedef struct pr_ioItem2 {
219    pr_ioItemPtr next;
220    picoos_int32 val;
221    struct picodata_itemhead head;
222    picoos_uchar * strci;
223    picoos_uchar * strcis;
224    picoos_bool alc;
225    picoos_bool auc;
226    picoos_bool suc;
227} pr_ioItem2;
228
229#define PR_IOITEM_MIN_SIZE sizeof(pr_ioItem2)
230
231typedef picoos_uint32 pr_MemState;
232typedef enum {pr_DynMem, pr_WorkMem} pr_MemTypes;
233
234/* *****************************************************************************/
235
236typedef struct pr_OutItemVar * pr_OutItemVarPtr;
237struct pr_OutItemVar {
238    pr_ioItemPtr first;
239    pr_ioItemPtr last;
240    picoos_int32 id;
241    pr_OutItemVarPtr next;
242};
243
244
245struct pr_WorkItem {
246    pr_ioItemPtr rit;
247};
248typedef pr_ioItemPtr pr_WorkItems[PR_MAX_PATH_LEN+1];
249
250struct pr_PathEle {
251    picokpr_Preproc rnetwork;
252    picoos_int16 ritemid;
253    picoos_int16 rcompare;
254    picoos_int16 rdepth;
255    picokpr_TokArrOffset rtok;
256    picokpr_StrArrOffset rprodname;
257    picoos_int32 rprodprefcost;
258    pr_LocalState rlState;
259};
260
261typedef struct pr_Path {
262    picoos_int32 rcost;
263    picoos_int32 rlen;
264    struct pr_PathEle rele[PR_MAX_PATH_LEN];
265} pr_Path;
266
267/* *****************************************************************************/
268
269/** subobject : PreprocUnit
270 *  shortcut  : pr
271 */
272typedef struct pr_subobj
273{
274    pr_ioItemPtr rinItemList;
275    pr_ioItemPtr rlastInItem;
276    pr_ioItemPtr routItemList;
277    pr_ioItemPtr rlastOutItem;
278    pr_GlobalState rgState;
279    pr_Path ractpath;
280    pr_Path rbestpath;
281    picoos_int32 rnritems;
282    pr_WorkItems ritems;
283    picoos_int32 rignore;
284    picoos_int32 spellMode;
285    picoos_int32 maxPathLen;
286    picoos_bool insidePhoneme;
287
288    picoos_uint8 inBuf[IN_BUF_SIZE+PICODATA_ITEM_HEADSIZE]; /* internal input buffer */
289    picoos_uint16 inBufLen;
290
291    picoos_uint8 outBuf[OUT_BUF_SIZE]; /* internal output buffer */
292    picoos_uint16 outReadPos; /* next pos to read from outBuf */
293    picoos_uint16 outWritePos; /* next pos to write to outBuf */
294
295    picokpr_Preproc preproc[PR_MAX_NR_PREPROC];
296    pr_ContextList ctxList;
297    pr_ProdList prodList;
298
299    pr_ContextList actCtx;
300    picoos_bool actCtxChanged;
301
302    picoos_uchar tmpStr1[PR_MAX_DATA_LEN_Z];
303    picoos_uchar tmpStr2[PR_MAX_DATA_LEN_Z];
304
305    picoos_uint8 pr_WorkMem[PR_WORK_MEM_SIZE];
306    picoos_uint32 workMemTop;
307    picoos_uint32 maxWorkMemTop;
308    picoos_uint8 pr_DynMem[PR_DYN_MEM_SIZE];
309    picoos_MemoryManager dynMemMM;
310    picoos_int32 dynMemSize;
311    picoos_int32 maxDynMemSize;
312
313    picoos_bool outOfMemory;
314
315    picoos_bool forceOutput;
316    picoos_int16 nrIterations;
317
318    picoos_uchar lspaces[128];
319    picoos_uchar saveFile[IN_BUF_SIZE];
320
321    pr_ioItem tmpItem;
322
323    picotrns_SimpleTransducer transducer;
324
325    /* kbs */
326
327    picoktab_Graphs graphs;
328    picokfst_FST xsampa_parser;
329    picokfst_FST svoxpa_parser;
330    picokfst_FST xsampa2svoxpa_mapper;
331
332} pr_subobj_t;
333
334/* *****************************************************************************/
335/* prototypes */
336
337static void pr_getOutputItemList (picodata_ProcessingUnit this, pr_subobj_t * pr,
338                                  picokpr_Preproc network,
339                                  picokpr_OutItemArrOffset outitem,
340                                  pr_OutItemVarPtr vars,
341                                  pr_ioItemPtr * first, pr_ioItemPtr * last);
342
343/* *****************************************************************************/
344
345#define pr_iABS(X) (((X) < 0) ? (-(X)) : (X))
346
347/* *****************************************************************************/
348/* module internal memory managment for dynamic and working memory using memory
349   partitions allocated with pr_subobj_t.
350   Dynamic memory is allocated in pr_subobj_t->pr_DynMem. Dynamic memory has
351   to be deallocated again with pr_DEALLOCATE.
352   Working memory is allocated in pr_subobj_t->pr_WorkMem. Working memory is stack
353   based and may not to be deallocated with pr_DEALLOCATE, but with pr_resetMemState
354   to a state previously saved with pr_getMemState.
355*/
356
357static void pr_ALLOCATE (picodata_ProcessingUnit this, pr_MemTypes mType, void * * adr, unsigned int byteSize)
358  /* allocates 'byteSize' bytes in the memery partition given by 'mType' */
359{
360    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
361    picoos_int32 incrUsedBytes, prevmaxDynMemSize;
362
363    if (mType == pr_WorkMem) {
364        if ((pr->workMemTop + byteSize) < PR_WORK_MEM_SIZE) {
365            (*adr) = (void *)(&(pr->pr_WorkMem[pr->workMemTop]));
366            byteSize = ((byteSize + PICOOS_ALIGN_SIZE - 1) / PICOOS_ALIGN_SIZE) * PICOOS_ALIGN_SIZE;
367            pr->workMemTop += byteSize;
368#if PR_TRACE_MEM
369            PICODBG_INFO(("pr_WorkMem: +%u, tot:%i of %i", byteSize, pr->workMemTop, PR_WORK_MEM_SIZE));
370#endif
371
372            if (pr->workMemTop > pr->maxWorkMemTop) {
373                pr->maxWorkMemTop = pr->workMemTop;
374#if PR_TRACE_MAX_MEM
375                PICODBG_INFO(("new max pr_WorkMem: %i of %i", pr->workMemTop, PR_WORK_MEM_SIZE));
376#endif
377            }
378        }
379        else {
380            (*adr) = NULL;
381            PICODBG_ERROR(("pr out of working memory"));
382            picoos_emRaiseException(this->common->em, PICO_EXC_OUT_OF_MEM, (picoos_char *)"pr out of dynamic memory", (picoos_char *)"");
383            pr->outOfMemory = TRUE;
384        }
385    }
386    else if (mType == pr_DynMem) {
387        (*adr) = picoos_allocate(pr->dynMemMM, byteSize);
388        if ((*adr) != NULL) {
389            prevmaxDynMemSize = pr->maxDynMemSize;
390            picoos_getMemUsage(pr->dynMemMM, 1, &pr->dynMemSize, &incrUsedBytes, &pr->maxDynMemSize);
391#if PR_TRACE_MEM
392            PICODBG_INFO(("pr_DynMem : +%i, tot:%i of %i", incrUsedBytes, pr->dynMemSize, PR_DYN_MEM_SIZE));
393#endif
394
395#if PR_TRACE_MAX_MEM
396            if (pr->maxDynMemSize > prevmaxDynMemSize) {
397                PICODBG_INFO(("new max pr_DynMem : %i of %i", pr->maxDynMemSize, PR_DYN_MEM_SIZE));
398            }
399#endif
400        }
401        else {
402            PICODBG_ERROR(("pr out of dynamic memory"));
403            picoos_emRaiseException(this->common->em, PICO_EXC_OUT_OF_MEM, (picoos_char *)"pr out of dynamic memory", (picoos_char *)"");
404            pr->outOfMemory = TRUE;
405        }
406    }
407    else {
408        (*adr) = NULL;
409    }
410}
411
412
413static void pr_DEALLOCATE (picodata_ProcessingUnit this, pr_MemTypes mType, void * * adr)
414{
415    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
416    picoos_int32 incrUsedBytes;
417    if (mType == pr_WorkMem) {
418        PICODBG_INFO(("not possible; use pr_resetMemState instead"));
419    }
420    else if (mType == pr_DynMem) {
421        picoos_deallocate(pr->dynMemMM, &(*adr));
422        picoos_getMemUsage(pr->dynMemMM, 1, &pr->dynMemSize, &incrUsedBytes, &pr->maxDynMemSize);
423#if PR_TRACE_MEM
424        PICODBG_INFO(("pr_DynMem : %i, tot:%i of %i: adr: %u", incrUsedBytes, pr->dynMemSize, PR_DYN_MEM_SIZE, *adr));
425#endif
426    }
427    else {
428        (*adr) = NULL;
429    }
430}
431
432
433static void pr_getMemState(picodata_ProcessingUnit this, pr_MemTypes mType, picoos_uint32 *lmemState)
434{
435    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
436    mType = mType;        /* avoid warning "var not used in this function"*/
437    *lmemState = pr->workMemTop;
438}
439
440
441static void pr_resetMemState(picodata_ProcessingUnit this, pr_MemTypes mType, picoos_uint32 lmemState)
442{
443    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
444
445#if PR_TRACE_MEM
446    PICODBG_INFO(("pr_WorkMem: -%i, tot:%i of %i", pr->workMemTop-lmemState, lmemState, PR_WORK_MEM_SIZE));
447#endif
448    mType = mType;        /* avoid warning "var not used in this function"*/
449    pr->workMemTop = lmemState;
450}
451
452
453/* *****************************************************************************/
454/* string operations */
455
456static picoos_int32 pr_strlen(const picoos_uchar * str)
457{
458    picoos_int32 i;
459
460    i=0;
461    while ((i<PR_MAX_DATA_LEN) && (str[i] != 0)) {
462        i++;
463    }
464    return i;
465}
466
467
468static picoos_uint32 pr_strcpy(picoos_uint8 * dest, const picoos_uint8 * src)
469{
470    picoos_int32 i;
471
472    i = 0;
473    while ((i<PR_MAX_DATA_LEN) && (src[i] != 0)) {
474        dest[i] = src[i];
475        i++;
476    }
477    dest[i] = 0;
478    return i;
479}
480
481
482static picoos_uint32 pr_strcat(picoos_uint8 * dest, const picoos_uint8 * src)
483{
484    picoos_int32 i, j;
485
486    i = 0;
487    while ((i<PR_MAX_DATA_LEN) && (dest[i] != 0)) {
488        i++;
489    }
490    j = 0;
491    while ((i<PR_MAX_DATA_LEN) && (j<PR_MAX_DATA_LEN) && (src[j] != 0)) {
492        dest[i] = src[j];
493        i++;
494        j++;
495    }
496    dest[i] = 0;
497    return i;
498}
499
500
501static void pr_getTermPartStr (picoos_uchar string[], picoos_int32 * ind, picoos_uchar termCh, picoos_uchar str[], picoos_bool * done)
502{
503    int j;
504    picoos_bool done1;
505
506    done1 = TRUE;
507    j = 0;
508    while ((*ind < PR_MAX_DATA_LEN) && (string[*ind] != termCh) && (string[*ind] != 0)) {
509        if (j < PR_MAX_DATA_LEN) {
510            str[j] = string[*ind];
511            j++;
512        } else {
513            done1 = FALSE;
514        }
515        (*ind)++;
516    }
517    if (j < PR_MAX_DATA_LEN) {
518        str[j] = 0;
519    }
520    *done = ((*ind < PR_MAX_DATA_LEN) && (string[*ind] == termCh));
521    if (*done) {
522        (*ind)++;
523    }
524    *done = *done && done1;
525}
526
527
528static picoos_int32 pr_removeSubstring (int pos, int len, unsigned char str[])
529{
530    int i;
531    int length;
532
533    length = pr_strlen(str);
534    if (pos >= length) {
535        return length;
536    } else {
537        i = pos + len;
538        while (i < length) {
539            str[pos] = str[i];
540            i++;
541            pos++;
542        }
543        str[pos] = 0;
544        return pos;
545    }
546}
547
548
549static picoos_bool pr_strEqual(picoos_uchar * str1, picoos_uchar * str2)
550{
551   return (picoos_strcmp((picoos_char *)str1, (picoos_char *)str2) == 0);
552}
553
554
555static void pr_int_to_string(picoos_int32 n, picoos_uchar * str, picoos_int32 maxstrlen)
556{
557    picoos_int32 i, len;
558    picoos_bool negative=FALSE;
559
560    len = 0;
561    str[0] = 0;
562    if (n<0) {
563        negative = TRUE;
564        n = -n;
565        len++;
566    }
567    i = n;
568
569    while (i>0) {
570        i = i / 10;
571        len++;
572    }
573
574    if (len<maxstrlen) {
575        str[len] = 0;
576        i = n;
577        while ((i>0) && (len>0)) {
578            len--;
579            str[len] = i % 10 + '0';
580            i = i / 10;
581        }
582        if (negative) {
583            len--;
584            str[len] = '-';
585        }
586    }
587}
588/* *****************************************************************************/
589
590static void pr_firstLetterToLowerCase (const picoos_uchar src[], picoos_uchar dest[])
591{
592
593    picoos_int32 i;
594    picoos_int32 j;
595    picoos_int32 l;
596    picoos_bool done;
597
598    i = 0;
599    j = 0;
600    l = picobase_det_utf8_length(src[0]);
601    while ((i < l) && (j < PR_MAX_DATA_LEN)) {
602        dest[j] = src[i];
603        i++;
604        j++;
605    }
606    if (j < PR_MAX_DATA_LEN) {
607        dest[j] = 0;
608    }
609    picobase_lowercase_utf8_str(dest, (picoos_char*)dest, PR_MAX_DATA_LEN, &done);
610    j = picobase_det_utf8_length(dest[0]);
611    l = pr_strlen(src);
612    while ((i < l) && (j < PR_MAX_DATA_LEN)) {
613        dest[j] = src[i];
614        i++;
615        j++;
616    }
617    dest[j] = 0;
618}
619
620
621static picoos_int32 tok_tokenDigitStrToInt (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_uchar stokenStr[])
622{
623    picoos_uint32 i;
624    picoos_uint32 l;
625    picoos_int32 id;
626    picoos_int32 val;
627    picoos_uint32 n;
628    picobase_utf8char utf8char;
629
630    val = 0;
631    i = 0;
632    l = pr_strlen(stokenStr);
633    while (i < l) {
634        picobase_get_next_utf8char(stokenStr, PR_MAX_DATA_LEN, & i, utf8char);
635        id = picoktab_graphOffset(pr->graphs, utf8char);
636        if (id > 0) {
637          if (picoktab_getIntPropValue(pr->graphs, id, &n)) {
638                val = (10 * val) + n;
639            } else {
640                val = ((10 * val) + (int)((int)utf8char[0] - (int)'0'));
641            }
642        } else if ((utf8char[0] >= '0') && (utf8char[0] <= '9')) {
643            val = 10 * val + ((int)utf8char[0] - (int)'0');
644        }
645    }
646    return val;
647}
648
649
650static picoos_bool pr_isLatinNumber (picoos_uchar str[], picoos_int32 * val)
651{
652
653    picoos_uint32 li;
654    picoos_uint32 llen;
655    picoos_uchar lact;
656    picoos_uchar lnext;
657    picoos_uchar lprev;
658    picoos_uchar llatinI;
659    picoos_uchar llatinV;
660    picoos_uchar llatinX;
661    picoos_uchar llatinL;
662    picoos_uchar llatinC;
663    picoos_uchar llatinD;
664    picoos_uchar llatinM;
665    picoos_int32 lseq;
666    picobase_utf8char utf8;
667
668    *val = 0;
669    llen = picobase_utf8_length(str, PR_MAX_DATA_LEN);
670    if (llen > 0) {
671        li = 0;
672        picobase_get_next_utf8char(str, PR_MAX_DATA_LEN, & li,utf8);
673        if (picobase_is_utf8_uppercase(utf8, PICOBASE_UTF8_MAXLEN)) {
674            llatinI = 'I';
675            llatinV = 'V';
676            llatinX = 'X';
677            llatinL = 'L';
678            llatinC = 'C';
679            llatinD = 'D';
680            llatinM = 'M';
681        } else {
682            llatinI = 'i';
683            llatinV = 'v';
684            llatinX = 'x';
685            llatinL = 'l';
686            llatinC = 'c';
687            llatinD = 'd';
688            llatinM = 'm';
689        }
690        lseq = 1000;
691        li = 0;
692        while (li < llen) {
693            if (li > 0) {
694                lprev = str[li - 1];
695            } else {
696                lprev = 0;
697            }
698            lact = str[li];
699            if (li < (llen - 1)) {
700                lnext = str[li + 1];
701            } else {
702                lnext = 0;
703            }
704            if ((lseq > 1) && (lact == llatinI)) {
705                if ((lprev != lact) && (lseq >= 4)) {
706                    if (lnext == llatinV) {
707                        *val = *val + 4;
708                        li++;
709                        lseq = 1;
710                    } else if (lnext == llatinX) {
711                        *val = *val + 9;
712                        li++;
713                        lseq = 1;
714                    } else {
715                        *val = *val + 1;
716                        lseq = 3;
717                    }
718                } else {
719                    *val = *val + 1;
720                    lseq = lseq - 1;
721                }
722            } else if ((lseq > 5) && (lact == llatinV)) {
723                *val = *val + 5;
724                lseq = 5;
725            } else if ((lseq > 10) && (lact == llatinX)) {
726                if ((lprev != lact) && (lseq >= 40)) {
727                    if (lnext == llatinL) {
728                        *val = *val + 40;
729                        li++;
730                        lseq = 10;
731                    } else if (lnext == llatinC) {
732                        *val = *val + 90;
733                        li++;
734                        lseq = 10;
735                    } else {
736                        *val = *val + 10;
737                        lseq = 30;
738                    }
739                } else {
740                    *val = *val + 10;
741                    lseq = lseq - 10;
742                }
743            } else if ((lseq > 50) && (lact == llatinL)) {
744                *val = *val + 50;
745                lseq = 50;
746            } else if ((lseq > 100) && (lact == llatinC)) {
747                if ((lprev != lact) && (lseq >= 400)) {
748                    if (lnext == llatinD) {
749                        *val = *val + 400;
750                        li++;
751                        lseq = 100;
752                    } else if (lnext == llatinM) {
753                        *val = *val + 900;
754                        li++;
755                        lseq = 100;
756                    } else {
757                        *val = *val + 100;
758                        lseq = 300;
759                    }
760                } else {
761                    *val = *val + 100;
762                    lseq = lseq - 100;
763                }
764            } else if ((lseq > 500) && (lact == llatinD)) {
765                *val = *val + 500;
766                lseq = 500;
767            } else if ((lseq >= 1000) && (lact == llatinM)) {
768                *val = *val + 1000;
769            } else {
770                return FALSE;
771            }
772            li++;
773        }
774    }
775    return TRUE;
776}
777
778
779static picoos_bool pr_isSUC (picoos_uchar str[])
780{
781
782    picoos_int32 li;
783    picoos_bool lis;
784    picobase_utf8char lutf;
785    picoos_int32 lj;
786    picoos_int32 ll;
787    picoos_bool luc;
788
789    li = 0;
790    lis = TRUE;
791    luc = TRUE;
792    while (lis && (li < PR_MAX_DATA_LEN) && (str[li] != 0)) {
793        lj = 0;
794        ll = picobase_det_utf8_length(str[li]);
795        while (lj < ll) {
796            lutf[lj] = str[li];
797            lj++;
798            li++;
799        }
800        lutf[lj] = 0;
801        if (luc) {
802            lis = lis && picobase_is_utf8_uppercase(lutf,PICOBASE_UTF8_MAXLEN+1);
803        } else {
804            lis = lis && picobase_is_utf8_lowercase(lutf,PICOBASE_UTF8_MAXLEN+1);
805        }
806        luc = FALSE;
807    }
808    return lis;
809}
810
811/* *****************************************************************************/
812
813static picoos_bool pr_isCmdType (pr_ioItemPtr it, picoos_uint8 type)
814{
815    if ((it != NULL) && (it->head.type == PICODATA_ITEM_CMD) && (it->head.info1 == type)) {
816        return TRUE;
817    } else {
818        return FALSE;
819    }
820}
821
822
823static picoos_bool pr_isCmdInfo2 (pr_ioItemPtr it, picoos_uint8 info2)
824{
825    if ((it != NULL) && (it->head.type == PICODATA_ITEM_CMD) && (it->head.info2 == info2)) {
826        return TRUE;
827    } else {
828        return FALSE;
829    }
830}
831
832
833static void pr_initPathEle (struct pr_PathEle * ele)
834{
835    ele->rnetwork = NULL;
836    ele->rtok = 0;
837    ele->ritemid =  -1;
838    ele->rdepth = 1;
839    ele->rlState = PR_LSInit;
840    ele->rcompare =  -1;
841    ele->rprodname = 0;
842    ele->rprodprefcost = 0;
843}
844
845/* *****************************************************************************/
846
847static void pr_disposeProdList (register picodata_ProcessingUnit this,  pr_ProdList * prodList)
848{
849    pr_ProdList p;
850
851    while ((*prodList) != NULL) {
852        p = (*prodList);
853        (*prodList) = (*prodList)->rNext;
854        picoos_deallocate(this->common->mm, (void *) &p);
855    }
856}
857
858
859static pico_Status pr_addContext (register picodata_ProcessingUnit this,  pr_subobj_t * pr, pr_ContextList * ctxList, picokpr_VarStrPtr contextNamePtr, picokpr_VarStrPtr netNamePtr, picokpr_VarStrPtr prodNamePtr)
860{
861    picokpr_Preproc net;
862    pr_ContextList ctx;
863    pr_ProdList prod;
864    int i;
865    picokpr_VarStrPtr strp;
866    picoos_int32 lprodarrlen;
867
868    ctx = (*ctxList);
869    while ((ctx != NULL) &&  !(pr_strEqual(contextNamePtr, ctx->rContextName))) {
870        ctx = ctx->rNext;
871    }
872    if (ctx == NULL) {
873        ctx = picoos_allocate(this->common->mm, sizeof(pr_Context));
874        if (ctx == NULL) {
875            return PICO_EXC_OUT_OF_MEM;
876        }
877        ctx->rNext = (*ctxList);
878        ctx->rProdList = NULL;
879        ctx->rContextName = contextNamePtr;
880        (*ctxList) = ctx;
881    }
882    i = 0;
883    net = pr->preproc[i];
884    while ((i<PR_MAX_NR_PREPROC) && (net != NULL) && !(pr_strEqual(netNamePtr, picokpr_getPreprocNetName(net)))) {
885        i++;
886        net = pr->preproc[i];
887    }
888    if (net != NULL) {
889        i = 0;
890        strp = picokpr_getVarStrPtr(net, picokpr_getProdNameOfs(net, i));
891        lprodarrlen = picokpr_getProdArrLen(net);
892        while ((i < lprodarrlen) &&  !(pr_strEqual(prodNamePtr, strp))) {
893            i++;
894            if (i < lprodarrlen) {
895                strp = picokpr_getVarStrPtr(net, picokpr_getProdNameOfs(net, i));
896            }
897        }
898        if (i < lprodarrlen) {
899            prod = picoos_allocate(this->common->mm, sizeof(pr_Prod));
900            if (prod == NULL) {
901              return PICO_EXC_OUT_OF_MEM;
902            }
903            prod->rNetwork = net;
904            prod->rProdOfs = i;
905            prod->rNext = ctx->rProdList;
906            ctx->rProdList = prod;
907        }
908    }
909    return PICO_OK;
910}
911
912
913static pico_Status pr_createContextList (register picodata_ProcessingUnit this)
914{
915    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
916    picokpr_VarStrPtr ctxNamePtr;
917    picokpr_VarStrPtr netNamePtr;
918    picokpr_VarStrPtr prodNamePtr;
919    picoos_int32 p, i, n;
920    pico_Status status;
921
922    pr->ctxList = NULL;
923    for (p=0; p<PR_MAX_NR_PREPROC; p++) {
924        if (pr->preproc[p] != NULL) {
925            n = picokpr_getCtxArrLen(pr->preproc[p]);
926            for (i = 1; i<n; i++) {
927                ctxNamePtr = picokpr_getVarStrPtr(pr->preproc[p], picokpr_getCtxCtxNameOfs(pr->preproc[p], i));
928                netNamePtr = picokpr_getVarStrPtr(pr->preproc[p], picokpr_getCtxNetNameOfs(pr->preproc[p], i));
929                prodNamePtr = picokpr_getVarStrPtr(pr->preproc[p], picokpr_getCtxProdNameOfs(pr->preproc[p], i));
930                status = pr_addContext(this, pr, &pr->ctxList, ctxNamePtr,netNamePtr, prodNamePtr);
931                if (status != PICO_OK) {
932                    return status;
933                }
934            }
935        }
936    }
937    return PICO_OK;
938}
939
940
941static void pr_disposeContextList (register picodata_ProcessingUnit this)
942{
943    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
944    pr_ContextList c;
945
946    while (pr->ctxList != NULL) {
947        c = pr->ctxList;
948        pr->ctxList = pr->ctxList->rNext;
949        pr_disposeProdList(this, & c->rProdList);
950        picoos_deallocate(this->common->mm, (void *) &c);
951    }
952}
953
954
955static pr_ContextList pr_findContext (pr_ContextList contextList, picoos_uchar contextName[])
956{
957    pr_ContextList context;
958
959    context = contextList;
960    while ((context != NULL) &&  !(pr_strEqual(context->rContextName,contextName))) {
961        context = context->rNext;
962    }
963    return context;
964}
965
966
967static void pr_setContext (register picodata_ProcessingUnit this,  pr_subobj_t * pr, picoos_uchar context[])
968{
969
970    pr_ContextList ctx;
971
972    ctx = pr_findContext(pr->ctxList,context);
973    if (ctx != NULL) {
974        pr->actCtx = ctx;
975        pr->actCtxChanged = TRUE;
976    } else {
977        PICODBG_WARN(("context '%s' not found; no change",context));
978        picoos_emRaiseWarning(this->common->em, PICO_ERR_CONTEXT_NOT_FOUND, (picoos_char*)"context '%s' not found; no change",(picoos_char*)context);
979    }
980}
981
982/* *****************************************************************************/
983/* item handling routines */
984
985
986static picoos_uint32 pr_copyData(picoos_uint8 * dest, const picoos_uint8 * src, picoos_int32 nrBytes, picoos_bool zeroTerm)
987{
988    picoos_int32 i=0;
989
990    if ((src != NULL) && (dest != NULL)) {
991        i = 0;
992        while ((i<nrBytes) && (i<PR_MAX_DATA_LEN)) {
993            dest[i] = src[i];
994            i++;
995        }
996        if (zeroTerm) {
997            dest[i] = 0;
998        }
999    }
1000    return i;
1001}
1002
1003
1004static void pr_initItem(picodata_ProcessingUnit this, pr_ioItem * item)
1005{
1006    item->next = NULL;
1007    item->val = 0;
1008    item->head.len = 0;
1009    item->strci = NULL;
1010    item->strcis = NULL;
1011    item->suc = FALSE;
1012    item->alc = FALSE;
1013    item->auc = FALSE;
1014}
1015
1016
1017static void pr_newItem (picodata_ProcessingUnit this, pr_MemTypes mType, pr_ioItemPtr * item, picoos_uint8 itemType, picoos_int32 size, picoos_bool inItem)
1018{
1019    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
1020
1021    if (mType == pr_WorkMem) {
1022        pr_ALLOCATE(this, mType, (void * *) & (*item),PR_IOITEM_MIN_SIZE+size+1);
1023        if (pr->outOfMemory) return;
1024        pr_initItem(this, *item);
1025    }
1026    else if ((mType == pr_DynMem) && inItem) {
1027        pr_ALLOCATE(this, mType, (void * *) & (*item), PR_IOITEM_MIN_SIZE+3*size+3);
1028        if (pr->outOfMemory) return;
1029        pr_initItem(this, *item);
1030        if (itemType == PICODATA_ITEM_TOKEN) {
1031            (*item)->strci = &((*item)->data[size+1]);
1032            (*item)->strcis = &((*item)->data[2*size+2]);
1033            (*item)->strci[0] = 0;
1034            (*item)->strcis[0] = 0;
1035        }
1036    }
1037    else if ((mType == pr_DynMem) && !inItem) {
1038        pr_ALLOCATE(this, mType, (void * *) & (*item), PR_IOITEM_MIN_SIZE+size+1);
1039        if (pr->outOfMemory) return;
1040        pr_initItem(this, *item);
1041    }
1042
1043    (*item)->data[0] = 0;
1044}
1045
1046
1047static void pr_copyItemContent (picodata_ProcessingUnit this, pr_ioItem * inItem, pr_ioItem * outItem)
1048{
1049    if (outItem != NULL) {
1050        outItem->next = inItem->next;
1051        outItem->val = inItem->val;
1052        outItem->head = inItem->head;
1053        outItem->suc = inItem->suc;
1054        outItem->alc = inItem->alc;
1055        outItem->auc = inItem->auc;
1056        if (inItem->head.len > 0 ) {
1057            pr_copyData(outItem->data, inItem->data, inItem->head.len, /*zeroTerm*/TRUE);
1058            pr_copyData(outItem->strci, inItem->strci, inItem->head.len, /*zeroTerm*/TRUE);
1059            pr_copyData(outItem->strcis, inItem->strcis, inItem->head.len, /*zeroTerm*/TRUE);
1060        }
1061    }
1062}
1063
1064
1065static void pr_copyItem (picodata_ProcessingUnit this, pr_MemTypes mType, pr_ioItemPtr inItem, pr_ioItemPtr * outItem)
1066{
1067    pr_subobj_t * pr = (pr_subobj_t *) this->subObj;
1068
1069    if (inItem != NULL) {
1070        pr_newItem(this, mType,& (*outItem), inItem->head.type, inItem->head.len, FALSE);
1071        if (pr->outOfMemory) return;
1072        pr_copyItemContent(this, inItem, *outItem);
1073    }
1074    else {
1075        outItem = NULL;
1076    }
1077}
1078
1079
1080static void pr_startItemList (pr_ioItemPtr * firstItem, pr_ioItemPtr * lastItem)
1081{
1082  *firstItem = NULL;
1083  *lastItem = NULL;
1084}
1085
1086
1087static void pr_appendItem (picodata_ProcessingUnit this, pr_ioItemPtr * firstItem, pr_ioItemPtr * lastItem, pr_ioItemPtr item)
1088{
1089    if (item != NULL) {
1090        item->next = NULL;
1091        if ((*lastItem) == NULL) {
1092            *firstItem = item;
1093        } else {
1094            (*lastItem)->next = item;
1095        }
1096        (*lastItem) = item;
1097    }
1098}
1099
1100
1101static void pr_disposeItem (picodata_ProcessingUnit this, pr_ioItemPtr * item)
1102{
1103    if ((*item) != NULL) {
1104        pr_DEALLOCATE(this, pr_DynMem, (void * *) & (*item));
1105    }
1106}
1107
1108
1109static void pr_putItem (picodata_ProcessingUnit this,  pr_subobj_t * pr,
1110                        pr_ioItemPtr * first, pr_ioItemPtr * last,
1111                        picoos_uint8 itemType, picoos_uint8 info1, picoos_uint8 info2,
1112                        picoos_uint16 val,
1113                        picoos_uchar str[])
1114{
1115    picoos_int32 i;
1116    pr_ioItemPtr item;
1117
1118    pr->tmpItem.next = NULL;
1119    pr->tmpItem.val = 0;
1120    pr->tmpItem.head.type = itemType;
1121    pr->tmpItem.head.info1 = info1;
1122    pr->tmpItem.head.info2 = info2;
1123
1124    pr_initItem(this, &pr->tmpItem);
1125    switch (itemType) {
1126    case PICODATA_ITEM_CMD:
1127        switch (info1) {
1128        case PICODATA_ITEMINFO1_CMD_CONTEXT:
1129        case PICODATA_ITEMINFO1_CMD_VOICE:
1130        case PICODATA_ITEMINFO1_CMD_MARKER:
1131        case PICODATA_ITEMINFO1_CMD_PLAY:
1132        case PICODATA_ITEMINFO1_CMD_SAVE:
1133        case PICODATA_ITEMINFO1_CMD_UNSAVE:
1134        case PICODATA_ITEMINFO1_CMD_PROSDOMAIN:
1135            pr->tmpItem.head.len = picoos_strlen((picoos_char*)str);
1136            for (i=0; i<pr->tmpItem.head.len; i++) {
1137                pr->tmpItem.data[i] = str[i];
1138            }
1139            pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
1140            if (pr->outOfMemory) return;
1141            pr_appendItem(this, & (*first),& (*last),item);
1142            break;
1143        case PICODATA_ITEMINFO1_CMD_IGNSIG:
1144        case PICODATA_ITEMINFO1_CMD_IGNORE:
1145        case PICODATA_ITEMINFO1_CMD_FLUSH:
1146            pr->tmpItem.head.len = 0;
1147            pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
1148            if (pr->outOfMemory) return;
1149            pr_appendItem(this, & (*first),& (*last),item);
1150            break;
1151        case PICODATA_ITEMINFO1_CMD_SPEED:
1152        case PICODATA_ITEMINFO1_CMD_PITCH:
1153        case PICODATA_ITEMINFO1_CMD_VOLUME:
1154        case PICODATA_ITEMINFO1_CMD_SPELL:
1155        case PICODATA_ITEMINFO1_CMD_SIL:
1156            pr->tmpItem.head.len = 2;
1157            pr->tmpItem.data[0] = val % 256;
1158            pr->tmpItem.data[1] = val / 256;
1159            pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
1160            if (pr->outOfMemory) return;
1161            pr_appendItem(this, & (*first),& (*last),item);
1162           break;
1163        case PICODATA_ITEMINFO1_CMD_PHONEME:
1164            PICODBG_WARN(("phoneme command not yet implemented"));
1165            break;
1166        default:
1167            PICODBG_WARN(("pr_putItem: unknown command type"));
1168        }
1169        break;
1170    case PICODATA_ITEM_TOKEN:
1171        pr->tmpItem.head.len = picoos_strlen((picoos_char*)str);
1172        for (i=0; i<pr->tmpItem.head.len; i++) {
1173            pr->tmpItem.data[i] = str[i];
1174        }
1175        pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & item);
1176        if (pr->outOfMemory) return;
1177        pr_appendItem(this, & (*first),& (*last),item);
1178        break;
1179     default:
1180         PICODBG_WARN(("pr_putItem: unknown item type"));
1181    }
1182}
1183
1184
1185static void pr_appendItemToOutItemList (picodata_ProcessingUnit this, pr_subobj_t * pr,
1186                                        pr_ioItemPtr * firstItem, pr_ioItemPtr * lastItem, pr_ioItemPtr item)
1187{
1188    pr_ioItemPtr litem;
1189    picoos_int32 li;
1190    picoos_int32 li2;
1191    picoos_int32 lid;
1192    picoos_int32 ln;
1193    picoos_int32 ln2;
1194    picoos_uint8 ltype;
1195    picoos_int8 lsubtype;
1196    picoos_uchar lstr[10];
1197    picoos_bool ldone;
1198
1199    item->next = NULL;
1200    if (((pr->spellMode != 0) && (item->head.type == PICODATA_ITEM_TOKEN) && (item->head.info1 != PICODATA_ITEMINFO1_TOKTYPE_SPACE))) {
1201        li = 0;
1202        ln = pr_strlen(item->data);
1203        while (li < ln) {
1204            ln2 = picobase_det_utf8_length(item->data[li]);
1205            for (li2 = 0; li2<ln2; li2++) {
1206                lstr[li2] = item->data[li];
1207                li++;
1208            }
1209            lstr[ln2] = 0;
1210            lid = picoktab_graphOffset(pr->graphs, lstr);
1211            if ((lid > 0) && picoktab_getIntPropTokenType(pr->graphs, lid, &ltype) &&
1212                ((ltype == PICODATA_ITEMINFO1_TOKTYPE_LETTERV) /*|| (ltype == PICODATA_ITEMINFO1_TOKTYPE_DIGIT)*/)) {
1213                ln2 = pr_strcat(lstr,(picoos_uchar*)SPEC_CHAR);
1214                picoktab_getIntPropTokenSubType(pr->graphs,lid, &lsubtype);
1215            }
1216            else {
1217                ltype = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
1218                lsubtype =  -(1);
1219            }
1220            pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_TOKEN, ln2, /*inItem*/FALSE);
1221            if (pr->outOfMemory) return;
1222            litem->head.type = PICODATA_ITEM_TOKEN;
1223            litem->head.info1 = item->head.info1;
1224            litem->head.info2 = item->head.info2;
1225            pr_strcpy(litem->data, lstr);
1226            litem->data[ln2] = 0;
1227            litem->head.len = ln2;
1228            if (litem->head.info1 == PICODATA_ITEMINFO1_TOKTYPE_DIGIT) {
1229                litem->val = tok_tokenDigitStrToInt(this, pr, litem->data);
1230            } else {
1231                litem->val = 0;
1232            }
1233            picobase_lowercase_utf8_str(litem->data,litem->strci,PR_MAX_DATA_LEN, &ldone);
1234            pr_firstLetterToLowerCase(litem->data,litem->strcis);
1235            litem->alc = picobase_is_utf8_lowercase(litem->data,PR_MAX_DATA_LEN);
1236            litem->auc = picobase_is_utf8_uppercase(litem->data,PR_MAX_DATA_LEN);
1237            litem->suc = pr_isSUC(litem->data);
1238
1239            pr_appendItem(this, firstItem, lastItem, litem);
1240            if (pr->spellMode == PR_SPELL_WITH_SENTENCE_BREAK) {
1241                pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_TOKEN, 2, /*inItem*/FALSE);
1242                if (pr->outOfMemory) return;
1243                litem->head.type = PICODATA_ITEM_TOKEN;
1244                litem->head.info1 = PICODATA_ITEMINFO1_TOKTYPE_CHAR;
1245                litem->head.info2 =  -1;
1246                litem->head.len =  1;
1247                litem->data[0] = ',';
1248                litem->data[1] = 0;
1249                litem->strci[0] = ',';
1250                litem->strci[1] = 0;
1251                litem->strcis[0] = ',';
1252                litem->strcis[1] = 0;
1253                litem->val = 0;
1254                pr_appendItem(this, firstItem, lastItem, litem);
1255            } else if (pr->spellMode == PR_SPELL_WITH_SENTENCE_BREAK) {
1256                pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_CMD, 0, /*inItem*/FALSE);
1257                if (pr->outOfMemory) return;
1258                litem->head.type = PICODATA_ITEM_CMD;
1259                litem->head.info1 = PICODATA_ITEMINFO1_CMD_FLUSH;
1260                litem->head.info2 = PICODATA_ITEMINFO2_NA;
1261                litem->head.len = 0;
1262                pr_appendItem(this, firstItem, lastItem,litem);
1263            } else if (pr->spellMode > 0) {
1264                pr_newItem(this, pr_DynMem,& litem, PICODATA_ITEM_CMD, 2, /*inItem*/FALSE);
1265                if (pr->outOfMemory) return;
1266                litem->head.type = PICODATA_ITEM_CMD;
1267                litem->head.info1 = PICODATA_ITEMINFO1_CMD_SIL;
1268                litem->head.info2 = PICODATA_ITEMINFO2_NA;
1269                litem->head.len = 2;
1270                litem->data[0] = pr->spellMode % 256;
1271                litem->data[1] = pr->spellMode / 256;
1272                pr_appendItem(this, firstItem, lastItem, litem);
1273            }
1274        }
1275        pr_disposeItem(this, & item);
1276    } else if (pr_isCmdType(item, PICODATA_ITEMINFO1_CMD_SPELL) && pr_isCmdInfo2(item, PICODATA_ITEMINFO2_CMD_START)) {
1277        pr->spellMode = item->data[0]+256*item->data[1];
1278        pr_disposeItem(this, & item);
1279    } else if (pr_isCmdType(item, PICODATA_ITEMINFO1_CMD_SPELL) && pr_isCmdInfo2(item, PICODATA_ITEMINFO2_CMD_END)) {
1280        pr->spellMode = 0;
1281        pr_disposeItem(this, & item);
1282    } else {
1283        pr_appendItem(this, firstItem,lastItem,item);
1284    }
1285}
1286
1287
1288/* *****************************************************************************/
1289
1290static pr_OutItemVarPtr pr_findVariable (pr_OutItemVarPtr vars, picoos_int32 id)
1291{
1292    while ((vars != NULL) && (vars->id != id)) {
1293        vars = vars->next;
1294    }
1295    if ((vars != NULL)) {
1296        return vars;
1297    } else {
1298        return NULL;
1299    }
1300}
1301
1302
1303static void pr_genCommands (picodata_ProcessingUnit this, pr_subobj_t * pr,
1304                            picokpr_Preproc network, picokpr_OutItemArrOffset outitem, pr_OutItemVarPtr vars, pr_ioItemPtr * first, pr_ioItemPtr * last)
1305{
1306
1307    pr_ioItemPtr litem;
1308    pr_OutItemVarPtr lvar;
1309    picoos_uint8 lcmd;
1310    picoos_uint8 linfo2;
1311    picoos_bool ldone;
1312#if 0
1313    picoos_int32 lphontype;
1314#endif
1315    picokpr_VarStrPtr lstrp;
1316    picoos_int32 lnum;
1317    pr_ioItemPtr lf;
1318    pr_ioItemPtr ll;
1319    picoos_int32 lf0beg;
1320    picoos_int32 lf0end;
1321    ShortStrParam lxfadebeg;
1322    ShortStrParam lxfadeend;
1323    picoos_bool lout;
1324    picoos_int32 ltype;
1325    picoos_int32 argOfs;
1326    picoos_int32 nextOfs;
1327    picoos_int32 nextOfs2;
1328    picoos_int32 nextOfs3;
1329    picoos_int32 nextOfs4;
1330    picoos_uchar alphabet[32];
1331
1332    lcmd = 0;
1333    lnum = 0;
1334    litem = NULL;
1335    ltype = picokpr_getOutItemType(network, outitem);
1336    switch (ltype) {
1337        case PR_OIgnore:
1338            if (picokpr_getOutItemVal(network, outitem) == 0) {
1339                pr_putItem(this, pr, & (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNORE,PICODATA_ITEMINFO2_CMD_START,0,(picoos_uchar*)"");
1340            } else {
1341                pr_putItem(this, pr, & (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNORE,PICODATA_ITEMINFO2_CMD_END,0,(picoos_uchar*)"");
1342            }
1343            break;
1344        case PR_OPitch:   case PR_OSpeed:   case PR_OVolume:
1345            switch (ltype) {
1346                case PR_OPitch:
1347                    lcmd = PICODATA_ITEMINFO1_CMD_PITCH;
1348                    lnum = PICO_PITCH_DEFAULT;
1349                    break;
1350                case PR_OSpeed:
1351                    lcmd = PICODATA_ITEMINFO1_CMD_SPEED;
1352                    lnum = PICO_SPEED_DEFAULT;
1353                    break;
1354                case PR_OVolume:
1355                    lcmd = PICODATA_ITEMINFO1_CMD_VOLUME;
1356                    lnum = PICO_VOLUME_DEFAULT;
1357                    break;
1358            default:
1359                break;
1360            }
1361            if ((picokpr_getOutItemArgOfs(network, outitem) != 0)) {
1362                switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
1363                    case PR_OVal:
1364                        pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE, picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)),(picoos_uchar*)"");
1365                        break;
1366                    case PR_OVar:
1367                        lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
1368                        if ((lvar != NULL) && (lvar->first != NULL) && (lvar->first->head.type == PICODATA_ITEM_TOKEN)) {
1369                            pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE,picoos_atoi((picoos_char*)lvar->first->data),(picoos_uchar*)"");
1370                        }
1371                        break;
1372                default:
1373                    pr_startItemList(& lf,& ll);
1374                    pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
1375                    if (pr->outOfMemory) return;
1376                    if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
1377                        pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE,picoos_atoi((picoos_char*)lf->data),(picoos_uchar*)"");
1378                    }
1379                    break;
1380                }
1381            } else {
1382                pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,PICODATA_ITEMINFO2_CMD_ABSOLUTE,lnum,(picoos_uchar*)"");
1383            }
1384            break;
1385
1386        case PR_OPhonSVOXPA:   case PR_OPhonSAMPA:
1387            if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
1388                if (ltype == PR_OPhonSVOXPA) {
1389                    picoos_strlcpy(alphabet, PICODATA_SVOXPA, sizeof(alphabet));
1390                }
1391                else {
1392                    picoos_strlcpy(alphabet, PICODATA_SAMPA, sizeof(alphabet));
1393                }
1394                pr_startItemList(& lf,& ll);
1395                pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
1396                if (pr->outOfMemory) return;
1397                if (lf != NULL) {
1398                    ldone = FALSE;
1399                    if (lf->head.type == PICODATA_ITEM_TOKEN) {
1400                        if (picodata_mapPAStrToPAIds(pr->transducer, this->common, pr->xsampa_parser, pr->svoxpa_parser, pr->xsampa2svoxpa_mapper, lf->data, alphabet, pr->tmpStr1, sizeof(pr->tmpStr1)-1) == PICO_OK) {
1401                            pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME,
1402                                PICODATA_ITEMINFO2_CMD_START, 0, pr->tmpStr1);
1403                            ldone = TRUE;
1404                        }
1405                        else {
1406                            PICODBG_WARN(("cannot map phonetic string '%s'; synthesizeing text instead", lf->data));
1407                            picoos_emRaiseWarning(this->common->em, PICO_ERR_OTHER,(picoos_char*)"", (picoos_char*)"cannot map phonetic string '%s'; synthesizing text instead", lf->data);
1408                        }
1409                    }
1410                    if (ldone) {
1411                        lf = lf->next;
1412                        while (lf != NULL) {
1413                            if (lf->head.type == PICODATA_ITEM_TOKEN) {
1414                                pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_TOKEN, PICODATA_ITEMINFO1_CMD_PHONEME,
1415                                               PICODATA_ITEMINFO2_CMD_END, 0, (picoos_char *)"");
1416                            }
1417                            lf = lf->next;
1418                        }
1419                        pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME,
1420                            PICODATA_ITEMINFO2_CMD_END, 0, (picoos_char *)"");
1421                    }
1422                }
1423            }
1424            break;
1425
1426        case PR_OSent:
1427            pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
1428            break;
1429        case PR_OPara:
1430            pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
1431            if (picokpr_getOutItemVal(network, outitem) == 1) {
1432                pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, PICO_PARAGRAPH_PAUSE_DUR, (picoos_uchar*)"");
1433            }
1434            break;
1435        case PR_OBreak:
1436            if ((picokpr_getOutItemArgOfs(network, outitem) != 0)) {
1437                switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
1438                    case PR_OVal:
1439                        pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)), (picoos_uchar*)"");
1440                        break;
1441                    case PR_OVar:
1442                        lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
1443                        if ((lvar != NULL) && (lvar->first != NULL) && (lvar->first->head.type == PICODATA_ITEM_TOKEN)) {
1444                            pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, picoos_atoi((picoos_char*)lvar->first->data), (picoos_uchar*)"");
1445                        }
1446                        break;
1447                default:
1448                    pr_startItemList(& lf,& ll);
1449                    pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
1450                    if (pr->outOfMemory) return;
1451                    if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
1452                        pr_putItem(this, pr, & (*first),& (*last), PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, picoos_atoi((picoos_char*)lf->data), (picoos_uchar*)"");
1453                    }
1454                    break;
1455                }
1456            }
1457            break;
1458        case PR_OVoice:   case PR_OContext:   case PR_OMark:
1459            if (picokpr_getOutItemType(network, outitem) == PR_OVoice) {
1460                lcmd = PICODATA_ITEMINFO1_CMD_VOICE;
1461                pr->tmpStr1[0] = 0;
1462                lnum = 1;
1463            } else if (picokpr_getOutItemType(network, outitem) == PR_OContext) {
1464                lcmd = PICODATA_ITEMINFO1_CMD_CONTEXT;
1465                pr_strcpy(pr->tmpStr1, (picoos_uchar*)PICO_CONTEXT_DEFAULT);
1466                lnum = 1;
1467            } else if ((picokpr_getOutItemType(network, outitem) == PR_OMark)) {
1468                lcmd = PICODATA_ITEMINFO1_CMD_MARKER;
1469                pr->tmpStr1[0] = 0;
1470                lnum = 0;
1471            }
1472            if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
1473                switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
1474                    case PR_OVar:
1475                        lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
1476                        if (lvar != NULL) {
1477                            litem = lvar->first;
1478                        }
1479                        pr->tmpStr1[0] = 0;
1480                        while (litem != NULL) {
1481                            if (litem->head.type == PICODATA_ITEM_TOKEN) {
1482                                pr_strcat(pr->tmpStr1,litem->data);
1483                            }
1484                            litem = litem->next;
1485                        }
1486                        pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,pr->tmpStr1);
1487                        break;
1488                    case PR_OStr:
1489                        if (picokpr_getOutItemStrOfs(network, picokpr_getOutItemArgOfs(network, outitem)) != 0) {
1490                            lstrp = picokpr_getOutItemStr(network, picokpr_getOutItemArgOfs(network, outitem));
1491                            pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,lstrp);
1492                        }
1493                        break;
1494                default:
1495                    pr_startItemList(& lf,& ll);
1496                    pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
1497                    if (pr->outOfMemory) return;
1498                    if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
1499                        pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,lf->data);
1500                    }
1501                    break;
1502                }
1503            } else {
1504                pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,pr->tmpStr1);
1505            }
1506            break;
1507        case PR_OGenFile:
1508            if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
1509                lcmd = PICODATA_ITEMINFO1_CMD_SAVE;
1510            } else {
1511                lcmd = PICODATA_ITEMINFO1_CMD_UNSAVE;
1512            }
1513            pr->tmpStr1[0] = 0;
1514            lnum = 0;
1515            if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
1516                switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
1517                    case PR_OVar:
1518                        lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
1519                        if (lvar != NULL) {
1520                            litem = lvar->first;
1521                        }
1522                        pr->tmpStr1[0] = 0;
1523                        while (litem != NULL) {
1524                            if (litem->head.type == PICODATA_ITEM_TOKEN) {
1525                                pr_strcat(pr->tmpStr1,litem->data);
1526                            }
1527                            litem = litem->next;
1528                        }
1529                        if ((lnum = picodata_getPuTypeFromExtension(pr->tmpStr1, /*input*/FALSE)) != PICODATA_ITEMINFO2_CMD_TO_UNKNOWN) {
1530                            if (pr->saveFile[0] != 0) {
1531                              pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
1532                                  picodata_getPuTypeFromExtension(pr->saveFile, /*input*/FALSE),0,pr->saveFile);
1533                            }
1534                            pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,pr->tmpStr1);
1535                            pr_strcpy(pr->saveFile, pr->tmpStr1);
1536                        }
1537                        break;
1538                    case PR_OStr:
1539                        if (picokpr_getOutItemStrOfs(network, picokpr_getOutItemArgOfs(network, outitem)) != 0) {
1540                            lstrp = picokpr_getOutItemStr(network, picokpr_getOutItemArgOfs(network, outitem));
1541                            if ((lnum = picodata_getPuTypeFromExtension(lstrp, /*input*/FALSE)) != PICODATA_ITEMINFO2_CMD_TO_UNKNOWN) {
1542                                if (pr->saveFile[0] != 0) {
1543                                    pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
1544                                        picodata_getPuTypeFromExtension(pr->saveFile, /*input*/FALSE),0,pr->saveFile);
1545                                }
1546                                pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,lstrp);
1547                                pr_strcpy(pr->saveFile, lstrp);
1548                            }
1549                            pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,lstrp);
1550                        }
1551                        break;
1552                default:
1553                    pr_startItemList(& lf,& ll);
1554                    pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
1555                    if (pr->outOfMemory) return;
1556                    if (((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN))) {
1557                        if ((lnum = picodata_getPuTypeFromExtension(lf->data, /*input*/FALSE)) != PICODATA_ITEMINFO2_CMD_TO_UNKNOWN) {
1558                            if (pr->saveFile[0] != 0) {
1559                                pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
1560                                    picodata_getPuTypeFromExtension(pr->saveFile, /*input*/FALSE),0,pr->saveFile);
1561                            }
1562                            pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, lcmd,lnum,0,lf->data);
1563                            pr_strcpy(pr->saveFile, lf->data);
1564                        }
1565                    }
1566                    break;
1567                }
1568/*
1569            } else {
1570                pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,lnum,0,pr->tmpStr1);
1571*/
1572            }
1573            break;
1574        case PR_OUseSig:   case PR_OPlay:
1575            lout = FALSE;
1576            lf0beg =  -(1);
1577            lf0end =  -(1);
1578            lxfadebeg[0] = 0;
1579            lxfadeend[0] = 0;
1580            pr->tmpStr1[0] = 0;
1581            if ((picokpr_getOutItemType(network, outitem) == PR_OUseSig)) {
1582                lcmd = PICODATA_ITEMINFO1_CMD_IGNSIG;
1583            } else {
1584                lcmd = PICODATA_ITEMINFO1_CMD_IGNORE;
1585            }
1586            if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
1587                linfo2 = PICODATA_ITEMINFO2_CMD_START;
1588            } else {
1589                linfo2 = PICODATA_ITEMINFO2_CMD_END;
1590            }
1591            if (picokpr_getOutItemArgOfs(network, outitem) != 0) {
1592                switch (picokpr_getOutItemType(network, picokpr_getOutItemArgOfs(network, outitem))) {
1593                    case PR_OVar:
1594                        lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, outitem)));
1595                        if (lvar != NULL) {
1596                            litem = lvar->first;
1597                        }
1598                        pr->tmpStr1[0] = 0;
1599                        while (litem != NULL) {
1600                            if (litem->head.type == PICODATA_ITEM_TOKEN) {
1601                                pr_strcat(pr->tmpStr1,litem->data);
1602                            }
1603                            litem = litem->next;
1604                        }
1605                        pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PLAY,
1606                            picodata_getPuTypeFromExtension(pr->tmpStr1, /*input*/TRUE),0, pr->tmpStr1);
1607                        lout = TRUE;
1608                        break;
1609                    case PR_OStr:
1610                        if (picokpr_getOutItemStrOfs(network, picokpr_getOutItemArgOfs(network, outitem)) != 0) {
1611                            lstrp = picokpr_getOutItemStr(network, picokpr_getOutItemArgOfs(network, outitem));
1612                            pr_strcpy(pr->tmpStr1, lstrp);
1613                            lout = TRUE;
1614                        }
1615                        break;
1616                default:
1617                    pr_startItemList(& lf,& ll);
1618                    pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, outitem),vars,& lf,& ll);
1619                    if (pr->outOfMemory) return;
1620                    if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
1621                        pr_strcpy(pr->tmpStr1, lf->data);
1622                        lout = TRUE;
1623                    }
1624                    break;
1625                }
1626            }
1627            argOfs = picokpr_getOutItemArgOfs(network, outitem);
1628            if (argOfs != 0) {
1629                nextOfs = picokpr_getOutItemNextOfs(network, outitem);
1630                if (nextOfs != 0) {
1631                    if (picokpr_getOutItemType(network, nextOfs) == PR_OVal) {
1632                        lf0beg = picokpr_getOutItemVal(network, nextOfs);
1633                    }
1634                    nextOfs2 = picokpr_getOutItemNextOfs(network, nextOfs);
1635                    if (nextOfs2 != 0) {
1636                        if (picokpr_getOutItemType(network, nextOfs2) == PR_OVal) {
1637                            lf0end = picokpr_getOutItemVal(network, nextOfs2);
1638                        }
1639                        nextOfs3 = picokpr_getOutItemNextOfs(network, nextOfs2);
1640                        if (nextOfs3 != 0) {
1641                            if ((picokpr_getOutItemType(network, nextOfs3) == PR_OStr) && (picokpr_getOutItemStrOfs(network, nextOfs3) != 0)) {
1642                                lstrp = picokpr_getOutItemStr(network, nextOfs3);
1643                                pr_strcpy(lxfadebeg, lstrp);
1644                            }
1645                            nextOfs4 = picokpr_getOutItemNextOfs(network, nextOfs3);
1646                            if (nextOfs4 != 0) {
1647                                if ((picokpr_getOutItemType(network, nextOfs4) == PR_OStr) && (picokpr_getOutItemStrOfs(network, nextOfs4) != 0)) {
1648                                    lstrp = picokpr_getOutItemStr(network, nextOfs4);
1649                                    pr_strcpy(lxfadeend, lstrp);
1650                                }
1651                            }
1652                        }
1653                    }
1654                }
1655            }
1656            if (lout) {
1657                pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,PICODATA_ITEMINFO1_CMD_PLAY,
1658                    picodata_getPuTypeFromExtension(pr->tmpStr1, /*input*/TRUE),0,pr->tmpStr1);
1659            }
1660            pr_putItem(this, pr,& (*first),& (*last),PICODATA_ITEM_CMD,lcmd,linfo2,0,(picoos_uchar*)"");
1661            break;
1662    default:
1663        PICODBG_INFO(("unknown command"));
1664        break;
1665    }
1666}
1667
1668
1669static void pr_getOutputItemList (picodata_ProcessingUnit this,
1670                                  pr_subobj_t * pr,
1671                                  picokpr_Preproc network,
1672                                  picokpr_OutItemArrOffset outitem,
1673                                  pr_OutItemVarPtr vars,
1674                                  pr_ioItemPtr * first,
1675                                  pr_ioItemPtr * last)
1676{
1677
1678    picokpr_OutItemArrOffset lo;
1679    picoos_int32 llen;
1680    picoos_int32 llen2;
1681    picokpr_VarStrPtr lstrp;
1682    picoos_int32 lval32;
1683    picoos_int32 li;
1684    picoos_int32 li2;
1685    picoos_int32 ln;
1686    picoos_int32 ln2;
1687    pr_ioItemPtr litem2;
1688    pr_ioItemPtr lf;
1689    pr_ioItemPtr ll;
1690    picoos_int32 lid;
1691    picoos_uint8 ltype;
1692    picoos_int8 lsubtype;
1693    pr_OutItemVarPtr lvar;
1694    picoos_int32 lspellmode;
1695    picoos_int32 largOfs;
1696    picoos_int32 lnextOfs;
1697
1698
1699    lo = outitem;
1700    while (lo != 0) {
1701        switch (picokpr_getOutItemType(network, lo)) {
1702            case PR_OStr:
1703                lstrp = picokpr_getOutItemStr(network, lo);
1704                if (pr->outOfMemory) return;
1705                pr_initItem(this, &pr->tmpItem);
1706                pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
1707                pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
1708                pr->tmpItem.head.info2 =  -1;
1709                pr->tmpItem.head.len = pr_strcpy(pr->tmpItem.data, lstrp);
1710                pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1711                if (pr->outOfMemory) return;
1712                pr_appendItem(this, & (*first),& (*last),litem2);
1713                break;
1714            case PR_OVar:
1715                lvar = pr_findVariable(vars,picokpr_getOutItemVal(network, lo));
1716                if (lvar != NULL) {
1717                    lf = lvar->first;
1718                } else {
1719                    lf = NULL;
1720                }
1721                while (lf != NULL) {
1722                    pr_copyItem(this, pr_WorkMem,& (*lf),& litem2);
1723                    if (pr->outOfMemory) return;
1724                    pr_appendItem(this, & (*first),& (*last),litem2);
1725                    lf = lf->next;
1726                }
1727                break;
1728            case PR_OSpell:
1729                lspellmode = PR_SPELL;
1730                largOfs = picokpr_getOutItemArgOfs(network, lo);
1731                if ((largOfs!= 0) && ((lnextOfs = picokpr_getOutItemNextOfs(network, largOfs)) != 0)) {
1732                    lspellmode = picokpr_getOutItemVal(network, lnextOfs);
1733                }
1734                pr_startItemList(& lf,& ll);
1735                pr_getOutputItemList(this, pr, network,largOfs,vars,& lf,& ll);
1736                if (pr->outOfMemory) return;
1737                while (lf != NULL) {
1738                    switch (lf->head.type) {
1739                        case PICODATA_ITEM_TOKEN:
1740                            li = 0;
1741                            ln = pr_strlen(lf->data);
1742                            while (li < ln) {
1743                                pr_initItem(this, &pr->tmpItem);
1744                                if (pr->outOfMemory) return;
1745                                pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
1746                                pr->tmpItem.head.info1 = lf->head.info1;
1747                                pr->tmpItem.head.info2 = lf->head.info2;
1748                                pr->tmpItem.head.len = picobase_det_utf8_length(lf->data[li]);
1749                                for (li2 = 0; li2 < pr->tmpItem.head.len; li2++) {
1750                                    pr->tmpItem.data[li2] = lf->data[li];
1751                                    li++;
1752                                }
1753                                pr->tmpItem.data[pr->tmpItem.head.len] = 0;
1754                                pr->tmpItem.val = 0;
1755                                lid = picoktab_graphOffset(pr->graphs,pr->tmpItem.data);
1756                                if (lid > 0) {
1757                                    if (picoktab_getIntPropTokenType(pr->graphs, lid, &ltype)) {
1758                                        if ((ltype == PICODATA_ITEMINFO1_TOKTYPE_LETTERV)/* || (ltype == PICODATA_ITEMINFO1_TOKTYPE_DIGIT)*/) {
1759                                            pr->tmpItem.head.len = pr_strcat(pr->tmpItem.data,(picoos_uchar*)SPEC_CHAR);
1760                                        }
1761                                    }
1762                                    picoktab_getIntPropTokenSubType(pr->graphs, lid, &lsubtype);
1763                                } else {
1764                                    ltype = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
1765                                    lsubtype =  -(1);
1766                                }
1767                                pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1768                                if (pr->outOfMemory) return;
1769                                pr_appendItem(this, & (*first),& (*last), litem2);
1770                                if (lspellmode == PR_SPELL_WITH_PHRASE_BREAK) {
1771                                    pr_initItem(this, &pr->tmpItem);
1772                                    pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
1773                                    pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_CHAR;
1774                                    pr->tmpItem.head.info2 = lsubtype;
1775                                    pr->tmpItem.head.len = 1;
1776                                    pr->tmpItem.data[0] = ',';
1777                                    pr->tmpItem.data[1] = 0;
1778                                    pr->tmpItem.val = 0;
1779                                    pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1780                                    if (pr->outOfMemory) return;
1781                                    pr_appendItem(this, & (*first),& (*last),litem2);
1782                                } else if (lspellmode == PR_SPELL_WITH_SENTENCE_BREAK) {
1783                                    pr_initItem(this, &pr->tmpItem);
1784                                    pr->tmpItem.head.type = PICODATA_ITEM_CMD;
1785                                    pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_CMD_FLUSH;
1786                                    pr->tmpItem.head.info2 = PICODATA_ITEMINFO2_NA;
1787                                    pr->tmpItem.head.len = 0;
1788                                    pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1789                                    if (pr->outOfMemory) return;
1790                                    pr_appendItem(this, & (*first),& (*last),litem2);
1791                                } else if (lspellmode > 0) {
1792                                    pr_initItem(this, &pr->tmpItem);
1793                                    pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
1794                                    pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_CHAR;
1795                                    pr->tmpItem.head.info2 = lsubtype;
1796                                    pr->tmpItem.head.len = 1;
1797                                    pr->tmpItem.data[0] = ',';
1798                                    pr->tmpItem.data[1] = 0;
1799                                    pr->tmpItem.val = 0;
1800                                    pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1801                                    if (pr->outOfMemory) return;
1802                                    pr_appendItem(this, & (*first),& (*last),litem2);
1803                                }
1804                            }
1805                            break;
1806                    default:
1807                        pr_copyItem(this, pr_WorkMem,& (*lf),& litem2);
1808                        if (pr->outOfMemory) return;
1809                        pr_appendItem(this, & (*first),& (*last),litem2);
1810                        break;
1811                    }
1812                    ll = lf;
1813                    lf = lf->next;
1814                    ll->next = NULL;
1815                }
1816                break;
1817            case PR_OConcat:
1818                pr_startItemList(& lf,& ll);
1819                pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, lo),vars,& lf,& ll);
1820                if (pr->outOfMemory) return;
1821                pr_initItem(this, &pr->tmpItem);
1822                pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
1823                pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
1824                pr->tmpItem.head.info2 =  -(1);
1825                pr->tmpItem.head.len = 0;
1826                pr->tmpItem.data[0] = 0;
1827                pr->tmpItem.val = 0;
1828                while (lf != NULL) {
1829                    switch (lf->head.type) {
1830                        case PICODATA_ITEM_TOKEN:
1831                            pr->tmpItem.head.len = pr_strcat(pr->tmpItem.data,lf->data);
1832                            break;
1833                        case PICODATA_ITEM_CMD:
1834                            pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1835                            if (pr->outOfMemory) return;
1836                            pr_appendItem(this, & (*first),& (*last),litem2);
1837
1838                            pr_copyItem(this, pr_WorkMem, lf, &litem2);
1839                            if (pr->outOfMemory) return;
1840                            pr_appendItem(this, & (*first),& (*last),litem2);
1841
1842                            pr_initItem(this, &pr->tmpItem);
1843                            pr->tmpItem.head.type = PICODATA_ITEM_TOKEN;
1844                            pr->tmpItem.head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
1845                            pr->tmpItem.head.info2 =  -(1);
1846                            pr->tmpItem.head.len = 0;
1847                            pr->tmpItem.data[0] = 0;
1848                            pr->tmpItem.val = 0;
1849                            break;
1850                    default:
1851                        break;
1852                    }
1853                    lf = lf->next;
1854                }
1855                pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1856                if (pr->outOfMemory) return;
1857                pr_appendItem(this, & (*first),& (*last),litem2);
1858                break;
1859            case PR_ORomanToCard:
1860                pr_startItemList(& lf,& ll);
1861                pr_getOutputItemList(this, pr, network,picokpr_getOutItemArgOfs(network, lo),vars,& lf,& ll);
1862                if (pr->outOfMemory) return;
1863                if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
1864                    pr_initItem(this, &pr->tmpItem);
1865                    pr_copyItemContent(this, lf, &pr->tmpItem);
1866                    if (pr_isLatinNumber(lf->data, & lval32)) {
1867                        pr_int_to_string(lval32, (picoos_char *)pr->tmpItem.data, PR_MAX_DATA_LEN_Z);
1868                        pr->tmpItem.head.len = pr_strlen(pr->tmpItem.data);
1869                        pr->tmpItem.val = lval32;
1870                    }
1871                    pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1872                    pr_appendItem(this, & (*first),& (*last),litem2);
1873                }
1874                break;
1875            case PR_OVal:
1876                break;
1877            case PR_OLeft:
1878                pr_startItemList(& lf,& ll);
1879                pr_getOutputItemList(this, pr, network,picokpr_getOutItemNextOfs(network, picokpr_getOutItemArgOfs(network, lo)),vars,& lf,& ll);
1880                if (pr->outOfMemory) return;
1881                if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
1882                    pr_initItem(this, &pr->tmpItem);
1883                    pr_copyItemContent(this, lf, &pr->tmpItem);
1884                    llen = lf->head.len;
1885                    llen2 = picobase_utf8_length(pr->tmpItem.data,PR_MAX_DATA_LEN);
1886                    ln = 0;
1887                    ln2 = 0;
1888                    largOfs = picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, lo));
1889                    while ((ln < llen) && (ln2 < llen2) && (ln2 < largOfs)) {
1890                        ln = ln + picobase_det_utf8_length(pr->tmpItem.data[ln]);
1891                        ln2++;
1892                    }
1893                    pr->tmpItem.data[ln] = 0;
1894                    pr->tmpItem.head.len = ln;
1895                    pr_copyItem(this, pr_WorkMem, &pr->tmpItem, &litem2);
1896                    if (pr->outOfMemory) return;
1897                    pr_appendItem(this, & (*first),& (*last),litem2);
1898                }
1899                break;
1900            case PR_ORight:
1901                pr_startItemList(& lf,& ll);
1902                pr_getOutputItemList(this, pr, network,picokpr_getOutItemNextOfs(network, picokpr_getOutItemArgOfs(network, lo)),vars,& lf,& ll);
1903                if (pr->outOfMemory) return;
1904                if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
1905                    pr_initItem(this, &pr->tmpItem);
1906                    pr_copyItemContent(this, lf, & pr->tmpItem);
1907                    llen = lf->head.len;
1908                    llen2 = picobase_utf8_length(pr->tmpItem.data,PR_MAX_DATA_LEN);
1909                    ln = 0;
1910                    ln2 = 0;
1911                    while ((ln < llen) && (ln2 < llen2) && (ln2 < (llen2 - picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, lo))))) {
1912                        ln = ln + picobase_det_utf8_length(pr->tmpItem.data[ln]);
1913                        ln2++;
1914                    }
1915                    pr->tmpItem.head.len = pr_removeSubstring(0,ln,pr->tmpItem.data);
1916                    pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & litem2);
1917                    if (pr->outOfMemory) return;
1918                    pr_appendItem(this, & (*first),& (*last),litem2);
1919                }
1920                break;
1921            case PR_OItem:
1922                pr_startItemList(& lf,& ll);
1923                pr_getOutputItemList(this, pr, network,picokpr_getOutItemNextOfs(network, picokpr_getOutItemArgOfs(network, lo)),vars,& lf,& ll);
1924                if (pr->outOfMemory) return;
1925                ln = picokpr_getOutItemVal(network, picokpr_getOutItemArgOfs(network, lo));
1926                li = 1;
1927                while ((li < ln) && (lf != NULL)) {
1928                    lf = lf->next;
1929                    li++;
1930                }
1931                if ((lf != NULL) && (li == ln) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
1932                    pr_copyItem(this, pr_WorkMem, lf, & litem2);
1933                    if (pr->outOfMemory) return;
1934                    pr_appendItem(this, & (*first),& (*last),litem2);
1935                }
1936                break;
1937            case PR_ORLZ:
1938                pr_startItemList(& lf,& ll);
1939                pr_getOutputItemList(this, pr, network, picokpr_getOutItemArgOfs(network, lo),vars,& lf,& ll);
1940                if (pr->outOfMemory) return;
1941                if ((lf != NULL) && (lf->head.type == PICODATA_ITEM_TOKEN)) {
1942                    pr_initItem(this, &pr->tmpItem);
1943                    pr_copyItemContent(this, lf, & pr->tmpItem);
1944                    li = 0;
1945                    while ((li < lf->head.len) && (pr->tmpItem.data[li] == '0')) {
1946                        li++;
1947                    }
1948                    pr->tmpItem.head.len = pr_removeSubstring(0,li,pr->tmpItem.data);
1949                    pr_copyItem(this, pr_WorkMem, &pr->tmpItem, & litem2);
1950                    if (pr->outOfMemory) return;
1951                    pr_appendItem(this, & (*first),& (*last),litem2);
1952                }
1953                break;
1954            case PR_OIgnore:   case PR_OPitch:   case PR_OSpeed:   case PR_OVolume:   case PR_OPhonSVOXPA:   case PR_OPhonSAMPA:   case PR_OBreak:   case PR_OMark:   case PR_OPara:   case PR_OSent:   case PR_OPlay:
1955            case PR_OUseSig:   case PR_OGenFile:   case PR_OAudioEdit:   case PR_OContext:   case PR_OVoice:
1956                pr_genCommands(this, pr, network,lo,vars,& (*first),& (*last));
1957                if (pr->outOfMemory) return;
1958                break;
1959        default:
1960            PICODBG_INFO(("unkown command"));
1961            break;
1962        }
1963        lo = picokpr_getOutItemNextOfs(network, lo);
1964    }
1965}
1966
1967
1968static picoos_int32 pr_attrVal (picokpr_Preproc network, picokpr_TokArrOffset tok, pr_TokSetEleWP type)
1969{
1970
1971    pr_TokSetEleWP tse;
1972    picoos_int32 n;
1973    picokpr_TokSetWP set;
1974
1975    n = 0;
1976    tse = PR_FIRST_TSE_WP;
1977    set = picokpr_getTokSetWP(network, tok);
1978    while (tse < type) {
1979        if (((1<<tse) & set) != 0) {
1980            n++;
1981        }
1982        tse = (pr_TokSetEleWP)((picoos_int32)tse+1);
1983    }
1984    return picokpr_getAttrValArrInt32(network, picokpr_getTokAttribOfs(network, tok) + n);
1985}
1986
1987
1988static void pr_getOutput (picodata_ProcessingUnit this, pr_subobj_t * pr,
1989                          picoos_int32 * i, picoos_int32 d, pr_ioItemPtr * o, pr_ioItemPtr * ol)
1990{
1991
1992    register struct pr_PathEle * with__0;
1993    pr_OutItemVarPtr lvars;
1994    pr_OutItemVarPtr lvar;
1995    pr_ioItemPtr lit;
1996    pr_ioItemPtr ldit;
1997    pr_ioItemPtr ldlit;
1998    picoos_bool lfirst;
1999    pr_ioItemPtr lcopy;
2000    picokpr_TokSetWP wpset;
2001    picokpr_TokSetNP npset;
2002    picoos_int32 li;
2003
2004    lfirst = TRUE;
2005    (*i)++;
2006    lit = NULL;
2007    lvars = NULL;
2008    ldit = NULL;
2009    ldlit = NULL;
2010    while ((*i) < pr->rbestpath.rlen) {
2011        with__0 = & pr->rbestpath.rele[*i];
2012        li = 0;
2013        if (*i > 0) {
2014            while ((li < 127) && (li < pr->rbestpath.rele[*i].rdepth)) {
2015                pr->lspaces[li++] = ' ';
2016            }
2017        }
2018        pr->lspaces[li] = 0;
2019        if (with__0->rprodname != 0) {
2020            PICODBG_INFO(("pp path  :%s%s(", pr->lspaces, picokpr_getVarStrPtr(with__0->rnetwork,with__0->rprodname)));
2021        }
2022        if ((pr->ritems[with__0->ritemid+1] != NULL) && (pr->ritems[with__0->ritemid+1]->head.type == PICODATA_ITEM_TOKEN)) {
2023            PICODBG_INFO(("pp in (1): %s'%s'", pr->lspaces, pr->ritems[with__0->ritemid+1]->data));
2024        }
2025        if ((pr->ritems[with__0->ritemid+1] != NULL)) {
2026            while ((pr->rinItemList != NULL) && (pr->rinItemList != pr->ritems[with__0->ritemid+1]) && (pr->rinItemList->head.type != PICODATA_ITEM_TOKEN)) {
2027                lit = pr->rinItemList;
2028                pr->rinItemList = pr->rinItemList->next;
2029                lit->next = NULL;
2030                pr_copyItem(this, pr_WorkMem,& (*lit),& lcopy);
2031                if (pr->outOfMemory) return;
2032                pr_disposeItem(this, & lit);
2033                pr_appendItem(this, & (*o),& (*ol),lcopy);
2034            }
2035            if (pr->rinItemList != NULL) {
2036                lit = pr->rinItemList;
2037                pr->rinItemList = pr->rinItemList->next;
2038                lit->next = NULL;
2039            } else {
2040                lit = NULL;
2041            }
2042        }
2043        wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
2044        npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
2045
2046        if ((PR_TSE_MASK_PROD & wpset) != 0) {
2047            if ((PR_TSE_MASK_VAR & wpset) != 0) {
2048                lvar = pr_findVariable(lvars,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar));
2049                if (lvar == NULL) {
2050                    pr_ALLOCATE(this, pr_WorkMem, (void *) & lvar,(sizeof (*lvar)));
2051                    lvar->next = lvars;
2052                    lvar->id = pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar);
2053                    lvars = lvar;
2054                }
2055                pr_startItemList(& lvar->first,& lvar->last);
2056                pr_getOutput(this, pr, & (*i),(d + 1),& lvar->first,& lvar->last);
2057                if (pr->outOfMemory) return;
2058            } else {
2059                pr_startItemList(& ldit,& ldlit);
2060                pr_getOutput(this, pr, & (*i),(d + 1),& ldit,& ldlit);
2061                if (pr->outOfMemory) return;
2062            }
2063            (*i)++;
2064        } else if ((PR_TSE_MASK_VAR & wpset) != 0) {
2065            lvar = pr_findVariable(lvars,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar));
2066            if (lvar == NULL) {
2067                pr_ALLOCATE(this, pr_WorkMem, (void *) & lvar,(sizeof (*lvar)));
2068                lvar->next = lvars;
2069                lvar->id = pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVar);
2070                lvars = lvar;
2071            }
2072            if (((PR_TSE_MASK_LEX & wpset) != 0) && ((PR_TSE_MASK_LETTER & npset) == 0)) {
2073                if (lfirst) {
2074                    pr_newItem(this, pr_WorkMem,& lit, PICODATA_ITEM_TOKEN, sizeof(struct pr_ioItem), /*inItem*/FALSE);
2075                    if (pr->outOfMemory) return;
2076                    lit->head.type = PICODATA_ITEM_TOKEN;
2077                    lit->head.info1 = pr->ritems[with__0->ritemid+1]->head.info1;
2078                    lit->head.info2 = pr->ritems[with__0->ritemid+1]->head.info2;
2079                    if (pr->ritems[with__0->ritemid+1]->head.info1 == PICODATA_ITEMINFO1_TOKTYPE_SPACE) {
2080                        lit->head.len = pr_strcpy(lit->data, (picoos_uchar*)"_");
2081                    } else {
2082                        lit->head.len = pr_strcpy(lit->data, pr->ritems[with__0->ritemid+1]->data);
2083                    }
2084                    lvar->first = lit;
2085                    lvar->last = lit;
2086                    lfirst = FALSE;
2087                } else {
2088                    if ((pr->ritems[with__0->ritemid+1]->head.info1 == PICODATA_ITEMINFO1_TOKTYPE_SPACE)) {
2089                        lvar->last->head.len = pr_strcat(lvar->last->data,(picoos_uchar*)"_");
2090                    } else {
2091                        lvar->last->head.len = pr_strcat(lvar->last->data,pr->ritems[with__0->ritemid+1]->data);
2092                    }
2093                    lvar->last->head.info1 = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
2094                    lvar->last->head.info2 =  -(1);
2095                }
2096            } else {
2097                lvar->first = pr->ritems[with__0->ritemid+1];
2098                lvar->last = pr->ritems[with__0->ritemid+1];
2099            }
2100            (*i)++;
2101        } else if ((PR_TSE_MASK_OUT & wpset) != 0) {
2102            pr_getOutputItemList(this, pr, with__0->rnetwork,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEOut),lvars,& (*o),& (*ol));
2103            if (pr->outOfMemory) return;
2104            (*i)++;
2105        } else if (((*i) < (pr->rbestpath.rlen - 1)) && (d != pr->rbestpath.rele[(*i) + 1].rdepth)) {
2106            if ((*i > 0) && (with__0->rdepth-1) == pr->rbestpath.rele[*i + 1].rdepth) {
2107                li = 0;
2108                while ((li < 127) && (li < with__0->rdepth-1)) {
2109                    pr->lspaces[li++] = ' ';
2110                }
2111                pr->lspaces[li] = 0;
2112                PICODBG_INFO(("pp path  :%s)", pr->lspaces));
2113            }
2114            return;
2115        } else {
2116            (*i)++;
2117        }
2118        if ((PR_TSE_MASK_LEX & wpset) == 0) {
2119            lfirst = TRUE;
2120        }
2121    }
2122    li = 0;
2123    while ((li < 127) && (li < pr->rbestpath.rele[*i].rdepth-1)) {
2124        pr->lspaces[li++] = ' ';
2125    }
2126    pr->lspaces[li] = 0;
2127    PICODBG_INFO(("pp path  :%s)", pr->lspaces));
2128}
2129
2130
2131static void pr_outputPath (picodata_ProcessingUnit this, pr_subobj_t * pr)
2132{
2133    register struct pr_PathEle * with__0;
2134    picoos_int32 li;
2135    pr_ioItemPtr lf;
2136    pr_ioItemPtr ll;
2137    pr_ioItemPtr lit;
2138    pr_ioItemPtr lit2;
2139    pr_MemState lmemState;
2140    picoos_bool lastPlayFileFound;
2141
2142    pr_getMemState(this, pr_WorkMem, & lmemState);
2143    lf = NULL;
2144    ll = NULL;
2145    li =  -(1);
2146    pr_getOutput(this, pr, & li,1,& lf,& ll);
2147    if (pr->outOfMemory) return;
2148    lastPlayFileFound = TRUE;
2149    while (lf != NULL) {
2150        lit = lf;
2151        lf = lf->next;
2152        lit->next = NULL;
2153        if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY)) {
2154            lastPlayFileFound = picoos_FileExists(this->common, (picoos_char*)lit->data);
2155            if (!lastPlayFileFound) {
2156                PICODBG_WARN(("file '%s' not found; synthesizing enclosed text instead", lit->data));
2157                picoos_emRaiseWarning(this->common->em, PICO_EXC_CANT_OPEN_FILE, (picoos_char*)"", (picoos_char*)"file '%s' not found; synthesizing enclosed text instead",lit->data);
2158            }
2159        }
2160        if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
2161            pr->insidePhoneme = TRUE;
2162        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME)&& pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
2163            pr->insidePhoneme = FALSE;
2164        }
2165        if ((pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) &&  !lastPlayFileFound)) {
2166        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNORE) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
2167            if (lastPlayFileFound) {
2168                pr->rignore++;
2169            }
2170        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNORE) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
2171            if (lastPlayFileFound) {
2172                if (pr->rignore > 0) {
2173                    pr->rignore--;
2174                }
2175            }
2176        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START) &&  !lastPlayFileFound) {
2177        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END) &&  !lastPlayFileFound) {
2178        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_CONTEXT)) {
2179            if (pr->rignore <= 0) {
2180                pr_setContext(this, pr, lit->data);
2181            }
2182        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_VOICE)) {
2183            if (pr->rignore <= 0) {
2184                pr_copyItem(this, pr_DynMem,lit,& lit2);
2185                if (pr->outOfMemory) return;
2186                pr_appendItem(this, & pr->routItemList,& pr->rlastOutItem, lit2);
2187            }
2188        } else {
2189            if ((pr->rignore <= 0) &&  !(pr->insidePhoneme && (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) ||
2190                                                               pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG)))) {
2191                PICODBG_INFO(("pp out(1): '%s'", lit->data));
2192                pr_copyItem(this, pr_DynMem,lit,& lit2);
2193                if (pr->outOfMemory) return;
2194                pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,lit2);
2195                if (pr->outOfMemory) return;
2196            }
2197        }
2198    }
2199    for (li = 0; li<pr->rbestpath.rlen; li++) {
2200        with__0 = & pr->rbestpath.rele[li];
2201        pr_disposeItem(this, & pr->ritems[with__0->ritemid+1]);
2202    }
2203    pr_resetMemState(this, pr_WorkMem, lmemState);
2204}
2205
2206/* *****************************************************************************/
2207
2208static void pr_compare (picoos_uchar str1lc[], picoos_uchar str2[], picoos_int16 * result)
2209{
2210
2211    picoos_int32 i;
2212    picoos_int32 j;
2213    picoos_int32 l;
2214    picoos_uint32 pos;
2215    picoos_bool finished1;
2216    picoos_bool finished2;
2217    picoos_bool done;
2218    picobase_utf8char utf8char;
2219
2220    pos = 0;
2221    l = picobase_get_next_utf8char(str2, PR_MAX_DATA_LEN, & pos,utf8char);
2222    picobase_lowercase_utf8_str(utf8char,(picoos_char*)utf8char,PICOBASE_UTF8_MAXLEN+1, &done);
2223    l = picobase_det_utf8_length(utf8char[0]);
2224    j = 0;
2225    i = 0;
2226    while ((i < PR_MAX_DATA_LEN) && (str1lc[i] != 0) && (l > 0) && (j <= 3) && (str1lc[i] == utf8char[j])) {
2227        i++;
2228        j++;
2229        if (j >= l) {
2230            l = picobase_get_next_utf8char(str2, PR_MAX_DATA_LEN, & pos,utf8char);
2231            picobase_lowercase_utf8_str(utf8char,(picoos_char*)utf8char,PICOBASE_UTF8_MAXLEN+1, &done);
2232            l = picobase_det_utf8_length(utf8char[0]);
2233            j = 0;
2234        }
2235    }
2236    finished1 = (i >= PR_MAX_DATA_LEN) || (str1lc[i] == 0);
2237    finished2 = (j > 3) || (utf8char[j] == 0);
2238    if (finished1 && finished2) {
2239        *result = PR_EQUAL;
2240    } else if (finished1) {
2241        *result = PR_SMALLER;
2242    } else if (finished2) {
2243        *result = PR_LARGER;
2244    } else {
2245        if (str1lc[i] < utf8char[j]) {
2246            *result = PR_SMALLER;
2247        } else {
2248            *result = PR_LARGER;
2249        }
2250    }
2251}
2252
2253
2254static picoos_bool pr_hasToken (picokpr_TokSetWP * tswp, picokpr_TokSetNP * tsnp)
2255{
2256    return ((((  PR_TSE_MASK_SPACE | PR_TSE_MASK_DIGIT | PR_TSE_MASK_LETTER | PR_TSE_MASK_SEQ
2257               | PR_TSE_MASK_CHAR | PR_TSE_MASK_BEGIN | PR_TSE_MASK_END) & (*tsnp)) != 0) ||
2258            ((PR_TSE_MASK_LEX & (*tswp)) != 0));
2259}
2260
2261
2262static picoos_bool pr_getNextToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
2263{
2264    register struct pr_PathEle * with__0;
2265    picoos_int32 len;
2266    picokpr_TokSetNP npset;
2267
2268    len = pr->ractpath.rlen;
2269    with__0 = & pr->ractpath.rele[len - 1];
2270    npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
2271    if ((len > 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN) && ((PR_TSE_MASK_NEXT & npset) != 0)) {
2272        pr_initPathEle(& pr->ractpath.rele[len]);
2273        pr->ractpath.rele[len].rnetwork = with__0->rnetwork;
2274        pr->ractpath.rele[len].rtok = picokpr_getTokNextOfs(with__0->rnetwork, with__0->rtok);
2275        pr->ractpath.rele[len].rdepth = with__0->rdepth;
2276        pr->ractpath.rlen++;
2277        return TRUE;
2278    } else {
2279        if (len >= PR_MAX_PATH_LEN) {
2280          PICODBG_INFO(("max path len reached (pr_getNextToken)"));
2281        }
2282        return FALSE;
2283    }
2284}
2285
2286
2287static picoos_bool pr_getAltToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
2288{
2289    register struct pr_PathEle * with__0;
2290    picokpr_TokArrOffset lTok;
2291    picokpr_TokSetNP npset;
2292
2293
2294    with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2295    if ((pr->ractpath.rlen > 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN)) {
2296        npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
2297        if (with__0->rcompare == PR_SMALLER) {
2298            if ((PR_TSE_MASK_ALTL & npset) != 0) {
2299                lTok = picokpr_getTokAltLOfs(with__0->rnetwork, with__0->rtok);
2300            } else {
2301                return FALSE;
2302            }
2303        } else {
2304            if ((PR_TSE_MASK_ALTR & npset) != 0) {
2305                lTok = picokpr_getTokAltROfs(with__0->rnetwork, with__0->rtok);
2306            } else {
2307                return FALSE;
2308            }
2309        }
2310        with__0->rlState = PR_LSInit;
2311        with__0->rtok = lTok;
2312        with__0->ritemid =  -1;
2313        with__0->rcompare =  -1;
2314        with__0->rprodname = 0;
2315        with__0->rprodprefcost = 0;
2316        return TRUE;
2317    } else {
2318        if (pr->ractpath.rlen >= PR_MAX_PATH_LEN) {
2319          PICODBG_INFO(("max path len reached (pr_getAltToken)"));
2320        }
2321        return FALSE;
2322    }
2323}
2324
2325
2326static picoos_bool pr_findProduction (picodata_ProcessingUnit this, pr_subobj_t * pr,
2327                                      picoos_uchar str[], picokpr_Preproc * network, picokpr_TokArrOffset * tokOfs)
2328{
2329    picoos_bool found;
2330    picoos_int32 p;
2331    picoos_int32 ind;
2332    picoos_int32 i;
2333    picoos_bool done;
2334    picokpr_VarStrPtr lstrp;
2335    picoos_int32 lprodarrlen;
2336
2337    ind = 0;
2338    pr_getTermPartStr(str,& ind,'.',pr->tmpStr1,& done);
2339    pr_getTermPartStr(str,& ind,'.',pr->tmpStr2,& done);
2340    found = FALSE;
2341
2342    for (p=0; p<PR_MAX_NR_PREPROC; p++) {
2343        if (!found && (pr->preproc[p] != NULL)) {
2344            if (pr_strEqual(pr->tmpStr1, picokpr_getPreprocNetName(pr->preproc[p]))) {
2345                i = 0;
2346                lprodarrlen = picokpr_getProdArrLen(pr->preproc[p]);
2347                while (!found && (i <= (lprodarrlen - 1))) {
2348                    lstrp = picokpr_getVarStrPtr(pr->preproc[p],picokpr_getProdNameOfs(pr->preproc[p], i));
2349                    if (pr_strEqual(pr->tmpStr2, lstrp)) {
2350                        *network = pr->preproc[p];
2351                        *tokOfs = picokpr_getProdATokOfs(pr->preproc[p], i);
2352                        return TRUE;
2353                    }
2354                    i++;
2355                }
2356           }
2357        }
2358    }
2359    return FALSE;
2360}
2361
2362
2363static picoos_bool pr_getProdToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
2364{
2365    register struct pr_PathEle * with__0;
2366    picokpr_VarStrPtr lstrp;
2367    picokpr_TokSetWP wpset;
2368
2369    if ((pr->ractpath.rlen > 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN)) {
2370        with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2371        wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
2372        if ((PR_TSE_MASK_PROD & wpset) != 0) {
2373            if ((PR_TSE_MASK_PRODEXT & wpset) != 0) {
2374                pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
2375                lstrp = picokpr_getVarStrPtr(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEProdExt));
2376                if (pr_findProduction(this, pr, lstrp,& pr->ractpath.rele[pr->ractpath.rlen].rnetwork,& pr->ractpath.rele[pr->ractpath.rlen].rtok)) {
2377                    with__0->rprodname = picokpr_getProdNameOfs(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok,PR_TSEProd));
2378                    with__0->rprodprefcost = picokpr_getProdPrefCost(with__0->rnetwork, pr_attrVal(with__0->rnetwork,with__0->rtok,PR_TSEProd));
2379                    pr->ractpath.rele[pr->ractpath.rlen].rdepth = with__0->rdepth + 1;
2380                    pr->ractpath.rlen++;
2381                    return TRUE;
2382                } else {
2383                    return FALSE;
2384                }
2385            } else {
2386                pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
2387                pr->ractpath.rele[pr->ractpath.rlen].rnetwork = with__0->rnetwork;
2388                pr->ractpath.rele[pr->ractpath.rlen].rtok = picokpr_getProdATokOfs(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok,PR_TSEProd));
2389                with__0->rprodname = picokpr_getProdNameOfs(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEProd));
2390                with__0->rprodprefcost = picokpr_getProdPrefCost(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEProd));
2391                pr->ractpath.rele[pr->ractpath.rlen].rdepth = with__0->rdepth + 1;
2392                pr->ractpath.rlen++;
2393                return TRUE;
2394            }
2395        }
2396    }
2397    if (pr->ractpath.rlen >= PR_MAX_PATH_LEN) {
2398        PICODBG_INFO(("max path len reached (pr_getProdToken)"));
2399    }
2400    return FALSE;
2401}
2402
2403
2404static picoos_bool pr_getProdContToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
2405{
2406    picoos_int32 li;
2407
2408    li = pr->ractpath.rlen - 1;
2409    while ((li > 0) &&  !((pr->ractpath.rele[li].rdepth == (pr->ractpath.rele[pr->ractpath.rlen - 1].rdepth - 1)) && ((PR_TSE_MASK_PROD &picokpr_getTokSetWP(pr->ractpath.rele[li].rnetwork, pr->ractpath.rele[li].rtok)) != 0))) {
2410        li--;
2411    }
2412    if (((li >= 0) && (pr->ractpath.rlen < PR_MAX_PATH_LEN) && (PR_TSE_MASK_NEXT &picokpr_getTokSetNP(pr->ractpath.rele[li].rnetwork, pr->ractpath.rele[li].rtok)) != 0)) {
2413        pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
2414        pr->ractpath.rele[pr->ractpath.rlen].rnetwork = pr->ractpath.rele[li].rnetwork;
2415        pr->ractpath.rele[pr->ractpath.rlen].rtok = picokpr_getTokNextOfs(pr->ractpath.rele[li].rnetwork, pr->ractpath.rele[li].rtok);
2416        pr->ractpath.rele[pr->ractpath.rlen].rdepth = pr->ractpath.rele[li].rdepth;
2417        pr->ractpath.rlen++;
2418        return TRUE;
2419    } else {
2420        if (pr->ractpath.rlen >= PR_MAX_PATH_LEN) {
2421            PICODBG_INFO(("max path len reached (pr_getProdContToken)"));
2422        }
2423        return FALSE;
2424    }
2425}
2426
2427/* *****************************************************************************/
2428
2429static picoos_bool pr_getTopLevelToken (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_bool firstprod)
2430{
2431    if (firstprod) {
2432        if (pr->actCtx != NULL) {
2433            pr->prodList = pr->actCtx->rProdList;
2434        } else {
2435            pr->prodList = NULL;
2436        }
2437    } else if (pr->prodList != NULL) {
2438        pr->prodList = pr->prodList->rNext;
2439    }
2440    if ((pr->prodList != NULL) && (pr->prodList->rProdOfs != 0) && (picokpr_getProdATokOfs(pr->prodList->rNetwork, pr->prodList->rProdOfs) != 0)) {
2441        pr_initPathEle(& pr->ractpath.rele[pr->ractpath.rlen]);
2442        pr->ractpath.rele[pr->ractpath.rlen].rdepth = 1;
2443        pr->ractpath.rele[pr->ractpath.rlen].rnetwork = pr->prodList->rNetwork;
2444        pr->ractpath.rele[pr->ractpath.rlen].rtok = picokpr_getProdATokOfs(pr->prodList->rNetwork, pr->prodList->rProdOfs);
2445        pr->ractpath.rele[pr->ractpath.rlen].rlState = PR_LSInit;
2446        pr->ractpath.rele[pr->ractpath.rlen].rcompare =  -1;
2447        pr->ractpath.rele[pr->ractpath.rlen].rprodname = picokpr_getProdNameOfs(pr->prodList->rNetwork, pr->prodList->rProdOfs);
2448        pr->ractpath.rele[pr->ractpath.rlen].rprodprefcost = picokpr_getProdPrefCost(pr->prodList->rNetwork, pr->prodList->rProdOfs);
2449        pr->ractpath.rlen++;
2450        return TRUE;
2451    } else {
2452        return FALSE;
2453    }
2454}
2455
2456
2457static picoos_bool pr_getToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
2458{
2459    picoos_int32 ln;
2460    picoos_int32 lid;
2461
2462    ln = (pr->ractpath.rlen - 2);
2463    while ((ln >= 0) && (pr->ractpath.rele[ln].ritemid ==  -1)) {
2464        ln = ln - 1;
2465    }
2466    if (ln >= 0) {
2467        lid = pr->ractpath.rele[ln].ritemid + 1;
2468    } else {
2469        lid = 0;
2470    }
2471    if (lid < pr->rnritems) {
2472        pr->ractpath.rele[pr->ractpath.rlen - 1].ritemid = lid;
2473    } else {
2474        pr->ractpath.rele[pr->ractpath.rlen - 1].ritemid =  -1;
2475    }
2476    return (lid < pr->rnritems);
2477}
2478
2479
2480static picoos_bool pr_getNextMultiToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
2481{
2482    picoos_int32 len;
2483
2484    len = pr->ractpath.rlen;
2485    if ((len > 0) && (len < PR_MAX_PATH_LEN)) {
2486        pr->ractpath.rele[len].rtok = pr->ractpath.rele[len - 1].rtok;
2487        pr->ractpath.rele[len].ritemid =  -(1);
2488        pr->ractpath.rele[len].rcompare = pr->ractpath.rele[len - 1].rcompare;
2489        pr->ractpath.rele[len].rdepth = pr->ractpath.rele[len - 1].rdepth;
2490        pr->ractpath.rele[len].rlState = PR_LSInit;
2491        pr->ractpath.rlen++;
2492        return TRUE;
2493    } else {
2494        if (len >= PR_MAX_PATH_LEN) {
2495            PICODBG_INFO(("max path len reached (pr_getNextMultiToken)"));
2496        }
2497        return FALSE;
2498    }
2499    return FALSE;
2500}
2501
2502
2503static pr_MatchState pr_matchMultiToken (picodata_ProcessingUnit this, pr_subobj_t * pr,
2504                                         picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2505{
2506    picoos_bool lcontinue=FALSE;
2507    picoos_bool lmatch=FALSE;
2508
2509    if (lmatch) {
2510        return PR_MSMatchedMulti;
2511    } else if (lcontinue) {
2512        return PR_MSMatchedContinue;
2513    } else {
2514        return PR_MSNotMatched;
2515    }
2516    pr = pr;        /* avoid warning "var not used in this function"*/
2517    npset = npset;    /* avoid warning "var not used in this function"*/
2518    wpset = wpset;    /* avoid warning "var not used in this function"*/
2519
2520}
2521
2522
2523static pr_MatchState pr_matchTokensSpace (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
2524                                          picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2525{
2526    register struct pr_PathEle * with__0;
2527    picoos_int32 llen;
2528    picoos_int32 lulen;
2529    picoos_int32 li;
2530    picokpr_VarStrPtr lstrp;
2531    picoos_int32 leol;
2532
2533    with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2534    if ((PR_TSE_MASK_SPACE & npset) == 0) {
2535        return PR_MSNotMatched;
2536    }
2537    lstrp = (picokpr_VarStrPtr)&pr->ritems[with__0->ritemid+1]->data;
2538    lulen = picobase_utf8_length(lstrp,PR_MAX_DATA_LEN);
2539    if (((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen))) {
2540        return PR_MSNotMatched;
2541    }
2542    if (((PR_TSE_MASK_MIN & wpset) != 0) && (lulen < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
2543        return PR_MSNotMatched;
2544    }
2545    if (((PR_TSE_MASK_MAX & wpset) != 0) && (lulen > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
2546        return PR_MSNotMatched;
2547    }
2548    if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
2549        return PR_MSNotMatched;
2550    }
2551    if ((PR_TSE_MASK_VAL & wpset) != 0) {
2552        leol = 0;
2553        llen = pr_strlen(lstrp);
2554        for (li = 0; li < llen; li++) {
2555            if (lstrp[li] == PR_EOL) {
2556                leol++;
2557            }
2558        }
2559        if (leol != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVal)) {
2560            return PR_MSNotMatched;
2561        }
2562    }
2563    if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
2564        return PR_MSNotMatched;
2565    }
2566    return PR_MSMatched;
2567}
2568
2569
2570static pr_MatchState pr_matchTokensDigit (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
2571                                          picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2572{
2573    register struct pr_PathEle * with__0;
2574    picoos_int32 lulen;
2575    picoos_int32 lval;
2576    picokpr_VarStrPtr lstrp;
2577
2578    with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2579    if ((PR_TSE_MASK_DIGIT & npset) == 0) {
2580        return PR_MSNotMatched;
2581    }
2582    lstrp = (picokpr_VarStrPtr)&pr->ritems[with__0->ritemid+1]->data;
2583    lulen = picobase_utf8_length(lstrp,PR_MAX_DATA_LEN);
2584    if ((((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen)))) {
2585        return PR_MSNotMatched;
2586    }
2587    lval = pr->ritems[with__0->ritemid+1]->val;
2588    if (((PR_TSE_MASK_MIN & wpset) != 0) && (lval < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
2589        return PR_MSNotMatched;
2590    }
2591    if (((PR_TSE_MASK_MAX & wpset) != 0) && (lval > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
2592        return PR_MSNotMatched;
2593    }
2594    if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
2595        return PR_MSNotMatched;
2596    }
2597    if (((PR_TSE_MASK_VAL & wpset) != 0) && (lval != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEVal))) {
2598        return PR_MSNotMatched;
2599    }
2600    if ((((PR_TSE_MASK_NLZ & npset) != 0) && lstrp[0] == '0')) {
2601        return PR_MSNotMatched;
2602    }
2603    if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
2604        return PR_MSNotMatched;
2605    }
2606    if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
2607        return PR_MSNotMatched;
2608    }
2609    if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
2610        return PR_MSNotMatched;
2611    }
2612    if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
2613        return PR_MSNotMatched;
2614    }
2615    return PR_MSMatched;
2616}
2617
2618
2619static pr_MatchState pr_matchTokensSeq (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
2620                                        picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2621{
2622
2623    register struct pr_PathEle * with__0;
2624    picoos_int32 lulen;
2625    picokpr_VarStrPtr lstrp;
2626
2627    with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2628
2629    if (!((PR_TSE_MASK_SEQ & npset) != 0)) {
2630        return PR_MSNotMatched;
2631    }
2632    lstrp = (picokpr_VarStrPtr)(void *) &pr->ritems[with__0->ritemid+1]->data;
2633    lulen = picobase_utf8_length(lstrp,PR_MAX_DATA_LEN);
2634    if (((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen))) {
2635        return PR_MSNotMatched;
2636    }
2637    if (((PR_TSE_MASK_MIN & wpset) != 0) && (lulen < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
2638        return PR_MSNotMatched;
2639    }
2640    if (((PR_TSE_MASK_MAX & wpset) != 0) && (lulen > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
2641        return PR_MSNotMatched;
2642    }
2643    if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
2644        return PR_MSNotMatched;
2645    }
2646    if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
2647        return PR_MSNotMatched;
2648    }
2649    if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN ,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
2650        return PR_MSNotMatched;
2651    }
2652    if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,lstrp,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
2653        return PR_MSNotMatched;
2654    }
2655    if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
2656        return PR_MSNotMatched;
2657    }
2658    return PR_MSMatched;
2659}
2660
2661
2662static pr_MatchState pr_matchTokensChar (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
2663                                         picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2664{
2665    register struct pr_PathEle * with__0;
2666
2667    with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2668
2669    if (!((PR_TSE_MASK_CHAR & npset) != 0)) {
2670        return PR_MSNotMatched;
2671    }
2672    if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
2673        return PR_MSNotMatched;
2674    }
2675    if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
2676        return PR_MSNotMatched;
2677    }
2678    return PR_MSMatched;
2679}
2680
2681
2682static pr_MatchState pr_matchTokensLetter (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int32 cmpres,
2683                                           picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2684{
2685
2686    register struct pr_PathEle * with__0;
2687    picoos_int32 lulen;
2688    picoos_int32 lromanval;
2689
2690    with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2691
2692    if ( !((PR_TSE_MASK_LETTER & npset) != 0)) {
2693        return PR_MSNotMatched;
2694    }
2695    lulen = picobase_utf8_length(pr->ritems[with__0->ritemid+1]->data, PR_MAX_DATA_LEN);
2696    if (((PR_TSE_MASK_LEN & wpset) != 0) && (lulen != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSELen))) {
2697        return PR_MSNotMatched;
2698    }
2699    if (((PR_TSE_MASK_MIN & wpset) != 0) && (lulen < pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMin))) {
2700        return PR_MSNotMatched;
2701    }
2702    if (((PR_TSE_MASK_MAX & wpset) != 0) && (lulen > pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMax))) {
2703        return PR_MSNotMatched;
2704    }
2705    if ((PR_TSE_MASK_CI & npset) != 0) {
2706        if (((PR_TSE_MASK_STR & wpset) != 0) && (cmpres != PR_EQUAL)) {
2707            return PR_MSNotMatched;
2708        }
2709        if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strci,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
2710            return PR_MSNotMatched;
2711        }
2712        if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strci,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
2713            return PR_MSNotMatched;
2714        }
2715        if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strci,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
2716            return PR_MSNotMatched;
2717        }
2718    } else if ((PR_TSE_MASK_CIS & npset) != 0) {
2719        if (((PR_TSE_MASK_STR & wpset) != 0) &&  !(picokpr_isEqual(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEStr)))) {
2720            return PR_MSNotMatched;
2721        }
2722        if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
2723            return PR_MSNotMatched;
2724        }
2725        if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
2726            return PR_MSNotMatched;
2727        }
2728        if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->strcis,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
2729            return PR_MSNotMatched;
2730        }
2731    } else {
2732        if (((PR_TSE_MASK_STR & wpset) != 0) &&  !(picokpr_isEqual(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEStr)))) {
2733            return PR_MSNotMatched;
2734        }
2735        if (((PR_TSE_MASK_HEAD & wpset) != 0) &&  !(picokpr_isEqualHead(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEHead)))) {
2736            return PR_MSNotMatched;
2737        }
2738        if (((PR_TSE_MASK_MID & wpset) != 0) &&  !(picokpr_isEqualMid(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEMid)))) {
2739            return PR_MSNotMatched;
2740        }
2741        if (((PR_TSE_MASK_TAIL & wpset) != 0) &&  !(picokpr_isEqualTail(with__0->rnetwork,pr->ritems[with__0->ritemid+1]->data,PR_MAX_DATA_LEN,pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSETail)))) {
2742            return PR_MSNotMatched;
2743        }
2744    }
2745    if (((PR_TSE_MASK_AUC & npset) != 0) &&  !(pr->ritems[with__0->ritemid+1]->auc)) {
2746        return PR_MSNotMatched;
2747    }
2748    if (((PR_TSE_MASK_ALC & npset) != 0) &&  !(pr->ritems[with__0->ritemid+1]->alc)) {
2749        return PR_MSNotMatched;
2750    }
2751    if (((PR_TSE_MASK_SUC & npset) != 0) &&  !(pr->ritems[with__0->ritemid+1]->suc)) {
2752        return PR_MSNotMatched;
2753    }
2754    if (((PR_TSE_MASK_ROMAN & npset) != 0) &&  !(pr_isLatinNumber(pr->ritems[with__0->ritemid+1]->data,& lromanval))) {
2755        return PR_MSNotMatched;
2756    }
2757    if (((PR_TSE_MASK_ID & wpset) != 0) && (pr->ritems[with__0->ritemid+1]->head.info2 != pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEID))) {
2758        return PR_MSNotMatched;
2759    }
2760    return PR_MSMatched;
2761}
2762
2763
2764static pr_MatchState pr_matchTokensBegin (picodata_ProcessingUnit this, pr_subobj_t * pr,
2765                                          picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2766{
2767    npset = npset;        /* avoid warning "var not used in this function"*/
2768    wpset = wpset;        /* avoid warning "var not used in this function"*/
2769    if ((PR_TSE_MASK_BEGIN &picokpr_getTokSetNP(pr->ractpath.rele[pr->ractpath.rlen - 1].rnetwork, pr->ractpath.rele[pr->ractpath.rlen - 1].rtok)) != 0) {
2770        return PR_MSMatched;
2771    } else {
2772        return PR_MSNotMatched;
2773    }
2774}
2775
2776
2777
2778static pr_MatchState pr_matchTokensEnd (picodata_ProcessingUnit this, pr_subobj_t * pr,
2779                                        picokpr_TokSetNP npset, picokpr_TokSetWP wpset)
2780{
2781    npset = npset;        /* avoid warning "var not used in this function"*/
2782    wpset = wpset;        /* avoid warning "var not used in this function"*/
2783    if ((PR_TSE_MASK_END &picokpr_getTokSetNP(pr->ractpath.rele[pr->ractpath.rlen - 1].rnetwork, pr->ractpath.rele[pr->ractpath.rlen - 1].rtok)) != 0) {
2784        return PR_MSMatched;
2785    } else {
2786        return PR_MSNotMatched;
2787    }
2788}
2789
2790
2791static pr_MatchState pr_matchTokens (picodata_ProcessingUnit this, pr_subobj_t * pr, picoos_int16 * cmpres)
2792{
2793
2794    register struct pr_PathEle * with__0;
2795    picokpr_VarStrPtr lstrp;
2796    picokpr_TokSetNP npset;
2797    picokpr_TokSetWP wpset;
2798
2799    with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2800    npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
2801    wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
2802
2803    *cmpres = PR_EQUAL;
2804    if ((PR_TSE_MASK_STR & wpset) != 0) {
2805        lstrp = picokpr_getVarStrPtr(with__0->rnetwork, pr_attrVal(with__0->rnetwork, with__0->rtok, PR_TSEStr));
2806        pr_compare(pr->ritems[with__0->ritemid+1]->strci,lstrp,cmpres);
2807    }
2808    if (((PR_TSE_MASK_LEX & wpset) == PR_TSE_MASK_LEX) && ((PR_TSE_MASK_LETTER & npset) == 0)) {
2809        return pr_matchMultiToken(this, pr, npset, wpset);
2810    } else {
2811        switch (pr->ritems[with__0->ritemid+1]->head.info1) {
2812            case PICODATA_ITEMINFO1_TOKTYPE_BEGIN:
2813                return pr_matchTokensBegin(this, pr, npset, wpset);
2814                break;
2815            case PICODATA_ITEMINFO1_TOKTYPE_END:
2816                return pr_matchTokensEnd(this, pr, npset, wpset);
2817                break;
2818            case PICODATA_ITEMINFO1_TOKTYPE_SPACE:
2819                return pr_matchTokensSpace(this, pr, *cmpres, npset, wpset);
2820                break;
2821            case PICODATA_ITEMINFO1_TOKTYPE_DIGIT:
2822                return pr_matchTokensDigit(this, pr, *cmpres, npset, wpset);
2823                break;
2824            case PICODATA_ITEMINFO1_TOKTYPE_LETTER:
2825                return pr_matchTokensLetter(this, pr, *cmpres, npset, wpset);
2826                break;
2827            case PICODATA_ITEMINFO1_TOKTYPE_SEQ:
2828                return pr_matchTokensSeq(this, pr, *cmpres, npset, wpset);
2829                break;
2830            case PICODATA_ITEMINFO1_TOKTYPE_CHAR:
2831                return pr_matchTokensChar(this, pr, *cmpres, npset, wpset);
2832                break;
2833        default:
2834            PICODBG_INFO(("pr_matchTokens: unknown token type"));
2835            return PR_MSNotMatched;
2836            break;
2837        }
2838    }
2839}
2840
2841
2842static void pr_calcPathCost (struct pr_Path * path)
2843{
2844    picoos_int32 li;
2845    picoos_bool lfirst;
2846    picokpr_TokSetWP wpset;
2847    picokpr_TokSetNP npset;
2848#if PR_TRACE_PATHCOST
2849    picoos_uchar str[1000];
2850    picoos_uchar * strp;
2851#endif
2852
2853#if PR_TRACE_PATHCOST
2854    str[0] = 0;
2855#endif
2856
2857    lfirst = TRUE;
2858    path->rcost = PR_COST_INIT;
2859    for (li = 0; li < path->rlen; li++) {
2860        if (li == 0) {
2861            path->rcost = path->rcost + path->rele[li].rprodprefcost;
2862        }
2863        wpset = picokpr_getTokSetWP(path->rele[li].rnetwork, path->rele[li].rtok);
2864        npset = picokpr_getTokSetNP(path->rele[li].rnetwork, path->rele[li].rtok);
2865        if ((PR_TSE_MASK_COST & wpset) != 0) {
2866            if (((PR_TSE_MASK_LEX & wpset) == PR_TSE_MASK_LEX) && ((PR_TSE_MASK_LETTER & npset) == 0)) {
2867                if (lfirst) {
2868                    path->rcost = path->rcost - PR_COST + pr_attrVal(path->rele[li].rnetwork, path->rele[li].rtok, PR_TSECost);
2869                } else {
2870                    path->rcost = path->rcost - PR_COST;
2871                }
2872                lfirst = FALSE;
2873            } else {
2874                path->rcost = path->rcost - PR_COST + pr_attrVal(path->rele[li].rnetwork, path->rele[li].rtok, PR_TSECost);
2875                lfirst = TRUE;
2876            }
2877        } else if (pr_hasToken(& wpset,& npset)) {
2878            path->rcost = path->rcost - PR_COST;
2879        }
2880#if PR_TRACE_PATHCOST
2881        if ((path->rele[li].rprodname != 0)) {
2882            strp = picokpr_getVarStrPtr(path->rele[li].rnetwork, path->rele[li].rprodname);
2883            picoos_strcat(str, (picoos_char *)" ");
2884            picoos_strcat(str, strp);
2885        }
2886#endif
2887    }
2888#if PR_TRACE_PATHCOST
2889    PICODBG_INFO(("pp cost: %i %s", path->rcost, str));
2890#endif
2891}
2892
2893
2894void pr_processToken (picodata_ProcessingUnit this, pr_subobj_t * pr)
2895{
2896    register struct pr_PathEle * with__0;
2897    picoos_bool ldummy;
2898    picoos_int32 li;
2899    picokpr_TokSetNP npset;
2900    picokpr_TokSetWP wpset;
2901
2902    do {
2903        pr->rgState = PR_GSContinue;
2904        if ((pr->ractpath.rlen == 0)) {
2905            if (pr_getTopLevelToken(this, pr, FALSE)) {
2906                pr->rgState = PR_GSContinue;
2907            } else if (pr->rbestpath.rlen == 0) {
2908                pr->rgState = PR_GSNotFound;
2909            } else {
2910                pr->rgState = PR_GSFound;
2911            }
2912        } else {
2913            if (pr->maxPathLen < pr->ractpath.rlen) {
2914                pr->maxPathLen = pr->ractpath.rlen;
2915            }
2916            with__0 = & pr->ractpath.rele[pr->ractpath.rlen - 1];
2917            switch (with__0->rlState) {
2918                case PR_LSInit:
2919                    npset = picokpr_getTokSetNP(with__0->rnetwork, with__0->rtok);
2920                    wpset = picokpr_getTokSetWP(with__0->rnetwork, with__0->rtok);
2921                    if ((PR_TSE_MASK_ACCEPT & npset) != 0){
2922                        if (with__0->rdepth == 1) {
2923                            pr_calcPathCost(&pr->ractpath);
2924                            if ((pr->rbestpath.rlen == 0) || (pr->ractpath.rcost < pr->rbestpath.rcost)) {
2925                                pr->rbestpath.rlen = pr->ractpath.rlen;
2926                                pr->rbestpath.rcost = pr->ractpath.rcost;
2927                                for (li = 0; li < pr->ractpath.rlen; li++) {
2928                                    pr->rbestpath.rele[li] = pr->ractpath.rele[li];
2929                                }
2930                            }
2931                            with__0->rlState = PR_LSGetNextToken;
2932                        } else {
2933                            with__0->rlState = PR_LSGetProdContToken;
2934                        }
2935                    } else if ((PR_TSE_MASK_PROD & wpset) != 0) {
2936                        with__0->rlState = PR_LSGetProdToken;
2937                    } else if ((PR_TSE_MASK_OUT & wpset) != 0) {
2938                        with__0->rlState = PR_LSGetNextToken;
2939                    } else if (pr_hasToken(& wpset,& npset)) {
2940                        with__0->rlState = PR_LSGetToken;
2941                    } else {
2942                        with__0->rlState = PR_LSGetNextToken;
2943                    }
2944                    break;
2945                case PR_LSGetProdToken:
2946                    with__0->rlState = PR_LSGetAltToken;
2947                    ldummy = pr_getProdToken(this, pr);
2948                    break;
2949                case PR_LSGetProdContToken:
2950                    with__0->rlState = PR_LSGetAltToken;
2951                    ldummy = pr_getProdContToken(this, pr);
2952                    break;
2953                case PR_LSGoBack:
2954                    pr->ractpath.rlen--;
2955                    break;
2956                case PR_LSGetToken:
2957                    if (pr_getToken(this, pr)) {
2958                        with__0->rlState = PR_LSMatch;
2959                    } else if (pr->forceOutput) {
2960                        with__0->rlState = PR_LSGetAltToken;
2961                    } else {
2962                        with__0->rlState = PR_LSGetToken2;
2963                        pr->rgState = PR_GSNeedToken;
2964                    }
2965                    break;
2966                case PR_LSGetToken2:
2967                    if (pr_getToken(this, pr)) {
2968                        with__0->rlState = PR_LSMatch;
2969                    } else {
2970                        with__0->rlState = PR_LSGoBack;
2971                    }
2972                    break;
2973                case PR_LSMatch:
2974                    switch (pr_matchTokens(this, pr, & with__0->rcompare)) {
2975                        case PR_MSMatched:
2976                            with__0->rlState = PR_LSGetNextToken;
2977                            break;
2978                        case PR_MSMatchedContinue:
2979                            with__0->rlState = PR_LSGetAltToken;
2980                            ldummy = pr_getNextMultiToken(this, pr);
2981                            break;
2982                        case PR_MSMatchedMulti:
2983                            with__0->rlState = PR_LSGetNextToken;
2984                            ldummy = pr_getNextMultiToken(this, pr);
2985                            break;
2986                    default:
2987                        with__0->rlState = PR_LSGetAltToken;
2988                        break;
2989                    }
2990                    break;
2991                case PR_LSGetNextToken:
2992                    with__0->rlState = PR_LSGetAltToken;
2993                    ldummy = pr_getNextToken(this, pr);
2994                    break;
2995                case PR_LSGetAltToken:
2996                    with__0->rlState = PR_LSGoBack;
2997                    ldummy = pr_getAltToken(this, pr);
2998                    break;
2999            default:
3000                PICODBG_INFO(("unhandled local state"));
3001                break;
3002            }
3003        }
3004        pr->nrIterations--;
3005    } while ((pr->rgState == PR_GSContinue) && (pr->nrIterations > 0));
3006}
3007
3008
3009void pr_process (picodata_ProcessingUnit this, pr_subobj_t * pr)
3010{
3011    switch (pr->rgState) {
3012        case PR_GS_START:
3013        case PR_GSFound:
3014        case PR_GSNotFound:
3015            pr->ractpath.rlen = 0;
3016            pr->ractpath.rcost = PR_COST_INIT;
3017            pr->rbestpath.rlen = 0;
3018            pr->rbestpath.rcost = PR_COST_INIT;
3019            if (pr_getTopLevelToken(this, pr, TRUE)) {
3020                pr->rgState = PR_GSContinue;
3021            } else {
3022                pr->rgState = PR_GSNotFound;
3023            }
3024            break;
3025        case PR_GSContinue:
3026            pr_processToken(this, pr);
3027            break;
3028        case PR_GSNeedToken:
3029            pr->rgState = PR_GSContinue;
3030            break;
3031    default:
3032        pr->rgState = PR_GS_START;
3033        break;
3034    }
3035}
3036
3037
3038static void pr_prepareItem (picodata_ProcessingUnit this, pr_subobj_t * pr, pr_ioItemPtr item)
3039{
3040    pr->ritems[pr->rnritems + 1] = item;
3041    pr->rnritems++;
3042}
3043
3044
3045static void pr_processItems (picodata_ProcessingUnit this, pr_subobj_t * pr)
3046{
3047    pr_ioItemPtr lit;
3048    pr_MemState lmemState;
3049
3050    pr_getMemState(this, pr_WorkMem,& lmemState);
3051
3052    while ((pr->rinItemList != NULL) && (pr->rinItemList->head.type != PICODATA_ITEM_TOKEN)) {
3053        lit = pr->rinItemList;
3054        PICODBG_INFO(("pp in (0)"));
3055        PICODBG_INFO(("pp out(0)"));
3056        pr->rinItemList = pr->rinItemList->next;
3057        lit->next = NULL;
3058        if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
3059            pr->insidePhoneme = TRUE;
3060        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
3061            pr->insidePhoneme = FALSE;
3062        }
3063        if (pr->insidePhoneme && (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) || pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG))) {
3064            pr_disposeItem(this, & lit);
3065        } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_CONTEXT)) {
3066            pr_setContext(this, pr, lit->data);
3067            pr_disposeItem(this, & lit);
3068        } else if (pr->rignore <= 0) {
3069            pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,lit);
3070            if (pr->outOfMemory) return;
3071        } else {
3072            pr_disposeItem(this, & lit);
3073        }
3074        pr->rgState = PR_GS_START;
3075    }
3076    if (pr->rinItemList != NULL) {
3077        pr_process(this, pr);
3078        if (pr->rgState == PR_GSNotFound) {
3079            lit = pr->rinItemList;
3080            pr->rinItemList = pr->rinItemList->next;
3081            lit->next = NULL;
3082            PICODBG_INFO(("pp in (2): '%s'", lit->data));
3083            if (pr->rignore <= 0) {
3084                PICODBG_INFO(("pp out(2): '%s'", lit->data));
3085            }
3086
3087            if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_START)) {
3088                pr->insidePhoneme = TRUE;
3089            } else if (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PHONEME) && pr_isCmdInfo2(lit, PICODATA_ITEMINFO2_CMD_END)) {
3090                pr->insidePhoneme = FALSE;
3091            }
3092            if (((pr->rignore <= 0) &&  !((pr->insidePhoneme && (pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_PLAY) || pr_isCmdType(lit,PICODATA_ITEMINFO1_CMD_IGNSIG)))))) {
3093                pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,lit);
3094                if (pr->outOfMemory) return;
3095            } else {
3096                pr_disposeItem(this, & lit);
3097            }
3098            pr->rgState = PR_GS_START;
3099            pr->rnritems = 0;
3100        } else if (pr->rgState == PR_GSFound) {
3101            pr_outputPath(this, pr);
3102            if (pr->outOfMemory) return;
3103            pr->rgState = PR_GS_START;
3104            pr->rnritems = 0;
3105        }
3106    }
3107    if (pr->rinItemList == NULL) {
3108        pr->rlastInItem = NULL;
3109    } else if (pr->rnritems == 0) {
3110        lit = pr->rinItemList;
3111        while (lit != NULL) {
3112            if (lit->head.type == PICODATA_ITEM_TOKEN) {
3113                pr_prepareItem(this, pr, lit);
3114            }
3115            lit = lit->next;
3116        }
3117    }
3118    pr_resetMemState(this, pr_WorkMem,lmemState);
3119}
3120
3121
3122
3123extern void pr_treatItem (picodata_ProcessingUnit this, pr_subobj_t * pr, pr_ioItemPtr item)
3124{
3125    pr_ioItemPtr lit;
3126
3127    pr_startItemList(& pr->routItemList,& pr->rlastOutItem);
3128
3129    if (!PR_ENABLED || (pr->rgState == PR_GSNoPreproc)) {
3130        /* preprocessing disabled or no preproc networks available:
3131           append items directly to output item list */
3132        PICODBG_INFO(("pp in (3): '%s'", item->data));
3133        PICODBG_INFO(("pp out(3): '%s'", item->data));
3134        pr_appendItemToOutItemList(this, pr, & pr->routItemList,& pr->rlastOutItem,item);
3135    } else {
3136
3137        if (pr->actCtxChanged) {
3138            pr->rgState = PR_GS_START;
3139            pr->ractpath.rcost = PR_COST_INIT;
3140            pr->ractpath.rlen = 0;
3141            pr->rbestpath.rcost = PR_COST_INIT;
3142            pr->rbestpath.rlen = 0;
3143            pr->prodList = NULL;
3144            pr->rnritems = 0;
3145            pr->actCtxChanged = FALSE;
3146        }
3147        if (pr_isCmdType(item , PICODATA_ITEMINFO1_CMD_CONTEXT) || pr_isCmdType(item, PICODATA_ITEMINFO1_CMD_FLUSH)) {
3148            /* context switch or flush: force processing and empty input item list */
3149            pr->forceOutput = TRUE;
3150        }
3151        pr_appendItem(this, & pr->rinItemList,& pr->rlastInItem, item);
3152        if (pr->rnritems == 0) {
3153            lit = pr->rinItemList;
3154            while (lit != NULL) {
3155                if (lit->head.type == PICODATA_ITEM_TOKEN) {
3156                    pr_prepareItem(this, pr, lit);
3157                }
3158                lit = lit->next;
3159            }
3160        } else if (item->head.type == PICODATA_ITEM_TOKEN) {
3161            pr_prepareItem(this, pr, item);
3162        }
3163    }
3164}
3165
3166/* *****************************************************************************/
3167/* *****************************************************************************/
3168/* *****************************************************************************/
3169
3170
3171pico_status_t prReset(register picodata_ProcessingUnit this, picoos_int32 resetMode)
3172{
3173
3174    picoos_int32 i;
3175    pr_subobj_t * pr;
3176
3177    if (NULL == this || NULL == this->subObj) {
3178        return PICO_ERR_OTHER;
3179    }
3180    pr = (pr_subobj_t *) this->subObj;
3181
3182    pr->rinItemList = NULL;
3183    pr->rlastInItem = NULL;
3184    pr->routItemList = NULL;
3185    pr->rlastOutItem = NULL;
3186    pr->ractpath.rcost = PR_COST_INIT;
3187    pr->ractpath.rlen = 0;
3188    pr->rbestpath.rcost = PR_COST_INIT;
3189    pr->rbestpath.rlen = 0;
3190    pr->rnritems = 0;
3191    pr->ritems[0] = NULL;
3192    pr->rignore = 0;
3193    pr->spellMode = 0;
3194    pr->maxPathLen = 0;
3195    pr->insidePhoneme = FALSE;
3196    pr->saveFile[0] = 0;
3197
3198    pr->outReadPos = 0;
3199    pr->outWritePos = 0;
3200    pr->inBufLen = 0;
3201
3202    pr->rgState = PR_GSNoPreproc;
3203    for (i=0; i<PR_MAX_NR_PREPROC; i++) {
3204        if (pr->preproc[i] != NULL) {
3205            pr->rgState = PR_GS_START;
3206        }
3207    }
3208    pr->actCtx = pr_findContext(pr->ctxList, (picoos_uchar*)PICO_CONTEXT_DEFAULT);
3209    pr->actCtxChanged = FALSE;
3210    pr->prodList = NULL;
3211
3212    if (((uintptr_t)pr->pr_WorkMem % PICOOS_ALIGN_SIZE) == 0) {
3213        pr->workMemTop = 0;
3214    }
3215    else {
3216        pr->workMemTop = PICOOS_ALIGN_SIZE - ((uintptr_t)pr->pr_WorkMem % PICOOS_ALIGN_SIZE);
3217    }
3218    pr->maxWorkMemTop=0;
3219    pr->dynMemSize=0;
3220    pr->maxDynMemSize=0;
3221    /* this is ok to be in 'initialize' because it is a private memory within pr. Creating a new mm
3222     * here amounts to resetting this internal memory
3223     */
3224    pr->dynMemMM = picoos_newMemoryManager((void *)pr->pr_DynMem, PR_DYN_MEM_SIZE,
3225            /*enableMemProt*/ FALSE);
3226    pr->outOfMemory = FALSE;
3227
3228    pr->forceOutput = FALSE;
3229
3230    if (resetMode == PICO_RESET_SOFT) {
3231        /*following initializations needed only at startup or after a full reset*/
3232        return PICO_OK;
3233    }
3234
3235    pr->xsampa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA_PARSE]);
3236
3237    pr->svoxpa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_SVOXPA_PARSE]);
3238
3239    pr->xsampa2svoxpa_mapper = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA2SVOXPA]);
3240
3241
3242
3243    return PICO_OK;
3244}
3245
3246
3247pico_status_t prInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode)
3248{
3249/*
3250    if (NULL == this || NULL == this->subObj) {
3251        return PICO_ERR_OTHER;
3252    }
3253*/
3254    return prReset(this, resetMode);
3255}
3256
3257
3258pico_status_t prTerminate(register picodata_ProcessingUnit this)
3259{
3260    return PICO_OK;
3261}
3262
3263picodata_step_result_t prStep(register picodata_ProcessingUnit this, picoos_int16 mode, picoos_uint16 * numBytesOutput);
3264
3265pico_status_t prSubObjDeallocate(register picodata_ProcessingUnit this,
3266        picoos_MemoryManager mm)
3267{
3268    pr_subobj_t * pr;
3269
3270    if (NULL != this) {
3271        pr = (pr_subobj_t *) this->subObj;
3272        mm = mm;        /* avoid warning "var not used in this function"*/
3273        PICODBG_INFO(("max pr_WorkMem: %i of %i", pr->maxWorkMemTop, PR_WORK_MEM_SIZE));
3274        PICODBG_INFO(("max pr_DynMem: %i of %i", pr->maxDynMemSize, PR_DYN_MEM_SIZE));
3275
3276        pr_disposeContextList(this);
3277        picoos_deallocate(this->common->mm, (void *) &this->subObj);
3278    }
3279    return PICO_OK;
3280}
3281
3282picodata_ProcessingUnit picopr_newPreprocUnit(picoos_MemoryManager mm, picoos_Common common,
3283        picodata_CharBuffer cbIn, picodata_CharBuffer cbOut,
3284        picorsrc_Voice voice)
3285{
3286    picoos_int32 i;
3287    pr_subobj_t * pr;
3288
3289
3290    picodata_ProcessingUnit this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice);
3291    if (this == NULL) {
3292        return NULL;
3293    }
3294
3295    this->initialize = prInitialize;
3296    PICODBG_DEBUG(("set this->step to prStep"));
3297    this->step = prStep;
3298    this->terminate = prTerminate;
3299    this->subDeallocate = prSubObjDeallocate;
3300    this->subObj = picoos_allocate(mm, sizeof(pr_subobj_t));
3301#if PR_TRACE_MEM || PR_TRACE_MAX_MEM
3302    PICODBG_INFO(("preproc alloc: %i", sizeof(pr_subobj_t)));
3303    PICODBG_INFO(("max dyn size: %i", PR_MAX_PATH_LEN*((((PR_IOITEM_MIN_SIZE+2) + PICOOS_ALIGN_SIZE - 1) / PICOOS_ALIGN_SIZE) * PICOOS_ALIGN_SIZE + 16)));
3304#endif
3305    if (this->subObj == NULL) {
3306        picoos_deallocate(mm, (void *)&this);
3307        return NULL;
3308    }
3309    pr = (pr_subobj_t *) this->subObj;
3310
3311    pr->graphs = picoktab_getGraphs(this->voice->kbArray[PICOKNOW_KBID_TAB_GRAPHS]);
3312    pr->preproc[0] = picokpr_getPreproc(this->voice->kbArray[PICOKNOW_KBID_TPP_MAIN]);
3313    for (i=0; i<PICOKNOW_MAX_NUM_UTPP; i++) {
3314      pr->preproc[1+i] = picokpr_getPreproc(this->voice->kbArray[PICOKNOW_KBID_TPP_USER_1+i]);
3315    }
3316
3317   if (pr_createContextList(this) != PICO_OK) {
3318        pr_disposeContextList(this);
3319        picoos_deallocate(mm, (void *)&this);
3320        return NULL;
3321    }
3322    prInitialize(this, PICO_RESET_FULL);
3323    return this;
3324}
3325
3326/**
3327 * fill up internal buffer
3328 */
3329picodata_step_result_t prStep(register picodata_ProcessingUnit this,
3330        picoos_int16 mode, picoos_uint16 * numBytesOutput)
3331{
3332    register pr_subobj_t * pr;
3333    pr_ioItemPtr it;
3334    picoos_int32 len, i;
3335    pico_status_t rv;
3336    picoos_int32 id;
3337    picoos_uint8 info1;
3338    picoos_uint8 info2;
3339    picoos_int32 nrUtfChars;
3340    picoos_uint32 pos;
3341    picobase_utf8char inUtf8char, outUtf8char;
3342    picoos_int32 inUtf8charlen, outUtf8charlen;
3343    picoos_int32 lenpos;
3344    picoos_bool ldone;
3345    picoos_bool split;
3346
3347    if (NULL == this || NULL == this->subObj) {
3348        return PICODATA_PU_ERROR;
3349    }
3350    pr = (pr_subobj_t *) this->subObj;
3351
3352    if (pr->outOfMemory) return PICODATA_PU_ERROR;
3353
3354    mode = mode;        /* avoid warning "var not used in this function"*/
3355    pr->nrIterations = PR_MAX_NR_ITERATIONS;
3356
3357    *numBytesOutput = 0;
3358    while (1) { /* exit via return */
3359        if ((pr->outWritePos - pr->outReadPos) > 0) {
3360            /* deliver the data in the output buffer */
3361            if (picodata_cbPutItem(this->cbOut, &pr->outBuf[pr->outReadPos], pr->outWritePos - pr->outReadPos, numBytesOutput) == PICO_OK) {
3362                pr->outReadPos += *numBytesOutput;
3363                if (pr->outWritePos == pr->outReadPos) {
3364                    pr->outWritePos = 0;
3365                    pr->outReadPos = 0;
3366                }
3367            }
3368            else {
3369                return PICODATA_PU_OUT_FULL;
3370            }
3371        }
3372        else if (pr->routItemList != NULL) {
3373            /* there are item(s) in the output item list, move them to the output buffer */
3374            it = pr->routItemList;
3375            pr->routItemList = pr->routItemList->next;
3376            if (pr->routItemList == NULL) {
3377                pr->rlastOutItem = NULL;
3378            }
3379            if (it->head.type == PICODATA_ITEM_TOKEN) {
3380                if ((it->head.info1 != PICODATA_ITEMINFO1_TOKTYPE_SPACE) && (it->head.len > 0)) {
3381                    nrUtfChars = picobase_utf8_length(it->data, PR_MAX_DATA_LEN);
3382                    if ((nrUtfChars == 1)
3383                        && (((id = picoktab_graphOffset(pr->graphs, it->data)) > 0))
3384                        && picoktab_getIntPropPunct(pr->graphs, id, &info1, &info2)) {
3385                        /* single punctuation chars have to be delivered as PICODATA_ITEM_PUNC items
3386                           instead as PICODATA_ITEM_WORDGRAPH items */
3387                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_PUNC;
3388                        pr->outBuf[pr->outWritePos++] = info1;
3389                        pr->outBuf[pr->outWritePos++] = info2;
3390                        pr->outBuf[pr->outWritePos++] = 0;
3391                        PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
3392                            (picoos_uint8 *)"pr: ", pr->outBuf, pr->outWritePos);
3393                    }
3394                    else {
3395                        /* do subgraphs substitutions and deliver token items as PICODATA_ITEM_WORDGRAPH
3396                           items to the output buffer */
3397                        split = FALSE;
3398                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_WORDGRAPH;
3399                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO1_NA;
3400                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO2_NA;
3401                        lenpos=pr->outWritePos;
3402                        pr->outBuf[pr->outWritePos++] = 0;
3403                        pos = 0;
3404                        len = pr_strlen(it->data);
3405                        while (pos < (picoos_uint32)len) {
3406                            if (picobase_get_next_utf8char(it->data, it->head.len, &pos, inUtf8char)) {
3407                                if (inUtf8char[0] <= 32) {
3408                                    /* do not add whitespace characters to the output buffer,
3409                                       but initiate token splitting instead
3410
3411                                    */
3412                                    split = TRUE;
3413                                }
3414                                else if (((id = picoktab_graphOffset(pr->graphs, inUtf8char)) > 0) && picoktab_getStrPropGraphsubs1(pr->graphs, id, outUtf8char)) {
3415                                    if (split) {
3416                                        /* split the token, eg. start a new item */
3417                                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_WORDGRAPH;
3418                                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO1_NA;
3419                                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO2_NA;
3420                                        lenpos=pr->outWritePos;
3421                                        pr->outBuf[pr->outWritePos++] = 0;
3422                                    }
3423                                    outUtf8charlen = picobase_det_utf8_length(outUtf8char[0]);
3424                                    for (i=0; i<outUtf8charlen; i++) {
3425                                        pr->outBuf[pr->outWritePos++] = outUtf8char[i];
3426                                        pr->outBuf[lenpos]++;
3427                                    }
3428                                    if (picoktab_getStrPropGraphsubs2(pr->graphs, id, outUtf8char)) {
3429                                        outUtf8charlen = picobase_det_utf8_length(outUtf8char[0]);
3430                                        for (i=0; i<outUtf8charlen; i++) {
3431                                            pr->outBuf[pr->outWritePos++] = outUtf8char[i];
3432                                            pr->outBuf[lenpos]++;
3433                                        }
3434                                    }
3435                                    split = FALSE;
3436                                }
3437                                else {
3438                                    if (split) {
3439                                        /* split the token, eg. start a new item */
3440                                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEM_WORDGRAPH;
3441                                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO1_NA;
3442                                        pr->outBuf[pr->outWritePos++] = PICODATA_ITEMINFO2_NA;
3443                                        lenpos=pr->outWritePos;
3444                                        pr->outBuf[pr->outWritePos++] = 0;
3445                                    }
3446                                    inUtf8charlen = picobase_det_utf8_length(inUtf8char[0]);
3447                                    for (i=0; i<inUtf8charlen; i++) {
3448                                        pr->outBuf[pr->outWritePos++] = inUtf8char[i];
3449                                        pr->outBuf[lenpos]++;
3450                                    }
3451                                    split = FALSE;
3452                                }
3453                            }
3454                        }
3455                        PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
3456                            (picoos_uint8 *)"pr: ", pr->outBuf, pr->outWritePos);
3457                    }
3458                }
3459            }
3460            else {
3461                /* handle all other item types and put them to the output buffer */
3462                pr->outBuf[pr->outWritePos++] = it->head.type;
3463                pr->outBuf[pr->outWritePos++] = it->head.info1;
3464                pr->outBuf[pr->outWritePos++] = it->head.info2;
3465                pr->outBuf[pr->outWritePos++] = it->head.len;
3466                for (i=0; i<it->head.len; i++) {
3467                    pr->outBuf[pr->outWritePos++] = it->data[i];
3468                }
3469                PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
3470                                   (picoos_uint8 *)"pr: ", pr->outBuf, pr->outWritePos);
3471            }
3472            pr_disposeItem(this, &it);
3473        }
3474        else if (pr->forceOutput) {
3475            pr_processItems(this, pr);
3476            if (pr->rinItemList == NULL) {
3477                pr->forceOutput = FALSE;
3478            }
3479        }
3480        else if ((pr->rgState != PR_GSNeedToken) && (pr->rinItemList != NULL)) {
3481            pr_processItems(this, pr);
3482        }
3483        else if (pr->inBufLen > 0) {
3484            /* input data is available in the input buffer, copy it to an input item
3485               and treat it */
3486            if (pr->dynMemSize < (45*PR_DYN_MEM_SIZE / 100)) {
3487                pr_newItem(this, pr_DynMem, &it, pr->inBuf[0], pr->inBuf[3], /*inItem*/TRUE);
3488                if (pr->outOfMemory) return PICODATA_PU_ERROR;
3489                it->head.type = pr->inBuf[0];
3490                it->head.info1 = pr->inBuf[1];
3491                it->head.info2 = pr->inBuf[2];
3492                it->head.len = pr->inBuf[3];
3493                for (i=0; i<pr->inBuf[3]; i++) {
3494                    it->data[i] = pr->inBuf[4+i];
3495                }
3496                it->data[pr->inBuf[3]] = 0;
3497                if ((pr->inBuf[0] == PICODATA_ITEM_TOKEN) && ((pr->inBuf[1] == PICODATA_ITEMINFO1_TOKTYPE_DIGIT))) {
3498                    it->val = tok_tokenDigitStrToInt(this, pr, it->data);
3499                } else {
3500                    it->val = 0;
3501                }
3502                if (pr->inBuf[0] == PICODATA_ITEM_TOKEN) {
3503                    picobase_lowercase_utf8_str(it->data,it->strci,PR_MAX_DATA_LEN, &ldone);
3504                    pr_firstLetterToLowerCase(it->data,it->strcis);
3505                    it->alc = picobase_is_utf8_lowercase(it->data,PR_MAX_DATA_LEN);
3506                    it->auc = picobase_is_utf8_uppercase(it->data,PR_MAX_DATA_LEN);
3507                    it->suc = pr_isSUC(it->data);
3508                }
3509
3510                pr_treatItem(this, pr, it);
3511                if (pr->outOfMemory) return PICODATA_PU_ERROR;
3512                pr_processItems(this, pr);
3513                pr->inBufLen = 0;
3514            }
3515            else {
3516                pr->forceOutput = TRUE;
3517            }
3518        }
3519        else {
3520            /* there is not data in the output buffer and there is no data in the output item list, so
3521               check whether input data is available */
3522            rv = picodata_cbGetItem(this->cbIn, pr->inBuf, IN_BUF_SIZE+PICODATA_ITEM_HEADSIZE, &pr->inBufLen);
3523            if (PICO_OK == rv) {
3524            } else if (PICO_EOF == rv) {
3525                /* there was no item in the char buffer */
3526                return PICODATA_PU_IDLE;
3527            } else if ((PICO_EXC_BUF_UNDERFLOW == rv) || (PICO_EXC_BUF_OVERFLOW == rv)) {
3528                pr->inBufLen = 0;
3529                PICODBG_ERROR(("problem getting item"));
3530                picoos_emRaiseException(this->common->em, rv, NULL, NULL);
3531                return PICODATA_PU_ERROR;
3532            } else {
3533                pr->inBufLen = 0;
3534                PICODBG_ERROR(("problem getting item, unhandled"));
3535                picoos_emRaiseException(this->common->em, rv, NULL, NULL);
3536                return PICODATA_PU_ERROR;
3537            }
3538        }
3539#if PR_TRACE_MEM
3540        PICODBG_INFO(("memory: dyn=%u, work=%u", pr->dynMemSize, pr->workMemTop));
3541#endif
3542        if (pr->nrIterations <= 0) {
3543            return PICODATA_PU_BUSY;
3544        }
3545    } /* while */
3546    return PICODATA_PU_ERROR;
3547}
3548
3549#ifdef __cplusplus
3550}
3551#endif
3552
3553
3554
3555/* end */
3556