1b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* 2b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland 3b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * 4b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * Licensed under the Apache License, Version 2.0 (the "License"); 5b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * you may not use this file except in compliance with the License. 6b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * You may obtain a copy of the License at 7b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * 8b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * http://www.apache.org/licenses/LICENSE-2.0 9b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * 10b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * Unless required by applicable law or agreed to in writing, software 11b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * distributed under the License is distributed on an "AS IS" BASIS, 12b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * See the License for the specific language governing permissions and 14b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * limitations under the License. 15b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen */ 16b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/** 17b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * @file picotok.c 18b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * 19b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * tokenizer 20b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * 21b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland 22b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * All rights reserved. 23b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * 24b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * History: 25b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * - 2009-04-20 -- initial version 26b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * 27b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen */ 28b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 29b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 30b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* ************************************************************/ 31b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* tokenisation and markup handling */ 32b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* ************************************************************/ 33b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 34b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/** @addtogroup picotok 35b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen @b tokenisation_overview 36b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 37b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen markup handling overview: 38b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 39b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen The following markups are recognized 40b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - ignore 41b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - speed 42b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - pitch 43b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - volume 44b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - voice 45b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - preproccontext 46b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - mark 47b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - play 48b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - usesig 49b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - genfile 50b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - sentence 51b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - s 52b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - paragraph 53b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - p 54b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - break 55b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - spell (pauses between letter) 56b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen - phoneme 57b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 58b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen All markups which are recognized but are not yet implemented in pico 59b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen system have the mark. 60b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen*/ 61b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 62b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 63b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#include "picodefs.h" 64b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#include "picoos.h" 65b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#include "picobase.h" 66b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#include "picodbg.h" 67b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#include "picodata.h" 68b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#include "picotok.h" 69b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#include "picoktab.h" 70b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 71b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#ifdef __cplusplus 72b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenextern "C" { 73b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#endif 74b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#if 0 75b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 76b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#endif 77b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 78b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* *****************************************************************************/ 79b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 80b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define IN_BUF_SIZE 255 81b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define OUT_BUF_SIZE IN_BUF_SIZE + 3 * PICODATA_ITEM_HEADSIZE + 3 82b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 83b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define MARKUP_STRING_BUF_SIZE (IN_BUF_SIZE*5) 84b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define MAX_NR_MARKUP_PARAMS 6 85b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define MARKUP_HANDLING_DISABLED 0 86b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define MARKUP_HANDLING_ENABLED 1 87b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define EOL '\n' 88b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 89b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 90b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef picoos_int8 pico_tokenSubType; 91b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef picoos_uint8 pico_tokenType; 92b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 93b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/** @todo : consider adding these specialized exception codes: */ 94b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 95b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE PICO_ERR_OTHER 96b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_ERR_INVALID_MARKUP_TAG PICO_ERR_OTHER 97b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_ERR_INTERNAL_LIMIT PICO_ERR_OTHER 98b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 99b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef enum {MIDummyStart, MIIgnore, 100b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MIPitch, MISpeed, MIVolume, 101b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MIVoice, MIPreprocContext, MIMarker, 102b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MIPlay, MIUseSig, MIGenFile, MIParagraph, 103b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MISentence, MIBreak, MISpell, MIPhoneme, MIItem, MISpeaker, MIDummyEnd 104b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } MarkupId; 105b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef enum {MSNotInMarkup, MSGotStart, MSExpectingmarkupTagName, MSInmarkupTagName, 106b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MSGotmarkupTagName, MSInAttrName, MSGotAttrName, MSGotEqual, MSInAttrValue, 107b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MSInAttrValueEscaped, MSGotAttrValue, MSGotEndSlash, MSGotEnd, 108b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MSError, MSErrorTooLong, MSErrorSyntax 109b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } MarkupState; 110b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef enum {MENone, MEMissingStart, MEUnknownTag, MEIdent, MEMissingEqual, 111b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MEMissingQuote, MEMissingEnd, MEUnexpectedChar, MEInterprete 112b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } MarkupParseError; 113b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 114b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef enum {MTNone, MTStart, MTEnd, MTEmpty} MarkupTagType; 115b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 116b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define UTF_CHAR_COMPLETE 2 117b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define UTF_CHAR_INCOMPLETE 1 118b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define UTF_CHAR_MALFORMED 0 119b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 120b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_IGNORE (picoos_uchar*)"ignore" 121b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_SPEED (picoos_uchar*)"speed" 122b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_PITCH (picoos_uchar*)"pitch" 123b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_VOLUME (picoos_uchar*)"volume" 124b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_VOICE (picoos_uchar*)"voice" 125b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_CONTEXT (picoos_uchar*)"preproccontext" 126b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_MARK (picoos_uchar*)"mark" 127b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_PLAY (picoos_uchar*)"play" 128b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_USESIG (picoos_uchar*)"usesig" 129b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_GENFILE (picoos_uchar*)"genfile" 130b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_SENTENCE (picoos_uchar*)"sentence" 131b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_S (picoos_uchar*)"s" 132b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_PARAGRAPH (picoos_uchar*)"paragraph" 133b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_P (picoos_uchar*)"p" 134b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_BREAK (picoos_uchar*)"break" 135b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_SPELL (picoos_uchar*)"spell" 136b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_PHONEME (picoos_uchar*)"phoneme" 137b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_ITEM (picoos_uchar*)"item" 138b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_MARKUP_KW_SPEAKER (picoos_uchar*)"speaker" 139b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 140b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWLevel (picoos_uchar *)"level" 141b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWName (picoos_uchar *)"name" 142b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWProsDomain (picoos_uchar *)"prosodydomain" 143b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWTime (picoos_uchar *)"time" 144b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWMode (picoos_uchar *)"mode" 145b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWSB (picoos_uchar *)"sb" 146b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWPB (picoos_uchar *)"pb" 147b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWFile (picoos_uchar *)"file" 148b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWType (picoos_uchar *)"type" 149b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWF0Beg (picoos_uchar *)"f0beg" 150b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWF0End (picoos_uchar *)"f0end" 151b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWXFadeBeg (picoos_uchar *)"xfadebeg" 152b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWXFadeEnd (picoos_uchar *)"xfadeend" 153b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWAlphabet (picoos_uchar *)"alphabet" 154b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWPH (picoos_uchar *)"ph" 155b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWOrthMode (picoos_uchar *)"orthmode" 156b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWIgnorePunct (picoos_uchar *)"ignorepunct" 157b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWInfo1 (picoos_uchar *)"info1" 158b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWInfo2 (picoos_uchar *)"info2" 159b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define KWDATA (picoos_uchar *)"data" 160b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 161b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEED_MIN 20 162b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEED_MAX 500 163b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEED_DEFAULT 100 164b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEED_FACTOR_MIN 500 165b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEED_FACTOR_MAX 2000 166b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 167b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_MIN 50 168b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_MAX 200 169b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_DEFAULT 100 170b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_FACTOR_MIN 500 171b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_FACTOR_MAX 2000 172b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_ADD_MIN -100 173b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_ADD_MAX 100 174b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_PITCH_ADD_DEFAULT 0 175b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 176b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_VOLUME_MIN 0 177b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_VOLUME_MAX 500 178b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_VOLUME_DEFAULT 100 179b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_VOLUME_FACTOR_MIN 500 180b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_VOLUME_FACTOR_MAX 2000 181b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 182b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEAKER_MIN 20 183b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEAKER_MAX 180 184b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEAKER_DEFAULT 100 185b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEAKER_FACTOR_MIN 500 186b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_SPEAKER_FACTOR_MAX 2000 187b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 188b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PICO_CONTEXT_DEFAULT (picoos_uchar*)"DEFAULT" 189b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 190b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define PARAGRAPH_PAUSE_DUR 500 191b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define SPELL_WITH_PHRASE_BREAK 1 192b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define SPELL_WITH_SENTENCE_BREAK 2 193b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 194b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* *****************************************************************************/ 195b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 196b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define TOK_PUNC_FLUSH (picoos_char) '\0' 197b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 198b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef picoos_uchar Word[MARKUP_STRING_BUF_SIZE]; 199b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 200b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 201b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstruct MarkupParam { 202b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen Word paramId; 203b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen Word paramVal; 204b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen}; 205b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 206b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef struct MarkupParam MarkupParams[MAX_NR_MARKUP_PARAMS]; 207b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 208e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivitypedef picoos_uchar utf8char0c[5]; /* one more than needed so it is ended always with 0c*/ 209b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 210b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/** subobject : TokenizeUnit 211b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * shortcut : tok 212b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen */ 213b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chentypedef struct tok_subobj 214b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 215b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 ignLevel; 216b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 217e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi utf8char0c utf; 218b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 utfpos; 219b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 utflen; 220b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 221b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MarkupParams markupParams; 222b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 nrMarkupParams; 223b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MarkupState markupState; 224b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar markupStr[MARKUP_STRING_BUF_SIZE]; 225b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 markupPos; 226b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 markupLevel[MIDummyEnd+1]; 227b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar markupTagName[IN_BUF_SIZE]; 228b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MarkupTagType markupTagType; 229b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MarkupParseError markupTagErr; 230b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 231b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 strPos; 232b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar strDelim; 233b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool isFileAttr; 234b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 235b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen pico_tokenType tokenType; 236b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen pico_tokenSubType tokenSubType; 237b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 238b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 tokenPos; 239b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar tokenStr[IN_BUF_SIZE]; 240b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 241b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 nrEOL; 242b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 243b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool markupHandlingMode; /* to be moved ??? */ 244b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool aborted; /* to be moved ??? */ 245b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 246b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool start; 247b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 248b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 outBuf[OUT_BUF_SIZE]; /* internal output buffer */ 249b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint16 outReadPos; /* next pos to read from outBuf */ 250b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint16 outWritePos; /* next pos to write to outBuf */ 251b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 252b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar saveFile[IN_BUF_SIZE]; 253b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen Word phonemes; 254b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 255b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picotrns_SimpleTransducer transducer; 256b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 257b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen /* kbs */ 258b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 259b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoktab_Graphs graphTab; 260b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picokfst_FST xsampa_parser; 261b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picokfst_FST svoxpa_parser; 262b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picokfst_FST xsampa2svoxpa_mapper; 263b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 264b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 265b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 266b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} tok_subobj_t; 267b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 268b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* *****************************************************************************/ 269b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 270b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatMarkupAsSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok); 271b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatChar (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar ch, picoos_bool markupHandling); 272b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok); 273b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_putToMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar str[]); 274b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok); 275b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic MarkupId tok_markupTagId (picoos_uchar tagId[]); 276b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 277b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* *****************************************************************************/ 278b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 279b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic picoos_bool tok_strEqual(picoos_uchar * str1, picoos_uchar * str2) 280b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 281b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return (picoos_strcmp((picoos_char*)str1, (picoos_char*)str2) == 0); 282b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 283b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 284b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_reduceBlanks(picoos_uchar * str) 285b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen /* Remove leading and trailing blanks of 'str' and reduce 286b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen groups of blanks within string to exactly one blank. */ 287b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 288b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 289b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int i = 0; 290b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int j = 0; 291b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 292b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while (str[j] != 0) { 293b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (str[j] == (picoos_uchar)' ') { 294b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen /* note one blank except at the beginning of string */ 295b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (i > 0) { 296b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen str[i] = (picoos_uchar)' '; 297b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i++; 298b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 299b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen j++; 300b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while (str[j] == (picoos_uchar)' ') { 301b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen j++; 302b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 303b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 304b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen str[i] = str[j]; 305b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen j++; 306b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i++; 307b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 308b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 309b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 310b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen /* remove blanks at end of string */ 311b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((i > 0) && (str[i - 1] == ' ')) { 312b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i--; 313b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 314b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen str[i] = 0; 315b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 316b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 317b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 318b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_startIgnore (tok_subobj_t * tok) 319b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 320b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->ignLevel++; 321b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 322b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 323b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 324b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_endIgnore (tok_subobj_t * tok) 325b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 326b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->ignLevel > 0) { 327b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->ignLevel--; 328b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 329b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 330b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 331b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 332b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_getParamIntVal (MarkupParams params, picoos_uchar paramId[], picoos_int32 * paramVal, picoos_bool * paramFound) 333b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 334b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int i=0; 335b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 336b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while ((i < MAX_NR_MARKUP_PARAMS) && !tok_strEqual(paramId,params[i].paramId)) { 337b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i++; 338b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 339b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((i < MAX_NR_MARKUP_PARAMS)) { 340b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramVal) = picoos_atoi((picoos_char*)params[i].paramVal); 341b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramFound) = TRUE; 342b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 343b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramVal) = -1; 344b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramFound) = FALSE; 345b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 346b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 347b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 348b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 349b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 350b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_getParamStrVal (MarkupParams params, picoos_uchar paramId[], picoos_uchar paramStrVal[], picoos_bool * paramFound) 351b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 352b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int i=0; 353b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 354b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while ((i < MAX_NR_MARKUP_PARAMS) && !tok_strEqual(paramId,params[i].paramId)) { 355b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i++; 356b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 357b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (i < MAX_NR_MARKUP_PARAMS) { 358b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_strcpy((picoos_char*)paramStrVal, (picoos_char*)params[i].paramVal); 359b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramFound) = TRUE; 360b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 361b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen paramStrVal[0] = 0; 362b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramFound) = FALSE; 363b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 364b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 365b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 366b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 367b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_getParamPhonesStr (MarkupParams params, picoos_uchar paramId[], picoos_uchar alphabet[], picoos_uchar phones[], picoos_int32 phoneslen, picoos_bool * paramFound) 368b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 369b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 370b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int i; 371b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool done; 372b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 373b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i = 0; 374b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while ((i < MAX_NR_MARKUP_PARAMS) && !tok_strEqual(paramId, params[i].paramId)) { 375b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i++; 376b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 377b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (i < MAX_NR_MARKUP_PARAMS) { 378b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(alphabet, PICODATA_XSAMPA) || tok_strEqual(alphabet, (picoos_uchar*)"")) { 379b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_strlcpy((picoos_char*)phones, (picoos_char*)params[i].paramVal, phoneslen); 380b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 381b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 382b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = FALSE; 383b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 384b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramFound) = TRUE; 385b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 386b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = FALSE; 387b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*paramFound) = FALSE; 388b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 389b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (!done) { 390b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen phones[0] = 0; 391b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 392b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 393b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 394b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 395b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_clearMarkupParams (MarkupParams params) 396b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 397b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int i; 398b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 399b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (i = 0; i<MAX_NR_MARKUP_PARAMS; i++) { 400b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen params[i].paramId[0] = 0; 401b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen params[i].paramVal[0] = 0; 402b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 403b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 404b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 405b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 406b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_getDur (picoos_uchar durStr[], picoos_uint32 * dur, picoos_bool * done) 407b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 408b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 409b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int num=0; 410b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int i=0; 411b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar tmpWord[IN_BUF_SIZE]; 412b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 413b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_strlcpy((picoos_char*)tmpWord, (picoos_char*)durStr, sizeof(tmpWord)); 414b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_reduceBlanks(tmpWord); 415b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while ((durStr[i] >= '0') && (durStr[i] <= '9')) { 416b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen num = 10 * num + (int)durStr[i] - (int)'0'; 417b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tmpWord[i] = ' '; 418b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i++; 419b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 420b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_reduceBlanks(tmpWord); 421b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(tmpWord, (picoos_uchar*)"s")) { 422b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*dur) = (1000 * num); 423b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*done) = TRUE; 424b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tmpWord,(picoos_uchar*)"ms")) { 425b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*dur) = num; 426b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*done) = TRUE; 427b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 428b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*dur) = 0; 429b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*done) = FALSE; 430b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 431b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 432b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 433b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 434b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic picoos_int32 tok_putToUtf (tok_subobj_t * tok, picoos_uchar ch) 435b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 436b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->utfpos < PICOBASE_UTF8_MAXLEN) { 437b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utf[tok->utfpos] = ch; 438b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->utfpos == 0) { 439b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utflen = picobase_det_utf8_length(ch); 440b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (((ch < (picoos_uchar)'\200') || (ch >= (picoos_uchar)'\300'))) { 441b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utflen = 0; 442b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 443b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (tok->utfpos)++; 444b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((tok->utfpos == tok->utflen)) { 445b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((tok->utfpos < PICOBASE_UTF8_MAXLEN)) { 446b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utf[tok->utfpos] = 0; 447b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 448b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return UTF_CHAR_COMPLETE; 449b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok->utfpos < tok->utflen) { 450b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return UTF_CHAR_INCOMPLETE; 451b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 452b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return UTF_CHAR_MALFORMED; 453b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 454b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 455b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return UTF_CHAR_MALFORMED; 456b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 457b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 458b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 459b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 460b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic picoos_bool tok_isRelative (picoos_uchar strval[], picoos_uint32 * val) 461b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 462b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 len; 463b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool rel; 464b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 465b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen rel = FALSE; 466b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen len = picoos_strlen((picoos_char*)strval); 467b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (len > 0) { 468b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (strval[len - 1] == '%') { 469b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen strval[len - 1] = 0; 470b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((strval[0] == '+') || (strval[0] == '-')) { 471b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*val) = 1000 + (picoos_atoi((picoos_char*)strval) * 10); 472b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 473b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*val) = picoos_atoi((picoos_char*)strval) * 10; 474b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 475b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen rel = TRUE; 476b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 477b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 478b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return rel; 479b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 480b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 481b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 482b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_putItem (picodata_ProcessingUnit this, tok_subobj_t * tok, 483b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 itemType, picoos_uint8 info1, picoos_uint8 info2, 484b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint16 val, 485b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar str[]) 486b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 487b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 len, i; 488b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 489b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((itemType == PICODATA_ITEM_CMD) && (info1 == PICODATA_ITEMINFO1_CMD_FLUSH)) { 490b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = itemType; 491b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info1; 492b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info2; 493b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = 0; 494b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 495b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else if (tok->ignLevel <= 0) { 496b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen switch (itemType) { 497b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEM_CMD: 498b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen switch (info1) { 499b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_CONTEXT: 500b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_VOICE: 501b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_MARKER: 502b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_PLAY: 503b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_SAVE: 504b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_UNSAVE: 505b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_PROSDOMAIN: 506b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_PHONEME: 507b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen len = picoos_strlen((picoos_char*)str); 508b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->outWritePos + 4 + len < OUT_BUF_SIZE) { 509b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = itemType; 510b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info1; 511b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info2; 512b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = len; 513b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (i=0; i<len; i++) { 514b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = str[i]; 515b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 516b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 517b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 518b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("tok_putItem: output buffer too small")); 519b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 520b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 521b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_IGNSIG: 522b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_IGNORE: 523b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->outWritePos + 4 < OUT_BUF_SIZE) { 524b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = itemType; 525b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info1; 526b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info2; 527b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = 0; 528b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 529b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 530b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("tok_putItem: output buffer too small")); 531b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 532b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 533b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_SPEED: 534b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_PITCH: 535b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_VOLUME: 536b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_SPELL: 537b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_SIL: 538b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEMINFO1_CMD_SPEAKER: 539b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->outWritePos + 4 + 2 < OUT_BUF_SIZE) { 540b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = itemType; 541b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info1; 542b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info2; 543b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = 2; 544b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = val % 256; 545b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = val / 256; 546b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 547b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 548b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("tok_putItem: output buffer too small")); 549b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 550b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 551b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen default: 552b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("tok_putItem: unknown command type")); 553b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 554b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 555b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case PICODATA_ITEM_TOKEN: 556b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen len = picoos_strlen((picoos_char*)str); 557b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->outWritePos + 4 + len < OUT_BUF_SIZE) { 558b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = itemType; 559b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info1; 560b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info2; 561b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = len; 562b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (i=0; i<len; i++) { 563b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = str[i]; 564b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 565b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 566b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 567b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("tok_putItem: output buffer too small")); 568b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 569b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 570b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen default: 571b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("tok_putItem: unknown item type")); 572b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 573b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 574b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 575b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 576b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 577b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_putItem2 (picodata_ProcessingUnit this, tok_subobj_t * tok, 578b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 type, 579b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 info1, picoos_uint8 info2, 580b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 len, 581b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 data[]) 582b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 583b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 i; 584b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 585b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (is_valid_itemtype(type)) { 586b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = type; 587b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info1; 588b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = info2; 589b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = len; 590b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (i=0; i<len; i++) { 591b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outBuf[tok->outWritePos++] = data[i]; 592b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 593b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 594b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 595b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 596b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 597b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic MarkupId tok_markupTagId (picoos_uchar tagId[]) 598b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 599b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (picoos_strstr(tagId,(picoos_char *)"svox:") == (picoos_char *)tagId) { 600b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tagId+=5; 601b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 602b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(tagId, TOK_MARKUP_KW_IGNORE)) { 603b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIIgnore; 604b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SPEED)) { 605b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MISpeed; 606b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PITCH)) { 607b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIPitch; 608b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_VOLUME)) { 609b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIVolume; 610b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SPEAKER)) { 611b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MISpeaker; 612b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_VOICE)) { 613b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIVoice; 614b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_CONTEXT)) { 615b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIPreprocContext; 616b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_MARK)) { 617b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIMarker; 618b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PLAY)) { 619b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIPlay; 620b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_USESIG)) { 621b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIUseSig; 622b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_GENFILE)) { 623b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIGenFile; 624b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SENTENCE) || tok_strEqual(tagId, TOK_MARKUP_KW_S)) { 625b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MISentence; 626b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PARAGRAPH) || tok_strEqual(tagId, TOK_MARKUP_KW_P)) { 627b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIParagraph; 628b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_BREAK)) { 629b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIBreak; 630b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SPELL)) { 631b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MISpell; 632b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PHONEME)) { 633b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIPhoneme; 634b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tagId, TOK_MARKUP_KW_ITEM)) { 635b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIItem; 636b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 637b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return MIDummyEnd; 638b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 639b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 640b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 641b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 642e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivistatic void tok_checkLimits (picodata_ProcessingUnit this, picoos_uint32 * value, picoos_uint32 min, picoos_uint32 max, picoos_uchar valueType[]) 643b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 644b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((((*value) < min) || ((*value) > max))) { 645b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE, (picoos_char*)"", (picoos_char*)"attempt to set illegal value %i for %s", *value, valueType); 646b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (((*value) < min)) { 647b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*value) = min; 648b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (((*value) > max)) { 649b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*value) = max; 650b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 651b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 652b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 653b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 654b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 655b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 656e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi/* 657e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi 658e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivistatic void tok_checkRealLimits (picodata_ProcessingUnit this, picoos_single * value, picoos_single min, picoos_single max, picoos_uchar valueType[]) 659b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 660b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((((*value) < min) || ((*value) > max))) { 661b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE, (picoos_char*)"", (picoos_char*)"attempt to set illegal value %f for %s", *value, valueType); 662b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (((*value) < min)) { 663b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*value) = min; 664b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (((*value) > max)) { 665b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*value) = max; 666b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 667b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 668b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 669e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi*/ 670b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 671b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#define VAL_STR_LEN 21 672b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 673b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_interpretMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_bool isStartTag, MarkupId mId) 674b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 675b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool done; 676b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 ival; 677b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint32 uval; 678b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 ival2; 679b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar valStr[VAL_STR_LEN]; 680b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar valStr2[VAL_STR_LEN]; 681b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar valStr3[VAL_STR_LEN]; 682b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 i2; 683b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint32 dur; 684b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool done1; 685b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool paramFound; 686b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 type, info1, info2; 687b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 data[256]; 688b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 pos, n, len; 689b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uchar part[10]; 690b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 691b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = FALSE; 692b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen switch (mId) { 693b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIIgnore: 694b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) { 695b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_startIgnore(tok); 696b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 697b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 698b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_endIgnore(tok); 699b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 700b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 701b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 702b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MISpeed: 703b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) { 704b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) { 705b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval, PICO_SPEED_FACTOR_MIN, PICO_SPEED_FACTOR_MAX,(picoos_uchar*)"relative speed factor"); 706b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEED, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)""); 707b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 708b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal); 709b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval, PICO_SPEED_MIN, PICO_SPEED_MAX,(picoos_uchar*)"speed"); 710b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEED, PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)""); 711b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 712b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 713b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 714b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEED, PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_SPEED_DEFAULT, (picoos_uchar*)""); 715b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 716b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 717b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 718b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIPitch: 719b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) { 720b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) { 721b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval,PICO_PITCH_FACTOR_MIN,PICO_PITCH_FACTOR_MAX, (picoos_uchar*)"relative pitch factor"); 722b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PITCH, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)""); 723b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 724b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal); 725b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval,PICO_PITCH_MIN,PICO_PITCH_MAX, (picoos_uchar*)"pitch"); 726b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PITCH,PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)""); 727b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 728b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 729b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 730b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PITCH,PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_PITCH_DEFAULT, (picoos_uchar*)""); 731b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 732b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 733b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 734b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIVolume: 735b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) { 736b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) { 737b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval, PICO_VOLUME_FACTOR_MIN, PICO_VOLUME_FACTOR_MAX, (picoos_uchar*)"relative volume factor"); 738b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOLUME, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)""); 739b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 740b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal); 741b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval, PICO_VOLUME_MIN, PICO_VOLUME_MAX, (picoos_uchar*)"volume"); 742b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOLUME, PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)""); 743b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 744b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 745b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 746b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOLUME, PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_VOLUME_DEFAULT, (picoos_uchar*)""); 747b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 748b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 749b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 750b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MISpeaker: 751b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) { 752b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) { 753b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval, PICO_SPEAKER_FACTOR_MIN, PICO_SPEAKER_FACTOR_MAX, (picoos_uchar*)"relative speaker factor"); 754b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEAKER, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)""); 755b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 756b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal); 757b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits(this, & uval, PICO_SPEAKER_MIN, PICO_SPEAKER_MAX, (picoos_uchar*)"volume"); 758b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEAKER, PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)""); 759b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 760b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 761b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 762b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEAKER, PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_SPEAKER_DEFAULT, (picoos_uchar*)""); 763b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 764b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 765b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 766b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 767b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIVoice: 768b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWName)) { 769b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOICE, PICODATA_ITEMINFO2_NA, 0, tok->markupParams[0].paramVal); 770b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 771b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 0, 0, (picoos_uchar*)""); 772b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 773b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) { 774b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOICE, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 775b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 776b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 0, 0, (picoos_uchar*)""); 777b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 778b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 779b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 780b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIPreprocContext: 781b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWName)) { 782b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_CONTEXT, PICODATA_ITEMINFO2_NA, 0, tok->markupParams[0].paramVal); 783b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 784b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) { 785b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_CONTEXT, PICODATA_ITEMINFO2_NA, 0, PICO_CONTEXT_DEFAULT); 786b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 787b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 788b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 789b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIMarker: 790b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWName)) { 791b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_MARKER, PICODATA_ITEMINFO2_NA, 0, tok->markupParams[0].paramVal); 792b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 793b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) { 794b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 795b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 796b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 797b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MISentence: 798b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag) { 799b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamStrVal(tok->markupParams, KWProsDomain, (picoos_uchar*)valStr, & paramFound); 800b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 801b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 2, 0, valStr); 802b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 803b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 804b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 805b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 2, 0, (picoos_uchar*)""); 806b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 807b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 808b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 809b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIParagraph: 810b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag) { 811b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamStrVal(tok->markupParams, KWProsDomain, (picoos_uchar*)valStr, & paramFound); 812b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 813b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 1, 0, valStr); 814b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 815b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 816b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 817b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, PARAGRAPH_PAUSE_DUR, (picoos_uchar*)""); 818b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 1, 0, (picoos_uchar*)""); 819b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 820b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 821b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 822b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIBreak: 823b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWTime)) { 824b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getDur(tok->markupParams[0].paramVal, & dur, & done1); 825b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits (this, &dur, 0, 65535, (picoos_uchar*)"time"); 826b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (done1) { 827b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, dur, (picoos_uchar*)""); 828b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 829b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 830b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 831b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 832b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 833b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 834b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MISpell: 835b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag) { 836b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(tok->markupParams[0].paramId, KWMode)) { 837b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(tok->markupParams[0].paramVal, KWPB)) { 838b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen uval = SPELL_WITH_PHRASE_BREAK; 839b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tok->markupParams[0].paramVal, KWSB)) { 840b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen uval = SPELL_WITH_SENTENCE_BREAK; 841b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 842b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getDur(tok->markupParams[0].paramVal, & uval, & done1); 843b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_checkLimits (this, & uval, 0, 65535, (picoos_uchar*)"time"); 844b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (done1) { 845b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 846b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 847b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 848b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 849b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen uval = SPELL_WITH_PHRASE_BREAK; 850b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 851b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPELL, PICODATA_ITEMINFO2_CMD_START, uval, (picoos_uchar*)""); 852b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 853b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 854b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPELL, PICODATA_ITEMINFO2_CMD_END, 0, (picoos_uchar*)""); 855b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 856b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 857b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 858b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIGenFile: 859b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWFile)) { 860b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->saveFile[0] != 0) { 861b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE, 862b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picodata_getPuTypeFromExtension(tok->saveFile, /*input*/FALSE), 0, tok->saveFile); 863b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->saveFile[0] = 0; 864b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 865b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SAVE, 866b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picodata_getPuTypeFromExtension(tok->markupParams[0].paramVal, /*input*/FALSE), 0, tok->markupParams[0].paramVal); 867b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_strcpy((picoos_char*)tok->saveFile, (picoos_char*)tok->markupParams[0].paramVal); 868b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 869b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 870b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->saveFile[0] != 0) { 871b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE, 872b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picodata_getPuTypeFromExtension(tok->saveFile, /*input*/FALSE), 0, (picoos_uchar*)""); 873b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->saveFile[0] = 0; 874b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 875b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 876b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 877b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 878b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIPlay: 879b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWFile)) { 880b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (picoos_FileExists(this->common, (picoos_char*)tok->markupParams[0].paramVal)) { 881b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamIntVal(tok->markupParams,KWF0Beg,& ival,& paramFound); 882b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamIntVal(tok->markupParams,KWF0End,& ival2,& paramFound); 883b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamStrVal(tok->markupParams,KWAlphabet,valStr3,& paramFound); 884b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamPhonesStr(tok->markupParams,KWXFadeBeg,valStr3,valStr,VAL_STR_LEN,& paramFound); 885b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamPhonesStr(tok->markupParams,KWXFadeEnd,valStr3,valStr2,VAL_STR_LEN,& paramFound); 886b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PLAY, 887b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picodata_getPuTypeFromExtension(tok->markupParams[0].paramVal, /*input*/TRUE), 0, tok->markupParams[0].paramVal); 888b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_startIgnore(tok); 889b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 890b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->ignLevel > 0) { 891b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_startIgnore(tok); 892b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 893b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_EXC_CANT_OPEN_FILE, (picoos_char*)"", (picoos_char*)"file '%s' not found; synthesizing enclosed text instead\n", tok->markupParams[0].paramVal); 894b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 895b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 896b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 897b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 898b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_endIgnore(tok); 899b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 900b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 901b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 902b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIUseSig: 903b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWFile)) { 904b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (picoos_FileExists(this->common, (picoos_char*)tok->markupParams[0].paramVal)) { 905b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamIntVal(tok->markupParams,KWF0Beg,& ival,& paramFound); 906b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamIntVal(tok->markupParams,KWF0End,& ival2,& paramFound); 907b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamStrVal(tok->markupParams,KWAlphabet,valStr3, & paramFound); 908b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamPhonesStr(tok->markupParams,KWXFadeBeg,valStr3,valStr,VAL_STR_LEN,& paramFound); 909b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_getParamPhonesStr(tok->markupParams,KWXFadeEnd,valStr3,valStr2,VAL_STR_LEN,& paramFound); 910b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PLAY, 911b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picodata_getPuTypeFromExtension(tok->markupParams[0].paramVal, /*input*/TRUE), 0, tok->markupParams[0].paramVal); 912b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNSIG, PICODATA_ITEMINFO2_CMD_START, 0, (picoos_uchar*)""); 913b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 914b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->ignLevel <= 0) { 915b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_EXC_CANT_OPEN_FILE, (picoos_char*)"", (picoos_char*)"file '%s' not found; synthesizing enclosed text instead", tok->markupParams[0].paramVal); 916b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 917b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 918b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 919b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 920b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNSIG, PICODATA_ITEMINFO2_CMD_END, 0, (picoos_uchar*)""); 921b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 922b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 923b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 924b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIPhoneme: 925b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i2 = 0; 926b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag) { 927b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(tok->markupParams[0].paramId, KWAlphabet) && tok_strEqual(tok->markupParams[1].paramId, KWPH)) { 928b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(tok->markupParams[2].paramId, KWOrthMode) 929b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen && tok_strEqual(tok->markupParams[2].paramVal, KWIgnorePunct)) { 930b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i2 = 1; 931b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 932b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (picodata_mapPAStrToPAIds(tok->transducer, this->common, tok->xsampa_parser, tok->svoxpa_parser, tok->xsampa2svoxpa_mapper, tok->markupParams[1].paramVal, tok->markupParams[0].paramVal, tok->phonemes, sizeof(tok->phonemes)-1) == PICO_OK) { 933b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME, 934b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODATA_ITEMINFO2_CMD_START, i2, tok->phonemes); 935b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 936b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 937b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("cannot map phonetic string '%s'; synthesizeing text instead", tok->markupParams[1].paramVal)); 938b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE,(picoos_char*)"", (picoos_char*)"cannot map phonetic string '%s'; synthesizeing text instead", tok->markupParams[1].paramVal); 939b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 940b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 941b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_strEqual(tok->markupParams[0].paramId, KWPH)) { 942b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_strEqual(tok->markupParams[1].paramId, KWOrthMode) 943b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen && tok_strEqual(tok->markupParams[1].paramVal, KWIgnorePunct)) { 944b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen i2 = 1; 945b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 946b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (picodata_mapPAStrToPAIds(tok->transducer, this->common, tok->xsampa_parser, tok->svoxpa_parser, tok->xsampa2svoxpa_mapper, tok->markupParams[0].paramVal, PICODATA_XSAMPA, tok->phonemes, sizeof(tok->phonemes)) == PICO_OK) { 947b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME, 948b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODATA_ITEMINFO2_CMD_START, i2, tok->phonemes); 949b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 950b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 951b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 952b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_WARN(("cannot map phonetic string '%s'; synthesizeing text instead", tok->markupParams[1].paramVal)); 953b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE,(picoos_char*)"", (picoos_char*)"cannot map phonetic string '%s'; synthesizing text instead", tok->markupParams[0].paramVal); 954b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 955b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 956b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 957b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) { 958b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME, 959b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODATA_ITEMINFO2_CMD_END, i2, (picoos_uchar*)""); 960b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 961b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 962b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 963b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MIItem: 964b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWType) && 965b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_strEqual(tok->markupParams[1].paramId, KWInfo1)&& 966b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_strEqual(tok->markupParams[2].paramId, KWInfo2)&& 967b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_strEqual(tok->markupParams[3].paramId, KWDATA)) { 968b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 len2, n2; 969b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen type = picoos_atoi(tok->markupParams[0].paramVal); 970b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen info1 = picoos_atoi(tok->markupParams[1].paramVal); 971b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen info2 = picoos_atoi(tok->markupParams[2].paramVal); 972b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen n = 0; n2 = 0; 973b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen len2 = (picoos_int32)picoos_strlen(tok->markupParams[3].paramVal); 974b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while (n<len2) { 975b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while ((tok->markupParams[3].paramVal[n] != 0) && (tok->markupParams[3].paramVal[n] <= 32)) { 976b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen n++; 977b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 978b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[3].paramVal[n2] = tok->markupParams[3].paramVal[n]; 979b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen n++; 980b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen n2++; 981b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 982b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (is_valid_itemtype(type)) { 983b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 984b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen len = 0; 985b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen pos = 0; 986b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_get_sep_part_str(tok->markupParams[3].paramVal, picoos_strlen(tok->markupParams[3].paramVal), 987b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen &pos, ',', part, 10, &done1); 988b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while (done && done1) { 989b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen n = picoos_atoi(part); 990b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((n>=0) && (n<256) && (len<256)) { 991b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen data[len++] = n; 992b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 993b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 994b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = FALSE; 995b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 996b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_get_sep_part_str(tok->markupParams[3].paramVal, picoos_strlen(tok->markupParams[3].paramVal), 997b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen &pos, ',', part, 10, &done1); 998b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 999b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (done) { 1000b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem2(this, tok, type, info1, info2, len, data); 1001b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1002b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1003b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 1004b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = FALSE; 1005b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1006b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) { 1007b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen done = TRUE; 1008b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1009b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1010b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen default: 1011b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1012b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1013b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (!done) { 1014b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEInterprete; 1015b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1016b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (isStartTag) { 1017b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupLevel[mId]++; 1018b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if ((tok->markupLevel[mId] > 0)) { 1019b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupLevel[mId]--; 1020b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1021b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1022b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1023b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1024b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic picoos_bool tok_attrChar (picoos_uchar ch, picoos_bool first) 1025b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1026b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return ((((ch >= (picoos_uchar)'A') && (ch <= (picoos_uchar)'Z')) || 1027b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen ((ch >= (picoos_uchar)'a') && (ch <= (picoos_uchar)'z'))) || 1028b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen ( !(first) && ((ch >= (picoos_uchar)'0') && (ch <= (picoos_uchar)'9')))); 1029b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1030b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1031b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1032b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1033b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic picoos_bool tok_idChar (picoos_uchar ch, picoos_bool first) 1034b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1035b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return tok_attrChar(ch, first) || ( !(first) && (ch == (picoos_uchar)':')); 1036b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1037b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1038b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1039b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_setIsFileAttr (picoos_uchar name[], picoos_bool * isFile) 1040b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1041b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (*isFile) = tok_strEqual(name, KWFile); 1042b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1043b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1044b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* *****************************************************************************/ 1045b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1046b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_putToSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar str[], pico_tokenType type, pico_tokenSubType subtype) 1047b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1048b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen int i, len; 1049b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1050b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (str[0] != 0) { 1051b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen len = picoos_strlen((picoos_char*)str); 1052b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (i = 0; i < len; i++) { 1053b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->tokenPos >= IN_BUF_SIZE) { 1054b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_INTERNAL_LIMIT, (picoos_char*)"", (picoos_char*)"simple token too long; forced treatment"); 1055b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1056b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1057b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenStr[tok->tokenPos] = str[i]; 1058b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenPos++; 1059b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1060b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1061b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenType = type; 1062b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenSubType = subtype; 1063b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1064b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1065b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1066b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_putToMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar str[]) 1067b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1068b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 i, len; 1069b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 ok; 1070b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1071b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MENone; 1072b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen len = picoos_strlen((picoos_char*)str); 1073b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (i = 0; i< len; i++) { 1074b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->markupPos >= (MARKUP_STRING_BUF_SIZE - 1)) { 1075b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((tok->markupPos == (MARKUP_STRING_BUF_SIZE - 1)) && (tok_markupTagId(tok->markupTagName) != MIDummyEnd)) { 1076b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_INTERNAL_LIMIT ,(picoos_char*)"", (picoos_char*)"markup tag too long"); 1077b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1078b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSErrorTooLong; 1079b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if ((str[i] == (picoos_uchar)' ') && ((tok->markupState == MSExpectingmarkupTagName) || (tok->markupState == MSGotmarkupTagName) || (tok->markupState == MSGotAttrName) || (tok->markupState == MSGotEqual) || (tok->markupState == MSGotAttrValue))) { 1080b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if ((str[i] == (picoos_uchar)'>') && ((tok->markupState == MSGotmarkupTagName) || (tok->markupState == MSInmarkupTagName) || (tok->markupState == MSGotAttrValue))) { 1081b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotEnd; 1082b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if ((str[i] == (picoos_uchar)'/') && ((tok->markupState == MSGotmarkupTagName) || (tok->markupState == MSInmarkupTagName) || (tok->markupState == MSGotAttrValue))) { 1083b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->markupTagType == MTEnd) { 1084b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEUnexpectedChar; 1085b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1086b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1087b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagType = MTEmpty; 1088b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotEndSlash; 1089b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1090b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1091b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen switch (tok->markupState) { 1092b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSNotInMarkup: 1093b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (str[i] == (picoos_uchar)'<') { 1094b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_clearMarkupParams(tok->markupParams); 1095b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->nrMarkupParams = 0; 1096b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos = 0; 1097b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagType = MTStart; 1098b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotStart; 1099b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1100b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEMissingStart; 1101b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1102b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1103b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1104b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSGotStart: 1105b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (str[i] == (picoos_uchar)'/') { 1106b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagType = MTEnd; 1107b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSExpectingmarkupTagName; 1108b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (str[i] == (picoos_uchar)' ') { 1109b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSExpectingmarkupTagName; 1110b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok_idChar(str[i],TRUE)) { 1111b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagType = MTStart; 1112b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagName[tok->strPos] = str[i]; 1113b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos++; 1114b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagName[tok->strPos] = 0; 1115b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInmarkupTagName; 1116b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1117b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEUnexpectedChar; 1118b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1119b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1120b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1121b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSInmarkupTagName: case MSExpectingmarkupTagName: 1122b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_idChar(str[i],tok->markupState == MSExpectingmarkupTagName)) { 1123b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagName[tok->strPos] = str[i]; 1124b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos++; 1125b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagName[(tok->strPos)] = 0; 1126b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInmarkupTagName; 1127b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if ((tok->markupState == MSInmarkupTagName) && (str[i] == (picoos_uchar)' ')) { 1128b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotmarkupTagName; 1129b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picobase_lowercase_utf8_str(tok->markupTagName, (picoos_char*)tok->markupTagName, IN_BUF_SIZE, &ok); 1130b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos = 0; 1131b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1132b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEIdent; 1133b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1134b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1135b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1136b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSGotmarkupTagName: case MSGotAttrValue: 1137b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_attrChar(str[i], TRUE)) { 1138b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->markupTagType == MTEnd) { 1139b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEUnexpectedChar; 1140b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1141b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1142b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) { 1143b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = str[i]; 1144b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos++; 1145b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = 0; 1146b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1147b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_INTERNAL_LIMIT ,(picoos_char*)"", (picoos_char*)"too many attributes in markup; ignoring"); 1148b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1149b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInAttrName; 1150b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1151b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1152b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEUnexpectedChar; 1153b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1154b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1155b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1156b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSInAttrName: 1157b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_attrChar(str[i], FALSE)) { 1158b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) { 1159b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = str[i]; 1160b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos++; 1161b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = 0; 1162b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1163b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInAttrName; 1164b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (str[i] == (picoos_uchar)' ') { 1165b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picobase_lowercase_utf8_str(tok->markupParams[tok->nrMarkupParams].paramId, (picoos_char*)tok->markupParams[tok->nrMarkupParams].paramId, IN_BUF_SIZE, &ok); 1166b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_setIsFileAttr(tok->markupParams[tok->nrMarkupParams].paramId, & tok->isFileAttr); 1167b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotAttrName; 1168b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (str[i] == (picoos_uchar)'=') { 1169b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picobase_lowercase_utf8_str(tok->markupParams[tok->nrMarkupParams].paramId, (picoos_char*)tok->markupParams[tok->nrMarkupParams].paramId, IN_BUF_SIZE, &ok); 1170b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_setIsFileAttr(tok->markupParams[tok->nrMarkupParams].paramId, & tok->isFileAttr); 1171b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotEqual; 1172b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1173b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEMissingEqual; 1174b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1175b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1176b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1177b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSGotAttrName: 1178b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (str[i] == (picoos_uchar)'=') { 1179b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotEqual; 1180b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1181b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEMissingEqual; 1182b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1183b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1184b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1185b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSGotEqual: 1186b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((str[i] == (picoos_uchar)'"') || (str[i] == (picoos_uchar)'\'')) { 1187b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strDelim = str[i]; 1188b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos = 0; 1189b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInAttrValue; 1190b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1191b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEMissingQuote; 1192b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1193b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1194b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1195b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSInAttrValue: 1196b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (!(tok->isFileAttr) && (str[i] == (picoos_uchar)'\\')) { 1197b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInAttrValueEscaped; 1198b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (str[i] == tok->strDelim) { 1199b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) { 1200b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->nrMarkupParams++; 1201b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1202b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos = 0; 1203b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotAttrValue; 1204b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1205b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) { 1206b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = str[i]; 1207b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos++; 1208b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = 0; 1209b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1210b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInAttrValue; 1211b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1212b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1213b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSInAttrValueEscaped: 1214b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) { 1215b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = str[i]; 1216b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos++; 1217b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = 0; 1218b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1219b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSInAttrValue; 1220b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1221b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case MSGotEndSlash: 1222b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (str[i] == (picoos_uchar)'>') { 1223b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSGotEnd; 1224b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1225b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEUnexpectedChar; 1226b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1227b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1228b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1229b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen default: 1230b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MEUnexpectedChar; 1231b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSError; 1232b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1233b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1234b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1235e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi if (tok->markupTagErr == MENone) { 1236e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi tok->markupStr[tok->markupPos] = str[i]; 1237e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi tok->markupPos++; 1238e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi } /* else restart parsing at current char */ 1239b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupStr[tok->markupPos] = 0; 1240b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1241b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen /* 1242b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_DEBUG(("putToMarkup %s", tok->markupStr)); 1243b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen */ 1244b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1245b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1246b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* *****************************************************************************/ 1247b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1248b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatMarkupAsSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok) 1249b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1250b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int32 i; 1251b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1252b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utfpos = 0; 1253b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utflen = 0; 1254b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSNotInMarkup; 1255b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (i = 0; i < tok->markupPos; i++) { 1256b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatChar(this, tok, tok->markupStr[i], FALSE); 1257b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1258b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupPos = 0; 1259b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos = 0; 1260b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1261b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1262b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1263b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok) 1264b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1265b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MarkupId mId; 1266b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1267b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok_markupTagId(tok->markupTagName) != MIDummyEnd) { 1268b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->markupTagErr == MENone) { 1269b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSNotInMarkup; 1270b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((tok->tokenType != PICODATA_ITEMINFO1_TOKTYPE_SPACE) && (tok->tokenType != PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED)) { 1271b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1272b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1273b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putToSimpleToken(this, tok, (picoos_uchar*)" ", PICODATA_ITEMINFO1_TOKTYPE_SPACE, -1); 1274b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen mId = tok_markupTagId(tok->markupTagName); 1275b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((tok->markupTagType == MTStart) || (tok->markupTagType == MTEmpty)) { 1276b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_interpretMarkup(this, tok, TRUE, mId); 1277b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1278b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (((tok->markupTagType == MTEnd) || (tok->markupTagType == MTEmpty))) { 1279b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_clearMarkupParams(tok->markupParams); 1280b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->nrMarkupParams = 0; 1281b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_interpretMarkup(this, tok, FALSE,mId); 1282b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1283b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1284b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->markupTagErr != MENone) { 1285b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (!tok->aborted) { 1286b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_INVALID_MARKUP_TAG, (picoos_char*)"", (picoos_char*)"syntax error in markup token '%s'",tok->markupStr); 1287b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1288b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatMarkupAsSimpleToken(this, tok); 1289b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1290b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1291b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatMarkupAsSimpleToken(this, tok); 1292b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1293b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSNotInMarkup; 1294b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupPos = 0; 1295b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos = 0; 1296b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1297b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1298b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1299b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1300b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatChar (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar ch, picoos_bool markupHandling) 1301b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1302e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi picoos_int32 i, id; 1303b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_uint8 uval8; 1304b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen pico_tokenType type = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED; 1305b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen pico_tokenSubType subtype = -1; 1306b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_bool dummy; 1307e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi utf8char0c utf2; 1308e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi picoos_int32 utf2pos; 1309b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1310b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (ch == NULLC) { 1311b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1312b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)""); 1313b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1314b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 1315b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen switch (tok_putToUtf(tok, ch)) { 1316b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case UTF_CHAR_MALFORMED: 1317b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utfpos = 0; 1318b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utflen = 0; 1319b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1320b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case UTF_CHAR_INCOMPLETE: 1321b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1322b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen case UTF_CHAR_COMPLETE: 1323b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen markupHandling = (markupHandling && (tok->markupHandlingMode == MARKUP_HANDLING_ENABLED)); 1324b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen id = picoktab_graphOffset(tok->graphTab, tok->utf); 1325b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (id > 0) { 1326b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (picoktab_getIntPropTokenType(tok->graphTab, id, &uval8)) { 1327b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen type = (pico_tokenType)uval8; 1328b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (type == PICODATA_ITEMINFO1_TOKTYPE_LETTERV) { 1329b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen type = PICODATA_ITEMINFO1_TOKTYPE_LETTER; 1330b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1331b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1332b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen dummy = picoktab_getIntPropTokenSubType(tok->graphTab, id, &subtype); 1333e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi } else if (tok->utf[tok->utfpos-1] <= (picoos_uchar)' ') { 1334b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen type = PICODATA_ITEMINFO1_TOKTYPE_SPACE; 1335b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen subtype = -1; 1336b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1337b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen type = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED; 1338b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen subtype = -1; 1339b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1340e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi if ((tok->utf[tok->utfpos-1] > (picoos_uchar)' ')) { 1341b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->nrEOL = 0; 1342e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi } else if ((tok->utf[tok->utfpos-1] == EOL)) { 1343b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->nrEOL++; 1344b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1345b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (markupHandling && (tok->markupState != MSNotInMarkup)) { 1346b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putToMarkup(this, tok, tok->utf); 1347b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->markupState >= MSError) { 1348e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi picoos_strlcpy(utf2, tok->utf, 5); 1349e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi utf2pos = tok->utfpos; 1350e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi /* treat string up to (but not including) current char as simple 1351e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi token and restart markup tag parsing with current char */ 1352b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatMarkupAsSimpleToken(this, tok); 1353e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi for (i = 0; i < utf2pos; i++) { 1354e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi tok_treatChar(this, tok, utf2[i], markupHandling); 1355e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi } 1356b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (tok->markupState == MSGotEnd) { 1357b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatMarkup(this, tok); 1358b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1359e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi } else if ((markupHandling && (tok->utf[tok->utfpos-1] == (picoos_uchar)'<'))) { 1360b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putToMarkup(this, tok, tok->utf); 1361b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if (type != PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED) { 1362b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((type != tok->tokenType) || (type == PICODATA_ITEMINFO1_TOKTYPE_CHAR) || (subtype != tok->tokenSubType)) { 1363b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1364e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi } else if ((tok->utf[tok->utfpos-1] == EOL) && (tok->nrEOL == 2)) { 1365b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1366b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putToSimpleToken(this, tok, (picoos_uchar*)".", PICODATA_ITEMINFO1_TOKTYPE_CHAR, -1); 1367b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1368b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1369b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putToSimpleToken(this, tok, tok->utf, type, subtype); 1370b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else { 1371b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1372b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1373b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utfpos = 0; 1374b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utflen = 0; 1375b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen break; 1376b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1377b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1378b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1379b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1380b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1381b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic void tok_treatSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok) 1382b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1383b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->tokenPos < IN_BUF_SIZE) { 1384b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenStr[tok->tokenPos] = 0; 1385b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1386b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->markupState != MSNotInMarkup) { 1387b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (!(tok->aborted) && (tok->markupState >= MSGotmarkupTagName) && (tok_markupTagId(tok->markupTagName) != MIDummyEnd)) { 1388b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_emRaiseWarning(this->common->em, PICO_ERR_INVALID_MARKUP_TAG, (picoos_char*)"", (picoos_char*)"unfinished markup tag '%s'",tok->markupStr); 1389b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1390b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatMarkupAsSimpleToken(this, tok); 1391b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatSimpleToken(this, tok); 1392b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } else if ((tok->tokenPos > 0) && ((tok->ignLevel <= 0) || (tok->tokenType == PICODATA_ITEMINFO1_TOKTYPE_SPACE))) { 1393b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_putItem(this, tok, PICODATA_ITEM_TOKEN, tok->tokenType, (picoos_uint8)tok->tokenSubType, 0, tok->tokenStr); 1394b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1395b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenPos = 0; 1396b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenType = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED; 1397b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenSubType = -1; 1398b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1399b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1400b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* *****************************************************************************/ 1401b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1402e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivistatic pico_status_t tokReset(register picodata_ProcessingUnit this, picoos_int32 resetMode) 1403b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1404b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_subobj_t * tok; 1405b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen MarkupId mId; 1406b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1407b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (NULL == this || NULL == this->subObj) { 1408b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICO_ERR_OTHER; 1409b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1410b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok = (tok_subobj_t *) this->subObj; 1411b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1412b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->ignLevel = 0; 1413b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1414b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utfpos = 0; 1415b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->utflen = 0; 1416b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1417b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_clearMarkupParams(tok->markupParams); 1418b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->nrMarkupParams = 0; 1419b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupState = MSNotInMarkup; 1420b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupPos = 0; 1421b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen for (mId = MIDummyStart; mId <= MIDummyEnd; mId++) { 1422b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupLevel[mId] = 0; 1423b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1424b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagName[0] = 0; 1425b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagType = MTNone; 1426b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupTagErr = MENone; 1427b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1428b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strPos = 0; 1429b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->strDelim = 0; 1430b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->isFileAttr = FALSE; 1431b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1432b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenType = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED; 1433b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenSubType = -1; 1434b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->tokenPos = 0; 1435b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1436b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->nrEOL = 0; 1437b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1438b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1439b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->markupHandlingMode = TRUE; 1440b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->aborted = FALSE; 1441b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1442b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->start = TRUE; 1443b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1444b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outReadPos = 0; 1445b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outWritePos = 0; 1446b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1447b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->saveFile[0] = 0; 1448b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1449b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1450b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->graphTab = picoktab_getGraphs(this->voice->kbArray[PICOKNOW_KBID_TAB_GRAPHS]); 1451b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1452b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->xsampa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA_PARSE]); 1453b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_TRACE(("got xsampa_parser @ %i",tok->xsampa_parser)); 1454b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1455b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->svoxpa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_SVOXPA_PARSE]); 1456b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_TRACE(("got svoxpa_parser @ %i",tok->svoxpa_parser)); 1457b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1458b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->xsampa2svoxpa_mapper = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA2SVOXPA]); 1459b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_TRACE(("got xsampa2svoxpa_mapper @ %i",tok->xsampa2svoxpa_mapper)); 1460b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1461b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1462b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1463b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICO_OK; 1464b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1465b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1466e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivistatic pico_status_t tokInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode) 1467b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1468b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* 1469b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1470b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_subobj_t * tok; 1471b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1472b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (NULL == this || NULL == this->subObj) { 1473b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICO_ERR_OTHER; 1474b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1475b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok = (tok_subobj_t *) this->subObj; 1476b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen*/ 1477e9f72c8954f29f10cb4feb16d328a1b5c1fd7169Jean-Michel Trivi return tokReset(this, resetMode); 1478b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1479b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1480b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1481b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic pico_status_t tokTerminate(register picodata_ProcessingUnit this) 1482b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1483b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICO_OK; 1484b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1485b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1486b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic picodata_step_result_t tokStep(register picodata_ProcessingUnit this, picoos_int16 mode, picoos_uint16 * numBytesOutput); 1487b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1488b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenstatic pico_status_t tokSubObjDeallocate(register picodata_ProcessingUnit this, 1489b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_MemoryManager mm) 1490b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1491b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1492b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (NULL != this) { 1493b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_deallocate(this->common->mm, (void *) &this->subObj); 1494b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1495b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen mm = mm; /* avoid warning "var not used in this function"*/ 1496b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICO_OK; 1497b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1498b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1499b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenpicodata_ProcessingUnit picotok_newTokenizeUnit(picoos_MemoryManager mm, picoos_Common common, 1500b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picodata_CharBuffer cbIn, picodata_CharBuffer cbOut, 1501b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picorsrc_Voice voice) 1502b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1503b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_subobj_t * tok; 1504b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picodata_ProcessingUnit this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice); 1505b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (this == NULL) { 1506b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return NULL; 1507b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1508b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen this->initialize = tokInitialize; 1509b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_DEBUG(("set this->step to tokStep")); 1510b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen this->step = tokStep; 1511b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen this->terminate = tokTerminate; 1512b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen this->subDeallocate = tokSubObjDeallocate; 1513b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen this->subObj = picoos_allocate(mm, sizeof(tok_subobj_t)); 1514b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (this->subObj == NULL) { 1515b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_deallocate(mm, (void *)&this); 1516b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return NULL; 1517b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1518b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok = (tok_subobj_t *) this->subObj; 1519b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->transducer = picotrns_newSimpleTransducer(mm, common, 10*(PICOTRNS_MAX_NUM_POSSYM+2)); 1520b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (NULL == tok->transducer) { 1521b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tokSubObjDeallocate(this,mm); 1522b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_deallocate(mm, (void *)&this); 1523b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return NULL; 1524b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 15257bc39b0d41efe0d8733490d54e14bc392d9f0b6dJean-Michel Trivi tokInitialize(this, PICO_RESET_FULL); 1526b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return this; 1527b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1528b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1529b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/** 1530b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen * fill up internal buffer, try to locate token, write token to output 1531b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen */ 1532b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chenpicodata_step_result_t tokStep(register picodata_ProcessingUnit this, 1533b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int16 mode, picoos_uint16 * numBytesOutput) 1534b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen{ 1535b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen register tok_subobj_t * tok; 1536b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1537b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (NULL == this || NULL == this->subObj) { 1538b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICODATA_PU_ERROR; 1539b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1540b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok = (tok_subobj_t *) this->subObj; 1541b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1542b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen mode = mode; /* avoid warning "var not used in this function"*/ 1543b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1544b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen *numBytesOutput = 0; 1545b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen while (1) { /* exit via return */ 1546b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen picoos_int16 ch; 1547b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1548b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if ((tok->outWritePos - tok->outReadPos) > 0) { 1549b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (picodata_cbPutItem(this->cbOut, &tok->outBuf[tok->outReadPos], tok->outWritePos - tok->outReadPos, numBytesOutput) == PICO_OK) { 1550b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG], 1551b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen (picoos_uint8 *)"tok:", &tok->outBuf[tok->outReadPos], tok->outWritePos - tok->outReadPos); 1552b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outReadPos += *numBytesOutput; 1553b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen if (tok->outWritePos == tok->outReadPos) { 1554b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outWritePos = 0; 1555b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok->outReadPos = 0; 1556b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1557b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1558b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 1559b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICODATA_PU_OUT_FULL; 1560b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1561b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1562b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1563b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else if (PICO_EOF != (ch = picodata_cbGetCh(this->cbIn))) { 1564b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen PICODBG_DEBUG(("read in %c", (picoos_char) ch)); 1565b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen tok_treatChar(this, tok, (picoos_uchar) ch, /*markupHandling*/TRUE); 1566b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1567b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen else { 1568b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen return PICODATA_PU_IDLE; 1569b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1570b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen } 1571b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1572b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1573b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#ifdef __cplusplus 1574b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen} 1575b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen#endif 1576b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen 1577b190149a69b110e6719ce0a41877a683f8db7ae7Charles Chen/* end */ 1578