xref: /external/svox/pico/lib/picowa.c

/*
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * @file picowa.c
 *
 * word analysis PU - lexicon lookup and POS prediction
 *
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 * All rights reserved.
 *
 * History:
 * - 2009-04-20 -- initial version
 *
 */

#include "picoos.h"
#include "picodbg.h"
#include "picodata.h"
#include "picowa.h"
#include "picoklex.h"
#include "picokdt.h"
#include "picoktab.h"

#ifdef __cplusplus
extern "C" {
#endif
#if 0
}
#endif

/* PU waStep states */
#define WA_STEPSTATE_COLLECT  0
#define WA_STEPSTATE_PROCESS  1
#define WA_STEPSTATE_FEED     2


/*  subobject    : WordAnaUnit
 *  shortcut     : wa
 *  context size : one item
 */
typedef struct wa_subobj {
    picoos_uint8 procState; /* for next processing step decision */

    /* one item only */
    picoos_uint8 inBuf[PICOWA_MAXITEMSIZE]; /* internal input buffer */
    picoos_uint16 inBufSize; /* actually allocated size */
    picoos_uint16 inLen; /* length of item in inBuf, 0 for empty buf */

    picoos_uint8 outBuf[PICOWA_MAXITEMSIZE]; /* internal output buffer */
    picoos_uint16 outBufSize; /* actually allocated size */
    picoos_uint16 outLen; /* length of item in outBuf, 0 for empty buf */

    /* lex knowledge base */
    picoklex_Lex lex;

    /* ulex knowledge bases */
    picoos_uint8 numUlex;
    picoklex_Lex ulex[PICOKNOW_MAX_NUM_ULEX];

    /* tab knowledge base */
    picoktab_Pos tabpos;

    /* dtposp knowledge base */
    picokdt_DtPosP dtposp;
} wa_subobj_t;


static pico_status_t waInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode) {
    picoos_uint8 i;
    picoklex_Lex ulex;
    wa_subobj_t * wa;

    picoknow_kb_id_t ulexKbIds[PICOKNOW_MAX_NUM_ULEX] = PICOKNOW_KBID_ULEX_ARRAY;

    PICODBG_DEBUG(("calling"));

    if (NULL == this || NULL == this->subObj) {
        return (picodata_step_result_t) picoos_emRaiseException(this->common->em,
                                       PICO_ERR_NULLPTR_ACCESS, NULL, NULL);
    }
    wa = (wa_subobj_t *) this->subObj;
    wa->procState = WA_STEPSTATE_COLLECT;
    wa->inBufSize = PICOWA_MAXITEMSIZE;
    wa->inLen = 0;
    wa->outBufSize = PICOWA_MAXITEMSIZE;
    wa->outLen = 0;

    if (resetMode == PICO_RESET_SOFT) {
        /*following initializations needed only at startup or after a full reset*/
        return PICO_OK;
    }
    /* kb lex */
    wa->lex = picoklex_getLex(this->voice->kbArray[PICOKNOW_KBID_LEX_MAIN]);
    if (wa->lex == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got lex"));

    /* kb ulex[] */
    wa->numUlex = 0;
    for (i = 0; i<PICOKNOW_MAX_NUM_ULEX; i++) {
        ulex = picoklex_getLex(this->voice->kbArray[ulexKbIds[i]]);
        if (NULL != ulex) {
            wa->ulex[wa->numUlex++] = ulex;
        }
    }
    PICODBG_DEBUG(("got %i user lexica", wa->numUlex));

    /* kb tabpos */
    wa->tabpos =
        picoktab_getPos(this->voice->kbArray[PICOKNOW_KBID_TAB_POS]);
    if (wa->tabpos == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got tabpos"));

    /* kb dtposp */
    wa->dtposp = picokdt_getDtPosP(this->voice->kbArray[PICOKNOW_KBID_DT_POSP]);
    if (wa->dtposp == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got dtposp"));
    return PICO_OK;
}

static picodata_step_result_t waStep(register picodata_ProcessingUnit this,
                                     picoos_int16 mode,
                                     picoos_uint16 *numBytesOutput);

static pico_status_t waTerminate(register picodata_ProcessingUnit this) {
    return PICO_OK;
}

static pico_status_t waSubObjDeallocate(register picodata_ProcessingUnit this,
                                        picoos_MemoryManager mm) {
    if (NULL != this) {
        picoos_deallocate(this->common->mm, (void *) &this->subObj);
    }
    mm = mm;        /* avoid warning "var not used in this function"*/
    return PICO_OK;
}


picodata_ProcessingUnit picowa_newWordAnaUnit(picoos_MemoryManager mm,
                                              picoos_Common common,
                                              picodata_CharBuffer cbIn,
                                              picodata_CharBuffer cbOut,
                                              picorsrc_Voice voice) {
    picodata_ProcessingUnit this;

    this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice);
    if (this == NULL) {
        return NULL;
    }

    this->initialize = waInitialize;
    PICODBG_DEBUG(("set this->step to waStep"));
    this->step = waStep;
    this->terminate = waTerminate;
    this->subDeallocate = waSubObjDeallocate;
    this->subObj = picoos_allocate(mm, sizeof(wa_subobj_t));
    if (this->subObj == NULL) {
        picoos_deallocate(mm, (void *)&this);
        picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, NULL, NULL);
        return NULL;
    }

    waInitialize(this, PICO_RESET_FULL);
    return this;
}

/* ***********************************************************************/
/*                       WORDGRAPH proc functions                        */
/* ***********************************************************************/

static picoos_uint8 waClassifyPos(register picodata_ProcessingUnit this,
                                  register wa_subobj_t *wa,
                                  const picoos_uint8 *graph,
                                  const picoos_uint16 graphlen) {
    picokdt_classify_result_t dtres;
    picoos_uint8 specchar;
    picoos_uint16 i;

    PICODBG_DEBUG(("graphlen %d", graphlen));

    /* check existence of special char (e.g. hyphen) in graph:
       for now, check existence of hard-coded ascii hyphen,
       ie. preproc needs to match all UTF8 hyphens to the ascii
       hyphen. */
    /*  @todo : consider specifying special char(s) in lingware. */
    specchar = FALSE;
    i = 0;
    while ((i < graphlen) && (!specchar)) {
        if (graph[i++] == '-') {
            specchar = TRUE;
        }
    }

    /* construct input vector, which is set in dtposp */
    if (!picokdt_dtPosPconstructInVec(wa->dtposp, graph, graphlen, specchar)) {
        /* error constructing invec */
        PICODBG_WARN(("problem with invec"));
        picoos_emRaiseWarning(this->common->em, PICO_WARN_INVECTOR, NULL, NULL);
        return PICODATA_ITEMINFO1_ERR;
    }

    /* classify */
    if (!picokdt_dtPosPclassify(wa->dtposp)) {
        /* error doing classification */
        PICODBG_WARN(("problem classifying"));
        picoos_emRaiseWarning(this->common->em, PICO_WARN_CLASSIFICATION,
                              NULL, NULL);
        return PICODATA_ITEMINFO1_ERR;
    }

    /* decompose */
    if (!picokdt_dtPosPdecomposeOutClass(wa->dtposp, &dtres)) {
        /* error decomposing */
        PICODBG_WARN(("problem decomposing"));
        picoos_emRaiseWarning(this->common->em, PICO_WARN_OUTVECTOR,
                              NULL, NULL);
        return PICODATA_ITEMINFO1_ERR;
    }

    if (dtres.set) {
        PICODBG_DEBUG(("class %d", dtres.class));
        return (picoos_uint8)dtres.class;
    } else {
        PICODBG_WARN(("result not set"));
        picoos_emRaiseWarning(this->common->em, PICO_WARN_CLASSIFICATION,
                              NULL, NULL);
        return PICODATA_ITEMINFO1_ERR;
    }
}


static pico_status_t waProcessWordgraph(register picodata_ProcessingUnit this,
                                        register wa_subobj_t *wa /*inout*/,
                                        picodata_itemhead_t *head /*inout*/,
                                        const picoos_uint8 *content) {
    pico_status_t status;
    picoklex_lexl_result_t lexres;
    picoos_uint8 posbuf[PICOKTAB_MAXNRPOS_IN_COMB];
    picoos_uint8 i;
    picoos_uint8 foundIndex;
    picoos_bool found;


    PICODBG_DEBUG(("type %c, len %d", head->type, head->len));

    /* do lookup
       if no entry found:
         do POS prediction:     -> WORDGRAPH(POSes,NA)graph
       else:
         if incl-phone:
           N entries possible  -> WORDINDEX(POSes,NA)POS1|ind1...POSN|indN
           (N in {1,...,PICOKLEX_MAX_NRRES}, now up to 4)
         else:
           no phone, one entry  -> WORDGRAPH(POS,NA)graph
    */

    found = FALSE;
    i = 0;
    while (!found && (i < wa->numUlex)) {
        found = picoklex_lexLookup(wa->ulex[i], content, head->len, &lexres);
        i++;
    }
    /* note that if found, i will be incremented nevertheless, so i >= 1 */
    if (found) {
        foundIndex = i;
    } else {
        foundIndex = 0;
    }
    if (!found && !picoklex_lexLookup(wa->lex, content, head->len, &lexres)) {
        /* no lex entry found, WORDGRAPH(POS,NA)graph */
        if (PICO_OK == picodata_copy_item(wa->inBuf, wa->inLen,
                                          wa->outBuf, wa->outBufSize,
                                          &wa->outLen)) {
            wa->inLen = 0;
            /* predict and modify pos in info1 */
            if (PICO_OK != picodata_set_iteminfo1(wa->outBuf, wa->outLen,
                                   waClassifyPos(this, wa, content, head->len))) {
                return picoos_emRaiseException(this->common->em,
                                               PICO_EXC_BUF_OVERFLOW,NULL,NULL);
            }
        }

    } else {    /* at least one entry found */
        PICODBG_DEBUG(("at least one entry found in lexicon %i",foundIndex));
        if (lexres.phonfound) {    /* incl. ind-phone and possibly multi-ent. */
            if (lexres.nrres > PICOKLEX_MAX_NRRES) {
                /* not possible with system lexicon, needs to be
                   ensured for user lex too */
                picoos_emRaiseWarning(this->common->em, PICO_WARN_FALLBACK,NULL,
                        (picoos_char *)"using %d lexicon lookup results",
                        PICOKLEX_MAX_NRRES);
                lexres.nrres = PICOKLEX_MAX_NRRES;
            }
            head->type = PICODATA_ITEM_WORDINDEX;
            if (lexres.nrres == 1) {
                head->info1 = lexres.posind[0];
            } else {
                /* more than one result, POSgroup info needs to be
                   determined for later POS disambiguation */
                for (i = 0; i < lexres.nrres; i++) {
                    posbuf[i] = lexres.posind[i * PICOKLEX_POSIND_SIZE];
                }
                head->info1 = picoktab_getPosGroup(wa->tabpos, posbuf,
                                                   lexres.nrres);
            }
            head->info2 = foundIndex;
            head->len = lexres.posindlen;
            if ((status = picodata_put_itemparts(head, lexres.posind,
                                                 lexres.posindlen,
                                                 wa->outBuf, wa->outBufSize,
                                                 &wa->outLen)) == PICO_OK) {
                wa->inLen = 0;
            } else {
                return picoos_emRaiseException(this->common->em, status,
                                               NULL, NULL);
            }

        } else {    /* no phone, :G2P, one entry: WORDGRAPH(POS,NA)graph */
            if (PICO_OK == picodata_copy_item(wa->inBuf, wa->inLen,
                                              wa->outBuf, wa->outBufSize,
                                              &wa->outLen)) {
                wa->inLen = 0;
                /* set lex pos in info1 */
                if (PICO_OK != picodata_set_iteminfo1(wa->outBuf, wa->outLen,
                                                      lexres.posind[0])) {
                    return picoos_emRaiseException(this->common->em,
                                                   PICO_EXC_BUF_OVERFLOW,
                                                   NULL, NULL);
                }
            }
        }
    }
    return PICO_OK;
}


/* ***********************************************************************/
/*                          waStep function                              */
/* ***********************************************************************/

/*
   collect into internal buffer, process, and then feed to output buffer

   init state: COLLECT      ext      ext
   state transitions:       in IN OUTout
   COLLECT | getOneItem  ->-1 +1  0  0   | (ATOMIC) -> PROCESS (got item)
   COLLECT | getOneItem  -> 0  0  0  0   | IDLE                (got no item)

   PROCESS | procOneItem -> 0 -1 +1  0   | (ATOMIC) -> FEED    (proc'ed item)
   PROCESS | procOneItem -> 0 -1  0  0   | BUSY     -> COLLECT (item skipped)

   FEED    | putOneItem  -> 0  0 -1 +1   | BUSY     -> COLLECT (put item)
   FEED    | putOneItem  -> 0  0  1  0   | OUT_FULL            (put no item)
*/

static picodata_step_result_t waStep(register picodata_ProcessingUnit this,
                                     picoos_int16 mode,
                                     picoos_uint16 * numBytesOutput) {
    register wa_subobj_t *wa;
    pico_status_t rv = PICO_OK;

    if (NULL == this || NULL == this->subObj) {
        return PICODATA_PU_ERROR;
    }
    wa = (wa_subobj_t *) this->subObj;
    mode = mode;        /* avoid warning "var not used in this function"*/
    *numBytesOutput = 0;
    while (1) { /* exit via return */
        PICODBG_DEBUG(("doing state %i, inLen: %d, outLen: %d",
                       wa->procState, wa->inLen, wa->outLen));

        switch (wa->procState) {
            /* collect state: get item from charBuf and store in
             * internal inBuf
             */
            case WA_STEPSTATE_COLLECT:
                if (wa->inLen == 0) { /* is input buffer empty? */
                    picoos_uint16 blen;
                    /* try to get one item */
                    rv = picodata_cbGetItem(this->cbIn, wa->inBuf,
                                            wa->inBufSize, &blen);
                    PICODBG_DEBUG(("after getting item, status: %d", rv));
                    if (PICO_OK == rv) {
                        /* we now have one item */
                        wa->inLen = blen;
                        wa->procState = WA_STEPSTATE_PROCESS;
                        /* uncomment next line to split into two steps */
                        /* return PICODATA_PU_ATOMIC; */
                    } else if (PICO_EOF == rv) {
                        /* there was no item in the char buffer */
                        return PICODATA_PU_IDLE;
                    } else if ((PICO_EXC_BUF_UNDERFLOW == rv)
                               || (PICO_EXC_BUF_OVERFLOW == rv)) {
                        PICODBG_ERROR(("problem getting item"));
                        picoos_emRaiseException(this->common->em, rv,
                                                NULL, NULL);
                        return PICODATA_PU_ERROR;
                    } else {
                        PICODBG_ERROR(("problem getting item, unhandled"));
                        picoos_emRaiseException(this->common->em, rv,
                                                NULL, NULL);
                        return PICODATA_PU_ERROR;
                    }
                } else { /* there already is an item in the input buffer */
                    PICODBG_WARN(("item already in input buffer"));
                    picoos_emRaiseWarning(this->common->em,
                                          PICO_WARN_PU_IRREG_ITEM, NULL, NULL);
                    wa->procState = WA_STEPSTATE_PROCESS;
                    /* uncomment next to split into two steps */
                    /* return PICODATA_PU_ATOMIC; */
                }
                break;


            /* process state: process item in internal inBuf and put
             * result in internal outBuf
             */
            case WA_STEPSTATE_PROCESS:

                /* ensure there is an item in inBuf and it is valid */
                if ((wa->inLen > 0) && picodata_is_valid_item(wa->inBuf,
                                                              wa->inLen)) {
                    picodata_itemhead_t ihead;
                    picoos_uint8 *icontent;
                    pico_status_t rvP = PICO_OK;

                    rv = picodata_get_iteminfo(wa->inBuf, wa->inLen, &ihead,
                                               &icontent);
                    if (PICO_OK == rv) {

                        switch (ihead.type) {
                            case PICODATA_ITEM_WORDGRAPH:

                                if (0 < ihead.len) {
                                    rvP = waProcessWordgraph(this, wa, &ihead,
                                                             icontent);
                                } else {
                                    /* else ignore empty WORDGRAPH */
                                    wa->inLen = 0;
                                    wa->procState = WA_STEPSTATE_COLLECT;
                                    return PICODATA_PU_BUSY;
                                }
                                break;
                            case PICODATA_ITEM_OTHER:
                                /* skip item */
                                rvP = PICO_WARN_PU_DISCARD_BUF;
                                break;
                            default:
                                /* copy item unmodified */
                                rvP = picodata_copy_item(wa->inBuf,
                                                         wa->inLen, wa->outBuf,
                                                         wa->outBufSize, &wa->outLen);
                                break;
                        }

                        if (PICO_OK == rvP) {
                            wa->inLen = 0;
                            wa->procState = WA_STEPSTATE_FEED;
                            /* uncomment next to split into two steps */
                            /* return PICODATA_PU_ATOMIC; */
                        } else if (PICO_WARN_PU_DISCARD_BUF == rvP) {
                            /* discard input buffer and get a new item */
                            PICODBG_INFO(("skipping OTHER item"));
/*                            picoos_emRaiseWarning(this->common->em,
                                                  PICO_WARN_PU_DISCARD_BUF, NULL, NULL);
*/
                            wa->inLen = 0;
                            wa->procState = WA_STEPSTATE_COLLECT;
                            return PICODATA_PU_BUSY;
                        } else {
                            /* PICO_EXC_BUF_OVERFLOW   <- overflow in outbuf
                               PICO_ERR_OTHER          <- no valid item in inbuf
                               or return from processWordgraph
                            */
                            PICODBG_ERROR(("problem processing item", rvP));
                            picoos_emRaiseException(this->common->em, rvP,
                                                    NULL, NULL);
                            return PICODATA_PU_ERROR;
                        }

                    } else {    /* could not get iteminfo */
                        /* PICO_EXC_BUF_OVERFLOW   <- overflow in outbuf
                           PICO_ERR_OTHER          <- no valid item in inbuf
                        */
                        PICODBG_ERROR(("problem getting item info, "
                                       "discard buffer content"));
                        wa->inLen = 0;
                        wa->procState = WA_STEPSTATE_COLLECT;
                        picoos_emRaiseException(this->common->em, rv,
                                                NULL, NULL);
                        return PICODATA_PU_ERROR;
                    }

                } else if (wa->inLen == 0) {    /* no item in inBuf */
                    PICODBG_INFO(("no item in inBuf"));
                    /* wa->inLen = 0;*/
                    wa->procState = WA_STEPSTATE_COLLECT;
                    return PICODATA_PU_BUSY;

                } else {    /* no valid item in inBuf */
                    /* bad state/item, discard buffer content */
                    PICODBG_WARN(("no valid item, discard buffer content"));
                    picoos_emRaiseWarning(this->common->em,
                                          PICO_WARN_PU_IRREG_ITEM, NULL, NULL);
                    picoos_emRaiseWarning(this->common->em,
                                          PICO_WARN_PU_DISCARD_BUF, NULL, NULL);
                    wa->inLen = 0;
                    wa->procState = WA_STEPSTATE_COLLECT;
                    return PICODATA_PU_BUSY;
                }
                break;


            /* feed state: copy item in internal outBuf to output charBuf */
            case WA_STEPSTATE_FEED:

                /* check that item fits in cb should not be needed */
                rv = picodata_cbPutItem(this->cbOut, wa->outBuf,
                                        wa->outLen, numBytesOutput);

                PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
                                   (picoos_uint8 *)"wana: ", wa->outBuf,
                                   wa->outLen);

                PICODBG_DEBUG(("put item, status: %d", rv));
                if (PICO_OK == rv) {
                    wa->outLen = 0;
                    wa->procState = WA_STEPSTATE_COLLECT;
                    return PICODATA_PU_BUSY;
                } else if (PICO_EXC_BUF_OVERFLOW == rv) {
                    PICODBG_INFO(("feeding, overflow, PICODATA_PU_OUT_FULL"));
                    return PICODATA_PU_OUT_FULL;
                } else if ((PICO_EXC_BUF_UNDERFLOW == rv)
                           || (PICO_ERR_OTHER == rv)) {
                    PICODBG_WARN(("feeding problem, discarding item"));
                    wa->outLen = 0;
                    wa->procState = WA_STEPSTATE_COLLECT;
                    picoos_emRaiseWarning(this->common->em, rv, NULL,NULL);
                    return PICODATA_PU_BUSY;
                }
                break;

            default:
                break;

        } /* switch */

    } /* while */

    /* should be never reached */
    PICODBG_ERROR(("reached end of function"));
    picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL);
    return PICODATA_PU_ERROR;
}

#ifdef __cplusplus
}
#endif


/* end */