pico/lib/picoktab.c

/*
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * @file picoktab.c
 *
 * symbol tables needed at runtime
 *
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 * All rights reserved.
 *
 * History:
 * - 2009-04-20 -- initial version
 *
 */

#include "picoos.h"
#include "picodbg.h"
#include "picoknow.h"
#include "picobase.h"
#include "picoktab.h"
#include "picodata.h"

#ifdef __cplusplus
extern "C" {
#endif
#if 0
}
#endif


/** @todo : the following would be better part of a knowledge base.
 * Make sure it is consistent with the phoneme symbol table used in the lingware */

/* PLANE_PHONEMES */

/* PLANE_POS */

/* PLANE_PB_STRENGTHS */

/* PLANE_ACCENTS */

/* PLANE_INTERN */
#define PICOKTAB_TMPID_PHONSTART      '\x26'  /* 38  '&' */
#define PICOKTAB_TMPID_PHONTERM       '\x23'  /* 35  '#' */


/* ************************************************************/
/* fixed ids */
/* ************************************************************/


static pico_status_t ktabIdsInitialize(register picoknow_KnowledgeBase this,
                                       picoos_Common common)
{
    picoktab_FixedIds ids;

    PICODBG_DEBUG(("start"));

    if (NULL == this || NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    ids = (picoktab_FixedIds) this->subObj;

    ids->phonStartId = PICOKTAB_TMPID_PHONSTART;
    ids->phonTermId = PICOKTAB_TMPID_PHONTERM;
    return PICO_OK;
}


static pico_status_t ktabIdsSubObjDeallocate(register picoknow_KnowledgeBase this,
                                             picoos_MemoryManager mm)
{
    if (NULL != this) {
        picoos_deallocate(mm, (void *) &this->subObj);
    }
    return PICO_OK;
}

pico_status_t picoktab_specializeIdsKnowledgeBase(picoknow_KnowledgeBase this,
                                                  picoos_Common common)
{
    if (NULL == this) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    this->subDeallocate = ktabIdsSubObjDeallocate;
    this->subObj = picoos_allocate(common->mm, sizeof(picoktab_fixed_ids_t));
    if (NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM,
                                       NULL, NULL);
    }
    return ktabIdsInitialize(this, common);
}

picoktab_FixedIds picoktab_getFixedIds(picoknow_KnowledgeBase this)
{
    return ((NULL == this) ? NULL : ((picoktab_FixedIds) this->subObj));
}


picoktab_FixedIds picoktab_newFixedIds(picoos_MemoryManager mm)
{
    picoktab_FixedIds this = (picoktab_FixedIds) picoos_allocate(mm,sizeof(*this));
    if (NULL != this) {
        /* initialize */
    }
    return this;
}


void picoktab_disposeFixedIds(picoos_MemoryManager mm, picoktab_FixedIds * this)
{
    if (NULL != (*this)) {
        /* terminate */
        picoos_deallocate(mm,(void *)this);
    }
}


/* ************************************************************/
/* Graphs */
/* ************************************************************/

/* overview binary file format for graphs kb:

    graphs-kb = NROFSENTRIES SIZEOFSENTRY ofstable graphs

    NROFSENTRIES  : 2 bytes, number of entries in offset table
    SIZEOFSENTRY  : 1 byte,  size of one entry in offset table

    ofstable = {OFFSET}=NROFSENTRIES (contains NROFSENTRIES entries of OFFSET)

    OFFSET: SIZEOFSENTRY bytes, offset to baseaddress of graphs-kb to entry in graphs

    graphs = {graph}=NROFSENTRIES (contains NROFSENTRIES entries of graph)

    graph = PROPSET FROM TO [TOKENTYPE] [TOKENSUBTYPE] [VALUE] [LOWERCASE] [GRAPHSUBS1] [GRAPHSUBS2]

    FROM          : 1..4 unsigned bytes, UTF8 character without terminating 0
    TO            : 1..4 unsigned bytes, UTF8 character without terminating 0
    PROPSET       : 1 unsigned byte, least significant bit : has TO field
                                                             next bit : has TOKENTYPE
                                                             next bit : has TOKENSUBTYPE
                                                             next bit : has VALUE
                                                             next bit : has LOWERCASE
                                                             next bit : has GRAPHSUBS1
                                                             next bit : has GRAPHSUBS2
                                                             next bit : has PUNC

    TOKENTYPE    : 1 unsigned byte
    TOKENSUBTYPE : 1 unsigned byte
    VALUE        : 1 unsigned byte
    LOWERCASE    : 1..4 unsigned bytes, UTF8 character without terminating 0
    GRAPHSUBS1   : 1..4 unsigned bytes, UTF8 character without terminating 0
    GRAPHSUBS2   : 1..4 unsigned bytes, UTF8 character without terminating 0
    PUNC         : 1 unsigned byte
*/

static picoos_uint32 ktab_propOffset (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint32 prop);

#define KTAB_START_GRAPHS_NR_OFFSET     0
#define KTAB_START_GRAPHS_SIZE_OFFSET   2
#define KTAB_START_GRAPHS_OFFSET_TABLE  3
#define KTAB_START_GRAPHS_GRAPH_TABLE   0

/* bitmasks to extract the grapheme properties info from the property set */
#define KTAB_GRAPH_PROPSET_TO            ((picoos_uint8)'\x01')
#define KTAB_GRAPH_PROPSET_TOKENTYPE     ((picoos_uint8)'\x02')
#define KTAB_GRAPH_PROPSET_TOKENSUBTYPE  ((picoos_uint8)'\x04')
#define KTAB_GRAPH_PROPSET_VALUE         ((picoos_uint8)'\x08')
#define KTAB_GRAPH_PROPSET_LOWERCASE     ((picoos_uint8)'\x010')
#define KTAB_GRAPH_PROPSET_GRAPHSUBS1    ((picoos_uint8)'\x020')
#define KTAB_GRAPH_PROPSET_GRAPHSUBS2    ((picoos_uint8)'\x040')
#define KTAB_GRAPH_PROPSET_PUNCT         ((picoos_uint8)'\x080')


typedef struct ktabgraphs_subobj *ktabgraphs_SubObj;

typedef struct ktabgraphs_subobj {
    picoos_uint16 nrOffset;
    picoos_uint16 sizeOffset;

    picoos_uint8 * offsetTable;
    picoos_uint8 * graphTable;
} ktabgraphs_subobj_t;


static pico_status_t ktabGraphsInitialize(register picoknow_KnowledgeBase this,
                                          picoos_Common common) {
    ktabgraphs_subobj_t * ktabgraphs;

    PICODBG_DEBUG(("start"));

    if (NULL == this || NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    ktabgraphs = (ktabgraphs_subobj_t *) this->subObj;
    ktabgraphs->nrOffset = ((int)(this->base[KTAB_START_GRAPHS_NR_OFFSET])) + 256*(int)(this->base[KTAB_START_GRAPHS_NR_OFFSET+1]);
    ktabgraphs->sizeOffset  = (int)(this->base[KTAB_START_GRAPHS_SIZE_OFFSET]);
    ktabgraphs->offsetTable = &(this->base[KTAB_START_GRAPHS_OFFSET_TABLE]);
    ktabgraphs->graphTable  = &(this->base[KTAB_START_GRAPHS_GRAPH_TABLE]);
    return PICO_OK;
}

static pico_status_t ktabGraphsSubObjDeallocate(register picoknow_KnowledgeBase this,
                                                picoos_MemoryManager mm) {
    if (NULL != this) {
        picoos_deallocate(mm, (void *) &this->subObj);
    }
    return PICO_OK;
}


pico_status_t picoktab_specializeGraphsKnowledgeBase(picoknow_KnowledgeBase this,
                                                     picoos_Common common) {
    if (NULL == this) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    this->subDeallocate = ktabGraphsSubObjDeallocate;
    this->subObj = picoos_allocate(common->mm, sizeof(ktabgraphs_subobj_t));
    if (NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM,
                                       NULL, NULL);
    }
    return ktabGraphsInitialize(this, common);
}


picoktab_Graphs picoktab_getGraphs(picoknow_KnowledgeBase this) {
    if (NULL == this) {
        return NULL;
    } else {
        return (picoktab_Graphs) this->subObj;
    }
}


/* Graphs methods */

picoos_uint8 picoktab_hasVowellikeProp(const picoktab_Graphs this,
                                       const picoos_uint8 *graph,
                                       const picoos_uint8 graphlenmax) {

  picoos_uint8 ui8App;
  picoos_uint32 graphsOffset;
  ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this;

  ui8App = graphlenmax;        /* avoid warning "var not used in this function"*/

  graphsOffset = picoktab_graphOffset (this, (picoos_uchar *)graph);
  return g->graphTable[graphsOffset + ktab_propOffset (this, graphsOffset, KTAB_GRAPH_PROPSET_TOKENTYPE)] == PICODATA_ITEMINFO1_TOKTYPE_LETTERV;
}


static void ktab_getStrProp (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint32 propOffset, picoos_uchar * str)
{
  ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this;
  picoos_uint32 i, l;

  i = 0;
  l = picobase_det_utf8_length(g->graphTable[graphsOffset+propOffset]);
  while (i<l) {
    str[i] = g->graphTable[graphsOffset+propOffset+i];
    i++;
  }
  str[l] = 0;
}


static picoos_uint32 ktab_propOffset(const picoktab_Graphs this,
        picoos_uint32 graphsOffset, picoos_uint32 prop)
/* Returns offset of property 'prop' inside the graph with offset 'graphsOffset' in graphs table;
 If the property is found, a value > 0 is returned otherwise 0 */
{
    picoos_uint32 n = 0;
    ktabgraphs_subobj_t * g = (ktabgraphs_SubObj) this;

    if ((g->graphTable[graphsOffset] & prop) == prop) {
        n = n + 1; /* overread PROPSET field */
        n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread FROM field */
        if (prop > KTAB_GRAPH_PROPSET_TO) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_TO)
                    == KTAB_GRAPH_PROPSET_TO) {
                n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread TO field */
            }
        } else {
            return n;
        }
        if (prop > KTAB_GRAPH_PROPSET_TOKENTYPE) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_TOKENTYPE)
                    == KTAB_GRAPH_PROPSET_TOKENTYPE) {
                n = n + 1; /* overread TOKENTYPE field */
            }
        } else {
            return n;
        }
        if (prop > KTAB_GRAPH_PROPSET_TOKENSUBTYPE) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_TOKENSUBTYPE)
                    == KTAB_GRAPH_PROPSET_TOKENSUBTYPE) {
                n = n + 1; /* overread stokentype field */
            }
        } else {
            return n;
        }
        if (prop > KTAB_GRAPH_PROPSET_VALUE) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_VALUE)
                    == KTAB_GRAPH_PROPSET_VALUE) {
                n = n + 1; /* overread value field */
            }
        } else {
            return n;
        }
        if (prop > KTAB_GRAPH_PROPSET_LOWERCASE) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_LOWERCASE)
                    == KTAB_GRAPH_PROPSET_LOWERCASE) {
                n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread lowercase field */
            }
        } else {
            return n;
        }
        if (prop > KTAB_GRAPH_PROPSET_GRAPHSUBS1) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_GRAPHSUBS1)
                    == KTAB_GRAPH_PROPSET_GRAPHSUBS1) {
                n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread graphsubs1 field */
            }
        } else {
            return n;
        }
        if (prop > KTAB_GRAPH_PROPSET_GRAPHSUBS2) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_GRAPHSUBS2)
                    == KTAB_GRAPH_PROPSET_GRAPHSUBS2) {
                n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread graphsubs2 field */
            }
        } else {
            return n;
        }
        if (prop > KTAB_GRAPH_PROPSET_PUNCT) {
            if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_PUNCT)
                    == KTAB_GRAPH_PROPSET_PUNCT) {
                n = n + 1; /* overread value field */
            }
        } else {
            return n;
        }
    }

    return n;
}


picoos_uint32 picoktab_graphOffset (const picoktab_Graphs this, picoos_uchar * utf8graph)
{  ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this;
   picoos_int32 a, b, m;
   picoos_uint32 graphsOffset;
   picoos_uint32 propOffset;
   picobase_utf8char from;
   picobase_utf8char to;
   picoos_bool utfGEfrom;
   picoos_bool utfLEto;

   if (g->nrOffset > 0) {
     a = 0;
     b = g->nrOffset-1;
     do  {
       m = (a+b) / 2;

       /* get offset to graph[m] */
       if (g->sizeOffset == 1) {
         graphsOffset = g->offsetTable[g->sizeOffset*m];
       }
       else {
         graphsOffset =     g->offsetTable[g->sizeOffset*m    ] +
                        256*g->offsetTable[g->sizeOffset*m + 1];
         /* PICODBG_DEBUG(("picoktab_graphOffset: %i %i %i %i", m, g->offsetTable[g->sizeOffset*m], g->offsetTable[g->sizeOffset*m + 1], graphsOffset));
         */
       }

       /* get FROM and TO field of graph[m] */
       ktab_getStrProp(this, graphsOffset, 1, from);
       propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_TO);
       if (propOffset > 0) {
         ktab_getStrProp(this, graphsOffset, propOffset, to);
       }
       else {
         picoos_strcpy((picoos_char *)to, (picoos_char *)from);
       }

       /* PICODBG_DEBUG(("picoktab_graphOffset: %i %i %i '%s' '%s' '%s'", a, m, b, from, utf8graph, to));
       */
       utfGEfrom = picoos_strcmp((picoos_char *)utf8graph, (picoos_char *)from) >= 0;
       utfLEto = picoos_strcmp((picoos_char *)utf8graph, (picoos_char *)to) <= 0;

       if (utfGEfrom && utfLEto) {
         /* PICODBG_DEBUG(("picoktab_graphOffset: utf char '%s' found", utf8graph));
          */
         return graphsOffset;
       }
       if (!utfGEfrom) {
         b = m-1;
       }
       else if (!utfLEto) {
         a = m+1;
       }
     } while (a<=b);
   }
   PICODBG_DEBUG(("picoktab_graphOffset: utf char '%s' not found", utf8graph));
   return 0;
}


picoos_bool  picoktab_getIntPropTokenType (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint8 * stokenType)
{
  picoos_uint32 propOffset;
  ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this;

  propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_TOKENTYPE);
  if (propOffset > 0) {
    *stokenType = (picoos_uint8)(g->graphTable[graphsOffset+propOffset]);
    return TRUE;
  }
  else {
    return FALSE;
  }
}


picoos_bool  picoktab_getIntPropTokenSubType (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_int8 * stokenSubType)
{
  picoos_uint32 propOffset;
  ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this;

  propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_TOKENSUBTYPE);
  if (propOffset > 0) {
    *stokenSubType = (picoos_int8)(g->graphTable[graphsOffset+propOffset]);
    return TRUE;
  }
  else {
    return FALSE;
  }
}

picoos_bool  picoktab_getIntPropValue (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint32 * value)
{
  picoos_uint32 propOffset;
  ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this;

  propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_VALUE);
  if (propOffset > 0) {
    *value = (picoos_uint32)(g->graphTable[graphsOffset+propOffset]);
    return TRUE;
  }
  else {
    return FALSE;
  }
}


picoos_bool  picoktab_getIntPropPunct (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint8 * info1, picoos_uint8 * info2)
{
  picoos_uint32 propOffset;
  ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this;

  propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_PUNCT);
  if (propOffset > 0) {
      if (g->graphTable[graphsOffset+propOffset] == 2) {
          *info1 = PICODATA_ITEMINFO1_PUNC_SENTEND;
      }
      else {
          *info1 = PICODATA_ITEMINFO1_PUNC_PHRASEEND;
      }
    if (g->graphTable[graphsOffset+1] == '.') {
        *info2 = PICODATA_ITEMINFO2_PUNC_SENT_T;
    }
    else if (g->graphTable[graphsOffset+1] == '?') {
        *info2 = PICODATA_ITEMINFO2_PUNC_SENT_Q;
    }
    else if (g->graphTable[graphsOffset+1] == '!') {
        *info2 = PICODATA_ITEMINFO2_PUNC_SENT_E;
    }
    else {
        *info2 = PICODATA_ITEMINFO2_PUNC_PHRASE;
    }
    return TRUE;
  }
  else {
    return FALSE;
  }
}


picoos_bool  picoktab_getStrPropLowercase (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uchar * lowercase)
{
  picoos_uint32 propOffset;

  propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_LOWERCASE);
  if (propOffset > 0) {
    ktab_getStrProp(this, graphsOffset, propOffset, lowercase);
    return TRUE;
  }
  else {
    return FALSE;
  }
}


picoos_bool  picoktab_getStrPropGraphsubs1 (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uchar * graphsubs1)
{
  picoos_uint32 propOffset;

  propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_GRAPHSUBS1);
  if (propOffset > 0) {
    ktab_getStrProp(this, graphsOffset, propOffset, graphsubs1);
    return TRUE;
  }
  else {
    return FALSE;
  }
}


picoos_bool  picoktab_getStrPropGraphsubs2 (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uchar * graphsubs2)
{
  picoos_uint32 propOffset;

  propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_GRAPHSUBS2);
  if (propOffset > 0) {
    ktab_getStrProp(this, graphsOffset, propOffset, graphsubs2);
    return TRUE;
  }
  else {
    return FALSE;
  }
}
/* *****************************************************************/
/* used for tools */

static void ktab_getUtf8 (picoos_uchar ** pos, picoos_uchar * to)
{
  picoos_uint32 l;
  l = picobase_det_utf8_length(**pos);
  while (l>0) {
    *(to++) = *((*pos)++);
    l--;
  }
  *to = 0;
}

picoos_uint16 picoktab_graphsGetNumEntries(const picoktab_Graphs this)
{
    ktabgraphs_subobj_t * g = (ktabgraphs_SubObj) this;
    return g->nrOffset;
}

void picoktab_graphsGetGraphInfo(const picoktab_Graphs this,
        picoos_uint16 graphIndex, picoos_uchar * from, picoos_uchar * to,
        picoos_uint8 * propset,
        picoos_uint8 * stokenType, picoos_uint8 * stokenSubType,
        picoos_uint8 * value, picoos_uchar * lowercase,
        picoos_uchar * graphsubs1, picoos_uchar * graphsubs2,
        picoos_uint8 * punct) {
    ktabgraphs_subobj_t * g = (ktabgraphs_SubObj) this;
    picoos_uint32 graphsOffset;
    picoos_uint8 * pos;

    /* calculate offset of graph[graphIndex] */
    if (g->sizeOffset == 1) {
        graphsOffset = g->offsetTable[graphIndex];
    } else {
        graphsOffset = g->offsetTable[2 * graphIndex]
                + (g->offsetTable[2 * graphIndex + 1] << 8);
    }
    pos = &(g->graphTable[graphsOffset]);
    *propset = *pos;

    pos++; /* advance to FROM */
    ktab_getUtf8(&pos, from); /* get FROM and advance */
    if ((*propset) & KTAB_GRAPH_PROPSET_TO) {
        ktab_getUtf8(&pos, to); /* get TO and advance */
    } else {
        picoos_strcpy((picoos_char *)to, (picoos_char *)from);
    }
    if ((*propset) & KTAB_GRAPH_PROPSET_TOKENTYPE) {
        (*stokenType) = *(pos++); /* get TOKENTYPE and advance */
    } else {
        (*stokenType) = -1;
    }
    if ((*propset) & KTAB_GRAPH_PROPSET_TOKENSUBTYPE) {
        (*stokenSubType) = *(pos++); /* get TOKENSUBTYPE and advance */
    } else {
        (*stokenSubType) = -1;
    }
    if ((*propset) & KTAB_GRAPH_PROPSET_VALUE) {
        (*value) = *(pos++); /* get VALUE and advance */
    } else {
        (*value) = -1;
    }
    if ((*propset) & KTAB_GRAPH_PROPSET_LOWERCASE) {
        ktab_getUtf8(&pos, lowercase); /* get LOWERCASE and advance */
    } else {
        lowercase[0] = NULLC;
    }
    if ((*propset) & KTAB_GRAPH_PROPSET_GRAPHSUBS1) {
        ktab_getUtf8(&pos, graphsubs1); /* get GRAPHSUBS1 and advance */
    } else {
        graphsubs1[0] = NULLC;
    }
    if ((*propset) & KTAB_GRAPH_PROPSET_GRAPHSUBS2) {
        ktab_getUtf8(&pos, graphsubs2); /* get GRAPHSUBS2 and advance */
    } else {
        graphsubs2[0] = NULLC;
    }
    if ((*propset) & KTAB_GRAPH_PROPSET_PUNCT) {
        (*punct) = *(pos++); /* get PUNCT and advance */
    } else {
        (*punct) = -1;
    }
}

/* ************************************************************/
/* Phones */
/* ************************************************************/

/* overview binary file format for phones kb:

    phones-kb = specids propertytable

    specids = PRIMSTRESSID1 SECSTRESSID1 SYLLBOUNDID1 PAUSEID1 WORDBOUNDID1
              RESERVE1 RESERVE1 RESERVE1

    propertytable = {PHONEPROP2}=256

    PRIMSTRESSID1: one byte, ID of primary stress
    SECSTRESSID1: one byte, ID of secondary stress
    SYLLBOUNDID1: one byte, ID of syllable boundary
    PAUSEID1: one byte, ID of pause
    RESERVE1: reserved for future use

    PHONEPROP2: one byte, max. of 256 phones directly access this table
                to check a property for a phone; binary properties
                encoded (1 bit per prop)
       least significant bit: vowel
                    next bit: diphth
                    next bit: glott
                    next bit: nonsyllvowel
                    next bit: syllcons
       3 bits spare
 */

#define KTAB_START_SPECIDS   0
#define KTAB_IND_PRIMSTRESS  0
#define KTAB_IND_SECSTRESS   1
#define KTAB_IND_SYLLBOUND   2
#define KTAB_IND_PAUSE       3
#define KTAB_IND_WORDBOUND   4

#define KTAB_START_PROPS     8


typedef struct ktabphones_subobj *ktabphones_SubObj;

typedef struct ktabphones_subobj {
    picoos_uint8 *specids;
    picoos_uint8 *props;
} ktabphones_subobj_t;


/* bitmasks to extract the property info from props */
#define KTAB_PPROP_VOWEL        '\x01'
#define KTAB_PPROP_DIPHTH       '\x02'
#define KTAB_PPROP_GLOTT        '\x04'
#define KTAB_PPROP_NONSYLLVOWEL '\x08'
#define KTAB_PPROP_SYLLCONS     '\x10'


static pico_status_t ktabPhonesInitialize(register picoknow_KnowledgeBase this,
                                          picoos_Common common) {
    ktabphones_subobj_t * ktabphones;

    PICODBG_DEBUG(("start"));

    if (NULL == this || NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    ktabphones = (ktabphones_subobj_t *) this->subObj;
    ktabphones->specids = &(this->base[KTAB_START_SPECIDS]);
    ktabphones->props   = &(this->base[KTAB_START_PROPS]);
    return PICO_OK;
}

static pico_status_t ktabPhonesSubObjDeallocate(register picoknow_KnowledgeBase this,
                                                picoos_MemoryManager mm) {
    if (NULL != this) {
        picoos_deallocate(mm, (void *) &this->subObj);
    }
    return PICO_OK;
}

pico_status_t picoktab_specializePhonesKnowledgeBase(picoknow_KnowledgeBase this,
                                                     picoos_Common common) {
    if (NULL == this) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    this->subDeallocate = ktabPhonesSubObjDeallocate;
    this->subObj = picoos_allocate(common->mm, sizeof(ktabphones_subobj_t));
    if (NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM,
                                       NULL, NULL);
    }
    return ktabPhonesInitialize(this, common);
}

picoktab_Phones picoktab_getPhones(picoknow_KnowledgeBase this) {
    if (NULL == this) {
        return NULL;
    } else {
        return (picoktab_Phones) this->subObj;
    }
}


/* Phones methods */

picoos_uint8 picoktab_hasVowelProp(const picoktab_Phones this,
                                   const picoos_uint8 ch) {
    return (KTAB_PPROP_VOWEL & ((ktabphones_SubObj)this)->props[ch]);
}
picoos_uint8 picoktab_hasDiphthProp(const picoktab_Phones this,
                                    const picoos_uint8 ch) {
    return (KTAB_PPROP_DIPHTH & ((ktabphones_SubObj)this)->props[ch]);
}
picoos_uint8 picoktab_hasGlottProp(const picoktab_Phones this,
                                   const picoos_uint8 ch) {
    return (KTAB_PPROP_GLOTT & ((ktabphones_SubObj)this)->props[ch]);
}
picoos_uint8 picoktab_hasNonsyllvowelProp(const picoktab_Phones this,
                                          const picoos_uint8 ch) {
    return (KTAB_PPROP_NONSYLLVOWEL & ((ktabphones_SubObj)this)->props[ch]);
}
picoos_uint8 picoktab_hasSyllconsProp(const picoktab_Phones this,
                                      const picoos_uint8 ch) {
    return (KTAB_PPROP_SYLLCONS & ((ktabphones_SubObj)this)->props[ch]);
}

picoos_bool picoktab_isSyllCarrier(const picoktab_Phones this,
                                    const picoos_uint8 ch) {
    picoos_uint8 props;
    props = ((ktabphones_SubObj)this)->props[ch];
    return (((KTAB_PPROP_VOWEL & props) &&
             !(KTAB_PPROP_NONSYLLVOWEL & props))
            || (KTAB_PPROP_SYLLCONS & props));
}

picoos_bool picoktab_isPrimstress(const picoktab_Phones this,
                                   const picoos_uint8 ch) {
    return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_PRIMSTRESS]);
}
picoos_bool picoktab_isSecstress(const picoktab_Phones this,
                                  const picoos_uint8 ch) {
    return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_SECSTRESS]);
}
picoos_bool picoktab_isSyllbound(const picoktab_Phones this,
                                  const picoos_uint8 ch) {
    return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_SYLLBOUND]);
}
picoos_bool picoktab_isWordbound(const picoktab_Phones this,
                                  const picoos_uint8 ch) {
    return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_WORDBOUND]);
}
picoos_bool picoktab_isPause(const picoktab_Phones this,
                              const picoos_uint8 ch) {
    return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_PAUSE]);
}

picoos_uint8 picoktab_getPrimstressID(const picoktab_Phones this) {
    return ((ktabphones_SubObj)this)->specids[KTAB_IND_PRIMSTRESS];
}
picoos_uint8 picoktab_getSecstressID(const picoktab_Phones this) {
    return ((ktabphones_SubObj)this)->specids[KTAB_IND_SECSTRESS];
}
picoos_uint8 picoktab_getSyllboundID(const picoktab_Phones this) {
    return ((ktabphones_SubObj)this)->specids[KTAB_IND_SYLLBOUND];
}
picoos_uint8 picoktab_getWordboundID(const picoktab_Phones this) {
    return ((ktabphones_SubObj)this)->specids[KTAB_IND_WORDBOUND];
}
picoos_uint8 picoktab_getPauseID(const picoktab_Phones this) {
    return ((ktabphones_SubObj)this)->specids[KTAB_IND_PAUSE];
}

/* ************************************************************/
/* Pos */
/* ************************************************************/

/* overview binary file format for pos kb:

    pos-kb = header posids
    header = {COUNT2 OFFS2}=8
    posids = {POSID1 {PARTID1}0:8}1:

    where POSID1 is the value of the (combined) part-of-speech symbol,
    and {PARTID1} are the symbol values of its components (empty if it
    is not a combined symbol). The {PARTID1} list is sorted.
    Part-of-speech symbols with equal number of components are grouped
    together.

    The header contains information about these groups:

    COUNT2 specifies the number of elements in the group, and OFFS2
    specifies the offset (relative to the beginning of the kb) where
    the group data starts, i.e.:

    25   32  -> 25 not-combined elements, starting at offset 32
    44   57  -> 44 elements composed of 2 symbols, starting at offset 57
    23  189  -> 23 elements composed of 3 symbols, starting at offset 189
    ...

    Currently, each symbol may be composed of up to 8 other symbols.
    Therefore, the header has 8 entries, too. The header starts with
    the unique POS list, and then in increasing order, 2 symbols, 3
    symbols,...

Zur Anschauung die ge-printf-te Version:

 25   32
 44   57
 23  189
 12  281
  4  341
  1  365
  0    0
  0    0
 33 |
 34 |
 35 |
 60 |
 etc.
 36 |  35  60
 50 |  35  95
 51 |  35  97
 58 |  35 120
 59 |  35 131
 61 |  60  75
 63 |  60  95
 64 |  60  97
 etc.
 42 |  35  60 117
 44 |  35  60 131
 45 |  35  73  97
 48 |  35  84  97
 54 |  35  97 131
 56 |  35 113 120
 57 |  35 117 120
 62 |  60  84 122
 etc.
 */

typedef struct ktabpos_subobj *ktabpos_SubObj;

typedef struct ktabpos_subobj {
    picoos_uint16 nrcomb[PICOKTAB_MAXNRPOS_IN_COMB];
    picoos_uint8 *nrcombstart[PICOKTAB_MAXNRPOS_IN_COMB];
} ktabpos_subobj_t;


static pico_status_t ktabPosInitialize(register picoknow_KnowledgeBase this,
                                       picoos_Common common) {
    ktabpos_subobj_t *ktabpos;
    picoos_uint16 osprev;
    picoos_uint16 os, pos;
    picoos_uint8 i;

    PICODBG_DEBUG(("start"));

    if (NULL == this || NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    ktabpos = (ktabpos_subobj_t *)this->subObj;

    os = 0;
    for (i = 0, pos = 0; i < PICOKTAB_MAXNRPOS_IN_COMB; i++, pos += 4) {
        ktabpos->nrcomb[i] = ((picoos_uint16)(this->base[pos+1])) << 8 |
            this->base[pos];
        if (ktabpos->nrcomb[i] > 0) {
            osprev = os;
            os = ((picoos_uint16)(this->base[pos+3])) << 8 | this->base[pos+2];
            ktabpos->nrcombstart[i] = &(this->base[os]);
            PICODBG_TRACE(("i %d, pos %d, nr %d, osprev %d, os %d", i, pos,
                           ktabpos->nrcomb[i], osprev, os));
            if (osprev >= os) {
                /* cannot be, in a valid kb */
                return picoos_emRaiseException(common->em,
                                               PICO_EXC_FILE_CORRUPT,
                                               NULL, NULL);
            }
        } else {
            if (i == 0) {
                /* cannot be, in a valid kb */
                return picoos_emRaiseException(common->em,
                                               PICO_EXC_FILE_CORRUPT,
                                               NULL, NULL);
            }
            ktabpos->nrcombstart[i] = NULL;
        }
    }
    return PICO_OK;
}

static pico_status_t ktabPosSubObjDeallocate(register picoknow_KnowledgeBase this,
                                             picoos_MemoryManager mm) {
    if (NULL != this) {
        picoos_deallocate(mm, (void *) &this->subObj);
    }
    return PICO_OK;
}

pico_status_t picoktab_specializePosKnowledgeBase(picoknow_KnowledgeBase this,
                                                  picoos_Common common) {
    if (NULL == this) {
        return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    this->subDeallocate = ktabPosSubObjDeallocate;
    this->subObj = picoos_allocate(common->mm, sizeof(ktabpos_subobj_t));
    if (NULL == this->subObj) {
        return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM,
                                       NULL, NULL);
    }
    return ktabPosInitialize(this, common);
}

picoktab_Pos picoktab_getPos(picoknow_KnowledgeBase this) {
    if (NULL == this) {
        return NULL;
    } else {
        return (picoktab_Pos) this->subObj;
    }
}


/* Pos methods */

static picoos_int16 ktab_isEqualPosGroup(const picoos_uint8 *grp1,
                                         const picoos_uint8 *grp2,
                                         picoos_uint8 len)
{
    /* if both, grp1 and grp2 would be sorted in ascending order
       we could implement a function picoktab_comparePosGroup in
       a similar manner as strcmp */

    picoos_uint16 i, j, equal;

    equal = 1;

    i = 0;
    while (equal && (i < len)) {
        /* search grp1[i] in grp2 */
        j = 0;
        while ((j < len) && (grp1[i] != grp2[j])) {
            j++;
        }
        equal = (j < len);
        i++;
    }

    return equal;
}


picoos_bool picoktab_isUniquePos(const picoktab_Pos this,
                                  const picoos_uint8 pos) {
    ktabpos_subobj_t *ktabpos;
    picoos_uint16 i;

    /* speed-up possible with e.g. binary search */

    ktabpos = (ktabpos_subobj_t *)this;
    PICODBG_TRACE(("pos %d, nrcombinations %d", pos, ktabpos->nrcomb[0]));
    i = 0;
    while ((i < ktabpos->nrcomb[0]) && (pos > ktabpos->nrcombstart[0][i])) {
        PICODBG_TRACE(("compare with pos %d at position %d",
                       ktabpos->nrcombstart[0][i], pos, i));
        i++;
    }
    return ((i < ktabpos->nrcomb[0]) && (pos == ktabpos->nrcombstart[0][i]));
}


picoos_bool picoktab_isPartOfPosGroup(const picoktab_Pos this,
                                       const picoos_uint8 pos,
                                       const picoos_uint8 posgroup)
{
    ktabpos_subobj_t *ktabpos;
    picoos_uint8 *grp;
    picoos_uint16 i, j, n, s, grplen;
    picoos_uint8 *e;
    picoos_uint8 found;

    ktabpos = (ktabpos_subobj_t *) this;

    grp = NULL;
    found = FALSE;
    grplen = 0;

    /* currently, a linear search is required to find 'posgroup'; the
       knowledge base should be extended to allow for a faster search */

    /* treat case i==0, grplen==0, ie. pos == posgroup */
    if (pos == posgroup) {
        found = TRUE;
    }

    i = 1;
    while ((grp == NULL) && (i < PICOKTAB_MAXNRPOS_IN_COMB)) {
        n = ktabpos->nrcomb[i];       /* number of entries */
        e = ktabpos->nrcombstart[i];  /* ptr to first entry */
        s = i + 2;                    /* size of an entry in bytes */
        /* was with while starting at 0:
        s = i > 0 ? i + 2 : 1;
        */
        j = 0;
        while ((grp == NULL) && (j < n)) {
            if (posgroup == e[0]) {
                grp = e + 1;
                grplen = s - 1;
            }
            e += s;
            j++;
        }
        i++;
    }

    /* test if 'pos' is contained in the components of 'posgroup' */
    if (grp != NULL) {
        for (i = 0; !found && (i < grplen); i++) {
            if (pos == grp[i]) {
                found = TRUE;
            }
        }

        /* just a way to test picoktab_getPosGroup */
        /*
        PICODBG_ASSERT(picoktab_getPosGroup(this, grp, grplen) == posgroup);
        */
    }

    return found;
}


picoos_uint8 picoktab_getPosGroup(const picoktab_Pos this,
                                  const picoos_uint8 *poslist,
                                  const picoos_uint8 poslistlen)
{
    picoos_uint8 poscomb;
    ktabpos_subobj_t *ktabpos;
    picoos_uint16 i, j, n, s;
    picoos_uint8 *e;

    ktabpos = (ktabpos_subobj_t *) this;
    poscomb = 0;

    if ((poslistlen > 0) && (poslistlen <= PICOKTAB_MAXNRPOS_IN_COMB)) {
        i = poslistlen - 1;
        if (i > 0) {
            n = ktabpos->nrcomb[i];       /* number of entries */
            e = ktabpos->nrcombstart[i];  /* ptr to first entry */
            s = i + 2;                    /* size of an entry in bytes */
            j = 0;
            while (!poscomb && (j < n)) {
                if (ktab_isEqualPosGroup(poslist, e + 1, poslistlen)) {
                    poscomb = *e;
                }
                e += s;
                j++;
            }
            if (!poscomb) {
                /* combination not found; shouldn't occur if lingware OK! */
                /* contingency solution: take first */
                PICODBG_WARN(("dynamically created POS combination not found in table; taking first (%i)",poslist[0]));
                poscomb = poslist[0];
            }
        } else {  /* not a composed POS */
            poscomb = poslist[0];
        }
    }

    return poscomb;
}

#ifdef __cplusplus
}
#endif


/* end */