1 /*
2 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 /**
17 * @file picospho.c
18 *
19 * sentence phonemic/phonetic FSTs PU
20 *
21 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
22 * All rights reserved.
23 *
24 * History:
25 * - 2009-04-20 -- initial version
26 *
27 */
28
29 #include "picoos.h"
30 #include "picodbg.h"
31 #include "picodata.h"
32
33 #include "picoknow.h"
34 #include "picokfst.h"
35 #include "picoktab.h"
36 #include "picotrns.h"
37
38 #include "picospho.h"
39
40 #ifdef __cplusplus
41 extern "C" {
42 #endif
43 #if 0
44 }
45 #endif
46
47 #define SPHO_BUFSIZE (3 * PICODATA_BUFSIZE_DEFAULT)
48
49
50
51 #define SPHO_MAX_ALTDESC_SIZE (60 * PICOTRNS_MAX_NUM_POSSYM)
52
53
54 #define SPHO_SMALLEST_SIL_DUR 1
55
56
57 /** @addtogroup picospho
58 *
59 * Algorithmic description
60 * =======================
61 * The main function, sphoStep, is divided into the subprocesses (processing states) described further down.
62 *
63 * Flow control:
64 * ------------
65 * The processing flow is controlled by setting
66 * - 'procState' : the next state to be processed
67 * - 'feedFollowState' : the state to be processed after the feed state (the feed state is treated like a primitive "subroutine")
68 * - some other flags
69 *
70 * Buffering:
71 * ---------
72 * - The input items are mainly stored and processed in two buffers, collectively called 'inBuf'
73 * - cbuf : unstructured buffer containing item contents
74 * - headx : structured buffer containing item heads, each expanded by a pointer to the item contents
75 * and space for a boundary potentially to be inserted (to the left of the original item)
76 * - For transduction, phonemes and their position are extracted from inBuf into
77 * - phonBuf,
78 * processed there, and the resulting phonemes realigned with inBuf.
79 * - Word items are split into syllables, stored in
80 * - sylBuf
81 * - Items to be output are stored in outBuf
82 *
83 * Windowing:
84 * ---------
85 * Optimal solutions are achieved if a whole sentence is processed at once. However, if any of the buffers are too small,
86 * only sentence parts are processed. To improve the quality of such sub-optimal solutions, a moving-window-with-overlap is applied:
87 * - [0,headxReadPos[ : the window considered for transduction
88 * - [activeStartPos,activeEndPos[ : the "active" subrange of the window actually used for output
89 * - penultima : the position (within the active range) that should be used as new window start when shifting the window
90 *
91 * After PROCESS_PARSE:
92 * 0 activeStartPos penultima activeEndPos headxReadPos headxWritePos
93 * | | | | | |
94 * |-------------=================================---------------| ['----': context '====' : active subrange)
95 *
96 * After PROCESS_SHIFT:
97 * 0 activeStartPos headWritePos
98 * | | | |
99 * |------------... (only left context is known; new active range, penultima, and right context to be established at next parse)
100 *
101 * Processing states:
102 * -----------------
103 * - INIT : initialize state variables
104 * - COLLECT : collect items into internal buffers ("inBuf")
105 * - PROCESS_PARSE : go through inBuf items and extract position/phoneme pairs into phoneme buffer 'phonBuf'
106 * word boundary phonemes are inserted between words
107 * - PROCESS_TRANSDUCE : transduce phonBuf
108 * - PROCESS_BOUNDS : go through inBuf items again and match against transduced pos/phoneme
109 * this is the first round of alignment, only inserting/deleting/modifying bounds, according to
110 * - existing BOUND items
111 * - newly produced word bounds separating WORDPHON items
112 * - bound upgrades/downgrades from transduction
113 * - bound upgrades/downgrades/insertions from SIL command items (originating e.g. from <break> text commands)
114 * all relevant bounds are placed in the corresponding headx extention; original bound items become invalid.
115 * - PROCESS_RECOMB : go through inBuf items again and match against transduced pos/phoneme
116 * this is the second round of alignment, treating non-BOUND items
117 * - WORDPHONs are broken into syllables by "calling" PROCESS_SYL
118 * - "side-bounds" (in the headx extension) are output by "calling" FEED
119 * - BOUND items are consumed with no effect
120 * - other items are output unchanged "calling" FEED
121 * - PROCESS_SYL : the WORDPHON coming from RECOMB is matched against the phonBuf and (new) SYLLPHON items
122 * are created. (the original wordphon is consumed)
123 * - FEED : feeds one item and returns to spho->feedFollowState
124 * - SHIFT : items in inBuf are shifted left to make room for new items. If a sentence doesn't fit
125 * inBuf in its entirety, left and/or right contexts are kept so they can be considered in
126 * the next transduction.
127 */
128
129
130
131 /* PU sphoStep states */
132 #define SPHO_STEPSTATE_INIT 0
133 #define SPHO_STEPSTATE_COLLECT 1
134 #define SPHO_STEPSTATE_PROCESS_PARSE 2
135 #define SPHO_STEPSTATE_PROCESS_TRANSDUCE 3
136 #define SPHO_STEPSTATE_PROCESS_BOUNDS 4
137 #define SPHO_STEPSTATE_PROCESS_RECOMB 5
138 #define SPHO_STEPSTATE_PROCESS_SYL 6
139 #define SPHO_STEPSTATE_FEED 7
140 #define SPHO_STEPSTATE_SHIFT 8
141
142 #define SPHO_POS_INVALID (PICOTRNS_POS_INVALID) /* indicates that no position was set yet */
143
144 /* nr item restriction: maximum number of extended item heads in headx */
145 #define SPHO_MAXNR_HEADX 60
146
147 /* nr item restriction: maximum size of all item contents together in cont */
148 #define SPHO_MAXSIZE_CBUF (30 * 255)
149
150 /* "expanded head": item head expanded by a content position and a by boundary information
151 * potentially inserted "to the left" of the item */
152 typedef struct {
153 picodata_itemhead_t head;
154 picoos_uint16 cind;
155 picoos_uint8 boundstrength; /* bstrength to the left, 0 if not set */
156 picoos_uint8 phrasetype; /* btype for following phrase, 0 if not set */
157 picoos_int16 sildur; /* silence duration for boundary, -1 if not set */
158 } picospho_headx_t;
159
160
161
162 #define SPHO_MSGSTR_SIZE 32
163
164 /** object : SentPhoUnit
165 * shortcut : spho
166 * derived from : picodata_ProcessingUnit
167 */
168 typedef struct spho_subobj {
169 picoos_Common common;
170
171 /* we use int16 for buffer positions so we can indicate exceptional positions (invalid etc.) with negative
172 * integers */
173 picoos_uint8 procState; /* for next processing step decision */
174
175 /* buffer for item headers */
176 picoos_uint8 tmpbuf[PICODATA_MAX_ITEMSIZE]; /* tmp. location for an item */
177
178 picospho_headx_t headx[SPHO_MAXNR_HEADX]; /* "expanded head" buffer */
179 picoos_uint16 headxBufSize; /* actually allocated size (if one day headxBuf is allocated dynamically) */
180 picoos_uint16 headxReadPos, headxWritePos;
181
182 picoos_uint8 cbuf[SPHO_MAXSIZE_CBUF];
183 picoos_uint16 cbufBufSize; /* actually allocated size */
184 picoos_uint16 cbufWritePos; /* next position to write to, 0 if buffer empty */
185
186 picoos_uint8 outBuf[PICODATA_BUFSIZE_DEFAULT]; /* internal output buffer to hold just one item */
187 picoos_uint16 outBufSize; /* actually allocated size (if one day outBuf is allocated dynamically) */
188 picoos_uint16 outReadPos; /* next pos to read from inBuf for output */
189
190 /* picoos_int16 outWritePos; */ /* next pos to output from in buf */
191
192 picoos_uint8 sylBuf[255]; /* internal buffer to hold contents of syl item to be output */
193 picoos_uint8 sylReadPos, sylWritePos; /* next pos to read from sylBuf, next pos to write to sylBuf */
194
195 /* buffer for internal calculation of transducer */
196 picotrns_AltDesc altDescBuf;
197 /* the number of AltDesc in the buffer */
198 picoos_uint16 maxAltDescLen;
199
200 /* the input to a transducer should not be larger than PICOTRNS_MAX_NUM_POSSYM
201 * so the output may expand (up to 4*PICOTRNS_MAX_NUM_POSSYM) */
202
203 picotrns_possym_t phonBufA[4 * PICOTRNS_MAX_NUM_POSSYM + 1];
204 picotrns_possym_t phonBufB[4 * PICOTRNS_MAX_NUM_POSSYM + 1];
205 picotrns_possym_t * phonBuf;
206 picotrns_possym_t * phonBufOut;
207 picoos_uint16 phonReadPos, phonWritePos; /* next pos to read from phonBufIn, next pos to write to phonBufIn */
208
209 picoos_int16 activeStartPos; /* start position of items to be treated (at end of left context) */
210 picoos_int16 penultima, activeEndPos; /* positions of last two bounds/words; SPHO_POS_INVALID means uninitialized */
211 picoos_int16 lastPhraseBoundPos; /* position of the last bound encountered (<0 if inexistent or not reachable */
212 picoos_uint8 lastPhraseType; /* phrase type of the last phrase boundary, 0 if not set */
213
214 picoos_uint8 needMoreInput, /* more data necessary to decide on token */
215 suppressParseWordBound, /* dont produce word boundary */
216 suppressRecombWordBound, /* dont produce word boundary */
217 breakPending, /* received a break but didn't interpret it yet */
218 /* sentEnd, */ /* sentence end detected */
219 force, /* in forced state */
220 wordStarted, /* is it the first syl in the word: expect POS */
221 sentenceStarted;
222
223 picoos_uint16 breakTime; /* time argument of the pending break command */
224
225 picoos_uint8 feedFollowState; /* where to return after feed */
226
227 /* fst knowledge bases */
228 picoos_uint8 numFsts;
229 picokfst_FST fst[PICOKNOW_MAX_NUM_SPHO_FSTS];
230 picoos_uint8 curFst; /* the fst to be applied next */
231
232 /* fixed ids knowledge base */
233 picoktab_FixedIds fixedIds;
234
235 /* phones kb */
236 picoktab_Phones phones;
237
238 /* some soecial ids from phones */
239 picoos_uint8 primStressId, secondStressId, syllSepId;
240
241 } spho_subobj_t;
242
243
sphoReset(register picodata_ProcessingUnit this)244 static pico_status_t sphoReset(register picodata_ProcessingUnit this)
245 {
246
247 spho_subobj_t * spho;
248
249 if (NULL == this || NULL == this->subObj) {
250 return picoos_emRaiseException(this->common->em,
251 PICO_ERR_NULLPTR_ACCESS, NULL, NULL);
252 }
253 spho = (spho_subobj_t *) this->subObj;
254
255 spho->curFst = 0;
256
257 /* processing state */
258 spho->procState = SPHO_STEPSTATE_INIT;
259 spho->needMoreInput = TRUE;
260 spho->suppressParseWordBound = FALSE;
261 spho->suppressRecombWordBound = FALSE;
262 spho->breakPending = FALSE;
263 spho->force = 0;
264 spho->sentenceStarted = 0;
265
266
267 /* item buffer headx/cbuf */
268 spho->headxBufSize = SPHO_MAXNR_HEADX;
269 spho->headxReadPos = 0;
270 spho->headxWritePos = 0;
271
272 spho->cbufWritePos = 0;
273 spho->cbufBufSize = SPHO_MAXSIZE_CBUF;
274
275 /* possym buffer */
276 spho->phonBuf = spho->phonBufA;
277 spho->phonBufOut = spho->phonBufB;
278 spho->phonReadPos = 0;
279
280 /* overlapping */
281 spho->activeStartPos = 0;
282 spho->penultima = SPHO_POS_INVALID;
283 spho->activeEndPos = SPHO_POS_INVALID;
284
285 return PICO_OK;
286 }
287
288
sphoInitialize(register picodata_ProcessingUnit this,picoos_int32 resetMode)289 static pico_status_t sphoInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode)
290 {
291 picoos_uint8 i;
292 spho_subobj_t * spho;
293 picokfst_FST fst;
294
295 picoknow_kb_id_t myKbIds[PICOKNOW_MAX_NUM_SPHO_FSTS] = PICOKNOW_KBID_SPHO_ARRAY;
296
297 PICODBG_DEBUG(("init"));
298
299 if (NULL == this || NULL == this->subObj) {
300 return picoos_emRaiseException(this->common->em,
301 PICO_ERR_NULLPTR_ACCESS, NULL, NULL);
302 }
303
304 spho = (spho_subobj_t *) this->subObj;
305
306 spho->numFsts = 0;
307
308 spho->curFst = 0;
309
310 for (i = 0; i<PICOKNOW_MAX_NUM_SPHO_FSTS; i++) {
311 fst = picokfst_getFST(this->voice->kbArray[myKbIds[i]]);
312 if (NULL != fst) {
313 spho->fst[spho->numFsts++] = fst;
314 }
315 }
316 spho->fixedIds = picoktab_getFixedIds(this->voice->kbArray[PICOKNOW_KBID_FIXED_IDS]);
317 spho->phones = picoktab_getPhones(this->voice->kbArray[PICOKNOW_KBID_TAB_PHONES]);
318
319 spho->syllSepId = picoktab_getSyllboundID(spho->phones);
320 spho->primStressId = picoktab_getPrimstressID(spho->phones);
321 spho->secondStressId = picoktab_getSecstressID(spho->phones);
322
323 PICODBG_DEBUG(("got %i fsts", spho->numFsts));
324
325
326 return sphoReset(this);
327
328 }
329
330 static picodata_step_result_t sphoStep(register picodata_ProcessingUnit this,
331 picoos_int16 mode, picoos_uint16 *numBytesOutput);
332
333
334
335
sphoTerminate(register picodata_ProcessingUnit this)336 static pico_status_t sphoTerminate(register picodata_ProcessingUnit this)
337 {
338 return PICO_OK;
339 }
340
341
sphoSubObjDeallocate(register picodata_ProcessingUnit this,picoos_MemoryManager mm)342 static pico_status_t sphoSubObjDeallocate(register picodata_ProcessingUnit this,
343 picoos_MemoryManager mm)
344 {
345 spho_subobj_t * spho;
346
347 spho = (spho_subobj_t *) this->subObj;
348
349 if (NULL != this) {
350 if (NULL != this->subObj) {
351 spho = (spho_subobj_t *) (this->subObj);
352 picotrns_deallocate_alt_desc_buf(spho->common->mm,&spho->altDescBuf);
353 picoos_deallocate(mm, (void *) &this->subObj);
354 }
355 }
356 return PICO_OK;
357 }
358
picospho_newSentPhoUnit(picoos_MemoryManager mm,picoos_Common common,picodata_CharBuffer cbIn,picodata_CharBuffer cbOut,picorsrc_Voice voice)359 picodata_ProcessingUnit picospho_newSentPhoUnit(picoos_MemoryManager mm,
360 picoos_Common common, picodata_CharBuffer cbIn,
361 picodata_CharBuffer cbOut, picorsrc_Voice voice)
362 {
363 spho_subobj_t * spho;
364
365 picodata_ProcessingUnit this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice);
366 if (this == NULL) {
367 return NULL;
368 }
369
370 this->initialize = sphoInitialize;
371 this->step = sphoStep;
372 this->terminate = sphoTerminate;
373 this->subDeallocate = sphoSubObjDeallocate;
374
375 this->subObj = picoos_allocate(mm, sizeof(spho_subobj_t));
376 if (this->subObj == NULL) {
377 picoos_deallocate(mm, (void **)(void*)&this);
378 return NULL;
379 }
380 spho = (spho_subobj_t *) this->subObj;
381
382 spho->common = this->common;
383
384 /* these are given by the pre-allocated array sizes */
385 spho->outBufSize = PICODATA_BUFSIZE_DEFAULT;
386
387
388 spho->altDescBuf = picotrns_allocate_alt_desc_buf(spho->common->mm, SPHO_MAX_ALTDESC_SIZE, &spho->maxAltDescLen);
389 if (NULL == spho->altDescBuf) {
390 picotrns_deallocate_alt_desc_buf(spho->common->mm,&spho->altDescBuf);
391 picoos_emRaiseException(spho->common->em,PICO_EXC_OUT_OF_MEM, NULL,NULL);
392 return NULL;
393 }
394
395 sphoInitialize(this, PICO_RESET_FULL);
396 return this;
397 }
398
399
400 /* ***********************************************************************/
401 /* process buffered item list */
402 /* ***********************************************************************/
403
404
405 /* shift relevant data in headx/'cbuf' (between 'readPos' incl and writePos non-incl) to 'start'.
406 * modify read/writePos accordingly */
shift_range_left_1(spho_subobj_t * spho,picoos_int16 * from,picoos_int16 to)407 static picoos_int16 shift_range_left_1(spho_subobj_t *spho, picoos_int16 * from, picoos_int16 to)
408 {
409
410 /* remember shift parameters for cbuf */
411 picoos_uint16
412 c_i,
413 c_j,
414 c_diff,
415 c_writePos,
416 i,
417 j,
418 diff,
419 writePos;
420 i = to;
421 j = *from;
422 diff = j-i;
423 writePos = spho->headxWritePos;
424 c_i = spho->headx[to].cind;
425 if (j < writePos) {
426 c_j = spho->headx[j].cind;
427 } else {
428 c_j = spho->cbufWritePos;
429 }
430 c_diff = c_j - c_i;
431 c_writePos = spho->cbufWritePos;
432
433 PICODBG_DEBUG((
434 "shifting buffer region [%i,%i[ down to %i",*from, writePos, to
435 ));
436
437
438 /* PICODBG_ASSERT((i<j)); */
439 if (i > j) {
440 return -1;
441 }
442 /* shift cbuf */
443 while (c_j < c_writePos) {
444 spho->cbuf[c_i++] = spho->cbuf[c_j++];
445 }
446 /* shift headx */
447 while (j < writePos) {
448 spho->headx[j].cind -= c_diff;
449 spho->headx[i++] = spho->headx[j++];
450 }
451 spho->headxWritePos -= diff;
452 *from = to;
453 spho->cbufWritePos -= c_diff;
454 /* */
455 PICODBG_DEBUG((
456 "readPos,WritePos are now [%i,%i[, returning shift amount %i",*from, spho->headxWritePos, diff
457 ));
458 return diff;
459 }
460
sphoAddPhoneme(register spho_subobj_t * spho,picoos_int16 pos,picoos_int16 sym)461 static pico_status_t sphoAddPhoneme(register spho_subobj_t *spho, picoos_int16 pos, picoos_int16 sym) {
462 picoos_uint8 plane, unshifted;
463 /* just for debuging */
464 unshifted = picotrns_unplane(sym,&plane);
465 PICODBG_TRACE(("adding %i/%i (%c on plane %i) at phonBuf[%i]",pos,sym,unshifted,plane,spho->phonWritePos));
466 if (2* PICOTRNS_MAX_NUM_POSSYM <= spho->phonWritePos) {
467 /* not an error! */
468 PICODBG_DEBUG(("couldn't add because phon buffer full"));
469 return PICO_EXC_BUF_OVERFLOW;
470 } else {
471 spho->phonBuf[spho->phonWritePos].pos = pos;
472 spho->phonBuf[spho->phonWritePos].sym = sym;
473 spho->phonWritePos++;
474 return PICO_OK;
475 }
476 }
477
sphoAddStartPhoneme(register spho_subobj_t * spho)478 static pico_status_t sphoAddStartPhoneme(register spho_subobj_t *spho) {
479 return sphoAddPhoneme(spho, PICOTRNS_POS_IGNORE,
480 (PICOKFST_PLANE_INTERN << 8) + spho->fixedIds->phonStartId);
481 }
482
sphoAddTermPhonemes(register spho_subobj_t * spho,picoos_uint16 pos)483 static pico_status_t sphoAddTermPhonemes(register spho_subobj_t *spho, picoos_uint16 pos) {
484 return sphoAddPhoneme(spho, pos,
485 (PICOKFST_PLANE_PB_STRENGTHS << 8) + PICODATA_ITEMINFO1_BOUND_SEND)
486 && sphoAddPhoneme(spho, PICOTRNS_POS_IGNORE,
487 (PICOKFST_PLANE_INTERN << 8) + spho->fixedIds->phonTermId);
488 }
489
490 /* return "syllable accent" (or prominence) symbol, given "word accent" symbol 'wacc' and stress value (no=0, primary=1, secondary=2) */
sphoGetSylAccent(register spho_subobj_t * spho,picoos_uint8 wacc,picoos_uint8 sylStress)491 static picoos_uint16 sphoGetSylAccent(register spho_subobj_t *spho,
492 picoos_uint8 wacc, picoos_uint8 sylStress)
493 {
494 PICODBG_ASSERT(sylStress <= 2);
495
496 spho = spho; /* avoid warning "var not used in this function"*/
497
498 switch (sylStress) {
499 case 0: /* non-stressed syllable gets no prominence */
500 /* return spho->fixedIds->accId[0]; */
501 return PICODATA_ACC0;
502 break;
503 case 1: /* primary-stressed syllable gets word prominence */
504 return wacc;
505 break;
506 case 2: /* secondary-stressed syllable gets no prominence or secondary stress prom. (4) */
507 return (PICODATA_ACC0 == wacc) ? PICODATA_ACC0
508 : PICODATA_ACC4;
509 /*return (spho->fixedIds->accId[0] == wacc) ? spho->fixedIds->accId[0]
510 : spho->fixedIds->accId[4]; */
511 break;
512 default:
513 /* never occurs :-) */
514 return PICODATA_ACC0;
515 break;
516 }
517 }
518
519
520 /* ***********************************************************************/
521 /* extract phonemes of an item into a phonBuf */
522 /* ***********************************************************************/
sphoExtractPhonemes(register picodata_ProcessingUnit this,register spho_subobj_t * spho,picoos_uint16 pos,picoos_uint8 convertAccents,picoos_uint8 * suppressWB)523 static pico_status_t sphoExtractPhonemes(register picodata_ProcessingUnit this,
524 register spho_subobj_t *spho, picoos_uint16 pos,
525 picoos_uint8 convertAccents, picoos_uint8 * suppressWB)
526 {
527 pico_status_t rv = PICO_OK;
528 picoos_uint16 i, j;
529 picoos_int16 fstSymbol;
530 picoos_uint8 curStress;
531 picotrns_possym_t tmpPosSym;
532 picoos_uint16 oldPos, curPos;
533 picodata_itemhead_t * head;
534 picoos_uint8* content;
535
536 #if defined(PICO_DEBUG)
537 picoos_char msgstr[SPHO_MSGSTR_SIZE];
538 #endif
539
540
541 /*
542 Items considered in a transduction are a BOUND or a WORDPHON item. its starting offset within the
543 headxBuf is given as 'pos'.
544 Elements that go into the transduction receive "their" position in the buffer.
545 */
546
547 oldPos = spho->phonWritePos;
548
549 head = &(spho->headx[pos].head);
550 content = spho->cbuf + spho->headx[pos].cind;
551
552 PICODBG_TRACE(("doing item %s\n",
553 picodata_head_to_string(head,msgstr,SPHO_MSGSTR_SIZE)));
554
555 switch (head->type) {
556 case PICODATA_ITEM_BOUND:
557 /* map SBEG, SEND and TERM (as sentence closing) to SEND */
558 fstSymbol = (PICODATA_ITEMINFO1_BOUND_SBEG == head->info1 || PICODATA_ITEMINFO1_BOUND_TERM == head->info1) ? PICODATA_ITEMINFO1_BOUND_SEND : head->info1;
559 PICODBG_TRACE(("found bound of type %c\n",head->info1));
560 /* BOUND(<bound strength><phrase type>) */
561 /* insert bound strength */
562 PICODBG_TRACE(("inserting phrase bound phoneme %c and setting suppresWB=1\n",fstSymbol));
563 fstSymbol += (PICOKFST_PLANE_PB_STRENGTHS << 8);
564 rv = sphoAddPhoneme(spho,pos,fstSymbol);
565 /* phrase type not used */
566 /* suppress next word boundary */
567 (*suppressWB) = 1;
568 break;
569
570 case PICODATA_ITEM_WORDPHON:
571 /* WORDPHON(POS,WACC)phon */
572 PICODBG_TRACE(("found WORDPHON"));
573 /* insert word boundary if not suppressed */
574 if (!(*suppressWB)) {
575 fstSymbol = (PICOKFST_PLANE_PB_STRENGTHS << 8) + PICODATA_ITEMINFO1_BOUND_PHR0;
576 PICODBG_TRACE(("adding word boundary phone"));
577 rv = sphoAddPhoneme(spho,pos,fstSymbol);
578 }
579 (*suppressWB) = 0;
580 /* for the time being, we force to use POS so we can transduce all fsts in a row without reconsulting the items */
581
582
583 /* If 'convertAccents' then the accentuation is not directly encoded. It rather influences the mapping of
584 * the word accent symbol to the actual accent phoneme which is put after the syllable separator. */
585 if (convertAccents) {
586 PICODBG_TRACE(("converting accents"));
587 /* extracting phonemes IN REVERSE order replacing syllable symbols with prominence symbols */
588 curPos = spho->phonWritePos;
589 curStress = 0; /* no stress */
590 for (i = head->len; i > 0 ;) {
591 i--;
592 if (spho->primStressId == content[i]) {
593 curStress = 1;
594 PICODBG_DEBUG(("skipping primary stress at pos %i (in 1 .. %i)",i, head->len));
595 continue; /* skip primary stress symbol */
596 } else if (spho->secondStressId == content[i]) {
597 curStress = 2;
598 PICODBG_DEBUG(("skipping secondary stress at pos %i (in 1 .. %i)",i, head->len));
599 continue; /* skip secundary stress symbol */
600 } else if (spho->syllSepId == content[i]) {
601 fstSymbol = (PICOKFST_PLANE_POS << 8) + head->info1;
602 rv = sphoAddPhoneme(spho, pos, fstSymbol);
603 /* replace syllSepId by combination of syllable stress and word prominence */
604 fstSymbol = sphoGetSylAccent(spho,head->info2,curStress);
605 curStress = 0;
606 /* add accent */
607 fstSymbol += (PICOKFST_PLANE_ACCENTS << 8);
608 rv = sphoAddPhoneme(spho,pos,fstSymbol);
609 if (PICO_OK != rv) {
610 break;
611 }
612 /* and keep syllable boundary */
613 fstSymbol = (PICOKFST_PLANE_PHONEMES << 8) + content[i];
614 } else {
615 /* normal phoneme */
616 fstSymbol = (PICOKFST_PLANE_PHONEMES << 8) + content[i];
617 }
618 if (PICO_OK == rv) {
619 rv = sphoAddPhoneme(spho,pos,fstSymbol);
620 }
621 }
622 if (PICO_OK == rv) {
623 /* bug 366: we position the "head" into the item header and not on the first phoneme
624 * because there might be no phonemes at all */
625 /* insert head of the first syllable of a word */
626 fstSymbol = (PICOKFST_PLANE_POS << 8) + head->info1;
627 rv = sphoAddPhoneme(spho,pos,fstSymbol);
628 fstSymbol = sphoGetSylAccent(spho,head->info2,curStress);
629 curStress = 0;
630 fstSymbol += (PICOKFST_PLANE_ACCENTS << 8);
631 rv = sphoAddPhoneme(spho,pos,fstSymbol);
632 }
633 if (PICO_OK == rv) {
634 /* invert sympos portion */
635 i = curPos;
636 j=spho->phonWritePos-1;
637 while (i < j) {
638 tmpPosSym.pos = spho->phonBuf[i].pos;
639 tmpPosSym.sym = spho->phonBuf[i].sym;
640 spho->phonBuf[i].pos = spho->phonBuf[j].pos;
641 spho->phonBuf[i].sym = spho->phonBuf[j].sym;
642 spho->phonBuf[j].pos = tmpPosSym.pos;
643 spho->phonBuf[j].sym = tmpPosSym.sym;
644 i++;
645 j--;
646 }
647 }
648 } else { /* convertAccents */
649 for (i = 0; i <head->len; i++) {
650 fstSymbol = (PICOKFST_PLANE_PHONEMES << 8) + content[i];
651 rv = sphoAddPhoneme(spho,pos,fstSymbol);
652 }
653 }
654 break;
655 default:
656 picoos_emRaiseException(this->common->em,rv,NULL,NULL);
657 break;
658 } /* switch(head->type) */
659 if (PICO_OK != rv) {
660 spho->phonWritePos = oldPos;
661 }
662 return rv;
663 }
664
665
666
667
668
669 #define SPHO_POSSYM_OK 0
670 #define SPHO_POSSYM_OUT_OF_RANGE 1
671 #define SPHO_POSSYM_END 2
672 #define SPHO_POSSYM_INVALID -3
673 /* *readPos is the next position in phonBuf to be read, and *writePos is the first position not to be read (may be outside
674 * buf).
675 * 'rangeEnd' is the first possym position outside the desired range.
676 * Possible return values:
677 * SPHO_POSSYM_OK : 'pos' and 'sym' are set to the read possym, *readPos is advanced
678 * SPHO_POSSYM_OUT_OF_RANGE : pos is out of range. 'pos' is set to that of the read possym, 'sym' is undefined
679 * SPHO_POSSYM_UNDERFLOW : no more data in buf. 'pos' is set to PICOTRNS_POS_INVALID, 'sym' is undefined
680 * SPHO_POSSYM_INVALID : "strange" pos. 'pos' is set to PICOTRNS_POS_INVALID, 'sym' is undefined
681 */
getNextPosSym(spho_subobj_t * spho,picoos_int16 * pos,picoos_int16 * sym,picoos_int16 rangeEnd)682 static pico_status_t getNextPosSym(spho_subobj_t * spho, picoos_int16 * pos, picoos_int16 * sym,
683 picoos_int16 rangeEnd) {
684 /* skip POS_IGNORE */
685 while ((spho->phonReadPos < spho->phonWritePos) && (PICOTRNS_POS_IGNORE == spho->phonBuf[spho->phonReadPos].pos)) {
686 PICODBG_DEBUG(("ignoring phone at spho->phonBuf[%i] because it has pos==IGNORE",spho->phonReadPos));
687 spho->phonReadPos++;
688 }
689 if ((spho->phonReadPos < spho->phonWritePos)) {
690 *pos = spho->phonBuf[spho->phonReadPos].pos;
691 if ((PICOTRNS_POS_INSERT == *pos) || ((0 <= *pos) && (*pos < rangeEnd))) {
692 *sym = spho->phonBuf[spho->phonReadPos++].sym;
693 return SPHO_POSSYM_OK;
694 } else if (*pos < 0){ /* *pos is "strange" (e.g. POS_INVALID) */
695 return SPHO_POSSYM_INVALID;
696 } else {
697 return SPHO_POSSYM_OUT_OF_RANGE;
698 }
699 } else {
700 /* no more possyms to read */
701 *pos = PICOTRNS_POS_INVALID;
702 return SPHO_POSSYM_END;
703 }
704 }
705
706
707
708 /** Calculate bound strength modified by transduction
709 *
710 * Given the original bound strength 'orig' and the desired target strength 'target' (suggested by fst),
711 * calculate the modified bound strength.
712 *
713 * @param orig original bound strength
714 * @param target target bound strength
715 * @return resulting bound strength
716 */
fstModifiedBoundStrength(picoos_uint8 orig,picoos_uint8 target)717 static picoos_uint8 fstModifiedBoundStrength(picoos_uint8 orig, picoos_uint8 target)
718 {
719 switch (orig) {
720 case PICODATA_ITEMINFO1_BOUND_PHR1:
721 case PICODATA_ITEMINFO1_BOUND_PHR2:
722 /* don't allow primary phrase bounds to be demoted to word bound */
723 if (PICODATA_ITEMINFO1_BOUND_PHR0 == target) {
724 return PICODATA_ITEMINFO1_BOUND_PHR3;
725 }
726 case PICODATA_ITEMINFO1_BOUND_PHR0:
727 case PICODATA_ITEMINFO1_BOUND_PHR3:
728 return target;
729 break;
730 default:
731 /* don't allow bounds other than phrase or word bounds to be changed */
732 return orig;
733 break;
734 }
735 }
736
737 /** Calculate bound strength modified by a \<break> command
738 *
739 * Given the original (predicted and possibly fst-modified) bound strength, and a time value from an
740 * overwriding \<break> command, calculate the modified bound strength.
741 *
742 * @param orig original bound strength
743 * @param time time given as property of \<break> command
744 * @param wasPrimary
745 * @return modified bound strength
746 */
breakModifiedBoundStrength(picoos_uint8 orig,picoos_uint16 time,picoos_bool wasPrimary)747 static picoos_uint8 breakModifiedBoundStrength(picoos_uint8 orig, picoos_uint16 time, picoos_bool wasPrimary)
748 {
749 picoos_uint8 modified = (0 == time) ? PICODATA_ITEMINFO1_BOUND_PHR3 :
750 (50 < time) ? PICODATA_ITEMINFO1_BOUND_PHR1 : PICODATA_ITEMINFO1_BOUND_PHR2;
751 switch (orig) {
752 /* for word and phrase breaks, return 'modified', unless a non-silence gets time==0, in which
753 * case return no break (word break) */
754 case PICODATA_ITEMINFO1_BOUND_PHR0:
755 if (0 == time) {
756 return PICODATA_ITEMINFO1_BOUND_PHR0;
757 }
758 case PICODATA_ITEMINFO1_BOUND_PHR3:
759 if (!wasPrimary && (0 == time)) {
760 return PICODATA_ITEMINFO1_BOUND_PHR0;
761 }
762 case PICODATA_ITEMINFO1_BOUND_PHR1:
763 case PICODATA_ITEMINFO1_BOUND_PHR2:
764 return modified;
765 break;
766 default:
767 return orig;
768 break;
769 }
770 }
771
breakStateInterrupting(picodata_itemhead_t * head,picoos_bool * breakBefore,picoos_bool * breakAfter)772 static picoos_bool breakStateInterrupting(picodata_itemhead_t * head,
773 picoos_bool * breakBefore, picoos_bool * breakAfter) {
774
775 picoos_bool result = 1;
776
777 *breakBefore = 0;
778 *breakAfter = 0;
779
780 if (PICODATA_ITEM_WORDPHON == head->type) {
781
782 } else if (PICODATA_ITEM_CMD == head->type) {
783 if ((PICODATA_ITEMINFO1_CMD_PLAY == head->info1)
784 || (PICODATA_ITEMINFO1_CMD_SAVE == head->info1)
785 || (PICODATA_ITEMINFO1_CMD_UNSAVE == head->info1)) {
786 *breakBefore = 1;
787 *breakAfter = 1;
788 } else if (PICODATA_ITEMINFO1_CMD_SAVE == head->info1) {
789 *breakBefore = 1;
790 } else if (PICODATA_ITEMINFO1_CMD_UNSAVE == head->info1) {
791 *breakAfter = 1;
792 } else if (PICODATA_ITEMINFO1_CMD_IGNSIG == head->info1) {
793 if (PICODATA_ITEMINFO2_CMD_START == head->info2) {
794 *breakBefore = 1;
795 } else {
796 *breakAfter = 1;
797 }
798 }
799 } else {
800 result = 0;
801 }
802 return result;
803 }
804
805
putSideBoundToOutput(spho_subobj_t * spho)806 static void putSideBoundToOutput(spho_subobj_t * spho)
807 {
808
809 picodata_itemhead_t ohead;
810 picoos_uint8 ocontent[2*sizeof(picoos_uint16)];
811 picoos_int16 sildur;
812 picoos_uint16 clen;
813
814 /* create boundary */
815 ohead.type = PICODATA_ITEM_BOUND;
816 ohead.info1 = spho->headx[spho->outReadPos].boundstrength;
817 ohead.info2 = spho->headx[spho->outReadPos].phrasetype;
818 sildur = spho->headx[spho->outReadPos].sildur;
819 if ((sildur < 0)
820 || (PICODATA_ITEMINFO1_BOUND_PHR0 == ohead.info1)
821 || (PICODATA_ITEMINFO1_BOUND_PHR3 == ohead.info1)) {
822 PICODBG_DEBUG(("outputting a bound of strength '%c' and type '%c' without duration constraints",ohead.info1, ohead.info2));
823 ohead.len = 0;
824 } else {
825 picoos_uint32 pos = 0;
826 picoos_write_mem_pi_uint16(ocontent,&pos,sildur);
827 picoos_write_mem_pi_uint16(ocontent,&pos,sildur);
828 PICODBG_DEBUG(("outputting a bound of strength '%c' and type '%c' with duration constraints [%i,%i]",ohead.info1, ohead.info2,sildur, sildur));
829 ohead.len = pos;
830 }
831 picodata_put_itemparts(&ohead, ocontent, ohead.len,
832 spho->outBuf, spho->outBufSize, &clen);
833 /* disable side bound */
834 spho->headx[spho->outReadPos].boundstrength = 0;
835 }
836
837 /** Set bound strength and sil dur.
838 *
839 * given the original bound strength 'orig_strength' and the fst-suggested bound strength 'fst_strength'
840 * and possibly being in a pending break state, calculate the resulting bound strength and set boundstrength
841 * and sildur of the current item (spho->headx[spho->outReadPos]) accordingly.
842 * if a boundstrength was set, also calculate the phrasetype and if necessary (and reachable), modify the phrase type
843 * of the previous phrase boundary.
844 *
845 * @param spho
846 * @param orig_strength
847 * @param orig_type
848 * @param fst_strength
849 */
setSideBound(spho_subobj_t * spho,picoos_uint8 orig_strength,picoos_uint8 orig_type,picoos_uint8 fst_strength)850 static void setSideBound(spho_subobj_t * spho, picoos_uint8 orig_strength, picoos_uint8 orig_type, picoos_uint8 fst_strength) {
851 picoos_uint8 strength;
852
853 /* insert modified bound according to transduction symbol, if any */
854 if (PICODATA_ITEMINFO1_NA == orig_strength) {
855 /* no original/fst strength given */
856 orig_strength = PICODATA_ITEMINFO1_BOUND_PHR0;
857 strength = PICODATA_ITEMINFO1_BOUND_PHR0;
858 } else {
859 strength = fstModifiedBoundStrength(orig_strength,fst_strength);
860 spho->headx[spho->outReadPos].boundstrength = strength;
861 spho->headx[spho->outReadPos].sildur = -1;
862 PICODBG_DEBUG(("setting bound strength to fst-suggested value %c (was %c)",strength, spho->headx[spho->outReadPos].boundstrength, spho->breakTime));
863 }
864
865 /* insert modified bound according to pending break, if any */
866 if (spho->breakPending) {
867 /* the calculation is based on the fst-modified value (because this is what the customer wants to
868 * override)
869 */
870 strength = breakModifiedBoundStrength(strength, spho->breakTime, (PICODATA_ITEMINFO1_BOUND_PHR1 == orig_strength));
871 PICODBG_DEBUG(("setting bound strength to break-imposed value %c (was %c) and time to %i",strength, spho->headx[spho->outReadPos].boundstrength, spho->breakTime));
872 spho->headx[spho->outReadPos].boundstrength = strength;
873 spho->headx[spho->outReadPos].sildur = spho->breakTime;
874 spho->breakPending = FALSE;
875 }
876 if (spho->headx[spho->outReadPos].boundstrength) {
877 /* we did set a bound strength, possibly promoting or demoting a boundary; now set the phrase type
878 * possibly also changing the phrase type of the previous phrase bound
879 */
880 picoos_uint8 fromPhrase = ((PICODATA_ITEMINFO1_BOUND_PHR0 != orig_strength));
881 picoos_uint8 toPhrase = ((PICODATA_ITEMINFO1_BOUND_PHR0 != strength));
882
883 PICODBG_DEBUG(("setting phrase type (wasPhrase=%i, isPhrase=%i)",fromPhrase,toPhrase));
884 if (toPhrase) {
885 if (fromPhrase) {
886 spho->lastPhraseType = orig_type;
887 } else { /*promote */
888 if (spho->activeStartPos <= spho->lastPhraseBoundPos) {
889 /* we still can change prev phrase bound */
890 /* since a new phrase boundary is introduced, we have to 'invent'
891 * an additional phrase type here. For that, we have to use some of the
892 * knowledge that otherwise is handled in picoacph.
893 */
894 spho->headx[spho->lastPhraseBoundPos].phrasetype
895 = PICODATA_ITEMINFO2_BOUNDTYPE_P;
896 }
897 }
898 spho->lastPhraseBoundPos = spho->outReadPos;
899 spho->headx[spho->lastPhraseBoundPos].phrasetype
900 = spho->lastPhraseType;
901
902 } else {
903 spho->headx[spho->outReadPos].phrasetype = PICODATA_ITEMINFO2_NA;
904 if (fromPhrase) { /* demote */
905 spho->lastPhraseType = orig_type;
906 if (spho->activeStartPos <= spho->lastPhraseBoundPos) {
907 /* we still can change prev phrase bound */
908 spho->headx[spho->lastPhraseBoundPos].phrasetype
909 = spho->lastPhraseType;
910 }
911 }
912 }
913 }
914 }
915
916
917 /* ***********************************************************************/
918 /* sphoStep function */
919 /* ***********************************************************************/
920
921
sphoStep(register picodata_ProcessingUnit this,picoos_int16 mode,picoos_uint16 * numBytesOutput)922 static picodata_step_result_t sphoStep(register picodata_ProcessingUnit this,
923 picoos_int16 mode, picoos_uint16 * numBytesOutput)
924 {
925
926 register spho_subobj_t *spho;
927 pico_status_t rv= PICO_OK;
928 picoos_uint16 blen;
929 picodata_itemhead_t ihead, ohead;
930 picoos_uint8 *icontent;
931 picoos_uint16 nextInPos;
932 #if defined(PICO_DEBUG)
933 picoos_char msgstr[SPHO_MSGSTR_SIZE];
934 #endif
935
936 /* used in FEED and FEED_SYM */
937 picoos_uint16 clen;
938 picoos_int16 pos, sym, sylsym;
939 picoos_uint8 plane;
940
941 /* used in BOUNDS */
942 picoos_bool breakBefore, breakAfter;
943
944 /* pico_status_t rvP= PICO_OK; */
945
946 picoos_uint16 curPos /*, nextPos */;
947 picoos_uint16 remHeadxSize, remCbufSize;
948
949
950 if (NULL == this || NULL == this->subObj) {
951 return PICODATA_PU_ERROR;
952 }
953 spho = (spho_subobj_t *) this->subObj;
954
955 mode = mode; /* avoid warning "var not used in this function"*/
956
957 *numBytesOutput = 0;
958 while (1) { /* exit via return */
959 PICODBG_INFO(("doing state %i, headxReadPos: %d, headxWritePos: %d",
960 spho->procState, spho->headxReadPos, spho->headxWritePos));
961
962 switch (spho->procState) {
963
964 case SPHO_STEPSTATE_INIT:
965 /* **********************************************************************/
966 /* INIT */
967 /* **********************************************************************/
968 PICODBG_DEBUG(("INIT"));
969 /* (re)set values for PARSE */
970 spho->penultima = SPHO_POS_INVALID;
971 spho->activeEndPos = SPHO_POS_INVALID;
972 spho->headxReadPos = 0;
973 spho->phonReadPos = 0;
974 spho->phonWritePos = 0;
975 spho->lastPhraseType = PICODATA_ITEMINFO2_NA;
976 spho->lastPhraseBoundPos = -1;
977
978 spho->procState = SPHO_STEPSTATE_COLLECT;
979 break;
980
981
982 case SPHO_STEPSTATE_COLLECT:
983 /* **********************************************************************/
984 /* COLLECT */
985 /* **********************************************************************/
986 /* collect state: get items from charBuf and store in
987 * internal inBuf
988 */
989 PICODBG_TRACE(("COLLECT"));
990 rv = PICO_OK;
991 remHeadxSize = spho->headxBufSize - spho->headxWritePos;
992 remCbufSize = spho->cbufBufSize - spho->cbufWritePos;
993 curPos = spho->headxWritePos;
994 while ((PICO_OK == rv) && (remHeadxSize > 0) && (remCbufSize > 0)) {
995 PICODBG_DEBUG(("COLLECT getting item at headxWritePos %i (remaining %i)",spho->headxWritePos, remHeadxSize));
996 rv = picodata_cbGetItem(this->cbIn, spho->tmpbuf, PICODATA_MAX_ITEMSIZE, &blen);
997 if (PICO_OK == rv) {
998 rv = picodata_get_itemparts(spho->tmpbuf,
999 PICODATA_MAX_ITEMSIZE, &(spho->headx[spho->headxWritePos].head),
1000 &(spho->cbuf[spho->cbufWritePos]), remCbufSize, &blen);
1001 if (PICO_OK == rv) {
1002 spho->headx[spho->headxWritePos].cind = spho->cbufWritePos;
1003 spho->headx[spho->headxWritePos].boundstrength = 0;
1004 spho->headxWritePos++;
1005 remHeadxSize--;
1006 spho->cbufWritePos += blen;
1007 remCbufSize -= blen;
1008 }
1009 }
1010 }
1011 if ((PICO_OK == rv) && ((remHeadxSize <= 0) || (remCbufSize <= 0))) {
1012 rv = PICO_EXC_BUF_OVERFLOW;
1013 }
1014
1015 /* in normal circumstances, rv is either PICO_EOF (no more items in cbIn) or PICO_BUF_OVERFLOW
1016 * (if no more items fit into headx) */
1017 if ((PICO_EOF != rv) && (PICO_EXC_BUF_OVERFLOW != rv)) {
1018 PICODBG_DEBUG(("COLLECT ** problem getting item, unhandled, rv: %i", rv));
1019 picoos_emRaiseException(this->common->em, rv,
1020 NULL, NULL);
1021 return PICODATA_PU_ERROR;
1022 }
1023 if (PICO_EOF == rv) { /* there are no more items available */
1024 if (curPos < spho->headxWritePos) { /* we did get some new items */
1025 PICODBG_DEBUG(("COLLECT read %i items",
1026 spho->headxWritePos - curPos));
1027 spho->needMoreInput = FALSE;
1028 }
1029 if (spho->needMoreInput) { /* not enough items to proceed */
1030 PICODBG_DEBUG(("COLLECT need more data, returning IDLE"));
1031 return PICODATA_PU_IDLE;
1032 } else {
1033 spho->procState = SPHO_STEPSTATE_PROCESS_PARSE;
1034 /* uncomment next to split into two steps */
1035 /* return PICODATA_PU_ATOMIC; */
1036 }
1037 } else { /* input buffer full */
1038 PICODBG_DEBUG(("COLLECT input buffer full"));
1039 if (spho->needMoreInput) { /* forced output because we can't get more data */
1040 spho->needMoreInput = FALSE;
1041 spho->force = TRUE;
1042 }
1043 spho->procState = SPHO_STEPSTATE_PROCESS_PARSE;
1044 }
1045 break;
1046
1047 case SPHO_STEPSTATE_PROCESS_PARSE:
1048
1049 /* **********************************************************************/
1050 /* PARSE: items -> input pos/phon pairs */
1051 /* **********************************************************************/
1052
1053 /* parse one item at a time */
1054 /* If
1055 * - the item is a sentence end or
1056 * - it is the last item and force=1 or
1057 * - the phon buffer is full
1058 * then set inReadPos to 0 and go to TRANSDUCE
1059 * else advance by one item */
1060
1061 /* look at the current item */
1062 PICODBG_TRACE(("PARSE"));
1063 if (spho->headxReadPos >= spho->headxWritePos) {
1064 /* no more items in headx */
1065 if (spho->force) {
1066 PICODBG_INFO(("no more items in headx but we are forced to transduce"));
1067
1068 /* headx is full; we are forced to transduce before reaching the sentence end */
1069 spho->force = FALSE;
1070 if (SPHO_POS_INVALID == spho->activeEndPos) {
1071 spho->activeEndPos = spho->headxReadPos;
1072 }
1073 spho->procState = SPHO_STEPSTATE_PROCESS_TRANSDUCE;
1074 } else {
1075 /* we try to get more data */
1076 PICODBG_INFO(("no more items in headx, try to collect more"));
1077 spho->needMoreInput = TRUE;
1078 spho->procState = SPHO_STEPSTATE_COLLECT;
1079 }
1080 break;
1081 }
1082
1083 ihead = spho->headx[spho->headxReadPos].head;
1084 icontent = spho->cbuf + spho->headx[spho->headxReadPos].cind;
1085
1086 PICODBG_DEBUG(("PARSE looking at item %s",picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE)));
1087 /* treat header */
1088 if (PICODATA_ITEM_BOUND == ihead.type) {
1089 /* see if it is a sentence end or termination boundary (flush) */
1090 if ((PICODATA_ITEMINFO1_BOUND_SEND == ihead.info1)
1091 || (PICODATA_ITEMINFO1_BOUND_TERM == ihead.info1)) {
1092 PICODBG_INFO(("PARSE found sentence end or term BOUND"));
1093
1094 if (spho->sentenceStarted) {
1095 /* its the end of the sentence */
1096 PICODBG_INFO(("PARSE found sentence end"));
1097 spho->sentenceStarted = 0;
1098 /* there is no need for a right context; move the active end to the end */
1099 /* add sentence termination phonemes */
1100 sphoAddTermPhonemes(spho, spho->headxReadPos);
1101 spho->headxReadPos++;
1102 spho->activeEndPos = spho->headxReadPos;
1103 /* we may discard all information up to activeEndPos, after processing of last
1104 * sentence part
1105 */
1106 spho->penultima = spho->activeEndPos;
1107
1108 /* transduce */
1109 spho->procState = SPHO_STEPSTATE_PROCESS_TRANSDUCE;
1110 /* uncomment to split */
1111 /* return PICODATA_PU_BUSY; */
1112 break;
1113 } else {
1114 if (PICODATA_ITEMINFO1_BOUND_TERM == ihead.info1) {
1115 /* its the end of input (flush) */
1116 PICODBG_INFO(("PARSE forwarding input end (flush)"));
1117 /* copy item unmodified */
1118 picodata_put_itemparts(&ihead,
1119 icontent,
1120 ihead.len,
1121 spho->outBuf, spho->outBufSize,
1122 &clen);
1123
1124 spho->headxReadPos++;
1125 spho->activeEndPos = spho->headxReadPos;
1126 spho->penultima = SPHO_POS_INVALID;
1127 spho->feedFollowState = SPHO_STEPSTATE_SHIFT;
1128 spho->procState = SPHO_STEPSTATE_FEED;
1129 break;
1130 } else {
1131 /* this should never happen */
1132 /* eliminate bound */
1133 spho->headxReadPos++;
1134 spho->activeEndPos = spho->headxReadPos;
1135 spho->penultima = SPHO_POS_INVALID;
1136 PICODBG_ERROR(("PARSE found a sentence end without a sentence start; eliminated"));
1137 }
1138 }
1139 } else if (PICODATA_ITEMINFO1_BOUND_SBEG == ihead.info1) {
1140 /* its the start of the sentence */
1141 PICODBG_INFO(("PARSE found sentence start"));
1142 /* add sentence starting phoneme */
1143 sphoAddStartPhoneme(spho);
1144
1145 spho->sentenceStarted = 1;
1146 }
1147 }
1148
1149 if ((PICODATA_ITEM_WORDPHON == ihead.type)
1150 || (PICODATA_ITEM_BOUND == ihead.type)) {
1151 /* if it is a word or a bound try to extract phonemes */
1152 PICODBG_INFO(("PARSE found WORD phon or phrase BOUND"));
1153 rv = sphoExtractPhonemes(this, spho, spho->headxReadPos,
1154 TRUE /* convertAccents */,
1155 &spho->suppressParseWordBound);
1156 if (PICO_OK == rv) {
1157 PICODBG_INFO(("PARSE successfully returned from phoneme extraction"));
1158 /* replace activeEndPos if the new item is a word, or activeEndPos was not set yet, or
1159 * activeEndPos was a bound */
1160 if ((spho->activeStartPos <= spho->headxReadPos) && ((PICODATA_ITEM_WORDPHON == ihead.type)
1161 || (SPHO_POS_INVALID == spho->activeEndPos)
1162 || (PICODATA_ITEM_BOUND == spho->headx[spho->activeEndPos].head.type))) {
1163 PICODBG_INFO(("PARSE found new activeEndPos: %i,%i -> %i,%i",
1164 spho->penultima,spho->activeEndPos,spho->activeEndPos,spho->headxReadPos));
1165 spho->penultima = spho->activeEndPos;
1166 spho->activeEndPos = spho->headxReadPos;
1167 }
1168
1169 } else if (PICO_EXC_BUF_OVERFLOW == rv) {
1170 /* phoneme buffer cannot take this item anymore;
1171 if the phoneme buffer has some contents, we are forced to transduce before reaching the sentence end
1172 else we skip the (too long word) */
1173 PICODBG_INFO(("PARSE returned from phoneme extraction with overflow, number of phonemes in phonBuf: %i; forced to TRANSDUCE", spho->phonWritePos));
1174 if ((SPHO_POS_INVALID == spho->activeEndPos) || (spho->activeStartPos == spho->activeEndPos)) {
1175 spho->activeEndPos = spho->headxReadPos;
1176 }
1177 spho->procState = SPHO_STEPSTATE_PROCESS_TRANSDUCE;
1178 break;
1179 } else {
1180 PICODBG_ERROR(("PARSE returned from phoneme extraction with exception %i",rv));
1181 return (picodata_step_result_t)picoos_emRaiseException(this->common->em,
1182 PICO_ERR_OTHER, NULL, NULL);
1183 }
1184 } else {
1185 PICODBG_INFO(("PARSE found other item, passing over"));
1186 /* it is "other" item, ignore */
1187 }
1188 /* set pos at next item */
1189 PICODBG_INFO(("PARSE going to next item: %i -> %i",spho->headxReadPos, spho->headxReadPos + 1));
1190 spho->headxReadPos++;
1191 break;
1192
1193 case SPHO_STEPSTATE_PROCESS_TRANSDUCE:
1194
1195 /* **********************************************************************/
1196 /* TRANSDUCE: transduction input pos/phon pairs to output pos/phon pairs */
1197 /* **********************************************************************/
1198 PICODBG_DEBUG(("TRANSDUCE (%i-th of %i fsts",spho->curFst+1, spho->numFsts));
1199
1200 /* termination condition first */
1201 if (spho->curFst >= spho->numFsts) {
1202
1203 #if defined(PICO_DEBUG)
1204 {
1205 PICODBG_INFO_CTX();
1206 PICODBG_INFO_MSG(("result of all transductions: "));
1207 PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], spho->phonBufOut, spho->phonWritePos);
1208 PICODBG_INFO_MSG(("\n"));
1209 }
1210 #endif
1211
1212 /* reset for next transduction */
1213 spho->curFst = 0;
1214 /* prepare BOUNDS */
1215 spho->outReadPos = 0;
1216 spho->phonReadPos = 0;
1217
1218 spho->procState = SPHO_STEPSTATE_PROCESS_BOUNDS;
1219 break;
1220 }
1221
1222 /* transduce from phonBufIn to PhonBufOut */
1223 {
1224
1225 picoos_uint32 nrSteps;
1226 #if defined(PICO_DEBUG)
1227 {
1228 PICODBG_INFO_CTX();
1229 PICODBG_INFO_MSG(("spho trying to transduce: "));
1230 PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], spho->phonBuf, spho->phonWritePos);
1231 PICODBG_INFO_MSG(("\n"));
1232 }
1233 #endif
1234 rv = picotrns_transduce(spho->fst[spho->curFst], FALSE,
1235 picotrns_printSolution, spho->phonBuf, spho->phonWritePos, spho->phonBufOut,
1236 &spho->phonWritePos,
1237 4*PICOTRNS_MAX_NUM_POSSYM, spho->altDescBuf,
1238 spho->maxAltDescLen, &nrSteps);
1239 if (PICO_OK == rv) {
1240 #if defined(PICO_DEBUG)
1241 {
1242 PICODBG_INFO_CTX();
1243 PICODBG_INFO_MSG(("result of transduction: (output symbols: %i)", spho->phonWritePos));
1244 PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], spho->phonBufOut, spho->phonWritePos);
1245 PICODBG_INFO_MSG(("\n"));
1246 }
1247 #endif
1248 PICODBG_TRACE(("number of steps done in tranduction: %i", nrSteps));
1249 } else {
1250 picoos_emRaiseWarning(this->common->em, PICO_WARN_FALLBACK,NULL,(picoos_char *)"phon buffer full");
1251 }
1252 }
1253 /* eliminate deep epsilons */
1254 picotrns_eliminate_epsilons(spho->phonBufOut, spho->phonWritePos, spho->phonBuf,
1255 &spho->phonWritePos,4*PICOTRNS_MAX_NUM_POSSYM);
1256
1257 spho->curFst++;
1258
1259 /* return PICODATA_PU_ATOMIC */
1260 break;
1261
1262
1263 case SPHO_STEPSTATE_PROCESS_BOUNDS:
1264 /* ************************************************************************/
1265 /* BOUNDS: combine input item with pos/phon pairs to insert/modify bounds */
1266 /* ************************************************************************/
1267
1268 PICODBG_INFO(("BOUNDS"));
1269
1270 /* get the suppressRecombWordBound in the left context */
1271 spho->suppressRecombWordBound = FALSE;
1272 while (spho->outReadPos < spho->activeStartPos) {
1273 /* look at the current item */
1274 ihead = spho->headx[spho->outReadPos].head;
1275 /* icontent = spho->cbuf + spho->headx[spho->outReadPos].cind; */
1276 PICODBG_INFO(("in position %i, looking at item %s",spho->outReadPos,picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE)));
1277 if (PICODATA_ITEM_BOUND == ihead.type) {
1278 spho->suppressRecombWordBound = TRUE;
1279 } else if (PICODATA_ITEM_WORDPHON == ihead.type) {
1280 spho->suppressRecombWordBound = FALSE;
1281 }
1282 spho->outReadPos++;
1283 }
1284 /* spho->outReadPos point now to the active region */
1285
1286 /* advance the phone reading pos to the active range */
1287 spho->phonReadPos = 0;
1288 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho, &pos, &sym,
1289 spho->activeStartPos))) {
1290 /* ignore */
1291 }
1292 PICODBG_INFO(("skipping left context phones results in %s", (SPHO_POSSYM_OUT_OF_RANGE==rv) ? "OUT_OF_RANGE" : (SPHO_POSSYM_END ==rv) ? "END" : "OTHER"));
1293
1294 /*
1295 * Align input items with transduced phones and note bound stregth changes and break commands
1296 */
1297
1298 while (spho->outReadPos < spho->activeEndPos) {
1299
1300 /* look at the current item */
1301 ihead = spho->headx[spho->outReadPos].head;
1302 icontent = spho->cbuf + spho->headx[spho->outReadPos].cind;
1303 nextInPos = spho->outReadPos + 1;
1304 /* */
1305 PICODBG_INFO(("in position %i, looking at item %s",spho->outReadPos,picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE)));
1306
1307 if ((PICODATA_ITEM_BOUND == ihead.type)
1308 || ((PICODATA_ITEM_WORDPHON == ihead.type)
1309 && (!spho->suppressRecombWordBound))) {
1310 /* there was a boundary originally */
1311 picoos_uint8 orig_strength, orig_type;
1312 if (PICODATA_ITEM_BOUND == ihead.type) {
1313 orig_strength = ihead.info1;
1314 orig_type = ihead.info2;
1315 spho->suppressRecombWordBound = TRUE;
1316 } else {
1317 orig_strength = PICODATA_ITEMINFO1_BOUND_PHR0;
1318 orig_type = PICODATA_ITEMINFO2_NA;
1319 }
1320 /* i expect a boundary phone here */
1321 /* consume FST bound phones, consider pending break and set the side-bound */
1322 PICODBG_INFO(("got BOUND or WORDPHON item and expects corresponding phone"));
1323 rv = getNextPosSym(spho, &pos, &sym, nextInPos);
1324 if (SPHO_POSSYM_OK != rv) {
1325 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list (%s)", (SPHO_POSSYM_OUT_OF_RANGE==rv) ? "OUT_OF_RANGE" : (SPHO_POSSYM_END ==rv) ? "END" :"OTHER"));
1326 return (picodata_step_result_t)picoos_emRaiseException(this->common->em,
1327 PICO_ERR_OTHER, NULL, NULL);
1328 }
1329 sym = picotrns_unplane(sym, &plane);
1330 /* */
1331 PICODBG_ASSERT((PICOKFST_PLANE_PB_STRENGTHS == plane));
1332
1333 /* insert modified bound according to transduction and possibly pending break */
1334 setSideBound(spho, orig_strength, orig_type,
1335 (picoos_uint8) sym);
1336 } else if ((PICODATA_ITEM_CMD == ihead.type)
1337 && (PICODATA_ITEMINFO1_CMD_SIL == ihead.info1)) {
1338 /* it's a SIL (break) command */
1339 picoos_uint16 time;
1340 picoos_uint32 pos = 0;
1341 picoos_read_mem_pi_uint16(icontent, &pos, &time);
1342 if (spho->breakPending) {
1343 spho->breakTime += time;
1344 } else {
1345 spho->breakTime = time;
1346 spho->breakPending = TRUE;
1347 }
1348 } else if ((PICODATA_ITEM_CMD == ihead.type) && (PICODATA_ITEMINFO1_CMD_PLAY == ihead.info1)) {
1349 /* insert break of at least one ms */
1350 if (!spho->breakPending || (spho->breakTime <= 0)) {
1351 spho->breakTime = SPHO_SMALLEST_SIL_DUR;
1352 spho->breakPending = TRUE;
1353 }
1354 setSideBound(spho, PICODATA_ITEMINFO1_NA,
1355 PICODATA_ITEMINFO2_NA, PICODATA_ITEMINFO1_NA);
1356 /* force following break to be at least one ms */
1357 spho->breakTime = SPHO_SMALLEST_SIL_DUR;
1358 spho->breakPending = TRUE;
1359 } else if (breakStateInterrupting(&ihead, &breakBefore, &breakAfter)) {
1360
1361 if (breakBefore &&(!spho->breakPending || (spho->breakTime <= 0))) {
1362 spho->breakTime = SPHO_SMALLEST_SIL_DUR;
1363 spho->breakPending = TRUE;
1364 }
1365 setSideBound(spho, PICODATA_ITEMINFO1_NA,
1366 PICODATA_ITEMINFO2_NA, PICODATA_ITEMINFO1_NA);
1367
1368 if (breakAfter) {
1369 spho->breakTime = SPHO_SMALLEST_SIL_DUR;
1370 spho->breakPending = TRUE;
1371 }
1372 if (PICODATA_ITEM_WORDPHON == ihead.type) {
1373 spho->suppressRecombWordBound = FALSE;
1374 }
1375 }
1376
1377 /* skip phones of that item */
1378 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho, &pos,
1379 &sym, nextInPos))) {
1380 /* ignore */
1381 }
1382 spho->outReadPos++;
1383 }
1384
1385 /* reset for RECOMB */
1386 spho->outReadPos = 0;
1387 spho->phonReadPos = 0;
1388 spho->suppressRecombWordBound = FALSE;
1389
1390 spho->procState = SPHO_STEPSTATE_PROCESS_RECOMB;
1391 return PICODATA_PU_ATOMIC;
1392
1393 break;
1394
1395 case SPHO_STEPSTATE_PROCESS_RECOMB:
1396 /* **********************************************************************/
1397 /* RECOMB: combine input item with pos/phon pairs to output item */
1398 /* **********************************************************************/
1399
1400 PICODBG_TRACE(("RECOMB"));
1401
1402 /* default place to come after feed: here */
1403 spho->feedFollowState = SPHO_STEPSTATE_PROCESS_RECOMB;
1404
1405 /* check termination condition first */
1406 if (spho->outReadPos >= spho->activeEndPos) {
1407 PICODBG_DEBUG(("RECOMB reached active region's end at %i",spho->outReadPos));
1408 spho->procState = SPHO_STEPSTATE_SHIFT;
1409 break;
1410 }
1411
1412 /* look at the current item */
1413 ihead = spho->headx[spho->outReadPos].head;
1414 icontent = spho->cbuf + spho->headx[spho->outReadPos].cind;
1415
1416 PICODBG_DEBUG(("RECOMB looking at item %s",picodata_head_to_string(&ihead,msgstr,SPHO_MSGSTR_SIZE)));
1417
1418 nextInPos = spho->outReadPos + 1;
1419
1420 PICODBG_DEBUG(("RECOMB treating item in headx at pos %i",spho->outReadPos));
1421 if (nextInPos <= spho->activeStartPos) { /* we're in the (passive) left context. Just skip it */
1422 PICODBG_DEBUG(("RECOMB skipping item in the left context (%i <= %i)",nextInPos, spho->activeStartPos));
1423 if (PICODATA_ITEM_BOUND == ihead.type) {
1424 spho->suppressRecombWordBound = 1;
1425 } else if (PICODATA_ITEM_WORDPHON == ihead.type) {
1426 spho->suppressRecombWordBound = 0;
1427 }
1428
1429 /* consume possyms */
1430 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho,&pos,&sym,nextInPos))) {
1431 /* ignore */
1432 }
1433 if (rv == SPHO_POSSYM_INVALID) {
1434 return (picodata_step_result_t)picoos_emRaiseException(this->common->em,
1435 PICO_ERR_OTHER, NULL, NULL);
1436 }
1437 spho->outReadPos = nextInPos;
1438 } else { /* active region */
1439 if (spho->headx[spho->outReadPos].boundstrength) {
1440 /* ***************** "side-bound" *********************/
1441 /* copy to outbuf */
1442 putSideBoundToOutput(spho);
1443 /* mark as processed */
1444 spho->headx[spho->outReadPos].boundstrength = 0;
1445 /* output it */
1446 spho->procState = SPHO_STEPSTATE_FEED;
1447 } else if (PICODATA_ITEM_BOUND == ihead.type) {
1448 /* ***************** BOUND *********************/
1449 /* expect a boundary phone here */
1450 PICODBG_DEBUG(("RECOMB got BOUND item and expects corresponding phone"));
1451 rv = getNextPosSym(spho, &pos, &sym, nextInPos);
1452 if (SPHO_POSSYM_OK != rv) {
1453 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list"));
1454 return (picodata_step_result_t)picoos_emRaiseException(
1455 this->common->em, PICO_ERR_OTHER, NULL,
1456 NULL);
1457 }
1458 sym = picotrns_unplane(sym, &plane);
1459 /* */
1460 PICODBG_ASSERT((PICOKFST_PLANE_PB_STRENGTHS == plane));
1461
1462 spho->suppressRecombWordBound = TRUE; /* if word following, don't need word boundary */
1463 /* just consume item and come back here*/
1464 spho->outReadPos = nextInPos;
1465
1466 } else if (PICODATA_ITEM_WORDPHON == ihead.type) {
1467 /* ***************** WORDPHON *********************/
1468 spho->wordStarted = TRUE;
1469 /* i expect a word boundary symbol in this range unless a phrase boundary was encountered before */
1470 if (spho->suppressRecombWordBound) {
1471 PICODBG_DEBUG(("RECOMB got WORDPHON item but skips expecting BOUND"));
1472 spho->suppressRecombWordBound = FALSE;
1473 } else {
1474 PICODBG_DEBUG(("RECOMB got WORDPHON item and expects corresponding bound phone"));
1475 rv = getNextPosSym(spho, &pos, &sym, nextInPos);
1476 if (SPHO_POSSYM_OK != rv) {
1477 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list"));
1478 return (picodata_step_result_t)picoos_emRaiseException(this->common->em,
1479 PICO_ERR_OTHER, NULL, NULL);
1480 }
1481 }
1482 spho->procState = SPHO_STEPSTATE_PROCESS_SYL;
1483 } else if ((PICODATA_ITEM_CMD == ihead.type) && (PICODATA_ITEMINFO1_CMD_SIL == ihead.info1)) {
1484 /* ***************** BREAK COMMAND *********************/
1485 /* just consume and come back here */
1486 PICODBG_DEBUG(("RECOMB consuming item from inBuf %i -> %i",spho->outReadPos, nextInPos));
1487 spho->outReadPos = nextInPos;
1488 } else {
1489 /* ***************** OTHER *********************/
1490 /* just copy item */
1491 PICODBG_DEBUG(("RECOMB found other item, just copying"));
1492 picodata_put_itemparts(&ihead, icontent, ihead.len,
1493 spho->outBuf, spho->outBufSize, &clen);
1494 PICODBG_DEBUG(("RECOMB consuming item from inBuf %i -> %i",spho->outReadPos, nextInPos));
1495 spho->outReadPos = nextInPos;
1496 /* and output it */
1497 spho->procState = SPHO_STEPSTATE_FEED;
1498 } /* if (ihead.type) */
1499
1500 }
1501
1502 /* return PICODATA_PU_BUSY; */
1503 break;
1504
1505 case SPHO_STEPSTATE_PROCESS_SYL:
1506 /* **********************************************************************/
1507 /* SYL: combine input word item with pos/phon pairs to syl output item */
1508 /* **********************************************************************/
1509
1510 /* consume all transduced phonemes with pos in in the range [spho->outReadPos,nextInPos[ */
1511 PICODBG_DEBUG(("SYL"));
1512
1513 spho->feedFollowState = SPHO_STEPSTATE_PROCESS_SYL;
1514
1515 /* look at the current item */
1516 ihead = spho->headx[spho->outReadPos].head;
1517 icontent = spho->cbuf + spho->headx[spho->outReadPos].cind;
1518 nextInPos = spho->outReadPos + 1;
1519 PICODBG_DEBUG(("SYL (1) treating item in headx at pos %i",spho->outReadPos));
1520 /* create syllable item in ohead (head) and sylBuf (contents) */
1521 ohead.type = PICODATA_ITEM_SYLLPHON;
1522
1523 PICODBG_TRACE(("SYL expects accent at phonBuf[%i] = (%i,%i) (outReadPos=%i)", spho->phonReadPos, spho->phonBuf[spho->phonReadPos].pos, spho->phonBuf[spho->phonReadPos].sym,spho->outReadPos));
1524 rv = getNextPosSym(spho,&pos,&sym,nextInPos);
1525 if (SPHO_POSSYM_OK != rv) {
1526 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list (%i)",rv));
1527 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL);
1528 }
1529 ohead.info2 = picotrns_unplane(sym, &plane);
1530 PICODBG_ASSERT((PICOKFST_PLANE_ACCENTS == plane));
1531 PICODBG_DEBUG(("SYL sets accent to %c", sym));
1532
1533 /* for the time being, we force to use POS so we can transduce all fsts in a row without reconsulting the items */
1534 PICODBG_TRACE(("SYL expects POS"));
1535 PICODBG_DEBUG(("SYL (2) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos));
1536 rv = getNextPosSym(spho,&pos,&sym,nextInPos);
1537 if (SPHO_POSSYM_OK != rv) {
1538 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list"));
1539 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL);
1540 }
1541 if (spho->wordStarted) {
1542 spho->wordStarted = FALSE;
1543 ohead.info1 = picotrns_unplane(sym, &plane);
1544 /* */
1545 PICODBG_ASSERT(PICOKFST_PLANE_POS == plane);
1546 /* */
1547 PICODBG_DEBUG(("SYL setting POS to %c", ohead.info1));
1548 } else {
1549 ohead.info1 = PICODATA_ITEMINFO1_NA;
1550 }
1551
1552 PICODBG_DEBUG(("SYL (3) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos));
1553 /* get phonemes of that syllable; stop if syllable boundary or outside word */
1554 sylsym = (PICOKFST_PLANE_PHONEMES << 8)
1555 + spho->syllSepId;
1556 PICODBG_DEBUG(("collecting syllable phonemes before headx position %i",nextInPos));
1557 spho->sylWritePos = 0;
1558 while (SPHO_POSSYM_OK == (rv = getNextPosSym(spho,&pos,&sym,nextInPos)) && (sym != sylsym)) {
1559 spho->sylBuf[spho->sylWritePos++] = picotrns_unplane(sym, &plane);
1560 /* */
1561 PICODBG_TRACE(("SYL adding phoneme to syllable: (pos %i,sym %i)[plane %i,sym %c]",pos,sym,plane,sym & 0xFF));
1562 PICODBG_ASSERT((PICOKFST_PLANE_PHONEMES == plane));
1563 }
1564 PICODBG_DEBUG(("SYL (4) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos));
1565 ohead.len = spho->sylWritePos;
1566 if (SPHO_POS_INVALID == rv) {
1567 PICODBG_ERROR(("unexpected symbol or unexpected end of phoneme list"));
1568 return (picodata_step_result_t)picoos_emRaiseException(this->common->em, PICO_WARN_INCOMPLETE, NULL, NULL);
1569 } else if ((SPHO_POSSYM_OUT_OF_RANGE == rv) || (SPHO_POSSYM_END == rv)) {
1570 PICODBG_DEBUG(("SYL arrived at end of word and/or end of phon buffer, go to next word"));
1571 spho->outReadPos = nextInPos; /* advance to next item */
1572 spho->feedFollowState = SPHO_STEPSTATE_PROCESS_RECOMB; /* go to RECOMB after feed */
1573 } else {
1574 PICODBG_ASSERT((sym == sylsym));
1575 }
1576 PICODBG_DEBUG(("SYL (5) treating item in inBuf range [%i,%i[",spho->outReadPos,nextInPos));
1577
1578 if (ohead.len > 0) {
1579 /* prepare syllable output */
1580 picodata_put_itemparts(&ohead, spho->sylBuf,
1581 PICODATA_BUFSIZE_DEFAULT, spho->outBuf,
1582 spho->outBufSize, &clen);
1583
1584 spho->procState = SPHO_STEPSTATE_FEED;
1585 } else { /* skip feeding output of empty syllable */
1586 spho->procState = spho->feedFollowState;
1587 }
1588 break;
1589
1590 case SPHO_STEPSTATE_FEED:
1591 /* **********************************************************************/
1592 /* FEED: output output item and proceed to feedFollowState */
1593 /* **********************************************************************/
1594
1595 PICODBG_DEBUG(("FEED"));
1596
1597 PICODBG_DEBUG(("FEED putting outBuf item into cb"));
1598
1599 /*feeding items to PU output buffer*/
1600 rv = picodata_cbPutItem(this->cbOut, spho->outBuf,
1601 spho->outBufSize, &clen);
1602
1603 PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
1604 (picoos_uint8 *)"spho: ",
1605 spho->outBuf, spho->outBufSize);
1606
1607 if (PICO_EXC_BUF_OVERFLOW == rv) {
1608 /* we have to redo this item */
1609 PICODBG_DEBUG(("FEED got overflow, returning ICODATA_PU_OUT_FULL"));
1610 return PICODATA_PU_OUT_FULL;
1611 } else if (PICO_OK == rv) {
1612 *numBytesOutput += clen;
1613 spho->procState = spho->feedFollowState;
1614 PICODBG_DEBUG(("FEED ok, going back to procState %i", spho->procState));
1615 return PICODATA_PU_BUSY;
1616 } else {
1617 PICODBG_DEBUG(("FEED got exception %i when trying to output item",rv));
1618 spho->procState = spho->feedFollowState;
1619 return (picodata_step_result_t)rv;
1620 }
1621 break;
1622
1623 case SPHO_STEPSTATE_SHIFT:
1624 /* **********************************************************************/
1625 /* SHIFT */
1626 /* **********************************************************************/
1627 /* If there exists a valid penultima, it should replace any left context (from 0 to activeStartPos)
1628 * else discard the current active range (from activeStartPos to activeEndPos), leaving the current
1629 * left context intact. Often, PARSE would move activeStartPos to 0, so that there is no left context
1630 * after the shift.
1631 */
1632
1633 PICODBG_DEBUG(("SHIFT"));
1634
1635 if (spho->penultima != SPHO_POS_INVALID) {
1636 picoos_int16 shift;
1637 /* set penultima as new left context and set activeStartPos to the shifted activeEndPos */
1638 PICODBG_DEBUG((
1639 "SHIFT shifting penultima from %i to 0",
1640 spho->penultima));
1641 shift = shift_range_left_1(spho, &spho->penultima, 0);
1642 if (shift < 0) {
1643 picoos_emRaiseException(this->common->em,PICO_ERR_OTHER,NULL,NULL);
1644 return PICODATA_PU_ERROR;
1645 }
1646 spho->activeStartPos = spho->activeEndPos
1647 - shift;
1648 spho->lastPhraseBoundPos -= shift;
1649 spho->suppressParseWordBound = FALSE;
1650 spho->suppressRecombWordBound = FALSE;
1651
1652 } else {
1653 picoos_int16 shift;
1654 picoos_bool lastPhraseBoundActive;
1655 if (spho->activeStartPos == spho->activeEndPos) {
1656 /* no items consumed; we have to abandon left context */
1657 spho->activeStartPos = 0;
1658 }
1659 lastPhraseBoundActive = (spho->lastPhraseBoundPos >= spho->activeStartPos);
1660 /* dummy comment */
1661 PICODBG_DEBUG(("SHIFT shift active end from %i to %i",
1662 spho->activeEndPos, spho->activeStartPos));
1663 shift = shift_range_left_1(spho, &spho->activeEndPos, spho->activeStartPos);
1664 if (shift < 0) {
1665 picoos_emRaiseException(this->common->em,PICO_ERR_OTHER,NULL,NULL);
1666 return PICODATA_PU_ERROR;
1667 }
1668 if (lastPhraseBoundActive) {
1669 spho->lastPhraseBoundPos -= shift;
1670 }
1671 }
1672
1673 spho->procState = SPHO_STEPSTATE_INIT;
1674 break;
1675
1676 default:
1677 picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL);
1678 return PICODATA_PU_ERROR;
1679 break;
1680
1681 } /* switch (spho->procState) */
1682
1683 } /* while (1) */
1684
1685 /* should be never reached */
1686 picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL);
1687 return PICODATA_PU_ERROR;
1688 }
1689
1690 #ifdef __cplusplus
1691 }
1692 #endif
1693
1694 /* end picospho.c */
1695