openwnn/JAJP/OpenWnnClauseConverterJAJP.java

/*
 * Copyright (C) 2008-2012  OMRON SOFTWARE Co., Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jp.co.omronsoft.openwnn.JAJP;

import jp.co.omronsoft.openwnn.*;
import java.util.*;

/**
 * The penWnn Clause Converter class for Japanese IME.
 *
 * @author Copyright (C) 2009 OMRON SOFTWARE CO., LTD.  All Rights Reserved.
 */
public class OpenWnnClauseConverterJAJP {
    /** Score(frequency value) of word in the learning dictionary */
    private static final int FREQ_LEARN = 600;
    /** Score(frequency value) of word in the user dictionary */
    private static final int FREQ_USER  = 500;

    /** Maximum limit length of input */
    public static final int MAX_INPUT_LENGTH = 50;

    /** search cache for unique independent words (jiritsugo) */
    private HashMap<String, ArrayList<WnnWord>> mIndepWordBag;
    /** search cache for all independent words (jiritsugo) */
    private HashMap<String, ArrayList<WnnWord>> mAllIndepWordBag;
    /** search cache for ancillary words (fuzokugo) */
    private HashMap<String, ArrayList<WnnWord>> mFzkPatterns;

    /** connect matrix for generating a clause */
    private byte[][] mConnectMatrix;

    /** dictionaries */
    private WnnDictionary mDictionary;

    /** candidates of conversion */
    private LinkedList mConvertResult;

    /** work area for consecutive clause conversion */
    private WnnSentence[] mSentenceBuffer;

    /** part of speech (default) */
    private WnnPOS mPosDefault;
    /** part of speech (end of clause/not end of sentence) */
    private WnnPOS mPosEndOfClause1;
    /** part of speech (end of clause/any place) */
    private WnnPOS mPosEndOfClause2;
    /** part of speech (end of sentence) */
    private WnnPOS mPosEndOfClause3;

    /** cost value of a clause */
    private static final int CLAUSE_COST = -1000;

    /** The candidate filter */
    private CandidateFilter mFilter = null;

    /**
     * Constructor
     */
    public OpenWnnClauseConverterJAJP() {
        mIndepWordBag  = new HashMap<String, ArrayList<WnnWord>>();
        mAllIndepWordBag  = new HashMap<String, ArrayList<WnnWord>>();
        mFzkPatterns   = new HashMap();
        mConvertResult = new LinkedList();

        mSentenceBuffer = new WnnSentence[MAX_INPUT_LENGTH];
    }

    /**
     * Set the dictionary
     *
     * @param dict  The dictionary for phrase conversion
     */
    public void setDictionary(WnnDictionary dict) {
        /* get connect matrix */
        mConnectMatrix = dict.getConnectMatrix();

        /* clear dictionary settings */
        mDictionary = dict;
        dict.clearDictionary();
        dict.clearApproxPattern();

        /* clear work areas */
        mIndepWordBag.clear();
        mAllIndepWordBag.clear();
        mFzkPatterns.clear();

        /* get part of speech tags */
        mPosDefault      = dict.getPOS(WnnDictionary.POS_TYPE_MEISI);
        mPosEndOfClause1 = dict.getPOS(WnnDictionary.POS_TYPE_V1);
        mPosEndOfClause2 = dict.getPOS(WnnDictionary.POS_TYPE_V2);
        mPosEndOfClause3 = dict.getPOS(WnnDictionary.POS_TYPE_V3);
    }

    /**
     * Set the candidate filter
     *
     * @param filter	The candidate filter
     */
    public void setFilter(CandidateFilter filter) {
    	mFilter = filter;
    }

    /**
     * Kana-to-Kanji conversion (single clause).
     * <br>
     * This method execute single clause conversion.
      *
     * @param input		The input string
     * @return			The candidates of conversion; {@code null} if an error occurs.
     */
     public Iterator convert(String input) {
        /* do nothing if no dictionary is specified */
        if (mConnectMatrix == null || mDictionary == null) {
            return null;
        }
        /* do nothing if the length of input exceeds the limit */
        if (input.length() > MAX_INPUT_LENGTH) {
            return null;
        }

        /* clear the candidates list */
        mConvertResult.clear();

        /* try single clause conversion */
        if (!singleClauseConvert(mConvertResult, input, mPosEndOfClause2, true)) {
            return null;
        }
        return mConvertResult.iterator();
    }

    /**
     * Consecutive clause conversion.
     *
     * @param input		The input string
     * @return			The result of consecutive clause conversion; {@code null} if fail.
     */
    public WnnSentence consecutiveClauseConvert(String input) {
        LinkedList clauses = new LinkedList();

        /* clear the cache which is not matched */
        for (int i = 0; i < input.length(); i++) {
            mSentenceBuffer[i] = null;
        }
        WnnSentence[] sentence = mSentenceBuffer;

        /* consecutive clause conversion */
        for (int start = 0; start < input.length(); start++) {
            if (start != 0 && sentence[start-1] == null) {
                continue;
            }

            /* limit the length of a clause */
            int end = input.length();
            if (end > start + 20) {
                end = start + 20;
            }
            /* make clauses */
            for ( ; end > start; end--) {
                int idx = end - 1;

                /* cutting a branch */
                if (sentence[idx] != null) {
                    if (start != 0) {
                        if (sentence[idx].frequency > sentence[start-1].frequency + CLAUSE_COST + FREQ_LEARN) {
                            /* there may be no way to be the best sequence from the 'start' */
                            break;
                        }
                    } else {
                        if (sentence[idx].frequency > CLAUSE_COST + FREQ_LEARN) {
                            /* there may be no way to be the best sequence from the 'start' */
                            break;
                        }
                    }
                }

                String key = input.substring(start, end);
                clauses.clear();
                WnnClause bestClause = null;
                if (end == input.length()) {
                    /* get the clause which can be the end of the sentence */
                    singleClauseConvert(clauses, key, mPosEndOfClause1, false);
                } else {
                    /* get the clause which is not the end of the sentence */
                    singleClauseConvert(clauses, key, mPosEndOfClause3, false);
                }
                if (clauses.isEmpty()) {
                    bestClause = defaultClause(key);
                } else {
                    bestClause = (WnnClause)clauses.get(0);
                }

                /* make a sub-sentence */
                WnnSentence ws;
                if (start == 0) {
                    ws = new WnnSentence(key, bestClause);
                } else {
                    ws = new WnnSentence(sentence[start-1], bestClause);
                }
                ws.frequency += CLAUSE_COST;

                /* update the best sub-sentence on the cache buffer */
                if (sentence[idx] == null || (sentence[idx].frequency < ws.frequency)) {
                    sentence[idx] = ws;
                }
            }
        }

        /* return the result of the consecutive clause conversion */
        if (sentence[input.length() - 1] != null) {
            return sentence[input.length() - 1];
        }
        return null;
    }

    /**
     * Consecutive clause conversion.
     *
     * @param resultList	Where to store the result
     * @param input			Input string
     * @return				{@code true} if success; {@code false} if fail.
     */
    private boolean consecutiveClauseConvert(LinkedList resultList, String input) {
        WnnSentence sentence = consecutiveClauseConvert(input);

        /* set the result of the consecutive clause conversion on the top of the list */
        if (sentence != null) {
            resultList.add(0, sentence);
            return true;
        }
        return false;
    }

    /**
     * Single clause conversion.
     *
     * @param clauseList	Where to store the results
     * @param input			Input string
     * @param terminal		Part of speech tag at the terminal
     * @param all			Get all candidates or not
     * @return				{@code true} if success; {@code false} if fail.
     */
    private boolean singleClauseConvert(LinkedList clauseList, String input, WnnPOS terminal, boolean all) {
        boolean ret = false;

        /* get clauses without ancillary word */
        ArrayList<WnnWord> stems = getIndependentWords(input, all);
        if (stems != null && (!stems.isEmpty())) {
            Iterator<WnnWord> stemsi = stems.iterator();
            while (stemsi.hasNext()) {
                WnnWord stem = stemsi.next();
                if (addClause(clauseList, input, stem, null, terminal, all)) {
                    ret = true;
                }
            }
        }

        /* get clauses with ancillary word */
        int max = CLAUSE_COST * 2;
        for (int split = 1; split < input.length(); split++) {
            /* get ancillary patterns */
            String str = input.substring(split);
            ArrayList<WnnWord> fzks = getAncillaryPattern(str);
            if (fzks == null || fzks.isEmpty()) {
                continue;
            }

            /* get candidates of stem in a clause */
            str = input.substring(0, split);
            stems = getIndependentWords(str, all);
            if (stems == null || stems.isEmpty()) {
                if (mDictionary.searchWord(WnnDictionary.SEARCH_PREFIX, WnnDictionary.ORDER_BY_FREQUENCY, str) <= 0) {
                    break;
                } else {
                    continue;
                }
            }
            /* make clauses */
            Iterator<WnnWord> stemsi = stems.iterator();
            while (stemsi.hasNext()) {
                WnnWord stem = stemsi.next();
                if (all || stem.frequency > max) {
                    Iterator<WnnWord> fzksi  = fzks.iterator();
                    while (fzksi.hasNext()) {
                        WnnWord fzk = fzksi.next();
                        if (addClause(clauseList, input, stem, fzk, terminal, all)) {
                            ret = true;
                            max = stem.frequency;
                        }
                    }
                }
            }
        }
        return ret;
    }

    /**
     * Add valid clause to the candidates list.
     *
     * @param clauseList	Where to store the results
     * @param input			Input string
     * @param stem			Stem of the clause (a independent word)
     * @param fzk			Ancillary pattern
     * @param terminal		Part of speech tag at the terminal
     * @param all			Get all candidates or not
     * @return				{@code true} if add the clause to the list; {@code false} if not.
     */
    private boolean addClause(LinkedList<WnnClause> clauseList, String input, WnnWord stem, WnnWord fzk,
                              WnnPOS terminal, boolean all) {
        WnnClause clause = null;
        /* check if the part of speech is valid */
        if (fzk == null) {
            if (connectible(stem.partOfSpeech.right, terminal.left)) {
                clause = new WnnClause(input, stem);
            }
        } else {
            if (connectible(stem.partOfSpeech.right, fzk.partOfSpeech.left)
                && connectible(fzk.partOfSpeech.right, terminal.left)) {
                clause = new WnnClause(input, stem, fzk);
            }
        }
        if (clause == null) {
            return false;
        }
        if (mFilter != null && !mFilter.isAllowed(clause)) {
        	return false;
        }

        /* store to the list */
        if (clauseList.isEmpty()) {
            /* add if the list is empty */
            clauseList.add(0, clause);
            return true;
        } else {
            if (!all) {
                /* reserve only the best clause */
                WnnClause best = (WnnClause)clauseList.get(0);
                if (best.frequency < clause.frequency) {
                    clauseList.set(0, clause);
                    return true;
                }
            } else {
                /* reserve all clauses */
                Iterator clauseListi = clauseList.iterator();
                int index = 0;
                while (clauseListi.hasNext()) {
                    WnnClause clausei = (WnnClause)clauseListi.next();
                    if (clausei.frequency < clause.frequency) {
                        break;
                    }
                    index++;
                }
                clauseList.add(index, clause);
                return true;
            }
        }

        return false;
    }

    /**
     * Check the part-of-speeches are connectable.
     *
     * @param right		Right attribute of the preceding word/clause
     * @param left		Left attribute of the following word/clause
     * @return			{@code true} if there are connectable; {@code false} if otherwise
     */
    private boolean connectible(int right, int left) {
        try {
            if (mConnectMatrix[left][right] != 0) {
                return true;
            }
        } catch (Exception ex) {
        }
        return false;
    }

    /**
     * Get all exact matched ancillary words(Fuzokugo) list.
     *
     * @param input		Search key
     * @return			List of ancillary words
     */
    private ArrayList<WnnWord> getAncillaryPattern(String input) {
        if (input.length() == 0) {
            return null;
        }

        HashMap<String,ArrayList<WnnWord>> fzkPat = mFzkPatterns;
        ArrayList<WnnWord> fzks = fzkPat.get(input);
        if (fzks != null) {
            return fzks;
        }

        /* set dictionaries */
        WnnDictionary dict = mDictionary;
        dict.clearDictionary();
        dict.clearApproxPattern();
        dict.setDictionary(6, 400, 500);

        for (int start = input.length() - 1; start >= 0; start--) {
            String key = input.substring(start);

            fzks = fzkPat.get(key);
            if (fzks != null) {
                continue;
            }

            fzks = new ArrayList<WnnWord>();
            mFzkPatterns.put(key, fzks);

            /* search ancillary words */
            dict.searchWord(WnnDictionary.SEARCH_EXACT, WnnDictionary.ORDER_BY_FREQUENCY, key);
            WnnWord word;
            while ((word = dict.getNextWord()) != null) {
                fzks.add(word);
            }

            /* concatenate sequence of ancillary words */
            for (int end = input.length() - 1; end > start; end--) {
                ArrayList<WnnWord> followFzks = fzkPat.get(input.substring(end));
                if (followFzks == null ||  followFzks.isEmpty()) {
                    continue;
                }
                dict.searchWord(WnnDictionary.SEARCH_EXACT, WnnDictionary.ORDER_BY_FREQUENCY, input.substring(start, end));
                while ((word = dict.getNextWord()) != null) {
                    Iterator<WnnWord> followFzksi = followFzks.iterator();
                    while (followFzksi.hasNext()) {
                        WnnWord follow = followFzksi.next();
                        if (connectible(word.partOfSpeech.right, follow.partOfSpeech.left)) {
                            fzks.add(new WnnWord(key, key, new WnnPOS(word.partOfSpeech.left, follow.partOfSpeech.right)));
                        }
                    }
                }
            }
        }
        return fzks;
    }

    /**
     * Get all exact matched independent words(Jiritsugo) list.
     *
     * @param input    Search key
     * @param all      {@code true} if list all words; {@code false} if list words which has an unique part of speech tag.
     * @return			List of words; {@code null} if {@code input.length() == 0}.
     */
    private ArrayList<WnnWord> getIndependentWords(String input, boolean all) {
        if (input.length() == 0) {
            return null;
        }

        ArrayList<WnnWord> words = (all)? mAllIndepWordBag.get(input) : mIndepWordBag.get(input);

        if (words == null) {
            /* set dictionaries */
            WnnDictionary dict = mDictionary;
            dict.clearDictionary();
            dict.clearApproxPattern();
            dict.setDictionary(4, 0, 10);
            dict.setDictionary(5, 400, 500);
            dict.setDictionary(WnnDictionary.INDEX_USER_DICTIONARY, FREQ_USER, FREQ_USER);
            dict.setDictionary(WnnDictionary.INDEX_LEARN_DICTIONARY, FREQ_LEARN, FREQ_LEARN);

            words = new ArrayList<WnnWord>();
            WnnWord word;
            if (all) {
            	mAllIndepWordBag.put(input, words);
                dict.searchWord(WnnDictionary.SEARCH_EXACT, WnnDictionary.ORDER_BY_FREQUENCY, input);
                /* store all words */
                while ((word = dict.getNextWord()) != null) {
                    if (input.equals(word.stroke)) {
                        words.add(word);
                    }
                }
            } else {
            	mIndepWordBag.put(input, words);
                dict.searchWord(WnnDictionary.SEARCH_EXACT, WnnDictionary.ORDER_BY_FREQUENCY, input);
                /* store a word which has an unique part of speech tag */
                while ((word = dict.getNextWord()) != null) {
                    if (input.equals(word.stroke)) {
                        Iterator<WnnWord> list = words.iterator();
                        boolean found = false;
                        while (list.hasNext()) {
                            WnnWord w = (WnnWord)list.next();
                                if (w.partOfSpeech.right == word.partOfSpeech.right) {
                                    found = true;
                                    break;
                                }
                        }
                        if (!found) {
                            words.add(word);
                        }
                        if (word.frequency < 400) {
                            break;
                        }
                    }
                }
            }
            addAutoGeneratedCandidates(input, words, all);
        }
        return words;
    }

    /**
     * Add some words not including in the dictionary.
     * <br>
     * This method adds some words which are not in the dictionary.
     *
     * @param input     Input string
     * @param wordList  List to store words
     * @param all       Get all candidates or not
     */
    private void addAutoGeneratedCandidates(String input, ArrayList wordList, boolean all) {
        wordList.add(new WnnWord(input, input, mPosDefault, (CLAUSE_COST - 1) * input.length()));
    }

    /**
     * Get a default clause.
     * <br>
     * This method generates a clause which has a string same as input
     * and the default part-of-speech tag.
     *
     * @param input    Input string
     * @return			Default clause
     */
    private WnnClause defaultClause(String input) {
        return (new WnnClause(input, input, mPosDefault, (CLAUSE_COST - 1) * input.length()));
    }
}