1 /*
2  * Copyright (C) 2012 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.inputmethod.latin.makedict;
18 
19 import com.android.inputmethod.annotations.UsedForTesting;
20 import com.android.inputmethod.latin.define.DecoderSpecificConstants;
21 
22 import java.util.Date;
23 import java.util.HashMap;
24 
25 /**
26  * Dictionary File Format Specification.
27  */
28 public final class FormatSpec {
29 
30     /*
31      * File header layout is as follows:
32      *
33      * v |
34      * e | MAGIC_NUMBER + version of the file format, 2 bytes.
35      * r |
36      * sion
37      *
38      * o |
39      * p | not used, 2 bytes.
40      * o |
41      * nflags
42      *
43      * h |
44      * e | size of the file header, 4bytes
45      * a |   including the size of the magic number, the option flags and the header size
46      * d |
47      * ersize
48      *
49      * attributes list
50      *
51      * attributes list is:
52      * <key>   = | string of characters at the char format described below, with the terminator used
53      *           | to signal the end of the string.
54      * <value> = | string of characters at the char format described below, with the terminator used
55      *           | to signal the end of the string.
56      * if the size of already read < headersize, goto key.
57      *
58      */
59 
60     /*
61      * Node array (FusionDictionary.PtNodeArray) layout is as follows:
62      *
63      * n |
64      * o | the number of PtNodes, 1 or 2 bytes.
65      * d | 1 byte = bbbbbbbb match
66      * e |   case 1xxxxxxx => xxxxxxx << 8 + next byte
67      * c |   otherwise => bbbbbbbb
68      * o |
69      * unt
70      *
71      * n |
72      * o | sequence of PtNodes,
73      * d | the layout of each PtNode is described below.
74      * e |
75      * s
76      *
77      * f |
78      * o | forward link address, 3byte
79      * r | 1 byte = bbbbbbbb match
80      * w |   case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte)
81      * a |   otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte
82      * r |
83      * dlinkaddress
84      */
85 
86     /* Node (FusionDictionary.PtNode) layout is as follows:
87      *   | CHILDREN_ADDRESS_TYPE  2 bits, 11          : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES
88      *   |                                10          : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES
89      * f |                                01          : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE
90      * l |                                00          : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS
91      * a | has several chars ?         1 bit, 1 = yes, 0 = no   : FLAG_HAS_MULTIPLE_CHARS
92      * g | has a terminal ?            1 bit, 1 = yes, 0 = no   : FLAG_IS_TERMINAL
93      * s | has shortcut targets ?      1 bit, 1 = yes, 0 = no   : FLAG_HAS_SHORTCUT_TARGETS
94      *   | has bigrams ?               1 bit, 1 = yes, 0 = no   : FLAG_HAS_BIGRAMS
95      *   | is not a word ?             1 bit, 1 = yes, 0 = no   : FLAG_IS_NOT_A_WORD
96      *   | is possibly offensive ?     1 bit, 1 = yes, 0 = no   : FLAG_IS_POSSIBLY_OFFENSIVE
97      *
98      * c | IF FLAG_HAS_MULTIPLE_CHARS
99      * h |   char, char, char, char    n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers
100      * a |   end                       1 byte, = 0
101      * r | ELSE
102      * s |   char                      1 or 3 bytes
103      *   | END
104      *
105      * f |
106      * r | IF FLAG_IS_TERMINAL
107      * e |   frequency                 1 byte
108      * q |
109      *
110      * c |
111      * h | children address, CHILDREN_ADDRESS_TYPE bytes
112      * i | This address is relative to the position of this field.
113      * l |
114      * drenaddress
115      *
116      *   | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS
117      *   | shortcut string list
118      *   | IF FLAG_IS_TERMINAL && FLAG_HAS_BIGRAMS
119      *   | bigrams address list
120      *
121      * Char format is:
122      * 1 byte = bbbbbbbb match
123      * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
124      * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
125      *       unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
126      *       00011111 would be outside unicode.
127      * else: iso-latin-1 code
128      * This allows for the whole unicode range to be encoded, including chars outside of
129      * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
130      * characters which should never happen anyway (and still work, but take 3 bytes).
131      *
132      * bigram address list is:
133      * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no     : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT
134      *           | addressSign = 1 bit,                 : FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE
135      *           |                      1 = must take -address, 0 = must take +address
136      *           |                         xx : mask with MASK_BIGRAM_ATTR_ADDRESS_TYPE
137      *           | addressFormat = 2 bits, 00 = unused  : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE
138      *           |                         01 = 1 byte  : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE
139      *           |                         10 = 2 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES
140      *           |                         11 = 3 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES
141      *           | 4 bits : frequency         : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY
142      * <address> | IF (01 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE == addressFormat)
143      *           |   read 1 byte, add top 4 bits
144      *           | ELSIF (10 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES == addressFormat)
145      *           |   read 2 bytes, add top 4 bits
146      *           | ELSE // 11 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES == addressFormat
147      *           |   read 3 bytes, add top 4 bits
148      *           | END
149      *           | if (FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) then address = -address
150      * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) goto bigram_and_shortcut_address_list_is
151      *
152      * shortcut string list is:
153      * <byte size> = PTNODE_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes.
154      * <flags>     = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT
155      *               | reserved = 3 bits, must be 0
156      *               | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY
157      * <shortcut>  = | string of characters at the char format described above, with the terminator
158      *               | used to signal the end of the string.
159      * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT goto flags
160      */
161 
162     public static final int MAGIC_NUMBER = 0x9BC13AFE;
163     static final int NOT_A_VERSION_NUMBER = -1;
164 
165     // These MUST have the same values as the relevant constants in format_utils.h.
166     // From version 2.01 on, we use version * 100 + revision as a version number. That allows
167     // us to change the format during development while having testing devices remove
168     // older files with each upgrade, while still having a readable versioning scheme.
169     // When we bump up the dictionary format version, we should update
170     // ExpandableDictionary.needsToMigrateDictionary() and
171     // ExpandableDictionary.matchesExpectedBinaryDictFormatVersionForThisType().
172     public static final int VERSION2 = 2;
173     public static final int VERSION201 = 201;
174     public static final int VERSION202 = 202;
175     // format version for Fava Dictionaries.
176     public static final int VERSION_DELIGHT3 = 86736212;
177     public static final int MINIMUM_SUPPORTED_VERSION_OF_CODE_POINT_TABLE = VERSION201;
178     // Dictionary version used for testing.
179     public static final int VERSION4_ONLY_FOR_TESTING = 399;
180     public static final int VERSION402 = 402;
181     public static final int VERSION403 = 403;
182     public static final int VERSION4 = VERSION403;
183     public static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202;
184     public static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION_DELIGHT3;
185     static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4;
186     static final int MAXIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION403;
187 
188     // TODO: Make this value adaptative to content data, store it in the header, and
189     // use it in the reading code.
190     static final int MAX_WORD_LENGTH = DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH;
191 
192     // These flags are used only in the static dictionary.
193     static final int MASK_CHILDREN_ADDRESS_TYPE = 0xC0;
194     static final int FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS = 0x00;
195     static final int FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE = 0x40;
196     static final int FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES = 0x80;
197     static final int FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = 0xC0;
198 
199     static final int FLAG_HAS_MULTIPLE_CHARS = 0x20;
200 
201     static final int FLAG_IS_TERMINAL = 0x10;
202     static final int FLAG_HAS_SHORTCUT_TARGETS = 0x08;
203     static final int FLAG_HAS_BIGRAMS = 0x04;
204     static final int FLAG_IS_NOT_A_WORD = 0x02;
205     static final int FLAG_IS_POSSIBLY_OFFENSIVE = 0x01;
206 
207     static final int FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT = 0x80;
208     static final int FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE = 0x40;
209     static final int MASK_BIGRAM_ATTR_ADDRESS_TYPE = 0x30;
210     static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE = 0x10;
211     static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES = 0x20;
212     static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES = 0x30;
213     static final int FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY = 0x0F;
214 
215     static final int PTNODE_CHARACTERS_TERMINATOR = 0x1F;
216 
217     static final int PTNODE_TERMINATOR_SIZE = 1;
218     static final int PTNODE_FLAGS_SIZE = 1;
219     static final int PTNODE_FREQUENCY_SIZE = 1;
220     static final int PTNODE_MAX_ADDRESS_SIZE = 3;
221     static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1;
222     static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3;
223     static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2;
224 
225     static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE;
226     static final int INVALID_CHARACTER = -1;
227 
228     static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127
229     // Large PtNode array size field size is 2 bytes.
230     static final int LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000;
231     static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767
232     static final int MAX_BIGRAMS_IN_A_PTNODE = 10000;
233     static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF;
234 
235     static final int MAX_TERMINAL_FREQUENCY = 255;
236     static final int MAX_BIGRAM_FREQUENCY = 15;
237 
238     public static final int SHORTCUT_WHITELIST_FREQUENCY = 15;
239 
240     // This option needs to be the same numeric value as the one in binary_format.h.
241     static final int NOT_VALID_WORD = -99;
242 
243     static final int UINT8_MAX = 0xFF;
244     static final int UINT16_MAX = 0xFFFF;
245     static final int UINT24_MAX = 0xFFFFFF;
246     static final int MSB8 = 0x80;
247     static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
248     static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
249 
250     /**
251      * Options about file format.
252      */
253     public static final class FormatOptions {
254         public final int mVersion;
255         public final boolean mHasTimestamp;
256 
257         @UsedForTesting
FormatOptions(final int version)258         public FormatOptions(final int version) {
259             this(version, false /* hasTimestamp */);
260         }
261 
FormatOptions(final int version, final boolean hasTimestamp)262         public FormatOptions(final int version, final boolean hasTimestamp) {
263             mVersion = version;
264             mHasTimestamp = hasTimestamp;
265         }
266     }
267 
268     /**
269      * Options global to the dictionary.
270      */
271     public static final class DictionaryOptions {
272         public final HashMap<String, String> mAttributes;
DictionaryOptions(final HashMap<String, String> attributes)273         public DictionaryOptions(final HashMap<String, String> attributes) {
274             mAttributes = attributes;
275         }
276         @Override
toString()277         public String toString() { // Convenience method
278             return toString(0, false);
279         }
toString(final int indentCount, final boolean plumbing)280         public String toString(final int indentCount, final boolean plumbing) {
281             final StringBuilder indent = new StringBuilder();
282             if (plumbing) {
283                 indent.append("H:");
284             } else {
285                 for (int i = 0; i < indentCount; ++i) {
286                     indent.append(" ");
287                 }
288             }
289             final StringBuilder s = new StringBuilder();
290             for (final String optionKey : mAttributes.keySet()) {
291                 s.append(indent);
292                 s.append(optionKey);
293                 s.append(" = ");
294                 if ("date".equals(optionKey) && !plumbing) {
295                     // Date needs a number of milliseconds, but the dictionary contains seconds
296                     s.append(new Date(
297                             1000 * Long.parseLong(mAttributes.get(optionKey))).toString());
298                 } else {
299                     s.append(mAttributes.get(optionKey));
300                 }
301                 s.append("\n");
302             }
303             return s.toString();
304         }
305     }
306 
FormatSpec()307     private FormatSpec() {
308         // This utility class is not publicly instantiable.
309     }
310 }
311