1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.inputmethod.latin.makedict; 18 19 import com.android.inputmethod.annotations.UsedForTesting; 20 import com.android.inputmethod.latin.define.DecoderSpecificConstants; 21 22 import java.util.Date; 23 import java.util.HashMap; 24 25 /** 26 * Dictionary File Format Specification. 27 */ 28 public final class FormatSpec { 29 30 /* 31 * File header layout is as follows: 32 * 33 * v | 34 * e | MAGIC_NUMBER + version of the file format, 2 bytes. 35 * r | 36 * sion 37 * 38 * o | 39 * p | not used, 2 bytes. 40 * o | 41 * nflags 42 * 43 * h | 44 * e | size of the file header, 4bytes 45 * a | including the size of the magic number, the option flags and the header size 46 * d | 47 * ersize 48 * 49 * attributes list 50 * 51 * attributes list is: 52 * <key> = | string of characters at the char format described below, with the terminator used 53 * | to signal the end of the string. 54 * <value> = | string of characters at the char format described below, with the terminator used 55 * | to signal the end of the string. 56 * if the size of already read < headersize, goto key. 57 * 58 */ 59 60 /* 61 * Node array (FusionDictionary.PtNodeArray) layout is as follows: 62 * 63 * n | 64 * o | the number of PtNodes, 1 or 2 bytes. 65 * d | 1 byte = bbbbbbbb match 66 * e | case 1xxxxxxx => xxxxxxx << 8 + next byte 67 * c | otherwise => bbbbbbbb 68 * o | 69 * unt 70 * 71 * n | 72 * o | sequence of PtNodes, 73 * d | the layout of each PtNode is described below. 74 * e | 75 * s 76 * 77 * f | 78 * o | forward link address, 3byte 79 * r | 1 byte = bbbbbbbb match 80 * w | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) 81 * a | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte 82 * r | 83 * dlinkaddress 84 */ 85 86 /* Node (FusionDictionary.PtNode) layout is as follows: 87 * | CHILDREN_ADDRESS_TYPE 2 bits, 11 : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES 88 * | 10 : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES 89 * f | 01 : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE 90 * l | 00 : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS 91 * a | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS 92 * g | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL 93 * s | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS 94 * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS 95 * | is not a word ? 1 bit, 1 = yes, 0 = no : FLAG_IS_NOT_A_WORD 96 * | is possibly offensive ? 1 bit, 1 = yes, 0 = no : FLAG_IS_POSSIBLY_OFFENSIVE 97 * 98 * c | IF FLAG_HAS_MULTIPLE_CHARS 99 * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers 100 * a | end 1 byte, = 0 101 * r | ELSE 102 * s | char 1 or 3 bytes 103 * | END 104 * 105 * f | 106 * r | IF FLAG_IS_TERMINAL 107 * e | frequency 1 byte 108 * q | 109 * 110 * c | 111 * h | children address, CHILDREN_ADDRESS_TYPE bytes 112 * i | This address is relative to the position of this field. 113 * l | 114 * drenaddress 115 * 116 * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS 117 * | shortcut string list 118 * | IF FLAG_IS_TERMINAL && FLAG_HAS_BIGRAMS 119 * | bigrams address list 120 * 121 * Char format is: 122 * 1 byte = bbbbbbbb match 123 * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte 124 * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because 125 * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with 126 * 00011111 would be outside unicode. 127 * else: iso-latin-1 code 128 * This allows for the whole unicode range to be encoded, including chars outside of 129 * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control 130 * characters which should never happen anyway (and still work, but take 3 bytes). 131 * 132 * bigram address list is: 133 * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT 134 * | addressSign = 1 bit, : FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE 135 * | 1 = must take -address, 0 = must take +address 136 * | xx : mask with MASK_BIGRAM_ATTR_ADDRESS_TYPE 137 * | addressFormat = 2 bits, 00 = unused : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE 138 * | 01 = 1 byte : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE 139 * | 10 = 2 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES 140 * | 11 = 3 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES 141 * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY 142 * <address> | IF (01 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE == addressFormat) 143 * | read 1 byte, add top 4 bits 144 * | ELSIF (10 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES == addressFormat) 145 * | read 2 bytes, add top 4 bits 146 * | ELSE // 11 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES == addressFormat 147 * | read 3 bytes, add top 4 bits 148 * | END 149 * | if (FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) then address = -address 150 * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) goto bigram_and_shortcut_address_list_is 151 * 152 * shortcut string list is: 153 * <byte size> = PTNODE_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. 154 * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT 155 * | reserved = 3 bits, must be 0 156 * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY 157 * <shortcut> = | string of characters at the char format described above, with the terminator 158 * | used to signal the end of the string. 159 * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT goto flags 160 */ 161 162 public static final int MAGIC_NUMBER = 0x9BC13AFE; 163 static final int NOT_A_VERSION_NUMBER = -1; 164 165 // These MUST have the same values as the relevant constants in format_utils.h. 166 // From version 2.01 on, we use version * 100 + revision as a version number. That allows 167 // us to change the format during development while having testing devices remove 168 // older files with each upgrade, while still having a readable versioning scheme. 169 // When we bump up the dictionary format version, we should update 170 // ExpandableDictionary.needsToMigrateDictionary() and 171 // ExpandableDictionary.matchesExpectedBinaryDictFormatVersionForThisType(). 172 public static final int VERSION2 = 2; 173 public static final int VERSION201 = 201; 174 public static final int VERSION202 = 202; 175 // format version for Fava Dictionaries. 176 public static final int VERSION_DELIGHT3 = 86736212; 177 public static final int VERSION402 = 402; 178 public static final int VERSION403 = 403; 179 public static final int VERSION4 = VERSION403; 180 public static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202; 181 public static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION_DELIGHT3; 182 static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4; 183 static final int MAXIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION403; 184 185 // TODO: Make this value adaptative to content data, store it in the header, and 186 // use it in the reading code. 187 static final int MAX_WORD_LENGTH = DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH; 188 189 // These flags are used only in the static dictionary. 190 static final int MASK_CHILDREN_ADDRESS_TYPE = 0xC0; 191 static final int FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS = 0x00; 192 static final int FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE = 0x40; 193 static final int FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES = 0x80; 194 static final int FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = 0xC0; 195 196 static final int FLAG_HAS_MULTIPLE_CHARS = 0x20; 197 198 static final int FLAG_IS_TERMINAL = 0x10; 199 static final int FLAG_HAS_SHORTCUT_TARGETS = 0x08; 200 static final int FLAG_HAS_BIGRAMS = 0x04; 201 static final int FLAG_IS_NOT_A_WORD = 0x02; 202 static final int FLAG_IS_POSSIBLY_OFFENSIVE = 0x01; 203 204 static final int FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT = 0x80; 205 static final int FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE = 0x40; 206 static final int MASK_BIGRAM_ATTR_ADDRESS_TYPE = 0x30; 207 static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE = 0x10; 208 static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES = 0x20; 209 static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES = 0x30; 210 static final int FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY = 0x0F; 211 212 static final int PTNODE_CHARACTERS_TERMINATOR = 0x1F; 213 214 static final int PTNODE_TERMINATOR_SIZE = 1; 215 static final int PTNODE_FLAGS_SIZE = 1; 216 static final int PTNODE_FREQUENCY_SIZE = 1; 217 static final int PTNODE_MAX_ADDRESS_SIZE = 3; 218 static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; 219 static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; 220 static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; 221 222 static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; 223 static final int INVALID_CHARACTER = -1; 224 225 static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 226 // Large PtNode array size field size is 2 bytes. 227 static final int LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; 228 static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 229 static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; 230 static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF; 231 232 static final int MAX_TERMINAL_FREQUENCY = 255; 233 static final int MAX_BIGRAM_FREQUENCY = 15; 234 235 public static final int SHORTCUT_WHITELIST_FREQUENCY = 15; 236 237 // This option needs to be the same numeric value as the one in binary_format.h. 238 static final int NOT_VALID_WORD = -99; 239 240 static final int UINT8_MAX = 0xFF; 241 static final int UINT16_MAX = 0xFFFF; 242 static final int UINT24_MAX = 0xFFFFFF; 243 static final int MSB8 = 0x80; 244 static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; 245 static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; 246 247 /** 248 * Options about file format. 249 */ 250 public static final class FormatOptions { 251 public final int mVersion; 252 public final boolean mHasTimestamp; 253 254 @UsedForTesting FormatOptions(final int version)255 public FormatOptions(final int version) { 256 this(version, false /* hasTimestamp */); 257 } 258 FormatOptions(final int version, final boolean hasTimestamp)259 public FormatOptions(final int version, final boolean hasTimestamp) { 260 mVersion = version; 261 mHasTimestamp = hasTimestamp; 262 } 263 } 264 265 /** 266 * Options global to the dictionary. 267 */ 268 public static final class DictionaryOptions { 269 public final HashMap<String, String> mAttributes; DictionaryOptions(final HashMap<String, String> attributes)270 public DictionaryOptions(final HashMap<String, String> attributes) { 271 mAttributes = attributes; 272 } 273 @Override toString()274 public String toString() { // Convenience method 275 return toString(0, false); 276 } toString(final int indentCount, final boolean plumbing)277 public String toString(final int indentCount, final boolean plumbing) { 278 final StringBuilder indent = new StringBuilder(); 279 if (plumbing) { 280 indent.append("H:"); 281 } else { 282 for (int i = 0; i < indentCount; ++i) { 283 indent.append(" "); 284 } 285 } 286 final StringBuilder s = new StringBuilder(); 287 for (final String optionKey : mAttributes.keySet()) { 288 s.append(indent); 289 s.append(optionKey); 290 s.append(" = "); 291 if ("date".equals(optionKey) && !plumbing) { 292 // Date needs a number of milliseconds, but the dictionary contains seconds 293 s.append(new Date( 294 1000 * Long.parseLong(mAttributes.get(optionKey))).toString()); 295 } else { 296 s.append(mAttributes.get(optionKey)); 297 } 298 s.append("\n"); 299 } 300 return s.toString(); 301 } 302 } 303 FormatSpec()304 private FormatSpec() { 305 // This utility class is not publicly instantiable. 306 } 307 } 308