1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * Copyright (C) 2013-2015, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ******************************************************************************* 8 * collationdatareader.h 9 * 10 * created on: 2013feb07 11 * created by: Markus W. Scherer 12 */ 13 14 #ifndef __COLLATIONDATAREADER_H__ 15 #define __COLLATIONDATAREADER_H__ 16 17 #include "unicode/utypes.h" 18 19 #if !UCONFIG_NO_COLLATION 20 21 #include "unicode/udata.h" 22 23 struct UDataMemory; 24 25 U_NAMESPACE_BEGIN 26 27 struct CollationTailoring; 28 29 /** 30 * Collation binary data reader. 31 */ 32 struct U_I18N_API CollationDataReader /* all static */ { 33 // The following constants are also copied into source/common/ucol_swp.cpp. 34 // Keep them in sync! 35 enum { 36 /** 37 * Number of int32_t indexes. 38 * 39 * Can be 2 if there are only options. 40 * Can be 7 or 8 if there are only options and a script reordering. 41 * The loader treats any index>=indexes[IX_INDEXES_LENGTH] as 0. 42 */ 43 IX_INDEXES_LENGTH, // 0 44 /** 45 * Bits 31..24: numericPrimary, for numeric collation 46 * 23..16: fast Latin format version (0 = no fast Latin table) 47 * 15.. 0: options bit set 48 */ 49 IX_OPTIONS, 50 IX_RESERVED2, 51 IX_RESERVED3, 52 53 /** Array offset to Jamo CE32s in ce32s[], or <0 if none. */ 54 IX_JAMO_CE32S_START, // 4 55 56 // Byte offsets from the start of the data, after the generic header. 57 // The indexes[] are at byte offset 0, other data follows. 58 // Each data item is aligned properly. 59 // The data items should be in descending order of unit size, 60 // to minimize the need for padding. 61 // Each item's byte length is given by the difference between its offset and 62 // the next index/offset value. 63 /** Byte offset to int32_t reorderCodes[]. */ 64 IX_REORDER_CODES_OFFSET, 65 /** 66 * Byte offset to uint8_t reorderTable[]. 67 * Empty table if <256 bytes (padding only). 68 * Otherwise 256 bytes or more (with padding). 69 */ 70 IX_REORDER_TABLE_OFFSET, 71 /** Byte offset to the collation trie. Its length is a multiple of 8 bytes. */ 72 IX_TRIE_OFFSET, 73 74 IX_RESERVED8_OFFSET, // 8 75 /** Byte offset to int64_t ces[]. */ 76 IX_CES_OFFSET, 77 IX_RESERVED10_OFFSET, 78 /** Byte offset to uint32_t ce32s[]. */ 79 IX_CE32S_OFFSET, 80 81 /** Byte offset to uint32_t rootElements[]. */ 82 IX_ROOT_ELEMENTS_OFFSET, // 12 83 /** Byte offset to UChar *contexts[]. */ 84 IX_CONTEXTS_OFFSET, 85 /** Byte offset to uint16_t [] with serialized unsafeBackwardSet. */ 86 IX_UNSAFE_BWD_OFFSET, 87 /** Byte offset to uint16_t fastLatinTable[]. */ 88 IX_FAST_LATIN_TABLE_OFFSET, 89 90 /** Byte offset to uint16_t scripts[]. */ 91 IX_SCRIPTS_OFFSET, // 16 92 /** 93 * Byte offset to UBool compressibleBytes[]. 94 * Empty table if <256 bytes (padding only). 95 * Otherwise 256 bytes or more (with padding). 96 */ 97 IX_COMPRESSIBLE_BYTES_OFFSET, 98 IX_RESERVED18_OFFSET, 99 IX_TOTAL_SIZE 100 }; 101 102 static void read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength, 103 CollationTailoring &tailoring, UErrorCode &errorCode); 104 105 static UBool U_CALLCONV 106 isAcceptable(void *context, const char *type, const char *name, const UDataInfo *pInfo); 107 108 private: 109 CollationDataReader(); // no constructor 110 }; 111 112 /* 113 * Format of collation data (ucadata.icu, binary data in coll/ *.res files). 114 * Format version 5. 115 * 116 * The root collation data is stored in the ucadata.icu file. 117 * Tailorings are stored inside .res resource bundle files, with a complete file header. 118 * 119 * Collation data begins with a standard ICU data file header 120 * (DataHeader, see ucmndata.h and unicode/udata.h). 121 * The UDataInfo.dataVersion field contains the UCA and other version numbers, 122 * see the comments for CollationTailoring.version. 123 * 124 * After the header, the file contains the following parts. 125 * Constants are defined as enum values of the CollationDataReader class. 126 * See also the Collation class. 127 * 128 * int32_t indexes[indexesLength]; 129 * The indexes array has variable length. 130 * Some tailorings only need the length and the options, 131 * others only add reorderCodes and the reorderTable, 132 * some need to store mappings. 133 * Only as many indexes are stored as needed to read all of the data. 134 * 135 * Index 0: indexesLength 136 * Index 1: numericPrimary, CollationFastLatin::VERSION, and options: see IX_OPTIONS 137 * Index 2..3: Unused/reserved/0. 138 * Index 4: Index into the ce32s array where the CE32s of the conjoining Jamo 139 * are stored in a short, contiguous part of the ce32s array. 140 * 141 * Indexes 5..19 are byte offsets in ascending order. 142 * Each byte offset marks the start of the next part in the data file, 143 * and the end of the previous one. 144 * When two consecutive byte offsets are the same (or too short), 145 * then the corresponding part is empty. 146 * Byte offsets are offsets from after the header, 147 * that is, from the beginning of the indexes[]. 148 * Each part starts at an offset with proper alignment for its data. 149 * If necessary, the previous part may include padding bytes to achieve this alignment. 150 * The last byte offset that is stored in the indexes indicates the total size of the data 151 * (starting with the indexes). 152 * 153 * int32_t reorderCodes[]; -- empty in root 154 * The list of script and reordering codes. 155 * 156 * Beginning with format version 5, this array may optionally 157 * have trailing entries with a full list of reorder ranges 158 * as described for CollationSettings::reorderRanges. 159 * 160 * Script or reorder codes are first and do not exceed 16-bit values. 161 * Range limits are stored in the upper 16 bits, and are never 0. 162 * Split this array into reorder codes and ranges at the first entry 163 * with non-zero upper 16 bits. 164 * 165 * If the ranges are missing but needed for split-reordered primary lead bytes, 166 * then they are regenerated at load time. 167 * 168 * uint8_t reorderTable[256]; -- empty in root; can be longer to include padding bytes 169 * Primary-weight lead byte permutation table. 170 * Normally present when the reorderCodes are, but can be built at load time. 171 * 172 * Beginning with format version 5, a 0 entry at a non-zero index 173 * (which is otherwise an illegal value) 174 * means that the primary lead byte is "split" 175 * (there are different offsets for primaries that share that lead byte) 176 * and the reordering offset must be determined via the reorder ranges 177 * that are either stored as part of the reorderCodes array 178 * or regenerated at load time. 179 * 180 * UTrie2 trie; -- see utrie2_impl.h and utrie2.h 181 * The trie holds the main collation data. Each code point is mapped to a 32-bit value. 182 * It encodes a simple collation element (CE) in compact form, unless bits 7..6 are both set, 183 * in which case it is a special CE32 and contains a 4-bit tag and further data. 184 * See the Collation class for details. 185 * 186 * The trie has a value for each lead surrogate code unit with some bits encoding 187 * collective properties of the 1024 supplementary characters whose UTF-16 form starts with 188 * the lead surrogate. See Collation::LEAD_SURROGATE_TAG.. 189 * 190 * int64_t ces[]; 191 * 64-bit CEs and expansions that cannot be stored in a more compact form. 192 * 193 * uint32_t ce32s[]; 194 * CE32s for expansions in compact form, and for characters whose trie values 195 * contain special data. 196 * 197 * uint32_t rootElements[]; -- empty in all tailorings 198 * Compact storage for all of the CEs that occur in the root collation. 199 * See the CollationRootElements class. 200 * 201 * UChar *contexts[]; 202 * Serialized UCharsTrie structures with prefix (pre-context) and contraction mappings. 203 * 204 * uint16_t unsafeBackwardSet[]; -- see UnicodeSet::serialize() 205 * Serialized form of characters that are unsafe when iterating backwards, 206 * and at the end of an identical string prefix. 207 * Back up to a safe character. 208 * Lead surrogates are "unsafe" when any of their corresponding supplementary 209 * code points are unsafe. 210 * Does not include [:^lccc=0:][:^tccc=0:]. 211 * For each tailoring, the root unsafeBackwardSet is subtracted. 212 * (As a result, in many tailorings no set needs to be stored.) 213 * 214 * uint16_t fastLatinTable[]; 215 * Optional optimization for Latin text. 216 * See the CollationFastLatin class. 217 * 218 * uint16_t scripts[]; -- empty in all tailorings 219 * Format version 5: 220 * uint16_t numScripts; 221 * uint16_t scriptsIndex[numScripts+16]; 222 * uint16_t scriptStarts[]; 223 * See CollationData::numScripts etc. 224 * 225 * Format version 4: 226 * Table of the reordering groups with their first and last lead bytes, 227 * and their script and reordering codes. 228 * See CollationData::scripts. 229 * 230 * UBool compressibleBytes[]; -- empty in all tailorings 231 * Flag for getSortKey(), indicating primary weight lead bytes that are compressible. 232 * 233 * ----------------- 234 * Changes for formatVersion 5 (ICU 55) 235 * 236 * Reordering moves single scripts, not groups of scripts. 237 * Reorder ranges are optionally appended to the reorderCodes, 238 * and a 0 entry in the reorderTable indicates a split lead byte. 239 * The scripts data has a new format. 240 * 241 * The rootElements may contain secondary and tertiary weights below common=05. 242 * (Used for small Hiragana letters.) 243 * Where is occurs, there is also an explicit unit with common secondary & tertiary weights. 244 * There are no other data structure changes, but builder code needs to be able to handle such data. 245 * 246 * The collation element for the merge separator code point U+FFFE 247 * does not necessarily have special, unique secondary/tertiary weights any more. 248 */ 249 250 U_NAMESPACE_END 251 252 #endif // !UCONFIG_NO_COLLATION 253 #endif // __COLLATIONDATAREADER_H__ 254