1 /*
2 *******************************************************************************
3 * Copyright (C) 2013-2015, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 *******************************************************************************
6 * collationdatareader.cpp
7 *
8 * created on: 2013feb07
9 * created by: Markus W. Scherer
10 */
11
12 #include "unicode/utypes.h"
13
14 #if !UCONFIG_NO_COLLATION
15
16 #include "unicode/ucol.h"
17 #include "unicode/udata.h"
18 #include "unicode/uscript.h"
19 #include "cmemory.h"
20 #include "collation.h"
21 #include "collationdata.h"
22 #include "collationdatareader.h"
23 #include "collationfastlatin.h"
24 #include "collationkeys.h"
25 #include "collationrootelements.h"
26 #include "collationsettings.h"
27 #include "collationtailoring.h"
28 #include "collunsafe.h"
29 #include "normalizer2impl.h"
30 #include "uassert.h"
31 #include "ucmndata.h"
32 #include "utrie2.h"
33
34 U_NAMESPACE_BEGIN
35
36 namespace {
37
getIndex(const int32_t * indexes,int32_t length,int32_t i)38 int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
39 return (i < length) ? indexes[i] : -1;
40 }
41
42 } // namespace
43
44 void
read(const CollationTailoring * base,const uint8_t * inBytes,int32_t inLength,CollationTailoring & tailoring,UErrorCode & errorCode)45 CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
46 CollationTailoring &tailoring, UErrorCode &errorCode) {
47 if(U_FAILURE(errorCode)) { return; }
48 if(base != NULL) {
49 if(inBytes == NULL || (0 <= inLength && inLength < 24)) {
50 errorCode = U_ILLEGAL_ARGUMENT_ERROR;
51 return;
52 }
53 const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes);
54 if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 &&
55 isAcceptable(tailoring.version, NULL, NULL, &header->info))) {
56 errorCode = U_INVALID_FORMAT_ERROR;
57 return;
58 }
59 if(base->getUCAVersion() != tailoring.getUCAVersion()) {
60 errorCode = U_COLLATOR_VERSION_MISMATCH;
61 return;
62 }
63 int32_t headerLength = header->dataHeader.headerSize;
64 inBytes += headerLength;
65 if(inLength >= 0) {
66 inLength -= headerLength;
67 }
68 }
69
70 if(inBytes == NULL || (0 <= inLength && inLength < 8)) {
71 errorCode = U_ILLEGAL_ARGUMENT_ERROR;
72 return;
73 }
74 const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes);
75 int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH];
76 if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) {
77 errorCode = U_INVALID_FORMAT_ERROR; // Not enough indexes.
78 return;
79 }
80
81 // Assume that the tailoring data is in initial state,
82 // with NULL pointers and 0 lengths.
83
84 // Set pointers to non-empty data parts.
85 // Do this in order of their byte offsets. (Should help porting to Java.)
86
87 int32_t index; // one of the indexes[] slots
88 int32_t offset; // byte offset for the index part
89 int32_t length; // number of bytes in the index part
90
91 if(indexesLength > IX_TOTAL_SIZE) {
92 length = inIndexes[IX_TOTAL_SIZE];
93 } else if(indexesLength > IX_REORDER_CODES_OFFSET) {
94 length = inIndexes[indexesLength - 1];
95 } else {
96 length = 0; // only indexes, and inLength was already checked for them
97 }
98 if(0 <= inLength && inLength < length) {
99 errorCode = U_INVALID_FORMAT_ERROR;
100 return;
101 }
102
103 const CollationData *baseData = base == NULL ? NULL : base->data;
104 const int32_t *reorderCodes = NULL;
105 int32_t reorderCodesLength = 0;
106 const uint32_t *reorderRanges = NULL;
107 int32_t reorderRangesLength = 0;
108 index = IX_REORDER_CODES_OFFSET;
109 offset = getIndex(inIndexes, indexesLength, index);
110 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
111 if(length >= 4) {
112 if(baseData == NULL) {
113 // We assume for collation settings that
114 // the base data does not have a reordering.
115 errorCode = U_INVALID_FORMAT_ERROR;
116 return;
117 }
118 reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
119 reorderCodesLength = length / 4;
120
121 // The reorderRanges (if any) are the trailing reorderCodes entries.
122 // Split the array at the boundary.
123 // Script or reorder codes do not exceed 16-bit values.
124 // Range limits are stored in the upper 16 bits, and are never 0.
125 while(reorderRangesLength < reorderCodesLength &&
126 (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
127 ++reorderRangesLength;
128 }
129 U_ASSERT(reorderRangesLength < reorderCodesLength);
130 if(reorderRangesLength != 0) {
131 reorderCodesLength -= reorderRangesLength;
132 reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
133 }
134 }
135
136 // There should be a reorder table only if there are reorder codes.
137 // However, when there are reorder codes the reorder table may be omitted to reduce
138 // the data size.
139 const uint8_t *reorderTable = NULL;
140 index = IX_REORDER_TABLE_OFFSET;
141 offset = getIndex(inIndexes, indexesLength, index);
142 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
143 if(length >= 256) {
144 if(reorderCodesLength == 0) {
145 errorCode = U_INVALID_FORMAT_ERROR; // Reordering table without reordering codes.
146 return;
147 }
148 reorderTable = inBytes + offset;
149 } else {
150 // If we have reorder codes, then build the reorderTable at the end,
151 // when the CollationData is otherwise complete.
152 }
153
154 if(baseData != NULL && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) {
155 errorCode = U_INVALID_FORMAT_ERROR;
156 return;
157 }
158 CollationData *data = NULL; // Remains NULL if there are no mappings.
159
160 index = IX_TRIE_OFFSET;
161 offset = getIndex(inIndexes, indexesLength, index);
162 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
163 if(length >= 8) {
164 if(!tailoring.ensureOwnedData(errorCode)) { return; }
165 data = tailoring.ownedData;
166 data->base = baseData;
167 data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000;
168 data->trie = tailoring.trie = utrie2_openFromSerialized(
169 UTRIE2_32_VALUE_BITS, inBytes + offset, length, NULL,
170 &errorCode);
171 if(U_FAILURE(errorCode)) { return; }
172 } else if(baseData != NULL) {
173 // Use the base data. Only the settings are tailored.
174 tailoring.data = baseData;
175 } else {
176 errorCode = U_INVALID_FORMAT_ERROR; // No mappings.
177 return;
178 }
179
180 index = IX_CES_OFFSET;
181 offset = getIndex(inIndexes, indexesLength, index);
182 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
183 if(length >= 8) {
184 if(data == NULL) {
185 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ces without tailored trie.
186 return;
187 }
188 data->ces = reinterpret_cast<const int64_t *>(inBytes + offset);
189 data->cesLength = length / 8;
190 }
191
192 index = IX_CE32S_OFFSET;
193 offset = getIndex(inIndexes, indexesLength, index);
194 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
195 if(length >= 4) {
196 if(data == NULL) {
197 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ce32s without tailored trie.
198 return;
199 }
200 data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset);
201 data->ce32sLength = length / 4;
202 }
203
204 int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START);
205 if(jamoCE32sStart >= 0) {
206 if(data == NULL || data->ce32s == NULL) {
207 errorCode = U_INVALID_FORMAT_ERROR; // Index into non-existent ce32s[].
208 return;
209 }
210 data->jamoCE32s = data->ce32s + jamoCE32sStart;
211 } else if(data == NULL) {
212 // Nothing to do.
213 } else if(baseData != NULL) {
214 data->jamoCE32s = baseData->jamoCE32s;
215 } else {
216 errorCode = U_INVALID_FORMAT_ERROR; // No Jamo CE32s for Hangul processing.
217 return;
218 }
219
220 index = IX_ROOT_ELEMENTS_OFFSET;
221 offset = getIndex(inIndexes, indexesLength, index);
222 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
223 if(length >= 4) {
224 length /= 4;
225 if(data == NULL || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) {
226 errorCode = U_INVALID_FORMAT_ERROR;
227 return;
228 }
229 data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset);
230 data->rootElementsLength = length;
231 uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE];
232 if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) {
233 errorCode = U_INVALID_FORMAT_ERROR;
234 return;
235 }
236 uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES];
237 if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) {
238 // [fixed last secondary common byte] is too low,
239 // and secondary weights would collide with compressed common secondaries.
240 errorCode = U_INVALID_FORMAT_ERROR;
241 return;
242 }
243 }
244
245 index = IX_CONTEXTS_OFFSET;
246 offset = getIndex(inIndexes, indexesLength, index);
247 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
248 if(length >= 2) {
249 if(data == NULL) {
250 errorCode = U_INVALID_FORMAT_ERROR; // Tailored contexts without tailored trie.
251 return;
252 }
253 data->contexts = reinterpret_cast<const UChar *>(inBytes + offset);
254 data->contextsLength = length / 2;
255 }
256
257 index = IX_UNSAFE_BWD_OFFSET;
258 offset = getIndex(inIndexes, indexesLength, index);
259 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
260 if(length >= 2) {
261 if(data == NULL) {
262 errorCode = U_INVALID_FORMAT_ERROR;
263 return;
264 }
265 if(baseData == NULL) {
266 #if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
267 tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
268 if(tailoring.unsafeBackwardSet == NULL) {
269 errorCode = U_MEMORY_ALLOCATION_ERROR;
270 return;
271 } else if (U_FAILURE(errorCode)) {
272 return;
273 }
274 #else
275 // Create the unsafe-backward set for the root collator.
276 // Include all non-zero combining marks and trail surrogates.
277 // We do this at load time, rather than at build time,
278 // to simplify Unicode version bootstrapping:
279 // The root data builder only needs the new FractionalUCA.txt data,
280 // but it need not be built with a version of ICU already updated to
281 // the corresponding new Unicode Character Database.
282 //
283 // The following is an optimized version of
284 // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
285 // It is faster and requires fewer code dependencies.
286 tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates
287 if(tailoring.unsafeBackwardSet == NULL) {
288 errorCode = U_MEMORY_ALLOCATION_ERROR;
289 return;
290 }
291 data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
292 #endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION
293 } else {
294 // Clone the root collator's set contents.
295 tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
296 baseData->unsafeBackwardSet->cloneAsThawed());
297 if(tailoring.unsafeBackwardSet == NULL) {
298 errorCode = U_MEMORY_ALLOCATION_ERROR;
299 return;
300 }
301 }
302 // Add the ranges from the data file to the unsafe-backward set.
303 USerializedSet sset;
304 const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset);
305 if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) {
306 errorCode = U_INVALID_FORMAT_ERROR;
307 return;
308 }
309 int32_t count = uset_getSerializedRangeCount(&sset);
310 for(int32_t i = 0; i < count; ++i) {
311 UChar32 start, end;
312 uset_getSerializedRange(&sset, i, &start, &end);
313 tailoring.unsafeBackwardSet->add(start, end);
314 }
315 // Mark each lead surrogate as "unsafe"
316 // if any of its 1024 associated supplementary code points is "unsafe".
317 UChar32 c = 0x10000;
318 for(UChar lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) {
319 if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) {
320 tailoring.unsafeBackwardSet->add(lead);
321 }
322 }
323 tailoring.unsafeBackwardSet->freeze();
324 data->unsafeBackwardSet = tailoring.unsafeBackwardSet;
325 } else if(data == NULL) {
326 // Nothing to do.
327 } else if(baseData != NULL) {
328 // No tailoring-specific data: Alias the root collator's set.
329 data->unsafeBackwardSet = baseData->unsafeBackwardSet;
330 } else {
331 errorCode = U_INVALID_FORMAT_ERROR; // No unsafeBackwardSet.
332 return;
333 }
334
335 // If the fast Latin format version is different,
336 // or the version is set to 0 for "no fast Latin table",
337 // then just always use the normal string comparison path.
338 if(data != NULL) {
339 data->fastLatinTable = NULL;
340 data->fastLatinTableLength = 0;
341 if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) {
342 index = IX_FAST_LATIN_TABLE_OFFSET;
343 offset = getIndex(inIndexes, indexesLength, index);
344 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
345 if(length >= 2) {
346 data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset);
347 data->fastLatinTableLength = length / 2;
348 if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) {
349 errorCode = U_INVALID_FORMAT_ERROR; // header vs. table version mismatch
350 return;
351 }
352 } else if(baseData != NULL) {
353 data->fastLatinTable = baseData->fastLatinTable;
354 data->fastLatinTableLength = baseData->fastLatinTableLength;
355 }
356 }
357 }
358
359 index = IX_SCRIPTS_OFFSET;
360 offset = getIndex(inIndexes, indexesLength, index);
361 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
362 if(length >= 2) {
363 if(data == NULL) {
364 errorCode = U_INVALID_FORMAT_ERROR;
365 return;
366 }
367 const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
368 int32_t scriptsLength = length / 2;
369 data->numScripts = scripts[0];
370 // There must be enough entries for both arrays, including more than two range starts.
371 data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
372 if(data->scriptStartsLength <= 2 ||
373 CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
374 errorCode = U_INVALID_FORMAT_ERROR;
375 return;
376 }
377 data->scriptsIndex = scripts + 1;
378 data->scriptStarts = scripts + 1 + data->numScripts + 16;
379 if(!(data->scriptStarts[0] == 0 &&
380 data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
381 data->scriptStarts[data->scriptStartsLength - 1] ==
382 (Collation::TRAIL_WEIGHT_BYTE << 8))) {
383 errorCode = U_INVALID_FORMAT_ERROR;
384 return;
385 }
386 } else if(data == NULL) {
387 // Nothing to do.
388 } else if(baseData != NULL) {
389 data->numScripts = baseData->numScripts;
390 data->scriptsIndex = baseData->scriptsIndex;
391 data->scriptStarts = baseData->scriptStarts;
392 data->scriptStartsLength = baseData->scriptStartsLength;
393 }
394
395 index = IX_COMPRESSIBLE_BYTES_OFFSET;
396 offset = getIndex(inIndexes, indexesLength, index);
397 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
398 if(length >= 256) {
399 if(data == NULL) {
400 errorCode = U_INVALID_FORMAT_ERROR;
401 return;
402 }
403 data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset);
404 } else if(data == NULL) {
405 // Nothing to do.
406 } else if(baseData != NULL) {
407 data->compressibleBytes = baseData->compressibleBytes;
408 } else {
409 errorCode = U_INVALID_FORMAT_ERROR; // No compressibleBytes[].
410 return;
411 }
412
413 const CollationSettings &ts = *tailoring.settings;
414 int32_t options = inIndexes[IX_OPTIONS] & 0xffff;
415 uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT];
416 int32_t fastLatinOptions = CollationFastLatin::getOptions(
417 tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries));
418 if(options == ts.options && ts.variableTop != 0 &&
419 reorderCodesLength == ts.reorderCodesLength &&
420 uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0 &&
421 fastLatinOptions == ts.fastLatinOptions &&
422 (fastLatinOptions < 0 ||
423 uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries,
424 sizeof(fastLatinPrimaries)) == 0)) {
425 return;
426 }
427
428 CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings);
429 if(settings == NULL) {
430 errorCode = U_MEMORY_ALLOCATION_ERROR;
431 return;
432 }
433 settings->options = options;
434 // Set variableTop from options and scripts data.
435 settings->variableTop = tailoring.data->getLastPrimaryForGroup(
436 UCOL_REORDER_CODE_FIRST + settings->getMaxVariable());
437 if(settings->variableTop == 0) {
438 errorCode = U_INVALID_FORMAT_ERROR;
439 return;
440 }
441
442 if(reorderCodesLength != 0) {
443 settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
444 reorderRanges, reorderRangesLength,
445 reorderTable, errorCode);
446 }
447
448 settings->fastLatinOptions = CollationFastLatin::getOptions(
449 tailoring.data, *settings,
450 settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
451 }
452
453 UBool U_CALLCONV
isAcceptable(void * context,const char *,const char *,const UDataInfo * pInfo)454 CollationDataReader::isAcceptable(void *context,
455 const char * /* type */, const char * /*name*/,
456 const UDataInfo *pInfo) {
457 if(
458 pInfo->size >= 20 &&
459 pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
460 pInfo->charsetFamily == U_CHARSET_FAMILY &&
461 pInfo->dataFormat[0] == 0x55 && // dataFormat="UCol"
462 pInfo->dataFormat[1] == 0x43 &&
463 pInfo->dataFormat[2] == 0x6f &&
464 pInfo->dataFormat[3] == 0x6c &&
465 pInfo->formatVersion[0] == 5
466 ) {
467 UVersionInfo *version = static_cast<UVersionInfo *>(context);
468 if(version != NULL) {
469 uprv_memcpy(version, pInfo->dataVersion, 4);
470 }
471 return TRUE;
472 } else {
473 return FALSE;
474 }
475 }
476
477 U_NAMESPACE_END
478
479 #endif // !UCONFIG_NO_COLLATION
480