1 /* 2 ******************************************************************************* 3 * Copyright (c) 1996-2010, International Business Machines Corporation 4 * and others. All Rights Reserved. 5 ******************************************************************************* 6 * File unorm.h 7 * 8 * Created by: Vladimir Weinstein 12052000 9 * 10 * Modification history : 11 * 12 * Date Name Description 13 * 02/01/01 synwee Added normalization quickcheck enum and method. 14 */ 15 #ifndef UNORM_H 16 #define UNORM_H 17 18 #include "unicode/utypes.h" 19 20 #if !UCONFIG_NO_NORMALIZATION 21 22 #include "unicode/uiter.h" 23 #include "unicode/unorm2.h" 24 25 /** 26 * \file 27 * \brief C API: Unicode Normalization 28 * 29 * <h2>Unicode normalization API</h2> 30 * 31 * Note: This API has been replaced by the unorm2.h API and is only available 32 * for backward compatibility. The functions here simply delegate to the 33 * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize(). 34 * There is one exception: The new API does not provide a replacement for unorm_compare(). 35 * 36 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or 37 * decomposed form, allowing for easier sorting and searching of text. 38 * <code>unorm_normalize</code> supports the standard normalization forms described in 39 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode"> 40 * Unicode Standard Annex #15: Unicode Normalization Forms</a>. 41 * 42 * Characters with accents or other adornments can be encoded in 43 * several different ways in Unicode. For example, take the character A-acute. 44 * In Unicode, this can be encoded as a single character (the 45 * "composed" form): 46 * 47 * \code 48 * 00C1 LATIN CAPITAL LETTER A WITH ACUTE 49 * \endcode 50 * 51 * or as two separate characters (the "decomposed" form): 52 * 53 * \code 54 * 0041 LATIN CAPITAL LETTER A 55 * 0301 COMBINING ACUTE ACCENT 56 * \endcode 57 * 58 * To a user of your program, however, both of these sequences should be 59 * treated as the same "user-level" character "A with acute accent". When you are searching or 60 * comparing text, you must ensure that these two sequences are treated 61 * equivalently. In addition, you must handle characters with more than one 62 * accent. Sometimes the order of a character's combining accents is 63 * significant, while in other cases accent sequences in different orders are 64 * really equivalent. 65 * 66 * Similarly, the string "ffi" can be encoded as three separate letters: 67 * 68 * \code 69 * 0066 LATIN SMALL LETTER F 70 * 0066 LATIN SMALL LETTER F 71 * 0069 LATIN SMALL LETTER I 72 * \endcode 73 * 74 * or as the single character 75 * 76 * \code 77 * FB03 LATIN SMALL LIGATURE FFI 78 * \endcode 79 * 80 * The ffi ligature is not a distinct semantic character, and strictly speaking 81 * it shouldn't be in Unicode at all, but it was included for compatibility 82 * with existing character sets that already provided it. The Unicode standard 83 * identifies such characters by giving them "compatibility" decompositions 84 * into the corresponding semantic characters. When sorting and searching, you 85 * will often want to use these mappings. 86 * 87 * <code>unorm_normalize</code> helps solve these problems by transforming text into the 88 * canonical composed and decomposed forms as shown in the first example above. 89 * In addition, you can have it perform compatibility decompositions so that 90 * you can treat compatibility characters the same as their equivalents. 91 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical 92 * order, so that you do not have to worry about accent rearrangement on your 93 * own. 94 * 95 * Form FCD, "Fast C or D", is also designed for collation. 96 * It allows to work on strings that are not necessarily normalized 97 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed 98 * characters and their decomposed equivalents the same. 99 * 100 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings 101 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical 102 * themselves. 103 * 104 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character, 105 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long 106 * as their decompositions do not need canonical reordering. 107 * 108 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts - 109 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will 110 * return UNORM_YES for most strings in practice. 111 * 112 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD. 113 * 114 * For more details on FCD see the collation design document: 115 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm 116 * 117 * ICU collation performs either NFD or FCD normalization automatically if normalization 118 * is turned on for the collator object. 119 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons, 120 * transliteration/transcription, unique representations, etc. 121 * 122 * The W3C generally recommends to exchange texts in NFC. 123 * Note also that most legacy character encodings use only precomposed forms and often do not 124 * encode any combining marks by themselves. For conversion to such character encodings the 125 * Unicode text needs to be normalized to NFC. 126 * For more usage examples, see the Unicode Standard Annex. 127 */ 128 129 /** 130 * Constants for normalization modes. 131 * @stable ICU 2.0 132 */ 133 typedef enum { 134 /** No decomposition/composition. @stable ICU 2.0 */ 135 UNORM_NONE = 1, 136 /** Canonical decomposition. @stable ICU 2.0 */ 137 UNORM_NFD = 2, 138 /** Compatibility decomposition. @stable ICU 2.0 */ 139 UNORM_NFKD = 3, 140 /** Canonical decomposition followed by canonical composition. @stable ICU 2.0 */ 141 UNORM_NFC = 4, 142 /** Default normalization. @stable ICU 2.0 */ 143 UNORM_DEFAULT = UNORM_NFC, 144 /** Compatibility decomposition followed by canonical composition. @stable ICU 2.0 */ 145 UNORM_NFKC =5, 146 /** "Fast C or D" form. @stable ICU 2.0 */ 147 UNORM_FCD = 6, 148 149 /** One more than the highest normalization mode constant. @stable ICU 2.0 */ 150 UNORM_MODE_COUNT 151 } UNormalizationMode; 152 153 /** 154 * Constants for options flags for normalization. 155 * Use 0 for default options, 156 * including normalization according to the Unicode version 157 * that is currently supported by ICU (see u_getUnicodeVersion). 158 * @stable ICU 2.6 159 */ 160 enum { 161 /** 162 * Options bit set value to select Unicode 3.2 normalization 163 * (except NormalizationCorrections). 164 * At most one Unicode version can be selected at a time. 165 * @stable ICU 2.6 166 */ 167 UNORM_UNICODE_3_2=0x20 168 }; 169 170 /** 171 * Lowest-order bit number of unorm_compare() options bits corresponding to 172 * normalization options bits. 173 * 174 * The options parameter for unorm_compare() uses most bits for 175 * itself and for various comparison and folding flags. 176 * The most significant bits, however, are shifted down and passed on 177 * to the normalization implementation. 178 * (That is, from unorm_compare(..., options, ...), 179 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the 180 * internal normalization functions.) 181 * 182 * @see unorm_compare 183 * @stable ICU 2.6 184 */ 185 #define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20 186 187 /** 188 * Normalize a string. 189 * The string will be normalized according the specified normalization mode 190 * and options. 191 * The source and result buffers must not be the same, nor overlap. 192 * 193 * @param source The string to normalize. 194 * @param sourceLength The length of source, or -1 if NUL-terminated. 195 * @param mode The normalization mode; one of UNORM_NONE, 196 * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT. 197 * @param options The normalization options, ORed together (0 for no options). 198 * @param result A pointer to a buffer to receive the result string. 199 * The result string is NUL-terminated if possible. 200 * @param resultLength The maximum size of result. 201 * @param status A pointer to a UErrorCode to receive any errors. 202 * @return The total buffer size needed; if greater than resultLength, 203 * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR. 204 * @stable ICU 2.0 205 */ 206 U_STABLE int32_t U_EXPORT2 207 unorm_normalize(const UChar *source, int32_t sourceLength, 208 UNormalizationMode mode, int32_t options, 209 UChar *result, int32_t resultLength, 210 UErrorCode *status); 211 212 /** 213 * Performing quick check on a string, to quickly determine if the string is 214 * in a particular normalization format. 215 * Three types of result can be returned UNORM_YES, UNORM_NO or 216 * UNORM_MAYBE. Result UNORM_YES indicates that the argument 217 * string is in the desired normalized format, UNORM_NO determines that 218 * argument string is not in the desired normalized format. A 219 * UNORM_MAYBE result indicates that a more thorough check is required, 220 * the user may have to put the string in its normalized form and compare the 221 * results. 222 * 223 * @param source string for determining if it is in a normalized format 224 * @param sourcelength length of source to test, or -1 if NUL-terminated 225 * @param mode which normalization form to test for 226 * @param status a pointer to a UErrorCode to receive any errors 227 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE 228 * 229 * @see unorm_isNormalized 230 * @stable ICU 2.0 231 */ 232 U_STABLE UNormalizationCheckResult U_EXPORT2 233 unorm_quickCheck(const UChar *source, int32_t sourcelength, 234 UNormalizationMode mode, 235 UErrorCode *status); 236 237 /** 238 * Performing quick check on a string; same as unorm_quickCheck but 239 * takes an extra options parameter like most normalization functions. 240 * 241 * @param src String that is to be tested if it is in a normalization format. 242 * @param srcLength Length of source to test, or -1 if NUL-terminated. 243 * @param mode Which normalization form to test for. 244 * @param options The normalization options, ORed together (0 for no options). 245 * @param pErrorCode ICU error code in/out parameter. 246 * Must fulfill U_SUCCESS before the function call. 247 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE 248 * 249 * @see unorm_quickCheck 250 * @see unorm_isNormalized 251 * @stable ICU 2.6 252 */ 253 U_STABLE UNormalizationCheckResult U_EXPORT2 254 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength, 255 UNormalizationMode mode, int32_t options, 256 UErrorCode *pErrorCode); 257 258 /** 259 * Test if a string is in a given normalization form. 260 * This is semantically equivalent to source.equals(normalize(source, mode)) . 261 * 262 * Unlike unorm_quickCheck(), this function returns a definitive result, 263 * never a "maybe". 264 * For NFD, NFKD, and FCD, both functions work exactly the same. 265 * For NFC and NFKC where quickCheck may return "maybe", this function will 266 * perform further tests to arrive at a TRUE/FALSE result. 267 * 268 * @param src String that is to be tested if it is in a normalization format. 269 * @param srcLength Length of source to test, or -1 if NUL-terminated. 270 * @param mode Which normalization form to test for. 271 * @param pErrorCode ICU error code in/out parameter. 272 * Must fulfill U_SUCCESS before the function call. 273 * @return Boolean value indicating whether the source string is in the 274 * "mode" normalization form. 275 * 276 * @see unorm_quickCheck 277 * @stable ICU 2.2 278 */ 279 U_STABLE UBool U_EXPORT2 280 unorm_isNormalized(const UChar *src, int32_t srcLength, 281 UNormalizationMode mode, 282 UErrorCode *pErrorCode); 283 284 /** 285 * Test if a string is in a given normalization form; same as unorm_isNormalized but 286 * takes an extra options parameter like most normalization functions. 287 * 288 * @param src String that is to be tested if it is in a normalization format. 289 * @param srcLength Length of source to test, or -1 if NUL-terminated. 290 * @param mode Which normalization form to test for. 291 * @param options The normalization options, ORed together (0 for no options). 292 * @param pErrorCode ICU error code in/out parameter. 293 * Must fulfill U_SUCCESS before the function call. 294 * @return Boolean value indicating whether the source string is in the 295 * "mode/options" normalization form. 296 * 297 * @see unorm_quickCheck 298 * @see unorm_isNormalized 299 * @stable ICU 2.6 300 */ 301 U_STABLE UBool U_EXPORT2 302 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength, 303 UNormalizationMode mode, int32_t options, 304 UErrorCode *pErrorCode); 305 306 /** 307 * Iterative normalization forward. 308 * This function (together with unorm_previous) is somewhat 309 * similar to the C++ Normalizer class (see its non-static functions). 310 * 311 * Iterative normalization is useful when only a small portion of a longer 312 * string/text needs to be processed. 313 * 314 * For example, the likelihood may be high that processing the first 10% of some 315 * text will be sufficient to find certain data. 316 * Another example: When one wants to concatenate two normalized strings and get a 317 * normalized result, it is much more efficient to normalize just a small part of 318 * the result around the concatenation place instead of re-normalizing everything. 319 * 320 * The input text is an instance of the C character iteration API UCharIterator. 321 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any 322 * other kind of text object. 323 * 324 * If a buffer overflow occurs, then the caller needs to reset the iterator to the 325 * old index and call the function again with a larger buffer - if the caller cares 326 * for the actual output. 327 * Regardless of the output buffer, the iterator will always be moved to the next 328 * normalization boundary. 329 * 330 * This function (like unorm_previous) serves two purposes: 331 * 332 * 1) To find the next boundary so that the normalization of the part of the text 333 * from the current position to that boundary does not affect and is not affected 334 * by the part of the text beyond that boundary. 335 * 336 * 2) To normalize the text up to the boundary. 337 * 338 * The second step is optional, per the doNormalize parameter. 339 * It is omitted for operations like string concatenation, where the two adjacent 340 * string ends need to be normalized together. 341 * In such a case, the output buffer will just contain a copy of the text up to the 342 * boundary. 343 * 344 * pNeededToNormalize is an output-only parameter. Its output value is only defined 345 * if normalization was requested (doNormalize) and successful (especially, no 346 * buffer overflow). 347 * It is useful for operations like a normalizing transliterator, where one would 348 * not want to replace a piece of text if it is not modified. 349 * 350 * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE 351 * if the normalization was necessary. 352 * 353 * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE. 354 * 355 * If the buffer overflows, then *pNeededToNormalize will be undefined; 356 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result 357 * will be undefined. 358 * 359 * @param src The input text in the form of a C character iterator. 360 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 361 * @param destCapacity The number of UChars that fit into dest. 362 * @param mode The normalization mode. 363 * @param options The normalization options, ORed together (0 for no options). 364 * @param doNormalize Indicates if the source text up to the next boundary 365 * is to be normalized (TRUE) or just copied (FALSE). 366 * @param pNeededToNormalize Output flag indicating if the normalization resulted in 367 * different text from the input. 368 * Not defined if an error occurs including buffer overflow. 369 * Always FALSE if !doNormalize. 370 * @param pErrorCode ICU error code in/out parameter. 371 * Must fulfill U_SUCCESS before the function call. 372 * @return Length of output (number of UChars) when successful or buffer overflow. 373 * 374 * @see unorm_previous 375 * @see unorm_normalize 376 * 377 * @stable ICU 2.1 378 */ 379 U_STABLE int32_t U_EXPORT2 380 unorm_next(UCharIterator *src, 381 UChar *dest, int32_t destCapacity, 382 UNormalizationMode mode, int32_t options, 383 UBool doNormalize, UBool *pNeededToNormalize, 384 UErrorCode *pErrorCode); 385 386 /** 387 * Iterative normalization backward. 388 * This function (together with unorm_next) is somewhat 389 * similar to the C++ Normalizer class (see its non-static functions). 390 * For all details see unorm_next. 391 * 392 * @param src The input text in the form of a C character iterator. 393 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 394 * @param destCapacity The number of UChars that fit into dest. 395 * @param mode The normalization mode. 396 * @param options The normalization options, ORed together (0 for no options). 397 * @param doNormalize Indicates if the source text up to the next boundary 398 * is to be normalized (TRUE) or just copied (FALSE). 399 * @param pNeededToNormalize Output flag indicating if the normalization resulted in 400 * different text from the input. 401 * Not defined if an error occurs including buffer overflow. 402 * Always FALSE if !doNormalize. 403 * @param pErrorCode ICU error code in/out parameter. 404 * Must fulfill U_SUCCESS before the function call. 405 * @return Length of output (number of UChars) when successful or buffer overflow. 406 * 407 * @see unorm_next 408 * @see unorm_normalize 409 * 410 * @stable ICU 2.1 411 */ 412 U_STABLE int32_t U_EXPORT2 413 unorm_previous(UCharIterator *src, 414 UChar *dest, int32_t destCapacity, 415 UNormalizationMode mode, int32_t options, 416 UBool doNormalize, UBool *pNeededToNormalize, 417 UErrorCode *pErrorCode); 418 419 /** 420 * Concatenate normalized strings, making sure that the result is normalized as well. 421 * 422 * If both the left and the right strings are in 423 * the normalization form according to "mode/options", 424 * then the result will be 425 * 426 * \code 427 * dest=normalize(left+right, mode, options) 428 * \endcode 429 * 430 * With the input strings already being normalized, 431 * this function will use unorm_next() and unorm_previous() 432 * to find the adjacent end pieces of the input strings. 433 * Only the concatenation of these end pieces will be normalized and 434 * then concatenated with the remaining parts of the input strings. 435 * 436 * It is allowed to have dest==left to avoid copying the entire left string. 437 * 438 * @param left Left source string, may be same as dest. 439 * @param leftLength Length of left source string, or -1 if NUL-terminated. 440 * @param right Right source string. Must not be the same as dest, nor overlap. 441 * @param rightLength Length of right source string, or -1 if NUL-terminated. 442 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 443 * @param destCapacity The number of UChars that fit into dest. 444 * @param mode The normalization mode. 445 * @param options The normalization options, ORed together (0 for no options). 446 * @param pErrorCode ICU error code in/out parameter. 447 * Must fulfill U_SUCCESS before the function call. 448 * @return Length of output (number of UChars) when successful or buffer overflow. 449 * 450 * @see unorm_normalize 451 * @see unorm_next 452 * @see unorm_previous 453 * 454 * @stable ICU 2.1 455 */ 456 U_STABLE int32_t U_EXPORT2 457 unorm_concatenate(const UChar *left, int32_t leftLength, 458 const UChar *right, int32_t rightLength, 459 UChar *dest, int32_t destCapacity, 460 UNormalizationMode mode, int32_t options, 461 UErrorCode *pErrorCode); 462 463 /** 464 * Option bit for unorm_compare: 465 * Both input strings are assumed to fulfill FCD conditions. 466 * @stable ICU 2.2 467 */ 468 #define UNORM_INPUT_IS_FCD 0x20000 469 470 /** 471 * Option bit for unorm_compare: 472 * Perform case-insensitive comparison. 473 * @stable ICU 2.2 474 */ 475 #define U_COMPARE_IGNORE_CASE 0x10000 476 477 #ifndef U_COMPARE_CODE_POINT_ORDER 478 /* see also unistr.h and ustring.h */ 479 /** 480 * Option bit for u_strCaseCompare, u_strcasecmp, unorm_compare, etc: 481 * Compare strings in code point order instead of code unit order. 482 * @stable ICU 2.2 483 */ 484 #define U_COMPARE_CODE_POINT_ORDER 0x8000 485 #endif 486 487 /** 488 * Compare two strings for canonical equivalence. 489 * Further options include case-insensitive comparison and 490 * code point order (as opposed to code unit order). 491 * 492 * Canonical equivalence between two strings is defined as their normalized 493 * forms (NFD or NFC) being identical. 494 * This function compares strings incrementally instead of normalizing 495 * (and optionally case-folding) both strings entirely, 496 * improving performance significantly. 497 * 498 * Bulk normalization is only necessary if the strings do not fulfill the FCD 499 * conditions. Only in this case, and only if the strings are relatively long, 500 * is memory allocated temporarily. 501 * For FCD strings and short non-FCD strings there is no memory allocation. 502 * 503 * Semantically, this is equivalent to 504 * strcmp[CodePointOrder](NFD(foldCase(NFD(s1))), NFD(foldCase(NFD(s2)))) 505 * where code point order and foldCase are all optional. 506 * 507 * UAX 21 2.5 Caseless Matching specifies that for a canonical caseless match 508 * the case folding must be performed first, then the normalization. 509 * 510 * @param s1 First source string. 511 * @param length1 Length of first source string, or -1 if NUL-terminated. 512 * 513 * @param s2 Second source string. 514 * @param length2 Length of second source string, or -1 if NUL-terminated. 515 * 516 * @param options A bit set of options: 517 * - U_FOLD_CASE_DEFAULT or 0 is used for default options: 518 * Case-sensitive comparison in code unit order, and the input strings 519 * are quick-checked for FCD. 520 * 521 * - UNORM_INPUT_IS_FCD 522 * Set if the caller knows that both s1 and s2 fulfill the FCD conditions. 523 * If not set, the function will quickCheck for FCD 524 * and normalize if necessary. 525 * 526 * - U_COMPARE_CODE_POINT_ORDER 527 * Set to choose code point order instead of code unit order 528 * (see u_strCompare for details). 529 * 530 * - U_COMPARE_IGNORE_CASE 531 * Set to compare strings case-insensitively using case folding, 532 * instead of case-sensitively. 533 * If set, then the following case folding options are used. 534 * 535 * - Options as used with case-insensitive comparisons, currently: 536 * 537 * - U_FOLD_CASE_EXCLUDE_SPECIAL_I 538 * (see u_strCaseCompare for details) 539 * 540 * - regular normalization options shifted left by UNORM_COMPARE_NORM_OPTIONS_SHIFT 541 * 542 * @param pErrorCode ICU error code in/out parameter. 543 * Must fulfill U_SUCCESS before the function call. 544 * @return <0 or 0 or >0 as usual for string comparisons 545 * 546 * @see unorm_normalize 547 * @see UNORM_FCD 548 * @see u_strCompare 549 * @see u_strCaseCompare 550 * 551 * @stable ICU 2.2 552 */ 553 U_STABLE int32_t U_EXPORT2 554 unorm_compare(const UChar *s1, int32_t length1, 555 const UChar *s2, int32_t length2, 556 uint32_t options, 557 UErrorCode *pErrorCode); 558 559 #endif /* #if !UCONFIG_NO_NORMALIZATION */ 560 561 #endif 562