1 /* 2 ******************************************************************************* 3 * Copyright (c) 1996-2015, International Business Machines Corporation 4 * and others. All Rights Reserved. 5 ******************************************************************************* 6 * File unorm.h 7 * 8 * Created by: Vladimir Weinstein 12052000 9 * 10 * Modification history : 11 * 12 * Date Name Description 13 * 02/01/01 synwee Added normalization quickcheck enum and method. 14 */ 15 #ifndef UNORM_H 16 #define UNORM_H 17 18 #include "unicode/utypes.h" 19 20 #if !UCONFIG_NO_NORMALIZATION 21 22 #include "unicode/uiter.h" 23 #include "unicode/unorm2.h" 24 25 #ifndef U_HIDE_DEPRECATED_API 26 27 /** 28 * \file 29 * \brief C API: Unicode Normalization 30 * 31 * Old Unicode normalization API. 32 * 33 * This API has been replaced by the unorm2.h API and is only available 34 * for backward compatibility. The functions here simply delegate to the 35 * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize(). 36 * There is one exception: The new API does not provide a replacement for unorm_compare(). 37 * Its declaration has been moved to unorm2.h. 38 * 39 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or 40 * decomposed form, allowing for easier sorting and searching of text. 41 * <code>unorm_normalize</code> supports the standard normalization forms described in 42 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode"> 43 * Unicode Standard Annex #15: Unicode Normalization Forms</a>. 44 * 45 * Characters with accents or other adornments can be encoded in 46 * several different ways in Unicode. For example, take the character A-acute. 47 * In Unicode, this can be encoded as a single character (the 48 * "composed" form): 49 * 50 * \code 51 * 00C1 LATIN CAPITAL LETTER A WITH ACUTE 52 * \endcode 53 * 54 * or as two separate characters (the "decomposed" form): 55 * 56 * \code 57 * 0041 LATIN CAPITAL LETTER A 58 * 0301 COMBINING ACUTE ACCENT 59 * \endcode 60 * 61 * To a user of your program, however, both of these sequences should be 62 * treated as the same "user-level" character "A with acute accent". When you are searching or 63 * comparing text, you must ensure that these two sequences are treated 64 * equivalently. In addition, you must handle characters with more than one 65 * accent. Sometimes the order of a character's combining accents is 66 * significant, while in other cases accent sequences in different orders are 67 * really equivalent. 68 * 69 * Similarly, the string "ffi" can be encoded as three separate letters: 70 * 71 * \code 72 * 0066 LATIN SMALL LETTER F 73 * 0066 LATIN SMALL LETTER F 74 * 0069 LATIN SMALL LETTER I 75 * \endcode 76 * 77 * or as the single character 78 * 79 * \code 80 * FB03 LATIN SMALL LIGATURE FFI 81 * \endcode 82 * 83 * The ffi ligature is not a distinct semantic character, and strictly speaking 84 * it shouldn't be in Unicode at all, but it was included for compatibility 85 * with existing character sets that already provided it. The Unicode standard 86 * identifies such characters by giving them "compatibility" decompositions 87 * into the corresponding semantic characters. When sorting and searching, you 88 * will often want to use these mappings. 89 * 90 * <code>unorm_normalize</code> helps solve these problems by transforming text into the 91 * canonical composed and decomposed forms as shown in the first example above. 92 * In addition, you can have it perform compatibility decompositions so that 93 * you can treat compatibility characters the same as their equivalents. 94 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical 95 * order, so that you do not have to worry about accent rearrangement on your 96 * own. 97 * 98 * Form FCD, "Fast C or D", is also designed for collation. 99 * It allows to work on strings that are not necessarily normalized 100 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed 101 * characters and their decomposed equivalents the same. 102 * 103 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings 104 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical 105 * themselves. 106 * 107 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character, 108 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long 109 * as their decompositions do not need canonical reordering. 110 * 111 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts - 112 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will 113 * return UNORM_YES for most strings in practice. 114 * 115 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD. 116 * 117 * For more details on FCD see the collation design document: 118 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm 119 * 120 * ICU collation performs either NFD or FCD normalization automatically if normalization 121 * is turned on for the collator object. 122 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons, 123 * transliteration/transcription, unique representations, etc. 124 * 125 * The W3C generally recommends to exchange texts in NFC. 126 * Note also that most legacy character encodings use only precomposed forms and often do not 127 * encode any combining marks by themselves. For conversion to such character encodings the 128 * Unicode text needs to be normalized to NFC. 129 * For more usage examples, see the Unicode Standard Annex. 130 */ 131 132 #ifndef U_HIDE_DEPRECATED_API 133 /** 134 * Constants for normalization modes. 135 * @deprecated ICU 56 Use unorm2.h instead. 136 */ 137 typedef enum { 138 /** No decomposition/composition. @deprecated ICU 56 Use unorm2.h instead. */ 139 UNORM_NONE = 1, 140 /** Canonical decomposition. @deprecated ICU 56 Use unorm2.h instead. */ 141 UNORM_NFD = 2, 142 /** Compatibility decomposition. @deprecated ICU 56 Use unorm2.h instead. */ 143 UNORM_NFKD = 3, 144 /** Canonical decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */ 145 UNORM_NFC = 4, 146 /** Default normalization. @deprecated ICU 56 Use unorm2.h instead. */ 147 UNORM_DEFAULT = UNORM_NFC, 148 /** Compatibility decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */ 149 UNORM_NFKC =5, 150 /** "Fast C or D" form. @deprecated ICU 56 Use unorm2.h instead. */ 151 UNORM_FCD = 6, 152 153 /** One more than the highest normalization mode constant. @deprecated ICU 56 Use unorm2.h instead. */ 154 UNORM_MODE_COUNT 155 } UNormalizationMode; 156 #endif /* U_HIDE_DEPRECATED_API */ 157 158 /** 159 * Constants for options flags for normalization. 160 * Use 0 for default options, 161 * including normalization according to the Unicode version 162 * that is currently supported by ICU (see u_getUnicodeVersion). 163 * @deprecated ICU 56 Use unorm2.h instead. 164 */ 165 enum { 166 /** 167 * Options bit set value to select Unicode 3.2 normalization 168 * (except NormalizationCorrections). 169 * At most one Unicode version can be selected at a time. 170 * @deprecated ICU 56 Use unorm2.h instead. 171 */ 172 UNORM_UNICODE_3_2=0x20 173 }; 174 175 /** 176 * Lowest-order bit number of unorm_compare() options bits corresponding to 177 * normalization options bits. 178 * 179 * The options parameter for unorm_compare() uses most bits for 180 * itself and for various comparison and folding flags. 181 * The most significant bits, however, are shifted down and passed on 182 * to the normalization implementation. 183 * (That is, from unorm_compare(..., options, ...), 184 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the 185 * internal normalization functions.) 186 * 187 * @see unorm_compare 188 * @deprecated ICU 56 Use unorm2.h instead. 189 */ 190 #define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20 191 192 /** 193 * Normalize a string. 194 * The string will be normalized according the specified normalization mode 195 * and options. 196 * The source and result buffers must not be the same, nor overlap. 197 * 198 * @param source The string to normalize. 199 * @param sourceLength The length of source, or -1 if NUL-terminated. 200 * @param mode The normalization mode; one of UNORM_NONE, 201 * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT. 202 * @param options The normalization options, ORed together (0 for no options). 203 * @param result A pointer to a buffer to receive the result string. 204 * The result string is NUL-terminated if possible. 205 * @param resultLength The maximum size of result. 206 * @param status A pointer to a UErrorCode to receive any errors. 207 * @return The total buffer size needed; if greater than resultLength, 208 * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR. 209 * @deprecated ICU 56 Use unorm2.h instead. 210 */ 211 U_STABLE int32_t U_EXPORT2 212 unorm_normalize(const UChar *source, int32_t sourceLength, 213 UNormalizationMode mode, int32_t options, 214 UChar *result, int32_t resultLength, 215 UErrorCode *status); 216 217 /** 218 * Performing quick check on a string, to quickly determine if the string is 219 * in a particular normalization format. 220 * Three types of result can be returned UNORM_YES, UNORM_NO or 221 * UNORM_MAYBE. Result UNORM_YES indicates that the argument 222 * string is in the desired normalized format, UNORM_NO determines that 223 * argument string is not in the desired normalized format. A 224 * UNORM_MAYBE result indicates that a more thorough check is required, 225 * the user may have to put the string in its normalized form and compare the 226 * results. 227 * 228 * @param source string for determining if it is in a normalized format 229 * @param sourcelength length of source to test, or -1 if NUL-terminated 230 * @param mode which normalization form to test for 231 * @param status a pointer to a UErrorCode to receive any errors 232 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE 233 * 234 * @see unorm_isNormalized 235 * @deprecated ICU 56 Use unorm2.h instead. 236 */ 237 U_STABLE UNormalizationCheckResult U_EXPORT2 238 unorm_quickCheck(const UChar *source, int32_t sourcelength, 239 UNormalizationMode mode, 240 UErrorCode *status); 241 242 /** 243 * Performing quick check on a string; same as unorm_quickCheck but 244 * takes an extra options parameter like most normalization functions. 245 * 246 * @param src String that is to be tested if it is in a normalization format. 247 * @param srcLength Length of source to test, or -1 if NUL-terminated. 248 * @param mode Which normalization form to test for. 249 * @param options The normalization options, ORed together (0 for no options). 250 * @param pErrorCode ICU error code in/out parameter. 251 * Must fulfill U_SUCCESS before the function call. 252 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE 253 * 254 * @see unorm_quickCheck 255 * @see unorm_isNormalized 256 * @deprecated ICU 56 Use unorm2.h instead. 257 */ 258 U_STABLE UNormalizationCheckResult U_EXPORT2 259 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength, 260 UNormalizationMode mode, int32_t options, 261 UErrorCode *pErrorCode); 262 263 /** 264 * Test if a string is in a given normalization form. 265 * This is semantically equivalent to source.equals(normalize(source, mode)) . 266 * 267 * Unlike unorm_quickCheck(), this function returns a definitive result, 268 * never a "maybe". 269 * For NFD, NFKD, and FCD, both functions work exactly the same. 270 * For NFC and NFKC where quickCheck may return "maybe", this function will 271 * perform further tests to arrive at a TRUE/FALSE result. 272 * 273 * @param src String that is to be tested if it is in a normalization format. 274 * @param srcLength Length of source to test, or -1 if NUL-terminated. 275 * @param mode Which normalization form to test for. 276 * @param pErrorCode ICU error code in/out parameter. 277 * Must fulfill U_SUCCESS before the function call. 278 * @return Boolean value indicating whether the source string is in the 279 * "mode" normalization form. 280 * 281 * @see unorm_quickCheck 282 * @deprecated ICU 56 Use unorm2.h instead. 283 */ 284 U_STABLE UBool U_EXPORT2 285 unorm_isNormalized(const UChar *src, int32_t srcLength, 286 UNormalizationMode mode, 287 UErrorCode *pErrorCode); 288 289 /** 290 * Test if a string is in a given normalization form; same as unorm_isNormalized but 291 * takes an extra options parameter like most normalization functions. 292 * 293 * @param src String that is to be tested if it is in a normalization format. 294 * @param srcLength Length of source to test, or -1 if NUL-terminated. 295 * @param mode Which normalization form to test for. 296 * @param options The normalization options, ORed together (0 for no options). 297 * @param pErrorCode ICU error code in/out parameter. 298 * Must fulfill U_SUCCESS before the function call. 299 * @return Boolean value indicating whether the source string is in the 300 * "mode/options" normalization form. 301 * 302 * @see unorm_quickCheck 303 * @see unorm_isNormalized 304 * @deprecated ICU 56 Use unorm2.h instead. 305 */ 306 U_STABLE UBool U_EXPORT2 307 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength, 308 UNormalizationMode mode, int32_t options, 309 UErrorCode *pErrorCode); 310 311 /** 312 * Iterative normalization forward. 313 * This function (together with unorm_previous) is somewhat 314 * similar to the C++ Normalizer class (see its non-static functions). 315 * 316 * Iterative normalization is useful when only a small portion of a longer 317 * string/text needs to be processed. 318 * 319 * For example, the likelihood may be high that processing the first 10% of some 320 * text will be sufficient to find certain data. 321 * Another example: When one wants to concatenate two normalized strings and get a 322 * normalized result, it is much more efficient to normalize just a small part of 323 * the result around the concatenation place instead of re-normalizing everything. 324 * 325 * The input text is an instance of the C character iteration API UCharIterator. 326 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any 327 * other kind of text object. 328 * 329 * If a buffer overflow occurs, then the caller needs to reset the iterator to the 330 * old index and call the function again with a larger buffer - if the caller cares 331 * for the actual output. 332 * Regardless of the output buffer, the iterator will always be moved to the next 333 * normalization boundary. 334 * 335 * This function (like unorm_previous) serves two purposes: 336 * 337 * 1) To find the next boundary so that the normalization of the part of the text 338 * from the current position to that boundary does not affect and is not affected 339 * by the part of the text beyond that boundary. 340 * 341 * 2) To normalize the text up to the boundary. 342 * 343 * The second step is optional, per the doNormalize parameter. 344 * It is omitted for operations like string concatenation, where the two adjacent 345 * string ends need to be normalized together. 346 * In such a case, the output buffer will just contain a copy of the text up to the 347 * boundary. 348 * 349 * pNeededToNormalize is an output-only parameter. Its output value is only defined 350 * if normalization was requested (doNormalize) and successful (especially, no 351 * buffer overflow). 352 * It is useful for operations like a normalizing transliterator, where one would 353 * not want to replace a piece of text if it is not modified. 354 * 355 * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE 356 * if the normalization was necessary. 357 * 358 * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE. 359 * 360 * If the buffer overflows, then *pNeededToNormalize will be undefined; 361 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result 362 * will be undefined. 363 * 364 * @param src The input text in the form of a C character iterator. 365 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 366 * @param destCapacity The number of UChars that fit into dest. 367 * @param mode The normalization mode. 368 * @param options The normalization options, ORed together (0 for no options). 369 * @param doNormalize Indicates if the source text up to the next boundary 370 * is to be normalized (TRUE) or just copied (FALSE). 371 * @param pNeededToNormalize Output flag indicating if the normalization resulted in 372 * different text from the input. 373 * Not defined if an error occurs including buffer overflow. 374 * Always FALSE if !doNormalize. 375 * @param pErrorCode ICU error code in/out parameter. 376 * Must fulfill U_SUCCESS before the function call. 377 * @return Length of output (number of UChars) when successful or buffer overflow. 378 * 379 * @see unorm_previous 380 * @see unorm_normalize 381 * 382 * @deprecated ICU 56 Use unorm2.h instead. 383 */ 384 U_STABLE int32_t U_EXPORT2 385 unorm_next(UCharIterator *src, 386 UChar *dest, int32_t destCapacity, 387 UNormalizationMode mode, int32_t options, 388 UBool doNormalize, UBool *pNeededToNormalize, 389 UErrorCode *pErrorCode); 390 391 /** 392 * Iterative normalization backward. 393 * This function (together with unorm_next) is somewhat 394 * similar to the C++ Normalizer class (see its non-static functions). 395 * For all details see unorm_next. 396 * 397 * @param src The input text in the form of a C character iterator. 398 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 399 * @param destCapacity The number of UChars that fit into dest. 400 * @param mode The normalization mode. 401 * @param options The normalization options, ORed together (0 for no options). 402 * @param doNormalize Indicates if the source text up to the next boundary 403 * is to be normalized (TRUE) or just copied (FALSE). 404 * @param pNeededToNormalize Output flag indicating if the normalization resulted in 405 * different text from the input. 406 * Not defined if an error occurs including buffer overflow. 407 * Always FALSE if !doNormalize. 408 * @param pErrorCode ICU error code in/out parameter. 409 * Must fulfill U_SUCCESS before the function call. 410 * @return Length of output (number of UChars) when successful or buffer overflow. 411 * 412 * @see unorm_next 413 * @see unorm_normalize 414 * 415 * @deprecated ICU 56 Use unorm2.h instead. 416 */ 417 U_STABLE int32_t U_EXPORT2 418 unorm_previous(UCharIterator *src, 419 UChar *dest, int32_t destCapacity, 420 UNormalizationMode mode, int32_t options, 421 UBool doNormalize, UBool *pNeededToNormalize, 422 UErrorCode *pErrorCode); 423 424 /** 425 * Concatenate normalized strings, making sure that the result is normalized as well. 426 * 427 * If both the left and the right strings are in 428 * the normalization form according to "mode/options", 429 * then the result will be 430 * 431 * \code 432 * dest=normalize(left+right, mode, options) 433 * \endcode 434 * 435 * With the input strings already being normalized, 436 * this function will use unorm_next() and unorm_previous() 437 * to find the adjacent end pieces of the input strings. 438 * Only the concatenation of these end pieces will be normalized and 439 * then concatenated with the remaining parts of the input strings. 440 * 441 * It is allowed to have dest==left to avoid copying the entire left string. 442 * 443 * @param left Left source string, may be same as dest. 444 * @param leftLength Length of left source string, or -1 if NUL-terminated. 445 * @param right Right source string. Must not be the same as dest, nor overlap. 446 * @param rightLength Length of right source string, or -1 if NUL-terminated. 447 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 448 * @param destCapacity The number of UChars that fit into dest. 449 * @param mode The normalization mode. 450 * @param options The normalization options, ORed together (0 for no options). 451 * @param pErrorCode ICU error code in/out parameter. 452 * Must fulfill U_SUCCESS before the function call. 453 * @return Length of output (number of UChars) when successful or buffer overflow. 454 * 455 * @see unorm_normalize 456 * @see unorm_next 457 * @see unorm_previous 458 * 459 * @deprecated ICU 56 Use unorm2.h instead. 460 */ 461 U_STABLE int32_t U_EXPORT2 462 unorm_concatenate(const UChar *left, int32_t leftLength, 463 const UChar *right, int32_t rightLength, 464 UChar *dest, int32_t destCapacity, 465 UNormalizationMode mode, int32_t options, 466 UErrorCode *pErrorCode); 467 468 #endif /* U_HIDE_DEPRECATED_API */ 469 #endif /* #if !UCONFIG_NO_NORMALIZATION */ 470 #endif 471