1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 2009-2013, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: unorm2.h 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2009dec15 14 * created by: Markus W. Scherer 15 */ 16 17 #ifndef __UNORM2_H__ 18 #define __UNORM2_H__ 19 20 /** 21 * \file 22 * \brief C API: New API for Unicode Normalization. 23 * 24 * Unicode normalization functionality for standard Unicode normalization or 25 * for using custom mapping tables. 26 * All instances of UNormalizer2 are unmodifiable/immutable. 27 * Instances returned by unorm2_getInstance() are singletons that must not be deleted by the caller. 28 * For more details see the Normalizer2 C++ class. 29 */ 30 31 #include "unicode/utypes.h" 32 #include "unicode/localpointer.h" 33 #include "unicode/uset.h" 34 35 /** 36 * Constants for normalization modes. 37 * For details about standard Unicode normalization forms 38 * and about the algorithms which are also used with custom mapping tables 39 * see http://www.unicode.org/unicode/reports/tr15/ 40 * @stable ICU 4.4 41 */ 42 typedef enum { 43 /** 44 * Decomposition followed by composition. 45 * Same as standard NFC when using an "nfc" instance. 46 * Same as standard NFKC when using an "nfkc" instance. 47 * For details about standard Unicode normalization forms 48 * see http://www.unicode.org/unicode/reports/tr15/ 49 * @stable ICU 4.4 50 */ 51 UNORM2_COMPOSE, 52 /** 53 * Map, and reorder canonically. 54 * Same as standard NFD when using an "nfc" instance. 55 * Same as standard NFKD when using an "nfkc" instance. 56 * For details about standard Unicode normalization forms 57 * see http://www.unicode.org/unicode/reports/tr15/ 58 * @stable ICU 4.4 59 */ 60 UNORM2_DECOMPOSE, 61 /** 62 * "Fast C or D" form. 63 * If a string is in this form, then further decomposition <i>without reordering</i> 64 * would yield the same form as DECOMPOSE. 65 * Text in "Fast C or D" form can be processed efficiently with data tables 66 * that are "canonically closed", that is, that provide equivalent data for 67 * equivalent text, without having to be fully normalized. 68 * Not a standard Unicode normalization form. 69 * Not a unique form: Different FCD strings can be canonically equivalent. 70 * For details see http://www.unicode.org/notes/tn5/#FCD 71 * @stable ICU 4.4 72 */ 73 UNORM2_FCD, 74 /** 75 * Compose only contiguously. 76 * Also known as "FCC" or "Fast C Contiguous". 77 * The result will often but not always be in NFC. 78 * The result will conform to FCD which is useful for processing. 79 * Not a standard Unicode normalization form. 80 * For details see http://www.unicode.org/notes/tn5/#FCC 81 * @stable ICU 4.4 82 */ 83 UNORM2_COMPOSE_CONTIGUOUS 84 } UNormalization2Mode; 85 86 /** 87 * Result values for normalization quick check functions. 88 * For details see http://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms 89 * @stable ICU 2.0 90 */ 91 typedef enum UNormalizationCheckResult { 92 /** 93 * The input string is not in the normalization form. 94 * @stable ICU 2.0 95 */ 96 UNORM_NO, 97 /** 98 * The input string is in the normalization form. 99 * @stable ICU 2.0 100 */ 101 UNORM_YES, 102 /** 103 * The input string may or may not be in the normalization form. 104 * This value is only returned for composition forms like NFC and FCC, 105 * when a backward-combining character is found for which the surrounding text 106 * would have to be analyzed further. 107 * @stable ICU 2.0 108 */ 109 UNORM_MAYBE 110 } UNormalizationCheckResult; 111 112 /** 113 * Opaque C service object type for the new normalization API. 114 * @stable ICU 4.4 115 */ 116 struct UNormalizer2; 117 typedef struct UNormalizer2 UNormalizer2; /**< C typedef for struct UNormalizer2. @stable ICU 4.4 */ 118 119 #if !UCONFIG_NO_NORMALIZATION 120 121 /** 122 * Returns a UNormalizer2 instance for Unicode NFC normalization. 123 * Same as unorm2_getInstance(NULL, "nfc", UNORM2_COMPOSE, pErrorCode). 124 * Returns an unmodifiable singleton instance. Do not delete it. 125 * @param pErrorCode Standard ICU error code. Its input value must 126 * pass the U_SUCCESS() test, or else the function returns 127 * immediately. Check for U_FAILURE() on output or use with 128 * function chaining. (See User Guide for details.) 129 * @return the requested Normalizer2, if successful 130 * @stable ICU 49 131 */ 132 U_STABLE const UNormalizer2 * U_EXPORT2 133 unorm2_getNFCInstance(UErrorCode *pErrorCode); 134 135 /** 136 * Returns a UNormalizer2 instance for Unicode NFD normalization. 137 * Same as unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, pErrorCode). 138 * Returns an unmodifiable singleton instance. Do not delete it. 139 * @param pErrorCode Standard ICU error code. Its input value must 140 * pass the U_SUCCESS() test, or else the function returns 141 * immediately. Check for U_FAILURE() on output or use with 142 * function chaining. (See User Guide for details.) 143 * @return the requested Normalizer2, if successful 144 * @stable ICU 49 145 */ 146 U_STABLE const UNormalizer2 * U_EXPORT2 147 unorm2_getNFDInstance(UErrorCode *pErrorCode); 148 149 /** 150 * Returns a UNormalizer2 instance for Unicode NFKC normalization. 151 * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_COMPOSE, pErrorCode). 152 * Returns an unmodifiable singleton instance. Do not delete it. 153 * @param pErrorCode Standard ICU error code. Its input value must 154 * pass the U_SUCCESS() test, or else the function returns 155 * immediately. Check for U_FAILURE() on output or use with 156 * function chaining. (See User Guide for details.) 157 * @return the requested Normalizer2, if successful 158 * @stable ICU 49 159 */ 160 U_STABLE const UNormalizer2 * U_EXPORT2 161 unorm2_getNFKCInstance(UErrorCode *pErrorCode); 162 163 /** 164 * Returns a UNormalizer2 instance for Unicode NFKD normalization. 165 * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, pErrorCode). 166 * Returns an unmodifiable singleton instance. Do not delete it. 167 * @param pErrorCode Standard ICU error code. Its input value must 168 * pass the U_SUCCESS() test, or else the function returns 169 * immediately. Check for U_FAILURE() on output or use with 170 * function chaining. (See User Guide for details.) 171 * @return the requested Normalizer2, if successful 172 * @stable ICU 49 173 */ 174 U_STABLE const UNormalizer2 * U_EXPORT2 175 unorm2_getNFKDInstance(UErrorCode *pErrorCode); 176 177 /** 178 * Returns a UNormalizer2 instance for Unicode NFKC_Casefold normalization. 179 * Same as unorm2_getInstance(NULL, "nfkc_cf", UNORM2_COMPOSE, pErrorCode). 180 * Returns an unmodifiable singleton instance. Do not delete it. 181 * @param pErrorCode Standard ICU error code. Its input value must 182 * pass the U_SUCCESS() test, or else the function returns 183 * immediately. Check for U_FAILURE() on output or use with 184 * function chaining. (See User Guide for details.) 185 * @return the requested Normalizer2, if successful 186 * @stable ICU 49 187 */ 188 U_STABLE const UNormalizer2 * U_EXPORT2 189 unorm2_getNFKCCasefoldInstance(UErrorCode *pErrorCode); 190 191 /** 192 * Returns a UNormalizer2 instance which uses the specified data file 193 * (packageName/name similar to ucnv_openPackage() and ures_open()/ResourceBundle) 194 * and which composes or decomposes text according to the specified mode. 195 * Returns an unmodifiable singleton instance. Do not delete it. 196 * 197 * Use packageName=NULL for data files that are part of ICU's own data. 198 * Use name="nfc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFC/NFD. 199 * Use name="nfkc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFKC/NFKD. 200 * Use name="nfkc_cf" and UNORM2_COMPOSE for Unicode standard NFKC_CF=NFKC_Casefold. 201 * 202 * @param packageName NULL for ICU built-in data, otherwise application data package name 203 * @param name "nfc" or "nfkc" or "nfkc_cf" or name of custom data file 204 * @param mode normalization mode (compose or decompose etc.) 205 * @param pErrorCode Standard ICU error code. Its input value must 206 * pass the U_SUCCESS() test, or else the function returns 207 * immediately. Check for U_FAILURE() on output or use with 208 * function chaining. (See User Guide for details.) 209 * @return the requested UNormalizer2, if successful 210 * @stable ICU 4.4 211 */ 212 U_STABLE const UNormalizer2 * U_EXPORT2 213 unorm2_getInstance(const char *packageName, 214 const char *name, 215 UNormalization2Mode mode, 216 UErrorCode *pErrorCode); 217 218 /** 219 * Constructs a filtered normalizer wrapping any UNormalizer2 instance 220 * and a filter set. 221 * Both are aliased and must not be modified or deleted while this object 222 * is used. 223 * The filter set should be frozen; otherwise the performance will suffer greatly. 224 * @param norm2 wrapped UNormalizer2 instance 225 * @param filterSet USet which determines the characters to be normalized 226 * @param pErrorCode Standard ICU error code. Its input value must 227 * pass the U_SUCCESS() test, or else the function returns 228 * immediately. Check for U_FAILURE() on output or use with 229 * function chaining. (See User Guide for details.) 230 * @return the requested UNormalizer2, if successful 231 * @stable ICU 4.4 232 */ 233 U_STABLE UNormalizer2 * U_EXPORT2 234 unorm2_openFiltered(const UNormalizer2 *norm2, const USet *filterSet, UErrorCode *pErrorCode); 235 236 /** 237 * Closes a UNormalizer2 instance from unorm2_openFiltered(). 238 * Do not close instances from unorm2_getInstance()! 239 * @param norm2 UNormalizer2 instance to be closed 240 * @stable ICU 4.4 241 */ 242 U_STABLE void U_EXPORT2 243 unorm2_close(UNormalizer2 *norm2); 244 245 #if U_SHOW_CPLUSPLUS_API 246 247 U_NAMESPACE_BEGIN 248 249 /** 250 * \class LocalUNormalizer2Pointer 251 * "Smart pointer" class, closes a UNormalizer2 via unorm2_close(). 252 * For most methods see the LocalPointerBase base class. 253 * 254 * @see LocalPointerBase 255 * @see LocalPointer 256 * @stable ICU 4.4 257 */ 258 U_DEFINE_LOCAL_OPEN_POINTER(LocalUNormalizer2Pointer, UNormalizer2, unorm2_close); 259 260 U_NAMESPACE_END 261 262 #endif 263 264 /** 265 * Writes the normalized form of the source string to the destination string 266 * (replacing its contents) and returns the length of the destination string. 267 * The source and destination strings must be different buffers. 268 * @param norm2 UNormalizer2 instance 269 * @param src source string 270 * @param length length of the source string, or -1 if NUL-terminated 271 * @param dest destination string; its contents is replaced with normalized src 272 * @param capacity number of UChars that can be written to dest 273 * @param pErrorCode Standard ICU error code. Its input value must 274 * pass the U_SUCCESS() test, or else the function returns 275 * immediately. Check for U_FAILURE() on output or use with 276 * function chaining. (See User Guide for details.) 277 * @return dest 278 * @stable ICU 4.4 279 */ 280 U_STABLE int32_t U_EXPORT2 281 unorm2_normalize(const UNormalizer2 *norm2, 282 const UChar *src, int32_t length, 283 UChar *dest, int32_t capacity, 284 UErrorCode *pErrorCode); 285 /** 286 * Appends the normalized form of the second string to the first string 287 * (merging them at the boundary) and returns the length of the first string. 288 * The result is normalized if the first string was normalized. 289 * The first and second strings must be different buffers. 290 * @param norm2 UNormalizer2 instance 291 * @param first string, should be normalized 292 * @param firstLength length of the first string, or -1 if NUL-terminated 293 * @param firstCapacity number of UChars that can be written to first 294 * @param second string, will be normalized 295 * @param secondLength length of the source string, or -1 if NUL-terminated 296 * @param pErrorCode Standard ICU error code. Its input value must 297 * pass the U_SUCCESS() test, or else the function returns 298 * immediately. Check for U_FAILURE() on output or use with 299 * function chaining. (See User Guide for details.) 300 * @return first 301 * @stable ICU 4.4 302 */ 303 U_STABLE int32_t U_EXPORT2 304 unorm2_normalizeSecondAndAppend(const UNormalizer2 *norm2, 305 UChar *first, int32_t firstLength, int32_t firstCapacity, 306 const UChar *second, int32_t secondLength, 307 UErrorCode *pErrorCode); 308 /** 309 * Appends the second string to the first string 310 * (merging them at the boundary) and returns the length of the first string. 311 * The result is normalized if both the strings were normalized. 312 * The first and second strings must be different buffers. 313 * @param norm2 UNormalizer2 instance 314 * @param first string, should be normalized 315 * @param firstLength length of the first string, or -1 if NUL-terminated 316 * @param firstCapacity number of UChars that can be written to first 317 * @param second string, should be normalized 318 * @param secondLength length of the source string, or -1 if NUL-terminated 319 * @param pErrorCode Standard ICU error code. Its input value must 320 * pass the U_SUCCESS() test, or else the function returns 321 * immediately. Check for U_FAILURE() on output or use with 322 * function chaining. (See User Guide for details.) 323 * @return first 324 * @stable ICU 4.4 325 */ 326 U_STABLE int32_t U_EXPORT2 327 unorm2_append(const UNormalizer2 *norm2, 328 UChar *first, int32_t firstLength, int32_t firstCapacity, 329 const UChar *second, int32_t secondLength, 330 UErrorCode *pErrorCode); 331 332 /** 333 * Gets the decomposition mapping of c. 334 * Roughly equivalent to normalizing the String form of c 335 * on a UNORM2_DECOMPOSE UNormalizer2 instance, but much faster, and except that this function 336 * returns a negative value and does not write a string 337 * if c does not have a decomposition mapping in this instance's data. 338 * This function is independent of the mode of the UNormalizer2. 339 * @param norm2 UNormalizer2 instance 340 * @param c code point 341 * @param decomposition String buffer which will be set to c's 342 * decomposition mapping, if there is one. 343 * @param capacity number of UChars that can be written to decomposition 344 * @param pErrorCode Standard ICU error code. Its input value must 345 * pass the U_SUCCESS() test, or else the function returns 346 * immediately. Check for U_FAILURE() on output or use with 347 * function chaining. (See User Guide for details.) 348 * @return the non-negative length of c's decomposition, if there is one; otherwise a negative value 349 * @stable ICU 4.6 350 */ 351 U_STABLE int32_t U_EXPORT2 352 unorm2_getDecomposition(const UNormalizer2 *norm2, 353 UChar32 c, UChar *decomposition, int32_t capacity, 354 UErrorCode *pErrorCode); 355 356 /** 357 * Gets the raw decomposition mapping of c. 358 * 359 * This is similar to the unorm2_getDecomposition() function but returns the 360 * raw decomposition mapping as specified in UnicodeData.txt or 361 * (for custom data) in the mapping files processed by the gennorm2 tool. 362 * By contrast, unorm2_getDecomposition() returns the processed, 363 * recursively-decomposed version of this mapping. 364 * 365 * When used on a standard NFKC Normalizer2 instance, 366 * unorm2_getRawDecomposition() returns the Unicode Decomposition_Mapping (dm) property. 367 * 368 * When used on a standard NFC Normalizer2 instance, 369 * it returns the Decomposition_Mapping only if the Decomposition_Type (dt) is Canonical (Can); 370 * in this case, the result contains either one or two code points (=1..4 UChars). 371 * 372 * This function is independent of the mode of the UNormalizer2. 373 * @param norm2 UNormalizer2 instance 374 * @param c code point 375 * @param decomposition String buffer which will be set to c's 376 * raw decomposition mapping, if there is one. 377 * @param capacity number of UChars that can be written to decomposition 378 * @param pErrorCode Standard ICU error code. Its input value must 379 * pass the U_SUCCESS() test, or else the function returns 380 * immediately. Check for U_FAILURE() on output or use with 381 * function chaining. (See User Guide for details.) 382 * @return the non-negative length of c's raw decomposition, if there is one; otherwise a negative value 383 * @stable ICU 49 384 */ 385 U_STABLE int32_t U_EXPORT2 386 unorm2_getRawDecomposition(const UNormalizer2 *norm2, 387 UChar32 c, UChar *decomposition, int32_t capacity, 388 UErrorCode *pErrorCode); 389 390 /** 391 * Performs pairwise composition of a & b and returns the composite if there is one. 392 * 393 * Returns a composite code point c only if c has a two-way mapping to a+b. 394 * In standard Unicode normalization, this means that 395 * c has a canonical decomposition to a+b 396 * and c does not have the Full_Composition_Exclusion property. 397 * 398 * This function is independent of the mode of the UNormalizer2. 399 * @param norm2 UNormalizer2 instance 400 * @param a A (normalization starter) code point. 401 * @param b Another code point. 402 * @return The non-negative composite code point if there is one; otherwise a negative value. 403 * @stable ICU 49 404 */ 405 U_STABLE UChar32 U_EXPORT2 406 unorm2_composePair(const UNormalizer2 *norm2, UChar32 a, UChar32 b); 407 408 /** 409 * Gets the combining class of c. 410 * The default implementation returns 0 411 * but all standard implementations return the Unicode Canonical_Combining_Class value. 412 * @param norm2 UNormalizer2 instance 413 * @param c code point 414 * @return c's combining class 415 * @stable ICU 49 416 */ 417 U_STABLE uint8_t U_EXPORT2 418 unorm2_getCombiningClass(const UNormalizer2 *norm2, UChar32 c); 419 420 /** 421 * Tests if the string is normalized. 422 * Internally, in cases where the quickCheck() method would return "maybe" 423 * (which is only possible for the two COMPOSE modes) this method 424 * resolves to "yes" or "no" to provide a definitive result, 425 * at the cost of doing more work in those cases. 426 * @param norm2 UNormalizer2 instance 427 * @param s input string 428 * @param length length of the string, or -1 if NUL-terminated 429 * @param pErrorCode Standard ICU error code. Its input value must 430 * pass the U_SUCCESS() test, or else the function returns 431 * immediately. Check for U_FAILURE() on output or use with 432 * function chaining. (See User Guide for details.) 433 * @return TRUE if s is normalized 434 * @stable ICU 4.4 435 */ 436 U_STABLE UBool U_EXPORT2 437 unorm2_isNormalized(const UNormalizer2 *norm2, 438 const UChar *s, int32_t length, 439 UErrorCode *pErrorCode); 440 441 /** 442 * Tests if the string is normalized. 443 * For the two COMPOSE modes, the result could be "maybe" in cases that 444 * would take a little more work to resolve definitively. 445 * Use spanQuickCheckYes() and normalizeSecondAndAppend() for a faster 446 * combination of quick check + normalization, to avoid 447 * re-checking the "yes" prefix. 448 * @param norm2 UNormalizer2 instance 449 * @param s input string 450 * @param length length of the string, or -1 if NUL-terminated 451 * @param pErrorCode Standard ICU error code. Its input value must 452 * pass the U_SUCCESS() test, or else the function returns 453 * immediately. Check for U_FAILURE() on output or use with 454 * function chaining. (See User Guide for details.) 455 * @return UNormalizationCheckResult 456 * @stable ICU 4.4 457 */ 458 U_STABLE UNormalizationCheckResult U_EXPORT2 459 unorm2_quickCheck(const UNormalizer2 *norm2, 460 const UChar *s, int32_t length, 461 UErrorCode *pErrorCode); 462 463 /** 464 * Returns the end of the normalized substring of the input string. 465 * In other words, with <code>end=spanQuickCheckYes(s, ec);</code> 466 * the substring <code>UnicodeString(s, 0, end)</code> 467 * will pass the quick check with a "yes" result. 468 * 469 * The returned end index is usually one or more characters before the 470 * "no" or "maybe" character: The end index is at a normalization boundary. 471 * (See the class documentation for more about normalization boundaries.) 472 * 473 * When the goal is a normalized string and most input strings are expected 474 * to be normalized already, then call this method, 475 * and if it returns a prefix shorter than the input string, 476 * copy that prefix and use normalizeSecondAndAppend() for the remainder. 477 * @param norm2 UNormalizer2 instance 478 * @param s input string 479 * @param length length of the string, or -1 if NUL-terminated 480 * @param pErrorCode Standard ICU error code. Its input value must 481 * pass the U_SUCCESS() test, or else the function returns 482 * immediately. Check for U_FAILURE() on output or use with 483 * function chaining. (See User Guide for details.) 484 * @return "yes" span end index 485 * @stable ICU 4.4 486 */ 487 U_STABLE int32_t U_EXPORT2 488 unorm2_spanQuickCheckYes(const UNormalizer2 *norm2, 489 const UChar *s, int32_t length, 490 UErrorCode *pErrorCode); 491 492 /** 493 * Tests if the character always has a normalization boundary before it, 494 * regardless of context. 495 * For details see the Normalizer2 base class documentation. 496 * @param norm2 UNormalizer2 instance 497 * @param c character to test 498 * @return TRUE if c has a normalization boundary before it 499 * @stable ICU 4.4 500 */ 501 U_STABLE UBool U_EXPORT2 502 unorm2_hasBoundaryBefore(const UNormalizer2 *norm2, UChar32 c); 503 504 /** 505 * Tests if the character always has a normalization boundary after it, 506 * regardless of context. 507 * For details see the Normalizer2 base class documentation. 508 * @param norm2 UNormalizer2 instance 509 * @param c character to test 510 * @return TRUE if c has a normalization boundary after it 511 * @stable ICU 4.4 512 */ 513 U_STABLE UBool U_EXPORT2 514 unorm2_hasBoundaryAfter(const UNormalizer2 *norm2, UChar32 c); 515 516 /** 517 * Tests if the character is normalization-inert. 518 * For details see the Normalizer2 base class documentation. 519 * @param norm2 UNormalizer2 instance 520 * @param c character to test 521 * @return TRUE if c is normalization-inert 522 * @stable ICU 4.4 523 */ 524 U_STABLE UBool U_EXPORT2 525 unorm2_isInert(const UNormalizer2 *norm2, UChar32 c); 526 527 #endif /* !UCONFIG_NO_NORMALIZATION */ 528 #endif /* __UNORM2_H__ */ 529