1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ********************************************************************** 5 * Copyright (c) 2001-2011, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ********************************************************************** 8 * Date Name Description 9 * 11/19/2001 aliu Creation. 10 ********************************************************************** 11 */ 12 13 #include "unicode/unimatch.h" 14 #include "unicode/utf16.h" 15 #include "patternprops.h" 16 #include "util.h" 17 18 // Define UChar constants using hex for EBCDIC compatibility 19 20 static const UChar BACKSLASH = 0x005C; /*\*/ 21 static const UChar UPPER_U = 0x0055; /*U*/ 22 static const UChar LOWER_U = 0x0075; /*u*/ 23 static const UChar APOSTROPHE = 0x0027; // '\'' 24 static const UChar SPACE = 0x0020; // ' ' 25 26 // "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 27 static const UChar DIGITS[] = { 28 48,49,50,51,52,53,54,55,56,57, 29 65,66,67,68,69,70,71,72,73,74, 30 75,76,77,78,79,80,81,82,83,84, 31 85,86,87,88,89,90 32 }; 33 34 U_NAMESPACE_BEGIN 35 36 UnicodeString& ICU_Utility::appendNumber(UnicodeString& result, int32_t n, 37 int32_t radix, int32_t minDigits) { 38 if (radix < 2 || radix > 36) { 39 // Bogus radix 40 return result.append((UChar)63/*?*/); 41 } 42 // Handle negatives 43 if (n < 0) { 44 n = -n; 45 result.append((UChar)45/*-*/); 46 } 47 // First determine the number of digits 48 int32_t nn = n; 49 int32_t r = 1; 50 while (nn >= radix) { 51 nn /= radix; 52 r *= radix; 53 --minDigits; 54 } 55 // Now generate the digits 56 while (--minDigits > 0) { 57 result.append(DIGITS[0]); 58 } 59 while (r > 0) { 60 int32_t digit = n / r; 61 result.append(DIGITS[digit]); 62 n -= digit * r; 63 r /= radix; 64 } 65 return result; 66 } 67 68 /** 69 * Return true if the character is NOT printable ASCII. 70 */ 71 UBool ICU_Utility::isUnprintable(UChar32 c) { 72 return !(c >= 0x20 && c <= 0x7E); 73 } 74 75 /** 76 * Escape unprintable characters using \uxxxx notation for U+0000 to 77 * U+FFFF and \Uxxxxxxxx for U+10000 and above. If the character is 78 * printable ASCII, then do nothing and return FALSE. Otherwise, 79 * append the escaped notation and return TRUE. 80 */ 81 UBool ICU_Utility::escapeUnprintable(UnicodeString& result, UChar32 c) { 82 if (isUnprintable(c)) { 83 result.append(BACKSLASH); 84 if (c & ~0xFFFF) { 85 result.append(UPPER_U); 86 result.append(DIGITS[0xF&(c>>28)]); 87 result.append(DIGITS[0xF&(c>>24)]); 88 result.append(DIGITS[0xF&(c>>20)]); 89 result.append(DIGITS[0xF&(c>>16)]); 90 } else { 91 result.append(LOWER_U); 92 } 93 result.append(DIGITS[0xF&(c>>12)]); 94 result.append(DIGITS[0xF&(c>>8)]); 95 result.append(DIGITS[0xF&(c>>4)]); 96 result.append(DIGITS[0xF&c]); 97 return TRUE; 98 } 99 return FALSE; 100 } 101 102 /** 103 * Returns the index of a character, ignoring quoted text. 104 * For example, in the string "abc'hide'h", the 'h' in "hide" will not be 105 * found by a search for 'h'. 106 */ 107 // FOR FUTURE USE. DISABLE FOR NOW for coverage reasons. 108 /* 109 int32_t ICU_Utility::quotedIndexOf(const UnicodeString& text, 110 int32_t start, int32_t limit, 111 UChar charToFind) { 112 for (int32_t i=start; i<limit; ++i) { 113 UChar c = text.charAt(i); 114 if (c == BACKSLASH) { 115 ++i; 116 } else if (c == APOSTROPHE) { 117 while (++i < limit 118 && text.charAt(i) != APOSTROPHE) {} 119 } else if (c == charToFind) { 120 return i; 121 } 122 } 123 return -1; 124 } 125 */ 126 127 /** 128 * Skip over a sequence of zero or more white space characters at pos. 129 * @param advance if true, advance pos to the first non-white-space 130 * character at or after pos, or str.length(), if there is none. 131 * Otherwise leave pos unchanged. 132 * @return the index of the first non-white-space character at or 133 * after pos, or str.length(), if there is none. 134 */ 135 int32_t ICU_Utility::skipWhitespace(const UnicodeString& str, int32_t& pos, 136 UBool advance) { 137 int32_t p = pos; 138 const UChar* s = str.getBuffer(); 139 p = (int32_t)(PatternProps::skipWhiteSpace(s + p, str.length() - p) - s); 140 if (advance) { 141 pos = p; 142 } 143 return p; 144 } 145 146 /** 147 * Skip over Pattern_White_Space in a Replaceable. 148 * Skipping may be done in the forward or 149 * reverse direction. In either case, the leftmost index will be 150 * inclusive, and the rightmost index will be exclusive. That is, 151 * given a range defined as [start, limit), the call 152 * skipWhitespace(text, start, limit) will advance start past leading 153 * whitespace, whereas the call skipWhitespace(text, limit, start), 154 * will back up limit past trailing whitespace. 155 * @param text the text to be analyzed 156 * @param pos either the start or limit of a range of 'text', to skip 157 * leading or trailing whitespace, respectively 158 * @param stop either the limit or start of a range of 'text', to skip 159 * leading or trailing whitespace, respectively 160 * @return the new start or limit, depending on what was passed in to 161 * 'pos' 162 */ 163 //?FOR FUTURE USE. DISABLE FOR NOW for coverage reasons. 164 //?int32_t ICU_Utility::skipWhitespace(const Replaceable& text, 165 //? int32_t pos, int32_t stop) { 166 //? UChar32 c; 167 //? UBool isForward = (stop >= pos); 168 //? 169 //? if (!isForward) { 170 //? --pos; // pos is a limit, so back up by one 171 //? } 172 //? 173 //? while (pos != stop && 174 //? PatternProps::isWhiteSpace(c = text.char32At(pos))) { 175 //? if (isForward) { 176 //? pos += U16_LENGTH(c); 177 //? } else { 178 //? pos -= U16_LENGTH(c); 179 //? } 180 //? } 181 //? 182 //? if (!isForward) { 183 //? ++pos; // make pos back into a limit 184 //? } 185 //? 186 //? return pos; 187 //?} 188 189 /** 190 * Parse a single non-whitespace character 'ch', optionally 191 * preceded by whitespace. 192 * @param id the string to be parsed 193 * @param pos INPUT-OUTPUT parameter. On input, pos[0] is the 194 * offset of the first character to be parsed. On output, pos[0] 195 * is the index after the last parsed character. If the parse 196 * fails, pos[0] will be unchanged. 197 * @param ch the non-whitespace character to be parsed. 198 * @return true if 'ch' is seen preceded by zero or more 199 * whitespace characters. 200 */ 201 UBool ICU_Utility::parseChar(const UnicodeString& id, int32_t& pos, UChar ch) { 202 int32_t start = pos; 203 skipWhitespace(id, pos, TRUE); 204 if (pos == id.length() || 205 id.charAt(pos) != ch) { 206 pos = start; 207 return FALSE; 208 } 209 ++pos; 210 return TRUE; 211 } 212 213 /** 214 * Parse a pattern string within the given Replaceable and a parsing 215 * pattern. Characters are matched literally and case-sensitively 216 * except for the following special characters: 217 * 218 * ~ zero or more Pattern_White_Space chars 219 * 220 * If end of pattern is reached with all matches along the way, 221 * pos is advanced to the first unparsed index and returned. 222 * Otherwise -1 is returned. 223 * @param pat pattern that controls parsing 224 * @param text text to be parsed, starting at index 225 * @param index offset to first character to parse 226 * @param limit offset after last character to parse 227 * @return index after last parsed character, or -1 on parse failure. 228 */ 229 int32_t ICU_Utility::parsePattern(const UnicodeString& pat, 230 const Replaceable& text, 231 int32_t index, 232 int32_t limit) { 233 int32_t ipat = 0; 234 235 // empty pattern matches immediately 236 if (ipat == pat.length()) { 237 return index; 238 } 239 240 UChar32 cpat = pat.char32At(ipat); 241 242 while (index < limit) { 243 UChar32 c = text.char32At(index); 244 245 // parse \s* 246 if (cpat == 126 /*~*/) { 247 if (PatternProps::isWhiteSpace(c)) { 248 index += U16_LENGTH(c); 249 continue; 250 } else { 251 if (++ipat == pat.length()) { 252 return index; // success; c unparsed 253 } 254 // fall thru; process c again with next cpat 255 } 256 } 257 258 // parse literal 259 else if (c == cpat) { 260 index += U16_LENGTH(c); 261 ipat += U16_LENGTH(cpat); 262 if (ipat == pat.length()) { 263 return index; // success; c parsed 264 } 265 // fall thru; get next cpat 266 } 267 268 // match failure of literal 269 else { 270 return -1; 271 } 272 273 cpat = pat.char32At(ipat); 274 } 275 276 return -1; // text ended before end of pat 277 } 278 279 /** 280 * Append a character to a rule that is being built up. To flush 281 * the quoteBuf to rule, make one final call with isLiteral == TRUE. 282 * If there is no final character, pass in (UChar32)-1 as c. 283 * @param rule the string to append the character to 284 * @param c the character to append, or (UChar32)-1 if none. 285 * @param isLiteral if true, then the given character should not be 286 * quoted or escaped. Usually this means it is a syntactic element 287 * such as > or $ 288 * @param escapeUnprintable if true, then unprintable characters 289 * should be escaped using \uxxxx or \Uxxxxxxxx. These escapes will 290 * appear outside of quotes. 291 * @param quoteBuf a buffer which is used to build up quoted 292 * substrings. The caller should initially supply an empty buffer, 293 * and thereafter should not modify the buffer. The buffer should be 294 * cleared out by, at the end, calling this method with a literal 295 * character. 296 */ 297 void ICU_Utility::appendToRule(UnicodeString& rule, 298 UChar32 c, 299 UBool isLiteral, 300 UBool escapeUnprintable, 301 UnicodeString& quoteBuf) { 302 // If we are escaping unprintables, then escape them outside 303 // quotes. \u and \U are not recognized within quotes. The same 304 // logic applies to literals, but literals are never escaped. 305 if (isLiteral || 306 (escapeUnprintable && ICU_Utility::isUnprintable(c))) { 307 if (quoteBuf.length() > 0) { 308 // We prefer backslash APOSTROPHE to double APOSTROPHE 309 // (more readable, less similar to ") so if there are 310 // double APOSTROPHEs at the ends, we pull them outside 311 // of the quote. 312 313 // If the first thing in the quoteBuf is APOSTROPHE 314 // (doubled) then pull it out. 315 while (quoteBuf.length() >= 2 && 316 quoteBuf.charAt(0) == APOSTROPHE && 317 quoteBuf.charAt(1) == APOSTROPHE) { 318 rule.append(BACKSLASH).append(APOSTROPHE); 319 quoteBuf.remove(0, 2); 320 } 321 // If the last thing in the quoteBuf is APOSTROPHE 322 // (doubled) then remove and count it and add it after. 323 int32_t trailingCount = 0; 324 while (quoteBuf.length() >= 2 && 325 quoteBuf.charAt(quoteBuf.length()-2) == APOSTROPHE && 326 quoteBuf.charAt(quoteBuf.length()-1) == APOSTROPHE) { 327 quoteBuf.truncate(quoteBuf.length()-2); 328 ++trailingCount; 329 } 330 if (quoteBuf.length() > 0) { 331 rule.append(APOSTROPHE); 332 rule.append(quoteBuf); 333 rule.append(APOSTROPHE); 334 quoteBuf.truncate(0); 335 } 336 while (trailingCount-- > 0) { 337 rule.append(BACKSLASH).append(APOSTROPHE); 338 } 339 } 340 if (c != (UChar32)-1) { 341 /* Since spaces are ignored during parsing, they are 342 * emitted only for readability. We emit one here 343 * only if there isn't already one at the end of the 344 * rule. 345 */ 346 if (c == SPACE) { 347 int32_t len = rule.length(); 348 if (len > 0 && rule.charAt(len-1) != c) { 349 rule.append(c); 350 } 351 } else if (!escapeUnprintable || !ICU_Utility::escapeUnprintable(rule, c)) { 352 rule.append(c); 353 } 354 } 355 } 356 357 // Escape ' and '\' and don't begin a quote just for them 358 else if (quoteBuf.length() == 0 && 359 (c == APOSTROPHE || c == BACKSLASH)) { 360 rule.append(BACKSLASH); 361 rule.append(c); 362 } 363 364 // Specials (printable ascii that isn't [0-9a-zA-Z]) and 365 // whitespace need quoting. Also append stuff to quotes if we are 366 // building up a quoted substring already. 367 else if (quoteBuf.length() > 0 || 368 (c >= 0x0021 && c <= 0x007E && 369 !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) || 370 (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) || 371 (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) || 372 PatternProps::isWhiteSpace(c)) { 373 quoteBuf.append(c); 374 // Double ' within a quote 375 if (c == APOSTROPHE) { 376 quoteBuf.append(c); 377 } 378 } 379 380 // Otherwise just append 381 else { 382 rule.append(c); 383 } 384 } 385 386 void ICU_Utility::appendToRule(UnicodeString& rule, 387 const UnicodeString& text, 388 UBool isLiteral, 389 UBool escapeUnprintable, 390 UnicodeString& quoteBuf) { 391 for (int32_t i=0; i<text.length(); ++i) { 392 appendToRule(rule, text[i], isLiteral, escapeUnprintable, quoteBuf); 393 } 394 } 395 396 /** 397 * Given a matcher reference, which may be null, append its 398 * pattern as a literal to the given rule. 399 */ 400 void ICU_Utility::appendToRule(UnicodeString& rule, 401 const UnicodeMatcher* matcher, 402 UBool escapeUnprintable, 403 UnicodeString& quoteBuf) { 404 if (matcher != NULL) { 405 UnicodeString pat; 406 appendToRule(rule, matcher->toPattern(pat, escapeUnprintable), 407 TRUE, escapeUnprintable, quoteBuf); 408 } 409 } 410 411 U_NAMESPACE_END 412