1 // © 2018 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 
4 #include "unicode/utypes.h"
5 
6 #if !UCONFIG_NO_FORMATTING
7 
8 #include "numbertest.h"
9 #include "numparse_impl.h"
10 #include "static_unicode_sets.h"
11 #include "unicode/dcfmtsym.h"
12 #include "unicode/testlog.h"
13 
14 #include <cmath>
15 #include <numparse_affixes.h>
16 
17 using icu::unisets::get;
18 
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)19 void NumberParserTest::runIndexedTest(int32_t index, UBool exec, const char*& name, char*) {
20     if (exec) {
21         logln("TestSuite NumberParserTest: ");
22     }
23     TESTCASE_AUTO_BEGIN;
24         TESTCASE_AUTO(testBasic);
25         TESTCASE_AUTO(testSeriesMatcher);
26         TESTCASE_AUTO(testCombinedCurrencyMatcher);
27         TESTCASE_AUTO(testAffixPatternMatcher);
28     TESTCASE_AUTO_END;
29 }
30 
testBasic()31 void NumberParserTest::testBasic() {
32     IcuTestErrorCode status(*this, "testBasic");
33 
34     static const struct TestCase {
35         int32_t flags;
36         const char16_t* inputString;
37         const char16_t* patternString;
38         int32_t expectedCharsConsumed;
39         double expectedResultDouble;
40     } cases[] = {{3, u"51423", u"0", 5, 51423.},
41                  {3, u"51423x", u"0", 5, 51423.},
42                  {3, u" 51423", u"0", 6, 51423.},
43                  {3, u"51423 ", u"0", 5, 51423.},
44                  {3, u"����������", u"0", 10, 51423.},
45                  {3, u"����������x", u"0", 10, 51423.},
46                  {3, u" ����������", u"0", 11, 51423.},
47                  {3, u"���������� ", u"0", 10, 51423.},
48                  {7, u"51,423", u"#,##,##0", 6, 51423.},
49                  {7, u" 51,423", u"#,##,##0", 7, 51423.},
50                  {7, u"51,423 ", u"#,##,##0", 6, 51423.},
51                  {7, u"51,423,", u"#,##,##0", 6, 51423.},
52                  {7, u"51,423,,", u"#,##,##0", 6, 51423.},
53                  {7, u"51,423.5", u"#,##,##0", 8, 51423.5},
54                  {7, u"51,423.5,", u"#,##,##0", 8, 51423.5},
55                  {7, u"51,423.5,,", u"#,##,##0", 8, 51423.5},
56                  {7, u"51,423.5.", u"#,##,##0", 8, 51423.5},
57                  {7, u"51,423.5..", u"#,##,##0", 8, 51423.5},
58                  {7, u"����,������", u"#,##,##0", 11, 51423.},
59                  {7, u"��,����,����,������", u"#,##,##0", 19, 78951423.},
60                  {7, u"����,������.������", u"#,##,##0", 18, 78951.423},
61                  {7, u"����,������", u"#,##,##0", 11, 78000.},
62                  {7, u"����,������.������", u"#,##,##0", 18, 78000.},
63                  {7, u"����,������.������", u"#,##,##0", 18, 78000.023},
64                  {7, u"����.������.������", u"#,##,##0", 11, 78.},
65                  {7, u"1,", u"#,##,##0", 1, 1.},
66                  {7, u"1,,", u"#,##,##0", 1, 1.},
67                  {7, u"1.,", u"#,##,##0", 2, 1.},
68                  {3, u"1,.", u"#,##,##0", 3, 1.},
69                  {7, u"1..", u"#,##,##0", 2, 1.},
70                  {3, u",1", u"#,##,##0", 2, 1.},
71                  {3, u"1,1", u"#,##,##0", 1, 1.},
72                  {3, u"1,1,", u"#,##,##0", 1, 1.},
73                  {3, u"1,1,,", u"#,##,##0", 1, 1.},
74                  {3, u"1,1,1", u"#,##,##0", 1, 1.},
75                  {3, u"1,1,1,", u"#,##,##0", 1, 1.},
76                  {3, u"1,1,1,,", u"#,##,##0", 1, 1.},
77                  {3, u"-51423", u"0", 6, -51423.},
78                  {3, u"51423-", u"0", 5, 51423.}, // plus and minus sign by default do NOT match after
79                  {3, u"+51423", u"0", 6, 51423.},
80                  {3, u"51423+", u"0", 5, 51423.}, // plus and minus sign by default do NOT match after
81                  {3, u"%51423", u"0", 6, 51423.},
82                  {3, u"51423%", u"0", 6, 51423.},
83                  {3, u"51423%%", u"0", 6, 51423.},
84                  {3, u"‰51423", u"0", 6, 51423.},
85                  {3, u"51423‰", u"0", 6, 51423.},
86                  {3, u"51423‰‰", u"0", 6, 51423.},
87                  {3, u"∞", u"0", 1, INFINITY},
88                  {3, u"-∞", u"0", 2, -INFINITY},
89                  {3, u"@@@123  @@", u"0", 6, 123.}, // TODO: Should padding be strong instead of weak?
90                  {3, u"@@@123@@  ", u"0", 6, 123.}, // TODO: Should padding be strong instead of weak?
91                  {3, u"a51423US dollars", u"a0¤¤¤", 16, 51423.},
92                  {3, u"a 51423 US dollars", u"a0¤¤¤", 18, 51423.},
93                  {3, u"514.23 USD", u"¤0", 10, 514.23},
94                  {3, u"514.23 GBP", u"¤0", 10, 514.23},
95                  {3, u"a ���������� b", u"a0b", 14, 51423.},
96                  {3, u"-a ���������� b", u"a0b", 15, -51423.},
97                  {3, u"a -���������� b", u"a0b", 15, -51423.},
98                  {3, u"����������", u"[0];(0)", 10, 51423.},
99                  {3, u"[����������", u"[0];(0)", 11, 51423.},
100                  {3, u"����������]", u"[0];(0)", 11, 51423.},
101                  {3, u"[����������]", u"[0];(0)", 12, 51423.},
102                  {3, u"(����������", u"[0];(0)", 11, -51423.},
103                  {3, u"����������)", u"[0];(0)", 11, -51423.},
104                  {3, u"(����������)", u"[0];(0)", 12, -51423.},
105                  {3, u"����������", u"{0};{0}", 10, 51423.},
106                  {3, u"{����������", u"{0};{0}", 11, 51423.},
107                  {3, u"����������}", u"{0};{0}", 11, 51423.},
108                  {3, u"{����������}", u"{0};{0}", 12, 51423.},
109                  {1, u"a40b", u"a0'0b'", 3, 40.}, // greedy code path thinks "40" is the number
110                  {2, u"a40b", u"a0'0b'", 4, 4.}, // slow code path finds the suffix "0b"
111                  {3, u"��.������E��", u"0", 12, 5142.},
112                  {3, u"��.������E-��", u"0", 13, 0.005142},
113                  {3, u"��.������e-��", u"0", 13, 0.005142},
114                  {7, u"5,142.50 Canadian dollars", u"#,##,##0 ¤¤¤", 25, 5142.5},
115                  {3, u"a$ b5", u"a ¤ b0", 5, 5.0},
116                  {3, u"��1.23", u"��0;��0", 6, 1.23},
117                  {3, u"��1.23", u"��0;��0", 6, -1.23},
118                  {3, u".00", u"0", 3, 0.0},
119                  {3, u"                              1,234", u"a0", 35, 1234.}, // should not hang
120                  {3, u"NaN", u"0", 3, NAN},
121                  {3, u"NaN E5", u"0", 6, NAN},
122                  {3, u"0", u"0", 1, 0.0}};
123 
124     parse_flags_t parseFlags = PARSE_FLAG_IGNORE_CASE | PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES;
125     for (auto& cas : cases) {
126         UnicodeString inputString(cas.inputString);
127         UnicodeString patternString(cas.patternString);
128         LocalPointer<const NumberParserImpl> parser(
129                 NumberParserImpl::createSimpleParser(
130                         Locale("en"), patternString, parseFlags, status));
131         if (status.errDataIfFailureAndReset("createSimpleParser() failed")) {
132             continue;
133         }
134         UnicodeString message =
135                 UnicodeString("Input <") + inputString + UnicodeString("> Parser ") + parser->toString();
136 
137         if (0 != (cas.flags & 0x01)) {
138             // Test greedy code path
139             ParsedNumber resultObject;
140             parser->parse(inputString, true, resultObject, status);
141             assertTrue("Greedy Parse failed: " + message, resultObject.success());
142             assertEquals(
143                     "Greedy Parse failed: " + message, cas.expectedCharsConsumed, resultObject.charEnd);
144             assertEquals(
145                     "Greedy Parse failed: " + message, cas.expectedResultDouble, resultObject.getDouble());
146         }
147 
148         if (0 != (cas.flags & 0x02)) {
149             // Test slow code path
150             ParsedNumber resultObject;
151             parser->parse(inputString, false, resultObject, status);
152             assertTrue("Non-Greedy Parse failed: " + message, resultObject.success());
153             assertEquals(
154                     "Non-Greedy Parse failed: " + message,
155                     cas.expectedCharsConsumed,
156                     resultObject.charEnd);
157             assertEquals(
158                     "Non-Greedy Parse failed: " + message,
159                     cas.expectedResultDouble,
160                     resultObject.getDouble());
161         }
162 
163         if (0 != (cas.flags & 0x04)) {
164             // Test with strict separators
165             parser.adoptInstead(
166                     NumberParserImpl::createSimpleParser(
167                             Locale("en"),
168                             patternString,
169                             parseFlags | PARSE_FLAG_STRICT_GROUPING_SIZE,
170                             status));
171             ParsedNumber resultObject;
172             parser->parse(inputString, true, resultObject, status);
173             assertTrue("Strict Parse failed: " + message, resultObject.success());
174             assertEquals(
175                     "Strict Parse failed: " + message, cas.expectedCharsConsumed, resultObject.charEnd);
176             assertEquals(
177                     "Strict Parse failed: " + message, cas.expectedResultDouble, resultObject.getDouble());
178         }
179     }
180 }
181 
testSeriesMatcher()182 void NumberParserTest::testSeriesMatcher() {
183     IcuTestErrorCode status(*this, "testSeriesMatcher");
184 
185     DecimalFormatSymbols symbols("en", status);
186     if (status.errDataIfFailureAndReset("Failure in DecimalFormtSymbols constructor")) {
187         return;
188     }
189     PlusSignMatcher m0(symbols, false);
190     MinusSignMatcher m1(symbols, false);
191     IgnorablesMatcher m2(unisets::DEFAULT_IGNORABLES);
192     PercentMatcher m3(symbols);
193     IgnorablesMatcher m4(unisets::DEFAULT_IGNORABLES);
194 
195     ArraySeriesMatcher::MatcherArray matchers(5);
196     matchers[0] = &m0;
197     matchers[1] = &m1;
198     matchers[2] = &m2;
199     matchers[3] = &m3;
200     matchers[4] = &m4;
201     ArraySeriesMatcher series(matchers, 5);
202 
203     assertFalse("", series.smokeTest(StringSegment(u"x", false)));
204     assertFalse("", series.smokeTest(StringSegment(u"-", false)));
205     assertTrue("", series.smokeTest(StringSegment(u"+", false)));
206 
207     static const struct TestCase {
208         const char16_t* input;
209         int32_t expectedOffset;
210         bool expectedMaybeMore;
211     } cases[] = {{u"", 0, true},
212                  {u" ", 0, false},
213                  {u"$", 0, false},
214                  {u"+", 0, true},
215                  {u" +", 0, false},
216                  {u"+-", 0, true},
217                  {u"+ -", 0, false},
218                  {u"+-  ", 0, true},
219                  {u"+-  $", 0, false},
220                  {u"+-%", 3, true},
221                  {u"  +-  %  ", 0, false},
222                  {u"+-  %  ", 7, true},
223                  {u"+-%$", 3, false}};
224 
225     for (auto& cas : cases) {
226         UnicodeString input(cas.input);
227 
228         StringSegment segment(input, false);
229         ParsedNumber result;
230         bool actualMaybeMore = series.match(segment, result, status);
231         int actualOffset = segment.getOffset();
232 
233         assertEquals("'" + input + "'", cas.expectedOffset, actualOffset);
234         assertEquals("'" + input + "'", cas.expectedMaybeMore, actualMaybeMore);
235     }
236 }
237 
testCombinedCurrencyMatcher()238 void NumberParserTest::testCombinedCurrencyMatcher() {
239     IcuTestErrorCode status(*this, "testCombinedCurrencyMatcher");
240 
241     IgnorablesMatcher ignorables(unisets::DEFAULT_IGNORABLES);
242     Locale locale = Locale::getEnglish();
243 
244     DecimalFormatSymbols dfs(locale, status);
245     if (status.errDataIfFailureAndReset("Failure in DecimalFormtSymbols constructor")) {
246         return;
247     }
248     dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$", status);
249     dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU", status);
250     CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status);
251 
252     AffixTokenMatcherSetupData affixSetupData = {
253             currencySymbols, {"en", status}, ignorables, "en", 0};
254     AffixTokenMatcherWarehouse warehouse(&affixSetupData);
255     NumberParseMatcher& matcher = warehouse.currency(status);
256     affixSetupData.parseFlags = PARSE_FLAG_NO_FOREIGN_CURRENCY;
257     AffixTokenMatcherWarehouse warehouseNoForeign(&affixSetupData);
258     NumberParseMatcher& matcherNoForeign = warehouseNoForeign.currency(status);
259 
260     static const struct TestCase {
261         const char16_t* input;
262         const char16_t* expectedCurrencyCode;
263         const char16_t* expectedNoForeignCurrencyCode;
264     } cases[]{{u"", u"", u""},
265               {u"FOO", u"", u""},
266               {u"USD", u"USD", u""},
267               {u"$", u"USD", u""},
268               {u"US dollars", u"USD", u""},
269               {u"eu", u"", u""},
270               {u"euros", u"EUR", u""},
271               {u"ICU", u"ICU", u"ICU"},
272               {u"IU$", u"ICU", u"ICU"}};
273     for (auto& cas : cases) {
274         UnicodeString input(cas.input);
275 
276         {
277             StringSegment segment(input, false);
278             ParsedNumber result;
279             matcher.match(segment, result, status);
280             assertEquals(
281                     "Parsing " + input,
282                     cas.expectedCurrencyCode,
283                     result.currencyCode);
284             assertEquals(
285                     "Whole string on " + input,
286                     cas.expectedCurrencyCode[0] == 0 ? 0 : input.length(),
287                     result.charEnd);
288         }
289         {
290             StringSegment segment(input, false);
291             ParsedNumber result;
292             matcherNoForeign.match(segment, result, status);
293             assertEquals(
294                     "[no foreign] Parsing " + input,
295                     cas.expectedNoForeignCurrencyCode,
296                     result.currencyCode);
297             assertEquals(
298                     "[no foreign] Whole string on " + input,
299                     cas.expectedNoForeignCurrencyCode[0] == 0 ? 0 : input.length(),
300                     result.charEnd);
301         }
302     }
303 }
304 
testAffixPatternMatcher()305 void NumberParserTest::testAffixPatternMatcher() {
306     IcuTestErrorCode status(*this, "testAffixPatternMatcher");
307     Locale locale = Locale::getEnglish();
308     IgnorablesMatcher ignorables(unisets::DEFAULT_IGNORABLES);
309 
310     DecimalFormatSymbols dfs(locale, status);
311     dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$", status);
312     dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU", status);
313     CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status);
314 
315     AffixTokenMatcherSetupData affixSetupData = {
316             currencySymbols, {"en", status}, ignorables, "en", 0};
317     AffixTokenMatcherWarehouse warehouse(&affixSetupData);
318 
319     static const struct TestCase {
320         bool exactMatch;
321         const char16_t* affixPattern;
322         int32_t expectedMatcherLength;
323         const char16_t* sampleParseableString;
324     } cases[] = {{false, u"-", 1, u"-"},
325                  {false, u"+-%", 5, u"+-%"},
326                  {true, u"+-%", 3, u"+-%"},
327                  {false, u"ab c", 5, u"a    bc"},
328                  {true, u"abc", 3, u"abc"},
329                  {false, u"hello-to+this%very¤long‰string", 59, u"hello-to+this%very USD long‰string"}};
330 
331     for (auto& cas : cases) {
332         UnicodeString affixPattern(cas.affixPattern);
333         UnicodeString sampleParseableString(cas.sampleParseableString);
334         int parseFlags = cas.exactMatch ? PARSE_FLAG_EXACT_AFFIX : 0;
335 
336         bool success;
337         AffixPatternMatcher matcher = AffixPatternMatcher::fromAffixPattern(
338                 affixPattern, warehouse, parseFlags, &success, status);
339         if (!status.errDataIfFailureAndReset("Creation should be successful")) {
340 
341             // Check that the matcher has the expected number of children
342             assertEquals(affixPattern + " " + cas.exactMatch, cas.expectedMatcherLength, matcher.length());
343 
344             // Check that the matcher works on a sample string
345             StringSegment segment(sampleParseableString, false);
346             ParsedNumber result;
347             matcher.match(segment, result, status);
348             assertEquals(affixPattern + " " + cas.exactMatch, sampleParseableString.length(), result.charEnd);
349         }
350     }
351 }
352 
353 
354 #endif
355