1 /*
2 **********************************************************************
3 * Copyright (C) 1999-2014, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 * Date Name Description
7 * 12/09/99 aliu Ported from Java.
8 **********************************************************************
9 */
10
11 #include "unicode/utypes.h"
12
13 #if !UCONFIG_NO_COLLATION
14
15 #include "thcoll.h"
16 #include "unicode/utypes.h"
17 #include "unicode/coll.h"
18 #include "unicode/localpointer.h"
19 #include "unicode/sortkey.h"
20 #include "unicode/tblcoll.h"
21 #include "unicode/ustring.h"
22 #include "cstring.h"
23 #include "filestrm.h"
24 #include "textfile.h"
25
26 /**
27 * The TestDictionary test expects a file of this name, with this
28 * encoding, to be present in the directory $ICU/source/test/testdata.
29 */
30 //#define TEST_FILE "th18057.txt"
31
32 /**
33 * This is the most failures we show in TestDictionary. If this number
34 * is < 0, we show all failures.
35 */
36 #define MAX_FAILURES_TO_SHOW -1
37
CollationThaiTest()38 CollationThaiTest::CollationThaiTest() {
39 UErrorCode status = U_ZERO_ERROR;
40 coll = Collator::createInstance(Locale("th", "TH", ""), status);
41 if (coll && U_SUCCESS(status)) {
42 //coll->setStrength(Collator::TERTIARY);
43 } else {
44 delete coll;
45 coll = 0;
46 }
47 }
48
~CollationThaiTest()49 CollationThaiTest::~CollationThaiTest() {
50 delete coll;
51 }
52
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)53 void CollationThaiTest::runIndexedTest(int32_t index, UBool exec, const char* &name,
54 char* /*par*/) {
55
56 if((!coll) && exec) {
57 dataerrln(__FILE__ " cannot test - failed to create collator.");
58 name = "some test";
59 return;
60 }
61
62 switch (index) {
63 TESTCASE(0,TestDictionary);
64 TESTCASE(1,TestCornerCases);
65 TESTCASE(2,TestNamesList);
66 TESTCASE(3,TestInvalidThai);
67 TESTCASE(4,TestReordering);
68 default: name = ""; break;
69 }
70 }
71
72 /**
73 * Read the external names list, and confirms that the collator
74 * gets the same results when comparing lines one to another
75 * using regular and iterative comparison.
76 */
TestNamesList(void)77 void CollationThaiTest::TestNamesList(void) {
78 if (coll == 0) {
79 errln("Error: could not construct Thai collator");
80 return;
81 }
82
83 UErrorCode ec = U_ZERO_ERROR;
84 TextFile names("TestNames_Thai.txt", "UTF16LE", ec);
85 if (U_FAILURE(ec)) {
86 logln("Can't open TestNames_Thai.txt: %s; skipping test",
87 u_errorName(ec));
88 return;
89 }
90
91 //
92 // Loop through each word in the dictionary and compare it to the previous
93 // word. They should be in sorted order.
94 //
95 UnicodeString lastWord, word;
96 //int32_t failed = 0;
97 int32_t wordCount = 0;
98 while (names.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
99
100 // Show the first 8 words being compared, so we can see what's happening
101 ++wordCount;
102 if (wordCount <= 8) {
103 UnicodeString str;
104 logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
105 }
106
107 if (lastWord.length() > 0) {
108 Collator::EComparisonResult result = coll->compare(lastWord, word);
109 doTest(coll, lastWord, word, result);
110 }
111 lastWord = word;
112 }
113
114 assertSuccess("readLine", ec);
115
116 logln((UnicodeString)"Words checked: " + wordCount);
117 }
118
119 /**
120 * Read the external dictionary file, which is already in proper
121 * sorted order, and confirm that the collator compares each line as
122 * preceding the following line.
123 */
TestDictionary(void)124 void CollationThaiTest::TestDictionary(void) {
125 if (coll == 0) {
126 errln("Error: could not construct Thai collator");
127 return;
128 }
129
130 UErrorCode ec = U_ZERO_ERROR;
131 TextFile riwords("riwords.txt", "UTF8", ec);
132 if (U_FAILURE(ec)) {
133 logln("Can't open riwords.txt: %s; skipping test",
134 u_errorName(ec));
135 return;
136 }
137
138 //
139 // Loop through each word in the dictionary and compare it to the previous
140 // word. They should be in sorted order.
141 //
142 UnicodeString lastWord, word;
143 int32_t failed = 0;
144 int32_t wordCount = 0;
145 while (riwords.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
146
147 // Show the first 8 words being compared, so we can see what's happening
148 ++wordCount;
149 if (wordCount <= 8) {
150 UnicodeString str;
151 logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
152 }
153
154 if (lastWord.length() > 0) {
155 int32_t result = coll->compare(lastWord, word);
156
157 if (result > 0) {
158 failed++;
159 if (MAX_FAILURES_TO_SHOW < 0 || failed <= MAX_FAILURES_TO_SHOW) {
160 UnicodeString str;
161 UnicodeString msg =
162 UnicodeString("--------------------------------------------\n")
163 + riwords.getLineNumber()
164 + " compare(" + IntlTest::prettify(lastWord, str);
165 msg += UnicodeString(", ")
166 + IntlTest::prettify(word, str) + ") returned " + result
167 + ", expected -1\n";
168 UErrorCode status = U_ZERO_ERROR;
169 CollationKey k1, k2;
170 coll->getCollationKey(lastWord, k1, status);
171 coll->getCollationKey(word, k2, status);
172 if (U_FAILURE(status)) {
173 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
174 return;
175 }
176 msg.append("key1: ").append(prettify(k1, str)).append("\n");
177 msg.append("key2: ").append(prettify(k2, str));
178 errln(msg);
179 }
180 }
181 }
182 lastWord = word;
183 }
184
185 assertSuccess("readLine", ec);
186
187 if (failed != 0) {
188 if (failed > MAX_FAILURES_TO_SHOW) {
189 errln((UnicodeString)"Too many failures; only the first " +
190 MAX_FAILURES_TO_SHOW + " failures were shown");
191 }
192 errln((UnicodeString)"Summary: " + failed + " of " + (riwords.getLineNumber() - 1) +
193 " comparisons failed");
194 }
195
196 logln((UnicodeString)"Words checked: " + wordCount);
197 }
198
199 /**
200 * Odd corner conditions taken from "How to Sort Thai Without Rewriting Sort",
201 * by Doug Cooper, http://seasrc.th.net/paper/thaisort.zip
202 */
TestCornerCases(void)203 void CollationThaiTest::TestCornerCases(void) {
204 const char* TESTS[] = {
205 // Shorter words precede longer
206 "\\u0e01", "<", "\\u0e01\\u0e01",
207
208 // Tone marks are considered after letters (i.e. are primary ignorable)
209 "\\u0e01\\u0e32", "<", "\\u0e01\\u0e49\\u0e32",
210
211 // ditto for other over-marks
212 "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32\\u0e4c",
213
214 // commonly used mark-in-context order.
215 // In effect, marks are sorted after each syllable.
216 "\\u0e01\\u0e32\\u0e01\\u0e49\\u0e32", "<", "\\u0e01\\u0e48\\u0e32\\u0e01\\u0e49\\u0e32",
217
218 // Hyphens and other punctuation follow whitespace but come before letters
219 "\\u0e01\\u0e32", "=", "\\u0e01\\u0e32-",
220 "\\u0e01\\u0e32-", "<", "\\u0e01\\u0e32\\u0e01\\u0e32",
221
222 // Doubler follows an indentical word without the doubler
223 "\\u0e01\\u0e32", "=", "\\u0e01\\u0e32\\u0e46",
224 "\\u0e01\\u0e32\\u0e46", "<", "\\u0e01\\u0e32\\u0e01\\u0e32",
225
226
227 // \\u0e45 after either \\u0e24 or \\u0e26 is treated as a single
228 // combining character, similar to "c < ch" in traditional spanish.
229 // TODO: beef up this case
230 "\\u0e24\\u0e29\\u0e35", "<", "\\u0e24\\u0e45\\u0e29\\u0e35",
231 "\\u0e26\\u0e29\\u0e35", "<", "\\u0e26\\u0e45\\u0e29\\u0e35",
232
233 // Vowels reorder, should compare \\u0e2d and \\u0e34
234 "\\u0e40\\u0e01\\u0e2d", "<", "\\u0e40\\u0e01\\u0e34",
235
236 // Tones are compared after the rest of the word (e.g. primary ignorable)
237 "\\u0e01\\u0e32\\u0e01\\u0e48\\u0e32", "<", "\\u0e01\\u0e49\\u0e32\\u0e01\\u0e32",
238
239 // Periods are ignored entirely
240 "\\u0e01.\\u0e01.", "<", "\\u0e01\\u0e32",
241 };
242 const int32_t TESTS_length = (int32_t)(sizeof(TESTS)/sizeof(TESTS[0]));
243
244 if (coll == 0) {
245 errln("Error: could not construct Thai collator");
246 return;
247 }
248 compareArray(*coll, TESTS, TESTS_length);
249 }
250
251 //------------------------------------------------------------------------
252 // Internal utilities
253 //------------------------------------------------------------------------
254
compareArray(Collator & c,const char * tests[],int32_t testsLength)255 void CollationThaiTest::compareArray(Collator& c, const char* tests[],
256 int32_t testsLength) {
257 for (int32_t i = 0; i < testsLength; i += 3) {
258
259 Collator::EComparisonResult expect;
260 if (tests[i+1][0] == '<') {
261 expect = Collator::LESS;
262 } else if (tests[i+1][0] == '>') {
263 expect = Collator::GREATER;
264 } else if (tests[i+1][0] == '=') {
265 expect = Collator::EQUAL;
266 } else {
267 // expect = Integer.decode(tests[i+1]).intValue();
268 errln((UnicodeString)"Error: unknown operator " + tests[i+1]);
269 return;
270 }
271
272 UnicodeString s1, s2;
273 parseChars(s1, tests[i]);
274 parseChars(s2, tests[i+2]);
275
276 doTest(&c, s1, s2, expect);
277 #if 0
278 UErrorCode status = U_ZERO_ERROR;
279 int32_t result = c.compare(s1, s2);
280 if (sign(result) != sign(expect))
281 {
282 UnicodeString t1, t2;
283 errln(UnicodeString("") +
284 i/3 + ": compare(" + IntlTest::prettify(s1, t1)
285 + " , " + IntlTest::prettify(s2, t2)
286 + ") got " + result + "; expected " + expect);
287
288 CollationKey k1, k2;
289 c.getCollationKey(s1, k1, status);
290 c.getCollationKey(s2, k2, status);
291 if (U_FAILURE(status)) {
292 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
293 return;
294 }
295 errln((UnicodeString)" key1: " + prettify(k1, t1) );
296 errln((UnicodeString)" key2: " + prettify(k2, t2) );
297 }
298 else
299 {
300 // Collator.compare worked OK; now try the collation keys
301 CollationKey k1, k2;
302 c.getCollationKey(s1, k1, status);
303 c.getCollationKey(s2, k2, status);
304 if (U_FAILURE(status)) {
305 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
306 return;
307 }
308
309 result = k1.compareTo(k2);
310 if (sign(result) != sign(expect)) {
311 UnicodeString t1, t2;
312 errln(UnicodeString("") +
313 i/3 + ": key(" + IntlTest::prettify(s1, t1)
314 + ").compareTo(key(" + IntlTest::prettify(s2, t2)
315 + ")) got " + result + "; expected " + expect);
316
317 errln((UnicodeString)" " + prettify(k1, t1) + " vs. " + prettify(k2, t2));
318 }
319 }
320 #endif
321 }
322 }
323
sign(int32_t i)324 int8_t CollationThaiTest::sign(int32_t i) {
325 if (i < 0) return -1;
326 if (i > 0) return 1;
327 return 0;
328 }
329
330 /**
331 * Set a UnicodeString corresponding to the given string. Use
332 * UnicodeString and the default converter, unless we see the sequence
333 * "\\u", in which case we interpret the subsequent escape.
334 */
parseChars(UnicodeString & result,const char * chars)335 UnicodeString& CollationThaiTest::parseChars(UnicodeString& result,
336 const char* chars) {
337 return result = CharsToUnicodeString(chars);
338 }
339
340 UCollator *thaiColl = NULL;
341
342 U_CDECL_BEGIN
343 static int U_CALLCONV
StrCmp(const void * p1,const void * p2)344 StrCmp(const void *p1, const void *p2) {
345 return ucol_strcoll(thaiColl, *(UChar **) p1, -1, *(UChar **)p2, -1);
346 }
347 U_CDECL_END
348
349
350 #define LINES 6
351
TestInvalidThai(void)352 void CollationThaiTest::TestInvalidThai(void) {
353 const char *tests[LINES] = {
354 "\\u0E44\\u0E01\\u0E44\\u0E01",
355 "\\u0E44\\u0E01\\u0E01\\u0E44",
356 "\\u0E01\\u0E44\\u0E01\\u0E44",
357 "\\u0E01\\u0E01\\u0E44\\u0E44",
358 "\\u0E44\\u0E44\\u0E01\\u0E01",
359 "\\u0E01\\u0E44\\u0E44\\u0E01",
360 };
361
362 UChar strings[LINES][20];
363
364 UChar *toSort[LINES];
365
366 int32_t i = 0, j = 0, len = 0;
367
368 UErrorCode coll_status = U_ZERO_ERROR;
369 UnicodeString iteratorText;
370
371 thaiColl = ucol_open ("th_TH", &coll_status);
372 if (U_FAILURE(coll_status)) {
373 errln("Error opening Thai collator: %s", u_errorName(coll_status));
374 return;
375 }
376
377 CollationElementIterator* c = ((RuleBasedCollator *)coll)->createCollationElementIterator( iteratorText );
378
379 for(i = 0; i < (int32_t)(sizeof(tests)/sizeof(tests[0])); i++) {
380 len = u_unescape(tests[i], strings[i], 20);
381 strings[i][len] = 0;
382 toSort[i] = strings[i];
383 }
384
385 qsort (toSort, LINES, sizeof (UChar *), StrCmp);
386
387 for (i=0; i < LINES; i++)
388 {
389 logln("%i", i);
390 for (j=i+1; j < LINES; j++) {
391 if (ucol_strcoll (thaiColl, toSort[i], -1, toSort[j], -1) == UCOL_GREATER)
392 {
393 // inconsistency ordering found!
394 errln("Inconsistent ordering between strings %i and %i", i, j);
395 }
396 }
397 iteratorText.setTo(toSort[i]);
398 c->setText(iteratorText, coll_status);
399 backAndForth(*c);
400 }
401
402
403 ucol_close(thaiColl);
404 delete c;
405 }
406
TestReordering(void)407 void CollationThaiTest::TestReordering(void) {
408 // Until UCA 4.1, the collation code swapped Thai/Lao prevowels with the following consonants,
409 // resulting in consonant+prevowel == prevowel+consonant.
410 // From UCA 5.0 on, there are order-reversing contractions for prevowel+consonant.
411 // From UCA 5.0 until UCA 6.1, there was a tertiary difference between
412 // consonant+prevowel and prevowel+consonant.
413 // In UCA 6.2, they compare equal again.
414 // The test was modified to using a collator with strength=secondary,
415 // ignoring possible tertiary differences.
416 const char *tests[] = {
417 "\\u0E41c\\u0301", "=", "\\u0E41\\u0107", // composition
418 "\\u0E41\\U0001D7CE", "<", "\\u0E41\\U0001D7CF", // supplementaries
419 "\\u0E41\\U0001D15F", "=", "\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary
420 "\\u0E41\\U0002F802", "=", "\\u0E41\\u4E41", // supplementary composition decomps to BMP
421 "\\u0E41\\u0301", "=", "\\u0E41\\u0301", // unsafe (just checking backwards iteration)
422 "\\u0E41\\u0301\\u0316", "=", "\\u0E41\\u0316\\u0301",
423
424 "\\u0e24\\u0e41", "=", "\\u0e41\\u0e24", // exiting contraction bug
425 "\\u0e3f\\u0e3f\\u0e24\\u0e41", "=", "\\u0e3f\\u0e3f\\u0e41\\u0e24",
426
427 "abc\\u0E41c\\u0301", "=", "abc\\u0E41\\u0107", // composition
428 "abc\\u0E41\\U0001D000", "<", "abc\\u0E41\\U0001D001", // supplementaries
429 "abc\\u0E41\\U0001D15F", "=", "abc\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary
430 "abc\\u0E41\\U0002F802", "=", "abc\\u0E41\\u4E41", // supplementary composition decomps to BMP
431 "abc\\u0E41\\u0301", "=", "abc\\u0E41\\u0301", // unsafe (just checking backwards iteration)
432 "abc\\u0E41\\u0301\\u0316", "=", "abc\\u0E41\\u0316\\u0301",
433
434 "\\u0E41c\\u0301abc", "=", "\\u0E41\\u0107abc", // composition
435 "\\u0E41\\U0001D000abc", "<", "\\u0E41\\U0001D001abc", // supplementaries
436 "\\u0E41\\U0001D15Fabc", "=", "\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary
437 "\\u0E41\\U0002F802abc", "=", "\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
438 "\\u0E41\\u0301abc", "=", "\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
439 "\\u0E41\\u0301\\u0316abc", "=", "\\u0E41\\u0316\\u0301abc",
440
441 "abc\\u0E41c\\u0301abc", "=", "abc\\u0E41\\u0107abc", // composition
442 "abc\\u0E41\\U0001D000abc", "<", "abc\\u0E41\\U0001D001abc", // supplementaries
443 "abc\\u0E41\\U0001D15Fabc", "=", "abc\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary
444 "abc\\u0E41\\U0002F802abc", "=", "abc\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
445 "abc\\u0E41\\u0301abc", "=", "abc\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
446 "abc\\u0E41\\u0301\\u0316abc", "=", "abc\\u0E41\\u0316\\u0301abc",
447 };
448
449 LocalPointer<Collator> coll2(coll->clone());
450 UErrorCode status = U_ZERO_ERROR;
451 coll2->setAttribute(UCOL_STRENGTH, UCOL_SECONDARY, status);
452 if(U_FAILURE(status)) {
453 errln("Unable to set the Thai collator clone to secondary strength");
454 return;
455 }
456 compareArray(*coll2, tests, sizeof(tests)/sizeof(tests[0]));
457
458 const char *rule = "& c < ab";
459 const char *testcontraction[] = { "\\u0E41ab", ">", "\\u0E41c"}; // After UCA 4.1 Thai are normal so won't break a contraction
460 UnicodeString rules;
461 parseChars(rules, rule);
462 LocalPointer<RuleBasedCollator> rcoll(new RuleBasedCollator(rules, status), status);
463 if(U_SUCCESS(status)) {
464 compareArray(*rcoll, testcontraction, 3);
465 } else {
466 errln("Couldn't instantiate collator from rules");
467 }
468
469 }
470
471
472 #endif /* #if !UCONFIG_NO_COLLATION */
473