1 package org.unicode.cldr.unittest; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.InputStream; 6 import java.io.PrintWriter; 7 import java.io.StringWriter; 8 import java.util.ArrayList; 9 import java.util.Arrays; 10 import java.util.Collection; 11 import java.util.Collections; 12 import java.util.Comparator; 13 import java.util.HashSet; 14 import java.util.Iterator; 15 import java.util.LinkedHashSet; 16 import java.util.List; 17 import java.util.Map; 18 import java.util.Map.Entry; 19 import java.util.Set; 20 import java.util.TreeMap; 21 import java.util.TreeSet; 22 23 import org.unicode.cldr.test.DisplayAndInputProcessor; 24 import org.unicode.cldr.tool.CldrVersion; 25 import org.unicode.cldr.tool.LikelySubtags; 26 import org.unicode.cldr.util.Builder; 27 import org.unicode.cldr.util.CLDRConfig; 28 import org.unicode.cldr.util.CLDRFile; 29 import org.unicode.cldr.util.CLDRFile.DraftStatus; 30 import org.unicode.cldr.util.CLDRFile.Status; 31 import org.unicode.cldr.util.CLDRFile.WinningChoice; 32 import org.unicode.cldr.util.CLDRPaths; 33 import org.unicode.cldr.util.ChainedMap; 34 import org.unicode.cldr.util.ChainedMap.M4; 35 import org.unicode.cldr.util.CharacterFallbacks; 36 import org.unicode.cldr.util.CldrUtility; 37 import org.unicode.cldr.util.Counter; 38 import org.unicode.cldr.util.DiscreteComparator; 39 import org.unicode.cldr.util.DiscreteComparator.Ordering; 40 import org.unicode.cldr.util.DtdData; 41 import org.unicode.cldr.util.DtdData.Attribute; 42 import org.unicode.cldr.util.DtdData.Element; 43 import org.unicode.cldr.util.DtdData.ElementType; 44 import org.unicode.cldr.util.DtdType; 45 import org.unicode.cldr.util.ElementAttributeInfo; 46 import org.unicode.cldr.util.Factory; 47 import org.unicode.cldr.util.InputStreamFactory; 48 import org.unicode.cldr.util.LanguageTagParser; 49 import org.unicode.cldr.util.Level; 50 import org.unicode.cldr.util.LocaleIDParser; 51 import org.unicode.cldr.util.Pair; 52 import org.unicode.cldr.util.PathHeader; 53 import org.unicode.cldr.util.PathUtilities; 54 import org.unicode.cldr.util.SupplementalDataInfo; 55 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo; 56 import org.unicode.cldr.util.SupplementalDataInfo.PluralType; 57 import org.unicode.cldr.util.XMLFileReader; 58 import org.unicode.cldr.util.XPathParts; 59 import org.xml.sax.ErrorHandler; 60 import org.xml.sax.InputSource; 61 import org.xml.sax.SAXException; 62 import org.xml.sax.SAXParseException; 63 import org.xml.sax.XMLReader; 64 65 import com.google.common.base.Joiner; 66 import com.google.common.base.Objects; 67 import com.google.common.collect.ImmutableMultimap; 68 import com.google.common.collect.ImmutableSet; 69 import com.google.common.collect.Multimap; 70 import com.google.common.collect.TreeMultimap; 71 import com.ibm.icu.impl.Relation; 72 import com.ibm.icu.impl.Row; 73 import com.ibm.icu.impl.Row.R2; 74 import com.ibm.icu.impl.Row.R3; 75 import com.ibm.icu.impl.Utility; 76 import com.ibm.icu.lang.UCharacter; 77 import com.ibm.icu.text.Collator; 78 import com.ibm.icu.text.DecimalFormat; 79 import com.ibm.icu.text.Normalizer; 80 import com.ibm.icu.text.NumberFormat; 81 import com.ibm.icu.text.UTF16; 82 import com.ibm.icu.text.UnicodeSet; 83 import com.ibm.icu.text.UnicodeSetIterator; 84 import com.ibm.icu.util.Currency; 85 import com.ibm.icu.util.ULocale; 86 87 public class TestBasic extends TestFmwkPlus { 88 89 private static final boolean DEBUG = false; 90 91 static CLDRConfig testInfo = CLDRConfig.getInstance(); 92 93 private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = testInfo 94 .getSupplementalDataInfo(); 95 96 private static final ImmutableSet<Pair<String, String>> knownElementExceptions = ImmutableSet.of( 97 Pair.of("ldml", "usesMetazone"), 98 Pair.of("ldmlICU", "usesMetazone")); 99 100 private static final ImmutableSet<Pair<String, String>> knownAttributeExceptions = ImmutableSet.of( 101 Pair.of("ldml", "version"), 102 Pair.of("supplementalData", "version"), 103 Pair.of("ldmlICU", "version"), 104 Pair.of("layout", "standard"), 105 Pair.of("currency", "id"), // for v1.1.1 106 Pair.of("monthNames", "type"), // for v1.1.1 107 Pair.of("alias", "type") // for v1.1.1 108 ); 109 110 private static final ImmutableSet<Pair<String, String>> knownChildExceptions = ImmutableSet.of( 111 Pair.of("abbreviationFallback", "special"), 112 Pair.of("inList", "special"), 113 Pair.of("preferenceOrdering", "special")); 114 115 /** 116 * Simple test that loads each file in the cldr directory, thus verifying 117 * that the DTD works, and also checks that the PrettyPaths work. 118 * 119 * @author markdavis 120 */ 121 main(String[] args)122 public static void main(String[] args) { 123 new TestBasic().run(args); 124 } 125 126 private static final ImmutableSet<String> skipAttributes = ImmutableSet.of( 127 "alt", "draft", "references"); 128 129 private final ImmutableSet<String> eightPointLocales = ImmutableSet.of( 130 "ar", "ca", "cs", "da", "de", "el", "es", "fi", "fr", "he", "hi", "hr", "hu", "id", 131 "it", "ja", "ko", "lt", "lv", "nb", "nl", "pl", "pt", "pt_PT", "ro", "ru", "sk", "sl", "sr", "sv", 132 "th", "tr", "uk", "vi", "zh", "zh_Hant"); 133 134 // private final boolean showForceZoom = Utility.getProperty("forcezoom", 135 // false); 136 137 private final boolean resolved = CldrUtility.getProperty("resolved", false); 138 139 private final Exception[] internalException = new Exception[1]; 140 TestDtds()141 public void TestDtds() throws IOException { 142 Relation<Row.R2<DtdType, String>, String> foundAttributes = Relation 143 .of(new TreeMap<Row.R2<DtdType, String>, Set<String>>(), 144 TreeSet.class); 145 final CLDRConfig config = CLDRConfig.getInstance(); 146 final File basedir = config.getCldrBaseDirectory(); 147 List<TimingInfo> data = new ArrayList<>(); 148 149 for (String subdir : config.getCLDRDataDirectories()) { 150 checkDtds(new File(basedir, subdir), 0, foundAttributes, data); 151 } 152 if (foundAttributes.size() > 0) { 153 showFoundElements(foundAttributes); 154 } 155 if (isVerbose()) { 156 long totalBytes = 0; 157 long totalNanos = 0; 158 for (TimingInfo i : data) { 159 long length = i.file.length(); 160 totalBytes += length; 161 totalNanos += i.nanos; 162 logln(i.nanos + "\t" + length + "\t" + i.file); 163 } 164 logln(totalNanos + "\t" + totalBytes); 165 } 166 } 167 checkDtds(File directoryFile, int level, Relation<R2<DtdType, String>, String> foundAttributes, List<TimingInfo> data)168 private void checkDtds(File directoryFile, int level, 169 Relation<R2<DtdType, String>, String> foundAttributes, 170 List<TimingInfo> data) throws IOException { 171 boolean deepCheck = getInclusion() >= 10; 172 File[] listFiles = directoryFile.listFiles(); 173 String normalizedPath = PathUtilities.getNormalizedPathString(directoryFile); 174 String indent = Utility.repeat("\t", level); 175 if (listFiles == null) { 176 throw new IllegalArgumentException(indent + "Empty directory: " 177 + normalizedPath); 178 } 179 logln("Checking files for DTD errors in: " + indent + normalizedPath); 180 for (File fileName : listFiles) { 181 String name = fileName.getName(); 182 if (CLDRConfig.isJunkFile(name)) { 183 continue; 184 } else if (fileName.isDirectory()) { 185 checkDtds(fileName, level + 1, foundAttributes, data); 186 } else if (name.endsWith(".xml")) { 187 data.add(check(fileName)); 188 if (deepCheck // takes too long to do all the time 189 ) { 190 CLDRFile cldrfile = CLDRFile.loadFromFile(fileName, "temp", 191 DraftStatus.unconfirmed); 192 for (String xpath : cldrfile) { 193 String fullPath = cldrfile.getFullXPath(xpath); 194 if (fullPath == null) { 195 fullPath = cldrfile.getFullXPath(xpath); 196 assertNotNull("", fullPath); 197 continue; 198 } 199 XPathParts parts = XPathParts 200 .getFrozenInstance(fullPath); 201 DtdType type = parts.getDtdData().dtdType; 202 for (int i = 0; i < parts.size(); ++i) { 203 String element = parts.getElement(i); 204 R2<DtdType, String> typeElement = Row.of(type, 205 element); 206 if (parts.getAttributeCount(i) == 0) { 207 foundAttributes.put(typeElement, "NONE"); 208 } else { 209 for (String attribute : parts 210 .getAttributeKeys(i)) { 211 foundAttributes.put(typeElement, attribute); 212 } 213 } 214 } 215 } 216 } 217 } 218 } 219 } 220 showFoundElements( Relation<Row.R2<DtdType, String>, String> foundAttributes)221 public void showFoundElements( 222 Relation<Row.R2<DtdType, String>, String> foundAttributes) { 223 Relation<Row.R2<DtdType, String>, String> theoryAttributes = Relation 224 .of(new TreeMap<Row.R2<DtdType, String>, Set<String>>(), 225 TreeSet.class); 226 for (DtdType type : DtdType.values()) { 227 DtdData dtdData = DtdData.getInstance(type); 228 for (Element element : dtdData.getElementFromName().values()) { 229 String name = element.getName(); 230 Set<Attribute> attributes = element.getAttributes().keySet(); 231 R2<DtdType, String> typeElement = Row.of(type, name); 232 if (attributes.isEmpty()) { 233 theoryAttributes.put(typeElement, "NONE"); 234 } else { 235 for (Attribute attribute : attributes) { 236 theoryAttributes.put(typeElement, attribute.name); 237 } 238 } 239 } 240 } 241 Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed = Relation 242 .of(new TreeMap<String, Set<R3<Boolean, DtdType, String>>>(), 243 LinkedHashSet.class); 244 245 for (Entry<R2<DtdType, String>, Set<String>> s : theoryAttributes 246 .keyValuesSet()) { 247 R2<DtdType, String> typeElement = s.getKey(); 248 Set<String> theoryAttributeSet = s.getValue(); 249 DtdType type = typeElement.get0(); 250 String element = typeElement.get1(); 251 if (element.equals("ANY") || element.equals("#PCDATA")) { 252 continue; 253 } 254 boolean deprecatedElement = SUPPLEMENTAL_DATA_INFO.isDeprecated( 255 type, element, "*", "*"); 256 String header = type + "\t" + element + "\t" 257 + (deprecatedElement ? "X" : "") + "\t"; 258 Set<String> usedAttributes = foundAttributes.get(typeElement); 259 Set<String> unusedAttributes = new LinkedHashSet<String>( 260 theoryAttributeSet); 261 if (usedAttributes == null) { 262 logln(header 263 + "<NOT-FOUND>\t\t" 264 + siftDeprecated(type, element, unusedAttributes, 265 attributesToTypeElementUsed, false)); 266 continue; 267 } 268 unusedAttributes.removeAll(usedAttributes); 269 logln(header 270 + siftDeprecated(type, element, usedAttributes, 271 attributesToTypeElementUsed, true) 272 + "\t" 273 + siftDeprecated(type, element, unusedAttributes, 274 attributesToTypeElementUsed, false)); 275 } 276 277 logln("Undeprecated Attributes\t"); 278 for (Entry<String, R3<Boolean, DtdType, String>> s : attributesToTypeElementUsed 279 .keyValueSet()) { 280 R3<Boolean, DtdType, String> typeElementUsed = s.getValue(); 281 logln(s.getKey() + "\t" + typeElementUsed.get0() 282 + "\t" + typeElementUsed.get1() + "\t" 283 + typeElementUsed.get2()); 284 } 285 } 286 siftDeprecated( DtdType type, String element, Set<String> attributeSet, Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed, boolean used)287 private String siftDeprecated( 288 DtdType type, 289 String element, 290 Set<String> attributeSet, 291 Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed, 292 boolean used) { 293 StringBuilder b = new StringBuilder(); 294 StringBuilder bdep = new StringBuilder(); 295 for (String attribute : attributeSet) { 296 String attributeName = "«" 297 + attribute 298 + (!"NONE".equals(attribute) && CLDRFile.isDistinguishing(type, element, attribute) ? "*" 299 : "") 300 + "»"; 301 if (!"NONE".equals(attribute) && SUPPLEMENTAL_DATA_INFO.isDeprecated(type, element, attribute, 302 "*")) { 303 if (bdep.length() != 0) { 304 bdep.append(" "); 305 } 306 bdep.append(attributeName); 307 } else { 308 if (b.length() != 0) { 309 b.append(" "); 310 } 311 b.append(attributeName); 312 if (!"NONE".equals(attribute)) { 313 attributesToTypeElementUsed.put(attribute, 314 Row.of(used, type, element)); 315 } 316 } 317 } 318 return b.toString() + "\t" + bdep.toString(); 319 } 320 321 class MyErrorHandler implements ErrorHandler { 322 @Override error(SAXParseException exception)323 public void error(SAXParseException exception) throws SAXException { 324 errln("error: " + XMLFileReader.showSAX(exception)); 325 throw exception; 326 } 327 328 @Override fatalError(SAXParseException exception)329 public void fatalError(SAXParseException exception) throws SAXException { 330 errln("fatalError: " + XMLFileReader.showSAX(exception)); 331 throw exception; 332 } 333 334 @Override warning(SAXParseException exception)335 public void warning(SAXParseException exception) throws SAXException { 336 errln("warning: " + XMLFileReader.showSAX(exception)); 337 throw exception; 338 } 339 } 340 341 private class TimingInfo { 342 File file; 343 long nanos; 344 } 345 check(File systemID)346 public TimingInfo check(File systemID) { 347 long start = System.nanoTime(); 348 try (InputStream fis = InputStreamFactory.createInputStream(systemID)) { 349 // FileInputStream fis = new FileInputStream(systemID); 350 XMLReader xmlReader = XMLFileReader.createXMLReader(true); 351 xmlReader.setErrorHandler(new MyErrorHandler()); 352 InputSource is = new InputSource(fis); 353 is.setSystemId(systemID.toString()); 354 xmlReader.parse(is); 355 // fis.close(); 356 } catch (SAXException | IOException e) { 357 errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" 358 + e.getMessage()); 359 } 360 // catch (SAXParseException e) { 361 // errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" + 362 // e.getMessage()); 363 // } catch (IOException e) { 364 // errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" + 365 // e.getMessage()); 366 // } 367 TimingInfo timingInfo = new TimingInfo(); 368 timingInfo.nanos = System.nanoTime() - start; 369 timingInfo.file = systemID; 370 return timingInfo; 371 } 372 TestCurrencyFallback()373 public void TestCurrencyFallback() { 374 Factory cldrFactory = testInfo.getCldrFactory(); 375 Set<String> currencies = testInfo.getStandardCodes().getAvailableCodes( 376 "currency"); 377 378 final UnicodeSet CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS = new UnicodeSet( 379 "[[:sc:]-[\\u0000-\\u00FF]]").freeze(); 380 381 CharacterFallbacks fallbacks = CharacterFallbacks.make(); 382 383 for (String locale : cldrFactory.getAvailable()) { 384 CLDRFile file = testInfo.getCLDRFile(locale, false); 385 if (file.isNonInheriting()) 386 continue; 387 388 final UnicodeSet OK_CURRENCY_FALLBACK = new UnicodeSet( 389 "[\\u0000-\\u00FF]").addAll(safeExemplars(file, "")) 390 .addAll(safeExemplars(file, "auxiliary")) 391 .freeze(); 392 UnicodeSet badSoFar = new UnicodeSet(); 393 394 for (Iterator<String> it = file.iterator(); it.hasNext();) { 395 String path = it.next(); 396 if (path.endsWith("/alias")) { 397 continue; 398 } 399 String value = file.getStringValue(path); 400 401 // check for special characters 402 if (CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS.containsSome(value)) { 403 XPathParts parts = XPathParts.getFrozenInstance(path); 404 if (!parts.getElement(-1).equals("symbol")) { 405 continue; 406 } 407 // We don't care about fallbacks for narrow currency symbols 408 if ("narrow".equals(parts.getAttributeValue(-1, "alt"))) { 409 continue; 410 } 411 String currencyType = parts.getAttributeValue(-2, "type"); 412 413 UnicodeSet fishy = new UnicodeSet().addAll(value) 414 .retainAll(CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS) 415 .removeAll(badSoFar); 416 for (UnicodeSetIterator it2 = new UnicodeSetIterator(fishy); it2 417 .next();) { 418 final int fishyCodepoint = it2.codepoint; 419 List<String> fallbackList = fallbacks 420 .getSubstitutes(fishyCodepoint); 421 422 String nfkc = Normalizer.normalize(fishyCodepoint, Normalizer.NFKC); 423 if (!nfkc.equals(UTF16.valueOf(fishyCodepoint))) { 424 if (fallbackList == null) { 425 fallbackList = new ArrayList<String>(); 426 } else { 427 fallbackList = new ArrayList<String>( 428 fallbackList); // writable 429 } 430 fallbackList.add(nfkc); 431 } 432 // later test for all Latin-1 433 if (fallbackList == null) { 434 errln("Locale:\t" + locale 435 + ";\tCharacter with no fallback:\t" 436 + it2.getString() + "\t" 437 + UCharacter.getName(fishyCodepoint)); 438 badSoFar.add(fishyCodepoint); 439 } else { 440 String fallback = null; 441 for (String fb : fallbackList) { 442 if (OK_CURRENCY_FALLBACK.containsAll(fb)) { 443 if (!fb.equals(currencyType) 444 && currencies.contains(fb)) { 445 errln("Locale:\t" 446 + locale 447 + ";\tCurrency:\t" 448 + currencyType 449 + ";\tFallback converts to different code!:\t" 450 + fb 451 + "\t" 452 + it2.getString() 453 + "\t" 454 + UCharacter 455 .getName(fishyCodepoint)); 456 } 457 if (fallback == null) { 458 fallback = fb; 459 } 460 } 461 } 462 if (fallback == null) { 463 errln("Locale:\t" 464 + locale 465 + ";\tCharacter with no good fallback (exemplars+Latin1):\t" 466 + it2.getString() + "\t" 467 + UCharacter.getName(fishyCodepoint)); 468 badSoFar.add(fishyCodepoint); 469 } else { 470 logln("Locale:\t" + locale 471 + ";\tCharacter with good fallback:\t" 472 + it2.getString() + " " 473 + UCharacter.getName(fishyCodepoint) 474 + " => " + fallback); 475 // badSoFar.add(fishyCodepoint); 476 } 477 } 478 } 479 } 480 } 481 } 482 } 483 TestAbstractPaths()484 public void TestAbstractPaths() { 485 Factory cldrFactory = testInfo.getCldrFactory(); 486 CLDRFile english = testInfo.getEnglish(); 487 Map<String, Counter<Level>> abstactPaths = new TreeMap<String, Counter<Level>>(); 488 RegexTransform abstractPathTransform = new RegexTransform( 489 RegexTransform.Processing.ONE_PASS).add("//ldml/", "") 490 .add("\\[@alt=\"[^\"]*\"\\]", "").add("=\"[^\"]*\"", "=\"*\"") 491 .add("([^]])\\[", "$1\t[").add("([^]])/", "$1\t/") 492 .add("/", "\t"); 493 494 for (String locale : getInclusion() <= 5 ? eightPointLocales : cldrFactory.getAvailable()) { 495 CLDRFile file = testInfo.getCLDRFile(locale, resolved); 496 if (file.isNonInheriting()) 497 continue; 498 logln(locale + "\t-\t" + english.getName(locale)); 499 500 for (Iterator<String> it = file.iterator(); it.hasNext();) { 501 String path = it.next(); 502 if (path.endsWith("/alias")) { 503 continue; 504 } 505 // collect abstracted paths 506 String abstractPath = abstractPathTransform.transform(path); 507 Level level = SUPPLEMENTAL_DATA_INFO.getCoverageLevel(path, 508 locale); 509 if (level == Level.OPTIONAL) { 510 level = Level.COMPREHENSIVE; 511 } 512 Counter<Level> row = abstactPaths.get(abstractPath); 513 if (row == null) { 514 abstactPaths.put(abstractPath, row = new Counter<Level>()); 515 } 516 row.add(level, 1); 517 } 518 } 519 logln(CldrUtility.LINE_SEPARATOR + "Abstract Paths"); 520 for (Entry<String, Counter<Level>> pathInfo : abstactPaths.entrySet()) { 521 String path = pathInfo.getKey(); 522 Counter<Level> counter = pathInfo.getValue(); 523 logln(counter.getTotal() + "\t" + getCoverage(counter) + "\t" 524 + path); 525 } 526 } 527 getCoverage(Counter<Level> counter)528 private CharSequence getCoverage(Counter<Level> counter) { 529 StringBuilder result = new StringBuilder(); 530 boolean first = true; 531 for (Level level : counter.getKeysetSortedByKey()) { 532 if (first) { 533 first = false; 534 } else { 535 result.append(' '); 536 } 537 result.append("L").append(level.ordinal()).append("=") 538 .append(counter.get(level)); 539 } 540 return result; 541 } 542 543 // public void TestCLDRFileCache() { 544 // long start = System.nanoTime(); 545 // Factory cldrFactory = testInfo.getCldrFactory(); 546 // String unusualLocale = "hi"; 547 // CLDRFile file = cldrFactory.make(unusualLocale, true); 548 // long afterOne = System.nanoTime(); 549 // logln("First: " + (afterOne-start)); 550 // CLDRFile file2 = cldrFactory.make(unusualLocale, true); 551 // long afterTwo = System.nanoTime(); 552 // logln("Second: " + (afterTwo-afterOne)); 553 // } 554 // TestPaths()555 public void TestPaths() { 556 Relation<String, String> distinguishing = Relation.of( 557 new TreeMap<String, Set<String>>(), TreeSet.class); 558 Relation<String, String> nonDistinguishing = Relation.of( 559 new TreeMap<String, Set<String>>(), TreeSet.class); 560 Factory cldrFactory = testInfo.getCldrFactory(); 561 CLDRFile english = testInfo.getEnglish(); 562 563 Relation<String, String> pathToLocale = Relation.of( 564 new TreeMap<String, Set<String>>(CLDRFile 565 .getComparator(DtdType.ldml)), 566 TreeSet.class, null); 567 Set<String> localesToTest = getInclusion() <= 5 ? eightPointLocales : cldrFactory.getAvailable(); 568 for (String locale : localesToTest) { 569 CLDRFile file = testInfo.getCLDRFile(locale, resolved); 570 DtdType dtdType = null; 571 if (file.isNonInheriting()) 572 continue; 573 DisplayAndInputProcessor displayAndInputProcessor = new DisplayAndInputProcessor( 574 file, false); 575 576 logln(locale + "\t-\t" + english.getName(locale)); 577 578 for (Iterator<String> it = file.iterator(); it.hasNext();) { 579 String path = it.next(); 580 if (dtdType == null) { 581 dtdType = DtdType.fromPath(path); 582 } 583 584 if (path.endsWith("/alias")) { 585 continue; 586 } 587 String value = file.getStringValue(path); 588 if (value == null) { 589 throw new IllegalArgumentException(locale 590 + "\tError: in null value at " + path); 591 } 592 593 String displayValue = displayAndInputProcessor 594 .processForDisplay(path, value); 595 if (!displayValue.equals(value)) { 596 logln("\t" 597 + locale 598 + "\tdisplayAndInputProcessor changes display value <" 599 + value + ">\t=>\t<" + displayValue + ">\t\t" 600 + path); 601 } 602 String inputValue = displayAndInputProcessor.processInput(path, 603 value, internalException); 604 if (internalException[0] != null) { 605 errln("\t" + locale 606 + "\tdisplayAndInputProcessor internal error <" 607 + value + ">\t=>\t<" + inputValue + ">\t\t" + path); 608 internalException[0].printStackTrace(System.out); 609 } 610 if (isVerbose() && !inputValue.equals(value)) { 611 displayAndInputProcessor.processInput(path, value, 612 internalException); // for 613 // debugging 614 logln("\t" 615 + locale 616 + "\tdisplayAndInputProcessor changes input value <" 617 + value + ">\t=>\t<" + inputValue + ">\t\t" + path); 618 } 619 620 pathToLocale.put(path, locale); 621 622 // also check for non-distinguishing attributes 623 if (path.contains("/identity")) 624 continue; 625 626 String fullPath = file.getFullXPath(path); 627 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 628 for (int i = 0; i < parts.size(); ++i) { 629 if (parts.getAttributeCount(i) == 0) { 630 continue; 631 } 632 String element = parts.getElement(i); 633 for (String attribute : parts.getAttributeKeys(i)) { 634 if (skipAttributes.contains(attribute)) 635 continue; 636 if (CLDRFile.isDistinguishing(dtdType, element, attribute)) { 637 distinguishing.put(element, attribute); 638 } else { 639 nonDistinguishing.put(element, attribute); 640 } 641 } 642 } 643 } 644 } 645 646 if (isVerbose()) { 647 System.out.format("Distinguishing Elements: %s" 648 + CldrUtility.LINE_SEPARATOR, distinguishing); 649 System.out.format("Nondistinguishing Elements: %s" 650 + CldrUtility.LINE_SEPARATOR, nonDistinguishing); 651 System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, 652 skipAttributes); 653 } 654 } 655 656 /** 657 * The verbose output shows the results of 1..3 \u00a4 signs. 658 */ checkCurrency()659 public void checkCurrency() { 660 Map<String, Set<R2<String, Integer>>> results = new TreeMap<String, Set<R2<String, Integer>>>( 661 Collator.getInstance(ULocale.ENGLISH)); 662 for (ULocale locale : ULocale.getAvailableLocales()) { 663 if (locale.getCountry().length() != 0) { 664 continue; 665 } 666 for (int i = 1; i < 4; ++i) { 667 NumberFormat format = getCurrencyInstance(locale, i); 668 for (Currency c : new Currency[] { Currency.getInstance("USD"), 669 Currency.getInstance("EUR"), 670 Currency.getInstance("INR") }) { 671 format.setCurrency(c); 672 final String formatted = format.format(12345.67); 673 Set<R2<String, Integer>> set = results.get(formatted); 674 if (set == null) { 675 results.put(formatted, 676 set = new TreeSet<R2<String, Integer>>()); 677 } 678 set.add(Row.of(locale.toString(), Integer.valueOf(i))); 679 } 680 } 681 } 682 for (String formatted : results.keySet()) { 683 logln(formatted + "\t" + results.get(formatted)); 684 } 685 } 686 getCurrencyInstance(ULocale locale, int type)687 private static NumberFormat getCurrencyInstance(ULocale locale, int type) { 688 NumberFormat format = NumberFormat.getCurrencyInstance(locale); 689 if (type > 1) { 690 DecimalFormat format2 = (DecimalFormat) format; 691 String pattern = format2.toPattern(); 692 String replacement = "\u00a4\u00a4"; 693 for (int i = 2; i < type; ++i) { 694 replacement += "\u00a4"; 695 } 696 pattern = pattern.replace("\u00a4", replacement); 697 format2.applyPattern(pattern); 698 } 699 return format; 700 } 701 safeExemplars(CLDRFile file, String string)702 private UnicodeSet safeExemplars(CLDRFile file, String string) { 703 final UnicodeSet result = file.getExemplarSet(string, 704 WinningChoice.NORMAL); 705 return result != null ? result : new UnicodeSet(); 706 } 707 TestAPath()708 public void TestAPath() { 709 // <month type="1">1</month> 710 String path = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month[@type=\"1\"]"; 711 CLDRFile root = testInfo.getRoot(); 712 logln("path: " + path); 713 String fullpath = root.getFullXPath(path); 714 logln("fullpath: " + fullpath); 715 String value = root.getStringValue(path); 716 logln("value: " + value); 717 Status status = new Status(); 718 String source = root.getSourceLocaleID(path, status); 719 logln("locale: " + source); 720 logln("status: " + status); 721 } 722 TestDefaultContents()723 public void TestDefaultContents() { 724 Set<String> defaultContents = Inheritance.defaultContents; 725 Multimap<String, String> parentToChildren = Inheritance.parentToChildren; 726 727 // Put a list of locales that should be default content here. 728 final String expectDC[] = { 729 "os_GE" // see CLDR-14118 730 }; 731 for(final String locale : expectDC) { 732 assertTrue("expect "+locale+" to be a default content locale", defaultContents.contains(locale)); 733 } 734 735 if (DEBUG) { 736 Inheritance.showChain("", "", "root"); 737 } 738 739 for (String locale : defaultContents) { 740 CLDRFile cldrFile; 741 try { 742 cldrFile = testInfo.getCLDRFile(locale, false); 743 } catch (RuntimeException e) { 744 logln("Can't open default content file:\t" + locale); 745 continue; 746 } 747 // we check that the default content locale is always empty 748 for (Iterator<String> it = cldrFile.iterator(); it.hasNext();) { 749 String path = it.next(); 750 if (path.contains("/identity")) { 751 continue; 752 } 753 errln("Default content file not empty:\t" + locale); 754 showDifferences(locale); 755 break; 756 } 757 } 758 759 // check that if a locale has any children, that exactly one of them is 760 // the default content. Ignore locales with variants 761 762 for (Entry<String, Collection<String>> localeAndKids : parentToChildren.asMap().entrySet()) { 763 String locale = localeAndKids.getKey(); 764 if (locale.equals("root")) { 765 continue; 766 } 767 768 Collection<String> rawChildren = localeAndKids.getValue(); 769 770 // remove variant children 771 Set<String> children = new LinkedHashSet<>(); 772 for (String child : rawChildren) { 773 if (new LocaleIDParser().set(child).getVariants().length == 0) { 774 children.add(child); 775 } 776 } 777 if (children.isEmpty()) { 778 continue; 779 } 780 781 Set<String> defaultContentChildren = new LinkedHashSet<String>(children); 782 defaultContentChildren.retainAll(defaultContents); 783 if (defaultContentChildren.size() == 1) { 784 continue; 785 // If we're already down to the region level then it's OK not to have 786 // default contents. 787 } else if (! new LocaleIDParser().set(locale).getRegion().isEmpty()) { 788 continue; 789 } else if (defaultContentChildren.isEmpty()) { 790 Object possible = highestShared(locale, children); 791 errln("Locale has children but is missing default contents locale: " 792 + locale + ", children: " + children + "; possible fixes for children:\n" + possible); 793 } else { 794 errln("Locale has too many defaultContent locales!!: " 795 + locale + ", defaultContents: " 796 + defaultContentChildren); 797 } 798 } 799 800 // check that each default content locale is likely-subtag equivalent to 801 // its parent. 802 803 for (String locale : defaultContents) { 804 String maxLocale = LikelySubtags.maximize(locale, likelyData); 805 String localeParent = LocaleIDParser.getParent(locale); 806 String maxLocaleParent = LikelySubtags.maximize(localeParent, 807 likelyData); 808 if (locale.equals("ar_001")) { 809 logln("Known exception to likelyMax(locale=" + locale + ")" 810 + " == " + "likelyMax(defaultContent=" + localeParent 811 + ")"); 812 continue; 813 } 814 assertEquals("likelyMax(locale=" + locale + ")" + " == " 815 + "likelyMax(defaultContent=" + localeParent + ")", 816 maxLocaleParent, maxLocale); 817 } 818 819 } 820 highestShared(String parent, Set<String> children)821 private String highestShared(String parent, Set<String> children) { 822 M4<PathHeader, String, String, Boolean> data = ChainedMap.of(new TreeMap<PathHeader, Object>(), new TreeMap<String, Object>(), 823 new TreeMap<String, Object>(), Boolean.class); 824 CLDRFile parentFile = testInfo.getCLDRFile(parent, true); 825 PathHeader.Factory phf = PathHeader.getFactory(testInfo.getEnglish()); 826 for (String child : children) { 827 CLDRFile cldrFile = testInfo.getCLDRFile(child, false); 828 for (String path : cldrFile) { 829 if (path.contains("/identity")) { 830 continue; 831 } 832 if (path.contains("provisional") || path.contains("unconfirmed")) { 833 continue; 834 } 835 String value = cldrFile.getStringValue(path); 836 // double-check 837 String parentValue = parentFile.getStringValue(path); 838 if (value.equals(parentValue)) { 839 continue; 840 } 841 PathHeader ph = phf.fromPath(path); 842 data.put(ph, value, child, Boolean.TRUE); 843 data.put(ph, parentValue == null ? "∅∅∅" : parentValue, child, Boolean.TRUE); 844 } 845 } 846 StringBuilder result = new StringBuilder(); 847 for (Entry<PathHeader, Map<String, Map<String, Boolean>>> entry : data) { 848 for (Entry<String, Map<String, Boolean>> item : entry.getValue().entrySet()) { 849 result.append("\n") 850 .append(entry.getKey()) 851 .append("\t") 852 .append(item.getKey() + "\t" + item.getValue().keySet()); 853 } 854 } 855 return result.toString(); 856 } 857 858 public static class Inheritance { 859 public static final Set<String> defaultContents = SUPPLEMENTAL_DATA_INFO 860 .getDefaultContentLocales(); 861 public static final Multimap<String, String> parentToChildren; 862 863 static { 864 Multimap<String, String> _parentToChildren = TreeMultimap.create(); 865 for (String child : testInfo.getCldrFactory().getAvailable()) { 866 if (child.equals("root")) { 867 continue; 868 } 869 String localeParent = LocaleIDParser.getParent(child); _parentToChildren.put(localeParent, child)870 _parentToChildren.put(localeParent, child); 871 } 872 parentToChildren = ImmutableMultimap.copyOf(_parentToChildren); 873 } 874 showChain(String prefix, String gparent, String current)875 public static void showChain(String prefix, String gparent, String current) { 876 Collection<String> children = parentToChildren.get(current); 877 if (children == null) { 878 throw new IllegalArgumentException(); 879 } 880 prefix += current + (defaultContents.contains(current) ? "*" : "") 881 + (isLikelyEquivalent(gparent, current) ? "~" : "") + "\t"; 882 883 // find leaves 884 Set<String> parents = new LinkedHashSet<>(children); 885 parents.retainAll(parentToChildren.keySet()); 886 Set<String> leaves = new LinkedHashSet<>(children); 887 leaves.removeAll(parentToChildren.keySet()); 888 if (!leaves.isEmpty()) { 889 List<String> presentation = new ArrayList<>(); 890 boolean gotDc = false; 891 for (String s : leaves) { 892 String shown = s; 893 if (isLikelyEquivalent(current, s)) { 894 shown += "~"; 895 } 896 if (defaultContents.contains(s)) { 897 gotDc = true; 898 shown += "*"; 899 } 900 if (!shown.equals(s)) { 901 presentation.add(0, shown); 902 } else { 903 presentation.add(shown); 904 } 905 } 906 if (!gotDc) { 907 int debug = 0; 908 } 909 if (leaves.size() == 1) { 910 System.out.println(prefix + Joiner.on(" ").join(presentation)); 911 } else { 912 System.out.println(prefix + "{" + Joiner.on(" ").join(presentation) + "}"); 913 } 914 } 915 for (String parent : parents) { 916 showChain(prefix, current, parent); 917 } 918 } 919 isLikelyEquivalent(String locale1, String locale2)920 static boolean isLikelyEquivalent(String locale1, String locale2) { 921 if (locale1.equals(locale2)) { 922 return true; 923 } 924 try { 925 String maxLocale1 = LikelySubtags.maximize(locale1, likelyData); 926 String maxLocale2 = LikelySubtags.maximize(locale2, likelyData); 927 return maxLocale1 != null && Objects.equal(maxLocale1, maxLocale2); 928 } catch (Exception e) { 929 return false; 930 } 931 } 932 } 933 934 static final Map<String, String> likelyData = SUPPLEMENTAL_DATA_INFO 935 .getLikelySubtags(); 936 TestLikelySubtagsComplete()937 public void TestLikelySubtagsComplete() { 938 LanguageTagParser ltp = new LanguageTagParser(); 939 for (String locale : testInfo.getCldrFactory().getAvailable()) { 940 if (locale.equals("root")) { 941 continue; 942 } 943 String maxLocale = LikelySubtags.maximize(locale, likelyData); 944 if (maxLocale == null) { 945 errln("Locale missing likely subtag: " + locale); 946 continue; 947 } 948 ltp.set(maxLocale); 949 if (ltp.getLanguage().isEmpty() || ltp.getScript().isEmpty() 950 || ltp.getRegion().isEmpty()) { 951 errln("Locale has defective likely subtag: " + locale + " => " 952 + maxLocale); 953 } 954 } 955 } 956 showDifferences(String locale)957 private void showDifferences(String locale) { 958 CLDRFile cldrFile = testInfo.getCLDRFile(locale, false); 959 final String localeParent = LocaleIDParser.getParent(locale); 960 CLDRFile parentFile = testInfo.getCLDRFile(localeParent, true); 961 int funnyCount = 0; 962 for (Iterator<String> it = cldrFile.iterator("", 963 cldrFile.getComparator()); it.hasNext();) { 964 String path = it.next(); 965 if (path.contains("/identity")) { 966 continue; 967 } 968 final String fullXPath = cldrFile.getFullXPath(path); 969 if (fullXPath.contains("[@draft=\"unconfirmed\"]") 970 || fullXPath.contains("[@draft=\"provisional\"]")) { 971 funnyCount++; 972 continue; 973 } 974 logln("\tpath:\t" + path); 975 logln("\t\t" + locale + " value:\t<" 976 + cldrFile.getStringValue(path) + ">"); 977 final String parentFullPath = parentFile.getFullXPath(path); 978 logln("\t\t" + localeParent + " value:\t<" 979 + parentFile.getStringValue(path) + ">"); 980 logln("\t\t" + locale + " fullpath:\t" + fullXPath); 981 logln("\t\t" + localeParent + " fullpath:\t" + parentFullPath); 982 } 983 logln("\tCount of non-approved:\t" + funnyCount); 984 } 985 986 enum MissingType { 987 plurals, main_exemplars, no_main, collation, index_exemplars, punct_exemplars 988 } 989 TestCoreData()990 public void TestCoreData() { 991 Set<String> availableLanguages = testInfo.getCldrFactory() 992 .getAvailableLanguages(); 993 PluralInfo rootRules = SUPPLEMENTAL_DATA_INFO.getPlurals( 994 PluralType.cardinal, "root"); 995 Multimap<MissingType, Comparable> errors = TreeMultimap.create(); 996 errors.put(MissingType.collation, "?"); 997 998 Multimap<MissingType, Comparable> warnings = TreeMultimap.create(); 999 warnings.put(MissingType.collation, "?"); 1000 warnings.put(MissingType.index_exemplars, "?"); 1001 warnings.put(MissingType.punct_exemplars, "?"); 1002 1003 Set<String> collations = new HashSet<String>(); 1004 1005 // collect collation info 1006 Factory collationFactory = Factory.make(CLDRPaths.COLLATION_DIRECTORY, 1007 ".*", DraftStatus.contributed); 1008 for (String localeID : collationFactory.getAvailable()) { 1009 if (isTopLevel(localeID)) { 1010 collations.add(localeID); 1011 } 1012 } 1013 logln(collations.toString()); 1014 1015 Set<String> allLanguages = Builder.with(new TreeSet<String>()) 1016 .addAll(collations).addAll(availableLanguages).freeze(); 1017 1018 for (String localeID : allLanguages) { 1019 if (localeID.equals("root")) { 1020 continue; // skip script locales 1021 } 1022 if (!isTopLevel(localeID)) { 1023 continue; 1024 } 1025 1026 errors.clear(); 1027 warnings.clear(); 1028 1029 String name = "Locale:" + localeID + " (" 1030 + testInfo.getEnglish().getName(localeID) + ")"; 1031 1032 if (!collations.contains(localeID)) { 1033 warnings.put(MissingType.collation, "missing"); 1034 logln(name + " is missing " + MissingType.collation.toString()); 1035 } 1036 1037 try { 1038 CLDRFile cldrFile = testInfo.getCldrFactory().make(localeID, 1039 false, DraftStatus.contributed); 1040 1041 String wholeFileAlias = cldrFile.getStringValue("//ldml/alias"); 1042 if (wholeFileAlias != null) { 1043 logln("Whole-file alias:" + name); 1044 continue; 1045 } 1046 1047 PluralInfo pluralInfo = SUPPLEMENTAL_DATA_INFO.getPlurals( 1048 PluralType.cardinal, localeID); 1049 if (pluralInfo == rootRules) { 1050 logln(name + " is missing " 1051 + MissingType.plurals.toString()); 1052 warnings.put(MissingType.plurals, "missing"); 1053 } 1054 UnicodeSet main = cldrFile.getExemplarSet("", 1055 WinningChoice.WINNING); 1056 if (main == null || main.isEmpty()) { 1057 errln(" " + name + " is missing " 1058 + MissingType.main_exemplars.toString()); 1059 errors.put(MissingType.main_exemplars, "missing"); 1060 } 1061 UnicodeSet index = cldrFile.getExemplarSet("index", 1062 WinningChoice.WINNING); 1063 if (index == null || index.isEmpty()) { 1064 logln(name + " is missing " 1065 + MissingType.index_exemplars.toString()); 1066 warnings.put(MissingType.index_exemplars, "missing"); 1067 } 1068 UnicodeSet punctuation = cldrFile.getExemplarSet("punctuation", 1069 WinningChoice.WINNING); 1070 if (punctuation == null || punctuation.isEmpty()) { 1071 logln(name + " is missing " 1072 + MissingType.punct_exemplars.toString()); 1073 warnings.put(MissingType.punct_exemplars, "missing"); 1074 } 1075 } catch (Exception e) { 1076 StringWriter x = new StringWriter(); 1077 PrintWriter pw = new PrintWriter(x); 1078 e.printStackTrace(pw); 1079 pw.flush(); 1080 errln(" " + name + " is missing main locale data." + x); 1081 errors.put(MissingType.no_main, x.toString()); 1082 } 1083 1084 // report errors 1085 1086 if (errors.isEmpty() && warnings.isEmpty()) { 1087 logln(name + ": No problems..."); 1088 } 1089 } 1090 } 1091 isTopLevel(String localeID)1092 private boolean isTopLevel(String localeID) { 1093 return "root".equals(LocaleIDParser.getParent(localeID)); 1094 } 1095 1096 /** 1097 * Tests that every dtd item is connected from root 1098 */ TestDtdCompleteness()1099 public void TestDtdCompleteness() { 1100 for (DtdType type : DtdType.values()) { 1101 DtdData dtdData = DtdData.getInstance(type); 1102 Set<Element> descendents = new LinkedHashSet<Element>(); 1103 dtdData.getDescendents(dtdData.ROOT, descendents); 1104 Set<Element> elements = dtdData.getElements(); 1105 if (!elements.equals(descendents)) { 1106 for (Element e : elements) { 1107 if (!descendents.contains(e) && !e.equals(dtdData.PCDATA) 1108 && !e.equals(dtdData.ANY)) { 1109 errln(type + ": Element " + e 1110 + " not contained in descendents of ROOT."); 1111 } 1112 } 1113 for (Element e : descendents) { 1114 if (!elements.contains(e)) { 1115 errln(type + ": Element " + e 1116 + ", descendent of ROOT, not in elements."); 1117 } 1118 } 1119 } 1120 LinkedHashSet<Element> all = new LinkedHashSet<Element>(descendents); 1121 all.addAll(elements); 1122 Set<Attribute> attributes = dtdData.getAttributes(); 1123 for (Attribute a : attributes) { 1124 if (!elements.contains(a.element)) { 1125 errln(type + ": Attribute " + a + " isn't for any element."); 1126 } 1127 } 1128 } 1129 } 1130 TestBasicDTDCompatibility()1131 public void TestBasicDTDCompatibility() { 1132 1133 if (logKnownIssue("cldrbug:11583", "Comment out test until last release data is available for unit tests")) { 1134 return; 1135 } 1136 1137 final String oldCommon = CldrVersion.v22_1.getBaseDirectory() + "/common"; 1138 1139 // set up exceptions 1140 Set<String> changedToEmpty = new HashSet<String>( 1141 Arrays.asList(new String[] { "version", "languageCoverage", 1142 "scriptCoverage", "territoryCoverage", 1143 "currencyCoverage", "timezoneCoverage", 1144 "skipDefaultLocale" })); 1145 Set<String> PCDATA = new HashSet<String>(); 1146 PCDATA.add("PCDATA"); 1147 Set<String> EMPTY = new HashSet<String>(); 1148 EMPTY.add("EMPTY"); 1149 Set<String> VERSION = new HashSet<String>(); 1150 VERSION.add("version"); 1151 1152 // test all DTDs 1153 for (DtdType dtd : DtdType.values()) { 1154 try { 1155 ElementAttributeInfo oldDtd = ElementAttributeInfo.getInstance( 1156 oldCommon, dtd); 1157 ElementAttributeInfo newDtd = ElementAttributeInfo 1158 .getInstance(dtd); 1159 1160 if (oldDtd == newDtd) { 1161 continue; 1162 } 1163 Relation<String, String> oldElement2Children = oldDtd 1164 .getElement2Children(); 1165 Relation<String, String> newElement2Children = newDtd 1166 .getElement2Children(); 1167 1168 Relation<String, String> oldElement2Attributes = oldDtd 1169 .getElement2Attributes(); 1170 Relation<String, String> newElement2Attributes = newDtd 1171 .getElement2Attributes(); 1172 1173 for (String element : oldElement2Children.keySet()) { 1174 Set<String> oldChildren = oldElement2Children 1175 .getAll(element); 1176 Set<String> newChildren = newElement2Children 1177 .getAll(element); 1178 if (newChildren == null) { 1179 if (!knownElementExceptions.contains(Pair.of(dtd.toString(), element))) { 1180 errln("Old " + dtd + " contains element not in new: <" 1181 + element + ">"); 1182 } 1183 continue; 1184 } 1185 Set<String> funny = containsInOrder(newChildren, 1186 oldChildren); 1187 if (funny != null) { 1188 if (changedToEmpty.contains(element) 1189 && oldChildren.equals(PCDATA) 1190 && newChildren.equals(EMPTY)) { 1191 // ok, skip 1192 } else { 1193 errln("Old " + dtd + " element <" + element 1194 + "> has children Missing/Misordered:\t" 1195 + funny + "\n\t\tOld:\t" + oldChildren 1196 + "\n\t\tNew:\t" + newChildren); 1197 } 1198 } 1199 1200 Set<String> oldAttributes = oldElement2Attributes 1201 .getAll(element); 1202 if (oldAttributes == null) { 1203 oldAttributes = Collections.emptySet(); 1204 } 1205 Set<String> newAttributes = newElement2Attributes 1206 .getAll(element); 1207 if (newAttributes == null) { 1208 newAttributes = Collections.emptySet(); 1209 } 1210 if (!newAttributes.containsAll(oldAttributes)) { 1211 LinkedHashSet<String> missing = new LinkedHashSet<String>( 1212 oldAttributes); 1213 missing.removeAll(newAttributes); 1214 if (element.equals(dtd.toString()) 1215 && missing.equals(VERSION)) { 1216 // ok, skip 1217 } else { 1218 errln("Old " + dtd + " element <" + element 1219 + "> has attributes Missing:\t" + missing 1220 + "\n\t\tOld:\t" + oldAttributes 1221 + "\n\t\tNew:\t" + newAttributes); 1222 } 1223 } 1224 } 1225 } catch (Exception e) { 1226 e.printStackTrace(); 1227 errln("Failure with " + dtd); 1228 } 1229 } 1230 } 1231 containsInOrder(Set<T> superset, Set<T> subset)1232 private <T> Set<T> containsInOrder(Set<T> superset, Set<T> subset) { 1233 if (!superset.containsAll(subset)) { 1234 LinkedHashSet<T> missing = new LinkedHashSet<T>(subset); 1235 missing.removeAll(superset); 1236 return missing; 1237 } 1238 // ok, we know that they are subsets, try order 1239 Set<T> result = null; 1240 DiscreteComparator<T> comp = new DiscreteComparator.Builder<T>( 1241 Ordering.ARBITRARY).add(superset).get(); 1242 T last = null; 1243 for (T item : subset) { 1244 if (last != null) { 1245 int order = comp.compare(last, item); 1246 if (order != -1) { 1247 if (result == null) { 1248 result = new HashSet<T>(); 1249 result.add(last); 1250 result.add(item); 1251 } 1252 } 1253 } 1254 last = item; 1255 } 1256 return result; 1257 } 1258 TestDtdCompatibility()1259 public void TestDtdCompatibility() { 1260 1261 for (DtdType type : DtdType.values()) { 1262 DtdData dtdData = DtdData.getInstance(type); 1263 Map<String, Element> currentElementFromName = dtdData 1264 .getElementFromName(); 1265 1266 // current has no orphan 1267 Set<Element> orphans = new LinkedHashSet<Element>(dtdData 1268 .getElementFromName().values()); 1269 orphans.remove(dtdData.ROOT); 1270 orphans.remove(dtdData.PCDATA); 1271 orphans.remove(dtdData.ANY); 1272 Set<String> elementsWithoutAlt = new TreeSet<String>(); 1273 Set<String> elementsWithoutDraft = new TreeSet<String>(); 1274 Set<String> elementsWithoutAlias = new TreeSet<String>(); 1275 Set<String> elementsWithoutSpecial = new TreeSet<String>(); 1276 1277 for (Element element : dtdData.getElementFromName().values()) { 1278 Set<Element> children = element.getChildren().keySet(); 1279 orphans.removeAll(children); 1280 if (type == DtdType.ldml 1281 && !SUPPLEMENTAL_DATA_INFO.isDeprecated(type, 1282 element.name, "*", "*")) { 1283 if (element.getType() == ElementType.PCDATA) { 1284 if (element.getAttributeNamed("alt") == null) { 1285 elementsWithoutAlt.add(element.name); 1286 } 1287 if (element.getAttributeNamed("draft") == null) { 1288 elementsWithoutDraft.add(element.name); 1289 } 1290 } else { 1291 if (children.size() != 0 && !"alias".equals(element.name)) { 1292 if (element.getChildNamed("alias") == null) { 1293 elementsWithoutAlias.add(element.name); 1294 } 1295 if (element.getChildNamed("special") == null) { 1296 elementsWithoutSpecial.add(element.name); 1297 } 1298 } 1299 } 1300 } 1301 } 1302 assertEquals(type + " DTD Must not have orphan elements", 1303 Collections.EMPTY_SET, orphans); 1304 assertEquals(type 1305 + " DTD elements with PCDATA must have 'alt' attributes", 1306 Collections.EMPTY_SET, elementsWithoutAlt); 1307 assertEquals(type 1308 + " DTD elements with PCDATA must have 'draft' attributes", 1309 Collections.EMPTY_SET, elementsWithoutDraft); 1310 assertEquals(type 1311 + " DTD elements with children must have 'alias' elements", 1312 Collections.EMPTY_SET, elementsWithoutAlias); 1313 assertEquals( 1314 type 1315 + " DTD elements with children must have 'special' elements", 1316 Collections.EMPTY_SET, elementsWithoutSpecial); 1317 1318 if (logKnownIssue("cldrbug:11583", "Comment out test until last release data is available for unit tests")) { 1319 return; 1320 } 1321 1322 for (CldrVersion version : CldrVersion.CLDR_VERSIONS_DESCENDING) { 1323 if (version == CldrVersion.unknown || version == CldrVersion.trunk) { 1324 continue; 1325 } 1326 DtdData dtdDataOld; 1327 try { 1328 dtdDataOld = DtdData.getInstance(type, version.toString()); 1329 } catch (IllegalArgumentException e) { 1330 boolean tooOld = false; 1331 switch (type) { 1332 case ldmlBCP47: 1333 case ldmlICU: 1334 tooOld = version.isOlderThan(CldrVersion.v1_7_2); 1335 break; 1336 case keyboard: 1337 case platform: 1338 tooOld = version.isOlderThan(CldrVersion.v22_1); 1339 break; 1340 default: 1341 break; 1342 } 1343 if (tooOld) { 1344 continue; 1345 } else { 1346 errln(version + ": " + e.getClass().getSimpleName() + ", " + e.getMessage()); 1347 continue; 1348 } 1349 } 1350 // verify that if E is in dtdDataOld, then it is in dtdData, and 1351 // has at least the same children and attributes 1352 for (Entry<String, Element> entry : dtdDataOld 1353 .getElementFromName().entrySet()) { 1354 Element oldElement = entry.getValue(); 1355 Element newElement = currentElementFromName.get(entry 1356 .getKey()); 1357 if (knownElementExceptions.contains(Pair.of(type.toString(), oldElement.getName()))) { 1358 continue; 1359 } 1360 if (assertNotNull(type 1361 + " DTD for trunk must be superset of v" + version 1362 + ", and must contain «" + oldElement.getName() 1363 + "»", newElement)) { 1364 // TODO Check order also 1365 for (Element oldChild : oldElement.getChildren() 1366 .keySet()) { 1367 if (oldChild == null) { 1368 continue; 1369 } 1370 Element newChild = newElement 1371 .getChildNamed(oldChild.getName()); 1372 1373 if (knownChildExceptions.contains(Pair.of(newElement.getName(), oldChild.getName()))) { 1374 continue; 1375 } 1376 assertNotNull( 1377 type + " DTD - Trunk children of «" 1378 + newElement.getName() 1379 + "» must be superset of v" 1380 + version + ", and must contain «" 1381 + oldChild.getName() + "»", 1382 newChild); 1383 } 1384 for (Attribute oldAttribute : oldElement 1385 .getAttributes().keySet()) { 1386 Attribute newAttribute = newElement 1387 .getAttributeNamed(oldAttribute.getName()); 1388 1389 if (knownAttributeExceptions.contains(Pair.of(newElement.getName(), oldAttribute.getName()))) { 1390 continue; 1391 } 1392 assertNotNull( 1393 type + " DTD - Trunk attributes of «" 1394 + newElement.getName() 1395 + "» must be superset of v" 1396 + version + ", and must contain «" 1397 + oldAttribute.getName() + "»", 1398 newAttribute); 1399 } 1400 } 1401 } 1402 } 1403 } 1404 } 1405 1406 /** 1407 * Compare each path to each other path for every single file in CLDR 1408 */ TestDtdComparison()1409 public void TestDtdComparison() { 1410 // try some simple paths for regression 1411 1412 sortPaths( 1413 DtdData.getInstance(DtdType.ldml).getDtdComparator(null), 1414 "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/dateTimeFormatLength[@type=\"full\"]/dateTimeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]", 1415 "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats"); 1416 1417 sortPaths( 1418 DtdData.getInstance(DtdType.supplementalData).getDtdComparator( 1419 null), 1420 "//supplementalData/territoryContainment/group[@type=\"419\"][@contains=\"013 029 005\"][@grouping=\"true\"]", 1421 "//supplementalData/territoryContainment/group[@type=\"003\"][@contains=\"021 013 029\"][@grouping=\"true\"]"); 1422 1423 } 1424 TestDtdComparisonsAll()1425 public void TestDtdComparisonsAll() { 1426 if (getInclusion() <= 5) { // Only run this test in exhaustive mode. 1427 return; 1428 } 1429 for (File file : CLDRConfig.getInstance().getAllCLDRFilesEndingWith(".xml")) { 1430 checkDtdComparatorFor(file, null); 1431 } 1432 } 1433 checkDtdComparatorForResource(String fileToRead, DtdType overrideDtdType)1434 public void checkDtdComparatorForResource(String fileToRead, 1435 DtdType overrideDtdType) { 1436 MyHandler myHandler = new MyHandler(overrideDtdType); 1437 XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); 1438 try { 1439 myHandler.fileName = fileToRead; 1440 xfr.read(myHandler.fileName, TestBasic.class, -1, true); 1441 logln(myHandler.fileName); 1442 } catch (Exception e) { 1443 Throwable t = e; 1444 StringBuilder b = new StringBuilder(); 1445 String indent = ""; 1446 while (t != null) { 1447 b.append(indent).append(t.getMessage()); 1448 indent = indent.isEmpty() ? "\n\t\t" : indent + "\t"; 1449 t = t.getCause(); 1450 } 1451 errln(b.toString()); 1452 return; 1453 } 1454 DtdData dtdData = DtdData.getInstance(myHandler.dtdType); 1455 sortPaths(dtdData.getDtdComparator(null), myHandler.data); 1456 } 1457 checkDtdComparatorFor(File fileToRead, DtdType overrideDtdType)1458 public void checkDtdComparatorFor(File fileToRead, DtdType overrideDtdType) { 1459 MyHandler myHandler = new MyHandler(overrideDtdType); 1460 XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); 1461 try { 1462 myHandler.fileName = PathUtilities.getNormalizedPathString(fileToRead); 1463 xfr.read(myHandler.fileName, -1, true); 1464 logln(myHandler.fileName); 1465 } catch (Exception e) { 1466 Throwable t = e; 1467 StringBuilder b = new StringBuilder(); 1468 String indent = ""; 1469 while (t != null) { 1470 b.append(indent).append(t.getMessage()); 1471 indent = indent.isEmpty() ? "\n\t\t" : indent + "\t"; 1472 t = t.getCause(); 1473 } 1474 errln(b.toString()); 1475 return; 1476 } 1477 DtdData dtdData = DtdData.getInstance(myHandler.dtdType); 1478 sortPaths(dtdData.getDtdComparator(null), myHandler.data); 1479 } 1480 1481 static class MyHandler extends XMLFileReader.SimpleHandler { 1482 private String fileName; 1483 private DtdType dtdType; 1484 private final Set<String> data = new LinkedHashSet<>(); 1485 MyHandler(DtdType overrideDtdType)1486 public MyHandler(DtdType overrideDtdType) { 1487 dtdType = overrideDtdType; 1488 } 1489 1490 @Override handlePathValue(String path, @SuppressWarnings("unused") String value)1491 public void handlePathValue(String path, @SuppressWarnings("unused") String value) { 1492 if (dtdType == null) { 1493 try { 1494 dtdType = DtdType.fromPath(path); 1495 } catch (Exception e) { 1496 throw new IllegalArgumentException( 1497 "Can't read " + fileName, e); 1498 } 1499 } 1500 data.add(path); 1501 } 1502 } 1503 sortPaths(Comparator<String> dc, Collection<String> paths)1504 public void sortPaths(Comparator<String> dc, Collection<String> paths) { 1505 String[] array = paths.toArray(new String[paths.size()]); 1506 sortPaths(dc, array); 1507 } 1508 sortPaths(Comparator<String> dc, String... array)1509 public void sortPaths(Comparator<String> dc, String... array) { 1510 Arrays.sort(array, 0, array.length, dc); 1511 } 1512 // public void TestNewDtdData() moved to TestDtdData 1513 } 1514