1 package org.unicode.cldr.test; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.PrintWriter; 6 import java.util.ArrayList; 7 import java.util.Arrays; 8 import java.util.Collection; 9 import java.util.Collections; 10 import java.util.Comparator; 11 import java.util.EnumSet; 12 import java.util.HashMap; 13 import java.util.HashSet; 14 import java.util.Iterator; 15 import java.util.List; 16 import java.util.Map; 17 import java.util.Set; 18 import java.util.TreeMap; 19 import java.util.TreeSet; 20 import java.util.regex.Matcher; 21 22 import org.unicode.cldr.draft.FileUtilities; 23 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 24 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 25 import org.unicode.cldr.test.CheckCLDR.CompoundCheckCLDR; 26 import org.unicode.cldr.test.CheckCLDR.FormatDemo; 27 import org.unicode.cldr.test.CheckCLDR.Options; 28 import org.unicode.cldr.test.CheckCLDR.Phase; 29 import org.unicode.cldr.test.CheckCLDR.SimpleDemo; 30 import org.unicode.cldr.tool.Option; 31 import org.unicode.cldr.tool.Option.Params; 32 import org.unicode.cldr.tool.ShowData; 33 import org.unicode.cldr.tool.TablePrinter; 34 import org.unicode.cldr.util.CLDRConfig; 35 import org.unicode.cldr.util.CLDRConfig.Environment; 36 import org.unicode.cldr.util.CLDRFile; 37 import org.unicode.cldr.util.CLDRFile.Status; 38 import org.unicode.cldr.util.CLDRPaths; 39 import org.unicode.cldr.util.CLDRTool; 40 import org.unicode.cldr.util.CldrUtility; 41 import org.unicode.cldr.util.Counter; 42 import org.unicode.cldr.util.CoverageInfo; 43 import org.unicode.cldr.util.Factory; 44 import org.unicode.cldr.util.LanguageTagParser; 45 import org.unicode.cldr.util.Level; 46 import org.unicode.cldr.util.LocaleIDParser; 47 import org.unicode.cldr.util.LogicalGrouping; 48 import org.unicode.cldr.util.Organization; 49 import org.unicode.cldr.util.Pair; 50 import org.unicode.cldr.util.PathDescription; 51 import org.unicode.cldr.util.PathHeader; 52 import org.unicode.cldr.util.PathUtilities; 53 import org.unicode.cldr.util.PatternCache; 54 import org.unicode.cldr.util.SimpleFactory; 55 import org.unicode.cldr.util.StandardCodes; 56 import org.unicode.cldr.util.StringId; 57 import org.unicode.cldr.util.SupplementalDataInfo; 58 import org.unicode.cldr.util.UnicodeSetPrettyPrinter; 59 import org.unicode.cldr.util.VoteResolver; 60 import org.unicode.cldr.util.VoteResolver.CandidateInfo; 61 import org.unicode.cldr.util.VoteResolver.UnknownVoterException; 62 import org.unicode.cldr.util.XMLSource; 63 64 import com.ibm.icu.dev.tool.UOption; 65 import com.ibm.icu.dev.util.ElapsedTimer; 66 import com.ibm.icu.impl.Relation; 67 import com.ibm.icu.impl.Row; 68 import com.ibm.icu.lang.UCharacter; 69 import com.ibm.icu.text.Collator; 70 import com.ibm.icu.text.UnicodeSet; 71 import com.ibm.icu.util.ULocale; 72 73 /** 74 * Console test for CheckCLDR. <br> 75 * Some common source directories: 76 * 77 * <pre> 78 * -s C:/cvsdata/unicode/cldr/incoming/vetted/main 79 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 80 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 81 * -s C:/cvsdata/unicode/cldr/testdata/main 82 * </pre> 83 * 84 * @author markdavis 85 * 86 */ 87 @CLDRTool(alias = "check", 88 description = "Run CheckCLDR against CLDR data") 89 public class ConsoleCheckCLDR { 90 private static final CLDRConfig CLDR_CONFIG = CLDRConfig.getInstance(); 91 private static final PathHeader.Factory PATH_HEADER_FACTORY = PathHeader.getFactory(); 92 public static boolean showStackTrace = false; 93 public static boolean errorsOnly = false; 94 static boolean SHOW_LOCALE = true; 95 static boolean SHOW_EXAMPLES = false; 96 // static PrettyPath prettyPathMaker = new PrettyPath(); 97 98 private static final int HELP1 = 0, 99 HELP2 = 1, 100 COVERAGE = 2, 101 EXAMPLES = 3, 102 FILE_FILTER = 4, 103 TEST_FILTER = 5, 104 DATE_FORMATS = 6, 105 ORGANIZATION = 7, 106 SHOWALL = 8, 107 PATH_FILTER = 9, 108 ERRORS_ONLY = 10, 109 CHECK_ON_SUBMIT = 11, 110 NO_ALIASES = 12, 111 SOURCE_DIRECTORY = 13, 112 USER = 14, 113 PHASE = 15, 114 GENERATE_HTML = 16, 115 VOTE_RESOLVE = 17, 116 ID_VIEW = 18, 117 SUBTYPE_FILTER = 19, 118 SOURCE_ALL = 20, 119 BAILEY = 21 120 // VOTE_RESOLVE2 = 21 121 ; 122 123 static final String SOURCE_DIRS = CLDRPaths.MAIN_DIRECTORY + "," + CLDRPaths.ANNOTATIONS_DIRECTORY + "," + CLDRPaths.SEED_DIRECTORY; 124 125 enum MyOptions { 126 coverage(new Params().setHelp("Set the coverage: eg -c comprehensive") 127 .setMatch("comprehensive|modern|moderate|basic")), // UOption.REQUIRES_ARG 128 examples(new Params().setHelp("Turn on examples (actually a summary of the demo)") 129 .setFlag('x')), //, 'x', UOption.NO_ARG), 130 file_filter(new Params().setHelp("Pick the locales (files) to check: arg is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)") 131 .setDefault(".*").setMatch(".*")), //, 'f', UOption.REQUIRES_ARG).setDefault(".*"), 132 test_filter(new Params() 133 .setHelp("Filter the Checks: arg is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)") 134 .setDefault(".*").setMatch(".*")), //, 't', UOption.REQUIRES_ARG).setDefault(".*"), 135 date_formats(new Params().setHelp("Turn on special date format checks")), //, 'd', UOption.NO_ARG), 136 organization(new Params().setHelp("Organization: ibm, google, ....; Uses Locales.txt for to filter locales and set coverage levels") 137 .setDefault(".*").setMatch(".*")), //, 'o', UOption.REQUIRES_ARG), 138 showall(new Params().setHelp("Show all paths, including aliased").setFlag('a')), //, 'a', UOption.NO_ARG), 139 path_filter(new Params().setHelp("Pick the paths to check, eg -p.*languages.*") 140 .setDefault(".*").setMatch(".*")), //, 'p', UOption.REQUIRES_ARG).setDefault(".*"), 141 errors_only(new Params().setHelp("Show errors only (with -ef, only final processing errors)")), //, 'e', UOption.NO_ARG), 142 check_on_submit(new Params().setHelp("") 143 .setFlag('k')), //, 'k', UOption.NO_ARG), 144 noaliases(new Params().setHelp("No aliases")), //, 'n', UOption.NO_ARG), 145 source_directory(new Params().setHelp("Fully qualified source directories. (Conflicts with -S.)") 146 .setDefault(SOURCE_DIRS).setMatch(".*")), //, 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 147 user(new Params().setHelp("User, eg -uu148") 148 .setMatch(".*")), //, 'u', UOption.REQUIRES_ARG), 149 phase(new Params().setHelp("?") 150 .setMatch(Phase.class).setFlag('z')), //, 'z', UOption.REQUIRES_ARG), 151 generate_html(new Params().setHelp("Generate HTML-style chart in directory.") 152 .setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/").setMatch(".*")), //, 'g', UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 153 vote_resolution(new Params().setHelp("")), //, 'v', UOption.NO_ARG), 154 id_view(new Params().setHelp("")), //, 'i', UOption.NO_ARG), 155 subtype_filter(new Params().setHelp("error/warning subtype filter, eg unexpectedOrderOfEraYear") 156 .setDefault(".*").setMatch(".*").setFlag('y')), //, 'y', UOption.REQUIRES_ARG), 157 source_all(new Params().setHelp( 158 "Partially qualified directories. Standard subdirectories added if not specified (/main, /annotations, /subdivisions). (Conflicts with -s.)") 159 .setMatch(".*").setFlag('S').setDefault("common,seed,exemplars")), //, 'S', <changed>), 160 bailey(new Params().setHelp("check bailey values (" + CldrUtility.INHERITANCE_MARKER + ")")), //, 'b', UOption.NO_ARG) 161 exemplarError(new Params().setFlag('E').setHelp("include to force strict Exemplar check")); 162 163 // BOILERPLATE TO COPY 164 final Option option; 165 MyOptions(Params params)166 private MyOptions(Params params) { 167 option = new Option(this, params); 168 } 169 170 private static Option.Options myOptions = new Option.Options(); 171 static { 172 for (MyOptions option : MyOptions.values()) { myOptions.add(option, option.option)173 myOptions.add(option, option.option); 174 } 175 } 176 parse(String[] args, boolean showArguments)177 private static Set<String> parse(String[] args, boolean showArguments) { 178 return myOptions.parse(MyOptions.values()[0], args, true); 179 } 180 } 181 182 private static final UOption[] options = { 183 UOption.HELP_H(), 184 UOption.HELP_QUESTION_MARK(), 185 UOption.create("coverage", 'c', UOption.REQUIRES_ARG), 186 UOption.create("examples", 'x', UOption.NO_ARG), 187 UOption.create("file_filter", 'f', UOption.REQUIRES_ARG).setDefault(".*"), 188 UOption.create("test_filter", 't', UOption.REQUIRES_ARG).setDefault(".*"), 189 UOption.create("date_formats", 'd', UOption.NO_ARG), 190 UOption.create("organization", 'o', UOption.REQUIRES_ARG), 191 UOption.create("showall", 'a', UOption.NO_ARG), 192 UOption.create("path_filter", 'p', UOption.REQUIRES_ARG).setDefault(".*"), 193 UOption.create("errors_only", 'e', UOption.NO_ARG), 194 UOption.create("check-on-submit", 'k', UOption.NO_ARG), 195 UOption.create("noaliases", 'n', UOption.NO_ARG), 196 UOption.create("source_directory", 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 197 UOption.create("user", 'u', UOption.REQUIRES_ARG), 198 UOption.create("phase", 'z', UOption.REQUIRES_ARG), 199 UOption.create("generate_html", 'g', UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 200 UOption.create("vote resolution", 'v', UOption.NO_ARG), 201 UOption.create("id view", 'i', UOption.NO_ARG), 202 UOption.create("subtype_filter", 'y', UOption.REQUIRES_ARG), 203 UOption.create("source_all", 'S', UOption.OPTIONAL_ARG).setDefault("common,seed,exemplars"), 204 UOption.create("bailey", 'b', UOption.NO_ARG), 205 UOption.create("exemplarError", 'E', UOption.NO_ARG) 206 // UOption.create("vote resolution2", 'w', UOption.OPTIONAL_ARG).setDefault(Utility.BASE_DIRECTORY + 207 // "incoming/vetted/main/votes/"), 208 }; 209 210 private static final Comparator<String> baseFirstCollator = new Comparator<String>() { 211 LanguageTagParser languageTagParser1 = new LanguageTagParser(); 212 LanguageTagParser languageTagParser2 = new LanguageTagParser(); 213 214 @Override 215 public int compare(String o1, String o2) { 216 String ls1 = languageTagParser1.set(o1).getLanguageScript(); 217 String ls2 = languageTagParser2.set(o2).getLanguageScript(); 218 int result = ls1.compareTo(ls2); 219 if (result != 0) return result; 220 return o1.compareTo(o2); 221 } 222 }; 223 private static final boolean PATH_IN_COUNT = false; 224 225 /* 226 * TODO: unused? Should be used? 227 */ 228 private static String[] HelpMessage = { 229 "-h \t This message", 230 "-s \t Source directory, default = " + SOURCE_DIRS, 231 "-S common,seed\t Use common AND seed directories. ( Set CLDR_DIR, don't use this with -s. )\n", 232 "-fxxx \t Pick the locales (files) to check: xxx is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)", 233 "-pxxx \t Pick the paths to check, eg -p(.*languages.*)", 234 "-cxxx \t Set the coverage: eg -c comprehensive or -c modern or -c moderate or -c basic", 235 "-txxx \t Filter the Checks: xxx is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)", 236 "-oxxx \t Organization: ibm, google, ....; filters locales and uses Locales.txt for coverage tests", 237 "-x \t Turn on examples (actually a summary of the demo).", 238 "-d \t Turn on special date format checks", 239 "-a \t Show all paths", 240 "-e \t Show errors only (with -ef, only final processing errors)", 241 "-n \t No aliases", 242 "-u \t User, eg -uu148", 243 "-y \t error/warning subtype filter, eg unexpectedOrderOfEraYear", 244 "-b \t check bailey values (" + CldrUtility.INHERITANCE_MARKER + ")", 245 }; 246 247 static Counter<ErrorType> subtotalCount = new Counter<>(true); // new ErrorCount(); 248 static Counter<ErrorType> totalCount = new Counter<>(true); 249 250 /** 251 * This will be the test framework way of using these tests. It is preliminary for now. 252 * The Survey Tool will call setDisplayInformation, and getCheckAll. 253 * For each cldrfile, it will set the cldrFile. 254 * Then on each path in the file it will call check. 255 * Right now it doesn't work with resolved files, so just use unresolved ones. 256 * 257 * @param args 258 * @throws IOException 259 */ main(String[] args)260 public static void main(String[] args) throws IOException { 261 MyOptions.parse(args, true); 262 ElapsedTimer totalTimer = new ElapsedTimer(); 263 //CldrUtility.showOptions(args); 264 UOption.parseArgs(args, options); 265 // if (options[HELP1].doesOccur || options[HELP2].doesOccur) { 266 // for (int i = 0; i < HelpMessage.length; ++i) { 267 // System.out.println(HelpMessage[i]); 268 // } 269 // return; 270 // } 271 String factoryFilter = options[FILE_FILTER].value; 272 if (factoryFilter.equals("key")) { 273 factoryFilter = "(en|ru|nl|fr|de|it|pl|es|tr|th|ja|zh|ko|ar|bg|sr|uk|ca|hr|cs|da|fil|fi|hu|id|lv|lt|nb|pt|ro|sk|sl|sv|vi|el|he|fa|hi|am|af|et|is|ms|sw|zu|bn|mr|ta|eu|gl|ur|gu|kn|ml|te|zh_Hant|pt_PT|en_GB)"; 274 } 275 String checkFilter = options[TEST_FILTER].value; 276 String subtypeFilterString = options[SUBTYPE_FILTER].value; 277 EnumSet<Subtype> subtypeFilter = null; 278 if (subtypeFilterString != null) { 279 subtypeFilter = EnumSet.noneOf(Subtype.class); 280 Matcher m = PatternCache.get(subtypeFilterString).matcher(""); 281 for (Subtype value : Subtype.values()) { 282 if (m.reset(value.toString()).find() || m.reset(value.name()).find()) { 283 subtypeFilter.add(value); 284 } 285 } 286 if (subtypeFilter.size() == 0) { 287 System.err.println("No subtype match for " + subtypeFilterString); 288 return; 289 } 290 } 291 292 errorsOnly = options[ERRORS_ONLY].doesOccur; 293 // if ("f".equals(options[ERRORS_ONLY].value)) { 294 // CheckCLDR.finalErrorType = CheckStatus.warningType; 295 // } 296 297 SHOW_EXAMPLES = options[EXAMPLES].doesOccur; 298 boolean showAll = options[SHOWALL].doesOccur; 299 boolean checkFlexibleDates = options[DATE_FORMATS].doesOccur; 300 String pathFilterString = options[PATH_FILTER].value; 301 Matcher pathFilter = null; 302 if (!pathFilterString.equals(".*")) { 303 pathFilter = PatternCache.get(pathFilterString).matcher(""); 304 } 305 boolean checkOnSubmit = options[CHECK_ON_SUBMIT].doesOccur; 306 boolean noaliases = options[NO_ALIASES].doesOccur; 307 308 Level coverageLevel = null; 309 String coverageLevelInput = options[COVERAGE].value; 310 if (coverageLevelInput != null) { 311 coverageLevel = Level.get(coverageLevelInput); 312 if (coverageLevel == Level.UNDETERMINED) { 313 throw new IllegalArgumentException("-c" + coverageLevelInput + "\t is invalid: must be one of: " 314 + "basic,moderate,..."); 315 } 316 } 317 318 Organization organization = options[ORGANIZATION].value == null ? null : Organization.fromString(options[ORGANIZATION].value); 319 if (organization != null) { 320 Set<Organization> organizations = StandardCodes.make().getLocaleCoverageOrganizations(); 321 if (!organizations.contains(organization)) { 322 throw new IllegalArgumentException("-o" + organization + "\t is invalid: must be one of: " 323 + organizations); 324 } 325 } 326 final CLDRConfig cldrConf = CLDR_CONFIG; 327 // set the envronment to UNITTEST as suggested 328 cldrConf.setEnvironment(Environment.UNITTEST); 329 // get the Phase from CLDRConfig object 330 final Phase phase; 331 // Phase phase = Phase.BUILD; 332 if (options[PHASE].doesOccur) { 333 String phaseVal = options[PHASE].value; 334 try { 335 // no null check for argument; if it is is null, Phase.forString would return the one from CLDRConfig 336 phase = Phase.forString(phaseVal); 337 } catch (IllegalArgumentException e) { 338 StringBuilder sb = new StringBuilder("Incorrect Phase value"); 339 if (phaseVal != null && !phaseVal.isEmpty()) { 340 sb.append(" '"); 341 sb.append(phaseVal); 342 sb.append("'"); 343 } 344 sb.append(": should be one of "); 345 for (Phase curPhase : Phase.values()) { 346 // implicitly does a toString; 347 sb.append(curPhase); 348 sb.append(", "); 349 } 350 int lastIdx = sb.lastIndexOf(","); 351 // remove the last comma, if it occurs 352 if (lastIdx > -1) { 353 String tmpBuf = sb.substring(0, lastIdx); 354 sb.setLength(0); 355 sb.append(tmpBuf); 356 } 357 sb.append("."); 358 // TODO: Reporting should be similar to an error (wrong parameter...), and not actually an Exception 359 throw new IllegalArgumentException(sb.toString(), e); 360 } 361 } else { 362 phase = cldrConf.getPhase(); 363 } 364 365 boolean baileyTest = options[BAILEY].doesOccur; 366 367 File sourceDirectories[] = null; 368 369 if (MyOptions.source_all.option.doesOccur()) { 370 if (MyOptions.source_directory.option.doesOccur()) { 371 throw new IllegalArgumentException("Don't use -s and -S together."); 372 } 373 sourceDirectories = cldrConf.addStandardSubdirectories(cldrConf.getCLDRDataDirectories(MyOptions.source_all.option.getValue())); 374 } else { 375 String[] sdirs = options[SOURCE_DIRECTORY].value.split(",\\s*"); 376 sourceDirectories = new File[sdirs.length]; 377 for (int i = 0; i < sdirs.length; ++i) { 378 sourceDirectories[i] = new File(CldrUtility.checkValidDirectory(sdirs[i], 379 "Fix with -s. Use -h for help.")); 380 } 381 } 382 383 if (options[GENERATE_HTML].doesOccur) { 384 coverageLevel = Level.MODERN; // reset 385 ErrorFile.generated_html_directory = options[GENERATE_HTML].value; 386 ErrorFile.generated_html_count = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "count.txt"); 387 // try { 388 // ErrorFile.voteFactory = CLDRFile.Factory.make(sourceDirectory + "../../proposed/main/", ".*"); 389 // } catch (RuntimeException e) { 390 // ErrorFile.voteFactory = null; 391 // } 392 // PrintWriter cssFile = FileUtilities.openUTF8Writer(generated_html_directory, "index.css"); 393 // Utility; 394 } 395 396 idView = options[ID_VIEW].doesOccur; 397 398 if (options[VOTE_RESOLVE].doesOccur) { 399 resolveVotesDirectory = CldrUtility.checkValidFile(CLDRPaths.BASE_DIRECTORY + "incoming/vetted/votes/", 400 true, null); 401 VoteResolver.setVoterToInfo(CldrUtility.checkValidFile(CLDRPaths.BASE_DIRECTORY 402 + "incoming/vetted/usersa/usersa.xml", false, null)); 403 voteResolver = new VoteResolver<>(); 404 } 405 406 // check stuff 407 // Comparator cc = StandardCodes.make().getTZIDComparator(); 408 // System.out.println(cc.compare("Antarctica/Rothera", "America/Cordoba")); 409 // System.out.println(cc.compare("Antarctica/Rothera", "America/Indianapolis")); 410 411 String user = options[USER].value; 412 413 System.out.println("Source directories:\n"); 414 for (File f : sourceDirectories) { 415 System.out.println(" " + f.getPath() + "\t(" 416 + PathUtilities.getNormalizedPathString(f) + ")"); 417 } 418 // System.out.println("factoryFilter: " + factoryFilter); 419 // System.out.println("test filter: " + checkFilter); 420 // System.out.println("organization: " + organization); 421 // System.out.println("show examples: " + SHOW_EXAMPLES); 422 // System.out.println("phase: " + phase); 423 // System.out.println("path filter: " + pathFilterString); 424 // System.out.println("coverage level: " + coverageLevel); 425 // System.out.println("checking dates: " + checkFlexibleDates); 426 // System.out.println("only check-on-submit: " + checkOnSubmit); 427 // System.out.println("show all: " + showAll); 428 // System.out.println("errors only?: " + errorsOnly); 429 // System.out.println("generate error counts: " + ErrorFile.generated_html_directory); 430 // // System.out.println("vote directory: " + (ErrorFile.voteFactory == null ? null : 431 // // ErrorFile.voteFactory.getSourceDirectory())); 432 // System.out.println("resolve votes: " + resolveVotesDirectory); 433 // System.out.println("id view: " + idView); 434 // System.out.println("subtype filter: " + subtypeFilter); 435 436 // set up the test 437 Factory cldrFactory = SimpleFactory.make(sourceDirectories, factoryFilter) 438 .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 439 CompoundCheckCLDR checkCldr = CheckCLDR.getCheckAll(cldrFactory, checkFilter); 440 if (checkCldr.getFilteredTestList().size() == 0) { 441 throw new IllegalArgumentException("The filter doesn't match any tests."); 442 } 443 System.out.println("filtered tests: " + checkCldr.getFilteredTests()); 444 Factory backCldrFactory = Factory.make(CLDRPaths.MAIN_DIRECTORY, factoryFilter) 445 .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 446 english = backCldrFactory.make("en", true); 447 448 CheckCLDR.setDisplayInformation(english); 449 checkCldr.setEnglishFile(english); 450 setExampleGenerator(new ExampleGenerator(english, english, CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 451 PathShower pathShower = new PathShower(); 452 453 // call on the files 454 Set<String> locales = new TreeSet<>(baseFirstCollator); 455 locales.addAll(cldrFactory.getAvailable()); 456 457 List<CheckStatus> result = new ArrayList<>(); 458 Set<PathHeader> paths = new TreeSet<>(); // CLDRFile.ldmlComparator); 459 Map m = new TreeMap(); 460 // double testNumber = 0; 461 Map<String, String> options = new HashMap<>(); 462 FlexibleDateFromCLDR fset = new FlexibleDateFromCLDR(); 463 Set<String> englishPaths = null; 464 465 Set<String> fatalErrors = new TreeSet<>(); 466 467 showHeaderLine(); 468 469 supplementalDataInfo = SupplementalDataInfo.getInstance(CLDRPaths.SUPPLEMENTAL_DIRECTORY); 470 471 LocaleIDParser localeIDParser = new LocaleIDParser(); 472 String lastBaseLanguage = ""; 473 PathHeader.Factory pathHeaderFactory = PathHeader.getFactory(english); 474 475 final List<String> specialPurposeLocales = new ArrayList<>(Arrays.asList("en_US_POSIX", "en_ZZ", "und", "und_ZZ")); 476 for (String localeID : locales) { 477 if (CLDRFile.isSupplementalName(localeID)) continue; 478 if (supplementalDataInfo.getDefaultContentLocales().contains(localeID)) { 479 System.out.println("# Skipping default content locale: " + localeID); 480 continue; 481 } 482 483 // We don't really need to check the POSIX locale, as it is a special purpose locale 484 if (specialPurposeLocales.contains(localeID)) { 485 System.out.println("# Skipping special purpose locale: " + localeID); 486 continue; 487 } 488 489 boolean isLanguageLocale = localeID.equals(localeIDParser.set(localeID).getLanguageScript()); 490 options.clear(); 491 492 if (MyOptions.exemplarError.option.doesOccur()) { 493 options.put(Options.Option.exemplarErrors.toString(), "true"); 494 } 495 496 // if the organization is set, skip any locale that doesn't have a value in Locales.txt 497 Level level = coverageLevel; 498 if (level == null) { 499 level = Level.BASIC; 500 } 501 if (organization != null) { 502 Map<String, Level> locale_status = StandardCodes.make().getLocaleToLevel(organization); 503 if (locale_status == null) continue; 504 level = locale_status.get(localeID); 505 if (level == null) continue; 506 if (level.compareTo(Level.BASIC) <= 0) continue; 507 } else if (!isLanguageLocale) { 508 // otherwise, skip all language locales 509 options.put(Options.Option.CheckCoverage_skip.getKey(), "true"); 510 } 511 512 // if (coverageLevel != null) options.put("CoverageLevel.requiredLevel", coverageLevel.toString()); 513 if (organization != null) options.put(Options.Option.CoverageLevel_localeType.getKey(), organization.toString()); 514 options.put(Options.Option.phase.getKey(), phase.toString()); 515 //options.put(Options.Option.SHOW_TIMES.getKey(), "true"); 516 517 if (SHOW_LOCALE) System.out.println(); 518 519 // options.put("CheckCoverage.requiredLevel","comprehensive"); 520 521 CLDRFile file; 522 CLDRFile englishFile = english; 523 CLDRFile parent = null; 524 525 ElapsedTimer timer = new ElapsedTimer(); 526 try { 527 file = cldrFactory.make(localeID, true); 528 if (ErrorFile.voteFactory != null) { 529 ErrorFile.voteFile = ErrorFile.voteFactory.make(localeID, true); 530 } 531 final String parentID = LocaleIDParser.getParent(localeID); 532 if (parentID != null) { 533 parent = cldrFactory.make(parentID, true); 534 } 535 //englishFile = cldrFactory.make("en", true); 536 } catch (RuntimeException e) { 537 fatalErrors.add(localeID); 538 System.out.println("FATAL ERROR: " + localeID); 539 e.printStackTrace(System.out); 540 continue; 541 } 542 543 // generate HTML if asked for 544 if (ErrorFile.generated_html_directory != null) { 545 String baseLanguage = localeIDParser.set(localeID).getLanguageScript(); 546 547 if (!baseLanguage.equals(lastBaseLanguage)) { 548 lastBaseLanguage = baseLanguage; 549 ErrorFile.openErrorFile(localeID, baseLanguage); 550 } 551 552 } 553 554 if (user != null) { 555 file = new CLDRFile.TestUser(file, user, isLanguageLocale); 556 if (parent != null) { 557 parent = new CLDRFile.TestUser(parent, user, isLanguageLocale); 558 } 559 } 560 checkCldr.setCldrFileToCheck(file, options, result); 561 562 subtotalCount.clear(); 563 564 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext();) { 565 CheckStatus status = it3.next(); 566 String statusString = status.toString(); // com.ibm.icu.impl.Utility.escape( 567 CheckStatus.Type statusType = status.getType(); 568 569 if (errorsOnly) { 570 if (!statusType.equals(CheckStatus.errorType)) continue; 571 } 572 573 if (subtypeFilter != null) { 574 if (!subtypeFilter.contains(status.getSubtype())) { 575 continue; 576 } 577 } 578 579 if (checkOnSubmit) { 580 if (!status.isCheckOnSubmit() || !statusType.equals(CheckStatus.errorType)) continue; 581 } 582 showValue(file, null, localeID, null, null, null, null, statusString, status.getSubtype()); 583 } 584 paths.clear(); 585 586 CoverageInfo covInfo = cldrConf.getCoverageInfo(); 587 for (String path : file.fullIterable()) { 588 if (pathFilter != null && !pathFilter.reset(path).find()) { 589 continue; 590 } 591 if (coverageLevel != null) { 592 Level currentLevel = covInfo.getCoverageLevel(path, localeID); 593 if (currentLevel.compareTo(coverageLevel) > 0) { 594 continue; 595 } 596 } 597 paths.add(pathHeaderFactory.fromPath(path)); 598 } 599 // addPrettyPaths(file, pathFilter, prettyPathMaker, noaliases, false, paths); 600 // addPrettyPaths(file, file.getExtraPaths(), pathFilter, prettyPathMaker, noaliases, false, paths); 601 602 // also add the English paths 603 // initialize the first time in. 604 if (englishPaths == null) { 605 englishPaths = new HashSet<>(); 606 final CLDRFile displayFile = CheckCLDR.getDisplayInformation(); 607 addPrettyPaths(displayFile, pathFilter, pathHeaderFactory, noaliases, true, englishPaths); 608 addPrettyPaths(displayFile, displayFile.getExtraPaths(), pathFilter, pathHeaderFactory, noaliases, 609 true, englishPaths); 610 englishPaths = Collections.unmodifiableSet(englishPaths); // for robustness 611 } 612 // paths.addAll(englishPaths); 613 614 UnicodeSet missingExemplars = new UnicodeSet(); 615 UnicodeSet missingCurrencyExemplars = new UnicodeSet(); 616 if (checkFlexibleDates) { 617 fset.set(file); 618 } 619 pathShower.set(localeID); 620 621 // only create if we are going to use 622 ExampleGenerator exampleGenerator = SHOW_EXAMPLES ? new ExampleGenerator(file, englishFile, 623 CLDRPaths.DEFAULT_SUPPLEMENTAL_DIRECTORY) : null; 624 625 // Status pathStatus = new Status(); 626 int pathCount = 0; 627 Status otherPath = new Status(); 628 629 for (PathHeader pathHeader : paths) { 630 pathCount++; 631 String path = pathHeader.getOriginalPath(); 632 String prettyPath = pathHeader.toString().replace('\t', '|').replace(' ', '_'); 633 // String prettyPath = it2.next(); 634 // String path = prettyPathMaker.getOriginal(prettyPath); 635 // if (path == null) { 636 // prettyPathMaker.getOriginal(prettyPath); 637 // } 638 639 if (!showAll && !file.isWinningPath(path)) { 640 continue; 641 } 642 if (!isLanguageLocale && !baileyTest) { 643 final String sourceLocaleID = file.getSourceLocaleID(path, otherPath); 644 if (!localeID.equals(sourceLocaleID)) { 645 continue; 646 } 647 // also skip aliases 648 if (!path.equals(otherPath.pathWhereFound)) { 649 continue; 650 } 651 } 652 653 if (path.contains("@alt")) { 654 if (path.contains("proposed")) continue; 655 } 656 String value = file.getStringValue(path); 657 if (baileyTest) { 658 value = CldrUtility.INHERITANCE_MARKER; 659 } 660 String fullPath = file.getFullXPath(path); 661 662 String example = ""; 663 664 if (SHOW_EXAMPLES) { 665 example = ExampleGenerator.simplify(exampleGenerator.getExampleHtml(path, value)); 666 showExamples(checkCldr, prettyPath, localeID, path, value, fullPath, example); 667 } 668 669 if (checkFlexibleDates) { 670 fset.checkFlexibles(path, value, fullPath); 671 } 672 673 if (path.contains("duration-century")) { 674 int debug = 0; 675 } 676 677 int limit = 1; 678 for (int jj = 0; jj < limit; ++jj) { 679 if (jj == 0) { 680 checkCldr.check(path, fullPath, value, new Options(options), result); 681 } else { 682 checkCldr.getExamples(path, fullPath, value, new Options(options), result); 683 } 684 685 boolean showedOne = false; 686 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext();) { 687 CheckStatus status = it3.next(); 688 String statusString = status.toString(); // com.ibm.icu.impl.Utility.escape( 689 CheckStatus.Type statusType = status.getType(); 690 Object[] parameters = status.getParameters(); 691 692 if (parameters != null) { 693 if (parameters.length >= 1 && status.getCause().getClass() == CheckForExemplars.class) { 694 try { 695 UnicodeSet set = new UnicodeSet(parameters[0].toString()); 696 if (status.getMessage().contains("currency")) { 697 missingCurrencyExemplars.addAll(set); 698 } else { 699 missingExemplars.addAll(set); 700 } 701 } catch (RuntimeException e) { 702 } // skip if not parseable as set 703 } 704 } 705 706 if (errorsOnly && !statusType.equals(CheckStatus.errorType)) { 707 continue; 708 } 709 710 if (subtypeFilter != null) { 711 if (!subtypeFilter.contains(status.getSubtype())) { 712 continue; 713 } 714 } 715 if (checkOnSubmit) { 716 if (!status.isCheckOnSubmit() || !statusType.equals(CheckStatus.errorType)) continue; 717 } 718 719 // System.out.print("Locale:\t" + getLocaleAndName(localeID) + "\t"); 720 if (statusType.equals(CheckStatus.demoType)) { 721 SimpleDemo d = status.getDemo(); 722 if (d != null && d instanceof FormatDemo) { 723 FormatDemo fd = (FormatDemo) d; 724 m.clear(); 725 // m.put("pattern", fd.getPattern()); 726 // m.put("input", fd.getRandomInput()); 727 if (d.processPost(m)) System.out.println("\tDemo:\t" + fd.getPlainText(m)); 728 } 729 continue; 730 } 731 732 if (parameters != null) { 733 for (int i = 0; i < parameters.length; ++i) { 734 if (showStackTrace && parameters[i] instanceof Throwable) { 735 ((Throwable) parameters[i]).printStackTrace(); 736 } 737 } 738 } 739 740 showValue(file, prettyPath, localeID, example, path, value, fullPath, statusString, 741 status.getSubtype()); 742 showedOne = true; 743 744 // survey tool will use: if (status.hasHTMLMessage()) 745 // System.out.println(status.getHTMLMessage()); 746 } 747 if (!showedOne && phase != Phase.FINAL_TESTING) { 748 if (!showedOne && showAll) { 749 showValue(file, prettyPath, localeID, example, path, value, fullPath, "ok", Subtype.none); 750 showedOne = true; 751 } 752 } 753 754 } 755 } 756 757 if (resolveVotesDirectory != null) { 758 LocaleVotingData.resolveErrors(localeID); 759 } 760 761 showSummary(localeID, level, "Items (including inherited):\t" + pathCount); 762 if (missingExemplars.size() != 0) { 763 missingExemplars.removeAll(new UnicodeSet("[[:Uppercase:]-[İ]]")); // remove uppercase #4670 764 if (missingExemplars.size() != 0) { 765 Collator col = Collator.getInstance(new ULocale(localeID)); 766 showSummary(localeID, level, "Total missing from general exemplars:\t" + 767 missingExemplars.size() 768 + "\t" + new UnicodeSetPrettyPrinter() 769 .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT)) 770 .setSpaceComparator(col != null ? col : Collator.getInstance(ULocale.ROOT) 771 .setStrength2(Collator.PRIMARY)) 772 .setCompressRanges(false) 773 .format(missingExemplars)); 774 } 775 } 776 if (missingCurrencyExemplars.size() != 0) { 777 Collator col = Collator.getInstance(new ULocale(localeID)); 778 showSummary(localeID, level, "Total missing from currency exemplars:\t" 779 + new UnicodeSetPrettyPrinter() 780 .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT)) 781 .setSpaceComparator(col != null ? col : Collator.getInstance(ULocale.ROOT) 782 .setStrength2(Collator.PRIMARY)) 783 .setCompressRanges(true) 784 .format(missingCurrencyExemplars)); 785 } 786 for (ErrorType type : subtotalCount.keySet()) { 787 showSummary(localeID, level, "Subtotal " + type + ":\t" + subtotalCount.getCount(type)); 788 } 789 if (checkFlexibleDates) { 790 fset.showFlexibles(); 791 } 792 if (SHOW_EXAMPLES) { 793 // ldml/dates/timeZoneNames/zone[@type="America/Argentina/San_Juan"]/exemplarCity 794 for (String zone : StandardCodes.make().getGoodAvailableCodes("tzid")) { 795 String path = "//ldml/dates/timeZoneNames/zone[@type=\"" + zone + "\"]/exemplarCity"; 796 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 797 String prettyPath = pathHeader.toString().replace('\t', '|').replace(' ', '_'); 798 if (pathFilter != null && !pathFilter.reset(path).matches()) { 799 continue; 800 } 801 String fullPath = file.getStringValue(path); 802 if (fullPath != null) { 803 continue; 804 } 805 /* 806 * TODO: fix this code. Calling getExampleHtml with value = null will always return null, 807 * so what's this supposed to accomplish? 808 */ 809 String example = ExampleGenerator.simplify(exampleGenerator.getExampleHtml(path, null /* value */)); 810 showExamples(checkCldr, prettyPath, localeID, path, null, fullPath, example); 811 } 812 } 813 System.out.println("# Elapsed time: " + timer); 814 System.out.flush(); 815 } 816 817 if (ErrorFile.errorFileWriter != null) { 818 ErrorFile.closeErrorFile(); 819 } 820 821 if (ErrorFile.generated_html_directory != null) { 822 ErrorFile.writeErrorCountsText(); 823 ErrorFile.writeErrorFileIndex(); 824 } 825 System.out.println(); 826 for (ErrorType type : totalCount.keySet()) { 827 System.out.println("# Total " + type + ":\t" + totalCount.getCount(type)); 828 } 829 830 System.out.println(); 831 System.out.println("# Total elapsed time: " + totalTimer); 832 if (fatalErrors.size() != 0) { 833 System.out.println("# FATAL ERRORS:"); 834 } 835 long errorCount = totalCount.getCount(ErrorType.error) + fatalErrors.size(); 836 if (errorCount != 0) { 837 // System.exit((int) errorCount); // cast is safe; we'll never have that many errors 838 System.out.println(); 839 System.out.println("<< FAILURE - Error count is " + errorCount + " . >>"); 840 System.exit(-1); 841 } else { 842 System.out.println(); 843 System.out.println("<< SUCCESS - No errors found. >>"); 844 } 845 if (LogicalGrouping.GET_TYPE_COUNTS) { 846 for (String s : LogicalGrouping.typeCount.keySet()) { 847 System.out.println(s + "=" + LogicalGrouping.typeCount.get(s)); 848 } 849 } 850 checkCldr.handleFinish(); 851 } 852 853 static class LocaleVotingData { 854 private int disputedCount = 0; 855 Counter<Organization> missingOrganizationCounter = new Counter<>(true); 856 Counter<Organization> goodOrganizationCounter = new Counter<>(true); 857 Counter<Organization> conflictedOrganizations = new Counter<>(true); 858 Counter<VoteResolver.Status> winningStatusCounter = new Counter<>(true); 859 860 static Map<String, LocaleVotingData> localeToErrors = new HashMap<>(); 861 private static Map<Integer, String> idToPath; 862 resolveErrors(String locale)863 public static void resolveErrors(String locale) { 864 localeToErrors.put(locale, new LocaleVotingData(locale)); 865 } 866 LocaleVotingData(String locale)867 public LocaleVotingData(String locale) { 868 869 Map<Organization, VoteResolver.Level> orgToMaxVote = VoteResolver.getOrganizationToMaxVote(locale); 870 871 Map<Integer, Map<Integer, CandidateInfo>> info = VoteResolver 872 .getBaseToAlternateToInfo(resolveVotesDirectory + locale + ".xml"); 873 874 Map<String, Integer> valueToItem = new HashMap<>(); 875 876 for (int basePath : info.keySet()) { 877 final Map<Integer, CandidateInfo> itemInfo = info.get(basePath); 878 879 // find the last release status and value 880 voteResolver.clear(); 881 valueToItem.clear(); 882 883 for (int item : itemInfo.keySet()) { 884 String itemValue = getValue(item); 885 valueToItem.put(itemValue, item); 886 887 CandidateInfo candidateInfo = itemInfo.get(item); 888 if (candidateInfo.oldStatus != null) { 889 voteResolver.setTrunk(itemValue, candidateInfo.oldStatus); 890 } 891 voteResolver.add(itemValue); 892 for (int voter : candidateInfo.voters) { 893 try { 894 voteResolver.add(itemValue, voter); 895 } catch (UnknownVoterException e) { 896 // skip 897 } 898 } 899 } 900 901 EnumSet<Organization> basePathConflictedOrganizations = voteResolver.getConflictedOrganizations(); 902 conflictedOrganizations.addAll(basePathConflictedOrganizations, 1); 903 904 VoteResolver.Status winningStatus = voteResolver.getWinningStatus(); 905 String winningValue = voteResolver.getWinningValue(); 906 907 winningStatusCounter.add(winningStatus, 1); 908 909 if (winningStatus == VoteResolver.Status.approved) { 910 continue; 911 } 912 913 CandidateInfo candidateInfo = itemInfo.get(valueToItem.get(winningValue)); 914 Map<Organization, VoteResolver.Level> orgToMaxVoteHere = VoteResolver 915 .getOrganizationToMaxVote(candidateInfo.voters); 916 917 // if the winning item is less than contributed, record the organizations that haven't given their 918 // maximum vote to the winning item. 919 if (winningStatus.compareTo(VoteResolver.Status.contributed) < 0) { 920 // showPaths(basePath, itemInfo); 921 for (Organization org : orgToMaxVote.keySet()) { 922 VoteResolver.Level maxVote = orgToMaxVote.get(org); 923 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 924 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 925 missingOrganizationCounter.add(org, 1); 926 } 927 } 928 if (voteResolver.isDisputed()) { 929 disputedCount++; 930 String path = getIdToPath(basePath); 931 ErrorFile.addDataToErrorFile(locale, path, ErrorType.disputed, Subtype.none); 932 } 933 } else { 934 for (Organization org : orgToMaxVote.keySet()) { 935 VoteResolver.Level maxVote = orgToMaxVote.get(org); 936 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 937 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 938 } else { 939 goodOrganizationCounter.add(org, 1); 940 } 941 } 942 } 943 } 944 System.out.println(getLocaleAndName(locale) + "\tEnabled Organizations:\t" + orgToMaxVote); 945 if (disputedCount != 0) { 946 System.out.println(getLocaleAndName(locale) + "\tDisputed Items:\t" + disputedCount); 947 } 948 949 if (missingOrganizationCounter.size() > 0) { 950 System.out.println(getLocaleAndName(locale) + "\tMIA organizations:\t" + missingOrganizationCounter); 951 System.out 952 .println(getLocaleAndName(locale) + "\tConflicted organizations:\t" + conflictedOrganizations); 953 System.out.println(getLocaleAndName(locale) + "\tCool organizations!:\t" + goodOrganizationCounter); 954 } 955 System.out.println(getLocaleAndName(locale) + "\tOptimal Status:\t" + winningStatusCounter); 956 } 957 getIdToPath(int basePath)958 private static String getIdToPath(int basePath) { 959 if (idToPath == null) { 960 idToPath = VoteResolver.getIdToPath(resolveVotesDirectory + "xpathTable.xml"); 961 } 962 return idToPath.get(basePath); 963 } 964 get(String locale)965 public static LocaleVotingData get(String locale) { 966 return localeToErrors.get(locale); 967 } 968 getDisputedCount()969 int getDisputedCount() { 970 return disputedCount; 971 } 972 getConflictedHTML()973 String getConflictedHTML() { 974 String result = conflictedOrganizations.toString(); 975 if (result.length() == 0) { 976 return ""; 977 } 978 result = result.substring(1, result.length() - 1); 979 result = result.replace(", ", "<br>"); 980 return result; 981 } 982 } 983 getValue(int item)984 private static String getValue(int item) { 985 return String.valueOf(item); 986 } 987 988 static Matcher draftStatusMatcher = PatternCache.get("\\[@draft=\"(provisional|unconfirmed)\"]").matcher(""); 989 990 enum ErrorType { 991 ok, error, disputed, warning, core, posix, minimal, basic, moderate, modern, comprehensive, optional, contributed, provisional, unconfirmed, unknown; 992 static EnumSet<ErrorType> unapproved = EnumSet.range(ErrorType.contributed, ErrorType.unconfirmed); 993 static EnumSet<ErrorType> coverage = EnumSet.range(ErrorType.posix, ErrorType.optional); 994 static EnumSet<ErrorType> showInSummary = EnumSet.of( 995 ErrorType.error, ErrorType.warning, ErrorType.posix, ErrorType.minimal, ErrorType.basic); 996 fromStatusString(String statusString)997 static ErrorType fromStatusString(String statusString) { 998 ErrorType shortStatus = statusString.equals("ok") ? ErrorType.ok 999 : statusString.startsWith("Error") ? ErrorType.error 1000 : statusString.equals("disputed") ? ErrorType.disputed 1001 : statusString.startsWith("Warning") ? ErrorType.warning 1002 : statusString.equals("contributed") ? ErrorType.contributed 1003 : statusString.equals("provisional") ? ErrorType.provisional 1004 : statusString.equals("unconfirmed") ? ErrorType.unconfirmed 1005 : ErrorType.unknown; 1006 if (shortStatus == ErrorType.unknown) { 1007 throw new IllegalArgumentException("Unknown error type: " + statusString); 1008 } else if (shortStatus == ErrorType.warning) { 1009 if (coverageMatcher.reset(statusString).find()) { 1010 shortStatus = ErrorType.valueOf(coverageMatcher.group(1)); 1011 } 1012 } 1013 return shortStatus; 1014 } 1015 } 1016 1017 /* 1018 * static class ErrorCount implements Comparable<ErrorCount> { 1019 * private Counter<ErrorType> counter = new Counter<ErrorType>(); 1020 * 1021 * public int compareTo(ErrorCount o) { 1022 * // we don't really need a good comparison - aren't going to be sorting 1023 * return total() < o.total() ? -1 : total() > o.total() ? 1 : 0; 1024 * } 1025 * public long total() { 1026 * return counter.getTotal(); 1027 * } 1028 * public void clear() { 1029 * counter.clear(); 1030 * } 1031 * public Set<ErrorType> keySet() { 1032 * return counter.getKeysetSortedByKey(); 1033 * } 1034 * public long getCount(ErrorType input) { 1035 * return counter.getCount(input); 1036 * } 1037 * public void increment(ErrorType errorType) { 1038 * counter.add(errorType, 1); 1039 * } 1040 * } 1041 */ 1042 1043 static class ErrorFile { 1044 1045 private static final boolean SHOW_VOTING_INFO = false; 1046 public static CLDRFile voteFile; 1047 public static Factory voteFactory; 1048 openErrorFile(String localeID, String baseLanguage)1049 private static void openErrorFile(String localeID, String baseLanguage) throws IOException { 1050 if (ErrorFile.errorFileWriter != null) { 1051 ErrorFile.closeErrorFile(); 1052 } 1053 ErrorFile.errorFileWriter = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, baseLanguage + ".html"); 1054 ErrorFile.errorFileTable = new TablePrinter(); 1055 errorFileCounter.clear(); 1056 ErrorFile.errorFileTable.setCaption("Problem Details") 1057 .addColumn("Problem").setCellAttributes("align=\"left\" class=\"{0}\"").setSortPriority(0) 1058 .setSpanRows(true) 1059 .setBreakSpans(true).setRepeatHeader(true).setHeaderCell(true) 1060 .addColumn("Subtype").setCellAttributes("align=\"left\" class=\"{1}\"").setSortPriority(1) 1061 .setSpanRows(true) 1062 .setBreakSpans(true).setRepeatHeader(true).setHeaderCell(true) 1063 .addColumn("Locale").setCellAttributes("class=\"{1}\"") 1064 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}\">{0}</a>").setSortPriority(2) 1065 .setSpanRows(true).setBreakSpans(true)//.setRepeatDivider(true) 1066 .addColumn("Name").setCellAttributes("class=\"{1}\"").setSpanRows(true) 1067 .setBreakSpans(true) 1068 .addColumn("Section").setCellAttributes("class=\"{1}\"").setSortPriority(3) 1069 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={3}&x={0}\">{0}</a>") 1070 .setSpanRows(true) 1071 .addColumn("Count").setCellAttributes("class=\"{1}\" align=\"right\""); 1072 1073 showIndexHead("", localeID, ErrorFile.errorFileWriter); 1074 } 1075 1076 static TablePrinter errorFileTable = new TablePrinter(); 1077 static Counter<Row.R4<String, String, ErrorType, Subtype>> errorFileCounter = new Counter<>( 1078 true); 1079 addDataToErrorFile(String localeID, String path, ErrorType shortStatus, Subtype subType)1080 private static void addDataToErrorFile(String localeID, String path, ErrorType shortStatus, 1081 Subtype subType) { 1082 String section = path == null ? null : XPathToMenu.xpathToMenu(path); 1083 if (section == null) { 1084 section = "general"; 1085 } 1086 errorFileCounter.add( 1087 new Row.R4<>(localeID, section, shortStatus, subType), 1); 1088 ErrorFile.sectionToProblemsToLocaleToCount.add( 1089 new Row.R4<>(section, shortStatus, subType, localeID), 1); 1090 } 1091 closeErrorFile()1092 private static void closeErrorFile() { 1093 Set<String> locales = new TreeSet<>(); 1094 for (Row.R4<String, String, ErrorType, Subtype> item : errorFileCounter.keySet()) { 1095 String localeID = item.get0(); 1096 locales.add(localeID); 1097 String section = item.get1(); 1098 ErrorType shortStatus = item.get2(); 1099 Subtype subtype = item.get3(); 1100 // final String prettyPath = path == null ? "general" : prettyPathMaker.getPrettyPath(path, true); 1101 // final String outputForm = path == null ? "general" : prettyPathMaker.getOutputForm(prettyPath); 1102 errorFileTable.addRow() 1103 .addCell(shortStatus) 1104 .addCell(subtype) 1105 .addCell(localeID) 1106 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1107 // .addCell(prettyPath) // menuPath == null ? "" : "<a href='" + link + "'>" + menuPath + "</a>" 1108 .addCell(section) // menuPath == null ? "" : "<a href='" + link + "'>" + menuPath + "</a>" 1109 .addCell(errorFileCounter.getCount(item)) 1110 // .addCell(ConsoleCheckCLDR.safeForHtml(path == null ? null : 1111 // ConsoleCheckCLDR.getEnglishPathValue(path))) 1112 // .addCell(ConsoleCheckCLDR.safeForHtml(value)) 1113 .finishRow(); 1114 } 1115 1116 if (SHOW_VOTING_INFO) { 1117 TablePrinter data = new TablePrinter().setCaption("Voting Information") 1118 .addColumn("Locale").setHeaderCell(true) 1119 .addColumn("Name").setHeaderCell(true) 1120 .addColumn("Organization") 1121 .addColumn("Missing") 1122 .addColumn("Conflicted") 1123 // .addColumn("Good") 1124 ; 1125 for (String localeID : locales) { 1126 // now the voting info 1127 LocaleVotingData localeVotingData = LocaleVotingData.localeToErrors.get(localeID); 1128 if (localeVotingData != null) { 1129 // find all the orgs with data 1130 EnumSet<Organization> orgs = EnumSet.noneOf(Organization.class); 1131 orgs.addAll(localeVotingData.missingOrganizationCounter.keySet()); 1132 orgs.addAll(localeVotingData.conflictedOrganizations.keySet()); 1133 orgs.addAll(localeVotingData.goodOrganizationCounter.keySet()); 1134 for (Organization org : orgs) { 1135 data.addRow() 1136 .addCell(ConsoleCheckCLDR.getLinkedLocale(localeID)) 1137 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1138 .addCell(org) 1139 .addCell(localeVotingData.missingOrganizationCounter.getCount(org)) 1140 .addCell(localeVotingData.conflictedOrganizations.getCount(org)) 1141 // .addCell(localeVotingData.goodOrganizationCounter.getCount(org)) 1142 .finishRow(); 1143 } 1144 } 1145 } 1146 ErrorFile.errorFileWriter.println(data.toTable()); 1147 ErrorFile.errorFileWriter.println("<p></p>"); 1148 } 1149 1150 // generated_html.println("<table border='1' style='border-collapse: collapse' bordercolor='#CCCCFF'>"); 1151 // Locale Group Error Warning Missing Votes: Contributed Missing Votes: Provisional Missing Votes: 1152 // Unconfirmed Missing Coverage: Posix Missing Coverage: Minimal Missing Coverage: Basic Missing Coverage: 1153 // Moderate Missing Coverage: Modern 1154 ErrorFile.errorFileWriter.println(ErrorFile.errorFileTable.toTable()); 1155 ErrorFile.errorFileWriter.println(ShowData.dateFooter()); 1156 ErrorFile.errorFileWriter.println(CldrUtility.ANALYTICS); 1157 ErrorFile.errorFileWriter.println("</body></html>"); 1158 ErrorFile.errorFileWriter.close(); 1159 ErrorFile.errorFileTable = null; 1160 } 1161 1162 // ================ Index File =================== 1163 showErrorFileIndex(PrintWriter generated_html_index)1164 static void showErrorFileIndex(PrintWriter generated_html_index) { 1165 1166 // get organizations 1167 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1168 1169 TablePrinter indexTablePrinter = new TablePrinter().setCaption("Problem Summary") 1170 .setTableAttributes("border='1' style='border-collapse: collapse' bordercolor='blue'") 1171 .addColumn("BASE").setHidden(true)//.setRepeatDivider(true) 1172 .addColumn("Locale").setCellPattern("<a name=\"{0}\" href=\"{1}.html\">{0}</a>") // link to base, anchor 1173 // with full 1174 .addColumn("Name"); 1175 if (SHOW_VOTING_INFO) { 1176 indexTablePrinter.addColumn("Summary") 1177 .addColumn("Missing"); 1178 } 1179 for (Organization org : orgToLocales.keySet()) { 1180 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1181 } 1182 indexTablePrinter 1183 .addColumn("Disputed").setHeaderAttributes("class='disputed'").setCellAttributes("class='disputed'") 1184 .addColumn("Conflicted").setHeaderAttributes("class='conflicted'") 1185 .setCellAttributes("class='conflicted'"); 1186 1187 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1188 String columnTitle = UCharacter.toTitleCase(type.toString(), null); 1189 final boolean coverage = ConsoleCheckCLDR.ErrorType.coverage.contains(type); 1190 if (coverage) { 1191 columnTitle = "MC: " + columnTitle; 1192 } else if (ConsoleCheckCLDR.ErrorType.unapproved.contains(type)) { 1193 columnTitle = "MV: " + columnTitle; 1194 } 1195 indexTablePrinter.addColumn(columnTitle).setHeaderAttributes("class='" + type + "'") 1196 .setCellAttributes("class='" + type + "'"); 1197 } 1198 1199 // now fill in the data 1200 LanguageTagParser ltp = new LanguageTagParser(); 1201 for (String key : ErrorFile.errorFileIndexData.keySet()) { 1202 Pair<String, Counter<ErrorType>> pair = ErrorFile.errorFileIndexData.get(key); 1203 String htmlOpenedFileLanguage = pair.getFirst(); 1204 Counter<ErrorType> counts = pair.getSecond(); 1205 LocaleVotingData votingData = LocaleVotingData.get(htmlOpenedFileLanguage); 1206 if (counts.getTotal() == 0) { 1207 continue; 1208 } 1209 final String baseLanguage = ltp.set(htmlOpenedFileLanguage).getLanguage(); 1210 indexTablePrinter.addRow() 1211 .addCell(baseLanguage) 1212 .addCell(htmlOpenedFileLanguage) 1213 .addCell(ConsoleCheckCLDR.getLocaleName(htmlOpenedFileLanguage)); 1214 if (SHOW_VOTING_INFO) { 1215 indexTablePrinter.addCell(votingData == null ? "" : votingData.winningStatusCounter.toString()) 1216 .addCell(votingData == null ? "" : votingData.missingOrganizationCounter.toString()); 1217 } 1218 for (Organization org : orgToLocales.keySet()) { 1219 indexTablePrinter.addCell(orgToLocales.getAll(org).contains(htmlOpenedFileLanguage) ? org.toString() 1220 .substring(0, 2) : ""); 1221 } 1222 indexTablePrinter 1223 .addCell(votingData == null ? "" : formatSkippingZero(votingData.getDisputedCount())) 1224 .addCell(votingData == null ? "" : votingData.getConflictedHTML()); 1225 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1226 indexTablePrinter.addCell(formatSkippingZero(counts.getCount(type))); 1227 } 1228 indexTablePrinter.finishRow(); 1229 } 1230 generated_html_index.println(indexTablePrinter.toTable()); 1231 generated_html_index.println(ShowData.dateFooter()); 1232 generated_html_index.println(CldrUtility.ANALYTICS); 1233 generated_html_index.println("</body></html>"); 1234 } 1235 1236 static Relation<Organization, String> orgToLocales; 1237 getOrgToLocales()1238 private static Relation<Organization, String> getOrgToLocales() { 1239 if (orgToLocales == null) { 1240 orgToLocales = Relation.of(new TreeMap<Organization, Set<String>>(), TreeSet.class); 1241 StandardCodes sc = StandardCodes.make(); 1242 for (Organization org : sc.getLocaleCoverageOrganizations()) { 1243 for (String locale : sc.getLocaleCoverageLocales(org)) { 1244 Level x = sc.getLocaleCoverageLevel(org, locale); 1245 if (x.compareTo(Level.BASIC) > 0) { 1246 orgToLocales.put(org, locale); 1247 } 1248 } 1249 } 1250 } 1251 return orgToLocales; 1252 } 1253 showSections()1254 static void showSections() throws IOException { 1255 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1256 TablePrinter indexTablePrinter = new TablePrinter().setCaption("Problem Summary") 1257 .setTableAttributes("border='1' style='border-collapse: collapse' bordercolor='blue'") 1258 .addColumn("Section").setSpanRows(true).setBreakSpans(true)//.setRepeatDivider(true) 1259 .addColumn("Problems").setCellAttributes("style=\"text-align:left\" class=\"{2}\"").setSpanRows(true) 1260 .addColumn("Subtype").setCellAttributes("style=\"text-align:left\" class=\"{2}\"").setSpanRows(true) 1261 .addColumn("Locale").setCellAttributes("class=\"{2}\"") 1262 .addColumn("Code").setCellAttributes("class=\"{2}\"") 1263 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}&x={1}\">{0}</a>") // TODO: use CLDRConfig.urls() 1264 .addColumn("Count").setCellAttributes("class=\"{2}\""); 1265 for (Organization org : orgToLocales.keySet()) { 1266 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1267 } 1268 1269 for (Row.R4<String, ErrorType, Subtype, String> sectionAndProblemsAndLocale : ErrorFile.sectionToProblemsToLocaleToCount 1270 .getKeysetSortedByKey()) { 1271 final ErrorType problem = sectionAndProblemsAndLocale.get1(); 1272 final Subtype subtype = sectionAndProblemsAndLocale.get2(); 1273 if (!ConsoleCheckCLDR.ErrorType.showInSummary.contains(problem)) { 1274 continue; 1275 } 1276 final String locale = sectionAndProblemsAndLocale.get3(); 1277 if (problem != ErrorType.error && problem != ErrorType.disputed && !orgToLocales.containsValue(locale)) { 1278 continue; 1279 } 1280 long count = ErrorFile.sectionToProblemsToLocaleToCount.getCount(sectionAndProblemsAndLocale); 1281 final String section = sectionAndProblemsAndLocale.get0(); 1282 indexTablePrinter.addRow() 1283 .addCell(section) 1284 .addCell(problem) 1285 .addCell(subtype) 1286 .addCell(ConsoleCheckCLDR.getLocaleName(locale)) 1287 .addCell(locale) 1288 .addCell(count); 1289 for (Organization org : orgToLocales.keySet()) { 1290 indexTablePrinter.addCell(orgToLocales.getAll(org).contains(locale) ? org.toString().substring(0, 2) : ""); 1291 } 1292 indexTablePrinter.finishRow(); 1293 } 1294 PrintWriter generated_html_index = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "sections.html"); 1295 ConsoleCheckCLDR.ErrorFile.showIndexHead("Error Report Index by Section", "", generated_html_index); 1296 generated_html_index.println(indexTablePrinter.toTable()); 1297 generated_html_index.println(ShowData.dateFooter()); 1298 generated_html_index.println(CldrUtility.ANALYTICS); 1299 generated_html_index.println("</body></html>"); 1300 generated_html_index.close(); 1301 } 1302 formatSkippingZero(long count)1303 static String formatSkippingZero(long count) { 1304 if (count == 0) { 1305 return ""; 1306 } 1307 return String.valueOf(count); 1308 } 1309 showIndexHead(String title, String localeID, PrintWriter generated_html_index)1310 static void showIndexHead(String title, String localeID, PrintWriter generated_html_index) { 1311 final boolean notLocaleSpecific = localeID.length() == 0; 1312 if ((!notLocaleSpecific)) { 1313 title = "Errors in " + ConsoleCheckCLDR.getNameAndLocale(localeID, false); 1314 } 1315 generated_html_index 1316 .println("<html>" + 1317 "<head><meta http-equiv='Content-Type' content='text/html; charset=utf-8'>" 1318 + CldrUtility.LINE_SEPARATOR 1319 + 1320 "<title>" 1321 + title 1322 + "</title>" 1323 + CldrUtility.LINE_SEPARATOR 1324 + 1325 "<link rel='stylesheet' href='errors.css' type='text/css'>" 1326 + CldrUtility.LINE_SEPARATOR 1327 + 1328 "<base target='_blank'>" 1329 + CldrUtility.LINE_SEPARATOR 1330 + 1331 "</head><body>" 1332 + CldrUtility.LINE_SEPARATOR 1333 + 1334 "<h1>" 1335 + title 1336 + "</h1>" 1337 + CldrUtility.LINE_SEPARATOR 1338 + 1339 "<p>" 1340 + 1341 "<a href='index.html" 1342 + (notLocaleSpecific ? "" : "#" + localeID) 1343 + "'>Index</a>" 1344 + 1345 " | " 1346 + 1347 "<a href='sections.html" 1348 + (notLocaleSpecific ? "" : "#" + localeID) 1349 + "'>Index by Section</a>" 1350 + 1351 " | " 1352 + 1353 "<a href='http://unicode.org/cldr/data/docs/survey/vetting.html'><b style='background-color: yellow;'><i>Help: How to Vet</i></b></a>" 1354 + 1355 "</p>" 1356 + 1357 "<p>The following errors have been detected in the locale" 1358 + 1359 (notLocaleSpecific 1360 ? "s. " + org.unicode.cldr.test.HelpMessages.getChartMessages("error_index_header") 1361 : " " + ConsoleCheckCLDR.getNameAndLocale(localeID, false) + ". " 1362 + ErrorFile.ERROR_CHART_HEADER)); 1363 } 1364 writeErrorFileIndex()1365 private static void writeErrorFileIndex() throws IOException { 1366 PrintWriter generated_html_index = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "index.html"); 1367 ConsoleCheckCLDR.ErrorFile.showIndexHead("Error Report Index", "", generated_html_index); 1368 ConsoleCheckCLDR.ErrorFile.showErrorFileIndex(generated_html_index); 1369 generated_html_index.close(); 1370 showSections(); 1371 } 1372 writeErrorCountsText()1373 private static void writeErrorCountsText() { 1374 // if (ErrorFile.htmlErrorsPerLocale.total() != 0) { 1375 1376 // do the plain text file 1377 ErrorFile.generated_html_count.print(ConsoleCheckCLDR.lastHtmlLocaleID + ";\tcounts"); 1378 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1379 ErrorFile.generated_html_count.print(";\t" + type + "=" + ErrorFile.htmlErrorsPerLocale.getCount(type)); 1380 } 1381 ErrorFile.generated_html_count.println(); 1382 ErrorFile.generated_html_count.flush(); 1383 1384 // now store the data for the index 1385 ErrorFile.errorFileIndexData.put(ConsoleCheckCLDR.lastHtmlLocaleID, 1386 new Pair<>(ConsoleCheckCLDR.lastHtmlLocaleID, ErrorFile.htmlErrorsPerLocale)); 1387 ErrorFile.htmlErrorsPerLocale = new Counter<>(); 1388 // } 1389 } 1390 1391 /* 1392 * static Counter<Organization> missingOrganizationCounter = new Counter<Organization>(true); 1393 * static Counter<Organization> goodOrganizationCounter = new Counter<Organization>(true); 1394 * static Counter<Organization> conflictedOrganizations = new Counter<Organization>(true); 1395 * static Counter<VoteResolver.Status> winningStatusCounter = new Counter<VoteResolver.Status>(true); 1396 */ 1397 1398 static Counter<ErrorType> htmlErrorsPerLocale = new Counter<>(); // ConsoleCheckCLDR.ErrorCount(); 1399 static PrintWriter generated_html_count = null; 1400 private static TreeMap<String, Pair<String, Counter<ErrorType>>> errorFileIndexData = new TreeMap<>(); 1401 1402 // private static ConsoleCheckCLDR.ErrorCount htmlErrorsPerBaseLanguage = new ConsoleCheckCLDR.ErrorCount(); 1403 static PrintWriter errorFileWriter = null; 1404 private static final String ERROR_CHART_HEADER = org.unicode.cldr.test.HelpMessages 1405 .getChartMessages("error_locale_header"); 1406 // "Please review and correct them. " + 1407 // "Note that errors in <i>sublocales</i> are often fixed by fixing the main locale.</p>" + 1408 // Utility.LINE_SEPARATOR + 1409 // "<p><i>This list is only generated daily, and so may not reflect fixes you have made until tomorrow. " + 1410 // "(There were production problems in integrating it fully into the Survey tool. " + 1411 // "However, it should let you see the problems and make sure that they get taken care of.)</i></p>" + 1412 // "<p>Coverage depends on your organizations goals: the highest tier languages should include up through all Modern values.</p>" 1413 // + Utility.LINE_SEPARATOR; 1414 static String generated_html_directory = null; 1415 public static Counter<Row.R4<String, ErrorType, Subtype, String>> sectionToProblemsToLocaleToCount = new Counter<>(); 1416 } 1417 showSummary(String localeID, Level level, String value)1418 private static void showSummary(String localeID, Level level, String value) { 1419 String line = "# " + getLocaleAndName(localeID) + "\tSummary\t" + level + "\t" + value; 1420 System.out.println(line); 1421 } 1422 showExamples(CheckCLDR checkCldr, String prettyPath, String localeID, String path, String value, String fullPath, String example)1423 private static void showExamples(CheckCLDR checkCldr, String prettyPath, String localeID, 1424 String path, String value, String fullPath, String example) { 1425 if (example != null) { 1426 showValue(checkCldr.getCldrFileToCheck(), prettyPath, localeID, example, path, value, fullPath, "ok", 1427 Subtype.none); 1428 } 1429 } 1430 addPrettyPaths(CLDRFile file, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1431 private static void addPrettyPaths(CLDRFile file, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, 1432 boolean noaliases, boolean filterDraft, Collection<String> target) { 1433 // Status pathStatus = new Status(); 1434 for (Iterator<String> pit = file.iterator(pathFilter); pit.hasNext();) { 1435 String path = pit.next(); 1436 if (file.isPathExcludedForSurvey(path)) { 1437 continue; 1438 } 1439 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1440 } 1441 } 1442 addPrettyPaths(CLDRFile file, Collection<String> paths, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1443 private static void addPrettyPaths(CLDRFile file, Collection<String> paths, Matcher pathFilter, 1444 PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target) { 1445 // Status pathStatus = new Status(); 1446 for (String path : paths) { 1447 if (pathFilter != null && !pathFilter.reset(path).matches()) continue; 1448 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1449 } 1450 } 1451 addPrettyPath(CLDRFile file, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target, String path)1452 private static void addPrettyPath(CLDRFile file, PathHeader.Factory pathHeaderFactory, boolean noaliases, 1453 boolean filterDraft, Collection<String> target, String path) { 1454 if (noaliases && XMLSource.Alias.isAliasPath(path)) { // this is just for console testing, the survey tool 1455 // shouldn't do it. 1456 return; 1457 // file.getSourceLocaleID(path, pathStatus); 1458 // if (!path.equals(pathStatus.pathWhereFound)) { 1459 // continue; 1460 // } 1461 } 1462 if (filterDraft) { 1463 String newPath = CLDRFile.getNondraftNonaltXPath(path); 1464 if (!newPath.equals(path)) { 1465 String value = file.getStringValue(newPath); 1466 if (value != null) { 1467 return; 1468 } 1469 } 1470 } 1471 String prettyPath = pathHeaderFactory.fromPath(path).toString(); // prettyPathMaker.getPrettyPath(path, true); 1472 // // get sortable version 1473 target.add(prettyPath); 1474 } 1475 setDisplayInformation(CLDRFile inputDisplayInformation, ExampleGenerator inputExampleGenerator)1476 public static synchronized void setDisplayInformation(CLDRFile inputDisplayInformation, 1477 ExampleGenerator inputExampleGenerator) { 1478 CheckCLDR.setDisplayInformation(inputDisplayInformation); 1479 englishExampleGenerator = inputExampleGenerator; 1480 } 1481 setExampleGenerator(ExampleGenerator inputExampleGenerator)1482 public static synchronized void setExampleGenerator(ExampleGenerator inputExampleGenerator) { 1483 englishExampleGenerator = inputExampleGenerator; 1484 } 1485 getExampleGenerator()1486 public static synchronized ExampleGenerator getExampleGenerator() { 1487 return englishExampleGenerator; 1488 } 1489 1490 private static ExampleGenerator englishExampleGenerator; 1491 1492 static Matcher coverageMatcher = PatternCache.get("meet ([a-z]*) coverage").matcher(""); // HACK TODO fix 1493 showHeaderLine()1494 private static void showHeaderLine() { 1495 if (SHOW_LOCALE) { 1496 if (idView) { 1497 System.out 1498 .println("Locale\tID\tDesc.\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯"); 1499 } else { 1500 System.out 1501 .println( 1502 "Locale\tStatus\t▸PPath◂\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t«fill-in»\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯\tFull Path\tAliasedSource/Path?"); 1503 } 1504 } 1505 } 1506 1507 private static PathDescription pathDescription = null; 1508 getIdString(String path, String value)1509 private static String getIdString(String path, String value) { 1510 if (pathDescription == null) { 1511 pathDescription = new PathDescription(supplementalDataInfo, english, null, null, 1512 PathDescription.ErrorHandling.CONTINUE); 1513 } 1514 final String description = pathDescription.getDescription(path, value, null, null); 1515 return "\t" + StringId.getId(path) + "" + "\t" + description + ""; 1516 } 1517 showValue(CLDRFile cldrFile, String prettyPath, String localeID, String example, String path, String value, String fullPath, String statusString, Subtype subType)1518 private static void showValue(CLDRFile cldrFile, String prettyPath, String localeID, String example, 1519 String path, String value, String fullPath, String statusString, Subtype subType) { 1520 ErrorType shortStatus = ErrorType.fromStatusString(statusString); 1521 subtotalCount.add(shortStatus, 1); 1522 totalCount.add(shortStatus, 1); 1523 if (subType == null) { 1524 subType = Subtype.none; 1525 } 1526 1527 if (ErrorFile.errorFileWriter == null) { 1528 example = example == null ? "" : example; 1529 String englishExample = null; 1530 final String englishPathValue = path == null ? null : getEnglishPathValue(path); 1531 if (SHOW_EXAMPLES && path != null) { 1532 englishExample = ExampleGenerator.simplify(getExampleGenerator().getExampleHtml(path, englishPathValue)); 1533 } 1534 englishExample = englishExample == null ? "" : englishExample; 1535 String cleanPrettyPath = path == null ? null : prettyPath; // prettyPathMaker.getOutputForm(prettyPath); 1536 Status status = new Status(); 1537 String sourceLocaleID = path == null ? null : cldrFile.getSourceLocaleID(path, status); 1538 String fillinValue = path == null ? null : cldrFile.getFillInValue(path); 1539 fillinValue = fillinValue == null ? "" : fillinValue.equals(value) ? "=" : fillinValue; 1540 1541 String pathLink = CLDR_CONFIG.urls().forXpath(localeID, path); 1542 1543 final String otherSource = path == null ? null 1544 : (sourceLocaleID.equals(localeID) ? "" 1545 : "\t" + sourceLocaleID); 1546 final String otherPath = path == null ? null 1547 : (status.pathWhereFound.equals(path) ? "" 1548 : "\t" + status.pathWhereFound); 1549 1550 String idViewString = idView ? (path == null ? "\tNO_ID" : getIdString(path, value)) : ""; 1551 System.out.println( 1552 getLocaleAndName(localeID) 1553 + (idViewString.isEmpty() ? 1554 // + "\t" + subtotalCount.getCount(shortStatus) 1555 "\t" + shortStatus 1556 + "\t▸" + cleanPrettyPath + "◂" 1557 + "\t〈" + englishPathValue + "〉" 1558 + "\t【" + englishExample + "】" 1559 + "\t〈" + value + "〉" 1560 + "\t«" + fillinValue + "»" 1561 + "\t【" + example + "】" 1562 + "\t⁅" + subType + "⁆" 1563 + "\t❮" + statusString + "❯" 1564 + "\t" + pathLink 1565 + otherSource 1566 + otherPath 1567 : idViewString 1568 + "\t〈" + englishPathValue + "〉" 1569 + "\t【" + englishExample + "】" 1570 + "\t" + value + "〉" 1571 + "\t【" + example + "】" 1572 + "\t⁅" + subType + "⁆" 1573 + "\t❮" + statusString + "❯")); 1574 } else if (ErrorFile.errorFileWriter != null) { 1575 if (shortStatus == ErrorType.contributed) { 1576 return; 1577 } 1578 if (shortStatus == ErrorType.posix) { 1579 shortStatus = ErrorType.minimal; 1580 } 1581 if (!localeID.equals(lastHtmlLocaleID)) { 1582 ErrorFile.writeErrorCountsText(); 1583 // startGeneratedTable(generated_html, generated_html_table); 1584 lastHtmlLocaleID = localeID; 1585 } 1586 addError(shortStatus); 1587 ErrorFile.addDataToErrorFile(localeID, path, shortStatus, subType); 1588 } 1589 if (PATH_IN_COUNT && ErrorFile.generated_html_count != null) { 1590 ErrorFile.generated_html_count.println(lastHtmlLocaleID + ";\tpath:\t" + path); 1591 } 1592 } 1593 addError(ErrorType shortStatus)1594 private static void addError(ErrorType shortStatus) { 1595 if (ErrorType.showInSummary.contains(shortStatus)) { 1596 ErrorFile.htmlErrorsPerLocale.increment(shortStatus); 1597 } 1598 } 1599 1600 static String lastHtmlLocaleID = ""; 1601 private static VoteResolver<String> voteResolver; 1602 private static String resolveVotesDirectory; 1603 private static boolean idView; 1604 private static SupplementalDataInfo supplementalDataInfo; 1605 private static CLDRFile english; 1606 1607 public static class PathShower { 1608 String localeID; 1609 boolean newLocale = true; 1610 String lastPath; 1611 String[] lastSplitPath; 1612 boolean showEnglish; 1613 String splitChar = "/"; 1614 1615 static final String lead = "****************************************"; 1616 set(String localeID)1617 public void set(String localeID) { 1618 this.localeID = localeID; 1619 newLocale = true; 1620 LocaleIDParser localeIDParser = new LocaleIDParser(); 1621 showEnglish = !localeIDParser.set(localeID).getLanguageScript().equals("en"); 1622 // localeID.equals(CheckCLDR.displayInformation.getLocaleID()); 1623 lastPath = null; 1624 lastSplitPath = null; 1625 } 1626 getSplitChar()1627 public String getSplitChar() { 1628 return splitChar; 1629 } 1630 setSplitChar(String splitChar)1631 public PathShower setSplitChar(String splitChar) { 1632 this.splitChar = splitChar; 1633 return this; 1634 } 1635 } 1636 showValue(String path, String value, boolean showEnglish, String localeID)1637 private static void showValue(String path, String value, boolean showEnglish, String localeID) { 1638 System.out.println("\tValue:\t" + value + (showEnglish ? "\t" + getEnglishPathValue(path) : "") + "\tLocale:\t" 1639 + localeID); 1640 } 1641 getEnglishPathValue(String path)1642 private static String getEnglishPathValue(String path) { 1643 String englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path); 1644 if (englishValue == null) { 1645 String path2 = CLDRFile.getNondraftNonaltXPath(path); 1646 englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path2); 1647 } 1648 return englishValue; 1649 } 1650 1651 /** 1652 * Utility for getting information. 1653 * 1654 * @param locale 1655 * @return 1656 */ getLocaleAndName(String locale)1657 public static String getLocaleAndName(String locale) { 1658 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1659 if (localizedName == null || localizedName.equals(locale)) return locale; 1660 return locale + " [" + localizedName + "]"; 1661 } 1662 1663 /** 1664 * Utility for getting information. 1665 * 1666 * @param locale 1667 * @param linkToXml 1668 * TODO 1669 * @return 1670 */ getNameAndLocale(String locale, boolean linkToXml)1671 public static String getNameAndLocale(String locale, boolean linkToXml) { 1672 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1673 if (localizedName == null || localizedName.equals(locale)) return locale; 1674 if (linkToXml) { 1675 locale = "<a href='http://unicode.org/cldr/data/common/main/" + locale + ".xml'>" + locale + "</a>"; 1676 } 1677 return localizedName + " [" + locale + "]"; 1678 } 1679 getLocaleName(String locale)1680 public static String getLocaleName(String locale) { 1681 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1682 if (localizedName == null || localizedName.equals(locale)) return locale; 1683 return localizedName; 1684 } 1685 getLinkedLocale(String locale)1686 public static String getLinkedLocale(String locale) { 1687 return "<a href='http://unicode.org/cldr/apps/survey?_=" + locale + "'>" + locale + "</a>"; 1688 } 1689 1690 1691 } 1692