1 package org.unicode.cldr.test; 2 3 import java.text.ParseException; 4 import java.util.ArrayList; 5 import java.util.Arrays; 6 import java.util.Calendar; 7 import java.util.Collection; 8 import java.util.Date; 9 import java.util.EnumMap; 10 import java.util.HashSet; 11 import java.util.Iterator; 12 import java.util.LinkedHashSet; 13 import java.util.List; 14 import java.util.Locale; 15 import java.util.Map; 16 import java.util.Random; 17 import java.util.Set; 18 import java.util.TreeSet; 19 import java.util.regex.Matcher; 20 import java.util.regex.Pattern; 21 22 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 23 import org.unicode.cldr.util.ApproximateWidth; 24 import org.unicode.cldr.util.CLDRFile; 25 import org.unicode.cldr.util.CLDRFile.Status; 26 import org.unicode.cldr.util.CLDRLocale; 27 import org.unicode.cldr.util.CldrUtility; 28 import org.unicode.cldr.util.DateTimeCanonicalizer.DateTimePatternType; 29 import org.unicode.cldr.util.DayPeriodInfo; 30 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 31 import org.unicode.cldr.util.DayPeriodInfo.Type; 32 import org.unicode.cldr.util.Factory; 33 import org.unicode.cldr.util.ICUServiceBuilder; 34 import org.unicode.cldr.util.Level; 35 import org.unicode.cldr.util.LocaleIDParser; 36 import org.unicode.cldr.util.LogicalGrouping; 37 import org.unicode.cldr.util.PathHeader; 38 import org.unicode.cldr.util.PathStarrer; 39 import org.unicode.cldr.util.PatternCache; 40 import org.unicode.cldr.util.PreferredAndAllowedHour; 41 import org.unicode.cldr.util.RegexUtilities; 42 import org.unicode.cldr.util.SupplementalDataInfo; 43 import org.unicode.cldr.util.XPathParts; 44 import org.unicode.cldr.util.props.UnicodeProperty.PatternMatcher; 45 46 import com.google.common.base.Joiner; 47 import com.ibm.icu.impl.Relation; 48 import com.ibm.icu.text.BreakIterator; 49 import com.ibm.icu.text.DateTimePatternGenerator; 50 import com.ibm.icu.text.DateTimePatternGenerator.VariableField; 51 import com.ibm.icu.text.MessageFormat; 52 import com.ibm.icu.text.NumberFormat; 53 import com.ibm.icu.text.SimpleDateFormat; 54 import com.ibm.icu.text.UnicodeSet; 55 import com.ibm.icu.util.Output; 56 import com.ibm.icu.util.ULocale; 57 58 public class CheckDates extends FactoryCheckCLDR { 59 static boolean GREGORIAN_ONLY = CldrUtility.getProperty("GREGORIAN", false); 60 61 ICUServiceBuilder icuServiceBuilder = new ICUServiceBuilder(); 62 NumberFormat english = NumberFormat.getNumberInstance(ULocale.ENGLISH); 63 PatternMatcher m; 64 DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 65 DateTimePatternGenerator dateTimePatternGenerator = DateTimePatternGenerator.getEmptyInstance(); 66 private CoverageLevel2 coverageLevel; 67 private SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(); 68 69 // Use the width of the character "0" as the basic unit for checking widths 70 // It's not perfect, but I'm not sure that anything can be. This helps us 71 // weed out some false positives in width checking, like 10月 vs. 十月 72 // in Chinese, which although technically longer, shouldn't trigger an 73 // error. 74 private static final int REFCHAR = ApproximateWidth.getWidth("0"); 75 76 private Level requiredLevel; 77 private String language; 78 private String territory; 79 80 private DayPeriodInfo dateFormatInfoFormat; 81 82 static String[] samples = { 83 // "AD 1970-01-01T00:00:00Z", 84 // "BC 4004-10-23T07:00:00Z", // try a BC date: creation according to Ussher & Lightfoot. Assuming garden of 85 // eden 2 hours ahead of UTC 86 "2005-12-02 12:15:16", 87 // "AD 2100-07-11T10:15:16Z", 88 }; // keep aligned with following 89 static String SampleList = "{0}" 90 // + Utility.LINE_SEPARATOR + "\t\u200E{1}\u200E" + Utility.LINE_SEPARATOR + "\t\u200E{2}\u200E" + 91 // Utility.LINE_SEPARATOR + "\t\u200E{3}\u200E" 92 ; // keep aligned with previous 93 94 private static final String DECIMAL_XPATH = "//ldml/numbers/symbols[@numberSystem='latn']/decimal"; 95 private static final Pattern HOUR_SYMBOL = PatternCache.get("H{1,2}"); 96 private static final Pattern MINUTE_SYMBOL = PatternCache.get("mm"); 97 private static final Pattern YEAR_FIELDS = PatternCache.get("(y|Y|u|U|r){1,5}"); 98 99 static String[] calTypePathsToCheck = { 100 "//ldml/dates/calendars/calendar[@type=\"buddhist\"]", 101 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]", 102 "//ldml/dates/calendars/calendar[@type=\"hebrew\"]", 103 "//ldml/dates/calendars/calendar[@type=\"islamic\"]", 104 "//ldml/dates/calendars/calendar[@type=\"japanese\"]", 105 "//ldml/dates/calendars/calendar[@type=\"roc\"]", 106 }; 107 static String[] calSymbolPathsWhichNeedDistinctValues = { 108 // === for months, days, quarters - format wide & abbrev sets must have distinct values === 109 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month", 110 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month", 111 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"abbreviated\"]/day", 112 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"short\"]/day", 113 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day", 114 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 115 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"wide\"]/quarter", 116 // === for dayPeriods - all values for a given context/width must be distinct === 117 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 118 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 119 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 120 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 121 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 122 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 123 // === for eras - all values for a given context/width should be distinct (warning) === 124 "/eras/eraNames/era", 125 "/eras/eraAbbr/era", // Hmm, root eraAbbr for japanese has many dups, should we change them or drop this test? 126 "/eras/eraNarrow/era", // We may need to allow dups here too 127 }; 128 129 // The following calendar symbol sets need not have distinct values 130 // "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"narrow\"]/month", 131 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"abbreviated\"]/month", 132 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"narrow\"]/month", 133 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"wide\"]/month", 134 // "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"narrow\"]/day", 135 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"abbreviated\"]/day", 136 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"narrow\"]/day", 137 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"wide\"]/day", 138 // "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"narrow\"]/quarter", 139 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 140 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"narrow\"]/quarter", 141 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"wide\"]/quarter", 142 143 // The above are followed by trailing pieces such as 144 // "[@type=\"am\"]", 145 // "[@type=\"sun\"]", 146 // "[@type=\"0\"]", 147 // "[@type=\"1\"]", 148 // "[@type=\"12\"]", 149 150 // Map<String, Set<String>> calPathsToSymbolSets; 151 // Map<String, Map<String, String>> calPathsToSymbolMaps = new HashMap<String, Map<String, String>>(); 152 CheckDates(Factory factory)153 public CheckDates(Factory factory) { 154 super(factory); 155 } 156 157 @Override setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)158 public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, 159 List<CheckStatus> possibleErrors) { 160 if (cldrFileToCheck == null) return this; 161 super.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 162 163 icuServiceBuilder.setCldrFile(getResolvedCldrFileToCheck()); 164 // the following is a hack to work around a bug in ICU4J (the snapshot, not the released version). 165 try { 166 bi = BreakIterator.getCharacterInstance(new ULocale(cldrFileToCheck.getLocaleID())); 167 } catch (RuntimeException e) { 168 bi = BreakIterator.getCharacterInstance(new ULocale("")); 169 } 170 CLDRFile resolved = getResolvedCldrFileToCheck(); 171 flexInfo = new FlexibleDateFromCLDR(); // ought to just clear(), but not available. 172 flexInfo.set(resolved); 173 174 // load decimal path specially 175 String decimal = resolved.getWinningValue(DECIMAL_XPATH); 176 if (decimal != null) { 177 flexInfo.checkFlexibles(DECIMAL_XPATH, decimal, DECIMAL_XPATH); 178 } 179 180 String localeID = cldrFileToCheck.getLocaleID(); 181 LocaleIDParser lp = new LocaleIDParser(); 182 territory = lp.set(localeID).getRegion(); 183 language = lp.getLanguage(); 184 if (territory == null || territory.length() == 0) { 185 if (language.equals("root")) { 186 territory = "001"; 187 } else { 188 CLDRLocale loc = CLDRLocale.getInstance(localeID); 189 CLDRLocale defContent = sdi.getDefaultContentFromBase(loc); 190 if (defContent == null) { 191 territory = "001"; 192 } else { 193 territory = defContent.getCountry(); 194 } 195 // Set territory for 12/24 hour clock to Egypt (12 hr) for ar_001 196 // instead of 24 hour (exception). 197 if (territory.equals("001") && language.equals("ar")) { 198 territory = "EG"; 199 } 200 } 201 } 202 coverageLevel = CoverageLevel2.getInstance(sdi, localeID); 203 requiredLevel = options.getRequiredLevel(localeID); 204 205 // load gregorian appendItems 206 for (Iterator<String> it = resolved.iterator("//ldml/dates/calendars/calendar[@type=\"gregorian\"]"); it.hasNext();) { 207 String path = it.next(); 208 String value = resolved.getWinningValue(path); 209 String fullPath = resolved.getFullXPath(path); 210 try { 211 flexInfo.checkFlexibles(path, value, fullPath); 212 } catch (Exception e) { 213 final String message = e.getMessage(); 214 CheckStatus item = new CheckStatus() 215 .setCause(this) 216 .setMainType(CheckStatus.errorType) 217 .setSubtype( 218 message.contains("Conflicting fields") ? Subtype.dateSymbolCollision : Subtype.internalError) 219 .setMessage(message); 220 possibleErrors.add(item); 221 } 222 // possibleErrors.add(flexInfo.getFailurePath(path)); 223 } 224 redundants.clear(); 225 /* 226 * TODO: NullPointerException may be thrown in ICU here during cldr-unittest TestAll 227 */ 228 flexInfo.getRedundants(redundants); 229 // Set baseSkeletons = flexInfo.gen.getBaseSkeletons(new TreeSet()); 230 // Set notCovered = new TreeSet(neededFormats); 231 // if (flexInfo.preferred12Hour()) { 232 // notCovered.addAll(neededHours12); 233 // } else { 234 // notCovered.addAll(neededHours24); 235 // } 236 // notCovered.removeAll(baseSkeletons); 237 // if (notCovered.size() != 0) { 238 // possibleErrors.add(new CheckStatus().setCause(this).setType(CheckCLDR.finalErrorType) 239 // .setCheckOnSubmit(false) 240 // .setMessage("Missing availableFormats: {0}", new Object[]{notCovered.toString()})); 241 // } 242 pathsWithConflictingOrder2sample = DateOrder.getOrderingInfo(cldrFileToCheck, resolved, flexInfo.fp); 243 if (pathsWithConflictingOrder2sample == null) { 244 CheckStatus item = new CheckStatus() 245 .setCause(this) 246 .setMainType(CheckStatus.errorType) 247 .setSubtype(Subtype.internalError) 248 .setMessage("DateOrder.getOrderingInfo fails"); 249 possibleErrors.add(item); 250 } 251 252 // calPathsToSymbolMaps.clear(); 253 // for (String calTypePath: calTypePathsToCheck) { 254 // for (String calSymbolPath: calSymbolPathsWhichNeedDistinctValues) { 255 // calPathsToSymbolMaps.put(calTypePath.concat(calSymbolPath), null); 256 // } 257 // } 258 259 dateFormatInfoFormat = sdi.getDayPeriods(Type.format, cldrFileToCheck.getLocaleID()); 260 return this; 261 } 262 263 Map<String, Map<DateOrder, String>> pathsWithConflictingOrder2sample; 264 265 // Set neededFormats = new TreeSet(Arrays.asList(new String[]{ 266 // "yM", "yMMM", "yMd", "yMMMd", "Md", "MMMd","yQ" 267 // })); 268 // Set neededHours12 = new TreeSet(Arrays.asList(new String[]{ 269 // "hm", "hms" 270 // })); 271 // Set neededHours24 = new TreeSet(Arrays.asList(new String[]{ 272 // "Hm", "Hms" 273 // })); 274 /** 275 * hour+minute, hour+minute+second (12 & 24) 276 * year+month, year+month+day (numeric & string) 277 * month+day (numeric & string) 278 * year+quarter 279 */ 280 BreakIterator bi; 281 FlexibleDateFromCLDR flexInfo; 282 Collection<String> redundants = new HashSet<>(); 283 Status status = new Status(); 284 PathStarrer pathStarrer = new PathStarrer(); 285 stripPrefix(String s)286 private String stripPrefix(String s) { 287 if (s != null) { 288 int prefEnd = s.lastIndexOf(" "); 289 if (prefEnd < 0 || prefEnd >= 3) { 290 prefEnd = s.lastIndexOf("\u2019"); // as in d’ 291 } 292 if (prefEnd >= 0 && prefEnd < 3) { 293 return s.substring(prefEnd + 1); 294 } 295 } 296 return s; 297 } 298 299 @Override handleCheck(String path, String fullPath, String value, Options options, List<CheckStatus> result)300 public CheckCLDR handleCheck(String path, String fullPath, String value, Options options, 301 List<CheckStatus> result) { 302 303 if (fullPath == null) { 304 return this; // skip paths that we don't have 305 } 306 307 if (path.indexOf("/dates") < 0 308 || path.endsWith("/default") 309 || path.endsWith("/alias")) { 310 return this; 311 } 312 313 String sourceLocale = getCldrFileToCheck().getSourceLocaleID(path, status); 314 315 if (!path.equals(status.pathWhereFound) || !sourceLocale.equals(getCldrFileToCheck().getLocaleID())) { 316 return this; 317 } 318 319 if (value == null) { 320 return this; 321 } 322 323 if (pathsWithConflictingOrder2sample != null) { 324 Map<DateOrder, String> problem = pathsWithConflictingOrder2sample.get(path); 325 if (problem != null) { 326 CheckStatus item = new CheckStatus() 327 .setCause(this) 328 .setMainType(CheckStatus.warningType) 329 .setSubtype(Subtype.incorrectDatePattern) 330 .setMessage("The ordering of date fields is inconsistent with others: {0}", 331 getValues(getResolvedCldrFileToCheck(), problem.values())); 332 result.add(item); 333 } 334 } 335 336 try { 337 if (path.indexOf("[@type=\"abbreviated\"]") >= 0) { 338 String pathToWide = path.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 339 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 340 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 341 CheckStatus item = new CheckStatus() 342 .setCause(this) 343 .setMainType(CheckStatus.errorType) 344 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 345 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 346 wideValue); 347 result.add(item); 348 } 349 Set<String> grouping = LogicalGrouping.getPaths(getCldrFileToCheck(), path); 350 if (grouping != null) { 351 for (String lgPath : grouping) { 352 String lgPathValue = getCldrFileToCheck().getWinningValueWithBailey(lgPath); 353 if (lgPathValue == null) { 354 continue; 355 } 356 String lgPathToWide = lgPath.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 357 String lgPathWideValue = getCldrFileToCheck().getWinningValueWithBailey(lgPathToWide); 358 // This helps us get around things like "de març" vs. "març" in Catalan 359 String thisValueStripped = stripPrefix(value); 360 String wideValueStripped = stripPrefix(wideValue); 361 String lgPathValueStripped = stripPrefix(lgPathValue); 362 String lgPathWideValueStripped = stripPrefix(lgPathWideValue); 363 boolean thisPathHasPeriod = value.contains("."); 364 boolean lgPathHasPeriod = lgPathValue.contains("."); 365 if (!thisValueStripped.equalsIgnoreCase(wideValueStripped) && !lgPathValueStripped.equalsIgnoreCase(lgPathWideValueStripped) && 366 thisPathHasPeriod != lgPathHasPeriod) { 367 CheckStatus.Type et = CheckStatus.errorType; 368 if (path.contains("dayPeriod")) { 369 et = CheckStatus.warningType; 370 } 371 CheckStatus item = new CheckStatus() 372 .setCause(this) 373 .setMainType(et) 374 .setSubtype(Subtype.inconsistentPeriods) 375 .setMessage("Inconsistent use of periods in abbreviations for this section."); 376 result.add(item); 377 break; 378 } 379 } 380 } 381 } else if (path.indexOf("[@type=\"narrow\"]") >= 0) { 382 String pathToAbbr = path.replace("[@type=\"narrow\"]", "[@type=\"abbreviated\"]"); 383 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 384 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 385 CheckStatus item = new CheckStatus() 386 .setCause(this) 387 .setMainType(CheckStatus.warningType) // Making this just a warning, because there are some oddball cases. 388 .setSubtype(Subtype.narrowDateFieldTooWide) 389 .setMessage("Narrow value \"{0}\" shouldn't be longer than the corresponding abbreviated value \"{1}\"", value, 390 abbrValue); 391 result.add(item); 392 } 393 } else if (path.indexOf("/eraNarrow") >= 0) { 394 String pathToAbbr = path.replace("/eraNarrow", "/eraAbbr"); 395 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 396 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 397 CheckStatus item = new CheckStatus() 398 .setCause(this) 399 .setMainType(CheckStatus.errorType) 400 .setSubtype(Subtype.narrowDateFieldTooWide) 401 .setMessage("Narrow value \"{0}\" can't be longer than the corresponding abbreviated value \"{1}\"", value, 402 abbrValue); 403 result.add(item); 404 } 405 } else if (path.indexOf("/eraAbbr") >= 0) { 406 String pathToWide = path.replace("/eraAbbr", "/eraNames"); 407 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 408 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 409 CheckStatus item = new CheckStatus() 410 .setCause(this) 411 .setMainType(CheckStatus.errorType) 412 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 413 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 414 wideValue); 415 result.add(item); 416 } 417 418 } 419 420 String failure = flexInfo.checkValueAgainstSkeleton(path, value); 421 if (failure != null) { 422 result.add(new CheckStatus() 423 .setCause(this) 424 .setMainType(CheckStatus.errorType) 425 .setSubtype(Subtype.illegalDatePattern) 426 .setMessage(failure)); 427 } 428 429 final String collisionPrefix = "//ldml/dates/calendars/calendar"; 430 main: if (path.startsWith(collisionPrefix)) { 431 int pos = path.indexOf("\"]"); // end of first type 432 if (pos < 0 || skipPath(path)) { // skip narrow, no-calendar 433 break main; 434 } 435 pos += 2; 436 String myType = getLastType(path); 437 if (myType == null) { 438 break main; 439 } 440 String myMainType = getMainType(path); 441 442 String calendarPrefix = path.substring(0, pos); 443 boolean endsWithDisplayName = path.endsWith("displayName"); // special hack, these shouldn't be in 444 // calendar. 445 446 Set<String> retrievedPaths = new HashSet<>(); 447 getResolvedCldrFileToCheck().getPathsWithValue(value, calendarPrefix, null, retrievedPaths); 448 if (retrievedPaths.size() < 2) { 449 break main; 450 } 451 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraAbbr/era[@type="0"], 452 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNames/era[@type="0"], 453 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNarrow/era[@type="0"]] 454 Type type = null; 455 DayPeriod dayPeriod = null; 456 final boolean isDayPeriod = path.contains("dayPeriod"); 457 if (isDayPeriod) { 458 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 459 type = Type.fromString(parts.getAttributeValue(5, "type")); 460 dayPeriod = DayPeriod.valueOf(parts.getAttributeValue(-1, "type")); 461 } 462 463 // TODO redo above and below in terms of parts instead of searching strings 464 465 Set<String> filteredPaths = new HashSet<>(); 466 Output<Integer> sampleError = new Output<>(); 467 468 for (String item : retrievedPaths) { 469 XPathParts itemParts = XPathParts.getFrozenInstance(item); 470 if (item.equals(path) 471 || skipPath(item) 472 || endsWithDisplayName != item.endsWith("displayName") 473 || itemParts.containsElement("alias")) { 474 continue; 475 } 476 String otherType = getLastType(item); 477 if (myType.equals(otherType)) { // we don't care about items with the same type value 478 continue; 479 } 480 String mainType = getMainType(item); 481 if (!myMainType.equals(mainType)) { // we *only* care about items with the same type value 482 continue; 483 } 484 if (isDayPeriod) { 485 //ldml/dates/calendars/calendar[@type="gregorian"]/dayPeriods/dayPeriodContext[@type="format"]/dayPeriodWidth[@type="wide"]/dayPeriod[@type="am"] 486 Type itemType = Type.fromString(itemParts.getAttributeValue(5, "type")); 487 DayPeriod itemDayPeriod = DayPeriod.valueOf(itemParts.getAttributeValue(-1, "type")); 488 489 if (!dateFormatInfoFormat.collisionIsError(type, dayPeriod, itemType, itemDayPeriod, sampleError)) { 490 continue; 491 } 492 } 493 filteredPaths.add(item); 494 } 495 if (filteredPaths.size() == 0) { 496 break main; 497 } 498 Set<String> others = new TreeSet<>(); 499 for (String path2 : filteredPaths) { 500 PathHeader pathHeader = getPathHeaderFactory().fromPath(path2); 501 others.add(pathHeader.getHeaderCode()); 502 } 503 CheckStatus.Type statusType = getPhase() == Phase.SUBMISSION || getPhase() == Phase.BUILD 504 ? CheckStatus.warningType 505 : CheckStatus.errorType; 506 final CheckStatus checkStatus = new CheckStatus() 507 .setCause(this) 508 .setMainType(statusType) 509 .setSubtype(Subtype.dateSymbolCollision); 510 if (sampleError.value == null) { 511 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}", 512 value, others.toString()); 513 } else { 514 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}. Sample problem: {2}", 515 value, others.toString(), sampleError.value / DayPeriodInfo.HOUR); 516 } 517 result.add(checkStatus); 518 } 519 DateTimePatternType dateTypePatternType = DateTimePatternType.fromPath(path); 520 if (DateTimePatternType.STOCK_AVAILABLE_INTERVAL_PATTERNS.contains(dateTypePatternType)) { 521 boolean patternBasicallyOk = false; 522 try { 523 formatParser.set(value); 524 patternBasicallyOk = true; 525 } catch (RuntimeException e) { 526 String message = e.getMessage(); 527 if (message.contains("Illegal datetime field:")) { 528 CheckStatus item = new CheckStatus().setCause(this) 529 .setMainType(CheckStatus.errorType) 530 .setSubtype(Subtype.illegalDatePattern) 531 .setMessage(message); 532 result.add(item); 533 } else { 534 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 535 .setSubtype(Subtype.illegalDatePattern) 536 .setMessage("Illegal date format pattern {0}", new Object[] { e }); 537 result.add(item); 538 } 539 } 540 if (patternBasicallyOk) { 541 checkPattern(dateTypePatternType, path, fullPath, value, result); 542 } 543 } else if (path.contains("hourFormat")) { 544 int semicolonPos = value.indexOf(';'); 545 if (semicolonPos < 0) { 546 CheckStatus item = new CheckStatus() 547 .setCause(this) 548 .setMainType(CheckStatus.errorType) 549 .setSubtype(Subtype.illegalDatePattern) 550 .setMessage( 551 "Value should contain a positive hour format and a negative hour format separated by a semicolon."); 552 result.add(item); 553 } else { 554 String[] formats = value.split(";"); 555 if (formats[0].equals(formats[1])) { 556 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 557 .setSubtype(Subtype.illegalDatePattern) 558 .setMessage("The hour formats should not be the same."); 559 result.add(item); 560 } else { 561 checkHasHourMinuteSymbols(formats[0], result); 562 checkHasHourMinuteSymbols(formats[1], result); 563 } 564 } 565 } 566 } catch (ParseException e) { 567 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 568 .setSubtype(Subtype.illegalDatePattern) 569 .setMessage("ParseException in creating date format {0}", new Object[] { e }); 570 result.add(item); 571 } catch (Exception e) { 572 // e.printStackTrace(); 573 // HACK 574 String msg = e.getMessage(); 575 if (msg == null || !HACK_CONFLICTING.matcher(msg).find()) { 576 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 577 .setSubtype(Subtype.illegalDatePattern) 578 .setMessage("Error in creating date format {0}", new Object[] { e }); 579 result.add(item); 580 } 581 } 582 return this; 583 } 584 isTooMuchWiderThan(String shortString, String longString)585 private boolean isTooMuchWiderThan(String shortString, String longString) { 586 // We all 1/3 the width of the reference character as a "fudge factor" in determining the allowable width 587 return ApproximateWidth.getWidth(shortString) > ApproximateWidth.getWidth(longString) + REFCHAR / 3; 588 } 589 590 /** 591 * Check for the presence of hour and minute symbols. 592 * 593 * @param value 594 * the value to be checked 595 * @param result 596 * the list to add any errors to. 597 */ checkHasHourMinuteSymbols(String value, List<CheckStatus> result)598 private void checkHasHourMinuteSymbols(String value, List<CheckStatus> result) { 599 boolean hasHourSymbol = HOUR_SYMBOL.matcher(value).find(); 600 boolean hasMinuteSymbol = MINUTE_SYMBOL.matcher(value).find(); 601 if (!hasHourSymbol && !hasMinuteSymbol) { 602 result.add(createErrorCheckStatus().setMessage("The hour and minute symbols are missing from {0}.", value)); 603 } else if (!hasHourSymbol) { 604 result.add(createErrorCheckStatus() 605 .setMessage("The hour symbol (H or HH) should be present in {0}.", value)); 606 } else if (!hasMinuteSymbol) { 607 result.add(createErrorCheckStatus().setMessage("The minute symbol (mm) should be present in {0}.", value)); 608 } 609 } 610 611 /** 612 * Convenience method for creating errors. 613 * 614 * @return 615 */ createErrorCheckStatus()616 private CheckStatus createErrorCheckStatus() { 617 return new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 618 .setSubtype(Subtype.illegalDatePattern); 619 } 620 skipPath(String path)621 public boolean skipPath(String path) { 622 return path.contains("arrow") 623 || path.contains("/availableFormats") 624 || path.contains("/interval") 625 || path.contains("/dateTimeFormat") 626 // || path.contains("/dayPeriod[") 627 // && !path.endsWith("=\"pm\"]") 628 // && !path.endsWith("=\"am\"]") 629 ; 630 } 631 getLastType(String path)632 public String getLastType(String path) { 633 int secondType = path.lastIndexOf("[@type=\""); 634 if (secondType < 0) { 635 return null; 636 } 637 secondType += 8; 638 int secondEnd = path.indexOf("\"]", secondType); 639 if (secondEnd < 0) { 640 return null; 641 } 642 return path.substring(secondType, secondEnd); 643 } 644 getMainType(String path)645 public String getMainType(String path) { 646 int secondType = path.indexOf("\"]/"); 647 if (secondType < 0) { 648 return null; 649 } 650 secondType += 3; 651 int secondEnd = path.indexOf("/", secondType); 652 if (secondEnd < 0) { 653 return null; 654 } 655 return path.substring(secondType, secondEnd); 656 } 657 getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values)658 private String getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values) { 659 Set<String> results = new TreeSet<>(); 660 for (String path : values) { 661 final String stringValue = resolvedCldrFileToCheck.getStringValue(path); 662 if (stringValue != null) { 663 results.add(stringValue); 664 } 665 } 666 return "{" + Joiner.on("},{").join(results) + "}"; 667 } 668 669 static final Pattern HACK_CONFLICTING = PatternCache.get("Conflicting fields:\\s+M+,\\s+l"); 670 671 @Override handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result)672 public CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result) { 673 if (path.indexOf("/dates") < 0 || path.indexOf("gregorian") < 0) return this; 674 try { 675 if (path.indexOf("/pattern") >= 0 && path.indexOf("/dateTimeFormat") < 0 676 || path.indexOf("/dateFormatItem") >= 0) { 677 checkPattern2(path, value, result); 678 } 679 } catch (Exception e) { 680 // don't worry about errors 681 } 682 return this; 683 } 684 685 static final SimpleDateFormat neutralFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", ULocale.ENGLISH); 686 static { 687 neutralFormat.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 688 } 689 690 // Get Date-Time in milliseconds getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second)691 private static long getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second) { 692 Calendar cal = Calendar.getInstance(); 693 cal.set(year, month, date, hourOfDay, minute, second); 694 return cal.getTimeInMillis(); 695 } 696 697 static long date1950 = getDateTimeinMillis(1950, 0, 1, 0, 0, 0); 698 static long date2010 = getDateTimeinMillis(2010, 0, 1, 0, 0, 0); 699 static long date4004BC = getDateTimeinMillis(-4004, 9, 23, 2, 0, 0); 700 static Random random = new Random(0); 701 checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result)702 private void checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result) 703 throws ParseException { 704 String skeleton = dateTimePatternGenerator.getSkeletonAllowingDuplicates(value); 705 String skeletonCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(value); 706 707 if (value.contains("MMM.") || value.contains("LLL.") || value.contains("E.") || value.contains("eee.") 708 || value.contains("ccc.") || value.contains("QQQ.") || value.contains("qqq.")) { 709 result 710 .add(new CheckStatus() 711 .setCause(this) 712 .setMainType(CheckStatus.warningType) 713 .setSubtype(Subtype.incorrectDatePattern) 714 .setMessage( 715 "Your pattern ({0}) is probably incorrect; abbreviated month/weekday/quarter names that need a period should include it in the name, rather than adding it to the pattern.", 716 value)); 717 } 718 XPathParts pathParts = XPathParts.getFrozenInstance(path); 719 String calendar = pathParts.findAttributeValue("calendar", "type"); 720 String id; 721 switch (dateTypePatternType) { 722 case AVAILABLE: 723 id = pathParts.getAttributeValue(-1, "id"); 724 break; 725 case INTERVAL: 726 id = pathParts.getAttributeValue(-2, "id"); 727 break; 728 case STOCK: 729 id = pathParts.getAttributeValue(-3, "type"); 730 break; 731 default: 732 throw new IllegalArgumentException(); 733 } 734 735 if (dateTypePatternType == DateTimePatternType.AVAILABLE || dateTypePatternType == DateTimePatternType.INTERVAL) { 736 String idCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(id); 737 if (skeleton.isEmpty()) { 738 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 739 .setSubtype(Subtype.incorrectDatePattern) 740 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 741 .setMessage("Your pattern ({1}) is incorrect for ID ({0}). " + 742 "You need to supply a pattern according to http://cldr.org/translation/date-time-patterns.", 743 id, value)); 744 } else if (!dateTimePatternGenerator.skeletonsAreSimilar(idCanonical, skeletonCanonical)) { 745 String fixedValue = dateTimePatternGenerator.replaceFieldTypes(value, id); 746 result 747 .add(new CheckStatus() 748 .setCause(this) 749 .setMainType(CheckStatus.errorType) 750 .setSubtype(Subtype.incorrectDatePattern) 751 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 752 .setMessage( 753 "Your pattern ({2}) doesn't correspond to what is asked for. Yours would be right for an ID ({1}) but not for the ID ({0}). " 754 + 755 "Please change your pattern to match what was asked, such as ({3}), with the right punctuation and/or ordering for your language. See http://cldr.org/translation/date-time-patterns.", 756 id, skeletonCanonical, value, fixedValue)); 757 } 758 if (dateTypePatternType == DateTimePatternType.AVAILABLE) { 759 // index y+w+ must correpond to pattern containing only Y+ and w+ 760 if (idCanonical.matches("y+w+") && !(skeleton.matches("Y+w+") || skeleton.matches("w+Y+"))) { 761 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 762 .setMessage("For id {0}, the pattern ({1}) must contain fields Y and w, and no others.", id, value)); 763 } 764 // index M+W msut correspond to pattern containing only M+/L+ and W 765 if (idCanonical.matches("M+W") && !(skeletonCanonical.matches("M+W") || skeletonCanonical.matches("WM+"))) { 766 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 767 .setMessage("For id {0}, the pattern ({1}) must contain fields M or L, plus W, and no others.", id, value)); 768 } 769 } 770 String failureMessage = (String) flexInfo.getFailurePath(path); 771 if (failureMessage != null) { 772 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 773 .setSubtype(Subtype.illegalDatePattern) 774 .setMessage("{0}", new Object[] { failureMessage })); 775 } 776 } 777 if (dateTypePatternType == DateTimePatternType.STOCK) { 778 int style = 0; 779 String len = pathParts.findAttributeValue("timeFormatLength", "type"); 780 DateOrTime dateOrTime = DateOrTime.time; 781 if (len == null) { 782 dateOrTime = DateOrTime.date; 783 style += 4; 784 len = pathParts.findAttributeValue("dateFormatLength", "type"); 785 if (len == null) { 786 len = pathParts.findAttributeValue("dateTimeFormatLength", "type"); 787 dateOrTime = DateOrTime.dateTime; 788 } 789 } 790 791 DateTimeLengths dateTimeLength = DateTimeLengths.valueOf(len.toUpperCase(Locale.ENGLISH)); 792 793 if (calendar.equals("gregorian") && !"root".equals(getCldrFileToCheck().getLocaleID())) { 794 checkValue(dateTimeLength, dateOrTime, value, result); 795 } 796 if (dateOrTime == DateOrTime.dateTime) { 797 return; // We don't need to do the rest for date/time combo patterns. 798 } 799 style += dateTimeLength.ordinal(); 800 // do regex match with skeletonCanonical but report errors using skeleton; they have corresponding field lengths 801 if (!dateTimePatterns[style].matcher(skeletonCanonical).matches() 802 && !calendar.equals("chinese") 803 && !calendar.equals("hebrew")) { 804 int i = RegexUtilities.findMismatch(dateTimePatterns[style], skeletonCanonical); 805 String skeletonPosition = skeleton.substring(0, i) + "☹" + skeleton.substring(i); 806 result.add(new CheckStatus() 807 .setCause(this) 808 .setMainType(CheckStatus.errorType) 809 .setSubtype(Subtype.missingOrExtraDateField) 810 .setMessage("Field is missing, extra, or the wrong length. Expected {0} [Internal: {1} / {2}]", 811 new Object[] { dateTimeMessage[style], skeletonPosition, dateTimePatterns[style].pattern() })); 812 } 813 } else if (dateTypePatternType == DateTimePatternType.INTERVAL) { 814 if (id.contains("y")) { 815 String greatestDifference = pathParts.findAttributeValue("greatestDifference", "id"); 816 int requiredYearFieldCount = 1; 817 if ("y".equals(greatestDifference)) { 818 requiredYearFieldCount = 2; 819 } 820 int yearFieldCount = 0; 821 Matcher yearFieldMatcher = YEAR_FIELDS.matcher(value); 822 while (yearFieldMatcher.find()) { 823 yearFieldCount++; 824 } 825 if (yearFieldCount < requiredYearFieldCount) { 826 result.add(new CheckStatus() 827 .setCause(this) 828 .setMainType(CheckStatus.errorType) 829 .setSubtype(Subtype.missingOrExtraDateField) 830 .setMessage("Not enough year fields in interval pattern. Must have {0} but only found {1}", 831 new Object[] { requiredYearFieldCount, yearFieldCount })); 832 } 833 } 834 } 835 836 if (value.contains("G") && calendar.equals("gregorian")) { 837 GyState actual = GyState.forPattern(value); 838 GyState expected = getExpectedGy(getCldrFileToCheck().getLocaleID()); 839 if (actual != expected) { 840 result.add(new CheckStatus() 841 .setCause(this) 842 .setMainType(CheckStatus.warningType) 843 .setSubtype(Subtype.unexpectedOrderOfEraYear) 844 .setMessage("Unexpected order of era/year. Expected {0}, but got {1} in 〈{2}〉 for {3}/{4}", 845 expected, actual, value, calendar, id)); 846 } 847 } 848 } 849 850 enum DateOrTime { 851 date, time, dateTime 852 } 853 854 static final Map<DateOrTime, Relation<DateTimeLengths, String>> STOCK_PATTERNS = new EnumMap<>( 855 DateOrTime.class); 856 857 // add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys)858 private static void add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, 859 DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys) { 860 Relation<DateTimeLengths, String> rel = STOCK_PATTERNS.get(dateOrTime); 861 if (rel == null) { 862 STOCK_PATTERNS.put(dateOrTime, rel = Relation.of(new EnumMap<DateTimeLengths, Set<String>>(DateTimeLengths.class), LinkedHashSet.class)); 863 } 864 rel.putAll(dateTimeLength, Arrays.asList(keys)); 865 } 866 867 /* Ticket #4936 868 value(short time) = value(hm) or value(Hm) 869 value(medium time) = value(hms) or value(Hms) 870 value(long time) = value(medium time+z) 871 value(full time) = value(medium time+zzzz) 872 */ 873 static { add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm")874 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms")875 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z")876 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz")877 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd")878 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd")879 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd")880 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd")881 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd"); 882 } 883 884 static final String AVAILABLE_PREFIX = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\""; 885 static final String AVAILABLE_SUFFIX = "\"]"; 886 static final String APPEND_TIMEZONE = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 887 checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result)888 private void checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result) { 889 // Check consistency of the pattern vs. supplemental wrt 12 vs. 24 hour clock. 890 if (dateOrTime == DateOrTime.time) { 891 PreferredAndAllowedHour pref = sdi.getTimeData().get(territory); 892 if (pref == null) { 893 pref = sdi.getTimeData().get("001"); 894 } 895 String checkForHour, clockType; 896 if (pref.preferred.equals(PreferredAndAllowedHour.HourStyle.h)) { 897 checkForHour = "h"; 898 clockType = "12"; 899 } else { 900 checkForHour = "H"; 901 clockType = "24"; 902 } 903 if (!value.contains(checkForHour)) { 904 CheckStatus.Type errType = CheckStatus.errorType; 905 // French/Canada is strange, they use 24 hr clock while en_CA uses 12. 906 if (language.equals("fr") && territory.equals("CA")) { 907 errType = CheckStatus.warningType; 908 } 909 910 result.add(new CheckStatus().setCause(this).setMainType(errType) 911 .setSubtype(Subtype.inconsistentTimePattern) 912 .setMessage("Time format inconsistent with supplemental time data for territory \"" + territory + "\"." 913 + " Use '" + checkForHour + "' for " + clockType + " hour clock.")); 914 } 915 } 916 if (dateOrTime == DateOrTime.dateTime) { 917 boolean inQuotes = false; 918 for (int i = 0; i < value.length(); i++) { 919 char ch = value.charAt(i); 920 if (ch == '\'') { 921 inQuotes = !inQuotes; 922 } 923 if (!inQuotes && (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) { 924 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 925 .setSubtype(Subtype.patternContainsInvalidCharacters) 926 .setMessage("Unquoted letter \"{0}\" in dateTime format.", ch)); 927 } 928 } 929 } else { 930 Set<String> keys = STOCK_PATTERNS.get(dateOrTime).get(dateTimeLength); 931 StringBuilder b = new StringBuilder(); 932 boolean onlyNulls = true; 933 int countMismatches = 0; 934 boolean errorOnMissing = false; 935 String timezonePattern = null; 936 Set<String> bases = new LinkedHashSet<>(); 937 for (String key : keys) { 938 int star = key.indexOf('*'); 939 boolean hasStar = star >= 0; 940 String base = !hasStar ? key : key.substring(0, star); 941 bases.add(base); 942 String xpath = AVAILABLE_PREFIX + base + AVAILABLE_SUFFIX; 943 String value1 = getCldrFileToCheck().getStringValue(xpath); 944 // String localeFound = getCldrFileToCheck().getSourceLocaleID(xpath, null); && !localeFound.equals("root") && !localeFound.equals("code-fallback") 945 if (value1 != null) { 946 onlyNulls = false; 947 if (hasStar) { 948 String zone = key.substring(star + 1); 949 timezonePattern = getResolvedCldrFileToCheck().getStringValue(APPEND_TIMEZONE); 950 value1 = MessageFormat.format(timezonePattern, value1, zone); 951 } 952 if (equalsExceptWidth(value, value1)) { 953 return; 954 } 955 } else { 956 // Example, if the requiredLevel for the locale is moderate, 957 // and the level for the path is modern, then we'll skip the error, 958 // but if the level for the path is basic, then we won't 959 Level pathLevel = coverageLevel.getLevel(xpath); 960 if (requiredLevel.compareTo(pathLevel) >= 0) { 961 errorOnMissing = true; 962 } 963 } 964 add(b, base, value1); 965 countMismatches++; 966 } 967 if (!onlyNulls) { 968 if (timezonePattern != null) { 969 b.append(" (with appendZonePattern: “" + timezonePattern + "”)"); 970 } 971 String msg = countMismatches != 1 972 ? "{1}-{0} → “{2}” didn't match any of the corresponding flexible skeletons: [{3}]. This or the flexible patterns needs to be changed." 973 : "{1}-{0} → “{2}” didn't match the corresponding flexible skeleton: {3}. This or the flexible pattern needs to be changed."; 974 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 975 .setSubtype(Subtype.inconsistentDatePattern) 976 .setMessage(msg, 977 dateTimeLength, dateOrTime, value, b)); 978 } else { 979 if (errorOnMissing) { 980 String msg = countMismatches != 1 981 ? "{1}-{0} → “{2}” doesn't have at least one value for a corresponding flexible skeleton {3}, which needs to be added." 982 : "{1}-{0} → “{2}” doesn't have a value for the corresponding flexible skeleton {3}, which needs to be added."; 983 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 984 .setSubtype(Subtype.missingDatePattern) 985 .setMessage(msg, 986 dateTimeLength, dateOrTime, value, Joiner.on(", ").join(bases))); 987 } 988 } 989 } 990 } 991 add(StringBuilder b, String key, String value1)992 private void add(StringBuilder b, String key, String value1) { 993 if (value1 == null) { 994 return; 995 } 996 if (b.length() != 0) { 997 b.append(" or "); 998 } 999 b.append(key + (value1 == null ? " - missing" : " → “" + value1 + "”")); 1000 } 1001 equalsExceptWidth(String value1, String value2)1002 private boolean equalsExceptWidth(String value1, String value2) { 1003 if (value1.equals(value2)) { 1004 return true; 1005 } else if (value2 == null) { 1006 return false; 1007 } 1008 1009 List<Object> items1 = new ArrayList<>(formatParser.set(value1).getItems()); // clone 1010 List<Object> items2 = formatParser.set(value2).getItems(); 1011 if (items1.size() != items2.size()) { 1012 return false; 1013 } 1014 Iterator<Object> it2 = items2.iterator(); 1015 for (Object item1 : items1) { 1016 Object item2 = it2.next(); 1017 if (item1.equals(item2)) { 1018 continue; 1019 } 1020 if (item1 instanceof VariableField && item2 instanceof VariableField) { 1021 // simple test for now, ignore widths 1022 if (item1.toString().charAt(0) == item2.toString().charAt(0)) { 1023 continue; 1024 } 1025 } 1026 return false; 1027 } 1028 return true; 1029 } 1030 1031 static final Set<String> YgLanguages = new HashSet<>(Arrays.asList( 1032 "ar", "cs", "da", "de", "en", "es", "fa", "fi", "fr", "he", "hr", "id", "it", "nb", "nl", "pt", "ru", "sv", "tr")); 1033 getExpectedGy(String localeID)1034 private GyState getExpectedGy(String localeID) { 1035 // hack for now 1036 int firstBar = localeID.indexOf('_'); 1037 String lang = firstBar < 0 ? localeID : localeID.substring(0, firstBar); 1038 return YgLanguages.contains(lang) ? GyState.YEAR_ERA : GyState.ERA_YEAR; 1039 } 1040 1041 enum GyState { 1042 YEAR_ERA, ERA_YEAR, OTHER; 1043 static DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 1044 1045 static synchronized GyState forPattern(String value) { 1046 formatParser.set(value); 1047 int last = -1; 1048 for (Object x : formatParser.getItems()) { 1049 if (x instanceof VariableField) { 1050 int type = ((VariableField) x).getType(); 1051 if (type == DateTimePatternGenerator.ERA && last == DateTimePatternGenerator.YEAR) { 1052 return GyState.YEAR_ERA; 1053 } else if (type == DateTimePatternGenerator.YEAR && last == DateTimePatternGenerator.ERA) { 1054 return GyState.ERA_YEAR; 1055 } 1056 last = type; 1057 } 1058 } 1059 return GyState.OTHER; 1060 } 1061 } 1062 1063 enum DateTimeLengths { 1064 SHORT, MEDIUM, LONG, FULL 1065 } 1066 1067 // The patterns below should only use the *canonical* characters for each field type: 1068 // y (not Y, u, U) 1069 // Q (not q) 1070 // M (not L) 1071 // E (not e, c) 1072 // a (not b, B) 1073 // H or h (not k or K) 1074 // v (not z, Z, V) 1075 static final Pattern[] dateTimePatterns = { 1076 PatternCache.get("a*(h|hh|H|HH)(m|mm)"), // time-short 1077 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)"), // time-medium 1078 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-long 1079 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-full 1080 PatternCache.get("G*y{1,4}M{1,2}(d|dd)"), // date-short; allow yyy for Minguo/ROC calendar 1081 PatternCache.get("G*y(yyy)?M{1,3}(d|dd)"), // date-medium 1082 PatternCache.get("G*y(yyy)?M{1,4}(d|dd)"), // date-long 1083 PatternCache.get("G*y(yyy)?M{1,4}E*(d|dd)"), // date-full 1084 PatternCache.get(".*"), // datetime-short 1085 PatternCache.get(".*"), // datetime-medium 1086 PatternCache.get(".*"), // datetime-long 1087 PatternCache.get(".*"), // datetime-full 1088 }; 1089 1090 static final String[] dateTimeMessage = { 1091 "hours (H, HH, h, or hh), and minutes (m or mm)", // time-short 1092 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss)", // time-medium 1093 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss); optionally timezone (z, zzzz, v, vvvv)", // time-long 1094 "hours (H, HH, h, or hh), minutes (m or mm), seconds (s or ss), and timezone (z, zzzz, v, vvvv)", // time-full 1095 "year (y, yy, yyyy), month (M or MM), and day (d or dd); optionally era (G)", // date-short 1096 "year (y), month (M, MM, or MMM), and day (d or dd); optionally era (G)", // date-medium 1097 "year (y), month (M, ... MMMM), and day (d or dd); optionally era (G)", // date-long 1098 "year (y), month (M, ... MMMM), and day (d or dd); optionally day of week (EEEE or cccc) or era (G)", // date-full 1099 }; 1100 1101 public String toString(DateTimePatternGenerator.FormatParser formatParser) { 1102 StringBuffer result = new StringBuffer(); 1103 for (Object x : formatParser.getItems()) { 1104 if (x instanceof DateTimePatternGenerator.VariableField) { 1105 result.append(x.toString()); 1106 } else { 1107 result.append(formatParser.quoteLiteral(x.toString())); 1108 } 1109 } 1110 return result.toString(); 1111 } 1112 1113 private void checkPattern2(String path, String value, List<CheckStatus> result) throws ParseException { 1114 XPathParts pathParts = XPathParts.getFrozenInstance(path); 1115 String calendar = pathParts.findAttributeValue("calendar", "type"); 1116 SimpleDateFormat x = icuServiceBuilder.getDateFormat(calendar, value); 1117 x.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 1118 result.add(new MyCheckStatus() 1119 .setFormat(x) 1120 .setCause(this).setMainType(CheckStatus.demoType)); 1121 } 1122 1123 static final UnicodeSet XGRAPHEME = new UnicodeSet("[[:mark:][:grapheme_extend:][:punctuation:]]"); 1124 static final UnicodeSet DIGIT = new UnicodeSet("[:decimal_number:]"); 1125 1126 static public class MyCheckStatus extends CheckStatus { 1127 private SimpleDateFormat df; 1128 1129 public MyCheckStatus setFormat(SimpleDateFormat df) { 1130 this.df = df; 1131 return this; 1132 } 1133 1134 @Override 1135 public SimpleDemo getDemo() { 1136 return new MyDemo().setFormat(df); 1137 } 1138 } 1139 1140 static class MyDemo extends FormatDemo { 1141 private SimpleDateFormat df; 1142 1143 @Override 1144 protected String getPattern() { 1145 return df.toPattern(); 1146 } 1147 1148 @Override 1149 protected String getSampleInput() { 1150 return neutralFormat.format(ExampleGenerator.DATE_SAMPLE); 1151 } 1152 1153 public MyDemo setFormat(SimpleDateFormat df) { 1154 this.df = df; 1155 return this; 1156 } 1157 1158 @Override 1159 protected void getArguments(Map<String, String> inout) { 1160 currentPattern = currentInput = currentFormatted = currentReparsed = "?"; 1161 Date d; 1162 try { 1163 currentPattern = inout.get("pattern"); 1164 if (currentPattern != null) 1165 df.applyPattern(currentPattern); 1166 else 1167 currentPattern = getPattern(); 1168 } catch (Exception e) { 1169 currentPattern = "Use format like: ##,###.##"; 1170 return; 1171 } 1172 try { 1173 currentInput = inout.get("input"); 1174 if (currentInput == null) { 1175 currentInput = getSampleInput(); 1176 } 1177 d = neutralFormat.parse(currentInput); 1178 } catch (Exception e) { 1179 currentInput = "Use neutral format like: 1993-11-31 13:49:02"; 1180 return; 1181 } 1182 try { 1183 currentFormatted = df.format(d); 1184 } catch (Exception e) { 1185 currentFormatted = "Can't format: " + e.getMessage(); 1186 return; 1187 } 1188 try { 1189 parsePosition.setIndex(0); 1190 Date n = df.parse(currentFormatted, parsePosition); 1191 if (parsePosition.getIndex() != currentFormatted.length()) { 1192 currentReparsed = "Couldn't parse past: " + "\u200E" 1193 + currentFormatted.substring(0, parsePosition.getIndex()) + "\u200E"; 1194 } else { 1195 currentReparsed = neutralFormat.format(n); 1196 } 1197 } catch (Exception e) { 1198 currentReparsed = "Can't parse: " + e.getMessage(); 1199 } 1200 } 1201 1202 } 1203 } 1204