1 package org.unicode.cldr.test;
2 
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.util.Arrays;
7 import java.util.Date;
8 import java.util.EnumSet;
9 import java.util.HashSet;
10 import java.util.Iterator;
11 import java.util.Objects;
12 import java.util.Set;
13 import java.util.TreeMap;
14 import java.util.TreeSet;
15 import java.util.regex.Matcher;
16 
17 import org.unicode.cldr.tool.ToolConfig;
18 import org.unicode.cldr.util.CLDRConfig;
19 import org.unicode.cldr.util.CLDRFile;
20 import org.unicode.cldr.util.CLDRPaths;
21 import org.unicode.cldr.util.CldrUtility;
22 import org.unicode.cldr.util.DateTimeFormats;
23 import org.unicode.cldr.util.DtdType;
24 import org.unicode.cldr.util.Factory;
25 import org.unicode.cldr.util.InputStreamFactory;
26 import org.unicode.cldr.util.LanguageTagParser;
27 import org.unicode.cldr.util.Level;
28 import org.unicode.cldr.util.Organization;
29 import org.unicode.cldr.util.PathUtilities;
30 import org.unicode.cldr.util.PatternCache;
31 import org.unicode.cldr.util.PrettyPath;
32 import org.unicode.cldr.util.StandardCodes;
33 import org.unicode.cldr.util.XMLFileReader;
34 import org.unicode.cldr.util.XPathParts;
35 import org.xml.sax.ErrorHandler;
36 import org.xml.sax.InputSource;
37 import org.xml.sax.SAXException;
38 import org.xml.sax.SAXParseException;
39 import org.xml.sax.XMLReader;
40 
41 import com.ibm.icu.impl.Relation;
42 import com.ibm.icu.text.DateFormatSymbols;
43 import com.ibm.icu.text.SimpleDateFormat;
44 import com.ibm.icu.util.ULocale;
45 
46 /**
47  * Simple test that loads each file in the cldr directory, thus verifying that
48  * the DTD works, and also checks that the PrettyPaths work.
49  *
50  * @author markdavis
51  */
52 public class QuickCheck {
53     private static final Set<String> skipAttributes = new HashSet<>(Arrays.asList(new String[] {
54         "alt", "draft", "references" }));
55 
56     private static String localeRegex;
57 
58     private static boolean showInfo = false;
59 
60     private static String commonDirectory;
61     private static String mainDirectory;
62 
63     private static boolean resolved;
64 
65     private static Exception[] internalException = new Exception[1];
66 
67     private static boolean verbose;
68 
main(String[] args)69     public static void main(String[] args) throws IOException {
70         CLDRConfig testInfo = ToolConfig.getToolInstance();
71         Factory factory = testInfo.getCldrFactory();
72         checkStock(factory);
73         if (true) return;
74         verbose = CldrUtility.getProperty("verbose", "false", "true").matches("(?i)T|TRUE");
75         localeRegex = CldrUtility.getProperty("locale", ".*");
76 
77         showInfo = CldrUtility.getProperty("showinfo", "false", "true").matches("(?i)T|TRUE");
78 
79         commonDirectory = CLDRPaths.COMMON_DIRECTORY; // Utility.getProperty("common", Utility.COMMON_DIRECTORY);
80         // if (commonDirectory == null) commonDirectory = Utility.COMMON_DIRECTORY
81         // System.out.println("Main Source Directory: " + commonDirectory +
82         // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)");
83 
84         mainDirectory = CldrUtility.getProperty("main", CLDRPaths.COMMON_DIRECTORY + "/main");
85         // System.out.println("Main Source Directory: " + commonDirectory +
86         // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)");
87 
88         resolved = CldrUtility.getProperty("resolved", "false", "true").matches("(?i)T|TRUE");
89 
90         boolean paths = CldrUtility.getProperty("paths", "true").matches("(?i)T|TRUE");
91 
92         pretty = CldrUtility.getProperty("pretty", "true").matches("(?i)T|TRUE");
93 
94         double startTime = System.currentTimeMillis();
95         checkDtds();
96         double deltaTime = System.currentTimeMillis() - startTime;
97         System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds");
98 
99         if (paths) {
100             System.out.println("Checking paths");
101             checkPaths();
102             deltaTime = System.currentTimeMillis() - startTime;
103             System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds");
104             System.out.println("Basic Test Passes");
105         }
106     }
107 
checkDtds()108     private static void checkDtds() throws IOException {
109         checkDtds(commonDirectory + "supplemental");
110         checkDtds(commonDirectory + "collation");
111         checkDtds(commonDirectory + "main");
112         checkDtds(commonDirectory + "rbnf");
113         checkDtds(commonDirectory + "segments");
114         checkDtds(commonDirectory + "../test");
115         checkDtds(commonDirectory + "transforms");
116     }
117 
checkDtds(String directory)118     private static void checkDtds(String directory) throws IOException {
119         File directoryFile = new File(directory);
120         File[] listFiles = directoryFile.listFiles();
121         String normalizedPath = PathUtilities.getNormalizedPathString(directoryFile);
122         if (listFiles == null) {
123             throw new IllegalArgumentException("Empty directory: " + normalizedPath);
124         }
125         System.out.println("Checking files for DTD errors in: " + normalizedPath);
126         for (File fileName : listFiles) {
127             if (!fileName.toString().endsWith(".xml")) {
128                 continue;
129             }
130             check(fileName);
131         }
132     }
133 
134     static class MyErrorHandler implements ErrorHandler {
135         @Override
error(SAXParseException exception)136         public void error(SAXParseException exception) throws SAXException {
137             System.out.println("\nerror: " + XMLFileReader.showSAX(exception));
138             throw exception;
139         }
140 
141         @Override
fatalError(SAXParseException exception)142         public void fatalError(SAXParseException exception) throws SAXException {
143             System.out.println("\nfatalError: " + XMLFileReader.showSAX(exception));
144             throw exception;
145         }
146 
147         @Override
warning(SAXParseException exception)148         public void warning(SAXParseException exception) throws SAXException {
149             System.out.println("\nwarning: " + XMLFileReader.showSAX(exception));
150             throw exception;
151         }
152     }
153 
check(File systemID)154     public static void check(File systemID) {
155         try (InputStream fis = InputStreamFactory.createInputStream(systemID)) {
156 //            FileInputStream fis = new FileInputStream(systemID);
157             XMLReader xmlReader = XMLFileReader.createXMLReader(true);
158             xmlReader.setErrorHandler(new MyErrorHandler());
159             InputSource is = new InputSource(fis);
160             is.setSystemId(systemID.toString());
161             xmlReader.parse(is);
162 //            fis.close();
163         } catch (SAXException | IOException e) { // SAXParseException is a Subtype of SaxException
164             System.out.println("\t" + "Can't read " + systemID);
165             System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
166         }
167 //        catch (SAXException e) {
168 //            System.out.println("\t" + "Can't read " + systemID);
169 //            System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
170 //        } catch (IOException e) {
171 //            System.out.println("\t" + "Can't read " + systemID);
172 //            System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
173 //        }
174     }
175 
176     static Matcher skipPaths = PatternCache.get("/identity" + "|/alias" + "|\\[@alt=\"proposed").matcher("");
177 
178     private static boolean pretty;
179 
checkPaths()180     private static void checkPaths() {
181         Relation<String, String> distinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null);
182         Relation<String, String> nonDistinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null);
183         Factory cldrFactory = Factory.make(mainDirectory, localeRegex);
184         CLDRFile english = cldrFactory.make("en", true);
185 
186         Relation<String, String> pathToLocale = Relation.of(
187             new TreeMap<String, Set<String>>(CLDRFile.getComparator(DtdType.ldml)),
188             TreeSet.class, null);
189         for (String locale : cldrFactory.getAvailable()) {
190             // if (locale.equals("root") && !localeRegex.equals("root"))
191             // continue;
192             CLDRFile file;
193             try {
194                 file = cldrFactory.make(locale, resolved);
195             } catch (Exception e) {
196                 System.out.println("\nfatalError: " + e.getMessage());
197                 continue;
198             }
199             if (file.isNonInheriting())
200                 continue;
201             DisplayAndInputProcessor displayAndInputProcessor = new DisplayAndInputProcessor(file, false);
202 
203             System.out.println(locale + "\t-\t" + english.getName(locale));
204             DtdType dtdType = null;
205 
206             for (Iterator<String> it = file.iterator(); it.hasNext();) {
207                 String path = it.next();
208                 if (path.endsWith("/alias")) {
209                     continue;
210                 }
211                 String value = file.getStringValue(path);
212                 if (value == null) {
213                     throw new IllegalArgumentException(locale + "\tError: in null value at " + path);
214                 }
215                 String displayValue = displayAndInputProcessor.processForDisplay(path, value);
216                 if (!displayValue.equals(value)) {
217                     System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes display value <" + value
218                         + ">\t=>\t<" + displayValue + ">\t\t" + path);
219                 }
220                 String inputValue = displayAndInputProcessor.processInput(path, value, internalException);
221                 if (internalException[0] != null) {
222                     System.out.println("\t" + locale + "\tdisplayAndInputProcessor internal error <" + value
223                         + ">\t=>\t<" + inputValue + ">\t\t" + path);
224                     internalException[0].printStackTrace(System.out);
225                 }
226                 if (verbose && !inputValue.equals(value)) {
227                     displayAndInputProcessor.processInput(path, value, internalException); // for debugging
228                     System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes input value <" + value
229                         + ">\t=>\t<" + inputValue + ">\t\t" + path);
230                 }
231 
232                 pathToLocale.put(path, locale);
233 
234                 // also check for non-distinguishing attributes
235                 if (path.contains("/identity")) continue;
236 
237                 // make sure we don't have problem alts
238                 if (path.contains("proposed")) {
239                     String sourceLocale = file.getSourceLocaleID(path, null);
240                     if (locale.equals(sourceLocale)) {
241                         String nonAltPath = CLDRFile.getNondraftNonaltXPath(path);
242                         if (!path.equals(nonAltPath)) {
243                             String nonAltLocale = file.getSourceLocaleID(nonAltPath, null);
244                             String nonAltValue = file.getStringValue(nonAltPath);
245                             if (nonAltValue == null || !locale.equals(nonAltLocale)) {
246                                 System.out.println("\t" + locale + "\tProblem alt=proposed <" + value + ">\t\t" + path);
247                             }
248                         }
249                     }
250                 }
251 
252                 String fullPath = file.getFullXPath(path);
253                 XPathParts parts = XPathParts.getFrozenInstance(fullPath);
254                 if (dtdType == null) {
255                     dtdType = DtdType.valueOf(parts.getElement(0));
256                 }
257                 for (int i = 0; i < parts.size(); ++i) {
258                     if (parts.getAttributeCount(i) == 0) continue;
259                     String element = parts.getElement(i);
260                     for (String attribute : parts.getAttributeKeys(i)) {
261                         if (skipAttributes.contains(attribute)) continue;
262                         if (CLDRFile.isDistinguishing(dtdType, element, attribute)) {
263                             distinguishing.put(element, attribute);
264                         } else {
265                             nonDistinguishing.put(element, attribute);
266                         }
267                     }
268                 }
269             }
270         }
271         System.out.println();
272 
273         System.out.format("Distinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, distinguishing);
274         System.out.format("Nondistinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, nonDistinguishing);
275         System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, skipAttributes);
276 
277         if (pretty) {
278             if (showInfo) {
279                 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Path to PrettyPath mapping"
280                     + CldrUtility.LINE_SEPARATOR);
281             }
282             PrettyPath prettyPath = new PrettyPath().setShowErrors(true);
283             Set<String> badPaths = new TreeSet<>();
284             for (String path : pathToLocale.keySet()) {
285                 String prettied = prettyPath.getPrettyPath(path, false);
286                 if (showInfo) System.out.println(prettied + "\t\t" + path);
287                 if (prettied.contains("%%") && !path.contains("/alias")) {
288                     badPaths.add(path);
289                 }
290             }
291             // now remove root
292 
293             if (showInfo) {
294                 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Paths not in root"
295                     + CldrUtility.LINE_SEPARATOR);
296             }
297 
298             CLDRFile root = cldrFactory.make("root", true);
299             for (Iterator<String> it = root.iterator(); it.hasNext();) {
300                 pathToLocale.removeAll(it.next());
301             }
302             if (showInfo) for (String path : pathToLocale.keySet()) {
303                 if (skipPaths.reset(path).find()) {
304                     continue;
305                 }
306                 System.out.println(path + "\t" + pathToLocale.getAll(path));
307             }
308 
309             if (badPaths.size() != 0) {
310                 System.out.println("Error: " + badPaths.size()
311                     + " Paths were not prettied: use -DSHOW and look for ones with %% in them.");
312             }
313         }
314     }
315 
checkStock(Factory factory)316     static void checkStock(Factory factory) {
317         String[][] items = {
318             { "full", "yMMMMEEEEd", "jmmsszzzz" },
319             { "long", "yMMMMd", "jmmssz" },
320             { "medium", "yMMMd", "jmmss" },
321             { "short", "yMd", "jmm" },
322         };
323         String calendarID = "gregorian";
324         String datetimePathPrefix = "//ldml/dates/calendars/calendar[@type=\"" + calendarID + "\"]/";
325 
326         int total = 0;
327         int mismatch = 0;
328         LanguageTagParser ltp = new LanguageTagParser();
329         Iterable<String> locales = StandardCodes.make().getLocaleCoverageLocales(Organization.cldr, EnumSet.of(Level.MODERN));
330         for (String locale : locales) {
331             if (!ltp.set(locale).getRegion().isEmpty()) {
332                 continue;
333             }
334             CLDRFile file = factory.make(locale, false);
335             DateTimeFormats dtf = new DateTimeFormats();
336             dtf.set(file, "gregorian", false);
337             for (String[] stockInfo : items) {
338                 String length = stockInfo[0];
339                 //ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="full"]/dateFormat[@type="standard"]/pattern[@type="standard"]
340                 String path = datetimePathPrefix + "dateFormats/dateFormatLength[@type=\"" +
341                     length + "\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]";
342                 String stockDatePattern = file.getStringValue(path);
343                 String flexibleDatePattern = dtf.getBestPattern(stockInfo[1]);
344                 mismatch += showStatus(++total, locale, "date", length, stockInfo[1], stockDatePattern, flexibleDatePattern);
345                 path = datetimePathPrefix + "timeFormats/timeFormatLength[@type=\"" + length +
346                     "\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]";
347                 String stockTimePattern = file.getStringValue(path);
348                 String flexibleTimePattern = dtf.getBestPattern(stockInfo[2]);
349                 mismatch += showStatus(++total, locale, "time", length, stockInfo[2], stockTimePattern, flexibleTimePattern);
350             }
351         }
352         System.out.println("Mismatches:\t" + mismatch + "\tTotal:\t" + total);
353     }
354 
355     static final Date SAMPLE_DATE = new Date(2013 - 1900, 1 - 1, 29, 13, 59, 59);
356 
showStatus(int total, String locale, String type, String length, String skeleton, String stockPattern, String flexiblePattern)357     private static int showStatus(int total, String locale, String type, String length,
358         String skeleton, String stockPattern, String flexiblePattern) {
359         ULocale ulocale = new ULocale(locale);
360         DateFormatSymbols dfs = new DateFormatSymbols(ulocale); // just use ICU for now
361         boolean areSame = Objects.equals(stockPattern, flexiblePattern);
362         System.out.println(total
363             + "\t" + (areSame ? "ok" : "diff")
364             + "\t" + locale
365             + "\t" + type
366             + "\t" + length
367             + "\t" + skeleton
368             + "\t" + stockPattern
369             + "\t" + (areSame ? "" : flexiblePattern)
370             + "\t'" + new SimpleDateFormat(stockPattern, dfs, ulocale).format(SAMPLE_DATE)
371             + "\t'" + (areSame ? "" : new SimpleDateFormat(flexiblePattern, dfs, ulocale).format(SAMPLE_DATE)));
372         return areSame ? 0 : 1;
373     }
374 
375 }
376