1 package org.unicode.cldr.tool;
2 
3 import java.io.File;
4 import java.io.PrintWriter;
5 import java.util.EnumMap;
6 import java.util.LinkedHashSet;
7 import java.util.Map;
8 import java.util.Set;
9 import java.util.TreeMap;
10 import java.util.TreeSet;
11 import java.util.regex.Matcher;
12 
13 import org.unicode.cldr.util.AttributeValueValidity;
14 import org.unicode.cldr.util.AttributeValueValidity.AttributeValueSpec;
15 import org.unicode.cldr.util.AttributeValueValidity.LocaleSpecific;
16 import org.unicode.cldr.util.AttributeValueValidity.Status;
17 import org.unicode.cldr.util.CLDRConfig;
18 import org.unicode.cldr.util.CLDRPaths;
19 import org.unicode.cldr.util.ChainedMap;
20 import org.unicode.cldr.util.DayPeriodInfo;
21 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod;
22 import org.unicode.cldr.util.DayPeriodInfo.Type;
23 import org.unicode.cldr.util.DtdData;
24 import org.unicode.cldr.util.DtdData.Attribute;
25 import org.unicode.cldr.util.DtdData.Element;
26 import org.unicode.cldr.util.DtdType;
27 import org.unicode.cldr.util.SupplementalDataInfo;
28 import org.unicode.cldr.util.SupplementalDataInfo.PluralType;
29 import org.unicode.cldr.util.XMLFileReader;
30 import org.unicode.cldr.util.XMLFileReader.SimpleHandler;
31 import org.unicode.cldr.util.XPathParts;
32 
33 import com.google.common.base.Joiner;
34 import com.google.common.base.Splitter;
35 import com.ibm.icu.impl.Row.R3;
36 import com.ibm.icu.util.Output;
37 
38 public class VerifyAttributeValues extends SimpleHandler {
39     private static final File BASE_DIR = new File(CLDRPaths.BASE_DIRECTORY);
40     private static final SupplementalDataInfo supplementalData = CLDRConfig.getInstance().getSupplementalDataInfo();
41 
42     public static final Joiner SPACE_JOINER = Joiner.on(' ');
43     public static final Splitter SPACE_SPLITTER = Splitter.on(' ').trimResults().omitEmptyStrings();
44 
45     public final static class Errors {
46 
47         @SuppressWarnings({ "unchecked", "rawtypes" })
48         final ChainedMap.M3<String, AttributeValueSpec, String> file_element_attribute = ChainedMap.of(new TreeMap(), new TreeMap(), String.class);
49 
put(String file, DtdType dtdType, String element, String attribute, String attributeValue, String problem)50         public void put(String file, DtdType dtdType, String element, String attribute, String attributeValue, String problem) {
51             file_element_attribute.put(file, new AttributeValueSpec(dtdType, element, attribute, attributeValue), problem);
52         }
53 
getRows()54         public Iterable<R3<String, AttributeValueSpec, String>> getRows() {
55             return file_element_attribute.rows();
56         }
57     }
58 
59     private DtdData dtdData; // set from first element read
60     private final Errors file_element_attribute;
61     private final String file;
62     private final EnumMap<LocaleSpecific, Set<String>> localeSpecific = new EnumMap<>(LocaleSpecific.class);
63     private final Set<AttributeValueSpec> missing;
64 
VerifyAttributeValues(String fileName, Errors file_element_attribute, Set<AttributeValueSpec> missing)65     private VerifyAttributeValues(String fileName, Errors file_element_attribute, Set<AttributeValueSpec> missing) {
66         this.file_element_attribute = file_element_attribute;
67         this.file = fileName.startsWith(BASE_DIR.toString()) ? fileName.substring(BASE_DIR.toString().length()) : fileName;
68         this.missing = missing;
69     }
70 
71     /**
72      * Check the filename—note that the errors and missing are <b>added to<b>, so clear if you want a fresh start!
73      * @param fileName
74      * @param errors
75      * @param missing
76      */
check(String fileName, Errors errors, Set<AttributeValueSpec> missing)77     public static void check(String fileName, Errors errors, Set<AttributeValueSpec> missing) {
78         try {
79             final VerifyAttributeValues platformHandler = new VerifyAttributeValues(fileName, errors, missing);
80             new XMLFileReader()
81                 .setHandler(platformHandler)
82                 .read(fileName, -1, true);
83         } catch (Exception e) {
84             throw new IllegalArgumentException(fileName, e);
85         }
86     }
87 
88     @Override
handlePathValue(String path, String value)89     public void handlePathValue(String path, String value) {
90         XPathParts parts = XPathParts.getFrozenInstance(path);
91         if (dtdData == null) {
92             dtdData = DtdData.getInstance(DtdType.valueOf(parts.getElement(0)));
93             if (dtdData.dtdType == DtdType.ldml) {
94                 String name = file;
95                 String locale = name.substring(name.lastIndexOf('/') + 1, name.lastIndexOf('.'));
96                 localeSpecific.put(LocaleSpecific.pluralCardinal, supplementalData.getPlurals(PluralType.cardinal, locale).getPluralRules().getKeywords());
97                 localeSpecific.put(LocaleSpecific.pluralOrdinal, supplementalData.getPlurals(PluralType.ordinal, locale).getPluralRules().getKeywords());
98                 localeSpecific.put(LocaleSpecific.dayPeriodFormat, getPeriods(Type.format, locale));
99                 localeSpecific.put(LocaleSpecific.dayPeriodSelection, getPeriods(Type.selection, locale));
100             } else {
101                 localeSpecific.clear();
102             }
103             AttributeValueValidity.setLocaleSpecifics(localeSpecific);
104         }
105 
106         for (int i = 0; i < parts.size(); ++i) {
107             if (parts.getAttributeCount(i) == 0) continue;
108             Map<String, String> attributes = parts.getAttributes(i);
109             String element = parts.getElement(i);
110             if (element.equals("attributeValues")) {
111                 continue; // don't look at ourselves in the mirror
112             }
113             Element elementInfo = dtdData.getElementFromName().get(element);
114 
115             for (String attribute : attributes.keySet()) {
116                 Attribute attributeInfo = elementInfo.getAttributeNamed(attribute);
117                 if (!attributeInfo.values.isEmpty()) {
118                     // we don't need to check, since the DTD will enforce values
119                     continue;
120                 }
121                 String attributeValue = attributes.get(attribute);
122                 if (dtdData.isDeprecated(element, attribute, attributeValue)) {
123                     file_element_attribute.put(file, dtdData.dtdType, element, attribute, attributeValue, "deprecated");
124                     continue;
125                 }
126 
127                 Output<String> reason = new Output<>();
128                 Status haveTest = AttributeValueValidity.check(dtdData, element, attribute, attributeValue, reason);
129                 switch (haveTest) {
130                 case ok:
131                     break;
132                 case deprecated:
133                 case illegal:
134                     file_element_attribute.put(file, dtdData.dtdType, element, attribute, attributeValue, reason.value);
135                     break;
136                 case noTest:
137                     missing.add(new AttributeValueSpec(dtdData.dtdType, element, attribute, attributeValue));
138                     break;
139                 }
140             }
141         }
142     }
143 
getPeriods(Type selection, String locale)144     private Set<String> getPeriods(Type selection, String locale) {
145         Set<String> result = new TreeSet<>();
146         final DayPeriodInfo dayPeriods = supplementalData.getDayPeriods(Type.format, locale);
147         for (DayPeriod period : dayPeriods.getPeriods()) {
148             result.add(period.toString());
149         }
150         result.add("am");
151         result.add("pm");
152         return new LinkedHashSet<>(result);
153     }
154 
findAttributeValues(File file, int max, Matcher fileMatcher, Errors errors, Set<AttributeValueSpec> allMissing, PrintWriter out)155     public static int findAttributeValues(File file, int max, Matcher fileMatcher, Errors errors, Set<AttributeValueSpec> allMissing, PrintWriter out) {
156         final String name = file.getName();
157         if (file.isDirectory()
158             && !name.equals("specs")
159             && !name.equals("tools")
160             && !file.toString().contains(".svn")
161         // && !name.equals("keyboards") // TODO reenable keyboards
162         ) {
163             int processed = 0;
164             int count = max;
165             for (File subfile : file.listFiles()) {
166                 final String subname = subfile.getName();
167                 if (--count < 0
168                     && !"en.xml".equals(subname)
169                     && !"root.xml".equals(subname)) {
170                     continue;
171                 }
172                 processed += findAttributeValues(subfile, max, fileMatcher, errors, allMissing, out);
173             }
174             if (out != null) {
175                 out.println("Processed files: " + processed + " \tin " + file);
176                 out.flush();
177             }
178             return processed;
179         } else if (name.endsWith(".xml")) {
180             if (fileMatcher == null || fileMatcher.reset(name.substring(0, name.length() - 4)).matches()) {
181                 check(file.toString(), errors, allMissing);
182                 return 1;
183             }
184         }
185         return 0;
186     }
187 
188 }