1 package org.unicode.cldr.tool; 2 3 import java.io.File; 4 import java.io.PrintWriter; 5 import java.util.EnumMap; 6 import java.util.LinkedHashSet; 7 import java.util.Map; 8 import java.util.Set; 9 import java.util.TreeMap; 10 import java.util.TreeSet; 11 import java.util.regex.Matcher; 12 13 import org.unicode.cldr.util.AttributeValueValidity; 14 import org.unicode.cldr.util.AttributeValueValidity.AttributeValueSpec; 15 import org.unicode.cldr.util.AttributeValueValidity.LocaleSpecific; 16 import org.unicode.cldr.util.AttributeValueValidity.Status; 17 import org.unicode.cldr.util.CLDRConfig; 18 import org.unicode.cldr.util.CLDRPaths; 19 import org.unicode.cldr.util.ChainedMap; 20 import org.unicode.cldr.util.DayPeriodInfo; 21 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 22 import org.unicode.cldr.util.DayPeriodInfo.Type; 23 import org.unicode.cldr.util.DtdData; 24 import org.unicode.cldr.util.DtdData.Attribute; 25 import org.unicode.cldr.util.DtdData.Element; 26 import org.unicode.cldr.util.DtdType; 27 import org.unicode.cldr.util.SupplementalDataInfo; 28 import org.unicode.cldr.util.SupplementalDataInfo.PluralType; 29 import org.unicode.cldr.util.XMLFileReader; 30 import org.unicode.cldr.util.XMLFileReader.SimpleHandler; 31 import org.unicode.cldr.util.XPathParts; 32 33 import com.google.common.base.Joiner; 34 import com.google.common.base.Splitter; 35 import com.ibm.icu.impl.Row.R3; 36 import com.ibm.icu.util.Output; 37 38 public class VerifyAttributeValues extends SimpleHandler { 39 private static final File BASE_DIR = new File(CLDRPaths.BASE_DIRECTORY); 40 private static final SupplementalDataInfo supplementalData = CLDRConfig.getInstance().getSupplementalDataInfo(); 41 42 public static final Joiner SPACE_JOINER = Joiner.on(' '); 43 public static final Splitter SPACE_SPLITTER = Splitter.on(' ').trimResults().omitEmptyStrings(); 44 45 public final static class Errors { 46 47 @SuppressWarnings({ "unchecked", "rawtypes" }) 48 final ChainedMap.M3<String, AttributeValueSpec, String> file_element_attribute = ChainedMap.of(new TreeMap(), new TreeMap(), String.class); 49 put(String file, DtdType dtdType, String element, String attribute, String attributeValue, String problem)50 public void put(String file, DtdType dtdType, String element, String attribute, String attributeValue, String problem) { 51 file_element_attribute.put(file, new AttributeValueSpec(dtdType, element, attribute, attributeValue), problem); 52 } 53 getRows()54 public Iterable<R3<String, AttributeValueSpec, String>> getRows() { 55 return file_element_attribute.rows(); 56 } 57 } 58 59 private DtdData dtdData; // set from first element read 60 private final Errors file_element_attribute; 61 private final String file; 62 private final EnumMap<LocaleSpecific, Set<String>> localeSpecific = new EnumMap<>(LocaleSpecific.class); 63 private final Set<AttributeValueSpec> missing; 64 VerifyAttributeValues(String fileName, Errors file_element_attribute, Set<AttributeValueSpec> missing)65 private VerifyAttributeValues(String fileName, Errors file_element_attribute, Set<AttributeValueSpec> missing) { 66 this.file_element_attribute = file_element_attribute; 67 this.file = fileName.startsWith(BASE_DIR.toString()) ? fileName.substring(BASE_DIR.toString().length()) : fileName; 68 this.missing = missing; 69 } 70 71 /** 72 * Check the filename—note that the errors and missing are <b>added to<b>, so clear if you want a fresh start! 73 * @param fileName 74 * @param errors 75 * @param missing 76 */ check(String fileName, Errors errors, Set<AttributeValueSpec> missing)77 public static void check(String fileName, Errors errors, Set<AttributeValueSpec> missing) { 78 try { 79 final VerifyAttributeValues platformHandler = new VerifyAttributeValues(fileName, errors, missing); 80 new XMLFileReader() 81 .setHandler(platformHandler) 82 .read(fileName, -1, true); 83 } catch (Exception e) { 84 throw new IllegalArgumentException(fileName, e); 85 } 86 } 87 88 @Override handlePathValue(String path, String value)89 public void handlePathValue(String path, String value) { 90 XPathParts parts = XPathParts.getFrozenInstance(path); 91 if (dtdData == null) { 92 dtdData = DtdData.getInstance(DtdType.valueOf(parts.getElement(0))); 93 if (dtdData.dtdType == DtdType.ldml) { 94 String name = file; 95 String locale = name.substring(name.lastIndexOf('/') + 1, name.lastIndexOf('.')); 96 localeSpecific.put(LocaleSpecific.pluralCardinal, supplementalData.getPlurals(PluralType.cardinal, locale).getPluralRules().getKeywords()); 97 localeSpecific.put(LocaleSpecific.pluralOrdinal, supplementalData.getPlurals(PluralType.ordinal, locale).getPluralRules().getKeywords()); 98 localeSpecific.put(LocaleSpecific.dayPeriodFormat, getPeriods(Type.format, locale)); 99 localeSpecific.put(LocaleSpecific.dayPeriodSelection, getPeriods(Type.selection, locale)); 100 } else { 101 localeSpecific.clear(); 102 } 103 AttributeValueValidity.setLocaleSpecifics(localeSpecific); 104 } 105 106 for (int i = 0; i < parts.size(); ++i) { 107 if (parts.getAttributeCount(i) == 0) continue; 108 Map<String, String> attributes = parts.getAttributes(i); 109 String element = parts.getElement(i); 110 if (element.equals("attributeValues")) { 111 continue; // don't look at ourselves in the mirror 112 } 113 Element elementInfo = dtdData.getElementFromName().get(element); 114 115 for (String attribute : attributes.keySet()) { 116 Attribute attributeInfo = elementInfo.getAttributeNamed(attribute); 117 if (!attributeInfo.values.isEmpty()) { 118 // we don't need to check, since the DTD will enforce values 119 continue; 120 } 121 String attributeValue = attributes.get(attribute); 122 if (dtdData.isDeprecated(element, attribute, attributeValue)) { 123 file_element_attribute.put(file, dtdData.dtdType, element, attribute, attributeValue, "deprecated"); 124 continue; 125 } 126 127 Output<String> reason = new Output<>(); 128 Status haveTest = AttributeValueValidity.check(dtdData, element, attribute, attributeValue, reason); 129 switch (haveTest) { 130 case ok: 131 break; 132 case deprecated: 133 case illegal: 134 file_element_attribute.put(file, dtdData.dtdType, element, attribute, attributeValue, reason.value); 135 break; 136 case noTest: 137 missing.add(new AttributeValueSpec(dtdData.dtdType, element, attribute, attributeValue)); 138 break; 139 } 140 } 141 } 142 } 143 getPeriods(Type selection, String locale)144 private Set<String> getPeriods(Type selection, String locale) { 145 Set<String> result = new TreeSet<>(); 146 final DayPeriodInfo dayPeriods = supplementalData.getDayPeriods(Type.format, locale); 147 for (DayPeriod period : dayPeriods.getPeriods()) { 148 result.add(period.toString()); 149 } 150 result.add("am"); 151 result.add("pm"); 152 return new LinkedHashSet<>(result); 153 } 154 findAttributeValues(File file, int max, Matcher fileMatcher, Errors errors, Set<AttributeValueSpec> allMissing, PrintWriter out)155 public static int findAttributeValues(File file, int max, Matcher fileMatcher, Errors errors, Set<AttributeValueSpec> allMissing, PrintWriter out) { 156 final String name = file.getName(); 157 if (file.isDirectory() 158 && !name.equals("specs") 159 && !name.equals("tools") 160 && !file.toString().contains(".svn") 161 // && !name.equals("keyboards") // TODO reenable keyboards 162 ) { 163 int processed = 0; 164 int count = max; 165 for (File subfile : file.listFiles()) { 166 final String subname = subfile.getName(); 167 if (--count < 0 168 && !"en.xml".equals(subname) 169 && !"root.xml".equals(subname)) { 170 continue; 171 } 172 processed += findAttributeValues(subfile, max, fileMatcher, errors, allMissing, out); 173 } 174 if (out != null) { 175 out.println("Processed files: " + processed + " \tin " + file); 176 out.flush(); 177 } 178 return processed; 179 } else if (name.endsWith(".xml")) { 180 if (fileMatcher == null || fileMatcher.reset(name.substring(0, name.length() - 4)).matches()) { 181 check(file.toString(), errors, allMissing); 182 return 1; 183 } 184 } 185 return 0; 186 } 187 188 }