1 package org.unicode.cldr.tool; 2 3 import java.util.HashMap; 4 import java.util.LinkedHashSet; 5 import java.util.List; 6 import java.util.Locale; 7 import java.util.Map; 8 import java.util.Map.Entry; 9 import java.util.Set; 10 import java.util.TreeMap; 11 import java.util.TreeSet; 12 import java.util.regex.Pattern; 13 14 import org.unicode.cldr.util.Builder; 15 import org.unicode.cldr.util.CldrUtility; 16 import org.unicode.cldr.util.FileProcessor; 17 import org.unicode.cldr.util.Iso639Data; 18 import org.unicode.cldr.util.IsoRegionData; 19 import org.unicode.cldr.util.PatternCache; 20 import org.unicode.cldr.util.StandardCodes; 21 import org.unicode.cldr.util.SupplementalDataInfo; 22 23 import com.google.common.base.Joiner; 24 import com.ibm.icu.impl.Relation; 25 import com.ibm.icu.impl.Row; 26 import com.ibm.icu.impl.Row.R2; 27 import com.ibm.icu.util.Output; 28 29 public class LocaleReplacements { 30 public static final Pattern WHITESPACE = PatternCache.get("\\s+"); 31 32 /** 33 * eg language, eng, <overlong,en> 34 */ 35 static Map<String, Map<String, Row.R2<Set<String>, String>>> type2item2replacementAndReason = new HashMap<>(); 36 static Map<String, Relation<String, Row.R2<String, Set<String>>>> type2reason2itemAndreplacement = new TreeMap<>(); 37 static Relation<String, String> fixed = Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class); 38 get(String old, Output<String> reason)39 public String get(String old, Output<String> reason) { 40 reason.value = null; 41 return old; 42 } 43 44 static { 45 Map<String, Map<String, Map<String, String>>> lstreg = StandardCodes.getLStreg(); 46 for (Entry<String, Map<String, Map<String, String>>> entry : lstreg.entrySet()) { 47 String type = entry.getKey(); 48 Map<String, Map<String, String>> subtype2data = entry.getValue(); 49 50 for (Entry<String, Map<String, String>> itemAndData : subtype2data.entrySet()) { 51 final Map<String, String> value = itemAndData.getValue(); 52 String deprecated = value.get("Deprecated"); 53 if (deprecated != null) { 54 String preferredValue = value.get("Preferred-Value"); 55 if (preferredValue == null) { 56 preferredValue = ""; 57 } 58 final String key = itemAndData.getKey(); 59 String type2 = type.equals("region") ? "territory" : type; addType2item2reasonNreplacement(type2, key, preferredValue, "deprecated", false)60 addType2item2reasonNreplacement(type2, key, preferredValue, "deprecated", false); 61 } 62 } 63 } 64 65 for (String lang : Iso639Data.getAvailable()) { 66 if (lang.length() != 2) continue; 67 String alpha3 = Iso639Data.toAlpha3(lang); 68 addType2item2reasonNreplacement("language", alpha3, lang, "overlong", false); 69 } 70 /* 71 * return IsoRegionData.get_alpha3(region); 72 * } 73 * }); 74 * addRegions(english, territories, "AC,CP,DG,EA,EU,IC,TA".split(","), new Transform<String,String>() { 75 * public String transform(String region) { 76 * return IsoRegionData.getNumeric(region); 77 */ 78 //Set<String> available2 = IsoRegionData.getAvailable(); 79 80 for (String region : IsoRegionData.getAvailable()) { 81 String alpha3 = IsoRegionData.get_alpha3(region); 82 addType2item2reasonNreplacement("territory", alpha3, region, "overlong", false); 83 String numeric = IsoRegionData.getNumeric(region); 84 addType2item2reasonNreplacement("territory", numeric, region, "overlong", false); 85 } 86 87 // Add overrides 88 FileProcessor myReader = new FileProcessor() { 89 @Override 90 protected boolean handleLine(int lineCount, String line) { 91 addType2item2reasonNreplacement(line); 92 return true; 93 } 94 }; 95 myReader.process(CldrUtility.class, "data/localeReplacements.txt")96 myReader.process(CldrUtility.class, "data/localeReplacements.txt"); 97 98 // fix up the data by recursing 99 100 for (Entry<String, Map<String, R2<Set<String>, String>>> entry : type2item2replacementAndReason.entrySet()) { 101 //String type = entry.getKey(); 102 final Map<String, R2<Set<String>, String>> item2replacementAndReason = entry.getValue(); 103 while (true) { 104 boolean keepGoing = false; 105 for (Entry<String, R2<Set<String>, String>> entry2 : item2replacementAndReason.entrySet()) { 106 String item = entry2.getKey(); 107 R2<Set<String>, String> replacementAndReason = entry2.getValue(); 108 Set<String> replacements = replacementAndReason.get0(); 109 //String reason = replacementAndReason.get1(); 110 Set<String> newReplacements = new LinkedHashSet<>(replacements.size()); 111 boolean gotChange = false; 112 for (String oldRep : replacements) { 113 R2<Set<String>, String> newRepAndReason = item2replacementAndReason.get(oldRep); 114 if (newRepAndReason != null) { fixed.put(item, oldRep + "\\t-->\\t" + newRepAndReason)115 fixed.put(item, oldRep + "\t-->\t" + newRepAndReason); newRepAndReason.get0()116 newReplacements.addAll(newRepAndReason.get0()); 117 gotChange = true; 118 } else { 119 newReplacements.add(oldRep); 120 } 121 } 122 if (gotChange) { 123 replacementAndReason.set0(newReplacements); 124 keepGoing = true; 125 } 126 } 127 if (!keepGoing) { 128 break; 129 } 130 } 131 } 132 133 for (Entry<String, Map<String, R2<Set<String>, String>>> entry : type2item2replacementAndReason.entrySet()) { 134 String type = entry.getKey(); 135 final Map<String, R2<Set<String>, String>> item2replacementAndReason = entry.getValue(); 136 for (Entry<String, R2<Set<String>, String>> entry2 : item2replacementAndReason.entrySet()) { 137 String item = entry2.getKey(); 138 R2<Set<String>, String> replacementAndReason = entry2.getValue(); 139 Set<String> replacements = replacementAndReason.get0(); 140 String reason = replacementAndReason.get1(); 141 142 Relation<String, R2<String, Set<String>>> reason2item2replacement = type2reason2itemAndreplacement 143 .get(type); 144 if (reason2item2replacement == null) { type2reason2itemAndreplacement.put( type, reason2item2replacement = Relation.of(new TreeMap<String, Set<R2<String, Set<String>>>>(), TreeSet.class))145 type2reason2itemAndreplacement.put( 146 type, 147 reason2item2replacement = Relation.of(new TreeMap<String, Set<R2<String, Set<String>>>>(), 148 TreeSet.class)); 149 } reason2item2replacement.put(reason, Row.of(item, replacements))150 reason2item2replacement.put(reason, Row.of(item, replacements)); 151 } 152 } 153 } 154 addType2item2reasonNreplacement(String line)155 private static void addType2item2reasonNreplacement(String line) { 156 String[] parts = WHITESPACE.split(line); 157 if (parts.length < 4) { 158 addType2item2reasonNreplacement(parts[0], parts[2], parts[1], "", true); 159 return; 160 } 161 // language macrolanguage bxk luy 162 for (int i = 3; i < parts.length; ++i) { 163 addType2item2reasonNreplacement(parts[0], parts[2], parts[i], parts[1], true); 164 } 165 } 166 addType2item2reasonNreplacement(String type, String key, String preferredValue, String reason, boolean ignoreDuplicates)167 private static void addType2item2reasonNreplacement(String type, String key, String preferredValue, String reason, 168 boolean ignoreDuplicates) { 169 if (key == null) { 170 return; 171 } 172 if (type.equals("legacy") || type.equals("redundant")) { 173 type = "language"; 174 } 175 176 key = key.replace('-', '_'); 177 if (type.equals("variant")) { 178 key = key.toUpperCase(Locale.US); 179 preferredValue = preferredValue.toUpperCase(Locale.US); 180 } 181 182 Map<String, R2<Set<String>, String>> item2replacementAndReason = type2item2replacementAndReason.get(type); 183 if (item2replacementAndReason == null) { 184 type2item2replacementAndReason.put(type, item2replacementAndReason = new HashMap<>()); 185 } 186 187 R2<Set<String>, String> oldReplacementAndReason = item2replacementAndReason.get(key); 188 if (oldReplacementAndReason != null) { 189 final String message = "duplicateReplacement\t" + type + "\t" + key + "\told: " 190 + oldReplacementAndReason + "\tnew:" + preferredValue + ", " + reason; 191 if (!ignoreDuplicates) { 192 throw new IllegalArgumentException(message); 193 } else { 194 fixed.put(key, message); 195 Set<String> list = oldReplacementAndReason.get0(); 196 list.add(preferredValue); 197 return; 198 } 199 } 200 Set<String> list = new LinkedHashSet<>(1); 201 if (!preferredValue.isEmpty()) { 202 list.add(preferredValue); 203 } 204 item2replacementAndReason.put(key, Row.of(list, reason)); 205 } 206 main(String[] args)207 public static void main(String[] args) { 208 Map<String, Map<String, R2<List<String>, String>>> localeAliasInfo = SupplementalDataInfo.getInstance() 209 .getLocaleAliasInfo(); 210 211 Set<String> newStuff = new TreeSet<>(); 212 Set<String> oldStuff = new TreeSet<>(); 213 for (Entry<String, Relation<String, R2<String, Set<String>>>> entry : type2reason2itemAndreplacement.entrySet()) { 214 String type = entry.getKey(); 215 for (Entry<String, R2<String, Set<String>>> entry2 : entry.getValue().entrySet()) { 216 String reason = entry2.getKey(); 217 R2<String, Set<String>> replacementAndReason = entry2.getValue(); 218 String key = replacementAndReason.get0(); 219 Set<String> replacements = replacementAndReason.get1(); 220 final String message = type + "\t" + reason + "\t" + key + "\t" 221 + Joiner.on(" ").join(replacements); 222 // System.out.println(message); 223 newStuff.add(message); 224 } 225 } 226 for (Entry<String, String> entry : fixed.entrySet()) { 227 System.out.println(entry.getKey() + "\t" + entry.getValue()); 228 } 229 // Returns type -> tag -> , like "language" -> "sh" -> <{"sr_Latn"}, reason> 230 for (Entry<String, Map<String, R2<List<String>, String>>> entry : localeAliasInfo.entrySet()) { 231 String type = entry.getKey(); 232 for (Entry<String, R2<List<String>, String>> entry2 : entry.getValue().entrySet()) { 233 String item = entry2.getKey(); 234 R2<List<String>, String> replacementAndReason = entry2.getValue(); 235 List<String> replacements = replacementAndReason.get0(); 236 String reason = replacementAndReason.get1(); 237 oldStuff.add(type + "\t" + reason + "\t" + item 238 + "\t" + (replacements == null ? "" : Joiner.on(" ").join(replacements))); 239 } 240 } 241 Set<Row.R2<String, String>> merged = new TreeSet<>(); 242 243 Set<String> oldNotNew = Builder.with(new TreeSet<>(oldStuff)).removeAll(newStuff).get(); 244 Set<String> newNotOld = Builder.with(new TreeSet<>(newStuff)).removeAll(oldStuff).get(); 245 //Set<String> shared = Builder.with(new TreeSet<String>(oldStuff)).retainAll(newStuff).get(); 246 // for (String s : shared) { 247 // merged.add(Row.of(s,"\tSAME")); 248 // } 249 for (String s : oldNotNew) { 250 merged.add(Row.of(s, "\tOLD")); 251 } 252 for (String s : newNotOld) { 253 merged.add(Row.of(s, "\tNEW")); 254 } 255 int i = 0; 256 for (R2<String, String> s : merged) { 257 System.out.println(++i + "\t" + s.get1() + "\t" + s.get0()); 258 } 259 System.out.println("DONE"); 260 } 261 }