package org.unicode.cldr.util; import java.io.File; import java.io.FilenameFilter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.unicode.cldr.test.CheckCLDR.Phase; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableSet; import com.ibm.icu.dev.test.TestFmwk; import com.ibm.icu.dev.test.TestLog; import com.ibm.icu.text.Collator; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; import com.ibm.icu.util.VersionInfo; public class CLDRConfig extends Properties { public static boolean SKIP_SEED = System.getProperty("CLDR_SKIP_SEED") != null; /** * */ private static final long serialVersionUID = -2605254975303398336L; public static boolean DEBUG = false; private static CLDRConfig INSTANCE = null; public static final String SUBCLASS = CLDRConfig.class.getName() + "Impl"; /** * Object to use for synchronization when interacting with Factory */ private static final Object CLDR_FACTORY_SYNC = new Object(); /** * Object to use for synchronization when interacting with Factory */ private static final Object FULL_FACTORY_SYNC = new Object(); /** * Object to use for synchronization when interacting with Factory */ private static final Object EXEMPLARS_FACTORY_SYNC = new Object(); /** * Object to use for synchronization when interacting with Factory */ private static final Object COLLATION_FACTORY_SYNC = new Object(); /** * Object to use for synchronization when interacting with Factory */ private static final Object RBNF_FACTORY_SYNC = new Object(); /** * Object to use for synchronization when interacting with Factory */ private static final Object ANNOTATIONS_FACTORY_SYNC = new Object(); /** * Object to use for synchronization when interacting with Factory */ private static final Object SUBDIVISION_FACTORY_SYNC = new Object(); /** * Object used for synchronization when interacting with SupplementalData */ private static final Object SUPPLEMENTAL_DATA_SYNC = new Object(); /** * Object used for synchronization in getCollator() */ private static final Object GET_COLLATOR_SYNC = new Object(); /** * Object used for synchronization in getCollator() */ private static final Object GET_COLLATOR_SYNC_ROOT = new Object(); /** * Object used for synchronization in getStandardCodes() */ private static final Object GET_STANDARD_CODES_SYNC = new Object(); /** * Object used for synchronization in getCoverageInfo() */ private static Object COVERAGE_INFO_SYNC = new Object(); public enum Environment { LOCAL, // < == unknown. SMOKETEST, // staging area PRODUCTION, // production server! UNITTEST // unit test setting } public static CLDRConfig getInstance() { synchronized (CLDRConfig.class) { if (INSTANCE == null) { final String env = System.getProperty("CLDR_ENVIRONMENT"); if (env != null && env.equals(Environment.UNITTEST.name())) { if (DEBUG) { System.err.println("-DCLDR_ENVIRONMENT=" + env + " - not loading " + SUBCLASS); } } else { try { // System.err.println("Attempting to new up a " + SUBCLASS); INSTANCE = (CLDRConfig) (Class.forName(SUBCLASS).newInstance()); if (INSTANCE != null) { System.err.println("Using CLDRConfig: " + INSTANCE.toString() + " - " + INSTANCE.getClass().getName()); } else { if (DEBUG) { // Probably occurred because ( config.getEnvironment() == Environment.UNITTEST ) // see CLDRConfigImpl System.err.println("Note: CLDRConfig Subclass " + SUBCLASS + ".newInstance() returned NULL " + "( this is OK if we aren't inside the SurveyTool's web server )"); } } } catch (ClassNotFoundException e) { // Expected - when not under cldr-apps, this class doesn't exist. } catch (InstantiationException | IllegalAccessException e) { // TODO: log a useful message } } } if (INSTANCE == null) { INSTANCE = new CLDRConfig(); CldrUtility.checkValidDirectory(INSTANCE.getProperty("CLDR_DIR"), "You have to set -DCLDR_DIR="); } } return INSTANCE; } String initStack = null; protected CLDRConfig() { initStack = StackTracker.currentStack(); } public String getInitStack() { return initStack; } private CoverageInfo coverageInfo = null; private SupplementalDataInfo supplementalDataInfo; private StandardCodes sc; private Factory cldrFactory; private Factory fullFactory; private Factory mainAndAnnotationsFactory; private Factory commonAndSeedAndMainAndAnnotationsFactory; private Factory exemplarsFactory; private Factory collationFactory; private Factory rbnfFactory; private Factory annotationsFactory; private Factory subdivisionFactory; private Factory supplementalFactory; private RuleBasedCollator colRoot; private RuleBasedCollator col; private Phase phase = null; // default private LoadingCache cldrFileResolvedCache = CacheBuilder.newBuilder() .maximumSize(200) .build( new CacheLoader() { @Override public CLDRFile load(String locale) { return getFullCldrFactory().make(locale, true); } }); // Unresolved CLDRFiles are smaller than resolved, so we can cache more of them safely. private LoadingCache cldrFileUnresolvedCache = CacheBuilder.newBuilder() .maximumSize(1000) .build( new CacheLoader() { @Override public CLDRFile load(String locale) { return getFullCldrFactory().make(locale, false); } }); private TestLog testLog = null; // base level public TestLog setTestLog(TestLog log) { testLog = log; return log; } // for calling "run" public TestFmwk setTestLog(TestFmwk log) { testLog = log; return log; } protected void logln(String msg) { if (testLog != null) { testLog.logln(msg); } else { System.out.println(msg); System.out.flush(); } } public SupplementalDataInfo getSupplementalDataInfo() { synchronized (SUPPLEMENTAL_DATA_SYNC) { if (supplementalDataInfo == null) { supplementalDataInfo = SupplementalDataInfo.getInstance(CLDRPaths.DEFAULT_SUPPLEMENTAL_DIRECTORY); } } return supplementalDataInfo; } public StandardCodes getStandardCodes() { synchronized (GET_STANDARD_CODES_SYNC) { if (sc == null) { sc = StandardCodes.make(); } } return sc; } public CoverageInfo getCoverageInfo() { synchronized (COVERAGE_INFO_SYNC) { if (coverageInfo == null) { coverageInfo = new CoverageInfo(getSupplementalDataInfo()); } } return coverageInfo; } public Factory getCldrFactory() { synchronized (CLDR_FACTORY_SYNC) { if (cldrFactory == null) { cldrFactory = Factory.make(CLDRPaths.MAIN_DIRECTORY, ".*"); } } return cldrFactory; } public Factory getExemplarsFactory() { synchronized (EXEMPLARS_FACTORY_SYNC) { if (exemplarsFactory == null) { exemplarsFactory = Factory.make(CLDRPaths.EXEMPLARS_DIRECTORY, ".*"); } } return exemplarsFactory; } public Factory getCollationFactory() { synchronized (COLLATION_FACTORY_SYNC) { if (collationFactory == null) { collationFactory = Factory.make(CLDRPaths.COLLATION_DIRECTORY, ".*"); } } return collationFactory; } public Factory getRBNFFactory() { synchronized (RBNF_FACTORY_SYNC) { if (rbnfFactory == null) { rbnfFactory = Factory.make(CLDRPaths.RBNF_DIRECTORY, ".*"); } } return rbnfFactory; } public Factory getAnnotationsFactory() { synchronized (ANNOTATIONS_FACTORY_SYNC) { if (annotationsFactory == null) { annotationsFactory = Factory.make(CLDRPaths.ANNOTATIONS_DIRECTORY, ".*"); } } return annotationsFactory; } public Factory getSubdivisionFactory() { synchronized (SUBDIVISION_FACTORY_SYNC) { if (subdivisionFactory == null) { subdivisionFactory = Factory.make(CLDRPaths.SUBDIVISIONS_DIRECTORY, ".*"); } } return subdivisionFactory; } public Factory getMainAndAnnotationsFactory() { synchronized (FULL_FACTORY_SYNC) { if (mainAndAnnotationsFactory == null) { File[] paths = { new File(CLDRPaths.MAIN_DIRECTORY), new File(CLDRPaths.ANNOTATIONS_DIRECTORY) }; mainAndAnnotationsFactory = SimpleFactory.make(paths, ".*"); } } return mainAndAnnotationsFactory; } static Factory allFactory; public Factory getCommonSeedExemplarsFactory() { synchronized (FULL_FACTORY_SYNC) { if (allFactory == null) { allFactory = SimpleFactory.make(addStandardSubdirectories(CLDR_DATA_DIRECTORIES), ".*"); } } return allFactory; } public Factory getCommonAndSeedAndMainAndAnnotationsFactory() { synchronized (FULL_FACTORY_SYNC) { if (commonAndSeedAndMainAndAnnotationsFactory == null) { File[] paths = { new File(CLDRPaths.MAIN_DIRECTORY), new File(CLDRPaths.ANNOTATIONS_DIRECTORY), SKIP_SEED ? null : new File(CLDRPaths.SEED_DIRECTORY), SKIP_SEED ? null : new File(CLDRPaths.SEED_ANNOTATIONS_DIRECTORY) }; commonAndSeedAndMainAndAnnotationsFactory = SimpleFactory.make(paths, ".*"); } } return commonAndSeedAndMainAndAnnotationsFactory; } public Factory getFullCldrFactory() { synchronized (FULL_FACTORY_SYNC) { if (fullFactory == null) { File[] paths = { new File(CLDRPaths.MAIN_DIRECTORY), SKIP_SEED ? null : new File(CLDRPaths.SEED_DIRECTORY)}; fullFactory = SimpleFactory.make(paths, ".*"); } } return fullFactory; } public Factory getSupplementalFactory() { synchronized (CLDR_FACTORY_SYNC) { if (supplementalFactory == null) { supplementalFactory = Factory.make(CLDRPaths.DEFAULT_SUPPLEMENTAL_DIRECTORY, ".*"); } } return supplementalFactory; } public CLDRFile getEnglish() { return getCLDRFile("en", true); } public CLDRFile getCLDRFile(String locale, boolean resolved) { return resolved ? cldrFileResolvedCache.getUnchecked(locale) : cldrFileUnresolvedCache.getUnchecked(locale); } public CLDRFile getRoot() { return getCLDRFile("root", true); } public Collator getCollatorRoot() { synchronized (GET_COLLATOR_SYNC_ROOT) { if (colRoot == null) { CLDRFile root = getCollationFactory().make("root", false); String rules = root.getStringValue("//ldml/collations/collation[@type=\"emoji\"][@visibility=\"external\"]/cr"); try { colRoot = new RuleBasedCollator(rules); } catch (Exception e) { colRoot = (RuleBasedCollator) getCollator(); return colRoot; } colRoot.setStrength(Collator.IDENTICAL); colRoot.setNumericCollation(true); colRoot.freeze(); } } return colRoot; } public Collator getCollator() { synchronized (GET_COLLATOR_SYNC) { if (col == null) { col = (RuleBasedCollator) Collator.getInstance(ULocale.forLanguageTag("en-u-co-emoji")); col.setStrength(Collator.IDENTICAL); col.setNumericCollation(true); col.freeze(); } } return col; } public synchronized Phase getPhase() { if (phase == null) { if (getEnvironment() == Environment.UNITTEST) { phase = Phase.BUILD; } else { phase = Phase.SUBMISSION; } } return phase; } @Override public String getProperty(String key, String d) { String result = getProperty(key); if (result == null) return d; return result; } private Set shown = new HashSet<>(); private Map localSet = null; @Override public String get(Object key) { return getProperty(key.toString()); } @Override public String getProperty(String key) { String result = null; if (localSet != null) { result = localSet.get(key); } if (result == null) { result = System.getProperty(key); } if (result == null) { result = System.getProperty(key.toUpperCase(Locale.ENGLISH)); } if (result == null) { result = System.getProperty(key.toLowerCase(Locale.ENGLISH)); } if (result == null) { result = System.getenv(key); } if (DEBUG && !shown.contains(key)) { logln("-D" + key + "=" + result); shown.add(key); } return result; } private Environment curEnvironment = null; public Environment getEnvironment() { if (curEnvironment == null) { String envString = getProperty("CLDR_ENVIRONMENT"); if (envString != null) { curEnvironment = Environment.valueOf(envString.trim()); } if (curEnvironment == null) { curEnvironment = getDefaultEnvironment(); } } return curEnvironment; } /** * If no environment is defined, what is the default? * @return */ protected Environment getDefaultEnvironment() { return Environment.LOCAL; } public void setEnvironment(Environment environment) { curEnvironment = environment; } /** * For test use only. Will throw an exception in non test environments. * @param k * @param v * @return */ @Override public Object setProperty(String k, String v) { if (getEnvironment() != Environment.UNITTEST) { throw new InternalError("setProperty() only valid in UNITTEST Environment."); } if (localSet == null) { localSet = new ConcurrentHashMap<>(); } shown.remove(k); // show it again with -D return localSet.put(k, v); } @Override public Object put(Object k, Object v) { return setProperty(k.toString(), v.toString()); } /** * Return true if the value indicates 'true' * @param k key * @param defVal default value * @return */ public boolean getProperty(String k, boolean defVal) { String val = getProperty(k, defVal ? "true" : null); if (val == null) { return false; } else { val = val.trim().toLowerCase(); return (val.equals("true") || val.equals("t") || val.equals("yes") || val.equals("y")); } } /** * Return a numeric property * @param k key * @param defVal default value * @return */ public int getProperty(String k, int defVal) { String val = getProperty(k, Integer.toString(defVal)); if (val == null) { return defVal; } else { try { return Integer.parseInt(val); } catch (NumberFormatException nfe) { return defVal; } } } private static class FileWrapper { private File cldrDir = null; private FileWrapper() { String dir = getInstance().getProperty("CLDR_DIR", null); if (dir != null) { cldrDir = new File(dir); } else { cldrDir = null; } } public File getCldrDir() { return this.cldrDir; } // singleton private static FileWrapper fileWrapperInstance = new FileWrapper(); public static FileWrapper getFileWrapperInstance() { return fileWrapperInstance; } } public File getCldrBaseDirectory() { return FileWrapper.getFileWrapperInstance().getCldrDir(); } /** * Get all CLDR XML files in the CLDR base directory. * @return */ public Set getAllCLDRFilesEndingWith(final String suffix) { FilenameFilter filter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.endsWith(suffix) && !isJunkFile(name); // skip junk and backup files } }; final File dir = getCldrBaseDirectory(); Set list; list = getCLDRFilesMatching(filter, dir); return list; } /** * Return all CLDR data files matching this filter * @param filter matching filter * @param baseDir base directory, see {@link #getCldrBaseDirectory()} * @return set of files */ public Set getCLDRFilesMatching(FilenameFilter filter, final File baseDir) { Set list; list = new LinkedHashSet<>(); for (String subdir : getCLDRDataDirectories()) { getFilesRecursively(new File(baseDir, subdir), filter, list); } return list; } /** * TODO: better place for these constants? */ private static final String COMMON_DIR = "common"; /** * TODO: better place for these constants? */ private static final String EXEMPLARS_DIR = "exemplars"; /** * TODO: better place for these constants? */ private static final String SEED_DIR = "seed"; /** * TODO: better place for these constants? */ private static final String KEYBOARDS_DIR = "keyboards"; private static final String MAIN_DIR = "main"; private static final String ANNOTATIONS_DIR = "annotations"; private static final String SUBDIVISIONS_DIR = "subdivisions"; /** * TODO: better place for these constants? */ private static final String CLDR_DATA_DIRECTORIES[] = { COMMON_DIR, SEED_DIR, KEYBOARDS_DIR, EXEMPLARS_DIR }; private static final ImmutableSet STANDARD_SUBDIRS = ImmutableSet.of(MAIN_DIR, ANNOTATIONS_DIR, SUBDIVISIONS_DIR); /** * Get a list of CLDR directories containing actual data * @return an iterable containing the names of all CLDR data subdirectories */ public Iterable getCLDRDataDirectories() { return Arrays.asList(CLDR_DATA_DIRECTORIES); } /** * Given comma separated list "common" or "common,main" return a list of actual files. * Adds subdirectories in STANDARD_SUBDIRS as necessary. */ public File[] getCLDRDataDirectories(String list) { final File dir = getCldrBaseDirectory(); String stubs[] = list.split(","); File[] ret = new File[stubs.length]; for (int i = 0; i < stubs.length; i++) { ret[i] = new File(dir, stubs[i]); } return ret; } /** * Add subdirectories to file list as needed, from STANDARD_SUBDIRS. *
  • map "common","seed" -> "common/main", "seed/main" *
  • but common/main -> common/main *
*/ public File[] addStandardSubdirectories(String... base) { return addStandardSubdirectories(fileArrayFromStringArray(getCldrBaseDirectory(), base)); } public File[] addStandardSubdirectories(File... base) { List ret = new ArrayList<>(); //File[] ret = new File[base.length * 2]; for (int i = 0; i < base.length; i++) { File baseFile = base[i]; String name = baseFile.getName(); if (STANDARD_SUBDIRS.contains(name)) { ret.add(baseFile); } else { for (String sub : STANDARD_SUBDIRS) { addIfExists(ret, baseFile, sub); } } } return ret.toArray(new File[ret.size()]); } public File[] fileArrayFromStringArray(File dir, String... subdirNames) { File[] fileList = new File[subdirNames.length]; int i = 0; for (String item : subdirNames) { fileList[i++] = new File(dir, item); } return fileList; } private void addIfExists(List ret, File baseFile, String sub) { File file = new File(baseFile, sub); if (file.exists()) { ret.add(file); } } /** * Utility function. Recursively add to a list of files. Skips ".svn" and junk directories. * @param directory base directory * @param filter filter to restrict files added * @param toAddTo set to add to * @return returns toAddTo. */ public Set getFilesRecursively(File directory, FilenameFilter filter, Set toAddTo) { File files[] = directory.listFiles(); if (files != null) { for (File subfile : files) { if (subfile.isDirectory()) { if (!isJunkFile(subfile.getName())) { getFilesRecursively(subfile, filter, toAddTo); } } else if (filter.accept(directory, subfile.getName())) { toAddTo.add(subfile); } } } return toAddTo; } /** * Is the filename junk? (subversion, backup, etc) * @param name * @return */ public static final boolean isJunkFile(String name) { return name.startsWith(".") || (name.startsWith("#")); // Skip: .svn, .BACKUP, #backup# files. } /** * Get the value of the debug setting for the calling class; assuming that no debugging is wanted if the property * value cannot be found * @param callingClass * @return * @see {@link #getDebugSettingsFor(Class, boolean)} */ public boolean getDebugSettingsFor(Class callingClass) { return getDebugSettingsFor(callingClass, false); } /** * Get the debug settings (whether debugging is enabled for the calling class; This will look for a property corresponding * to the canonical classname +".debug"; if that property cannot be found, the default value will be returned. * @param callingClass * @param defaultValue * @return */ public boolean getDebugSettingsFor(Class callingClass, boolean defaultValue) { // avoid NPE if (callingClass == null) { return defaultValue; } return getProperty(callingClass.getCanonicalName() + ".debug", defaultValue); } /** * Get the URL generator for "general purpose" (non chart) use. * @return */ public CLDRURLS urls() { if (urls == null) { synchronized (this) { urls = internalGetUrls(); } } return urls; } /** * Get the URL generator for "absolute" (chart, email) use. * By default, this is the same as urls. */ public CLDRURLS absoluteUrls() { if (absoluteUrls == null) { synchronized (this) { absoluteUrls = internalGetAbsoluteUrls(); } } return absoluteUrls; } /** * Probably would not need to override this. */ protected CLDRURLS internalGetAbsoluteUrls() { return new StaticCLDRURLS(this.getProperty(CLDRURLS.CLDR_SURVEY_BASE, CLDRURLS.DEFAULT_BASE)); } /** * Override this to provide a different URL source for non-absolute URLs. */ protected CLDRURLS internalGetUrls() { return absoluteUrls(); } private CLDRURLS urls = null; private CLDRURLS absoluteUrls = null; public boolean isCldrVersionBefore(int... version) { return getEnglish().getDtdVersionInfo() .compareTo(getVersion(version)) < 0; } public static VersionInfo getVersion(int... versionInput) { int[] version = new int[4]; for (int i = 0; i < versionInput.length; ++i) { version[i] = versionInput[i]; } return VersionInfo.getInstance(version[0], version[1], version[2], version[3]); } }