textclassifier/common/TextClassifierSettings.java

/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.textclassifier.common;

import static java.util.concurrent.TimeUnit.HOURS;

import android.provider.DeviceConfig;
import android.provider.DeviceConfig.Properties;
import android.view.textclassifier.ConversationAction;
import android.view.textclassifier.TextClassifier;
import androidx.annotation.NonNull;
import com.android.textclassifier.utils.IndentingPrintWriter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;

/**
 * TextClassifier specific settings.
 *
 * <p>Currently, this class does not guarantee co-diverted flags are updated atomically.
 *
 * <p>Example of setting the values for testing.
 *
 * <pre>
 * adb shell cmd device_config put textclassifier system_textclassifier_enabled true
 * </pre>
 *
 * @see android.provider.DeviceConfig#NAMESPACE_TEXTCLASSIFIER
 */
public final class TextClassifierSettings {
  private static final String TAG = "TextClassifierSettings";
  public static final String NAMESPACE = DeviceConfig.NAMESPACE_TEXTCLASSIFIER;

  private static final String DELIMITER = ":";

  /** Whether the user language profile feature is enabled. */
  private static final String USER_LANGUAGE_PROFILE_ENABLED = "user_language_profile_enabled";
  /** Max length of text that suggestSelection can accept. */
  @VisibleForTesting
  static final String SUGGEST_SELECTION_MAX_RANGE_LENGTH = "suggest_selection_max_range_length";
  /** Max length of text that classifyText can accept. */
  private static final String CLASSIFY_TEXT_MAX_RANGE_LENGTH = "classify_text_max_range_length";
  /** Max length of text that generateLinks can accept. */
  private static final String GENERATE_LINKS_MAX_TEXT_LENGTH = "generate_links_max_text_length";
  /** Sampling rate for generateLinks logging. */
  private static final String GENERATE_LINKS_LOG_SAMPLE_RATE = "generate_links_log_sample_rate";
  /**
   * Extra count that is added to some languages, e.g. system languages, when deducing the frequent
   * languages in {@link
   * com.android.textclassifier.ulp.LanguageProfileAnalyzer#getFrequentLanguages(int)}.
   */

  /**
   * A colon(:) separated string that specifies the default entities types for generateLinks when
   * hint is not given.
   */
  @VisibleForTesting static final String ENTITY_LIST_DEFAULT = "entity_list_default";
  /**
   * A colon(:) separated string that specifies the default entities types for generateLinks when
   * the text is in a not editable UI widget.
   */
  private static final String ENTITY_LIST_NOT_EDITABLE = "entity_list_not_editable";
  /**
   * A colon(:) separated string that specifies the default entities types for generateLinks when
   * the text is in an editable UI widget.
   */
  private static final String ENTITY_LIST_EDITABLE = "entity_list_editable";
  /**
   * A colon(:) separated string that specifies the default action types for
   * suggestConversationActions when the suggestions are used in an app.
   */
  private static final String IN_APP_CONVERSATION_ACTION_TYPES_DEFAULT =
      "in_app_conversation_action_types_default";
  /**
   * A colon(:) separated string that specifies the default action types for
   * suggestConversationActions when the suggestions are used in a notification.
   */
  private static final String NOTIFICATION_CONVERSATION_ACTION_TYPES_DEFAULT =
      "notification_conversation_action_types_default";
  /** Threshold to accept a suggested language from LangID model. */
  @VisibleForTesting static final String LANG_ID_THRESHOLD_OVERRIDE = "lang_id_threshold_override";
  /** Whether to enable {@link com.android.textclassifier.intent.TemplateIntentFactory}. */
  @VisibleForTesting
  static final String TEMPLATE_INTENT_FACTORY_ENABLED = "template_intent_factory_enabled";
  /** Whether to enable "translate" action in classifyText. */
  private static final String TRANSLATE_IN_CLASSIFICATION_ENABLED =
      "translate_in_classification_enabled";
  /**
   * Whether to detect the languages of the text in request by using langId for the native model.
   */
  private static final String DETECT_LANGUAGES_FROM_TEXT_ENABLED =
      "detect_languages_from_text_enabled";

  /** Whether to enable model downloading with ModelDownloadManager */
  @VisibleForTesting
  public static final String MODEL_DOWNLOAD_MANAGER_ENABLED = "model_download_manager_enabled";
  /** Type of network to download model manifest. A String value of androidx.work.NetworkType. */
  private static final String MANIFEST_DOWNLOAD_REQUIRED_NETWORK_TYPE =
      "manifest_download_required_network_type";
  /** Max attempts allowed for a single ModelDownloader downloading task. */
  @VisibleForTesting
  static final String MODEL_DOWNLOAD_MAX_ATTEMPTS = "model_download_max_attempts";

  @VisibleForTesting
  static final String MODEL_DOWNLOAD_BACKOFF_DELAY_IN_MILLIS =
      "model_download_backoff_delay_in_millis";
  /** Flag name for manifest url is dynamically formatted based on model type and model language. */
  @VisibleForTesting public static final String MANIFEST_URL_TEMPLATE = "manifest_url_%s_%s";
  /** Sampling rate for TextClassifier API logging. */
  static final String TEXTCLASSIFIER_API_LOG_SAMPLE_RATE = "textclassifier_api_log_sample_rate";

  /** The size of the cache of the mapping of session id to text classification context. */
  private static final String SESSION_ID_TO_CONTEXT_CACHE_SIZE = "session_id_to_context_cache_size";

  /**
   * A colon(:) separated string that specifies the configuration to use when including surrounding
   * context text in language detection queries.
   *
   * <p>Format= minimumTextSize<int>:penalizeRatio<float>:textScoreRatio<float>
   *
   * <p>e.g. 20:1.0:0.4
   *
   * <p>Accept all text lengths with minimumTextSize=0
   *
   * <p>Reject all text less than minimumTextSize with penalizeRatio=0
   *
   * @see {@code TextClassifierImpl#detectLanguages(String, int, int)} for reference.
   */
  @VisibleForTesting static final String LANG_ID_CONTEXT_SETTINGS = "lang_id_context_settings";
  /** Default threshold to translate the language of the context the user selects */
  private static final String TRANSLATE_ACTION_THRESHOLD = "translate_action_threshold";

  // Sync this with ConversationAction.TYPE_ADD_CONTACT;
  public static final String TYPE_ADD_CONTACT = "add_contact";
  // Sync this with ConversationAction.COPY;
  public static final String TYPE_COPY = "copy";

  private static final int SUGGEST_SELECTION_MAX_RANGE_LENGTH_DEFAULT = 10 * 1000;
  private static final int CLASSIFY_TEXT_MAX_RANGE_LENGTH_DEFAULT = 10 * 1000;
  private static final int GENERATE_LINKS_MAX_TEXT_LENGTH_DEFAULT = 100 * 1000;
  private static final int GENERATE_LINKS_LOG_SAMPLE_RATE_DEFAULT = 100;

  private static final ImmutableList<String> ENTITY_LIST_DEFAULT_VALUE =
      ImmutableList.of(
          TextClassifier.TYPE_ADDRESS,
          TextClassifier.TYPE_EMAIL,
          TextClassifier.TYPE_PHONE,
          TextClassifier.TYPE_URL,
          TextClassifier.TYPE_DATE,
          TextClassifier.TYPE_DATE_TIME,
          TextClassifier.TYPE_FLIGHT_NUMBER);
  private static final ImmutableList<String> CONVERSATION_ACTIONS_TYPES_DEFAULT_VALUES =
      ImmutableList.of(
          ConversationAction.TYPE_TEXT_REPLY,
          ConversationAction.TYPE_CREATE_REMINDER,
          ConversationAction.TYPE_CALL_PHONE,
          ConversationAction.TYPE_OPEN_URL,
          ConversationAction.TYPE_SEND_EMAIL,
          ConversationAction.TYPE_SEND_SMS,
          ConversationAction.TYPE_TRACK_FLIGHT,
          ConversationAction.TYPE_VIEW_CALENDAR,
          ConversationAction.TYPE_VIEW_MAP,
          TYPE_ADD_CONTACT,
          TYPE_COPY);
  /**
   * < 0 : Not set. Use value from LangId model. 0 - 1: Override value in LangId model.
   *
   * @see EntityConfidence
   */
  private static final float LANG_ID_THRESHOLD_OVERRIDE_DEFAULT = -1f;

  private static final float TRANSLATE_ACTION_THRESHOLD_DEFAULT = 0.5f;

  private static final boolean USER_LANGUAGE_PROFILE_ENABLED_DEFAULT = true;
  private static final boolean TEMPLATE_INTENT_FACTORY_ENABLED_DEFAULT = true;
  private static final boolean TRANSLATE_IN_CLASSIFICATION_ENABLED_DEFAULT = true;
  private static final boolean DETECT_LANGUAGES_FROM_TEXT_ENABLED_DEFAULT = true;
  private static final boolean MODEL_DOWNLOAD_MANAGER_ENABLED_DEFAULT = false;
  private static final String MANIFEST_DOWNLOAD_REQUIRED_NETWORK_TYPE_DEFAULT = "UNMETERED";
  private static final int MODEL_DOWNLOAD_MAX_ATTEMPTS_DEFAULT = 5;
  private static final long MODEL_DOWNLOAD_BACKOFF_DELAY_IN_MILLIS_DEFAULT = HOURS.toMillis(1);
  private static final String MANIFEST_URL_DEFAULT = "";
  private static final float[] LANG_ID_CONTEXT_SETTINGS_DEFAULT = new float[] {20f, 1.0f, 0.4f};
  /**
   * Sampling rate for API logging. For example, 100 means there is a 0.01 chance that the API call
   * is the logged.
   */
  private static final int TEXTCLASSIFIER_API_LOG_SAMPLE_RATE_DEFAULT = 10;

  private static final int SESSION_ID_TO_CONTEXT_CACHE_SIZE_DEFAULT = 10;

  // TODO(licha): Consider removing this. We can use real device config for testing.
  /** DeviceConfig interface to facilitate testing. */
  @VisibleForTesting
  public interface IDeviceConfig {
    default Properties getProperties(@NonNull String namespace, @NonNull String... names) {
      return new Properties.Builder(namespace).build();
    }

    default int getInt(@NonNull String namespace, @NonNull String name, @NonNull int defaultValue) {
      return defaultValue;
    }

    default long getLong(
        @NonNull String namespace, @NonNull String name, @NonNull long defaultValue) {
      return defaultValue;
    }

    default float getFloat(
        @NonNull String namespace, @NonNull String name, @NonNull float defaultValue) {
      return defaultValue;
    }

    default String getString(
        @NonNull String namespace, @NonNull String name, @Nullable String defaultValue) {
      return defaultValue;
    }

    default boolean getBoolean(
        @NonNull String namespace, @NonNull String name, boolean defaultValue) {
      return defaultValue;
    }
  }

  private static final IDeviceConfig DEFAULT_DEVICE_CONFIG =
      new IDeviceConfig() {
        @Override
        public Properties getProperties(@NonNull String namespace, @NonNull String... names) {
          return DeviceConfig.getProperties(namespace, names);
        }

        @Override
        public int getInt(
            @NonNull String namespace, @NonNull String name, @NonNull int defaultValue) {
          return DeviceConfig.getInt(namespace, name, defaultValue);
        }

        @Override
        public long getLong(
            @NonNull String namespace, @NonNull String name, @NonNull long defaultValue) {
          return DeviceConfig.getLong(namespace, name, defaultValue);
        }

        @Override
        public float getFloat(
            @NonNull String namespace, @NonNull String name, @NonNull float defaultValue) {
          return DeviceConfig.getFloat(namespace, name, defaultValue);
        }

        @Override
        public String getString(
            @NonNull String namespace, @NonNull String name, @NonNull String defaultValue) {
          return DeviceConfig.getString(namespace, name, defaultValue);
        }

        @Override
        public boolean getBoolean(
            @NonNull String namespace, @NonNull String name, @NonNull boolean defaultValue) {
          return DeviceConfig.getBoolean(namespace, name, defaultValue);
        }
      };

  private final IDeviceConfig deviceConfig;

  public TextClassifierSettings() {
    this(DEFAULT_DEVICE_CONFIG);
  }

  @VisibleForTesting
  public TextClassifierSettings(IDeviceConfig deviceConfig) {
    this.deviceConfig = deviceConfig;
  }

  public int getSuggestSelectionMaxRangeLength() {
    return deviceConfig.getInt(
        NAMESPACE, SUGGEST_SELECTION_MAX_RANGE_LENGTH, SUGGEST_SELECTION_MAX_RANGE_LENGTH_DEFAULT);
  }

  public int getClassifyTextMaxRangeLength() {
    return deviceConfig.getInt(
        NAMESPACE, CLASSIFY_TEXT_MAX_RANGE_LENGTH, CLASSIFY_TEXT_MAX_RANGE_LENGTH_DEFAULT);
  }

  public int getGenerateLinksMaxTextLength() {
    return deviceConfig.getInt(
        NAMESPACE, GENERATE_LINKS_MAX_TEXT_LENGTH, GENERATE_LINKS_MAX_TEXT_LENGTH_DEFAULT);
  }

  public int getGenerateLinksLogSampleRate() {
    return deviceConfig.getInt(
        NAMESPACE, GENERATE_LINKS_LOG_SAMPLE_RATE, GENERATE_LINKS_LOG_SAMPLE_RATE_DEFAULT);
  }

  public List<String> getEntityListDefault() {
    return getDeviceConfigStringList(ENTITY_LIST_DEFAULT, ENTITY_LIST_DEFAULT_VALUE);
  }

  public List<String> getEntityListNotEditable() {
    return getDeviceConfigStringList(ENTITY_LIST_NOT_EDITABLE, ENTITY_LIST_DEFAULT_VALUE);
  }

  public List<String> getEntityListEditable() {
    return getDeviceConfigStringList(ENTITY_LIST_EDITABLE, ENTITY_LIST_DEFAULT_VALUE);
  }

  public List<String> getInAppConversationActionTypes() {
    return getDeviceConfigStringList(
        IN_APP_CONVERSATION_ACTION_TYPES_DEFAULT, CONVERSATION_ACTIONS_TYPES_DEFAULT_VALUES);
  }

  public List<String> getNotificationConversationActionTypes() {
    return getDeviceConfigStringList(
        NOTIFICATION_CONVERSATION_ACTION_TYPES_DEFAULT, CONVERSATION_ACTIONS_TYPES_DEFAULT_VALUES);
  }

  public float getLangIdThresholdOverride() {
    return deviceConfig.getFloat(
        NAMESPACE, LANG_ID_THRESHOLD_OVERRIDE, LANG_ID_THRESHOLD_OVERRIDE_DEFAULT);
  }

  public float getTranslateActionThreshold() {
    return deviceConfig.getFloat(
        NAMESPACE, TRANSLATE_ACTION_THRESHOLD, TRANSLATE_ACTION_THRESHOLD_DEFAULT);
  }

  public boolean isUserLanguageProfileEnabled() {
    return deviceConfig.getBoolean(
        NAMESPACE, USER_LANGUAGE_PROFILE_ENABLED, USER_LANGUAGE_PROFILE_ENABLED_DEFAULT);
  }

  public boolean isTemplateIntentFactoryEnabled() {
    return deviceConfig.getBoolean(
        NAMESPACE, TEMPLATE_INTENT_FACTORY_ENABLED, TEMPLATE_INTENT_FACTORY_ENABLED_DEFAULT);
  }

  public boolean isTranslateInClassificationEnabled() {
    return deviceConfig.getBoolean(
        NAMESPACE,
        TRANSLATE_IN_CLASSIFICATION_ENABLED,
        TRANSLATE_IN_CLASSIFICATION_ENABLED_DEFAULT);
  }

  public boolean isDetectLanguagesFromTextEnabled() {
    return deviceConfig.getBoolean(
        NAMESPACE, DETECT_LANGUAGES_FROM_TEXT_ENABLED, DETECT_LANGUAGES_FROM_TEXT_ENABLED_DEFAULT);
  }

  public float[] getLangIdContextSettings() {
    return getDeviceConfigFloatArray(LANG_ID_CONTEXT_SETTINGS, LANG_ID_CONTEXT_SETTINGS_DEFAULT);
  }

  public boolean isModelDownloadManagerEnabled() {
    return deviceConfig.getBoolean(
        NAMESPACE, MODEL_DOWNLOAD_MANAGER_ENABLED, MODEL_DOWNLOAD_MANAGER_ENABLED_DEFAULT);
  }

  /** Returns a string which represents a androidx.work.NetworkType enum. */
  public String getManifestDownloadRequiredNetworkType() {
    return deviceConfig.getString(
        NAMESPACE,
        MANIFEST_DOWNLOAD_REQUIRED_NETWORK_TYPE,
        MANIFEST_DOWNLOAD_REQUIRED_NETWORK_TYPE_DEFAULT);
  }

  public int getModelDownloadMaxAttempts() {
    return deviceConfig.getInt(
        NAMESPACE, MODEL_DOWNLOAD_MAX_ATTEMPTS, MODEL_DOWNLOAD_MAX_ATTEMPTS_DEFAULT);
  }

  public long getModelDownloadBackoffDelayInMillis() {
    return deviceConfig.getLong(
        NAMESPACE,
        MODEL_DOWNLOAD_BACKOFF_DELAY_IN_MILLIS,
        MODEL_DOWNLOAD_BACKOFF_DELAY_IN_MILLIS_DEFAULT);
  }

  /**
   * Get model's manifest url for given model type and language.
   *
   * @param modelType the type of model for the target url
   * @param modelLanguageTag the language tag for the model (e.g. en), but can also be "universal"
   * @return DeviceConfig configured url or empty string if not set
   */
  public String getManifestURL(@ModelType.ModelTypeDef String modelType, String modelLanguageTag) {
    // E.g: manifest_url_annotator_zh, manifest_url_lang_id_universal,
    // manifest_url_actions_suggestions_en
    String urlFlagName = String.format(MANIFEST_URL_TEMPLATE, modelType, modelLanguageTag);
    return deviceConfig.getString(NAMESPACE, urlFlagName, MANIFEST_URL_DEFAULT);
  }

  /**
   * Gets all language variants configured for a specific ModelType.
   *
   * <p>For a specific language, there can be many variants: de-CH, de-LI, zh-Hans, zh-Hant. There
   * is no easy way to hardcode the list in client. Therefore, we parse all configured flag's name
   * in DeviceConfig, and let the client to choose the best variant to download.
   */
  public ImmutableList<String> getLanguageTagsForManifestURL(
      @ModelType.ModelTypeDef String modelType) {
    String urlFlagBaseName = String.format(MANIFEST_URL_TEMPLATE, modelType, /* language */ "");
    Properties properties = deviceConfig.getProperties(NAMESPACE);
    ImmutableList.Builder<String> variantsBuilder = ImmutableList.builder();
    for (String name : properties.getKeyset()) {
      if (name.startsWith(urlFlagBaseName)
          && properties.getString(name, /* defaultValue= */ null) != null) {
        variantsBuilder.add(name.substring(urlFlagBaseName.length()));
      }
    }
    return variantsBuilder.build();
  }

  public int getTextClassifierApiLogSampleRate() {
    return deviceConfig.getInt(
        NAMESPACE, TEXTCLASSIFIER_API_LOG_SAMPLE_RATE, TEXTCLASSIFIER_API_LOG_SAMPLE_RATE_DEFAULT);
  }

  public int getSessionIdToContextCacheSize() {
    return deviceConfig.getInt(
        NAMESPACE, SESSION_ID_TO_CONTEXT_CACHE_SIZE, SESSION_ID_TO_CONTEXT_CACHE_SIZE_DEFAULT);
  }

  public void dump(IndentingPrintWriter pw) {
    pw.println("TextClassifierSettings:");
    pw.increaseIndent();
    pw.printPair(CLASSIFY_TEXT_MAX_RANGE_LENGTH, getClassifyTextMaxRangeLength());
    pw.printPair(DETECT_LANGUAGES_FROM_TEXT_ENABLED, isDetectLanguagesFromTextEnabled());
    pw.printPair(ENTITY_LIST_DEFAULT, getEntityListDefault());
    pw.printPair(ENTITY_LIST_EDITABLE, getEntityListEditable());
    pw.printPair(ENTITY_LIST_NOT_EDITABLE, getEntityListNotEditable());
    pw.printPair(GENERATE_LINKS_LOG_SAMPLE_RATE, getGenerateLinksLogSampleRate());
    pw.printPair(GENERATE_LINKS_MAX_TEXT_LENGTH, getGenerateLinksMaxTextLength());
    pw.printPair(IN_APP_CONVERSATION_ACTION_TYPES_DEFAULT, getInAppConversationActionTypes());
    pw.printPair(LANG_ID_CONTEXT_SETTINGS, Arrays.toString(getLangIdContextSettings()));
    pw.printPair(LANG_ID_THRESHOLD_OVERRIDE, getLangIdThresholdOverride());
    pw.printPair(TRANSLATE_ACTION_THRESHOLD, getTranslateActionThreshold());
    pw.printPair(
        NOTIFICATION_CONVERSATION_ACTION_TYPES_DEFAULT, getNotificationConversationActionTypes());
    pw.printPair(SUGGEST_SELECTION_MAX_RANGE_LENGTH, getSuggestSelectionMaxRangeLength());
    pw.printPair(USER_LANGUAGE_PROFILE_ENABLED, isUserLanguageProfileEnabled());
    pw.printPair(TEMPLATE_INTENT_FACTORY_ENABLED, isTemplateIntentFactoryEnabled());
    pw.printPair(TRANSLATE_IN_CLASSIFICATION_ENABLED, isTranslateInClassificationEnabled());
    pw.printPair(MODEL_DOWNLOAD_MANAGER_ENABLED, isModelDownloadManagerEnabled());
    pw.printPair(MODEL_DOWNLOAD_MAX_ATTEMPTS, getModelDownloadMaxAttempts());
    pw.decreaseIndent();
    pw.printPair(TEXTCLASSIFIER_API_LOG_SAMPLE_RATE, getTextClassifierApiLogSampleRate());
    pw.printPair(SESSION_ID_TO_CONTEXT_CACHE_SIZE, getSessionIdToContextCacheSize());
    pw.decreaseIndent();
  }

  private List<String> getDeviceConfigStringList(String key, List<String> defaultValue) {
    return parse(deviceConfig.getString(NAMESPACE, key, null), defaultValue);
  }

  private float[] getDeviceConfigFloatArray(String key, float[] defaultValue) {
    return parse(deviceConfig.getString(NAMESPACE, key, null), defaultValue);
  }

  private static List<String> parse(@Nullable String listStr, List<String> defaultValue) {
    if (listStr != null) {
      return Collections.unmodifiableList(Arrays.asList(listStr.split(DELIMITER)));
    }
    return defaultValue;
  }

  private static float[] parse(@Nullable String arrayStr, float[] defaultValue) {
    if (arrayStr != null) {
      final List<String> split = Splitter.onPattern(DELIMITER).splitToList(arrayStr);
      if (split.size() != defaultValue.length) {
        return defaultValue;
      }
      final float[] result = new float[split.size()];
      for (int i = 0; i < split.size(); i++) {
        try {
          result[i] = Float.parseFloat(split.get(i));
        } catch (NumberFormatException e) {
          return defaultValue;
        }
      }
      return result;
    } else {
      return defaultValue;
    }
  }
}