Searched defs:BertTokenizerOptions (Results 1 – 2 of 2) sorted by relevance
59 struct BertTokenizerOptions { struct60 int max_bytes_per_token = kDefaultMaxBytesPerToken;61 int max_chars_per_subtoken = kDefaultMaxCharsPerSubToken;62 std::string suffix_indicator = kDefaultSuffixIndicator;63 bool use_unknown_token = kDefaultUseUnknownToken;64 std::string unknown_token = kDefaultUnknownToken;65 bool split_unknown_chars = kDefaultSplitUnknownChars;66 std::string delim_str = kDefaultDelimRe;67 std::string include_delim_str = kDefaultIncludeDelimRe;
57 struct BertTokenizerOptions { struct58 int max_bytes_per_token = kDefaultMaxBytesPerToken;59 int max_chars_per_subtoken = kDefaultMaxCharsPerSubToken;60 std::string suffix_indicator = kDefaultSuffixIndicator;61 bool use_unknown_token = kDefaultUseUnknownToken;62 std::string unknown_token = kDefaultUnknownToken;63 bool split_unknown_chars = kDefaultSplitUnknownChars;