Home
last modified time | relevance | path

Searched refs:regex_tokenizer (Results 1 – 4 of 4) sorted by relevance

/external/tflite-support/tensorflow_lite_support/cc/text/tokenizers/
Dtokenizer_utils.cc100 std::unique_ptr<RegexTokenizer> regex_tokenizer = in CreateTokenizerFromProcessUnit() local
106 if (!regex_tokenizer->GetUnknownToken(&unknown_token_id)) { in CreateTokenizerFromProcessUnit()
114 if (!regex_tokenizer->GetPadToken(&pad_token_id)) { in CreateTokenizerFromProcessUnit()
121 return regex_tokenizer; in CreateTokenizerFromProcessUnit()
DBUILD164 ":regex_tokenizer",
177 name = "regex_tokenizer",
179 "regex_tokenizer.cc",
182 "regex_tokenizer.h",
/external/tflite-support/tensorflow_lite_support/cc/task/text/nlclassifier/
Dnl_classifier.cc116 std::unique_ptr<RegexTokenizer> regex_tokenizer = in CreateRegexTokenizerFromProcessUnit() local
122 if (!regex_tokenizer->GetUnknownToken(&unknown_token_id)) { in CreateRegexTokenizerFromProcessUnit()
130 if (!regex_tokenizer->GetPadToken(&pad_token_id)) { in CreateRegexTokenizerFromProcessUnit()
136 return regex_tokenizer; in CreateRegexTokenizerFromProcessUnit()
DBUILD28 "//tensorflow_lite_support/cc/text/tokenizers:regex_tokenizer",