Searched refs:GetUnknownToken (Results 1 – 4 of 4) sorted by relevance
46 bool GetUnknownToken(int* unknown_token);
118 bool RegexTokenizer::GetUnknownToken(int* unknown_token) { in GetUnknownToken() function in tflite::support::text::tokenizer::RegexTokenizer
106 if (!regex_tokenizer->GetUnknownToken(&unknown_token_id)) { in CreateTokenizerFromProcessUnit()
122 if (!regex_tokenizer->GetUnknownToken(&unknown_token_id)) { in CreateRegexTokenizerFromProcessUnit()211 tokenizer_->GetUnknownToken(&unknown_token_id); in Preprocess()