Searched refs:unknown_token_id (Results 1 – 2 of 2) sorted by relevance
105 int unknown_token_id = 0; in CreateTokenizerFromProcessUnit() local106 if (!regex_tokenizer->GetUnknownToken(&unknown_token_id)) { in CreateTokenizerFromProcessUnit()
121 int unknown_token_id = 0; in CreateRegexTokenizerFromProcessUnit() local122 if (!regex_tokenizer->GetUnknownToken(&unknown_token_id)) { in CreateRegexTokenizerFromProcessUnit()210 int unknown_token_id = 0; in Preprocess() local211 tokenizer_->GetUnknownToken(&unknown_token_id); in Preprocess()232 input_tokens[input_token_index] = unknown_token_id; in Preprocess()