Searched refs:pad_token_id (Results 1 – 2 of 2) sorted by relevance
113 int pad_token_id = 0; in CreateTokenizerFromProcessUnit() local114 if (!regex_tokenizer->GetPadToken(&pad_token_id)) { in CreateTokenizerFromProcessUnit()
129 int pad_token_id = 0; in CreateRegexTokenizerFromProcessUnit() local130 if (!regex_tokenizer->GetPadToken(&pad_token_id)) { in CreateRegexTokenizerFromProcessUnit()213 int pad_token_id = 0; in Preprocess() local214 tokenizer_->GetPadToken(&pad_token_id); in Preprocess()216 std::vector<int> input_tokens(max_sentence_length, pad_token_id); in Preprocess()