Lines Matching refs:BertTokenizer

30 void AssertTokenizerResults(std::unique_ptr<BertTokenizer> tokenizer) {  in AssertTokenizerResults()
40       absl::make_unique<BertTokenizer>(buffer.data(), buffer.size());  in TEST()
47       absl::make_unique<BertTokenizer>(GetTestDataPath(kTestVocabPath));  in TEST()
58   auto tokenizer = absl::make_unique<BertTokenizer>(vocab);  in TEST()
65       absl::make_unique<BertTokenizer>(GetTestDataPath(kTestVocabPath));  in TEST()
79   auto tokenizer = absl::make_unique<BertTokenizer>(vocab);  in TEST()
93   auto tokenizer = absl::make_unique<BertTokenizer>(vocab);  in TEST()
114   auto tokenizer = absl::make_unique<BertTokenizer>(vocab);  in TEST()
135   auto tokenizer = absl::make_unique<BertTokenizer>(vocab);  in TEST()
157   auto tokenizer = absl::make_unique<BertTokenizer>(vocab);  in TEST()
165   std::vector<std::string> tokens = BertTokenizer::PreTokenize(input);  in TEST()