Searched refs:PreTokenize (Results 1 – 3 of 3) sorted by relevance
68 std::vector<std::string> tokens = PreTokenize(input); in Tokenize()104 std::vector<std::string> BertTokenizer::PreTokenize( in PreTokenize() function in libtextclassifier3::BertTokenizer
131 static std::vector<std::string> PreTokenize(const absl::string_view input);
165 std::vector<std::string> tokens = BertTokenizer::PreTokenize(input); in TEST()