Searched refs:SentencePieceTokenizer (Results 1 – 6 of 6) sorted by relevance
32 class SentencePieceTokenizer : public Tokenizer {35 explicit SentencePieceTokenizer(const std::string& path_to_model) { in SentencePieceTokenizer() function39 explicit SentencePieceTokenizer(const char* spmodel_buffer_data, in SentencePieceTokenizer() function
32 using ::tflite::support::text::tokenizer::SentencePieceTokenizer;40 absl::make_unique<SentencePieceTokenizer>(model.data(), model.size()); in Java_org_tensorflow_lite_support_text_tokenizers_SentencePieceTokenizer_nativeLoadResource()47 delete reinterpret_cast<SentencePieceTokenizer*>(handle); in Java_org_tensorflow_lite_support_text_tokenizers_SentencePieceTokenizer_nativeUnloadResource()
84 return absl::make_unique<SentencePieceTokenizer>(model_buffer.data(), in CreateTokenizerFromProcessUnit()
42 using ::tflite::support::text::tokenizer::SentencePieceTokenizer;381 tokenizer_ = absl::make_unique<SentencePieceTokenizer>(path_to_spmodel); in InitializeSentencepieceTokenizer()386 tokenizer_ = absl::make_unique<SentencePieceTokenizer>(spmodel_buffer_data, in InitializeSentencepieceTokenizerFromBinary()
22 using SentencepieceTokenizerCPP = ::tflite::support::text::tokenizer::SentencePieceTokenizer;
445 // The SentencePiece model files used in the SentencePieceTokenizer.448 // The optional vocabulary model files used in the SentencePieceTokenizer.