Searched refs:BertTokenizer (Results 1 – 12 of 12) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | bert_tokenizer_test.cc | 30 void AssertTokenizerResults(std::unique_ptr<BertTokenizer> tokenizer) { in AssertTokenizerResults()
40 absl::make_unique<BertTokenizer>(buffer.data(), buffer.size()); in TEST()
47 absl::make_unique<BertTokenizer>(GetTestDataPath(kTestVocabPath)); in TEST()
58 auto tokenizer = absl::make_unique<BertTokenizer>(vocab); in TEST()
65 absl::make_unique<BertTokenizer>(GetTestDataPath(kTestVocabPath)); in TEST()
79 auto tokenizer = absl::make_unique<BertTokenizer>(vocab); in TEST()
93 auto tokenizer = absl::make_unique<BertTokenizer>(vocab); in TEST()
114 auto tokenizer = absl::make_unique<BertTokenizer>(vocab); in TEST()
135 auto tokenizer = absl::make_unique<BertTokenizer>(vocab); in TEST()
157 auto tokenizer = absl::make_unique<BertTokenizer>(vocab); in TEST()
[all …]
|
| D | bert_tokenizer.cc | 62 TokenizerResult BertTokenizer::TokenizeSingleToken(const std::string& token) { in TokenizeSingleToken()
64 return BertTokenizer::Tokenize(tokens); in TokenizeSingleToken()
67 TokenizerResult BertTokenizer::Tokenize(const std::string& input) { in Tokenize()
69 return BertTokenizer::Tokenize(tokens); in Tokenize()
72 TokenizerResult BertTokenizer::Tokenize( in Tokenize()
104 std::vector<std::string> BertTokenizer::PreTokenize( in PreTokenize()
|
| D | bert_tokenizer.h | 84 class BertTokenizer : public tflite::support::text::tokenizer::Tokenizer {
87 explicit BertTokenizer(const std::vector<std::string>& vocab,
92 explicit BertTokenizer(const std::string& path_to_vocab,
94 : BertTokenizer(LoadVocabFromFile(path_to_vocab), options) {} in BertTokenizer() function
98 BertTokenizer(const char* vocab_buffer_data, size_t vocab_buffer_size,
100 : BertTokenizer(LoadVocabFromBuffer(vocab_buffer_data, vocab_buffer_size), in BertTokenizer() function
|
| /external/tflite-support/tensorflow_lite_support/cc/text/tokenizers/ |
| D | bert_tokenizer.h | 89 class BertTokenizer : public tflite::support::text::tokenizer::Tokenizer {
92 explicit BertTokenizer(const std::vector<std::string>& vocab,
100 explicit BertTokenizer(const std::string& path_to_vocab,
102 : BertTokenizer(utils::LoadVocabFromFile(path_to_vocab), options) {} in BertTokenizer() function
106 BertTokenizer(const char* vocab_buffer_data, size_t vocab_buffer_size,
108 : BertTokenizer( in BertTokenizer() function
|
| D | bert_tokenizer_jni.cc | 28 using ::tflite::support::text::tokenizer::BertTokenizer;
50 auto handle = absl::make_unique<BertTokenizer>( in Java_org_tensorflow_lite_support_text_tokenizers_BertTokenizer_nativeLoadResource()
70 delete reinterpret_cast<BertTokenizer*>(handle); in Java_org_tensorflow_lite_support_text_tokenizers_BertTokenizer_nativeUnloadResource()
|
| D | bert_tokenizer.cc | 58 TokenizerResult BertTokenizer::Tokenize(const std::string& input) { in Tokenize()
62 WordpieceTokenizerResult BertTokenizer::TokenizeWordpiece( in TokenizeWordpiece()
|
| D | tokenizer_utils.cc | 74 return absl::make_unique<BertTokenizer>(vocab_buffer.data(), in CreateTokenizerFromProcessUnit()
|
| /external/libtextclassifier/native/annotator/pod_ner/ |
| D | pod-ner-impl.cc | 142 bool FindSpecialWordpieceIds(const std::unique_ptr<BertTokenizer> &tokenizer, in FindSpecialWordpieceIds()
165 std::unique_ptr<BertTokenizer> CreateTokenizer(const PodNerModel *model) { in CreateTokenizer()
173 return std::unique_ptr<BertTokenizer>(new BertTokenizer( in CreateTokenizer()
187 std::unique_ptr<BertTokenizer> tokenizer = CreateTokenizer(model); in Create()
|
| D | pod-ner-impl.h | 89 std::unique_ptr<BertTokenizer> tokenizer_;
|
| /external/tflite-support/tensorflow_lite_support/cc/task/text/qa/ |
| D | bert_question_answerer.cc | 40 using ::tflite::support::text::tokenizer::BertTokenizer;
370 tokenizer_ = absl::make_unique<BertTokenizer>(path_to_vocab); in InitializeBertTokenizer()
376 absl::make_unique<BertTokenizer>(vocab_buffer_data, vocab_buffer_size); in InitializeBertTokenizerFromBinary()
|
| /external/tflite-support/tensorflow_lite_support/ios/text/tokenizers/Sources/ |
| D | TFLBertTokenizer.mm | 21 using BertTokenizerCPP = ::tflite::support::text::tokenizer::BertTokenizer;
|
| /external/tflite-support/tensorflow_lite_support/metadata/ |
| D | metadata_schema.fbs | 433 // Performs Bert tokenization as in tf.text.BertTokenizer
434 …flow/text/blob/3599f6fcd2b780a2dc413b90fb9315464f10b314/docs/api_docs/python/text/BertTokenizer.md)
437 // The vocabulary files used in the BertTokenizer.
|