| /external/tflite-support/tensorflow_lite_support/cc/text/tokenizers/ |
| D | regex_tokenizer.cc | 92 bool RegexTokenizer::LookupId(absl::string_view key, int* result) const { in LookupId() function in tflite::support::text::tokenizer::RegexTokenizer
111 return LookupId(kStart, start_token); in GetStartToken()
115 return LookupId(kPad, pad_token); in GetPadToken()
119 return LookupId(kUnknown, unknown_token); in GetUnknownToken()
|
| D | bert_tokenizer.h | 78 bool LookupId(absl::string_view key, int* result) const;
126 bool LookupId(absl::string_view key, int* result) const override { in LookupId() function
127 return vocab_.LookupId(key, result); in LookupId()
|
| D | tokenizer.h | 41 virtual bool LookupId(absl::string_view key, int* result) const = 0;
|
| D | regex_tokenizer.h | 40 bool LookupId(absl::string_view key, int* result) const override;
|
| D | sentencepiece_tokenizer.h | 54 bool LookupId(absl::string_view key, int* result) const override { in LookupId() function
|
| D | tokenizer_jni_lib.cc | 77 tokenizer->LookupId(token, &id); in nativeConvertTokensToIds()
|
| D | bert_tokenizer.cc | 39 bool FlatHashMapBackedWordpiece::LookupId(const absl::string_view key, in LookupId() function in tflite::support::text::tokenizer::FlatHashMapBackedWordpiece
|
| /external/libtextclassifier/native/utils/ |
| D | bert_tokenizer_test.cc | 96 ASSERT_FALSE(tokenizer->LookupId("iDontExist", &i)); in TEST()
98 ASSERT_TRUE(tokenizer->LookupId("i", &i)); in TEST()
100 ASSERT_TRUE(tokenizer->LookupId("'", &i)); in TEST()
102 ASSERT_TRUE(tokenizer->LookupId("m", &i)); in TEST()
104 ASSERT_TRUE(tokenizer->LookupId("question", &i)); in TEST()
|
| D | bert_tokenizer.h | 73 bool LookupId(absl::string_view key, int* result) const;
120 bool LookupId(absl::string_view key, int* result) const override { in LookupId() function
121 return vocab_.LookupId(key, result); in LookupId()
|
| D | bert_tokenizer.cc | 43 bool FlatHashMapBackedWordpiece::LookupId(const absl::string_view key, in LookupId() function in libtextclassifier3::FlatHashMapBackedWordpiece
|
| /external/libtextclassifier/native/annotator/pod_ner/ |
| D | pod-ner-impl.cc | 145 if (!tokenizer->LookupId("[CLS]", cls_id)) { in FindSpecialWordpieceIds()
149 if (!tokenizer->LookupId("[SEP]", sep_id)) { in FindSpecialWordpieceIds()
153 if (!tokenizer->LookupId(".", period_id)) { in FindSpecialWordpieceIds()
157 if (!tokenizer->LookupId("[UNK]", unknown_id)) { in FindSpecialWordpieceIds()
371 if (!tokenizer_->LookupId(wordpiece, &(wordpiece_ids.emplace_back()))) { in PrepareText()
|
| /external/tflite-support/tensorflow_lite_support/ios/text/tokenizers/Sources/ |
| D | TFLTokenizerUtil.mm | 37 tokenizer->LookupId(cToken, &id);
|
| /external/tflite-support/tensorflow_lite_support/cc/task/text/nlclassifier/ |
| D | bert_nl_classifier.cc | 103 tokenizer_->LookupId(tokens[i], &input_ids[i]); in Preprocess()
|
| D | nl_classifier.cc | 229 if (tokenizer_->LookupId(token, &token_id)) { in Preprocess()
|
| /external/tflite-support/tensorflow_lite_support/cc/task/text/qa/ |
| D | bert_question_answerer.cc | 264 tokenizer_->LookupId(token, &input_ids[i]); in Preprocess()
|