Home
last modified time | relevance | path

Searched refs:first_token_index (Results 1 – 3 of 3) sorted by relevance

/external/libtextclassifier/native/annotator/pod_ner/
Dutils_test.cc701 int first_token_index, num_tokens; in TEST() local
704 /*max_num_wordpieces=*/6, /*num_wordpieces=*/12, &first_token_index, in TEST()
707 EXPECT_EQ(first_token_index, 0); in TEST()
712 /*max_num_wordpieces=*/6, /*num_wordpieces=*/12, &first_token_index, in TEST()
715 EXPECT_EQ(first_token_index, 1); in TEST()
721 int first_token_index, num_tokens; in TEST() local
724 /*max_num_wordpieces=*/6, /*num_wordpieces=*/12, &first_token_index, in TEST()
727 EXPECT_EQ(first_token_index, 0); in TEST()
733 int first_token_index, num_tokens; in TEST() local
736 /*max_num_wordpieces=*/6, /*num_wordpieces=*/12, &first_token_index, in TEST()
[all …]
Dutils.cc131 int num_wordpieces, int *first_token_index, int *num_tokens) { in FindFullTokensSpanInWindow() argument
133 *first_token_index = internal::FindFirstFullTokenIndex( in FindFullTokensSpanInWindow()
135 window_first_wordpiece_index = word_starts[*first_token_index]; in FindFullTokensSpanInWindow()
147 *num_tokens = last_token_index - *first_token_index + 1; in FindFullTokensSpanInWindow()
Dutils.h72 int num_wordpieces, int *first_token_index, int *num_tokens);