Home
last modified time | relevance | path

Searched refs:word_starts (Results 1 – 3 of 3) sorted by relevance

/external/libtextclassifier/native/annotator/pod_ner/
Dutils.cc41 const std::vector<int32_t> &word_starts, int num_wordpieces) { in CodepointSpanToWordpieceSpan() argument
46 span_first_wordpiece_index = word_starts[i]; in CodepointSpanToWordpieceSpan()
50 (i + 1) < word_starts.size() ? word_starts[i + 1] : num_wordpieces; in CodepointSpanToWordpieceSpan()
65 int FindLastFullTokenIndex(const std::vector<int32_t> &word_starts, in FindLastFullTokenIndex() argument
67 if (word_starts.empty()) { in FindLastFullTokenIndex()
70 if (*word_starts.rbegin() < wordpiece_end && in FindLastFullTokenIndex()
73 return word_starts.size() - 1; in FindLastFullTokenIndex()
75 for (int i = word_starts.size() - 1; i > 0; --i) { in FindLastFullTokenIndex()
76 if (word_starts[i] <= wordpiece_end) { in FindLastFullTokenIndex()
83 int FindFirstFullTokenIndex(const std::vector<int32_t> &word_starts, in FindFirstFullTokenIndex() argument
[all …]
Dutils_test.cc626 std::vector<int32_t> word_starts{0, 2, 3, 5, 6, 7, 10, 11}; in TEST() local
629 {2, 3}, tokens, word_starts, in TEST()
639 std::vector<int32_t> word_starts{0, 2, 3, 5, 6, 7, 10, 11}; in TEST() local
642 {6, 7}, tokens, word_starts, in TEST()
648 {6, 7}, tokens, word_starts, in TEST()
654 {12, 14}, tokens, word_starts, in TEST()
664 std::vector<int32_t> word_starts{0, 2, 3, 5, 6, 7, 10, 11}; in TEST() local
667 {2, 3}, tokens, word_starts, in TEST()
677 std::vector<int32_t> word_starts{0, 2, 3, 5, 6, 7, 10, 11}; in TEST() local
680 {15, 19}, tokens, word_starts, in TEST()
[all …]
Dutils.h49 const std::vector<int32_t> &word_starts, int num_wordpieces,
57 int FindLastFullTokenIndex(const std::vector<int32_t> &word_starts,
60 int FindFirstFullTokenIndex(const std::vector<int32_t> &word_starts,
70 const std::vector<int32_t> &word_starts,