Lines Matching refs:line_tokens
1981 std::vector<Token> line_tokens; in ModelAnnotate() local
1982 line_tokens = selection_feature_processor_->Tokenize(line_str); in ModelAnnotate()
1987 &line_tokens, in ModelAnnotate()
1990 0, static_cast<TokenIndex>(line_tokens.size())}; in ModelAnnotate()
1994 line_tokens, full_line_span)) { in ModelAnnotate()
2000 line_tokens, full_line_span, in ModelAnnotate()
2012 if (!ModelChunk(line_tokens.size(), /*span_of_interest=*/full_line_span, in ModelAnnotate()
2032 TokenSpanToCodepointSpan(line_tokens, chunk); in ModelAnnotate()
2062 line_unicode, line_tokens, detected_text_language_tags, in ModelAnnotate()
2073 if (!ModelClassifyText(line_str, line_tokens, in ModelAnnotate()
2101 tokens->insert(tokens->end(), line_tokens.begin(), line_tokens.end()); in ModelAnnotate()
2103 *tokens = std::move(line_tokens); in ModelAnnotate()