Lines Matching refs:token_span

207   TokenSpan token_span;  in LabelToSpan()  local
208 if (!LabelToTokenSpan(label, &token_span)) { in LabelToSpan()
212 const int result_begin_token_index = token_span.first; in LabelToSpan()
216 const int result_end_token_index = token_span.second; in LabelToSpan()
251 TokenSpan* token_span) const { in LabelToTokenSpan()
253 *token_span = label_to_selection_[label]; in LabelToTokenSpan()
362 const std::vector<Token>& selectable_tokens, TokenSpan token_span) { in TokenSpanToCodepointSpan() argument
363 return {selectable_tokens[token_span.first].start, in TokenSpanToCodepointSpan()
364 selectable_tokens[token_span.second - 1].end}; in TokenSpanToCodepointSpan()
600 const TokenSpan& token_span, const std::vector<Token>& tokens) const { in SupportedCodepointsRatio() argument
603 for (int i = token_span.first; i < token_span.second; ++i) { in SupportedCodepointsRatio()
662 TokenSpan token_span{l, r}; in MakeLabelMaps() local
663 selection_to_label_[token_span] = selection_label_id; in MakeLabelMaps()
664 label_to_selection_.push_back(token_span); in MakeLabelMaps()
747 const std::vector<Token>& tokens, TokenSpan token_span) const { in HasEnoughSupportedCodepoints()
750 SupportedCodepointsRatio(token_span, tokens); in HasEnoughSupportedCodepoints()
761 const std::vector<Token>& tokens, TokenSpan token_span, in ExtractFeatures() argument
767 features->reserve(feature_vector_size * TokenSpanSize(token_span)); in ExtractFeatures()
768 for (int i = token_span.first; i < token_span.second; ++i) { in ExtractFeatures()
787 *cached_features = CachedFeatures::Create(token_span, std::move(features), in ExtractFeatures()