Searched refs:normalized_text (Results 1 – 5 of 5) sorted by relevance
| /external/icing/icing/transform/map/ |
| D | map-normalizer.cc | 34 std::string normalized_text; in NormalizeTerm() local
35 normalized_text.reserve(term.length()); in NormalizeTerm()
40 normalized_text.push_back(std::tolower(term[i])); in NormalizeTerm()
53 absl_ports::StrAppend(&normalized_text, term.substr(i, utf8_length)); in NormalizeTerm()
65 normalized_text.push_back( in NormalizeTerm()
69 i18n_utils::AppendUchar32ToUtf8(&normalized_text, iterator->second); in NormalizeTerm()
73 absl_ports::StrAppend(&normalized_text, term.substr(i, utf8_length)); in NormalizeTerm()
78 if (normalized_text.length() > max_term_byte_size_) { in NormalizeTerm()
79 i18n_utils::SafeTruncateUtf8(&normalized_text, max_term_byte_size_); in NormalizeTerm()
82 return normalized_text; in NormalizeTerm()
|
| /external/libtextclassifier/native/utils/sentencepiece/ |
| D | encoder.cc | 21 bool Encoder::Encode(StringPiece normalized_text, in Encode() argument
23 const int len = normalized_text.size(); in Encode()
37 normalized_text.RemovePrefix(1); in Encode()
61 if (!pieces_->FindAllPrefixMatches(normalized_text, &matches)) { in Encode()
78 normalized_text.RemovePrefix(1); in Encode()
|
| D | encoder.h | 58 bool Encode(StringPiece normalized_text,
|
| /external/icing/icing/transform/icu/ |
| D | icu-normalizer.cc | 123 std::string normalized_text; in NormalizeTerm() local
126 return normalized_text; in NormalizeTerm()
144 normalized_text = NormalizeLatin(normalizer2, term); in NormalizeTerm()
146 normalized_text = term_transformer_->Transform(term); in NormalizeTerm()
149 if (normalized_text.length() > max_term_byte_size_) { in NormalizeTerm()
150 i18n_utils::SafeTruncateUtf8(&normalized_text, max_term_byte_size_); in NormalizeTerm()
153 return normalized_text; in NormalizeTerm()
|
| /external/icing/icing/query/ |
| D | query-processor.cc | 232 std::string normalized_text = normalizer_.NormalizeTerm(token.text); in ParseRawQuery() local
260 index_.GetIterator(normalized_text, kSectionIdMaskAll, in ParseRawQuery()
271 index_.GetIterator(normalized_text, kSectionIdMaskAll, in ParseRawQuery()
274 results.query_term_iterators[normalized_text] = in ParseRawQuery()
280 std::move(normalized_text)); in ParseRawQuery()
|