Home
last modified time | relevance | path

Searched refs:text_unicode (Results 1 – 4 of 4) sorted by relevance

/external/libtextclassifier/utils/
Dtokenizer.cc91 UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false); in Tokenize() local
92 return Tokenize(text_unicode); in Tokenize()
95 std::vector<Token> Tokenizer::Tokenize(const UnicodeText& text_unicode) const { in Tokenize()
98 return InternalTokenize(text_unicode); in Tokenize()
103 if (!ICUTokenize(text_unicode, &result)) { in Tokenize()
107 InternalRetokenize(text_unicode, &result); in Tokenize()
113 return InternalTokenize(text_unicode); in Tokenize()
118 const UnicodeText& text_unicode) const { in InternalTokenize()
124 for (auto it = text_unicode.begin(); it != text_unicode.end(); in InternalTokenize()
Dtokenizer.h69 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;
88 std::vector<Token> InternalTokenize(const UnicodeText& text_unicode) const;
/external/libtextclassifier/annotator/
Dfeature-processor.h112 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;
Dfeature-processor.cc196 const UnicodeText& text_unicode) const { in Tokenize()
197 return tokenizer_.Tokenize(text_unicode); in Tokenize()