Searched refs:text_unicode (Results 1 – 4 of 4) sorted by relevance
91 UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false); in Tokenize() local92 return Tokenize(text_unicode); in Tokenize()95 std::vector<Token> Tokenizer::Tokenize(const UnicodeText& text_unicode) const { in Tokenize()98 return InternalTokenize(text_unicode); in Tokenize()103 if (!ICUTokenize(text_unicode, &result)) { in Tokenize()107 InternalRetokenize(text_unicode, &result); in Tokenize()113 return InternalTokenize(text_unicode); in Tokenize()118 const UnicodeText& text_unicode) const { in InternalTokenize()124 for (auto it = text_unicode.begin(); it != text_unicode.end(); in InternalTokenize()
69 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;88 std::vector<Token> InternalTokenize(const UnicodeText& text_unicode) const;
112 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;
196 const UnicodeText& text_unicode) const { in Tokenize()197 return tokenizer_.Tokenize(text_unicode); in Tokenize()