Searched refs:unicode_text (Results 1 – 3 of 3) sorted by relevance
/external/libtextclassifier/utils/ |
D | tokenizer.cc | 160 void Tokenizer::TokenizeSubstring(const UnicodeText& unicode_text, in TokenizeSubstring() argument 169 UnicodeText text = UnicodeText::Substring(unicode_text, span.first, in TokenizeSubstring() 185 void Tokenizer::InternalRetokenize(const UnicodeText& unicode_text, in InternalRetokenize() argument 207 TokenizeSubstring(unicode_text, span, &result); in InternalRetokenize() 212 TokenizeSubstring(unicode_text, span, &result); in InternalRetokenize()
|
D | tokenizer.h | 85 void TokenizeSubstring(const UnicodeText& unicode_text, CodepointSpan span, 92 void InternalRetokenize(const UnicodeText& unicode_text,
|
/external/python/cpython2/Parser/ |
D | tokenizer.c | 1711 PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace"); in dec_utf8() local 1712 if (unicode_text) { in dec_utf8() 1713 ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace"); in dec_utf8() 1714 Py_DECREF(unicode_text); in dec_utf8()
|