Home
last modified time | relevance | path

Searched refs:UTF8ToUnicodeText (Results 1 – 25 of 40) sorted by relevance

12

/external/libtextclassifier/native/annotator/number/
Dnumber_test-include.cc119 UTF8ToUnicodeText("... 12345 ..."), {4, 9}, in TEST_F()
130 UTF8ToUnicodeText("... 12345.12345 ..."), {4, 15}, in TEST_F()
144 UTF8ToUnicodeText("... 12345. ..."), {4, 9}, in TEST_F()
147 UTF8ToUnicodeText("... 12345. ..."), {4, 10}, in TEST_F()
155 UTF8ToUnicodeText("... 12345. ..."), {4, 9}, in TEST_F()
168 UTF8ToUnicodeText("how much is 2 plus 5 divided by 7% minus 3.14 " in TEST_F()
194 UTF8ToUnicodeText("... 123a45 ..."), {4, 10}, in TEST_F()
197 UTF8ToUnicodeText("... 12345..12345 ..."), {4, 16}, in TEST_F()
200 UTF8ToUnicodeText("... 12345a ..."), {4, 11}, in TEST_F()
208 UTF8ToUnicodeText("... 14, ..."), {4, 6}, in TEST_F()
[all …]
Dnumber.cc92 UTF8ToUnicodeText(tokens[prefix_end_index].value, /*do_copy=*/false) in TokensAreValidNumberPrefix()
109 return TryParseNumber(UTF8ToUnicodeText(tokens[prefix_end_index - 1].value, in TokensAreValidNumberPrefix()
127 UTF8ToUnicodeText(tokens[ending_index].value, /*do_copy=*/false).begin(); in TokensAreValidEnding()
150 UTF8ToUnicodeText(tokens[suffix_start_index].value, /*do_copy=*/false) in TokensAreValidNumberSuffix()
167 UTF8ToUnicodeText(tokens[suffix_start_index + 1].value, in TokensAreValidNumberSuffix()
229 *UTF8ToUnicodeText(tokens[i].value, /*do_copy=*/false).begin())) { in FindAll()
234 UTF8ToUnicodeText(token.value, /*do_copy=*/false); in FindAll()
240 *UTF8ToUnicodeText(tokens[i - 1].value, /*do_copy=*/false).begin()); in FindAll()
/external/libtextclassifier/native/utils/utf8/
Dunilib_test-include.cc85 EXPECT_EQ(unilib_->ToLowerText(UTF8ToUnicodeText("Never gonna give you up.")) in TEST_F()
91 EXPECT_EQ(unilib_->ToUpperText(UTF8ToUnicodeText("Never gonna let you down.")) in TEST_F()
160 unilib_->ToLowerText(UTF8ToUnicodeText("Κανένας άνθρωπος δεν ξέρει")) in TEST_F()
163 EXPECT_TRUE(unilib_->IsLowerText(UTF8ToUnicodeText("ξέρει"))); in TEST_F()
170 unilib_->ToUpperText(UTF8ToUnicodeText("Κανένας άνθρωπος δεν ξέρει")) in TEST_F()
173 EXPECT_TRUE(unilib_->IsUpperText(UTF8ToUnicodeText("ΚΑΝΈΝΑΣ"))); in TEST_F()
180 UTF8ToUnicodeText("[0-9]+", /*do_copy=*/true); in TEST_F()
183 const UnicodeText input = UTF8ToUnicodeText("hello 0123", /*do_copy=*/false); in TEST_F()
198 UTF8ToUnicodeText("[0-9]+��", /*do_copy=*/false); in TEST_F()
204 matcher = pattern->Matcher(UTF8ToUnicodeText("0123��", /*do_copy=*/false)); in TEST_F()
[all …]
Dunicodetext_test.cc40 UnicodeText text = UTF8ToUnicodeText("1234��hello", /*do_copy=*/false); in TEST()
47 UnicodeText text = UTF8ToUnicodeText("1234��hello", /*do_copy=*/false); in TEST()
62 UTF8ToUnicodeText(StringPiece(raw_text), /*do_copy=*/false); in TEST()
75 UnicodeText text = UTF8ToUnicodeText("1234��hello", /*do_copy=*/false); in TEST()
80 UTF8ToUnicodeText("��h")); in TEST()
84 UTF8ToUnicodeText("��h")); in TEST()
86 UTF8ToUnicodeText("��h")); in TEST()
88 UTF8ToUnicodeText("��h")); in TEST()
108 EXPECT_TRUE(UTF8ToUnicodeText("1234��hello", /*do_copy=*/false).is_valid()); in TEST()
110 UTF8ToUnicodeText("\u304A\u00B0\u106B", /*do_copy=*/false).is_valid()); in TEST()
[all …]
Dunicodetext.cc318 UnicodeText UTF8ToUnicodeText(const char* utf8_buf, int len, bool do_copy) { in UTF8ToUnicodeText() function
328 UnicodeText UTF8ToUnicodeText(const char* utf8_buf, bool do_copy) { in UTF8ToUnicodeText() function
329 return UTF8ToUnicodeText(utf8_buf, strlen(utf8_buf), do_copy); in UTF8ToUnicodeText()
332 UnicodeText UTF8ToUnicodeText(const std::string& str, bool do_copy) { in UTF8ToUnicodeText() function
333 return UTF8ToUnicodeText(str.data(), str.size(), do_copy); in UTF8ToUnicodeText()
336 UnicodeText UTF8ToUnicodeText(StringPiece str, bool do_copy) { in UTF8ToUnicodeText() function
337 return UTF8ToUnicodeText(str.data(), str.size(), do_copy); in UTF8ToUnicodeText()
340 UnicodeText UTF8ToUnicodeText(absl::string_view str, bool do_copy) { in UTF8ToUnicodeText() function
341 return UTF8ToUnicodeText(str.data(), str.size(), do_copy); in UTF8ToUnicodeText()
Dunicodetext.h236 UnicodeText UTF8ToUnicodeText(const char* utf8_buf, int len,
238 UnicodeText UTF8ToUnicodeText(const char* utf8_buf, bool do_copy = true);
239 UnicodeText UTF8ToUnicodeText(const std::string& str, bool do_copy = true);
240 UnicodeText UTF8ToUnicodeText(StringPiece str, bool do_copy = true);
241 UnicodeText UTF8ToUnicodeText(absl::string_view str, bool do_copy = true);
Dunilib-javaicu.cc427 return UTF8ToUnicodeText("", /*do_copy=*/false); in Group()
434 return UTF8ToUnicodeText("", /*do_copy=*/false); in Group()
437 return UTF8ToUnicodeText(status_or_result.ValueOrDie(), /*do_copy=*/true); in Group()
440 return UTF8ToUnicodeText("", /*do_copy=*/false); in Group()
454 return UTF8ToUnicodeText("", /*do_copy=*/false); in Group()
462 return UTF8ToUnicodeText("", /*do_copy=*/false); in Group()
469 return UTF8ToUnicodeText("", /*do_copy=*/false); in Group()
472 return UTF8ToUnicodeText(status_or_result.ValueOrDie(), /*do_copy=*/true); in Group()
475 return UTF8ToUnicodeText("", /*do_copy=*/false); in Group()
/external/libtextclassifier/native/annotator/translate/
Dtranslate_test.cc78 UTF8ToUnicodeText("Třista třicet tři stříbrných stříkaček."), {18, 28}, in TEST_F()
95 EXPECT_TRUE(translate_annotator_.ClassifyText(UTF8ToUnicodeText("学校"), in TEST_F()
117 UTF8ToUnicodeText("This is utterly unutterable."), {8, 15}, "en", in TEST_F()
125 UTF8ToUnicodeText("Třista třicet tři stříbrných stříkaček."), {8, 15}, in TEST_F()
136 UTF8ToUnicodeText("This is utterly unutterable."), {8, 15}, "cs,en,de,ja", in TEST_F()
142 UTF8ToUnicodeText("Třista třicet, tři stříbrných stříkaček"); in TEST_F()
160 UTF8ToUnicodeText("Třista třicet, tři stříbrných stříkaček"); in TEST_F()
167 UTF8ToUnicodeText("ač")); in TEST_F()
170 UTF8ToUnicodeText("stříkaček")); in TEST_F()
173 UTF8ToUnicodeText("stříkaček")); in TEST_F()
[all …]
/external/libtextclassifier/native/annotator/duration/
Dduration_test.cc124 UTF8ToUnicodeText("Wake me up in 15 minutes ok?"), {14, 24}, in TEST_F()
135 UTF8ToUnicodeText("Wake me up in15 minutesok?"), {13, 23}, in TEST_F()
146 UTF8ToUnicodeText("Weird space"), {5, 6}, in TEST_F()
151 const UnicodeText text = UTF8ToUnicodeText("Wake me up in 15 minutes ok?"); in TEST_F()
170 UTF8ToUnicodeText("Set a timer for 3 and half minutes ok?"); in TEST_F()
189 UTF8ToUnicodeText("Wake me up in 3 hours and 5 seconds ok?"); in TEST_F()
207 const UnicodeText text = UTF8ToUnicodeText( in TEST_F()
227 const UnicodeText text = UTF8ToUnicodeText("Set a timer for half an hour"); in TEST_F()
246 UTF8ToUnicodeText("Set a timer for 1 hour and a half"); in TEST_F()
265 UTF8ToUnicodeText("Set a timer for an hour and a half"); in TEST_F()
[all …]
/external/libtextclassifier/native/annotator/pod_ner/
Dpod-ner-impl_test.cc112 UTF8ToUnicodeText("Google New York , in New York"), &annotations)); in TEST_F()
119 UTF8ToUnicodeText("Jamie I'm in the first picture and Cameron and Zach " in TEST_F()
134 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(""), &annotations)); in TEST_F()
195 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(multi_word_location), in TEST_F()
203 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(single_word_location), in TEST_F()
243 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(multi_word_location), in TEST_F()
251 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(single_word_location), in TEST_F()
268 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(text), &annotations)); in TEST_F()
289 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(text), &annotations)); in TEST_F()
302 ASSERT_TRUE(annotator->Annotate(UTF8ToUnicodeText(text), &annotations)); in TEST_F()
[all …]
/external/libtextclassifier/native/annotator/grammar/
Dgrammar-annotator_test.cc71 UTF8ToUnicodeText( in TEST_F()
114 UTF8ToUnicodeText("My flight: LX 38 arriving at 4pm, I'll fly back on " in TEST_F()
157 UTF8ToUnicodeText("Please call 911 before 10 am!", /*do_copy=*/false), in TEST_F()
189 UTF8ToUnicodeText( in TEST_F()
244 UTF8ToUnicodeText("See LX 38.00", /*do_copy=*/false), CodepointSpan{4, 9}, in TEST_F()
248 UTF8ToUnicodeText("See LX 38 00", /*do_copy=*/false), CodepointSpan{4, 9}, in TEST_F()
253 UTF8ToUnicodeText("See LX 38, seat 5", /*do_copy=*/false), in TEST_F()
305 UTF8ToUnicodeText("Use tracking number 012345 for live parcel tracking.", in TEST_F()
312 UTF8ToUnicodeText("Call phone 012345 for live parcel tracking.", in TEST_F()
344 UTF8ToUnicodeText( in TEST_F()
[all …]
/external/libtextclassifier/native/utils/
Dregex-match_test.cc44 UnicodeText pattern = UTF8ToUnicodeText("(\\d{16})", in TEST_F()
46 UnicodeText message = UTF8ToUnicodeText("cc: 4012888888881881", in TEST_F()
83 UTF8ToUnicodeText("never gonna (?:give (you) up|let (you) down)", in TEST_F()
89 UTF8ToUnicodeText("never gonna give you up - never gonna let you down"); in TEST_F()
Dtoken-feature-extractor.cc57 UnicodeText word = UTF8ToUnicodeText(token, /*do_copy=*/false); in RemapTokenUnicode()
77 unilib_.CreateRegexPattern(UTF8ToUnicodeText( in TokenFeatureExtractor()
111 UTF8ToUnicodeText(token.value, /*do_copy=*/false); in ExtractDenseFeatures()
141 UTF8ToUnicodeText(token.value, /*do_copy=*/false); in ExtractDenseFeatures()
237 UnicodeText word = UTF8ToUnicodeText(token.value, /*do_copy=*/false); in ExtractCharactergramFeaturesUnicode()
266 UTF8ToUnicodeText(feature_word, /*do_copy=*/false); in ExtractCharactergramFeaturesUnicode()
Dnormalization_test.cc40 UTF8ToUnicodeText(text, /*do_copy=*/false)) in NormalizeTextCodepointWise()
Dtokenizer-utils.h69 const UnicodeText input_unicode = UTF8ToUnicodeText(input, /*do_copy=*/false); in TokenizeWithFilter()
Dtokenizer.cc98 UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false); in Tokenize()
208 UTF8ToUnicodeText(token.value, /*do_copy=*/false); in InternalRetokenize()
/external/libtextclassifier/native/utils/zlib/
Dzlib_regex.cc42 UTF8ToUnicodeText(decompressed_pattern.data(), in UncompressMakeRegexPattern()
50 UTF8ToUnicodeText(uncompressed_pattern->c_str(), in UncompressMakeRegexPattern()
/external/libtextclassifier/native/annotator/
Dfeature-processor.cc89 UTF8ToUnicodeText(it->value, /*do_copy=*/false); in SplitTokensOnSelectionBoundaries()
142 const UnicodeText context_unicode = UTF8ToUnicodeText(context, in StripTokensFromOtherLines()
224 UTF8ToUnicodeText(result_begin_token.value, /*do_copy=*/false); in LabelToSpan()
227 UTF8ToUnicodeText(result_end_token.value, /*do_copy=*/false); in LabelToSpan()
294 const UnicodeText token_left_unicode = UTF8ToUnicodeText( in SpanToLabel()
296 const UnicodeText token_right_unicode = UTF8ToUnicodeText( in SpanToLabel()
584 UTF8ToUnicodeText(context, /*do_copy=*/false); in StripBoundaryCodepoints()
626 UTF8ToUnicodeText(tokens[i].value, /*do_copy=*/false); in SupportedCodepointsRatio()
643 const UnicodeText value_unicode = UTF8ToUnicodeText(value, /*do_copy=*/false); in StripBoundaryCodepoints()
702 UTF8ToUnicodeText(context, /*do_copy=*/false); in RetokenizeAndFindClick()
Dfeature-processor_test.cc304 const UnicodeText context_unicode = UTF8ToUnicodeText(context, in TEST_F()
327 const UnicodeText context_unicode = UTF8ToUnicodeText(context, in TEST_F()
920 const UnicodeText text1 = UTF8ToUnicodeText(text1_utf8, /*do_copy=*/false); in TEST_F()
931 const UnicodeText text2 = UTF8ToUnicodeText(text2_utf8, /*do_copy=*/false); in TEST_F()
942 const UnicodeText text3 = UTF8ToUnicodeText(text3_utf8, /*do_copy=*/false); in TEST_F()
953 const UnicodeText text4 = UTF8ToUnicodeText(text4_utf8, /*do_copy=*/false); in TEST_F()
964 const UnicodeText text5 = UTF8ToUnicodeText(text5_utf8, /*do_copy=*/false); in TEST_F()
975 const UnicodeText text6 = UTF8ToUnicodeText(text6_utf8, /*do_copy=*/false); in TEST_F()
988 const UnicodeText text7 = UTF8ToUnicodeText(text7_utf8, /*do_copy=*/false); in TEST_F()
Dannotator.cc842 const UnicodeText context_unicode = UTF8ToUnicodeText(context, in SuggestSelection()
896 if (!DatetimeChunk(UTF8ToUnicodeText(context, /*do_copy=*/false), in SuggestSelection()
1427 UTF8ToUnicodeText(context, /*do_copy=*/false); in ModelClassifyText()
1619 UTF8ToUnicodeText(context, /*do_copy=*/false) in RegexClassifyText()
1622 UTF8ToUnicodeText(selection_text, /*do_copy=*/false)); in RegexClassifyText()
1682 UTF8ToUnicodeText(context, /*do_copy=*/false) in DatetimeClassifyText()
1755 UTF8ToUnicodeText(context, /*do_copy=*/false); in ClassifyText()
1960 const UnicodeText context_unicode = UTF8ToUnicodeText(context, in ModelAnnotate()
2026 line_unicode = UTF8ToUnicodeText(line_str, /*do_copy=*/false); in ModelAnnotate()
2166 UTF8ToUnicodeText(context, /*do_copy=*/false); in AnnotateSingleInput()
[all …]
Dstrip-unpaired-brackets.cc82 return StripUnpairedBrackets(UTF8ToUnicodeText(context, /*do_copy=*/false), in StripUnpairedBrackets()
/external/libtextclassifier/native/utils/grammar/
Danalyzer_test.cc65 UTF8ToUnicodeText("The month is January 2020", /*do_copy=*/false), in TEST_F()
82 analyzer.Parse(UTF8ToUnicodeText("february", /*do_copy=*/false), in TEST_F()
/external/libtextclassifier/native/actions/
Dregex-actions.cc136 UTF8ToUnicodeText(message, /*do_copy=*/false)); in IsLowConfidenceInput()
171 UTF8ToUnicodeText(action.response_text, /*do_copy=*/false)); in FilterConfidenceOutput()
201 UTF8ToUnicodeText(message, /*do_copy=*/false)); in SuggestActions()
/external/libtextclassifier/native/utils/grammar/parsing/
Dlexer.cc35 UnicodeText token_unicode = UTF8ToUnicodeText(value.data(), value.size(), in AppendTokenSymbols()
/external/libtextclassifier/native/annotator/datetime/
Dgrammar-parser.cc47 return Parse(UTF8ToUnicodeText(input, /*do_copy=*/false), in Parse()

12