1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "annotator/number/number_test-include.h"
18 
19 #include <string>
20 #include <vector>
21 
22 #include "annotator/collections.h"
23 #include "annotator/model_generated.h"
24 #include "annotator/types-test-util.h"
25 #include "annotator/types.h"
26 #include "utils/tokenizer-utils.h"
27 #include "utils/utf8/unicodetext.h"
28 #include "gmock/gmock.h"
29 #include "gtest/gtest.h"
30 
31 namespace libtextclassifier3 {
32 namespace test_internal {
33 
34 using ::testing::AllOf;
35 using ::testing::ElementsAre;
36 using ::testing::Field;
37 using ::testing::Matcher;
38 using ::testing::UnorderedElementsAre;
39 
40 const NumberAnnotatorOptions*
TestingNumberAnnotatorOptions()41 NumberAnnotatorTest::TestingNumberAnnotatorOptions() {
42   static const flatbuffers::DetachedBuffer* options_data = []() {
43     NumberAnnotatorOptionsT options;
44     options.enabled = true;
45     options.priority_score = -10.0;
46     options.float_number_priority_score = 1.0;
47     options.enabled_annotation_usecases =
48         1 << AnnotationUsecase_ANNOTATION_USECASE_RAW;
49     options.max_number_of_digits = 20;
50 
51     options.percentage_priority_score = 1.0;
52     options.percentage_annotation_usecases =
53         (1 << AnnotationUsecase_ANNOTATION_USECASE_RAW) +
54         (1 << AnnotationUsecase_ANNOTATION_USECASE_SMART);
55     std::set<std::string> percent_suffixes({"パーセント", "percent", "pércént",
56                                             "pc", "pct", "%", "٪", "﹪", "％"});
57     for (const std::string& string_value : percent_suffixes) {
58       options.percentage_pieces_string.append(string_value);
59       options.percentage_pieces_string.push_back('\0');
60     }
61 
62     flatbuffers::FlatBufferBuilder builder;
63     builder.Finish(NumberAnnotatorOptions::Pack(builder, &options));
64     return new flatbuffers::DetachedBuffer(builder.Release());
65   }();
66 
67   return flatbuffers::GetRoot<NumberAnnotatorOptions>(options_data->data());
68 }
69 
70 MATCHER_P(IsCorrectCollection, collection, "collection is " + collection) {
71   return arg.collection == collection;
72 }
73 
74 MATCHER_P(IsCorrectNumericValue, numeric_value,
75           "numeric value is " + std::to_string(numeric_value)) {
76   return arg.numeric_value == numeric_value;
77 }
78 
79 MATCHER_P(IsCorrectNumericDoubleValue, numeric_double_value,
80           "numeric double value is " + std::to_string(numeric_double_value)) {
81   return arg.numeric_double_value == numeric_double_value;
82 }
83 
84 MATCHER_P(IsCorrectScore, score, "score is " + std::to_string(score)) {
85   return arg.score == score;
86 }
87 
88 MATCHER_P(IsCorrectPriortyScore, priority_score,
89           "priority score is " + std::to_string(priority_score)) {
90   return arg.priority_score == priority_score;
91 }
92 
93 MATCHER_P(IsCorrectSpan, span,
94           "span is (" + std::to_string(span.first) + "," +
95               std::to_string(span.second) + ")") {
96   return arg.span == span;
97 }
98 
99 MATCHER_P(Classification, inner, "") {
100   return testing::ExplainMatchResult(inner, arg.classification,
101                                      result_listener);
102 }
103 
IsAnnotatedSpan(const CodepointSpan & codepoint_span,const std::string & collection,const int int_value,const double double_value,const float priority_score=-10,const float score=1)104 static Matcher<AnnotatedSpan> IsAnnotatedSpan(
105     const CodepointSpan& codepoint_span, const std::string& collection,
106     const int int_value, const double double_value,
107     const float priority_score = -10, const float score = 1) {
108   return AllOf(
109       IsCorrectSpan(codepoint_span),
110       Classification(ElementsAre(AllOf(
111           IsCorrectCollection(collection), IsCorrectNumericValue(int_value),
112           IsCorrectNumericDoubleValue(double_value), IsCorrectScore(score),
113           IsCorrectPriortyScore(priority_score)))));
114 }
115 
TEST_F(NumberAnnotatorTest,ClassifiesAndParsesNumberCorrectly)116 TEST_F(NumberAnnotatorTest, ClassifiesAndParsesNumberCorrectly) {
117   ClassificationResult classification_result;
118   EXPECT_TRUE(number_annotator_.ClassifyText(
119       UTF8ToUnicodeText("... 12345 ..."), {4, 9},
120       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
121 
122   EXPECT_EQ(classification_result.collection, "number");
123   EXPECT_EQ(classification_result.numeric_value, 12345);
124   EXPECT_FLOAT_EQ(classification_result.numeric_double_value, 12345);
125 }
126 
TEST_F(NumberAnnotatorTest,ClassifiesAndParsesNumberAsFloatCorrectly)127 TEST_F(NumberAnnotatorTest, ClassifiesAndParsesNumberAsFloatCorrectly) {
128   ClassificationResult classification_result;
129   EXPECT_TRUE(number_annotator_.ClassifyText(
130       UTF8ToUnicodeText("... 12345.12345 ..."), {4, 15},
131       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
132 
133   EXPECT_EQ(classification_result.collection, "number");
134   EXPECT_EQ(classification_result.numeric_value, 12345);
135   EXPECT_FLOAT_EQ(classification_result.numeric_double_value, 12345.12345);
136 }
137 
TEST_F(NumberAnnotatorTest,ClassifiesAndParsesNumberAsFloatCorrectlyWithoutDecimals)138 TEST_F(NumberAnnotatorTest,
139        ClassifiesAndParsesNumberAsFloatCorrectlyWithoutDecimals) {
140   ClassificationResult classification_result;
141   // The dot after a number is considered punctuation, not part of a floating
142   // number.
143   EXPECT_TRUE(number_annotator_.ClassifyText(
144       UTF8ToUnicodeText("... 12345. ..."), {4, 9},
145       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
146   EXPECT_FALSE(number_annotator_.ClassifyText(
147       UTF8ToUnicodeText("... 12345. ..."), {4, 10},
148       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
149 
150   EXPECT_EQ(classification_result.collection, "number");
151   EXPECT_EQ(classification_result.numeric_value, 12345);
152   EXPECT_FLOAT_EQ(classification_result.numeric_double_value, 12345);
153 
154   EXPECT_TRUE(number_annotator_.ClassifyText(
155       UTF8ToUnicodeText("... 12345. ..."), {4, 9},
156       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
157   EXPECT_EQ(classification_result.collection, "number");
158   EXPECT_EQ(classification_result.numeric_value, 12345);
159   EXPECT_FLOAT_EQ(classification_result.numeric_double_value, 12345);
160 }
161 
TEST_F(NumberAnnotatorTest,FindsAllIntegerAndFloatNumbersInText)162 TEST_F(NumberAnnotatorTest, FindsAllIntegerAndFloatNumbersInText) {
163   std::vector<AnnotatedSpan> result;
164   // In the context "68.9#" -> 68.9 is a number because # is punctuation.
165   // In the context "68.9#?" -> 68.9 is not a number because is followed by two
166   // punctuation signs.
167   EXPECT_TRUE(number_annotator_.FindAll(
168       UTF8ToUnicodeText("how much is 2 plus 5 divided by 7% minus 3.14 "
169                         "what about 68.9# or 68.9#?"),
170       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
171 
172   EXPECT_THAT(result,
173               UnorderedElementsAre(
174                   IsAnnotatedSpan(CodepointSpan(12, 13), "number",
175                                   /*int_value=*/2, /*double_value=*/2.0),
176                   IsAnnotatedSpan(CodepointSpan(19, 20), "number",
177                                   /*int_value=*/5, /*double_value=*/5.0),
178                   IsAnnotatedSpan(CodepointSpan(32, 33), "number",
179                                   /*int_value=*/7, /*double_value=*/7.0),
180                   IsAnnotatedSpan(CodepointSpan(32, 34), "percentage",
181                                   /*int_value=*/7, /*double_value=*/7.0,
182                                   /*priority_score=*/1),
183                   IsAnnotatedSpan(CodepointSpan(41, 45), "number",
184                                   /*int_value=*/3, /*double_value=*/3.14,
185                                   /*priority_score=*/1),
186                   IsAnnotatedSpan(CodepointSpan(57, 61), "number",
187                                   /*int_value=*/68, /*double_value=*/68.9,
188                                   /*priority_score=*/1)));
189 }
190 
TEST_F(NumberAnnotatorTest,ClassifiesNonNumberCorrectly)191 TEST_F(NumberAnnotatorTest, ClassifiesNonNumberCorrectly) {
192   ClassificationResult classification_result;
193   EXPECT_FALSE(number_annotator_.ClassifyText(
194       UTF8ToUnicodeText("... 123a45 ..."), {4, 10},
195       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
196   EXPECT_FALSE(number_annotator_.ClassifyText(
197       UTF8ToUnicodeText("... 12345..12345 ..."), {4, 16},
198       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
199   EXPECT_FALSE(number_annotator_.ClassifyText(
200       UTF8ToUnicodeText("... 12345a ..."), {4, 11},
201       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
202 }
203 
TEST_F(NumberAnnotatorTest,ClassifiesNumberSelectionCorrectly)204 TEST_F(NumberAnnotatorTest, ClassifiesNumberSelectionCorrectly) {
205   ClassificationResult classification_result;
206   // Punctuation after a number is not part of the number.
207   EXPECT_TRUE(number_annotator_.ClassifyText(
208       UTF8ToUnicodeText("... 14, ..."), {4, 6},
209       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
210   EXPECT_EQ(classification_result.collection, "number");
211   EXPECT_EQ(classification_result.numeric_value, 14);
212   EXPECT_EQ(classification_result.numeric_double_value, 14);
213 
214   EXPECT_FALSE(number_annotator_.ClassifyText(
215       UTF8ToUnicodeText("... 14, ..."), {4, 7},
216       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
217 }
218 
TEST_F(NumberAnnotatorTest,ClassifiesPercentageSignCorrectly)219 TEST_F(NumberAnnotatorTest, ClassifiesPercentageSignCorrectly) {
220   ClassificationResult classification_result;
221   EXPECT_TRUE(number_annotator_.ClassifyText(
222       UTF8ToUnicodeText("... 99% ..."), {4, 7},
223       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
224 
225   EXPECT_EQ(classification_result.collection, "percentage");
226   EXPECT_EQ(classification_result.numeric_value, 99);
227   EXPECT_EQ(classification_result.numeric_double_value, 99);
228 }
229 
TEST_F(NumberAnnotatorTest,ClassifiesPercentageWordCorrectly)230 TEST_F(NumberAnnotatorTest, ClassifiesPercentageWordCorrectly) {
231   ClassificationResult classification_result;
232   EXPECT_TRUE(number_annotator_.ClassifyText(
233       UTF8ToUnicodeText("... 15 percent ..."), {4, 14},
234       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
235 
236   EXPECT_EQ(classification_result.collection, "percentage");
237   EXPECT_EQ(classification_result.numeric_value, 15);
238   EXPECT_EQ(classification_result.numeric_double_value, 15);
239 }
240 
TEST_F(NumberAnnotatorTest,ClassifiesNonAsciiPercentageIncorrectSuffix)241 TEST_F(NumberAnnotatorTest, ClassifiesNonAsciiPercentageIncorrectSuffix) {
242   ClassificationResult classification_result;
243   EXPECT_FALSE(number_annotator_.ClassifyText(
244       UTF8ToUnicodeText("15 café"), {0, 7},
245       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
246 }
247 
TEST_F(NumberAnnotatorTest,ClassifiesNonAsciiFrPercentageCorrectSuffix)248 TEST_F(NumberAnnotatorTest, ClassifiesNonAsciiFrPercentageCorrectSuffix) {
249   ClassificationResult classification_result;
250   EXPECT_TRUE(number_annotator_.ClassifyText(
251       UTF8ToUnicodeText("25 pércént"), {0, 10},
252       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
253 
254   EXPECT_EQ(classification_result.collection, "percentage");
255   EXPECT_EQ(classification_result.numeric_value, 25);
256   EXPECT_EQ(classification_result.numeric_double_value, 25);
257 }
258 
TEST_F(NumberAnnotatorTest,ClassifiesNonAsciiJaPercentageCorrectSuffix)259 TEST_F(NumberAnnotatorTest, ClassifiesNonAsciiJaPercentageCorrectSuffix) {
260   ClassificationResult classification_result;
261   EXPECT_TRUE(number_annotator_.ClassifyText(
262       UTF8ToUnicodeText("10パーセント"), {0, 7},
263       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
264   EXPECT_EQ(classification_result.collection, "percentage");
265   EXPECT_EQ(classification_result.numeric_value, 10);
266   EXPECT_EQ(classification_result.numeric_double_value, 10);
267 
268   std::vector<AnnotatedSpan> result;
269   EXPECT_TRUE(number_annotator_.FindAll(
270       UTF8ToUnicodeText("明日の降水確率は10パーセント  音量を12にセット"),
271       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
272   EXPECT_THAT(result,
273               UnorderedElementsAre(
274                   IsAnnotatedSpan(CodepointSpan(8, 10), "number",
275                                   /*int_value=*/10, /*double_value=*/10.0),
276                   IsAnnotatedSpan(CodepointSpan(8, 15), "percentage",
277                                   /*int_value=*/10, /*double_value=*/10.0,
278                                   /*priority_score=*/1),
279                   IsAnnotatedSpan(CodepointSpan(20, 22), "number",
280                                   /*int_value=*/12, /*double_value=*/12.0)));
281 }
282 
TEST_F(NumberAnnotatorTest,FindsAllNumbersInText)283 TEST_F(NumberAnnotatorTest, FindsAllNumbersInText) {
284   std::vector<AnnotatedSpan> result;
285   EXPECT_TRUE(number_annotator_.FindAll(
286       UTF8ToUnicodeText("... 12345 ... 9 is my number and 27% or 68# #38 ＃39 "
287                         "but not $99."),
288       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
289 
290   EXPECT_THAT(
291       result,
292       UnorderedElementsAre(
293           IsAnnotatedSpan(CodepointSpan(4, 9), "number",
294                           /*int_value=*/12345, /*double_value=*/12345.0),
295           IsAnnotatedSpan(CodepointSpan(14, 15), "number",
296                           /*int_value=*/9, /*double_value=*/9.0),
297           IsAnnotatedSpan(CodepointSpan(33, 35), "number",
298                           /*int_value=*/27, /*double_value=*/27.0),
299           IsAnnotatedSpan(CodepointSpan(33, 36), "percentage",
300                           /*int_value=*/27, /*double_value=*/27.0,
301                           /*priority_score=*/1),
302           IsAnnotatedSpan(CodepointSpan(40, 42), "number",
303                           /*int_value=*/68, /*double_value=*/68.0),
304           IsAnnotatedSpan(CodepointSpan(45, 47), "number",
305                           /*int_value=*/38, /*double_value=*/38.0),
306           IsAnnotatedSpan(CodepointSpan(49, 51), "number",
307                           /*int_value=*/39, /*double_value=*/39.0)));
308 }
309 
TEST_F(NumberAnnotatorTest,FindsNoNumberInText)310 TEST_F(NumberAnnotatorTest, FindsNoNumberInText) {
311   std::vector<AnnotatedSpan> result;
312   EXPECT_TRUE(number_annotator_.FindAll(
313       UTF8ToUnicodeText("... 12345a ... 12345..12345 and 123a45 are not valid. "
314                         "And -#5% is also bad."),
315       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
316   ASSERT_EQ(result.size(), 0);
317 }
318 
TEST_F(NumberAnnotatorTest,FindsNumberWithPunctuation)319 TEST_F(NumberAnnotatorTest, FindsNumberWithPunctuation) {
320   std::vector<AnnotatedSpan> result;
321   // A number should be followed by only one punctuation signs => 15 is not a
322   // number.
323   EXPECT_TRUE(number_annotator_.FindAll(
324       UTF8ToUnicodeText(
325           "It's 12, 13, 14! Or 15??? For sure 16: 17; 18. and －19"),
326       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
327 
328   EXPECT_THAT(result,
329               UnorderedElementsAre(
330                   IsAnnotatedSpan(CodepointSpan(5, 7), "number",
331                                   /*int_value=*/12, /*double_value=*/12.0),
332                   IsAnnotatedSpan(CodepointSpan(9, 11), "number",
333                                   /*int_value=*/13, /*double_value=*/13.0),
334                   IsAnnotatedSpan(CodepointSpan(13, 15), "number",
335                                   /*int_value=*/14, /*double_value=*/14.0),
336                   IsAnnotatedSpan(CodepointSpan(35, 37), "number",
337                                   /*int_value=*/16, /*double_value=*/16.0),
338                   IsAnnotatedSpan(CodepointSpan(39, 41), "number",
339                                   /*int_value=*/17, /*double_value=*/17.0),
340                   IsAnnotatedSpan(CodepointSpan(43, 45), "number",
341                                   /*int_value=*/18, /*double_value=*/18.0),
342                   IsAnnotatedSpan(CodepointSpan(51, 54), "number",
343                                   /*int_value=*/-19, /*double_value=*/-19.0)));
344 }
345 
TEST_F(NumberAnnotatorTest,FindsFloatNumberWithPunctuation)346 TEST_F(NumberAnnotatorTest, FindsFloatNumberWithPunctuation) {
347   std::vector<AnnotatedSpan> result;
348   EXPECT_TRUE(number_annotator_.FindAll(
349       UTF8ToUnicodeText("It's 12.123, 13.45, 14.54321! Or 15.1? Maybe 16.33: "
350                         "17.21; but for sure 18.90."),
351       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
352 
353   EXPECT_THAT(result,
354               UnorderedElementsAre(
355                   IsAnnotatedSpan(CodepointSpan(5, 11), "number",
356                                   /*int_value=*/12, /*double_value=*/12.123,
357                                   /*priority_score=*/1),
358                   IsAnnotatedSpan(CodepointSpan(13, 18), "number",
359                                   /*int_value=*/13, /*double_value=*/13.45,
360                                   /*priority_score=*/1),
361                   IsAnnotatedSpan(CodepointSpan(20, 28), "number",
362                                   /*int_value=*/14, /*double_value=*/14.54321,
363                                   /*priority_score=*/1),
364                   IsAnnotatedSpan(CodepointSpan(33, 37), "number",
365                                   /*int_value=*/15, /*double_value=*/15.1,
366                                   /*priority_score=*/1),
367                   IsAnnotatedSpan(CodepointSpan(45, 50), "number",
368                                   /*int_value=*/16, /*double_value=*/16.33,
369                                   /*priority_score=*/1),
370                   IsAnnotatedSpan(CodepointSpan(52, 57), "number",
371                                   /*int_value=*/17, /*double_value=*/17.21,
372                                   /*priority_score=*/1),
373                   IsAnnotatedSpan(CodepointSpan(72, 77), "number",
374                                   /*int_value=*/18, /*double_value=*/18.9,
375                                   /*priority_score=*/1)));
376 }
377 
TEST_F(NumberAnnotatorTest,HandlesNumbersAtBeginning)378 TEST_F(NumberAnnotatorTest, HandlesNumbersAtBeginning) {
379   std::vector<AnnotatedSpan> result;
380   EXPECT_TRUE(number_annotator_.FindAll(
381       UTF8ToUnicodeText("-5"), AnnotationUsecase_ANNOTATION_USECASE_RAW,
382       &result));
383 
384   EXPECT_THAT(result, UnorderedElementsAre(IsAnnotatedSpan(
385                           CodepointSpan(0, 2), "number",
386                           /*int_value=*/-5, /*double_value=*/-5)));
387 }
388 
TEST_F(NumberAnnotatorTest,HandlesNegativeNumbers)389 TEST_F(NumberAnnotatorTest, HandlesNegativeNumbers) {
390   std::vector<AnnotatedSpan> result;
391   EXPECT_TRUE(number_annotator_.FindAll(
392       UTF8ToUnicodeText("Number -5 and -5% and not number --5%"),
393       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
394 
395   EXPECT_THAT(result,
396               UnorderedElementsAre(
397                   IsAnnotatedSpan(CodepointSpan(7, 9), "number",
398                                   /*int_value=*/-5, /*double_value=*/-5),
399                   IsAnnotatedSpan(CodepointSpan(14, 16), "number",
400                                   /*int_value=*/-5, /*double_value=*/-5),
401                   IsAnnotatedSpan(CodepointSpan(14, 17), "percentage",
402                                   /*int_value=*/-5, /*double_value=*/-5,
403                                   /*priority_score=*/1)));
404 }
405 
TEST_F(NumberAnnotatorTest,FindGoodPercentageContexts)406 TEST_F(NumberAnnotatorTest, FindGoodPercentageContexts) {
407   std::vector<AnnotatedSpan> result;
408   EXPECT_TRUE(number_annotator_.FindAll(
409       UTF8ToUnicodeText(
410           "5 percent, 10 pct, 25 pc and 17%, -5 percent, 10％ are percentages"),
411       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
412 
413   EXPECT_THAT(result,
414               UnorderedElementsAre(
415                   IsAnnotatedSpan(CodepointSpan(0, 1), "number",
416                                   /*int_value=*/5, /*double_value=*/5),
417                   IsAnnotatedSpan(CodepointSpan(0, 9), "percentage",
418                                   /*int_value=*/5, /*double_value=*/5,
419                                   /*priority_score=*/1),
420                   IsAnnotatedSpan(CodepointSpan(11, 13), "number",
421                                   /*int_value=*/10, /*double_value=*/10),
422                   IsAnnotatedSpan(CodepointSpan(11, 17), "percentage",
423                                   /*int_value=*/10, /*double_value=*/10,
424                                   /*priority_score=*/1),
425                   IsAnnotatedSpan(CodepointSpan(19, 21), "number",
426                                   /*int_value=*/25, /*double_value=*/25),
427                   IsAnnotatedSpan(CodepointSpan(19, 24), "percentage",
428                                   /*int_value=*/25, /*double_value=*/25,
429                                   /*priority_score=*/1),
430                   IsAnnotatedSpan(CodepointSpan(29, 31), "number",
431                                   /*int_value=*/17, /*double_value=*/17),
432                   IsAnnotatedSpan(CodepointSpan(29, 32), "percentage",
433                                   /*int_value=*/17, /*double_value=*/17,
434                                   /*priority_score=*/1),
435                   IsAnnotatedSpan(CodepointSpan(34, 36), "number",
436                                   /*int_value=*/-5, /*double_value=*/-5),
437                   IsAnnotatedSpan(CodepointSpan(34, 44), "percentage",
438                                   /*int_value=*/-5, /*double_value=*/-5,
439                                   /*priority_score=*/1),
440                   IsAnnotatedSpan(CodepointSpan(46, 48), "number",
441                                   /*int_value=*/10, /*double_value=*/10),
442                   IsAnnotatedSpan(CodepointSpan(46, 49), "percentage",
443                                   /*int_value=*/10, /*double_value=*/10,
444                                   /*priority_score=*/1)));
445 }
446 
TEST_F(NumberAnnotatorTest,FindSinglePercentageInContext)447 TEST_F(NumberAnnotatorTest, FindSinglePercentageInContext) {
448   std::vector<AnnotatedSpan> result;
449   EXPECT_TRUE(number_annotator_.FindAll(
450       UTF8ToUnicodeText("5%"), AnnotationUsecase_ANNOTATION_USECASE_RAW,
451       &result));
452 
453   EXPECT_THAT(result, UnorderedElementsAre(
454                           IsAnnotatedSpan(CodepointSpan(0, 1), "number",
455                                           /*int_value=*/5, /*double_value=*/5),
456                           IsAnnotatedSpan(CodepointSpan(0, 2), "percentage",
457                                           /*int_value=*/5, /*double_value=*/5,
458                                           /*priority_score=*/1)));
459 }
460 
TEST_F(NumberAnnotatorTest,IgnoreBadPercentageContexts)461 TEST_F(NumberAnnotatorTest, IgnoreBadPercentageContexts) {
462   std::vector<AnnotatedSpan> result;
463   // A valid number is followed by only one punctuation element.
464   EXPECT_TRUE(number_annotator_.FindAll(
465       UTF8ToUnicodeText("10, pct, 25 prc, 5#: percentage are not percentages"),
466       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
467 
468   EXPECT_THAT(result,
469               UnorderedElementsAre(
470                   IsAnnotatedSpan(CodepointSpan(0, 2), "number",
471                                   /*int_value=*/10, /*double_value=*/10),
472                   IsAnnotatedSpan(CodepointSpan(9, 11), "number",
473                                   /*int_value=*/25, /*double_value=*/25)));
474 }
475 
TEST_F(NumberAnnotatorTest,IgnoreBadPercentagePunctuationContexts)476 TEST_F(NumberAnnotatorTest, IgnoreBadPercentagePunctuationContexts) {
477   std::vector<AnnotatedSpan> result;
478   EXPECT_TRUE(number_annotator_.FindAll(
479       UTF8ToUnicodeText(
480           "#!24% or :?33 percent are not valid percentages, nor numbers."),
481       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
482 
483   EXPECT_TRUE(result.empty());
484 }
485 
TEST_F(NumberAnnotatorTest,FindPercentageInNonAsciiContext)486 TEST_F(NumberAnnotatorTest, FindPercentageInNonAsciiContext) {
487   std::vector<AnnotatedSpan> result;
488   EXPECT_TRUE(number_annotator_.FindAll(
489       UTF8ToUnicodeText(
490           "At the café 10% or 25 percent of people are nice. Only 10%!"),
491       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
492 
493   EXPECT_THAT(result,
494               UnorderedElementsAre(
495                   IsAnnotatedSpan(CodepointSpan(12, 14), "number",
496                                   /*int_value=*/10, /*double_value=*/10),
497                   IsAnnotatedSpan(CodepointSpan(12, 15), "percentage",
498                                   /*int_value=*/10, /*double_value=*/10,
499                                   /*priority_score=*/1),
500                   IsAnnotatedSpan(CodepointSpan(19, 21), "number",
501                                   /*int_value=*/25, /*double_value=*/25),
502                   IsAnnotatedSpan(CodepointSpan(19, 29), "percentage",
503                                   /*int_value=*/25, /*double_value=*/25,
504                                   /*priority_score=*/1),
505                   IsAnnotatedSpan(CodepointSpan(55, 57), "number",
506                                   /*int_value=*/10, /*double_value=*/10),
507                   IsAnnotatedSpan(CodepointSpan(55, 58), "percentage",
508                                   /*int_value=*/10, /*double_value=*/10,
509                                   /*priority_score=*/1)));
510 }
511 
TEST_F(NumberAnnotatorTest,WhenPercentSuffixWithAdditionalIgnoredCharactersDoesNotParseIt)512 TEST_F(NumberAnnotatorTest,
513        WhenPercentSuffixWithAdditionalIgnoredCharactersDoesNotParseIt) {
514   ClassificationResult classification_result;
515   EXPECT_FALSE(number_annotator_.ClassifyText(
516       UTF8ToUnicodeText("23#!? percent"), {0, 13},
517       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
518 }
519 
TEST_F(NumberAnnotatorTest,WhenPercentSuffixWithAdditionalRandomTokensDoesNotParseIt)520 TEST_F(NumberAnnotatorTest,
521        WhenPercentSuffixWithAdditionalRandomTokensDoesNotParseIt) {
522   ClassificationResult classification_result;
523   EXPECT_FALSE(number_annotator_.ClassifyText(
524       UTF8ToUnicodeText("23 asdf 3.14 pct asdf"), {0, 21},
525       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
526 }
527 
TEST_F(NumberAnnotatorTest,WhenPercentSuffixWithAdditionalRandomPrefixSuffixDoesNotParseIt)528 TEST_F(NumberAnnotatorTest,
529        WhenPercentSuffixWithAdditionalRandomPrefixSuffixDoesNotParseIt) {
530   ClassificationResult classification_result;
531   EXPECT_FALSE(number_annotator_.ClassifyText(
532       UTF8ToUnicodeText("abdf23 percentabdf"), {0, 18},
533       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
534 }
535 
TEST_F(NumberAnnotatorTest,WhenPercentSuffixWithAdditionalRandomStringsDoesNotParsesIt)536 TEST_F(NumberAnnotatorTest,
537        WhenPercentSuffixWithAdditionalRandomStringsDoesNotParsesIt) {
538   ClassificationResult classification_result;
539   EXPECT_FALSE(number_annotator_.ClassifyText(
540       UTF8ToUnicodeText("#?!23 percent#!?"), {0, 16},
541       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
542 }
543 
TEST_F(NumberAnnotatorTest,WhenBothPercentSymbolAndSuffixDoesNotParseIt)544 TEST_F(NumberAnnotatorTest, WhenBothPercentSymbolAndSuffixDoesNotParseIt) {
545   ClassificationResult classification_result;
546   EXPECT_FALSE(number_annotator_.ClassifyText(
547       UTF8ToUnicodeText("23% percent"), {0, 11},
548       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
549 }
550 
TEST_F(NumberAnnotatorTest,WhenPercentSymbolWithAdditionalPrefixCharactersDoesNotParsesIt)551 TEST_F(NumberAnnotatorTest,
552        WhenPercentSymbolWithAdditionalPrefixCharactersDoesNotParsesIt) {
553   ClassificationResult classification_result;
554   EXPECT_FALSE(number_annotator_.ClassifyText(
555       UTF8ToUnicodeText("#?23%"), {0, 5},
556       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
557 }
558 
TEST_F(NumberAnnotatorTest,WhenNumberWithAdditionalCharactersDoesNotParsesIt)559 TEST_F(NumberAnnotatorTest, WhenNumberWithAdditionalCharactersDoesNotParsesIt) {
560   ClassificationResult classification_result;
561   EXPECT_FALSE(number_annotator_.ClassifyText(
562       UTF8ToUnicodeText("23#!?"), {0, 5},
563       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
564 }
565 
TEST_F(NumberAnnotatorTest,WhenPercentSymbolWithAdditionalCharactersDoesNotParsesIt)566 TEST_F(NumberAnnotatorTest,
567        WhenPercentSymbolWithAdditionalCharactersDoesNotParsesIt) {
568   ClassificationResult classification_result;
569   // ! does not belong to the percentage annotation
570   EXPECT_TRUE(number_annotator_.ClassifyText(
571       UTF8ToUnicodeText("23%!"), {0, 3},
572       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
573   EXPECT_EQ(classification_result.collection, "percentage");
574   EXPECT_EQ(classification_result.numeric_value, 23);
575   EXPECT_EQ(classification_result.numeric_double_value, 23);
576 
577   EXPECT_FALSE(number_annotator_.ClassifyText(
578       UTF8ToUnicodeText("23%!"), {0, 4},
579       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
580 }
581 
TEST_F(NumberAnnotatorTest,WhenAdditionalCharactersWithMisplacedPercentSymbolDoesNotParsesIt)582 TEST_F(NumberAnnotatorTest,
583        WhenAdditionalCharactersWithMisplacedPercentSymbolDoesNotParsesIt) {
584   ClassificationResult classification_result;
585   EXPECT_FALSE(number_annotator_.ClassifyText(
586       UTF8ToUnicodeText("23.:;%"), {0, 6},
587       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
588 }
589 
TEST_F(NumberAnnotatorTest,WhenMultipleMinusSignsDoesNotParsesIt)590 TEST_F(NumberAnnotatorTest, WhenMultipleMinusSignsDoesNotParsesIt) {
591   ClassificationResult classification_result;
592   EXPECT_TRUE(number_annotator_.ClassifyText(
593       UTF8ToUnicodeText("--11"), {1, 4},
594       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
595   EXPECT_THAT(classification_result,
596               AllOf(Field(&ClassificationResult::collection, "number"),
597                     Field(&ClassificationResult::numeric_value, -11),
598                     Field(&ClassificationResult::numeric_double_value, -11)));
599 
600   EXPECT_FALSE(number_annotator_.ClassifyText(
601       UTF8ToUnicodeText("--11"), {0, 4},
602       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
603 }
604 
TEST_F(NumberAnnotatorTest,WhenMultipleMinusSignsPercentSignDoesNotParsesIt)605 TEST_F(NumberAnnotatorTest, WhenMultipleMinusSignsPercentSignDoesNotParsesIt) {
606   ClassificationResult classification_result;
607   EXPECT_TRUE(number_annotator_.ClassifyText(
608       UTF8ToUnicodeText("--11%"), {1, 5},
609       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
610   EXPECT_THAT(classification_result,
611               AllOf(Field(&ClassificationResult::collection, "percentage"),
612                     Field(&ClassificationResult::numeric_value, -11),
613                     Field(&ClassificationResult::numeric_double_value, -11)));
614 
615   EXPECT_FALSE(number_annotator_.ClassifyText(
616       UTF8ToUnicodeText("--11%"), {0, 5},
617       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
618 }
619 
TEST_F(NumberAnnotatorTest,WhenPlusMinusSignsDoesNotParsesIt)620 TEST_F(NumberAnnotatorTest, WhenPlusMinusSignsDoesNotParsesIt) {
621   ClassificationResult classification_result;
622   EXPECT_TRUE(number_annotator_.ClassifyText(
623       UTF8ToUnicodeText("+-11"), {1, 4},
624       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
625   EXPECT_THAT(classification_result,
626               AllOf(Field(&ClassificationResult::collection, "number"),
627                     Field(&ClassificationResult::numeric_value, -11),
628                     Field(&ClassificationResult::numeric_double_value, -11)));
629 
630   EXPECT_FALSE(number_annotator_.ClassifyText(
631       UTF8ToUnicodeText("+-11"), {0, 4},
632       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
633 }
634 
TEST_F(NumberAnnotatorTest,WhenMinusPlusSignsDoesNotParsesIt)635 TEST_F(NumberAnnotatorTest, WhenMinusPlusSignsDoesNotParsesIt) {
636   ClassificationResult classification_result;
637   // + right before a number is not included in the number annotation
638   EXPECT_FALSE(number_annotator_.ClassifyText(
639       UTF8ToUnicodeText("-+11"), {1, 4},
640       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
641   EXPECT_FALSE(number_annotator_.ClassifyText(
642       UTF8ToUnicodeText("-+11"), {0, 4},
643       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
644 }
645 
TEST_F(NumberAnnotatorTest,WhenMinusSignSuffixDoesNotParsesIt)646 TEST_F(NumberAnnotatorTest, WhenMinusSignSuffixDoesNotParsesIt) {
647   ClassificationResult classification_result;
648   EXPECT_FALSE(number_annotator_.ClassifyText(
649       UTF8ToUnicodeText("10-"), {0, 3},
650       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
651 }
652 
TEST_F(NumberAnnotatorTest,WhenMultipleCharSuffixDoesNotParsesIt)653 TEST_F(NumberAnnotatorTest, WhenMultipleCharSuffixDoesNotParsesIt) {
654   ClassificationResult classification_result;
655   EXPECT_TRUE(number_annotator_.ClassifyText(
656       UTF8ToUnicodeText("10**"), {0, 2},
657       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
658   EXPECT_THAT(classification_result,
659               AllOf(Field(&ClassificationResult::collection, "number"),
660                     Field(&ClassificationResult::numeric_value, 10),
661                     Field(&ClassificationResult::numeric_double_value, 10)));
662 
663   EXPECT_FALSE(number_annotator_.ClassifyText(
664       UTF8ToUnicodeText("10**"), {0, 3},
665       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
666   EXPECT_FALSE(number_annotator_.ClassifyText(
667       UTF8ToUnicodeText("10**"), {0, 4},
668       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
669 }
670 
TEST_F(NumberAnnotatorTest,WhenMultipleCharPrefixDoesNotParsesIt)671 TEST_F(NumberAnnotatorTest, WhenMultipleCharPrefixDoesNotParsesIt) {
672   ClassificationResult classification_result;
673   EXPECT_FALSE(number_annotator_.ClassifyText(
674       UTF8ToUnicodeText("**10"), {1, 4},
675       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
676   EXPECT_FALSE(number_annotator_.ClassifyText(
677       UTF8ToUnicodeText("**10"), {0, 4},
678       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
679 }
680 
TEST_F(NumberAnnotatorTest,WhenLowestSupportedNumberParsesIt)681 TEST_F(NumberAnnotatorTest, WhenLowestSupportedNumberParsesIt) {
682   ClassificationResult classification_result;
683   EXPECT_TRUE(number_annotator_.ClassifyText(
684       UTF8ToUnicodeText("-1000000000"), {0, 11},
685       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
686 
687   EXPECT_THAT(
688       classification_result,
689       AllOf(Field(&ClassificationResult::collection, "number"),
690             Field(&ClassificationResult::numeric_value, -1000000000),
691             Field(&ClassificationResult::numeric_double_value, -1000000000)));
692 }
693 
TEST_F(NumberAnnotatorTest,WhenLargestSupportedNumberParsesIt)694 TEST_F(NumberAnnotatorTest, WhenLargestSupportedNumberParsesIt) {
695   ClassificationResult classification_result;
696   EXPECT_TRUE(number_annotator_.ClassifyText(
697       UTF8ToUnicodeText("1000000000"), {0, 10},
698       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
699 
700   EXPECT_THAT(
701       classification_result,
702       AllOf(Field(&ClassificationResult::collection, "number"),
703             Field(&ClassificationResult::numeric_value, 1000000000),
704             Field(&ClassificationResult::numeric_double_value, 1000000000)));
705 }
706 
TEST_F(NumberAnnotatorTest,WhenLowestSupportedFloatNumberParsesIt)707 TEST_F(NumberAnnotatorTest, WhenLowestSupportedFloatNumberParsesIt) {
708   ClassificationResult classification_result;
709   EXPECT_TRUE(number_annotator_.ClassifyText(
710       UTF8ToUnicodeText("-999999999.999999999"), {0, 20},
711       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
712 
713   EXPECT_THAT(classification_result,
714               AllOf(Field(&ClassificationResult::collection, "number"),
715                     Field(&ClassificationResult::numeric_value, -1000000000),
716                     Field(&ClassificationResult::numeric_double_value,
717                           -999999999.999999999)));
718 }
719 
TEST_F(NumberAnnotatorTest,WhenLargestFloatSupportedNumberParsesIt)720 TEST_F(NumberAnnotatorTest, WhenLargestFloatSupportedNumberParsesIt) {
721   ClassificationResult classification_result;
722   EXPECT_TRUE(number_annotator_.ClassifyText(
723       UTF8ToUnicodeText("999999999.999999999"), {0, 19},
724       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
725 
726   EXPECT_THAT(classification_result,
727               AllOf(Field(&ClassificationResult::collection, "number"),
728                     Field(&ClassificationResult::numeric_value, 1000000000),
729                     Field(&ClassificationResult::numeric_double_value,
730                           999999999.999999999)));
731 }
732 
TEST_F(NumberAnnotatorTest,WhenLargeNumberDoesNotParseIt)733 TEST_F(NumberAnnotatorTest, WhenLargeNumberDoesNotParseIt) {
734   ClassificationResult classification_result;
735   EXPECT_FALSE(number_annotator_.ClassifyText(
736       UTF8ToUnicodeText("1234567890123456789012345678901234567890"), {0, 40},
737       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
738 }
739 
TEST_F(NumberAnnotatorTest,WhenMinusInTheMiddleDoesNotParseIt)740 TEST_F(NumberAnnotatorTest, WhenMinusInTheMiddleDoesNotParseIt) {
741   ClassificationResult classification_result;
742   EXPECT_FALSE(number_annotator_.ClassifyText(
743       UTF8ToUnicodeText("2016-2017"), {0, 9},
744       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
745 }
746 
TEST_F(NumberAnnotatorTest,WhenSuffixWithoutNumberDoesNotParseIt)747 TEST_F(NumberAnnotatorTest, WhenSuffixWithoutNumberDoesNotParseIt) {
748   std::vector<AnnotatedSpan> result;
749   EXPECT_TRUE(number_annotator_.FindAll(
750       UTF8ToUnicodeText("... % ..."), AnnotationUsecase_ANNOTATION_USECASE_RAW,
751       &result));
752 
753   ASSERT_EQ(result.size(), 0);
754 }
755 
TEST_F(NumberAnnotatorTest,WhenPrefixWithoutNumberDoesNotParseIt)756 TEST_F(NumberAnnotatorTest, WhenPrefixWithoutNumberDoesNotParseIt) {
757   std::vector<AnnotatedSpan> result;
758   EXPECT_TRUE(number_annotator_.FindAll(
759       UTF8ToUnicodeText("... $ ..."), AnnotationUsecase_ANNOTATION_USECASE_RAW,
760       &result));
761 
762   ASSERT_EQ(result.size(), 0);
763 }
764 
TEST_F(NumberAnnotatorTest,WhenPrefixAndSuffixWithoutNumberDoesNotParseIt)765 TEST_F(NumberAnnotatorTest, WhenPrefixAndSuffixWithoutNumberDoesNotParseIt) {
766   std::vector<AnnotatedSpan> result;
767   EXPECT_TRUE(number_annotator_.FindAll(
768       UTF8ToUnicodeText("... $% ..."), AnnotationUsecase_ANNOTATION_USECASE_RAW,
769       &result));
770 
771   ASSERT_EQ(result.size(), 0);
772 }
773 
TEST_F(NumberAnnotatorTest,ForNumberAnnotationsSetsScoreAndPriorityScore)774 TEST_F(NumberAnnotatorTest, ForNumberAnnotationsSetsScoreAndPriorityScore) {
775   ClassificationResult classification_result;
776   EXPECT_TRUE(number_annotator_.ClassifyText(
777       UTF8ToUnicodeText("... 12345 ..."), {4, 9},
778       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
779 
780   EXPECT_EQ(classification_result.collection, "number");
781   EXPECT_EQ(classification_result.numeric_value, 12345);
782   EXPECT_EQ(classification_result.numeric_double_value, 12345);
783   EXPECT_EQ(classification_result.score, 1);
784   EXPECT_EQ(classification_result.priority_score, -10);
785 
786   std::vector<AnnotatedSpan> result;
787   EXPECT_TRUE(number_annotator_.FindAll(
788       UTF8ToUnicodeText("Come at 9 or 10 ok?"),
789       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
790 
791   EXPECT_THAT(result,
792               UnorderedElementsAre(
793                   IsAnnotatedSpan(CodepointSpan(8, 9), "number",
794                                   /*int_value=*/9, /*double_value=*/9),
795                   IsAnnotatedSpan(CodepointSpan(13, 15), "number",
796                                   /*int_value=*/10, /*double_value=*/10)));
797 }
798 
TEST_F(NumberAnnotatorTest,ForFloatNumberAnnotationsSetsScoreAndPriorityScore)799 TEST_F(NumberAnnotatorTest,
800        ForFloatNumberAnnotationsSetsScoreAndPriorityScore) {
801   ClassificationResult classification_result;
802   EXPECT_TRUE(number_annotator_.ClassifyText(
803       UTF8ToUnicodeText("... 12345.12345 ..."), {4, 15},
804       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
805   EXPECT_EQ(classification_result.collection, "number");
806   EXPECT_EQ(classification_result.numeric_value, 12345);
807   EXPECT_EQ(classification_result.numeric_double_value, 12345.12345);
808   EXPECT_EQ(classification_result.score, 1);
809   EXPECT_EQ(classification_result.priority_score, 1);
810 
811   std::vector<AnnotatedSpan> result;
812   EXPECT_TRUE(number_annotator_.FindAll(
813       UTF8ToUnicodeText("Results are between 12.5 and 13.5, right?"),
814       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
815   EXPECT_THAT(result,
816               UnorderedElementsAre(
817                   IsAnnotatedSpan(CodepointSpan(20, 24), "number",
818                                   /*int_value=*/12, /*double_value=*/12.5,
819                                   /*priority_score=*/1),
820                   IsAnnotatedSpan(CodepointSpan(29, 33), "number",
821                                   /*int_value=*/13, /*double_value=*/13.5,
822                                   /*priority_score=*/1)));
823 }
824 
TEST_F(NumberAnnotatorTest,ForPercentageAnnotationsSetsScoreAndPriorityScore)825 TEST_F(NumberAnnotatorTest, ForPercentageAnnotationsSetsScoreAndPriorityScore) {
826   ClassificationResult classification_result;
827   EXPECT_TRUE(number_annotator_.ClassifyText(
828       UTF8ToUnicodeText("... 12345% ..."), {4, 10},
829       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
830   EXPECT_EQ(classification_result.collection, "percentage");
831   EXPECT_EQ(classification_result.numeric_value, 12345);
832   EXPECT_EQ(classification_result.numeric_double_value, 12345);
833   EXPECT_EQ(classification_result.score, 1);
834   EXPECT_EQ(classification_result.priority_score, 1);
835 
836   EXPECT_TRUE(number_annotator_.ClassifyText(
837       UTF8ToUnicodeText("... 12345 percent ..."), {4, 17},
838       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
839   EXPECT_EQ(classification_result.collection, "percentage");
840   EXPECT_EQ(classification_result.numeric_value, 12345);
841   EXPECT_EQ(classification_result.numeric_double_value, 12345);
842   EXPECT_EQ(classification_result.score, 1);
843   EXPECT_EQ(classification_result.priority_score, 1);
844 
845   std::vector<AnnotatedSpan> result;
846   EXPECT_TRUE(number_annotator_.FindAll(
847       UTF8ToUnicodeText("Results are between 9% and 10 percent."),
848       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
849   EXPECT_THAT(result,
850               UnorderedElementsAre(
851                   IsAnnotatedSpan(CodepointSpan(20, 21), "number",
852                                   /*int_value=*/9, /*double_value=*/9),
853                   IsAnnotatedSpan(CodepointSpan(20, 22), "percentage",
854                                   /*int_value=*/9, /*double_value=*/9,
855                                   /*priority_score=*/1),
856                   IsAnnotatedSpan(CodepointSpan(27, 29), "number",
857                                   /*int_value=*/10, /*double_value=*/10),
858                   IsAnnotatedSpan(CodepointSpan(27, 37), "percentage",
859                                   /*int_value=*/10, /*double_value=*/10,
860                                   /*priority_score=*/1)));
861 }
862 
TEST_F(NumberAnnotatorTest,NumberDisabledPercentageEnabledForSmartUsecase)863 TEST_F(NumberAnnotatorTest, NumberDisabledPercentageEnabledForSmartUsecase) {
864   ClassificationResult classification_result;
865   EXPECT_FALSE(number_annotator_.ClassifyText(
866       UTF8ToUnicodeText("... 12345 ..."), {4, 9},
867       AnnotationUsecase_ANNOTATION_USECASE_SMART, &classification_result));
868 
869   EXPECT_TRUE(number_annotator_.ClassifyText(
870       UTF8ToUnicodeText("... 12345% ..."), {4, 10},
871       AnnotationUsecase_ANNOTATION_USECASE_SMART, &classification_result));
872   EXPECT_EQ(classification_result.collection, "percentage");
873   EXPECT_EQ(classification_result.numeric_value, 12345);
874   EXPECT_EQ(classification_result.numeric_double_value, 12345.0);
875   EXPECT_EQ(classification_result.score, 1);
876   EXPECT_EQ(classification_result.priority_score, 1);
877 
878   EXPECT_TRUE(number_annotator_.ClassifyText(
879       UTF8ToUnicodeText("... 12345percent ..."), {4, 16},
880       AnnotationUsecase_ANNOTATION_USECASE_SMART, &classification_result));
881   EXPECT_EQ(classification_result.collection, "percentage");
882   EXPECT_EQ(classification_result.numeric_value, 12345);
883   EXPECT_EQ(classification_result.numeric_double_value, 12345);
884   EXPECT_EQ(classification_result.score, 1);
885   EXPECT_EQ(classification_result.priority_score, 1);
886 
887   std::vector<AnnotatedSpan> result;
888   EXPECT_TRUE(number_annotator_.FindAll(
889       UTF8ToUnicodeText("Accuracy for experiment 3 is 9%."),
890       AnnotationUsecase_ANNOTATION_USECASE_SMART, &result));
891   EXPECT_THAT(result, UnorderedElementsAre(
892                           IsAnnotatedSpan(CodepointSpan(29, 31), "percentage",
893                                           /*int_value=*/9, /*double_value=*/9.0,
894                                           /*priority_score=*/1)));
895 }
896 
TEST_F(NumberAnnotatorTest,MathOperatorsNotAnnotatedAsNumbersFindAll)897 TEST_F(NumberAnnotatorTest, MathOperatorsNotAnnotatedAsNumbersFindAll) {
898   std::vector<AnnotatedSpan> result;
899   EXPECT_TRUE(number_annotator_.FindAll(
900       UTF8ToUnicodeText("how much is 2 + 2 or 5 - 96 * 89"),
901       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
902 
903   EXPECT_THAT(result,
904               UnorderedElementsAre(
905                   IsAnnotatedSpan(CodepointSpan(12, 13), "number",
906                                   /*int_value=*/2, /*double_value=*/2),
907                   IsAnnotatedSpan(CodepointSpan(16, 17), "number",
908                                   /*int_value=*/2, /*double_value=*/2),
909                   IsAnnotatedSpan(CodepointSpan(21, 22), "number",
910                                   /*int_value=*/5, /*double_value=*/5),
911                   IsAnnotatedSpan(CodepointSpan(25, 27), "number",
912                                   /*int_value=*/96, /*double_value=*/96),
913                   IsAnnotatedSpan(CodepointSpan(30, 32), "number",
914                                   /*int_value=*/89, /*double_value=*/89)));
915 }
916 
TEST_F(NumberAnnotatorTest,MathOperatorsNotAnnotatedAsNumbersClassifyText)917 TEST_F(NumberAnnotatorTest, MathOperatorsNotAnnotatedAsNumbersClassifyText) {
918   ClassificationResult classification_result;
919   EXPECT_FALSE(number_annotator_.ClassifyText(
920       UTF8ToUnicodeText("2 + 2"), {2, 3},
921       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
922   EXPECT_FALSE(number_annotator_.ClassifyText(
923       UTF8ToUnicodeText("2 - 96 * 89"), {2, 3},
924       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
925 }
926 
TEST_F(NumberAnnotatorTest,SlashSeparatesTwoNumbersFindAll)927 TEST_F(NumberAnnotatorTest, SlashSeparatesTwoNumbersFindAll) {
928   std::vector<AnnotatedSpan> result;
929   EXPECT_TRUE(number_annotator_.FindAll(
930       UTF8ToUnicodeText("what's 1 + 2/3 * 4／5 * 6 / 7"),
931       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
932 
933   EXPECT_THAT(result,
934               UnorderedElementsAre(
935                   IsAnnotatedSpan(CodepointSpan(7, 8), "number",
936                                   /*int_value=*/1, /*double_value=*/1),
937                   IsAnnotatedSpan(CodepointSpan(11, 12), "number",
938                                   /*int_value=*/2, /*double_value=*/2),
939                   IsAnnotatedSpan(CodepointSpan(13, 14), "number",
940                                   /*int_value=*/3, /*double_value=*/3),
941                   IsAnnotatedSpan(CodepointSpan(17, 18), "number",
942                                   /*int_value=*/4, /*double_value=*/4),
943                   IsAnnotatedSpan(CodepointSpan(19, 20), "number",
944                                   /*int_value=*/5, /*double_value=*/5),
945                   IsAnnotatedSpan(CodepointSpan(23, 24), "number",
946                                   /*int_value=*/6, /*double_value=*/6),
947                   IsAnnotatedSpan(CodepointSpan(27, 28), "number",
948                                   /*int_value=*/7, /*double_value=*/7)));
949 }
950 
TEST_F(NumberAnnotatorTest,SlashSeparatesTwoNumbersClassifyText)951 TEST_F(NumberAnnotatorTest, SlashSeparatesTwoNumbersClassifyText) {
952   ClassificationResult classification_result;
953   EXPECT_TRUE(number_annotator_.ClassifyText(
954       UTF8ToUnicodeText("what's 1 + 2/3 * 4"), {11, 12},
955       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
956   EXPECT_EQ(classification_result.collection, "number");
957   EXPECT_EQ(classification_result.numeric_value, 2);
958   EXPECT_EQ(classification_result.numeric_double_value, 2);
959   EXPECT_EQ(classification_result.score, 1);
960 
961   EXPECT_TRUE(number_annotator_.ClassifyText(
962       UTF8ToUnicodeText("what's 1 + 2/3 * 4"), {13, 14},
963       AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification_result));
964   EXPECT_EQ(classification_result.collection, "number");
965   EXPECT_EQ(classification_result.numeric_value, 3);
966   EXPECT_EQ(classification_result.numeric_double_value, 3);
967   EXPECT_EQ(classification_result.score, 1);
968 }
969 
TEST_F(NumberAnnotatorTest,SlashDoesNotSeparatesTwoNumbersFindAll)970 TEST_F(NumberAnnotatorTest, SlashDoesNotSeparatesTwoNumbersFindAll) {
971   std::vector<AnnotatedSpan> result;
972   // 2 in the "2/" context is a number because / is punctuation
973   EXPECT_TRUE(number_annotator_.FindAll(
974       UTF8ToUnicodeText("what's 2a2/3 or 2/s4 or 2/ or /3 or //3 or 2//"),
975       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
976 
977   EXPECT_THAT(result, UnorderedElementsAre(IsAnnotatedSpan(
978                           CodepointSpan(24, 25), "number",
979                           /*int_value=*/2, /*double_value=*/2)));
980 }
981 
TEST_F(NumberAnnotatorTest,BracketsContextAnnotatedFindAll)982 TEST_F(NumberAnnotatorTest, BracketsContextAnnotatedFindAll) {
983   std::vector<AnnotatedSpan> result;
984   EXPECT_TRUE(number_annotator_.FindAll(
985       UTF8ToUnicodeText("The interval is: (12, 13) or [-12, -4.5)"),
986       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
987 
988   EXPECT_THAT(result,
989               UnorderedElementsAre(
990                   IsAnnotatedSpan(CodepointSpan(18, 20), "number",
991                                   /*int_value=*/12, /*double_value=*/12),
992                   IsAnnotatedSpan(CodepointSpan(22, 24), "number",
993                                   /*int_value=*/13, /*double_value=*/13),
994                   IsAnnotatedSpan(CodepointSpan(30, 33), "number",
995                                   /*int_value=*/-12, /*double_value=*/-12),
996                   IsAnnotatedSpan(CodepointSpan(35, 39), "number",
997                                   /*int_value=*/-4, /*double_value=*/-4.5,
998                                   /*priority_score=*/1)));
999 }
1000 
TEST_F(NumberAnnotatorTest,BracketsContextNotAnnotatedFindAll)1001 TEST_F(NumberAnnotatorTest, BracketsContextNotAnnotatedFindAll) {
1002   std::vector<AnnotatedSpan> result;
1003   EXPECT_TRUE(number_annotator_.FindAll(
1004       UTF8ToUnicodeText("The interval is: -(12, 138*)"),
1005       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
1006 
1007   EXPECT_TRUE(result.empty());
1008 }
1009 
TEST_F(NumberAnnotatorTest,FractionalNumberDotsFindAll)1010 TEST_F(NumberAnnotatorTest, FractionalNumberDotsFindAll) {
1011   std::vector<AnnotatedSpan> result;
1012   // Dots source: https://unicode-search.net/unicode-namesearch.pl?term=period
1013   EXPECT_TRUE(number_annotator_.FindAll(
1014       UTF8ToUnicodeText("3.1 3﹒2 3．3"),
1015       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
1016 
1017   EXPECT_THAT(result, UnorderedElementsAre(
1018                           IsAnnotatedSpan(CodepointSpan(0, 3), "number",
1019                                           /*int_value=*/3, /*double_value=*/3.1,
1020                                           /*priority_score=*/1),
1021                           IsAnnotatedSpan(CodepointSpan(4, 7), "number",
1022                                           /*int_value=*/3, /*double_value=*/3.2,
1023                                           /*priority_score=*/1),
1024                           IsAnnotatedSpan(CodepointSpan(8, 11), "number",
1025                                           /*int_value=*/3, /*double_value=*/3.3,
1026                                           /*priority_score=*/1)));
1027 }
1028 
TEST_F(NumberAnnotatorTest,NonAsciiDigitsFindAll)1029 TEST_F(NumberAnnotatorTest, NonAsciiDigitsFindAll) {
1030   std::vector<AnnotatedSpan> result;
1031   // Dots source: https://unicode-search.net/unicode-namesearch.pl?term=period
1032   // Digits source: https://unicode-search.net/unicode-namesearch.pl?term=digit
1033   EXPECT_TRUE(number_annotator_.FindAll(
1034       UTF8ToUnicodeText("３ ３﹒２ ３．３%"),
1035       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
1036 
1037   EXPECT_THAT(result, UnorderedElementsAre(
1038                           IsAnnotatedSpan(CodepointSpan(0, 1), "number",
1039                                           /*int_value=*/3, /*double_value=*/3),
1040                           IsAnnotatedSpan(CodepointSpan(2, 5), "number",
1041                                           /*int_value=*/3, /*double_value=*/3.2,
1042                                           /*priority_score=*/1),
1043                           IsAnnotatedSpan(CodepointSpan(6, 9), "number",
1044                                           /*int_value=*/3, /*double_value=*/3.3,
1045                                           /*priority_score=*/1),
1046                           IsAnnotatedSpan(CodepointSpan(6, 10), "percentage",
1047                                           /*int_value=*/3, /*double_value=*/3.3,
1048                                           /*priority_score=*/1)));
1049 }
1050 
TEST_F(NumberAnnotatorTest,AnnotatedZeroPrecededNumbersFindAll)1051 TEST_F(NumberAnnotatorTest, AnnotatedZeroPrecededNumbersFindAll) {
1052   std::vector<AnnotatedSpan> result;
1053   EXPECT_TRUE(number_annotator_.FindAll(
1054       UTF8ToUnicodeText("Numbers: 0.9 or 09 or 09.9 or 032310"),
1055       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
1056 
1057   EXPECT_THAT(result, UnorderedElementsAre(
1058                           IsAnnotatedSpan(CodepointSpan(9, 12), "number",
1059                                           /*int_value=*/0, /*double_value=*/0.9,
1060                                           /*priority_score=*/1),
1061                           IsAnnotatedSpan(CodepointSpan(16, 18), "number",
1062                                           /*int_value=*/9, /*double_value=*/9),
1063                           IsAnnotatedSpan(CodepointSpan(22, 26), "number",
1064                                           /*int_value=*/9, /*double_value=*/9.9,
1065                                           /*priority_score=*/1),
1066                           IsAnnotatedSpan(CodepointSpan(30, 36), "number",
1067                                           /*int_value=*/32310,
1068                                           /*double_value=*/32310)));
1069 }
1070 
TEST_F(NumberAnnotatorTest,ZeroAfterDotFindAll)1071 TEST_F(NumberAnnotatorTest, ZeroAfterDotFindAll) {
1072   std::vector<AnnotatedSpan> result;
1073   EXPECT_TRUE(number_annotator_.FindAll(
1074       UTF8ToUnicodeText("15.0 16.00"), AnnotationUsecase_ANNOTATION_USECASE_RAW,
1075       &result));
1076 
1077   EXPECT_THAT(result,
1078               UnorderedElementsAre(
1079                   IsAnnotatedSpan(CodepointSpan(0, 4), "number",
1080                                   /*int_value=*/15, /*double_value=*/15),
1081                   IsAnnotatedSpan(CodepointSpan(5, 10), "number",
1082                                   /*int_value=*/16, /*double_value=*/16)));
1083 }
1084 
TEST_F(NumberAnnotatorTest,NineDotNineFindAll)1085 TEST_F(NumberAnnotatorTest, NineDotNineFindAll) {
1086   std::vector<AnnotatedSpan> result;
1087   EXPECT_TRUE(number_annotator_.FindAll(
1088       UTF8ToUnicodeText("9.9 9.99 99.99 99.999 99.9999"),
1089       AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
1090 
1091   EXPECT_THAT(result,
1092               UnorderedElementsAre(
1093                   IsAnnotatedSpan(CodepointSpan(0, 3), "number",
1094                                   /*int_value=*/9, /*double_value=*/9.9,
1095                                   /*priority_score=*/1),
1096                   IsAnnotatedSpan(CodepointSpan(4, 8), "number",
1097                                   /*int_value=*/9, /*double_value=*/9.99,
1098                                   /*priority_score=*/1),
1099                   IsAnnotatedSpan(CodepointSpan(9, 14), "number",
1100                                   /*int_value=*/99, /*double_value=*/99.99,
1101                                   /*priority_score=*/1),
1102                   IsAnnotatedSpan(CodepointSpan(15, 21), "number",
1103                                   /*int_value=*/99, /*double_value=*/99.999,
1104                                   /*priority_score=*/1),
1105                   IsAnnotatedSpan(CodepointSpan(22, 29), "number",
1106                                   /*int_value=*/99, /*double_value=*/99.9999,
1107                                   /*priority_score=*/1)));
1108 }
1109 
1110 }  // namespace test_internal
1111 }  // namespace libtextclassifier3
1112