Lines Matching refs:extractor
44 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
49 extractor.Extract(Token{"Hello", 0, 5}, true, &sparse_features, in TEST_F()
55 extractor.HashToken("H"), in TEST_F()
56 extractor.HashToken("e"), in TEST_F()
57 extractor.HashToken("l"), in TEST_F()
58 extractor.HashToken("l"), in TEST_F()
59 extractor.HashToken("o"), in TEST_F()
60 extractor.HashToken("^H"), in TEST_F()
61 extractor.HashToken("He"), in TEST_F()
62 extractor.HashToken("el"), in TEST_F()
63 extractor.HashToken("ll"), in TEST_F()
64 extractor.HashToken("lo"), in TEST_F()
65 extractor.HashToken("o$"), in TEST_F()
66 extractor.HashToken("^He"), in TEST_F()
67 extractor.HashToken("Hel"), in TEST_F()
68 extractor.HashToken("ell"), in TEST_F()
69 extractor.HashToken("llo"), in TEST_F()
70 extractor.HashToken("lo$") in TEST_F()
77 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features, in TEST_F()
83 extractor.HashToken("w"), in TEST_F()
84 extractor.HashToken("o"), in TEST_F()
85 extractor.HashToken("r"), in TEST_F()
86 extractor.HashToken("l"), in TEST_F()
87 extractor.HashToken("d"), in TEST_F()
88 extractor.HashToken("!"), in TEST_F()
89 extractor.HashToken("^w"), in TEST_F()
90 extractor.HashToken("wo"), in TEST_F()
91 extractor.HashToken("or"), in TEST_F()
92 extractor.HashToken("rl"), in TEST_F()
93 extractor.HashToken("ld"), in TEST_F()
94 extractor.HashToken("d!"), in TEST_F()
95 extractor.HashToken("!$"), in TEST_F()
96 extractor.HashToken("^wo"), in TEST_F()
97 extractor.HashToken("wor"), in TEST_F()
98 extractor.HashToken("orl"), in TEST_F()
99 extractor.HashToken("rld"), in TEST_F()
100 extractor.HashToken("ld!"), in TEST_F()
101 extractor.HashToken("d!$"), in TEST_F()
114 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
119 extractor.Extract(Token{"Hello", 0, 5}, true, &sparse_features, in TEST_F()
123 testing::ElementsAreArray({extractor.HashToken("^Hello$")})); in TEST_F()
128 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features, in TEST_F()
132 testing::ElementsAreArray({extractor.HashToken("^world!$")})); in TEST_F()
143 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
148 extractor.Extract(Token{"Hělló", 0, 5}, true, &sparse_features, in TEST_F()
154 extractor.HashToken("H"), in TEST_F()
155 extractor.HashToken("ě"), in TEST_F()
156 extractor.HashToken("l"), in TEST_F()
157 extractor.HashToken("l"), in TEST_F()
158 extractor.HashToken("ó"), in TEST_F()
159 extractor.HashToken("^H"), in TEST_F()
160 extractor.HashToken("Hě"), in TEST_F()
161 extractor.HashToken("ěl"), in TEST_F()
162 extractor.HashToken("ll"), in TEST_F()
163 extractor.HashToken("ló"), in TEST_F()
164 extractor.HashToken("ó$"), in TEST_F()
165 extractor.HashToken("^Hě"), in TEST_F()
166 extractor.HashToken("Hěl"), in TEST_F()
167 extractor.HashToken("ěll"), in TEST_F()
168 extractor.HashToken("lló"), in TEST_F()
169 extractor.HashToken("ló$") in TEST_F()
176 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features, in TEST_F()
182 extractor.HashToken("w"), in TEST_F()
183 extractor.HashToken("o"), in TEST_F()
184 extractor.HashToken("r"), in TEST_F()
185 extractor.HashToken("l"), in TEST_F()
186 extractor.HashToken("d"), in TEST_F()
187 extractor.HashToken("!"), in TEST_F()
188 extractor.HashToken("^w"), in TEST_F()
189 extractor.HashToken("wo"), in TEST_F()
190 extractor.HashToken("or"), in TEST_F()
191 extractor.HashToken("rl"), in TEST_F()
192 extractor.HashToken("ld"), in TEST_F()
193 extractor.HashToken("d!"), in TEST_F()
194 extractor.HashToken("!$"), in TEST_F()
195 extractor.HashToken("^wo"), in TEST_F()
196 extractor.HashToken("wor"), in TEST_F()
197 extractor.HashToken("orl"), in TEST_F()
198 extractor.HashToken("rld"), in TEST_F()
199 extractor.HashToken("ld!"), in TEST_F()
200 extractor.HashToken("d!$"), in TEST_F()
213 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
218 extractor.Extract(Token{"Hělló", 0, 5}, true, &sparse_features, in TEST_F()
222 testing::ElementsAreArray({extractor.HashToken("^Hělló$")})); in TEST_F()
227 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features, in TEST_F()
231 extractor.HashToken("^world!$"), in TEST_F()
244 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
248 extractor.Extract(Token{"Hělló", 0, 5}, true, &sparse_features, in TEST_F()
254 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features, in TEST_F()
260 extractor.Extract(Token{"Ř", 23, 29}, false, &sparse_features, in TEST_F()
266 extractor.Extract(Token{"ř", 23, 29}, false, &sparse_features, in TEST_F()
278 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
282 extractor.Extract(Token{"9:30am", 0, 6}, true, &sparse_features, in TEST_F()
286 extractor.Extract(Token{"5:32am", 0, 6}, true, &sparse_features2, in TEST_F()
290 extractor.Extract(Token{"10:32am", 0, 6}, true, &sparse_features2, in TEST_F()
302 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
306 extractor.Extract(Token{"9:30am", 0, 6}, true, &sparse_features, in TEST_F()
310 extractor.Extract(Token{"5:32am", 0, 6}, true, &sparse_features2, in TEST_F()
314 extractor.Extract(Token{"10:32am", 0, 6}, true, &sparse_features2, in TEST_F()
326 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
330 extractor.Extract(Token{"AABB", 0, 6}, true, &sparse_features, in TEST_F()
334 extractor.Extract(Token{"aaBB", 0, 6}, true, &sparse_features2, in TEST_F()
338 extractor.Extract(Token{"aAbB", 0, 6}, true, &sparse_features2, in TEST_F()
350 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
354 extractor.Extract(Token{"ŘŘ", 0, 6}, true, &sparse_features, &dense_features); in TEST_F()
357 extractor.Extract(Token{"řř", 0, 6}, true, &sparse_features2, in TEST_F()
372 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
376 extractor.Extract(Token{"abCde", 0, 6}, true, &sparse_features, in TEST_F()
381 extractor.Extract(Token{"abcde", 0, 6}, true, &sparse_features, in TEST_F()
386 extractor.Extract(Token{"12c45", 0, 6}, true, &sparse_features, in TEST_F()
391 extractor.Extract(Token{"12345", 0, 6}, true, &sparse_features, in TEST_F()
404 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
409 extractor.Extract(Token{"abcdefghijklmnopqřstuvwxyz", 0, 0}, true, in TEST_F()
415 extractor.HashToken("^abcdefghij\1qřstuvwxyz"), in TEST_F()
416 extractor.HashToken("abcdefghij\1qřstuvwxyz$"), in TEST_F()
463 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
468 extractor.Extract(Token(), false, &sparse_features, &dense_features); in TEST_F()
471 testing::ElementsAreArray({extractor.HashToken("<PAD>")})); in TEST_F()
489 TestingTokenFeatureExtractor extractor(options, unilib_); in TEST_F() local
494 extractor.Extract(Token{"Hěllo", 0, 5}, true, &sparse_features, in TEST_F()
501 extractor.HashToken("\xc4"), in TEST_F()
506 extractor.HashToken("^H"), in TEST_F()
510 extractor.HashToken("ll"), in TEST_F()
517 extractor.HashToken("llo"), in TEST_F()
525 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features, in TEST_F()
530 extractor.HashToken("w"), in TEST_F()
535 extractor.HashToken("!"), in TEST_F()
552 EXPECT_EQ(extractor.HashToken("<PAD>"), 1); in TEST_F()