1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/str_util.h"
17
18 #include <vector>
19
20 #include "tensorflow/core/platform/test.h"
21
22 namespace tensorflow {
23
TEST(CEscape,Basic)24 TEST(CEscape, Basic) {
25 EXPECT_EQ(str_util::CEscape("hello"), "hello");
26 EXPECT_EQ(str_util::CEscape("hello\n"), "hello\\n");
27 EXPECT_EQ(str_util::CEscape("hello\r"), "hello\\r");
28 EXPECT_EQ(str_util::CEscape("\t\r\"'"), "\\t\\r\\\"\\'");
29 EXPECT_EQ(str_util::CEscape("\320hi\200"), "\\320hi\\200");
30 }
31
ExpectCUnescapeSuccess(StringPiece source)32 string ExpectCUnescapeSuccess(StringPiece source) {
33 string dest;
34 string error;
35 EXPECT_TRUE(str_util::CUnescape(source, &dest, &error)) << error;
36 return dest;
37 }
38
TEST(CUnescape,Basic)39 TEST(CUnescape, Basic) {
40 EXPECT_EQ("hello", ExpectCUnescapeSuccess("hello"));
41 EXPECT_EQ("hello\n", ExpectCUnescapeSuccess("hello\\n"));
42 EXPECT_EQ("hello\r", ExpectCUnescapeSuccess("hello\\r"));
43 EXPECT_EQ("\t\r\"'", ExpectCUnescapeSuccess("\\t\\r\\\"\\'"));
44 EXPECT_EQ("\320hi\200", ExpectCUnescapeSuccess("\\320hi\\200"));
45 }
46
TEST(CUnescape,HandlesCopyOnWriteStrings)47 TEST(CUnescape, HandlesCopyOnWriteStrings) {
48 string dest = "hello";
49 string read = dest;
50 // For std::string, read and dest now share the same buffer.
51
52 string error;
53 StringPiece source = "llohe";
54 // CUnescape is going to write "llohe" to dest, so dest's buffer will be
55 // reallocated, and read's buffer remains untouched.
56 EXPECT_TRUE(str_util::CUnescape(source, &dest, &error));
57 EXPECT_EQ("hello", read);
58 }
59
TEST(StripTrailingWhitespace,Basic)60 TEST(StripTrailingWhitespace, Basic) {
61 string test;
62 test = "hello";
63 str_util::StripTrailingWhitespace(&test);
64 EXPECT_EQ(test, "hello");
65
66 test = "foo ";
67 str_util::StripTrailingWhitespace(&test);
68 EXPECT_EQ(test, "foo");
69
70 test = " ";
71 str_util::StripTrailingWhitespace(&test);
72 EXPECT_EQ(test, "");
73
74 test = "";
75 str_util::StripTrailingWhitespace(&test);
76 EXPECT_EQ(test, "");
77
78 test = " abc\t";
79 str_util::StripTrailingWhitespace(&test);
80 EXPECT_EQ(test, " abc");
81 }
82
TEST(RemoveLeadingWhitespace,Basic)83 TEST(RemoveLeadingWhitespace, Basic) {
84 string text = " \t \n \r Quick\t";
85 StringPiece data(text);
86 // check that all whitespace is removed
87 EXPECT_EQ(str_util::RemoveLeadingWhitespace(&data), 11);
88 EXPECT_EQ(data, StringPiece("Quick\t"));
89 // check that non-whitespace is not removed
90 EXPECT_EQ(str_util::RemoveLeadingWhitespace(&data), 0);
91 EXPECT_EQ(data, StringPiece("Quick\t"));
92 }
93
TEST(RemoveLeadingWhitespace,TerminationHandling)94 TEST(RemoveLeadingWhitespace, TerminationHandling) {
95 // check termination handling
96 string text = "\t";
97 StringPiece data(text);
98 EXPECT_EQ(str_util::RemoveLeadingWhitespace(&data), 1);
99 EXPECT_EQ(data, StringPiece(""));
100
101 // check termination handling again
102 EXPECT_EQ(str_util::RemoveLeadingWhitespace(&data), 0);
103 EXPECT_EQ(data, StringPiece(""));
104 }
105
TEST(RemoveTrailingWhitespace,Basic)106 TEST(RemoveTrailingWhitespace, Basic) {
107 string text = " \t \n \r Quick \t";
108 StringPiece data(text);
109 // check that all whitespace is removed
110 EXPECT_EQ(str_util::RemoveTrailingWhitespace(&data), 2);
111 EXPECT_EQ(data, StringPiece(" \t \n \r Quick"));
112 // check that non-whitespace is not removed
113 EXPECT_EQ(str_util::RemoveTrailingWhitespace(&data), 0);
114 EXPECT_EQ(data, StringPiece(" \t \n \r Quick"));
115 }
116
TEST(RemoveTrailingWhitespace,TerminationHandling)117 TEST(RemoveTrailingWhitespace, TerminationHandling) {
118 // check termination handling
119 string text = "\t";
120 StringPiece data(text);
121 EXPECT_EQ(str_util::RemoveTrailingWhitespace(&data), 1);
122 EXPECT_EQ(data, StringPiece(""));
123
124 // check termination handling again
125 EXPECT_EQ(str_util::RemoveTrailingWhitespace(&data), 0);
126 EXPECT_EQ(data, StringPiece(""));
127 }
128
TEST(RemoveWhitespaceContext,Basic)129 TEST(RemoveWhitespaceContext, Basic) {
130 string text = " \t \n \r Quick \t";
131 StringPiece data(text);
132 // check that all whitespace is removed
133 EXPECT_EQ(str_util::RemoveWhitespaceContext(&data), 13);
134 EXPECT_EQ(data, StringPiece("Quick"));
135 // check that non-whitespace is not removed
136 EXPECT_EQ(str_util::RemoveWhitespaceContext(&data), 0);
137 EXPECT_EQ(data, StringPiece("Quick"));
138
139 // Test empty string
140 text = "";
141 data = text;
142 EXPECT_EQ(str_util::RemoveWhitespaceContext(&data), 0);
143 EXPECT_EQ(data, StringPiece(""));
144 }
145
TestConsumeLeadingDigits(StringPiece s,int64 expected,StringPiece remaining)146 void TestConsumeLeadingDigits(StringPiece s, int64 expected,
147 StringPiece remaining) {
148 uint64 v;
149 StringPiece input(s);
150 if (str_util::ConsumeLeadingDigits(&input, &v)) {
151 EXPECT_EQ(v, static_cast<uint64>(expected));
152 EXPECT_EQ(input, remaining);
153 } else {
154 EXPECT_LT(expected, 0);
155 EXPECT_EQ(input, remaining);
156 }
157 }
158
TEST(ConsumeLeadingDigits,Basic)159 TEST(ConsumeLeadingDigits, Basic) {
160 using str_util::ConsumeLeadingDigits;
161
162 TestConsumeLeadingDigits("123", 123, "");
163 TestConsumeLeadingDigits("a123", -1, "a123");
164 TestConsumeLeadingDigits("9_", 9, "_");
165 TestConsumeLeadingDigits("11111111111xyz", 11111111111ll, "xyz");
166
167 // Overflow case
168 TestConsumeLeadingDigits("1111111111111111111111111111111xyz", -1,
169 "1111111111111111111111111111111xyz");
170
171 // 2^64
172 TestConsumeLeadingDigits("18446744073709551616xyz", -1,
173 "18446744073709551616xyz");
174 // 2^64-1
175 TestConsumeLeadingDigits("18446744073709551615xyz", 18446744073709551615ull,
176 "xyz");
177 // (2^64-1)*10+9
178 TestConsumeLeadingDigits("184467440737095516159yz", -1,
179 "184467440737095516159yz");
180 }
181
TestConsumeNonWhitespace(StringPiece s,StringPiece expected,StringPiece remaining)182 void TestConsumeNonWhitespace(StringPiece s, StringPiece expected,
183 StringPiece remaining) {
184 StringPiece v;
185 StringPiece input(s);
186 if (str_util::ConsumeNonWhitespace(&input, &v)) {
187 EXPECT_EQ(v, expected);
188 EXPECT_EQ(input, remaining);
189 } else {
190 EXPECT_EQ(expected, "");
191 EXPECT_EQ(input, remaining);
192 }
193 }
194
TEST(ConsumeNonWhitespace,Basic)195 TEST(ConsumeNonWhitespace, Basic) {
196 TestConsumeNonWhitespace("", "", "");
197 TestConsumeNonWhitespace(" ", "", " ");
198 TestConsumeNonWhitespace("abc", "abc", "");
199 TestConsumeNonWhitespace("abc ", "abc", " ");
200 }
201
TEST(ConsumePrefix,Basic)202 TEST(ConsumePrefix, Basic) {
203 string s("abcdef");
204 StringPiece input(s);
205 EXPECT_FALSE(str_util::ConsumePrefix(&input, "abcdefg"));
206 EXPECT_EQ(input, "abcdef");
207
208 EXPECT_FALSE(str_util::ConsumePrefix(&input, "abce"));
209 EXPECT_EQ(input, "abcdef");
210
211 EXPECT_TRUE(str_util::ConsumePrefix(&input, ""));
212 EXPECT_EQ(input, "abcdef");
213
214 EXPECT_FALSE(str_util::ConsumePrefix(&input, "abcdeg"));
215 EXPECT_EQ(input, "abcdef");
216
217 EXPECT_TRUE(str_util::ConsumePrefix(&input, "abcdef"));
218 EXPECT_EQ(input, "");
219
220 input = s;
221 EXPECT_TRUE(str_util::ConsumePrefix(&input, "abcde"));
222 EXPECT_EQ(input, "f");
223 }
224
TEST(StripPrefix,Basic)225 TEST(StripPrefix, Basic) {
226 EXPECT_EQ(str_util::StripPrefix("abcdef", "abcdefg"), "abcdef");
227 EXPECT_EQ(str_util::StripPrefix("abcdef", "abce"), "abcdef");
228 EXPECT_EQ(str_util::StripPrefix("abcdef", ""), "abcdef");
229 EXPECT_EQ(str_util::StripPrefix("abcdef", "abcdeg"), "abcdef");
230 EXPECT_EQ(str_util::StripPrefix("abcdef", "abcdef"), "");
231 EXPECT_EQ(str_util::StripPrefix("abcdef", "abcde"), "f");
232 }
233
TEST(JoinStrings,Basic)234 TEST(JoinStrings, Basic) {
235 std::vector<string> s;
236 s = {"hi"};
237 EXPECT_EQ(str_util::Join(s, " "), "hi");
238 s = {"hi", "there", "strings"};
239 EXPECT_EQ(str_util::Join(s, " "), "hi there strings");
240
241 std::vector<StringPiece> sp;
242 sp = {"hi"};
243 EXPECT_EQ(str_util::Join(sp, ",,"), "hi");
244 sp = {"hi", "there", "strings"};
245 EXPECT_EQ(str_util::Join(sp, "--"), "hi--there--strings");
246 }
247
TEST(JoinStrings,Join3)248 TEST(JoinStrings, Join3) {
249 std::vector<string> s;
250 s = {"hi"};
251 auto l1 = [](string* out, string s) { *out += s; };
252 EXPECT_EQ(str_util::Join(s, " ", l1), "hi");
253 s = {"hi", "there", "strings"};
254 auto l2 = [](string* out, string s) { *out += s[0]; };
255 EXPECT_EQ(str_util::Join(s, " ", l2), "h t s");
256 }
257
TEST(Split,Basic)258 TEST(Split, Basic) {
259 EXPECT_TRUE(str_util::Split("", ',').empty());
260 EXPECT_EQ(str_util::Join(str_util::Split("a", ','), "|"), "a");
261 EXPECT_EQ(str_util::Join(str_util::Split(",", ','), "|"), "|");
262 EXPECT_EQ(str_util::Join(str_util::Split("a,b,c", ','), "|"), "a|b|c");
263 EXPECT_EQ(str_util::Join(str_util::Split("a,,,b,,c,", ','), "|"),
264 "a|||b||c|");
265 EXPECT_EQ(str_util::Join(str_util::Split("a!,!b,!c,", ",!"), "|"),
266 "a|||b||c|");
267 EXPECT_EQ(str_util::Join(
268 str_util::Split("a,,,b,,c,", ',', str_util::SkipEmpty()), "|"),
269 "a|b|c");
270 EXPECT_EQ(
271 str_util::Join(
272 str_util::Split("a, ,b,,c,", ',', str_util::SkipWhitespace()), "|"),
273 "a|b|c");
274 EXPECT_EQ(str_util::Join(str_util::Split("a. !b,;c,", ".,;!",
275 str_util::SkipWhitespace()),
276 "|"),
277 "a|b|c");
278 }
279
TEST(Lowercase,Basic)280 TEST(Lowercase, Basic) {
281 EXPECT_EQ("", str_util::Lowercase(""));
282 EXPECT_EQ("hello", str_util::Lowercase("hello"));
283 EXPECT_EQ("hello world", str_util::Lowercase("Hello World"));
284 }
285
TEST(Uppercase,Basic)286 TEST(Uppercase, Basic) {
287 EXPECT_EQ("", str_util::Uppercase(""));
288 EXPECT_EQ("HELLO", str_util::Uppercase("hello"));
289 EXPECT_EQ("HELLO WORLD", str_util::Uppercase("Hello World"));
290 }
291
TEST(SnakeCase,Basic)292 TEST(SnakeCase, Basic) {
293 EXPECT_EQ("", str_util::ArgDefCase(""));
294 EXPECT_EQ("", str_util::ArgDefCase("!"));
295 EXPECT_EQ("", str_util::ArgDefCase("5"));
296 EXPECT_EQ("", str_util::ArgDefCase("!:"));
297 EXPECT_EQ("", str_util::ArgDefCase("5-5"));
298 EXPECT_EQ("", str_util::ArgDefCase("_!"));
299 EXPECT_EQ("", str_util::ArgDefCase("_5"));
300 EXPECT_EQ("a", str_util::ArgDefCase("_a"));
301 EXPECT_EQ("a", str_util::ArgDefCase("_A"));
302 EXPECT_EQ("i", str_util::ArgDefCase("I"));
303 EXPECT_EQ("i", str_util::ArgDefCase("i"));
304 EXPECT_EQ("i_", str_util::ArgDefCase("I%"));
305 EXPECT_EQ("i_", str_util::ArgDefCase("i%"));
306 EXPECT_EQ("i", str_util::ArgDefCase("%I"));
307 EXPECT_EQ("i", str_util::ArgDefCase("-i"));
308 EXPECT_EQ("i", str_util::ArgDefCase("3i"));
309 EXPECT_EQ("i", str_util::ArgDefCase("32i"));
310 EXPECT_EQ("i3", str_util::ArgDefCase("i3"));
311 EXPECT_EQ("i_a3", str_util::ArgDefCase("i_A3"));
312 EXPECT_EQ("i_i", str_util::ArgDefCase("II"));
313 EXPECT_EQ("i_i", str_util::ArgDefCase("I_I"));
314 EXPECT_EQ("i__i", str_util::ArgDefCase("I__I"));
315 EXPECT_EQ("i_i_32", str_util::ArgDefCase("II-32"));
316 EXPECT_EQ("ii_32", str_util::ArgDefCase("Ii-32"));
317 EXPECT_EQ("hi_there", str_util::ArgDefCase("HiThere"));
318 EXPECT_EQ("hi_hi", str_util::ArgDefCase("Hi!Hi"));
319 EXPECT_EQ("hi_hi", str_util::ArgDefCase("HiHi"));
320 EXPECT_EQ("hihi", str_util::ArgDefCase("Hihi"));
321 EXPECT_EQ("hi_hi", str_util::ArgDefCase("Hi_Hi"));
322 }
323
TEST(TitlecaseString,Basic)324 TEST(TitlecaseString, Basic) {
325 string s = "sparse_lookup";
326 str_util::TitlecaseString(&s, "_");
327 ASSERT_EQ(s, "Sparse_Lookup");
328
329 s = "sparse_lookup";
330 str_util::TitlecaseString(&s, " ");
331 ASSERT_EQ(s, "Sparse_lookup");
332
333 s = "dense";
334 str_util::TitlecaseString(&s, " ");
335 ASSERT_EQ(s, "Dense");
336 }
337
TEST(StringReplace,Basic)338 TEST(StringReplace, Basic) {
339 EXPECT_EQ("XYZ_XYZ_XYZ", str_util::StringReplace("ABC_ABC_ABC", "ABC", "XYZ",
340 /*replace_all=*/true));
341 }
342
TEST(StringReplace,OnlyFirst)343 TEST(StringReplace, OnlyFirst) {
344 EXPECT_EQ("XYZ_ABC_ABC", str_util::StringReplace("ABC_ABC_ABC", "ABC", "XYZ",
345 /*replace_all=*/false));
346 }
347
TEST(StringReplace,IncreaseLength)348 TEST(StringReplace, IncreaseLength) {
349 EXPECT_EQ("a b c",
350 str_util::StringReplace("abc", "b", " b ", /*replace_all=*/true));
351 }
352
TEST(StringReplace,IncreaseLengthMultipleMatches)353 TEST(StringReplace, IncreaseLengthMultipleMatches) {
354 EXPECT_EQ("a b b c",
355 str_util::StringReplace("abbc", "b", " b ", /*replace_all=*/true));
356 }
357
TEST(StringReplace,NoChange)358 TEST(StringReplace, NoChange) {
359 EXPECT_EQ("abc",
360 str_util::StringReplace("abc", "d", "X", /*replace_all=*/true));
361 }
362
TEST(StringReplace,EmptyStringReplaceFirst)363 TEST(StringReplace, EmptyStringReplaceFirst) {
364 EXPECT_EQ("", str_util::StringReplace("", "a", "X", /*replace_all=*/false));
365 }
366
TEST(StringReplace,EmptyStringReplaceAll)367 TEST(StringReplace, EmptyStringReplaceAll) {
368 EXPECT_EQ("", str_util::StringReplace("", "a", "X", /*replace_all=*/true));
369 }
370
TEST(Strnlen,Basic)371 TEST(Strnlen, Basic) {
372 EXPECT_EQ(0, str_util::Strnlen("ab", 0));
373 EXPECT_EQ(1, str_util::Strnlen("a", 1));
374 EXPECT_EQ(2, str_util::Strnlen("abcd", 2));
375 EXPECT_EQ(3, str_util::Strnlen("abc", 10));
376 EXPECT_EQ(4, str_util::Strnlen("a \t\n", 10));
377 }
378
379 } // namespace tensorflow
380