1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <gtest/gtest.h>
18 
19 #include <iconv.h>
20 
21 #define INVALID_ICONV_T reinterpret_cast<iconv_t>(-1)
22 
TEST(iconv,iconv_open_EINVAL)23 TEST(iconv, iconv_open_EINVAL) {
24   errno = 0;
25   ASSERT_EQ(INVALID_ICONV_T, iconv_open("silly", "silly"));
26   ASSERT_EQ(EINVAL, errno);
27   errno = 0;
28   ASSERT_EQ(INVALID_ICONV_T, iconv_open("silly", "UTF-8"));
29   ASSERT_EQ(EINVAL, errno);
30   errno = 0;
31   ASSERT_EQ(INVALID_ICONV_T, iconv_open("UTF-8", "silly"));
32   ASSERT_EQ(EINVAL, errno);
33 }
34 
TEST(iconv,iconv_open_comparator)35 TEST(iconv, iconv_open_comparator) {
36   // Examples from http://www.unicode.org/reports/tr22/#Charset_Alias_Matching:
37   // "For example, the following names should match: "UTF-8", "utf8", "u.t.f-008", ..."
38   iconv_t c;
39   ASSERT_NE(INVALID_ICONV_T, c = iconv_open("UTF-8", "utf8"));
40   ASSERT_EQ(0, iconv_close(c));
41   ASSERT_NE(INVALID_ICONV_T, c = iconv_open("UTF-8", "u.t.f-008"));
42   ASSERT_EQ(0, iconv_close(c));
43 
44   // "...but not "utf-80" or "ut8"."
45   errno = 0;
46   ASSERT_EQ(INVALID_ICONV_T, iconv_open("UTF-8", "utf-80"));
47   ASSERT_EQ(EINVAL, errno);
48   errno = 0;
49   ASSERT_EQ(INVALID_ICONV_T, iconv_open("UTF-8", "ut80"));
50   ASSERT_EQ(EINVAL, errno);
51 }
52 
TEST(iconv,iconv_smoke)53 TEST(iconv, iconv_smoke) {
54   const char* utf8 = "a٦ᄀ"; // U+0666 ٦ 0xd9 0xa6 // U+1100 ᄀ 0xe1 0x84 0x80
55   char buf[BUFSIZ] = {};
56 
57   iconv_t c = iconv_open("UTF-32LE", "UTF-8");
58   ASSERT_NE(INVALID_ICONV_T, c);
59 
60   char* in = const_cast<char*>(utf8);
61   size_t in_bytes = strlen(in);
62 
63   char* out = buf;
64   size_t out_bytes = sizeof(buf);
65 
66   EXPECT_EQ(0U, iconv(c, &in, &in_bytes, &out, &out_bytes));
67 
68   wchar_t* utf16 = reinterpret_cast<wchar_t*>(buf);
69   EXPECT_EQ(L'a', utf16[0]);
70   EXPECT_EQ(L'٦', utf16[1]);
71   EXPECT_EQ(L'ᄀ', utf16[2]);
72   EXPECT_EQ(L'\0', utf16[3]);
73   EXPECT_EQ(0U, in_bytes);
74   EXPECT_EQ(sizeof(buf) - (3 /* chars */ * 4 /* bytes each */), out_bytes);
75 
76   ASSERT_EQ(0, iconv_close(c));
77 }
78 
TEST(iconv,iconv_lossy_TRANSLIT)79 TEST(iconv, iconv_lossy_TRANSLIT) {
80   const char* utf8 = "a٦ᄀz"; // U+0666 ٦ 0xd9 0xa6 // U+1100 ᄀ 0xe1 0x84 0x80
81   char buf[BUFSIZ] = {};
82 
83   iconv_t c = iconv_open("ASCII//TRANSLIT", "UTF-8");
84   ASSERT_NE(INVALID_ICONV_T, c);
85 
86   char* in = const_cast<char*>(utf8);
87   size_t in_bytes = strlen(in);
88 
89   char* out = buf;
90   size_t out_bytes = sizeof(buf);
91 
92   // Two of the input characters (5 input bytes) aren't representable as ASCII.
93   // With "//TRANSLIT", we use a replacement character, and report the number
94   // of replacements.
95   EXPECT_EQ(2U, iconv(c, &in, &in_bytes, &out, &out_bytes));
96 
97   EXPECT_EQ('a', buf[0]);
98   EXPECT_EQ('?', buf[1]);
99   EXPECT_EQ('?', buf[2]);
100   EXPECT_EQ('z', buf[3]);
101   EXPECT_EQ(0, buf[4]);
102   EXPECT_EQ(0U, in_bytes);
103   EXPECT_EQ(sizeof(buf) - 4, out_bytes);
104 
105   ASSERT_EQ(0, iconv_close(c));
106 }
107 
TEST(iconv,iconv_lossy_IGNORE)108 TEST(iconv, iconv_lossy_IGNORE) {
109   const char* utf8 = "a٦ᄀz"; // U+0666 ٦ 0xd9 0xa6 // U+1100 ᄀ 0xe1 0x84 0x80
110   char buf[BUFSIZ] = {};
111 
112   iconv_t c = iconv_open("ASCII//IGNORE", "UTF-8");
113   ASSERT_NE(INVALID_ICONV_T, c);
114 
115   char* in = const_cast<char*>(utf8);
116   size_t in_bytes = strlen(in);
117 
118   char* out = buf;
119   size_t out_bytes = sizeof(buf);
120 
121   // Two of the input characters (5 input bytes) aren't representable as ASCII.
122   // With "//IGNORE", we just skip them (but return failure).
123   errno = 0;
124   EXPECT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
125   EXPECT_EQ(EILSEQ, errno);
126 
127   EXPECT_EQ('a', buf[0]);
128   EXPECT_EQ('z', buf[1]);
129   EXPECT_EQ(0, buf[2]);
130   EXPECT_EQ(0U, in_bytes);
131   EXPECT_EQ(sizeof(buf) - 2, out_bytes);
132 
133   ASSERT_EQ(0, iconv_close(c));
134 }
135 
TEST(iconv,iconv_lossy)136 TEST(iconv, iconv_lossy) {
137   const char* utf8 = "a٦ᄀz"; // U+0666 ٦ 0xd9 0xa6 // U+1100 ᄀ 0xe1 0x84 0x80
138   char buf[BUFSIZ] = {};
139 
140   iconv_t c = iconv_open("ASCII", "UTF-8");
141   ASSERT_NE(INVALID_ICONV_T, c);
142 
143   char* in = const_cast<char*>(utf8);
144   size_t in_bytes = strlen(in);
145 
146   char* out = buf;
147   size_t out_bytes = sizeof(buf);
148 
149   // The second input character isn't representable as ASCII, so we stop there.
150   errno = 0;
151   EXPECT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
152   EXPECT_EQ(EILSEQ, errno);
153 
154   EXPECT_EQ('a', buf[0]);
155   EXPECT_EQ(0, buf[1]);
156   EXPECT_EQ(6U, in_bytes); // Two bytes for ٦, three bytes for ᄀ, and one byte for z.
157   EXPECT_EQ(sizeof(buf) - 1, out_bytes);
158 
159   ASSERT_EQ(0, iconv_close(c));
160 }
161 
TEST(iconv,iconv_malformed_sequence_EILSEQ)162 TEST(iconv, iconv_malformed_sequence_EILSEQ) {
163   const char* utf8 = "a\xd9z"; // 0xd9 is the first byte of the two-byte U+0666 ٦.
164   char buf[BUFSIZ] = {};
165 
166   iconv_t c = iconv_open("UTF-8", "UTF-8");
167   ASSERT_NE(INVALID_ICONV_T, c);
168 
169   char* in = const_cast<char*>(utf8);
170   size_t in_bytes = strlen(in);
171 
172   char* out = buf;
173   size_t out_bytes = sizeof(buf);
174 
175   // The second input byte is a malformed character, so we stop there.
176   errno = 0;
177   EXPECT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
178   EXPECT_EQ(EILSEQ, errno);
179   EXPECT_EQ('\xd9', *in); // *in is left pointing to the start of the invalid sequence.
180   ++in;
181   --in_bytes;
182   errno = 0;
183   EXPECT_EQ(0U, iconv(c, &in, &in_bytes, &out, &out_bytes));
184   EXPECT_EQ(0, errno);
185 
186   EXPECT_EQ('a', buf[0]);
187   EXPECT_EQ('z', buf[1]);
188   EXPECT_EQ(0, buf[2]);
189   EXPECT_EQ(0U, in_bytes);
190   EXPECT_EQ(sizeof(buf) - 2, out_bytes);
191 
192   ASSERT_EQ(0, iconv_close(c));
193 }
194 
TEST(iconv,iconv_incomplete_sequence_EINVAL)195 TEST(iconv, iconv_incomplete_sequence_EINVAL) {
196   const char* utf8 = "a\xd9"; // 0xd9 is the first byte of the two-byte U+0666 ٦.
197   char buf[BUFSIZ] = {};
198 
199   iconv_t c = iconv_open("UTF-8", "UTF-8");
200   ASSERT_NE(INVALID_ICONV_T, c);
201 
202   char* in = const_cast<char*>(utf8);
203   size_t in_bytes = strlen(in);
204 
205   char* out = buf;
206   size_t out_bytes = sizeof(buf);
207 
208   // The second input byte is just the start of a character, and we don't have any more bytes.
209   errno = 0;
210   EXPECT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
211   EXPECT_EQ(EINVAL, errno);
212   EXPECT_EQ('\xd9', *in); // *in is left pointing to the start of the incomplete sequence.
213 
214   EXPECT_EQ('a', buf[0]);
215   EXPECT_EQ(0, buf[1]);
216   EXPECT_EQ(1U, in_bytes);
217   EXPECT_EQ(sizeof(buf) - 1, out_bytes);
218 
219   ASSERT_EQ(0, iconv_close(c));
220 }
221 
TEST(iconv,iconv_E2BIG)222 TEST(iconv, iconv_E2BIG) {
223   const char* utf8 = "abc";
224   char buf[BUFSIZ] = {};
225 
226   iconv_t c = iconv_open("UTF-8", "UTF-8");
227   ASSERT_NE(INVALID_ICONV_T, c);
228 
229   char* in = const_cast<char*>(utf8);
230   size_t in_bytes = strlen(in);
231 
232   char* out = buf;
233   size_t out_bytes = 1;
234 
235   // We need three bytes, so one isn't enough (but we will make progress).
236   out_bytes = 1;
237   errno = 0;
238   EXPECT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
239   EXPECT_EQ(E2BIG, errno);
240   EXPECT_EQ(2U, in_bytes);
241   EXPECT_EQ(0U, out_bytes);
242 
243   // Two bytes left, so zero isn't enough (and we can't even make progress).
244   out_bytes = 0;
245   errno = 0;
246   EXPECT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
247   EXPECT_EQ(E2BIG, errno);
248   EXPECT_EQ(2U, in_bytes);
249   EXPECT_EQ(0U, out_bytes);
250 
251   // Two bytes left, so one isn't enough (but we will make progress).
252   out_bytes = 1;
253   errno = 0;
254   EXPECT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
255   EXPECT_EQ(E2BIG, errno);
256   EXPECT_EQ(1U, in_bytes);
257   EXPECT_EQ(0U, out_bytes);
258 
259   // One byte left, so one byte is now enough.
260   out_bytes = 1;
261   errno = 0;
262   EXPECT_EQ(0U, iconv(c, &in, &in_bytes, &out, &out_bytes));
263   EXPECT_EQ(0, errno);
264   EXPECT_EQ(0U, in_bytes);
265   EXPECT_EQ(0U, out_bytes);
266 
267   EXPECT_EQ('a', buf[0]);
268   EXPECT_EQ('b', buf[1]);
269   EXPECT_EQ('c', buf[2]);
270   EXPECT_EQ(0, buf[3]);
271 
272   ASSERT_EQ(0, iconv_close(c));
273 }
274 
TEST(iconv,iconv_invalid_converter_EBADF)275 TEST(iconv, iconv_invalid_converter_EBADF) {
276   char* in = nullptr;
277   char* out = nullptr;
278   size_t in_bytes = 0;
279   size_t out_bytes = 0;
280   errno = 0;
281   ASSERT_EQ(static_cast<size_t>(-1), iconv(INVALID_ICONV_T, &in, &in_bytes, &out, &out_bytes));
282   ASSERT_EQ(EBADF, errno);
283 }
284 
TEST(iconv,iconv_close_invalid_converter_EBADF)285 TEST(iconv, iconv_close_invalid_converter_EBADF) {
286   errno = 0;
287   ASSERT_EQ(-1, iconv_close(INVALID_ICONV_T));
288   ASSERT_EQ(EBADF, errno);
289 }
290 
RoundTrip(const char * dst_enc,const char * expected_bytes,size_t n)291 static void RoundTrip(const char* dst_enc, const char* expected_bytes, size_t n) {
292   // Examples from https://en.wikipedia.org/wiki/UTF-16.
293   const char* utf8 = "$€��"; // U+0024, U+20AC, U+10437.
294 
295   iconv_t c = iconv_open(dst_enc, "UTF-8");
296   ASSERT_NE(INVALID_ICONV_T, c) << dst_enc;
297 
298   char* in = const_cast<char*>(utf8);
299   size_t in_bytes = strlen(utf8);
300   char buf[BUFSIZ] = {};
301   char* out = buf;
302   size_t out_bytes = sizeof(buf);
303   size_t replacement_count = iconv(c, &in, &in_bytes, &out, &out_bytes);
304 
305   // Check we got the bytes we were expecting.
306   for (size_t i = 0; i < n; ++i) {
307     EXPECT_EQ(expected_bytes[i], buf[i]) << i << ' '<< dst_enc;
308   }
309 
310   ASSERT_EQ(0, iconv_close(c));
311 
312   // We can't round-trip if there were replacements.
313   if (strstr(dst_enc, "ascii")) {
314     GTEST_LOG_(INFO) << "can't round-trip " << dst_enc << "\n";
315     return;
316   }
317   ASSERT_EQ(0U, replacement_count);
318 
319   c = iconv_open("UTF-8", dst_enc);
320   ASSERT_NE(INVALID_ICONV_T, c) << dst_enc;
321 
322   in = buf;
323   in_bytes = n;
324   char buf2[BUFSIZ] = {};
325   out = buf2;
326   out_bytes = sizeof(buf2);
327   iconv(c, &in, &in_bytes, &out, &out_bytes);
328 
329   ASSERT_STREQ(utf8, buf2) << dst_enc;
330 
331   ASSERT_EQ(0, iconv_close(c));
332 }
333 
TEST(iconv,iconv_round_trip_ascii)334 TEST(iconv, iconv_round_trip_ascii) {
335   RoundTrip("ascii//TRANSLIT", "$??", 3);
336 }
337 
TEST(iconv,iconv_round_trip_utf8)338 TEST(iconv, iconv_round_trip_utf8) {
339   RoundTrip("utf8", "\x24\xe2\x82\xac\xf0\x90\x90\xb7", 8);
340 }
341 
TEST(iconv,iconv_round_trip_utf16be)342 TEST(iconv, iconv_round_trip_utf16be) {
343   RoundTrip("utf16be", "\x00\x24" "\x20\xac" "\xd8\x01\xdc\x37", 8);
344 }
345 
TEST(iconv,iconv_round_trip_utf16le)346 TEST(iconv, iconv_round_trip_utf16le) {
347   RoundTrip("utf16le", "\x24\x00" "\xac\x20" "\x01\xd8\x37\xdc", 8);
348 }
349 
TEST(iconv,iconv_round_trip_utf32be)350 TEST(iconv, iconv_round_trip_utf32be) {
351   RoundTrip("utf32be", "\x00\x00\x00\x24" "\x00\x00\x20\xac" "\x00\x01\x04\x37", 12);
352 }
353 
TEST(iconv,iconv_round_trip_utf32le)354 TEST(iconv, iconv_round_trip_utf32le) {
355   RoundTrip("utf32le", "\x24\x00\x00\x00" "\xac\x20\x00\x00" "\x37\x04\x01\x00", 12);
356 }
357 
TEST(iconv,iconv_round_trip_wchar_t)358 TEST(iconv, iconv_round_trip_wchar_t) {
359   RoundTrip("wchar_t", "\x24\x00\x00\x00" "\xac\x20\x00\x00" "\x37\x04\x01\x00", 12);
360 }
361 
Check(int expected_errno,const char * src_enc,const char * src,size_t n)362 static void Check(int expected_errno, const char* src_enc, const char* src, size_t n) {
363   iconv_t c = iconv_open("wchar_t", src_enc);
364   char* in = const_cast<char*>(src);
365   size_t in_bytes = n;
366   wchar_t out_buf[16];
367   size_t out_bytes = sizeof(out_buf);
368   char* out = reinterpret_cast<char*>(out_buf);
369   errno = 0;
370   ASSERT_EQ(static_cast<size_t>(-1), iconv(c, &in, &in_bytes, &out, &out_bytes));
371   EXPECT_EQ(expected_errno, errno);
372   EXPECT_EQ(0, iconv_close(c));
373 }
374 
TEST(iconv,iconv_EILSEQ_ascii)375 TEST(iconv, iconv_EILSEQ_ascii) {
376   Check(EILSEQ, "ASCII", "\xac", 1); // > 0x7f, so not ASCII.
377 }
378 
TEST(iconv,iconv_EILSEQ_utf8_initial)379 TEST(iconv, iconv_EILSEQ_utf8_initial) {
380   Check(EILSEQ, "utf8", "\x82", 1); // Invalid initial byte.
381 }
382 
TEST(iconv,iconv_EILSEQ_utf8_non_initial)383 TEST(iconv, iconv_EILSEQ_utf8_non_initial) {
384   Check(EILSEQ, "utf8", "\xe2\xe2\x82", 3); // Invalid second byte.
385 }
386 
TEST(iconv,iconv_EILSEQ_utf16be_low_surrogate_first)387 TEST(iconv, iconv_EILSEQ_utf16be_low_surrogate_first) {
388   Check(EILSEQ, "utf16be", "\xdc\x37" "\xd8\x01", 4);
389 }
390 
TEST(iconv,iconv_EILSEQ_utf16le_low_surrogate_first)391 TEST(iconv, iconv_EILSEQ_utf16le_low_surrogate_first) {
392   Check(EILSEQ, "utf16le", "\x37\xdc" "\x01\xd8", 4);
393 }
394 
TEST(iconv,iconv_EINVAL_utf8_short)395 TEST(iconv, iconv_EINVAL_utf8_short) {
396   Check(EINVAL, "utf8", "\xe2\x82", 2); // Missing final byte of 3-byte sequence.
397 }
398 
TEST(iconv,iconv_EINVAL_utf16be_short)399 TEST(iconv, iconv_EINVAL_utf16be_short) {
400   Check(EINVAL, "utf16be", "\x00", 1); // Missing second byte.
401 }
402 
TEST(iconv,iconv_EINVAL_utf16be_missing_low_surrogate)403 TEST(iconv, iconv_EINVAL_utf16be_missing_low_surrogate) {
404   Check(EINVAL, "utf16be", "\xd8\x01", 2);
405 }
406 
TEST(iconv,iconv_EINVAL_utf16be_half_low_surrogate)407 TEST(iconv, iconv_EINVAL_utf16be_half_low_surrogate) {
408   Check(EINVAL, "utf16be", "\xd8\x01\xdc", 3);
409 }
410 
TEST(iconv,iconv_EINVAL_utf16le_short)411 TEST(iconv, iconv_EINVAL_utf16le_short) {
412   Check(EINVAL, "utf16le", "\x24", 1); // Missing second byte.
413 }
414 
TEST(iconv,iconv_EINVAL_utf16le_missing_low_surrogate)415 TEST(iconv, iconv_EINVAL_utf16le_missing_low_surrogate) {
416   Check(EINVAL, "utf16le", "\x01\xd8", 2);
417 }
418 
TEST(iconv,iconv_EINVAL_utf16le_half_low_surrogate)419 TEST(iconv, iconv_EINVAL_utf16le_half_low_surrogate) {
420   Check(EINVAL, "utf16le", "\x01\xd8\x37", 3);
421 }
422 
TEST(iconv,iconv_EINVAL_utf32be_short)423 TEST(iconv, iconv_EINVAL_utf32be_short) {
424   Check(EINVAL, "utf32be", "\x00\x00\x00", 3); // Missing final byte.
425 }
426 
TEST(iconv,iconv_EINVAL_utf32le_short)427 TEST(iconv, iconv_EINVAL_utf32le_short) {
428   Check(EINVAL, "utf32le", "\x24\x00\x00", 3); // Missing final byte.
429 }
430 
TEST(iconv,iconv_initial_shift_state)431 TEST(iconv, iconv_initial_shift_state) {
432   // POSIX: "For state-dependent encodings, the conversion descriptor
433   // cd is placed into its initial shift state by a call for which inbuf
434   // is a null pointer, or for which inbuf points to a null pointer."
435   iconv_t c = iconv_open("utf8", "utf8");
436   char* in = nullptr;
437   size_t in_bytes = 0;
438   wchar_t out_buf[16];
439   size_t out_bytes = sizeof(out_buf);
440   char* out = reinterpret_cast<char*>(out_buf);
441 
442   // Points to a null pointer...
443   errno = 0;
444   ASSERT_EQ(static_cast<size_t>(0), iconv(c, &in, &in_bytes, &out, &out_bytes));
445   EXPECT_EQ(0, errno);
446   EXPECT_EQ(sizeof(out_buf), out_bytes);
447 
448   // Is a null pointer...
449   errno = 0;
450   ASSERT_EQ(static_cast<size_t>(0), iconv(c, nullptr, &in_bytes, &out, &out_bytes));
451   EXPECT_EQ(0, errno);
452   EXPECT_EQ(sizeof(out_buf), out_bytes);
453 
454   EXPECT_EQ(0, iconv_close(c));
455 }
456