1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef LIBTEXTCLASSIFIER_UTILS_STRINGS_UTF8_H_
18 #define LIBTEXTCLASSIFIER_UTILS_STRINGS_UTF8_H_
19
20 #include "utils/base/integral_types.h"
21
22 namespace libtextclassifier3 {
23
24 // Returns the length (number of bytes) of the Unicode code point starting at
25 // src, based on inspecting just that one byte. Preconditions: src != NULL,
26 // *src can be read.
GetNumBytesForUTF8Char(const char * src)27 static inline int GetNumBytesForUTF8Char(const char *src) {
28 // On most platforms, char is unsigned by default, but iOS is an exception.
29 // The cast below makes sure we always interpret *src as an unsigned char.
30 return "\1\1\1\1\1\1\1\1\1\1\1\1\2\2\3\4"
31 [(*(reinterpret_cast<const unsigned char *>(src)) & 0xFF) >> 4];
32 }
33
34 // Returns true if this byte is a trailing UTF-8 byte (10xx xxxx)
IsTrailByte(char x)35 static inline bool IsTrailByte(char x) {
36 // return (x & 0xC0) == 0x80;
37 // Since trail bytes are always in [0x80, 0xBF], we can optimize:
38 return static_cast<signed char>(x) < -0x40;
39 }
40
41 // Returns true iff src points to a well-formed UTF-8 string.
42 bool IsValidUTF8(const char *src, int size);
43
44 // Helper to ensure that strings are not truncated in the middle of
45 // multi-byte UTF-8 characters.
46 // Given a string, and a position at which to truncate, returns the
47 // last position not after the provided cut point, that would truncate a
48 // full character.
49 int SafeTruncateLength(const char *str, int truncate_at);
50
51 // Gets a unicode codepoint from a valid utf8 encoding.
52 char32 ValidCharToRune(const char *str);
53
54 // Checks whether a utf8 encoding is a valid codepoint and returns the number of
55 // bytes of the codepoint.
56 bool IsValidChar(const char *str, int size, int *num_bytes);
57
58 // Converts a valid codepoint to utf8.
59 // Returns the length of the encoding.
60 int ValidRuneToChar(const char32 rune, char *dest);
61
62 } // namespace libtextclassifier3
63
64 #endif // LIBTEXTCLASSIFIER_UTILS_STRINGS_UTF8_H_
65