1 //===-- Memcpy utils --------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_UTILS_H
10 #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_UTILS_H
11
12 #include "src/string/memory_utils/utils.h"
13 #include <stddef.h> // size_t
14
15 // __builtin_memcpy_inline guarantees to never call external functions.
16 // Unfortunately it is not widely available.
17 #ifdef __clang__
18 #if __has_builtin(__builtin_memcpy_inline)
19 #define USE_BUILTIN_MEMCPY_INLINE
20 #endif
21 #elif defined(__GNUC__)
22 #define USE_BUILTIN_MEMCPY
23 #endif
24
25 namespace __llvm_libc {
26
27 // This is useful for testing.
28 #if defined(LLVM_LIBC_MEMCPY_MONITOR)
29 extern "C" void LLVM_LIBC_MEMCPY_MONITOR(char *__restrict,
30 const char *__restrict, size_t);
31 #endif
32
33 // Copies `kBlockSize` bytes from `src` to `dst`.
34 template <size_t kBlockSize>
CopyBlock(char * __restrict dst,const char * __restrict src)35 static void CopyBlock(char *__restrict dst, const char *__restrict src) {
36 #if defined(LLVM_LIBC_MEMCPY_MONITOR)
37 LLVM_LIBC_MEMCPY_MONITOR(dst, src, kBlockSize);
38 #elif defined(USE_BUILTIN_MEMCPY_INLINE)
39 __builtin_memcpy_inline(dst, src, kBlockSize);
40 #elif defined(USE_BUILTIN_MEMCPY)
41 __builtin_memcpy(dst, src, kBlockSize);
42 #else
43 for (size_t i = 0; i < kBlockSize; ++i)
44 dst[i] = src[i];
45 #endif
46 }
47
48 // Copies `kBlockSize` bytes from `src + count - kBlockSize` to
49 // `dst + count - kBlockSize`.
50 // Precondition: `count >= kBlockSize`.
51 template <size_t kBlockSize>
CopyLastBlock(char * __restrict dst,const char * __restrict src,size_t count)52 static void CopyLastBlock(char *__restrict dst, const char *__restrict src,
53 size_t count) {
54 const size_t offset = count - kBlockSize;
55 CopyBlock<kBlockSize>(dst + offset, src + offset);
56 }
57
58 // Copies `kBlockSize` bytes twice with an overlap between the two.
59 //
60 // [1234567812345678123]
61 // [__XXXXXXXXXXXXXX___]
62 // [__XXXXXXXX_________]
63 // [________XXXXXXXX___]
64 //
65 // Precondition: `count >= kBlockSize && count <= kBlockSize`.
66 template <size_t kBlockSize>
CopyBlockOverlap(char * __restrict dst,const char * __restrict src,size_t count)67 static void CopyBlockOverlap(char *__restrict dst, const char *__restrict src,
68 size_t count) {
69 CopyBlock<kBlockSize>(dst, src);
70 CopyLastBlock<kBlockSize>(dst, src, count);
71 }
72
73 // Copies `count` bytes by blocks of `kBlockSize` bytes.
74 // Copies at the start and end of the buffer are unaligned.
75 // Copies in the middle of the buffer are aligned to `kBlockSize`.
76 //
77 // e.g. with
78 // [12345678123456781234567812345678]
79 // [__XXXXXXXXXXXXXXXXXXXXXXXXXXX___]
80 // [__XXXXXXXX______________________]
81 // [________XXXXXXXX________________]
82 // [________________XXXXXXXX________]
83 // [_____________________XXXXXXXX___]
84 //
85 // Precondition: `count > 2 * kBlockSize` for efficiency.
86 // `count >= kBlockSize` for correctness.
87 template <size_t kBlockSize>
CopyAlignedBlocks(char * __restrict dst,const char * __restrict src,size_t count)88 static void CopyAlignedBlocks(char *__restrict dst, const char *__restrict src,
89 size_t count) {
90 CopyBlock<kBlockSize>(dst, src); // Copy first block
91
92 // Copy aligned blocks
93 const size_t ofla = offset_from_last_aligned<kBlockSize>(dst);
94 const size_t limit = count + ofla - kBlockSize;
95 for (size_t offset = kBlockSize; offset < limit; offset += kBlockSize)
96 CopyBlock<kBlockSize>(dst - ofla + offset, src - ofla + offset);
97
98 CopyLastBlock<kBlockSize>(dst, src, count); // Copy last block
99 }
100
101 } // namespace __llvm_libc
102
103 #endif // LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_UTILS_H
104