1 #define JEMALLOC_CHUNK_MMAP_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 /******************************************************************************/
5 /* Defines/includes needed for special android code. */
6
7 #if defined(__ANDROID__)
8 #include <sys/prctl.h>
9
10 /* Definitions of prctl arguments to set a vma name in Android kernels. */
11 #define ANDROID_PR_SET_VMA 0x53564d41
12 #define ANDROID_PR_SET_VMA_ANON_NAME 0
13 #endif
14
15 /******************************************************************************/
16
17 /******************************************************************************/
18 /* Function prototypes for non-inline static functions. */
19
20 static void *pages_map(void *addr, size_t size);
21 static void pages_unmap(void *addr, size_t size);
22 static void *chunk_alloc_mmap_slow(size_t size, size_t alignment,
23 bool *zero);
24
25 /******************************************************************************/
26
27 static void *
pages_map(void * addr,size_t size)28 pages_map(void *addr, size_t size)
29 {
30 void *ret;
31
32 assert(size != 0);
33
34 #ifdef _WIN32
35 /*
36 * If VirtualAlloc can't allocate at the given address when one is
37 * given, it fails and returns NULL.
38 */
39 ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
40 PAGE_READWRITE);
41 #else
42 /*
43 * We don't use MAP_FIXED here, because it can cause the *replacement*
44 * of existing mappings, and we only want to create new mappings.
45 */
46 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
47 -1, 0);
48 assert(ret != NULL);
49
50 if (ret == MAP_FAILED)
51 ret = NULL;
52 else if (addr != NULL && ret != addr) {
53 /*
54 * We succeeded in mapping memory, but not in the right place.
55 */
56 pages_unmap(ret, size);
57 ret = NULL;
58 }
59 #endif
60 #if defined(__ANDROID__)
61 if (ret != NULL) {
62 /* Name this memory as being used by libc */
63 prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, ret,
64 size, "libc_malloc");
65 }
66 #endif
67 assert(ret == NULL || (addr == NULL && ret != addr)
68 || (addr != NULL && ret == addr));
69 return (ret);
70 }
71
72 static void
pages_unmap(void * addr,size_t size)73 pages_unmap(void *addr, size_t size)
74 {
75
76 #ifdef _WIN32
77 if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
78 #else
79 if (munmap(addr, size) == -1)
80 #endif
81 {
82 char buf[BUFERROR_BUF];
83
84 buferror(get_errno(), buf, sizeof(buf));
85 malloc_printf("<jemalloc>: Error in "
86 #ifdef _WIN32
87 "VirtualFree"
88 #else
89 "munmap"
90 #endif
91 "(): %s\n", buf);
92 if (opt_abort)
93 abort();
94 }
95 }
96
97 static void *
pages_trim(void * addr,size_t alloc_size,size_t leadsize,size_t size)98 pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
99 {
100 void *ret = (void *)((uintptr_t)addr + leadsize);
101
102 assert(alloc_size >= leadsize + size);
103 #ifdef _WIN32
104 {
105 void *new_addr;
106
107 pages_unmap(addr, alloc_size);
108 new_addr = pages_map(ret, size);
109 if (new_addr == ret)
110 return (ret);
111 if (new_addr)
112 pages_unmap(new_addr, size);
113 return (NULL);
114 }
115 #else
116 {
117 size_t trailsize = alloc_size - leadsize - size;
118
119 if (leadsize != 0)
120 pages_unmap(addr, leadsize);
121 if (trailsize != 0)
122 pages_unmap((void *)((uintptr_t)ret + size), trailsize);
123 return (ret);
124 }
125 #endif
126 }
127
128 bool
pages_purge(void * addr,size_t length)129 pages_purge(void *addr, size_t length)
130 {
131 bool unzeroed;
132
133 #ifdef _WIN32
134 VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE);
135 unzeroed = true;
136 #elif defined(JEMALLOC_HAVE_MADVISE)
137 # ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
138 # define JEMALLOC_MADV_PURGE MADV_DONTNEED
139 # define JEMALLOC_MADV_ZEROS true
140 # elif defined(JEMALLOC_PURGE_MADVISE_FREE)
141 # define JEMALLOC_MADV_PURGE MADV_FREE
142 # define JEMALLOC_MADV_ZEROS false
143 # else
144 # error "No madvise(2) flag defined for purging unused dirty pages."
145 # endif
146 int err = madvise(addr, length, JEMALLOC_MADV_PURGE);
147 unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
148 # undef JEMALLOC_MADV_PURGE
149 # undef JEMALLOC_MADV_ZEROS
150 #else
151 /* Last resort no-op. */
152 unzeroed = true;
153 #endif
154 return (unzeroed);
155 }
156
157 static void *
chunk_alloc_mmap_slow(size_t size,size_t alignment,bool * zero)158 chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero)
159 {
160 void *ret, *pages;
161 size_t alloc_size, leadsize;
162
163 alloc_size = size + alignment - PAGE;
164 /* Beware size_t wrap-around. */
165 if (alloc_size < size)
166 return (NULL);
167 do {
168 pages = pages_map(NULL, alloc_size);
169 if (pages == NULL)
170 return (NULL);
171 leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
172 (uintptr_t)pages;
173 ret = pages_trim(pages, alloc_size, leadsize, size);
174 } while (ret == NULL);
175
176 assert(ret != NULL);
177 *zero = true;
178 return (ret);
179 }
180
181 void *
chunk_alloc_mmap(size_t size,size_t alignment,bool * zero)182 chunk_alloc_mmap(size_t size, size_t alignment, bool *zero)
183 {
184 void *ret;
185 size_t offset;
186
187 /*
188 * Ideally, there would be a way to specify alignment to mmap() (like
189 * NetBSD has), but in the absence of such a feature, we have to work
190 * hard to efficiently create aligned mappings. The reliable, but
191 * slow method is to create a mapping that is over-sized, then trim the
192 * excess. However, that always results in one or two calls to
193 * pages_unmap().
194 *
195 * Optimistically try mapping precisely the right amount before falling
196 * back to the slow method, with the expectation that the optimistic
197 * approach works most of the time.
198 */
199
200 assert(alignment != 0);
201 assert((alignment & chunksize_mask) == 0);
202
203 ret = pages_map(NULL, size);
204 if (ret == NULL)
205 return (NULL);
206 offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
207 if (offset != 0) {
208 pages_unmap(ret, size);
209 return (chunk_alloc_mmap_slow(size, alignment, zero));
210 }
211
212 assert(ret != NULL);
213 *zero = true;
214 return (ret);
215 }
216
217 bool
chunk_dalloc_mmap(void * chunk,size_t size)218 chunk_dalloc_mmap(void *chunk, size_t size)
219 {
220
221 if (config_munmap)
222 pages_unmap(chunk, size);
223
224 return (!config_munmap);
225 }
226