1 #define	JEMALLOC_PROF_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3 /******************************************************************************/
4 
5 #ifdef JEMALLOC_PROF_LIBUNWIND
6 #define	UNW_LOCAL_ONLY
7 #include <libunwind.h>
8 #endif
9 
10 #ifdef JEMALLOC_PROF_LIBGCC
11 #include <unwind.h>
12 #endif
13 
14 /******************************************************************************/
15 /* Data. */
16 
17 bool		opt_prof = false;
18 bool		opt_prof_active = true;
19 bool		opt_prof_thread_active_init = true;
20 size_t		opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
21 ssize_t		opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
22 bool		opt_prof_gdump = false;
23 bool		opt_prof_final = false;
24 bool		opt_prof_leak = false;
25 bool		opt_prof_accum = false;
26 char		opt_prof_prefix[
27     /* Minimize memory bloat for non-prof builds. */
28 #ifdef JEMALLOC_PROF
29     PATH_MAX +
30 #endif
31     1];
32 
33 /*
34  * Initialized as opt_prof_active, and accessed via
35  * prof_active_[gs]et{_unlocked,}().
36  */
37 bool			prof_active;
38 static malloc_mutex_t	prof_active_mtx;
39 
40 /*
41  * Initialized as opt_prof_thread_active_init, and accessed via
42  * prof_thread_active_init_[gs]et().
43  */
44 static bool		prof_thread_active_init;
45 static malloc_mutex_t	prof_thread_active_init_mtx;
46 
47 /*
48  * Initialized as opt_prof_gdump, and accessed via
49  * prof_gdump_[gs]et{_unlocked,}().
50  */
51 bool			prof_gdump_val;
52 static malloc_mutex_t	prof_gdump_mtx;
53 
54 uint64_t	prof_interval = 0;
55 
56 size_t		lg_prof_sample;
57 
58 /*
59  * Table of mutexes that are shared among gctx's.  These are leaf locks, so
60  * there is no problem with using them for more than one gctx at the same time.
61  * The primary motivation for this sharing though is that gctx's are ephemeral,
62  * and destroying mutexes causes complications for systems that allocate when
63  * creating/destroying mutexes.
64  */
65 static malloc_mutex_t	*gctx_locks;
66 static unsigned		cum_gctxs; /* Atomic counter. */
67 
68 /*
69  * Table of mutexes that are shared among tdata's.  No operations require
70  * holding multiple tdata locks, so there is no problem with using them for more
71  * than one tdata at the same time, even though a gctx lock may be acquired
72  * while holding a tdata lock.
73  */
74 static malloc_mutex_t	*tdata_locks;
75 
76 /*
77  * Global hash of (prof_bt_t *)-->(prof_gctx_t *).  This is the master data
78  * structure that knows about all backtraces currently captured.
79  */
80 static ckh_t		bt2gctx;
81 static malloc_mutex_t	bt2gctx_mtx;
82 
83 /*
84  * Tree of all extant prof_tdata_t structures, regardless of state,
85  * {attached,detached,expired}.
86  */
87 static prof_tdata_tree_t	tdatas;
88 static malloc_mutex_t	tdatas_mtx;
89 
90 static uint64_t		next_thr_uid;
91 static malloc_mutex_t	next_thr_uid_mtx;
92 
93 static malloc_mutex_t	prof_dump_seq_mtx;
94 static uint64_t		prof_dump_seq;
95 static uint64_t		prof_dump_iseq;
96 static uint64_t		prof_dump_mseq;
97 static uint64_t		prof_dump_useq;
98 
99 /*
100  * This buffer is rather large for stack allocation, so use a single buffer for
101  * all profile dumps.
102  */
103 static malloc_mutex_t	prof_dump_mtx;
104 static char		prof_dump_buf[
105     /* Minimize memory bloat for non-prof builds. */
106 #ifdef JEMALLOC_PROF
107     PROF_DUMP_BUFSIZE
108 #else
109     1
110 #endif
111 ];
112 static size_t		prof_dump_buf_end;
113 static int		prof_dump_fd;
114 
115 /* Do not dump any profiles until bootstrapping is complete. */
116 static bool		prof_booted = false;
117 
118 /******************************************************************************/
119 /*
120  * Function prototypes for static functions that are referenced prior to
121  * definition.
122  */
123 
124 static bool	prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx);
125 static void	prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
126 static bool	prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
127     bool even_if_attached);
128 static void	prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
129     bool even_if_attached);
130 static char	*prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name);
131 
132 /******************************************************************************/
133 /* Red-black trees. */
134 
135 JEMALLOC_INLINE_C int
prof_tctx_comp(const prof_tctx_t * a,const prof_tctx_t * b)136 prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b)
137 {
138 	uint64_t a_thr_uid = a->thr_uid;
139 	uint64_t b_thr_uid = b->thr_uid;
140 	int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
141 	if (ret == 0) {
142 		uint64_t a_thr_discrim = a->thr_discrim;
143 		uint64_t b_thr_discrim = b->thr_discrim;
144 		ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
145 		    b_thr_discrim);
146 		if (ret == 0) {
147 			uint64_t a_tctx_uid = a->tctx_uid;
148 			uint64_t b_tctx_uid = b->tctx_uid;
149 			ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
150 			    b_tctx_uid);
151 		}
152 	}
153 	return (ret);
154 }
155 
rb_gen(static UNUSED,tctx_tree_,prof_tctx_tree_t,prof_tctx_t,tctx_link,prof_tctx_comp)156 rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
157     tctx_link, prof_tctx_comp)
158 
159 JEMALLOC_INLINE_C int
160 prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b)
161 {
162 	unsigned a_len = a->bt.len;
163 	unsigned b_len = b->bt.len;
164 	unsigned comp_len = (a_len < b_len) ? a_len : b_len;
165 	int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
166 	if (ret == 0)
167 		ret = (a_len > b_len) - (a_len < b_len);
168 	return (ret);
169 }
170 
rb_gen(static UNUSED,gctx_tree_,prof_gctx_tree_t,prof_gctx_t,dump_link,prof_gctx_comp)171 rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
172     prof_gctx_comp)
173 
174 JEMALLOC_INLINE_C int
175 prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b)
176 {
177 	int ret;
178 	uint64_t a_uid = a->thr_uid;
179 	uint64_t b_uid = b->thr_uid;
180 
181 	ret = ((a_uid > b_uid) - (a_uid < b_uid));
182 	if (ret == 0) {
183 		uint64_t a_discrim = a->thr_discrim;
184 		uint64_t b_discrim = b->thr_discrim;
185 
186 		ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
187 	}
188 	return (ret);
189 }
190 
rb_gen(static UNUSED,tdata_tree_,prof_tdata_tree_t,prof_tdata_t,tdata_link,prof_tdata_comp)191 rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
192     prof_tdata_comp)
193 
194 /******************************************************************************/
195 
196 void
197 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
198 {
199 	prof_tdata_t *tdata;
200 
201 	cassert(config_prof);
202 
203 	if (updated) {
204 		/*
205 		 * Compute a new sample threshold.  This isn't very important in
206 		 * practice, because this function is rarely executed, so the
207 		 * potential for sample bias is minimal except in contrived
208 		 * programs.
209 		 */
210 		tdata = prof_tdata_get(tsd, true);
211 		if (tdata != NULL)
212 			prof_sample_threshold_update(tdata);
213 	}
214 
215 	if ((uintptr_t)tctx > (uintptr_t)1U) {
216 		malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
217 		tctx->prepared = false;
218 		if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx))
219 			prof_tctx_destroy(tsd, tctx);
220 		else
221 			malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
222 	}
223 }
224 
225 void
prof_malloc_sample_object(tsdn_t * tsdn,const void * ptr,size_t usize,prof_tctx_t * tctx)226 prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
227     prof_tctx_t *tctx)
228 {
229 
230 	prof_tctx_set(tsdn, ptr, usize, tctx);
231 
232 	malloc_mutex_lock(tsdn, tctx->tdata->lock);
233 	tctx->cnts.curobjs++;
234 	tctx->cnts.curbytes += usize;
235 	if (opt_prof_accum) {
236 		tctx->cnts.accumobjs++;
237 		tctx->cnts.accumbytes += usize;
238 	}
239 	tctx->prepared = false;
240 	malloc_mutex_unlock(tsdn, tctx->tdata->lock);
241 }
242 
243 void
prof_free_sampled_object(tsd_t * tsd,size_t usize,prof_tctx_t * tctx)244 prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
245 {
246 
247 	malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
248 	assert(tctx->cnts.curobjs > 0);
249 	assert(tctx->cnts.curbytes >= usize);
250 	tctx->cnts.curobjs--;
251 	tctx->cnts.curbytes -= usize;
252 
253 	if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx))
254 		prof_tctx_destroy(tsd, tctx);
255 	else
256 		malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
257 }
258 
259 void
bt_init(prof_bt_t * bt,void ** vec)260 bt_init(prof_bt_t *bt, void **vec)
261 {
262 
263 	cassert(config_prof);
264 
265 	bt->vec = vec;
266 	bt->len = 0;
267 }
268 
269 JEMALLOC_INLINE_C void
prof_enter(tsd_t * tsd,prof_tdata_t * tdata)270 prof_enter(tsd_t *tsd, prof_tdata_t *tdata)
271 {
272 
273 	cassert(config_prof);
274 	assert(tdata == prof_tdata_get(tsd, false));
275 
276 	if (tdata != NULL) {
277 		assert(!tdata->enq);
278 		tdata->enq = true;
279 	}
280 
281 	malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
282 }
283 
284 JEMALLOC_INLINE_C void
prof_leave(tsd_t * tsd,prof_tdata_t * tdata)285 prof_leave(tsd_t *tsd, prof_tdata_t *tdata)
286 {
287 
288 	cassert(config_prof);
289 	assert(tdata == prof_tdata_get(tsd, false));
290 
291 	malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
292 
293 	if (tdata != NULL) {
294 		bool idump, gdump;
295 
296 		assert(tdata->enq);
297 		tdata->enq = false;
298 		idump = tdata->enq_idump;
299 		tdata->enq_idump = false;
300 		gdump = tdata->enq_gdump;
301 		tdata->enq_gdump = false;
302 
303 		if (idump)
304 			prof_idump(tsd_tsdn(tsd));
305 		if (gdump)
306 			prof_gdump(tsd_tsdn(tsd));
307 	}
308 }
309 
310 #ifdef JEMALLOC_PROF_LIBUNWIND
311 void
prof_backtrace(prof_bt_t * bt)312 prof_backtrace(prof_bt_t *bt)
313 {
314 	int nframes;
315 
316 	cassert(config_prof);
317 	assert(bt->len == 0);
318 	assert(bt->vec != NULL);
319 
320 	nframes = unw_backtrace(bt->vec, PROF_BT_MAX);
321 	if (nframes <= 0)
322 		return;
323 	bt->len = nframes;
324 }
325 #elif (defined(JEMALLOC_PROF_LIBGCC))
326 static _Unwind_Reason_Code
prof_unwind_init_callback(struct _Unwind_Context * context,void * arg)327 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg)
328 {
329 
330 	cassert(config_prof);
331 
332 	return (_URC_NO_REASON);
333 }
334 
335 static _Unwind_Reason_Code
prof_unwind_callback(struct _Unwind_Context * context,void * arg)336 prof_unwind_callback(struct _Unwind_Context *context, void *arg)
337 {
338 	prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
339 	void *ip;
340 
341 	cassert(config_prof);
342 
343 	ip = (void *)_Unwind_GetIP(context);
344 	if (ip == NULL)
345 		return (_URC_END_OF_STACK);
346 	data->bt->vec[data->bt->len] = ip;
347 	data->bt->len++;
348 	if (data->bt->len == data->max)
349 		return (_URC_END_OF_STACK);
350 
351 	return (_URC_NO_REASON);
352 }
353 
354 void
prof_backtrace(prof_bt_t * bt)355 prof_backtrace(prof_bt_t *bt)
356 {
357 	prof_unwind_data_t data = {bt, PROF_BT_MAX};
358 
359 	cassert(config_prof);
360 
361 	_Unwind_Backtrace(prof_unwind_callback, &data);
362 }
363 #elif (defined(JEMALLOC_PROF_GCC))
364 void
prof_backtrace(prof_bt_t * bt)365 prof_backtrace(prof_bt_t *bt)
366 {
367 #define	BT_FRAME(i)							\
368 	if ((i) < PROF_BT_MAX) {					\
369 		void *p;						\
370 		if (__builtin_frame_address(i) == 0)			\
371 			return;						\
372 		p = __builtin_return_address(i);			\
373 		if (p == NULL)						\
374 			return;						\
375 		bt->vec[(i)] = p;					\
376 		bt->len = (i) + 1;					\
377 	} else								\
378 		return;
379 
380 	cassert(config_prof);
381 
382 	BT_FRAME(0)
383 	BT_FRAME(1)
384 	BT_FRAME(2)
385 	BT_FRAME(3)
386 	BT_FRAME(4)
387 	BT_FRAME(5)
388 	BT_FRAME(6)
389 	BT_FRAME(7)
390 	BT_FRAME(8)
391 	BT_FRAME(9)
392 
393 	BT_FRAME(10)
394 	BT_FRAME(11)
395 	BT_FRAME(12)
396 	BT_FRAME(13)
397 	BT_FRAME(14)
398 	BT_FRAME(15)
399 	BT_FRAME(16)
400 	BT_FRAME(17)
401 	BT_FRAME(18)
402 	BT_FRAME(19)
403 
404 	BT_FRAME(20)
405 	BT_FRAME(21)
406 	BT_FRAME(22)
407 	BT_FRAME(23)
408 	BT_FRAME(24)
409 	BT_FRAME(25)
410 	BT_FRAME(26)
411 	BT_FRAME(27)
412 	BT_FRAME(28)
413 	BT_FRAME(29)
414 
415 	BT_FRAME(30)
416 	BT_FRAME(31)
417 	BT_FRAME(32)
418 	BT_FRAME(33)
419 	BT_FRAME(34)
420 	BT_FRAME(35)
421 	BT_FRAME(36)
422 	BT_FRAME(37)
423 	BT_FRAME(38)
424 	BT_FRAME(39)
425 
426 	BT_FRAME(40)
427 	BT_FRAME(41)
428 	BT_FRAME(42)
429 	BT_FRAME(43)
430 	BT_FRAME(44)
431 	BT_FRAME(45)
432 	BT_FRAME(46)
433 	BT_FRAME(47)
434 	BT_FRAME(48)
435 	BT_FRAME(49)
436 
437 	BT_FRAME(50)
438 	BT_FRAME(51)
439 	BT_FRAME(52)
440 	BT_FRAME(53)
441 	BT_FRAME(54)
442 	BT_FRAME(55)
443 	BT_FRAME(56)
444 	BT_FRAME(57)
445 	BT_FRAME(58)
446 	BT_FRAME(59)
447 
448 	BT_FRAME(60)
449 	BT_FRAME(61)
450 	BT_FRAME(62)
451 	BT_FRAME(63)
452 	BT_FRAME(64)
453 	BT_FRAME(65)
454 	BT_FRAME(66)
455 	BT_FRAME(67)
456 	BT_FRAME(68)
457 	BT_FRAME(69)
458 
459 	BT_FRAME(70)
460 	BT_FRAME(71)
461 	BT_FRAME(72)
462 	BT_FRAME(73)
463 	BT_FRAME(74)
464 	BT_FRAME(75)
465 	BT_FRAME(76)
466 	BT_FRAME(77)
467 	BT_FRAME(78)
468 	BT_FRAME(79)
469 
470 	BT_FRAME(80)
471 	BT_FRAME(81)
472 	BT_FRAME(82)
473 	BT_FRAME(83)
474 	BT_FRAME(84)
475 	BT_FRAME(85)
476 	BT_FRAME(86)
477 	BT_FRAME(87)
478 	BT_FRAME(88)
479 	BT_FRAME(89)
480 
481 	BT_FRAME(90)
482 	BT_FRAME(91)
483 	BT_FRAME(92)
484 	BT_FRAME(93)
485 	BT_FRAME(94)
486 	BT_FRAME(95)
487 	BT_FRAME(96)
488 	BT_FRAME(97)
489 	BT_FRAME(98)
490 	BT_FRAME(99)
491 
492 	BT_FRAME(100)
493 	BT_FRAME(101)
494 	BT_FRAME(102)
495 	BT_FRAME(103)
496 	BT_FRAME(104)
497 	BT_FRAME(105)
498 	BT_FRAME(106)
499 	BT_FRAME(107)
500 	BT_FRAME(108)
501 	BT_FRAME(109)
502 
503 	BT_FRAME(110)
504 	BT_FRAME(111)
505 	BT_FRAME(112)
506 	BT_FRAME(113)
507 	BT_FRAME(114)
508 	BT_FRAME(115)
509 	BT_FRAME(116)
510 	BT_FRAME(117)
511 	BT_FRAME(118)
512 	BT_FRAME(119)
513 
514 	BT_FRAME(120)
515 	BT_FRAME(121)
516 	BT_FRAME(122)
517 	BT_FRAME(123)
518 	BT_FRAME(124)
519 	BT_FRAME(125)
520 	BT_FRAME(126)
521 	BT_FRAME(127)
522 #undef BT_FRAME
523 }
524 #else
525 void
prof_backtrace(prof_bt_t * bt)526 prof_backtrace(prof_bt_t *bt)
527 {
528 
529 	cassert(config_prof);
530 	not_reached();
531 }
532 #endif
533 
534 static malloc_mutex_t *
prof_gctx_mutex_choose(void)535 prof_gctx_mutex_choose(void)
536 {
537 	unsigned ngctxs = atomic_add_u(&cum_gctxs, 1);
538 
539 	return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]);
540 }
541 
542 static malloc_mutex_t *
prof_tdata_mutex_choose(uint64_t thr_uid)543 prof_tdata_mutex_choose(uint64_t thr_uid)
544 {
545 
546 	return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]);
547 }
548 
549 static prof_gctx_t *
prof_gctx_create(tsdn_t * tsdn,prof_bt_t * bt)550 prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt)
551 {
552 	/*
553 	 * Create a single allocation that has space for vec of length bt->len.
554 	 */
555 	size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
556 	prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
557 	    size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
558 	    true);
559 	if (gctx == NULL)
560 		return (NULL);
561 	gctx->lock = prof_gctx_mutex_choose();
562 	/*
563 	 * Set nlimbo to 1, in order to avoid a race condition with
564 	 * prof_tctx_destroy()/prof_gctx_try_destroy().
565 	 */
566 	gctx->nlimbo = 1;
567 	tctx_tree_new(&gctx->tctxs);
568 	/* Duplicate bt. */
569 	memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
570 	gctx->bt.vec = gctx->vec;
571 	gctx->bt.len = bt->len;
572 	return (gctx);
573 }
574 
575 static void
prof_gctx_try_destroy(tsd_t * tsd,prof_tdata_t * tdata_self,prof_gctx_t * gctx,prof_tdata_t * tdata)576 prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
577     prof_tdata_t *tdata)
578 {
579 
580 	cassert(config_prof);
581 
582 	/*
583 	 * Check that gctx is still unused by any thread cache before destroying
584 	 * it.  prof_lookup() increments gctx->nlimbo in order to avoid a race
585 	 * condition with this function, as does prof_tctx_destroy() in order to
586 	 * avoid a race between the main body of prof_tctx_destroy() and entry
587 	 * into this function.
588 	 */
589 	prof_enter(tsd, tdata_self);
590 	malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
591 	assert(gctx->nlimbo != 0);
592 	if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
593 		/* Remove gctx from bt2gctx. */
594 		if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL))
595 			not_reached();
596 		prof_leave(tsd, tdata_self);
597 		/* Destroy gctx. */
598 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
599 		idalloctm(tsd_tsdn(tsd), gctx, NULL, true, true);
600 	} else {
601 		/*
602 		 * Compensate for increment in prof_tctx_destroy() or
603 		 * prof_lookup().
604 		 */
605 		gctx->nlimbo--;
606 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
607 		prof_leave(tsd, tdata_self);
608 	}
609 }
610 
611 static bool
prof_tctx_should_destroy(tsdn_t * tsdn,prof_tctx_t * tctx)612 prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx)
613 {
614 
615 	malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
616 
617 	if (opt_prof_accum)
618 		return (false);
619 	if (tctx->cnts.curobjs != 0)
620 		return (false);
621 	if (tctx->prepared)
622 		return (false);
623 	return (true);
624 }
625 
626 static bool
prof_gctx_should_destroy(prof_gctx_t * gctx)627 prof_gctx_should_destroy(prof_gctx_t *gctx)
628 {
629 
630 	if (opt_prof_accum)
631 		return (false);
632 	if (!tctx_tree_empty(&gctx->tctxs))
633 		return (false);
634 	if (gctx->nlimbo != 0)
635 		return (false);
636 	return (true);
637 }
638 
639 static void
prof_tctx_destroy(tsd_t * tsd,prof_tctx_t * tctx)640 prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
641 {
642 	prof_tdata_t *tdata = tctx->tdata;
643 	prof_gctx_t *gctx = tctx->gctx;
644 	bool destroy_tdata, destroy_tctx, destroy_gctx;
645 
646 	malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
647 
648 	assert(tctx->cnts.curobjs == 0);
649 	assert(tctx->cnts.curbytes == 0);
650 	assert(!opt_prof_accum);
651 	assert(tctx->cnts.accumobjs == 0);
652 	assert(tctx->cnts.accumbytes == 0);
653 
654 	ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
655 	destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);
656 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
657 
658 	malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
659 	switch (tctx->state) {
660 	case prof_tctx_state_nominal:
661 		tctx_tree_remove(&gctx->tctxs, tctx);
662 		destroy_tctx = true;
663 		if (prof_gctx_should_destroy(gctx)) {
664 			/*
665 			 * Increment gctx->nlimbo in order to keep another
666 			 * thread from winning the race to destroy gctx while
667 			 * this one has gctx->lock dropped.  Without this, it
668 			 * would be possible for another thread to:
669 			 *
670 			 * 1) Sample an allocation associated with gctx.
671 			 * 2) Deallocate the sampled object.
672 			 * 3) Successfully prof_gctx_try_destroy(gctx).
673 			 *
674 			 * The result would be that gctx no longer exists by the
675 			 * time this thread accesses it in
676 			 * prof_gctx_try_destroy().
677 			 */
678 			gctx->nlimbo++;
679 			destroy_gctx = true;
680 		} else
681 			destroy_gctx = false;
682 		break;
683 	case prof_tctx_state_dumping:
684 		/*
685 		 * A dumping thread needs tctx to remain valid until dumping
686 		 * has finished.  Change state such that the dumping thread will
687 		 * complete destruction during a late dump iteration phase.
688 		 */
689 		tctx->state = prof_tctx_state_purgatory;
690 		destroy_tctx = false;
691 		destroy_gctx = false;
692 		break;
693 	default:
694 		not_reached();
695 		destroy_tctx = false;
696 		destroy_gctx = false;
697 	}
698 	malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
699 	if (destroy_gctx) {
700 		prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
701 		    tdata);
702 	}
703 
704 	malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);
705 
706 	if (destroy_tdata)
707 		prof_tdata_destroy(tsd, tdata, false);
708 
709 	if (destroy_tctx)
710 		idalloctm(tsd_tsdn(tsd), tctx, NULL, true, true);
711 }
712 
713 static bool
prof_lookup_global(tsd_t * tsd,prof_bt_t * bt,prof_tdata_t * tdata,void ** p_btkey,prof_gctx_t ** p_gctx,bool * p_new_gctx)714 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
715     void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx)
716 {
717 	union {
718 		prof_gctx_t	*p;
719 		void		*v;
720 	} gctx;
721 	union {
722 		prof_bt_t	*p;
723 		void		*v;
724 	} btkey;
725 	bool new_gctx;
726 
727 	prof_enter(tsd, tdata);
728 	if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
729 		/* bt has never been seen before.  Insert it. */
730 		gctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
731 		if (gctx.v == NULL) {
732 			prof_leave(tsd, tdata);
733 			return (true);
734 		}
735 		btkey.p = &gctx.p->bt;
736 		if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
737 			/* OOM. */
738 			prof_leave(tsd, tdata);
739 			idalloctm(tsd_tsdn(tsd), gctx.v, NULL, true, true);
740 			return (true);
741 		}
742 		new_gctx = true;
743 	} else {
744 		/*
745 		 * Increment nlimbo, in order to avoid a race condition with
746 		 * prof_tctx_destroy()/prof_gctx_try_destroy().
747 		 */
748 		malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
749 		gctx.p->nlimbo++;
750 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
751 		new_gctx = false;
752 	}
753 	prof_leave(tsd, tdata);
754 
755 	*p_btkey = btkey.v;
756 	*p_gctx = gctx.p;
757 	*p_new_gctx = new_gctx;
758 	return (false);
759 }
760 
761 prof_tctx_t *
prof_lookup(tsd_t * tsd,prof_bt_t * bt)762 prof_lookup(tsd_t *tsd, prof_bt_t *bt)
763 {
764 	union {
765 		prof_tctx_t	*p;
766 		void		*v;
767 	} ret;
768 	prof_tdata_t *tdata;
769 	bool not_found;
770 
771 	cassert(config_prof);
772 
773 	tdata = prof_tdata_get(tsd, false);
774 	if (tdata == NULL)
775 		return (NULL);
776 
777 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
778 	not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
779 	if (!not_found) /* Note double negative! */
780 		ret.p->prepared = true;
781 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
782 	if (not_found) {
783 		void *btkey;
784 		prof_gctx_t *gctx;
785 		bool new_gctx, error;
786 
787 		/*
788 		 * This thread's cache lacks bt.  Look for it in the global
789 		 * cache.
790 		 */
791 		if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
792 		    &new_gctx))
793 			return (NULL);
794 
795 		/* Link a prof_tctx_t into gctx for this thread. */
796 		ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
797 		    size2index(sizeof(prof_tctx_t)), false, NULL, true,
798 		    arena_ichoose(tsd, NULL), true);
799 		if (ret.p == NULL) {
800 			if (new_gctx)
801 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
802 			return (NULL);
803 		}
804 		ret.p->tdata = tdata;
805 		ret.p->thr_uid = tdata->thr_uid;
806 		ret.p->thr_discrim = tdata->thr_discrim;
807 		memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
808 		ret.p->gctx = gctx;
809 		ret.p->tctx_uid = tdata->tctx_uid_next++;
810 		ret.p->prepared = true;
811 		ret.p->state = prof_tctx_state_initializing;
812 		malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
813 		error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
814 		malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
815 		if (error) {
816 			if (new_gctx)
817 				prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
818 			idalloctm(tsd_tsdn(tsd), ret.v, NULL, true, true);
819 			return (NULL);
820 		}
821 		malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
822 		ret.p->state = prof_tctx_state_nominal;
823 		tctx_tree_insert(&gctx->tctxs, ret.p);
824 		gctx->nlimbo--;
825 		malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
826 	}
827 
828 	return (ret.p);
829 }
830 
831 /*
832  * The bodies of this function and prof_leakcheck() are compiled out unless heap
833  * profiling is enabled, so that it is possible to compile jemalloc with
834  * floating point support completely disabled.  Avoiding floating point code is
835  * important on memory-constrained systems, but it also enables a workaround for
836  * versions of glibc that don't properly save/restore floating point registers
837  * during dynamic lazy symbol loading (which internally calls into whatever
838  * malloc implementation happens to be integrated into the application).  Note
839  * that some compilers (e.g.  gcc 4.8) may use floating point registers for fast
840  * memory moves, so jemalloc must be compiled with such optimizations disabled
841  * (e.g.
842  * -mno-sse) in order for the workaround to be complete.
843  */
844 void
prof_sample_threshold_update(prof_tdata_t * tdata)845 prof_sample_threshold_update(prof_tdata_t *tdata)
846 {
847 #ifdef JEMALLOC_PROF
848 	uint64_t r;
849 	double u;
850 
851 	if (!config_prof)
852 		return;
853 
854 	if (lg_prof_sample == 0) {
855 		tdata->bytes_until_sample = 0;
856 		return;
857 	}
858 
859 	/*
860 	 * Compute sample interval as a geometrically distributed random
861 	 * variable with mean (2^lg_prof_sample).
862 	 *
863 	 *                             __        __
864 	 *                             |  log(u)  |                     1
865 	 * tdata->bytes_until_sample = | -------- |, where p = ---------------
866 	 *                             | log(1-p) |             lg_prof_sample
867 	 *                                                     2
868 	 *
869 	 * For more information on the math, see:
870 	 *
871 	 *   Non-Uniform Random Variate Generation
872 	 *   Luc Devroye
873 	 *   Springer-Verlag, New York, 1986
874 	 *   pp 500
875 	 *   (http://luc.devroye.org/rnbookindex.html)
876 	 */
877 	r = prng_lg_range_u64(&tdata->prng_state, 53);
878 	u = (double)r * (1.0/9007199254740992.0L);
879 	tdata->bytes_until_sample = (uint64_t)(log(u) /
880 	    log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
881 	    + (uint64_t)1U;
882 #endif
883 }
884 
885 #ifdef JEMALLOC_JET
886 static prof_tdata_t *
prof_tdata_count_iter(prof_tdata_tree_t * tdatas,prof_tdata_t * tdata,void * arg)887 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
888 {
889 	size_t *tdata_count = (size_t *)arg;
890 
891 	(*tdata_count)++;
892 
893 	return (NULL);
894 }
895 
896 size_t
prof_tdata_count(void)897 prof_tdata_count(void)
898 {
899 	size_t tdata_count = 0;
900 	tsdn_t *tsdn;
901 
902 	tsdn = tsdn_fetch();
903 	malloc_mutex_lock(tsdn, &tdatas_mtx);
904 	tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
905 	    (void *)&tdata_count);
906 	malloc_mutex_unlock(tsdn, &tdatas_mtx);
907 
908 	return (tdata_count);
909 }
910 #endif
911 
912 #ifdef JEMALLOC_JET
913 size_t
prof_bt_count(void)914 prof_bt_count(void)
915 {
916 	size_t bt_count;
917 	tsd_t *tsd;
918 	prof_tdata_t *tdata;
919 
920 	tsd = tsd_fetch();
921 	tdata = prof_tdata_get(tsd, false);
922 	if (tdata == NULL)
923 		return (0);
924 
925 	malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
926 	bt_count = ckh_count(&bt2gctx);
927 	malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
928 
929 	return (bt_count);
930 }
931 #endif
932 
933 #ifdef JEMALLOC_JET
934 #undef prof_dump_open
935 #define	prof_dump_open JEMALLOC_N(prof_dump_open_impl)
936 #endif
937 static int
prof_dump_open(bool propagate_err,const char * filename)938 prof_dump_open(bool propagate_err, const char *filename)
939 {
940 	int fd;
941 
942 	fd = creat(filename, 0644);
943 	if (fd == -1 && !propagate_err) {
944 		malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n",
945 		    filename);
946 		if (opt_abort)
947 			abort();
948 	}
949 
950 	return (fd);
951 }
952 #ifdef JEMALLOC_JET
953 #undef prof_dump_open
954 #define	prof_dump_open JEMALLOC_N(prof_dump_open)
955 prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl);
956 #endif
957 
958 static bool
prof_dump_flush(bool propagate_err)959 prof_dump_flush(bool propagate_err)
960 {
961 	bool ret = false;
962 	ssize_t err;
963 
964 	cassert(config_prof);
965 
966 	err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
967 	if (err == -1) {
968 		if (!propagate_err) {
969 			malloc_write("<jemalloc>: write() failed during heap "
970 			    "profile flush\n");
971 			if (opt_abort)
972 				abort();
973 		}
974 		ret = true;
975 	}
976 	prof_dump_buf_end = 0;
977 
978 	return (ret);
979 }
980 
981 static bool
prof_dump_close(bool propagate_err)982 prof_dump_close(bool propagate_err)
983 {
984 	bool ret;
985 
986 	assert(prof_dump_fd != -1);
987 	ret = prof_dump_flush(propagate_err);
988 	close(prof_dump_fd);
989 	prof_dump_fd = -1;
990 
991 	return (ret);
992 }
993 
994 static bool
prof_dump_write(bool propagate_err,const char * s)995 prof_dump_write(bool propagate_err, const char *s)
996 {
997 	size_t i, slen, n;
998 
999 	cassert(config_prof);
1000 
1001 	i = 0;
1002 	slen = strlen(s);
1003 	while (i < slen) {
1004 		/* Flush the buffer if it is full. */
1005 		if (prof_dump_buf_end == PROF_DUMP_BUFSIZE)
1006 			if (prof_dump_flush(propagate_err) && propagate_err)
1007 				return (true);
1008 
1009 		if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
1010 			/* Finish writing. */
1011 			n = slen - i;
1012 		} else {
1013 			/* Write as much of s as will fit. */
1014 			n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
1015 		}
1016 		memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
1017 		prof_dump_buf_end += n;
1018 		i += n;
1019 	}
1020 
1021 	return (false);
1022 }
1023 
1024 JEMALLOC_FORMAT_PRINTF(2, 3)
1025 static bool
prof_dump_printf(bool propagate_err,const char * format,...)1026 prof_dump_printf(bool propagate_err, const char *format, ...)
1027 {
1028 	bool ret;
1029 	va_list ap;
1030 	char buf[PROF_PRINTF_BUFSIZE];
1031 
1032 	va_start(ap, format);
1033 	malloc_vsnprintf(buf, sizeof(buf), format, ap);
1034 	va_end(ap);
1035 	ret = prof_dump_write(propagate_err, buf);
1036 
1037 	return (ret);
1038 }
1039 
1040 static void
prof_tctx_merge_tdata(tsdn_t * tsdn,prof_tctx_t * tctx,prof_tdata_t * tdata)1041 prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata)
1042 {
1043 
1044 	malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
1045 
1046 	malloc_mutex_lock(tsdn, tctx->gctx->lock);
1047 
1048 	switch (tctx->state) {
1049 	case prof_tctx_state_initializing:
1050 		malloc_mutex_unlock(tsdn, tctx->gctx->lock);
1051 		return;
1052 	case prof_tctx_state_nominal:
1053 		tctx->state = prof_tctx_state_dumping;
1054 		malloc_mutex_unlock(tsdn, tctx->gctx->lock);
1055 
1056 		memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
1057 
1058 		tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1059 		tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1060 		if (opt_prof_accum) {
1061 			tdata->cnt_summed.accumobjs +=
1062 			    tctx->dump_cnts.accumobjs;
1063 			tdata->cnt_summed.accumbytes +=
1064 			    tctx->dump_cnts.accumbytes;
1065 		}
1066 		break;
1067 	case prof_tctx_state_dumping:
1068 	case prof_tctx_state_purgatory:
1069 		not_reached();
1070 	}
1071 }
1072 
1073 static void
prof_tctx_merge_gctx(tsdn_t * tsdn,prof_tctx_t * tctx,prof_gctx_t * gctx)1074 prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx)
1075 {
1076 
1077 	malloc_mutex_assert_owner(tsdn, gctx->lock);
1078 
1079 	gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1080 	gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1081 	if (opt_prof_accum) {
1082 		gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
1083 		gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
1084 	}
1085 }
1086 
1087 static prof_tctx_t *
prof_tctx_merge_iter(prof_tctx_tree_t * tctxs,prof_tctx_t * tctx,void * arg)1088 prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1089 {
1090 	tsdn_t *tsdn = (tsdn_t *)arg;
1091 
1092 	malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
1093 
1094 	switch (tctx->state) {
1095 	case prof_tctx_state_nominal:
1096 		/* New since dumping started; ignore. */
1097 		break;
1098 	case prof_tctx_state_dumping:
1099 	case prof_tctx_state_purgatory:
1100 		prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
1101 		break;
1102 	default:
1103 		not_reached();
1104 	}
1105 
1106 	return (NULL);
1107 }
1108 
1109 struct prof_tctx_dump_iter_arg_s {
1110 	tsdn_t	*tsdn;
1111 	bool	propagate_err;
1112 };
1113 
1114 static prof_tctx_t *
prof_tctx_dump_iter(prof_tctx_tree_t * tctxs,prof_tctx_t * tctx,void * opaque)1115 prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque)
1116 {
1117 	struct prof_tctx_dump_iter_arg_s *arg =
1118 	    (struct prof_tctx_dump_iter_arg_s *)opaque;
1119 
1120 	malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);
1121 
1122 	switch (tctx->state) {
1123 	case prof_tctx_state_initializing:
1124 	case prof_tctx_state_nominal:
1125 		/* Not captured by this dump. */
1126 		break;
1127 	case prof_tctx_state_dumping:
1128 	case prof_tctx_state_purgatory:
1129 		if (prof_dump_printf(arg->propagate_err,
1130 		    "  t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
1131 		    "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
1132 		    tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
1133 		    tctx->dump_cnts.accumbytes))
1134 			return (tctx);
1135 		break;
1136 	default:
1137 		not_reached();
1138 	}
1139 	return (NULL);
1140 }
1141 
1142 static prof_tctx_t *
prof_tctx_finish_iter(prof_tctx_tree_t * tctxs,prof_tctx_t * tctx,void * arg)1143 prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1144 {
1145 	tsdn_t *tsdn = (tsdn_t *)arg;
1146 	prof_tctx_t *ret;
1147 
1148 	malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
1149 
1150 	switch (tctx->state) {
1151 	case prof_tctx_state_nominal:
1152 		/* New since dumping started; ignore. */
1153 		break;
1154 	case prof_tctx_state_dumping:
1155 		tctx->state = prof_tctx_state_nominal;
1156 		break;
1157 	case prof_tctx_state_purgatory:
1158 		ret = tctx;
1159 		goto label_return;
1160 	default:
1161 		not_reached();
1162 	}
1163 
1164 	ret = NULL;
1165 label_return:
1166 	return (ret);
1167 }
1168 
1169 static void
prof_dump_gctx_prep(tsdn_t * tsdn,prof_gctx_t * gctx,prof_gctx_tree_t * gctxs)1170 prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
1171 {
1172 
1173 	cassert(config_prof);
1174 
1175 	malloc_mutex_lock(tsdn, gctx->lock);
1176 
1177 	/*
1178 	 * Increment nlimbo so that gctx won't go away before dump.
1179 	 * Additionally, link gctx into the dump list so that it is included in
1180 	 * prof_dump()'s second pass.
1181 	 */
1182 	gctx->nlimbo++;
1183 	gctx_tree_insert(gctxs, gctx);
1184 
1185 	memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
1186 
1187 	malloc_mutex_unlock(tsdn, gctx->lock);
1188 }
1189 
1190 struct prof_gctx_merge_iter_arg_s {
1191 	tsdn_t	*tsdn;
1192 	size_t	leak_ngctx;
1193 };
1194 
1195 static prof_gctx_t *
prof_gctx_merge_iter(prof_gctx_tree_t * gctxs,prof_gctx_t * gctx,void * opaque)1196 prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque)
1197 {
1198 	struct prof_gctx_merge_iter_arg_s *arg =
1199 	    (struct prof_gctx_merge_iter_arg_s *)opaque;
1200 
1201 	malloc_mutex_lock(arg->tsdn, gctx->lock);
1202 	tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
1203 	    (void *)arg->tsdn);
1204 	if (gctx->cnt_summed.curobjs != 0)
1205 		arg->leak_ngctx++;
1206 	malloc_mutex_unlock(arg->tsdn, gctx->lock);
1207 
1208 	return (NULL);
1209 }
1210 
1211 static void
prof_gctx_finish(tsd_t * tsd,prof_gctx_tree_t * gctxs)1212 prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
1213 {
1214 	prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1215 	prof_gctx_t *gctx;
1216 
1217 	/*
1218 	 * Standard tree iteration won't work here, because as soon as we
1219 	 * decrement gctx->nlimbo and unlock gctx, another thread can
1220 	 * concurrently destroy it, which will corrupt the tree.  Therefore,
1221 	 * tear down the tree one node at a time during iteration.
1222 	 */
1223 	while ((gctx = gctx_tree_first(gctxs)) != NULL) {
1224 		gctx_tree_remove(gctxs, gctx);
1225 		malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
1226 		{
1227 			prof_tctx_t *next;
1228 
1229 			next = NULL;
1230 			do {
1231 				prof_tctx_t *to_destroy =
1232 				    tctx_tree_iter(&gctx->tctxs, next,
1233 				    prof_tctx_finish_iter,
1234 				    (void *)tsd_tsdn(tsd));
1235 				if (to_destroy != NULL) {
1236 					next = tctx_tree_next(&gctx->tctxs,
1237 					    to_destroy);
1238 					tctx_tree_remove(&gctx->tctxs,
1239 					    to_destroy);
1240 					idalloctm(tsd_tsdn(tsd), to_destroy,
1241 					    NULL, true, true);
1242 				} else
1243 					next = NULL;
1244 			} while (next != NULL);
1245 		}
1246 		gctx->nlimbo--;
1247 		if (prof_gctx_should_destroy(gctx)) {
1248 			gctx->nlimbo++;
1249 			malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
1250 			prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
1251 		} else
1252 			malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
1253 	}
1254 }
1255 
1256 struct prof_tdata_merge_iter_arg_s {
1257 	tsdn_t		*tsdn;
1258 	prof_cnt_t	cnt_all;
1259 };
1260 
1261 static prof_tdata_t *
prof_tdata_merge_iter(prof_tdata_tree_t * tdatas,prof_tdata_t * tdata,void * opaque)1262 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
1263     void *opaque)
1264 {
1265 	struct prof_tdata_merge_iter_arg_s *arg =
1266 	    (struct prof_tdata_merge_iter_arg_s *)opaque;
1267 
1268 	malloc_mutex_lock(arg->tsdn, tdata->lock);
1269 	if (!tdata->expired) {
1270 		size_t tabind;
1271 		union {
1272 			prof_tctx_t	*p;
1273 			void		*v;
1274 		} tctx;
1275 
1276 		tdata->dumping = true;
1277 		memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
1278 		for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
1279 		    &tctx.v);)
1280 			prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
1281 
1282 		arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;
1283 		arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;
1284 		if (opt_prof_accum) {
1285 			arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;
1286 			arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;
1287 		}
1288 	} else
1289 		tdata->dumping = false;
1290 	malloc_mutex_unlock(arg->tsdn, tdata->lock);
1291 
1292 	return (NULL);
1293 }
1294 
1295 static prof_tdata_t *
prof_tdata_dump_iter(prof_tdata_tree_t * tdatas,prof_tdata_t * tdata,void * arg)1296 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1297 {
1298 	bool propagate_err = *(bool *)arg;
1299 
1300 	if (!tdata->dumping)
1301 		return (NULL);
1302 
1303 	if (prof_dump_printf(propagate_err,
1304 	    "  t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
1305 	    tdata->thr_uid, tdata->cnt_summed.curobjs,
1306 	    tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
1307 	    tdata->cnt_summed.accumbytes,
1308 	    (tdata->thread_name != NULL) ? " " : "",
1309 	    (tdata->thread_name != NULL) ? tdata->thread_name : ""))
1310 		return (tdata);
1311 	return (NULL);
1312 }
1313 
1314 #ifdef JEMALLOC_JET
1315 #undef prof_dump_header
1316 #define	prof_dump_header JEMALLOC_N(prof_dump_header_impl)
1317 #endif
1318 static bool
prof_dump_header(tsdn_t * tsdn,bool propagate_err,const prof_cnt_t * cnt_all)1319 prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all)
1320 {
1321 	bool ret;
1322 
1323 	if (prof_dump_printf(propagate_err,
1324 	    "heap_v2/%"FMTu64"\n"
1325 	    "  t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1326 	    ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
1327 	    cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes))
1328 		return (true);
1329 
1330 	malloc_mutex_lock(tsdn, &tdatas_mtx);
1331 	ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
1332 	    (void *)&propagate_err) != NULL);
1333 	malloc_mutex_unlock(tsdn, &tdatas_mtx);
1334 	return (ret);
1335 }
1336 #ifdef JEMALLOC_JET
1337 #undef prof_dump_header
1338 #define	prof_dump_header JEMALLOC_N(prof_dump_header)
1339 prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
1340 #endif
1341 
1342 static bool
prof_dump_gctx(tsdn_t * tsdn,bool propagate_err,prof_gctx_t * gctx,const prof_bt_t * bt,prof_gctx_tree_t * gctxs)1343 prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx,
1344     const prof_bt_t *bt, prof_gctx_tree_t *gctxs)
1345 {
1346 	bool ret;
1347 	unsigned i;
1348 	struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg;
1349 
1350 	cassert(config_prof);
1351 	malloc_mutex_assert_owner(tsdn, gctx->lock);
1352 
1353 	/* Avoid dumping such gctx's that have no useful data. */
1354 	if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
1355 	    (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
1356 		assert(gctx->cnt_summed.curobjs == 0);
1357 		assert(gctx->cnt_summed.curbytes == 0);
1358 		assert(gctx->cnt_summed.accumobjs == 0);
1359 		assert(gctx->cnt_summed.accumbytes == 0);
1360 		ret = false;
1361 		goto label_return;
1362 	}
1363 
1364 	if (prof_dump_printf(propagate_err, "@")) {
1365 		ret = true;
1366 		goto label_return;
1367 	}
1368 	for (i = 0; i < bt->len; i++) {
1369 		if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
1370 		    (uintptr_t)bt->vec[i])) {
1371 			ret = true;
1372 			goto label_return;
1373 		}
1374 	}
1375 
1376 	if (prof_dump_printf(propagate_err,
1377 	    "\n"
1378 	    "  t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1379 	    gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
1380 	    gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
1381 		ret = true;
1382 		goto label_return;
1383 	}
1384 
1385 	prof_tctx_dump_iter_arg.tsdn = tsdn;
1386 	prof_tctx_dump_iter_arg.propagate_err = propagate_err;
1387 	if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
1388 	    (void *)&prof_tctx_dump_iter_arg) != NULL) {
1389 		ret = true;
1390 		goto label_return;
1391 	}
1392 
1393 	ret = false;
1394 label_return:
1395 	return (ret);
1396 }
1397 
1398 #ifndef _WIN32
1399 JEMALLOC_FORMAT_PRINTF(1, 2)
1400 static int
prof_open_maps(const char * format,...)1401 prof_open_maps(const char *format, ...)
1402 {
1403 	int mfd;
1404 	va_list ap;
1405 	char filename[PATH_MAX + 1];
1406 
1407 	va_start(ap, format);
1408 	malloc_vsnprintf(filename, sizeof(filename), format, ap);
1409 	va_end(ap);
1410 	mfd = open(filename, O_RDONLY);
1411 
1412 	return (mfd);
1413 }
1414 #endif
1415 
1416 static int
prof_getpid(void)1417 prof_getpid(void)
1418 {
1419 
1420 #ifdef _WIN32
1421 	return (GetCurrentProcessId());
1422 #else
1423 	return (getpid());
1424 #endif
1425 }
1426 
1427 static bool
prof_dump_maps(bool propagate_err)1428 prof_dump_maps(bool propagate_err)
1429 {
1430 	bool ret;
1431 	int mfd;
1432 
1433 	cassert(config_prof);
1434 #ifdef __FreeBSD__
1435 	mfd = prof_open_maps("/proc/curproc/map");
1436 #elif defined(_WIN32)
1437 	mfd = -1; // Not implemented
1438 #else
1439 	{
1440 		int pid = prof_getpid();
1441 
1442 		mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
1443 		if (mfd == -1)
1444 			mfd = prof_open_maps("/proc/%d/maps", pid);
1445 	}
1446 #endif
1447 	if (mfd != -1) {
1448 		ssize_t nread;
1449 
1450 		if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
1451 		    propagate_err) {
1452 			ret = true;
1453 			goto label_return;
1454 		}
1455 		nread = 0;
1456 		do {
1457 			prof_dump_buf_end += nread;
1458 			if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
1459 				/* Make space in prof_dump_buf before read(). */
1460 				if (prof_dump_flush(propagate_err) &&
1461 				    propagate_err) {
1462 					ret = true;
1463 					goto label_return;
1464 				}
1465 			}
1466 			nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
1467 			    PROF_DUMP_BUFSIZE - prof_dump_buf_end);
1468 		} while (nread > 0);
1469 	} else {
1470 		ret = true;
1471 		goto label_return;
1472 	}
1473 
1474 	ret = false;
1475 label_return:
1476 	if (mfd != -1)
1477 		close(mfd);
1478 	return (ret);
1479 }
1480 
1481 /*
1482  * See prof_sample_threshold_update() comment for why the body of this function
1483  * is conditionally compiled.
1484  */
1485 static void
prof_leakcheck(const prof_cnt_t * cnt_all,size_t leak_ngctx,const char * filename)1486 prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
1487     const char *filename)
1488 {
1489 
1490 #ifdef JEMALLOC_PROF
1491 	/*
1492 	 * Scaling is equivalent AdjustSamples() in jeprof, but the result may
1493 	 * differ slightly from what jeprof reports, because here we scale the
1494 	 * summary values, whereas jeprof scales each context individually and
1495 	 * reports the sums of the scaled values.
1496 	 */
1497 	if (cnt_all->curbytes != 0) {
1498 		double sample_period = (double)((uint64_t)1 << lg_prof_sample);
1499 		double ratio = (((double)cnt_all->curbytes) /
1500 		    (double)cnt_all->curobjs) / sample_period;
1501 		double scale_factor = 1.0 / (1.0 - exp(-ratio));
1502 		uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
1503 		    * scale_factor);
1504 		uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
1505 		    scale_factor);
1506 
1507 		malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64
1508 		    " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
1509 		    curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
1510 		    1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1511 		malloc_printf(
1512 		    "<jemalloc>: Run jeprof on \"%s\" for leak detail\n",
1513 		    filename);
1514 	}
1515 #endif
1516 }
1517 
1518 struct prof_gctx_dump_iter_arg_s {
1519 	tsdn_t	*tsdn;
1520 	bool	propagate_err;
1521 };
1522 
1523 static prof_gctx_t *
prof_gctx_dump_iter(prof_gctx_tree_t * gctxs,prof_gctx_t * gctx,void * opaque)1524 prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque)
1525 {
1526 	prof_gctx_t *ret;
1527 	struct prof_gctx_dump_iter_arg_s *arg =
1528 	    (struct prof_gctx_dump_iter_arg_s *)opaque;
1529 
1530 	malloc_mutex_lock(arg->tsdn, gctx->lock);
1531 
1532 	if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt,
1533 	    gctxs)) {
1534 		ret = gctx;
1535 		goto label_return;
1536 	}
1537 
1538 	ret = NULL;
1539 label_return:
1540 	malloc_mutex_unlock(arg->tsdn, gctx->lock);
1541 	return (ret);
1542 }
1543 
1544 static bool
prof_dump(tsd_t * tsd,bool propagate_err,const char * filename,bool leakcheck)1545 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
1546 {
1547 	prof_tdata_t *tdata;
1548 	struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
1549 	size_t tabind;
1550 	union {
1551 		prof_gctx_t	*p;
1552 		void		*v;
1553 	} gctx;
1554 	struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
1555 	struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg;
1556 	prof_gctx_tree_t gctxs;
1557 
1558 	cassert(config_prof);
1559 
1560 	tdata = prof_tdata_get(tsd, true);
1561 	if (tdata == NULL)
1562 		return (true);
1563 
1564 	malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
1565 	prof_enter(tsd, tdata);
1566 
1567 	/*
1568 	 * Put gctx's in limbo and clear their counters in preparation for
1569 	 * summing.
1570 	 */
1571 	gctx_tree_new(&gctxs);
1572 	for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);)
1573 		prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, &gctxs);
1574 
1575 	/*
1576 	 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1577 	 * stats and merge them into the associated gctx's.
1578 	 */
1579 	prof_tdata_merge_iter_arg.tsdn = tsd_tsdn(tsd);
1580 	memset(&prof_tdata_merge_iter_arg.cnt_all, 0, sizeof(prof_cnt_t));
1581 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1582 	tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
1583 	    (void *)&prof_tdata_merge_iter_arg);
1584 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1585 
1586 	/* Merge tctx stats into gctx's. */
1587 	prof_gctx_merge_iter_arg.tsdn = tsd_tsdn(tsd);
1588 	prof_gctx_merge_iter_arg.leak_ngctx = 0;
1589 	gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter,
1590 	    (void *)&prof_gctx_merge_iter_arg);
1591 
1592 	prof_leave(tsd, tdata);
1593 
1594 	/* Create dump file. */
1595 	if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1)
1596 		goto label_open_close_error;
1597 
1598 	/* Dump profile header. */
1599 	if (prof_dump_header(tsd_tsdn(tsd), propagate_err,
1600 	    &prof_tdata_merge_iter_arg.cnt_all))
1601 		goto label_write_error;
1602 
1603 	/* Dump per gctx profile stats. */
1604 	prof_gctx_dump_iter_arg.tsdn = tsd_tsdn(tsd);
1605 	prof_gctx_dump_iter_arg.propagate_err = propagate_err;
1606 	if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter,
1607 	    (void *)&prof_gctx_dump_iter_arg) != NULL)
1608 		goto label_write_error;
1609 
1610 	/* Dump /proc/<pid>/maps if possible. */
1611 	if (prof_dump_maps(propagate_err))
1612 		goto label_write_error;
1613 
1614 	if (prof_dump_close(propagate_err))
1615 		goto label_open_close_error;
1616 
1617 	prof_gctx_finish(tsd, &gctxs);
1618 	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
1619 
1620 	if (leakcheck) {
1621 		prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all,
1622 		    prof_gctx_merge_iter_arg.leak_ngctx, filename);
1623 	}
1624 	return (false);
1625 label_write_error:
1626 	prof_dump_close(propagate_err);
1627 label_open_close_error:
1628 	prof_gctx_finish(tsd, &gctxs);
1629 	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
1630 	return (true);
1631 }
1632 
1633 #define	DUMP_FILENAME_BUFSIZE	(PATH_MAX + 1)
1634 #define	VSEQ_INVALID		UINT64_C(0xffffffffffffffff)
1635 static void
prof_dump_filename(char * filename,char v,uint64_t vseq)1636 prof_dump_filename(char *filename, char v, uint64_t vseq)
1637 {
1638 
1639 	cassert(config_prof);
1640 
1641 	if (vseq != VSEQ_INVALID) {
1642 	        /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
1643 		malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1644 		    "%s.%d.%"FMTu64".%c%"FMTu64".heap",
1645 		    opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq);
1646 	} else {
1647 	        /* "<prefix>.<pid>.<seq>.<v>.heap" */
1648 		malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1649 		    "%s.%d.%"FMTu64".%c.heap",
1650 		    opt_prof_prefix, prof_getpid(), prof_dump_seq, v);
1651 	}
1652 	prof_dump_seq++;
1653 }
1654 
1655 static void
prof_fdump(void)1656 prof_fdump(void)
1657 {
1658 	tsd_t *tsd;
1659 	char filename[DUMP_FILENAME_BUFSIZE];
1660 
1661 	cassert(config_prof);
1662 	assert(opt_prof_final);
1663 	assert(opt_prof_prefix[0] != '\0');
1664 
1665 	if (!prof_booted)
1666 		return;
1667 	tsd = tsd_fetch();
1668 
1669 	malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1670 	prof_dump_filename(filename, 'f', VSEQ_INVALID);
1671 	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1672 	prof_dump(tsd, false, filename, opt_prof_leak);
1673 }
1674 
1675 void
prof_idump(tsdn_t * tsdn)1676 prof_idump(tsdn_t *tsdn)
1677 {
1678 	tsd_t *tsd;
1679 	prof_tdata_t *tdata;
1680 
1681 	cassert(config_prof);
1682 
1683 	if (!prof_booted || tsdn_null(tsdn))
1684 		return;
1685 	tsd = tsdn_tsd(tsdn);
1686 	tdata = prof_tdata_get(tsd, false);
1687 	if (tdata == NULL)
1688 		return;
1689 	if (tdata->enq) {
1690 		tdata->enq_idump = true;
1691 		return;
1692 	}
1693 
1694 	if (opt_prof_prefix[0] != '\0') {
1695 		char filename[PATH_MAX + 1];
1696 		malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1697 		prof_dump_filename(filename, 'i', prof_dump_iseq);
1698 		prof_dump_iseq++;
1699 		malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1700 		prof_dump(tsd, false, filename, false);
1701 	}
1702 }
1703 
1704 bool
prof_mdump(tsd_t * tsd,const char * filename)1705 prof_mdump(tsd_t *tsd, const char *filename)
1706 {
1707 	char filename_buf[DUMP_FILENAME_BUFSIZE];
1708 
1709 	cassert(config_prof);
1710 
1711 	if (!opt_prof || !prof_booted)
1712 		return (true);
1713 
1714 	if (filename == NULL) {
1715 		/* No filename specified, so automatically generate one. */
1716 		if (opt_prof_prefix[0] == '\0')
1717 			return (true);
1718 		malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1719 		prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
1720 		prof_dump_mseq++;
1721 		malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1722 		filename = filename_buf;
1723 	}
1724 	return (prof_dump(tsd, true, filename, false));
1725 }
1726 
1727 void
prof_gdump(tsdn_t * tsdn)1728 prof_gdump(tsdn_t *tsdn)
1729 {
1730 	tsd_t *tsd;
1731 	prof_tdata_t *tdata;
1732 
1733 	cassert(config_prof);
1734 
1735 	if (!prof_booted || tsdn_null(tsdn))
1736 		return;
1737 	tsd = tsdn_tsd(tsdn);
1738 	tdata = prof_tdata_get(tsd, false);
1739 	if (tdata == NULL)
1740 		return;
1741 	if (tdata->enq) {
1742 		tdata->enq_gdump = true;
1743 		return;
1744 	}
1745 
1746 	if (opt_prof_prefix[0] != '\0') {
1747 		char filename[DUMP_FILENAME_BUFSIZE];
1748 		malloc_mutex_lock(tsdn, &prof_dump_seq_mtx);
1749 		prof_dump_filename(filename, 'u', prof_dump_useq);
1750 		prof_dump_useq++;
1751 		malloc_mutex_unlock(tsdn, &prof_dump_seq_mtx);
1752 		prof_dump(tsd, false, filename, false);
1753 	}
1754 }
1755 
1756 static void
prof_bt_hash(const void * key,size_t r_hash[2])1757 prof_bt_hash(const void *key, size_t r_hash[2])
1758 {
1759 	prof_bt_t *bt = (prof_bt_t *)key;
1760 
1761 	cassert(config_prof);
1762 
1763 	hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1764 }
1765 
1766 static bool
prof_bt_keycomp(const void * k1,const void * k2)1767 prof_bt_keycomp(const void *k1, const void *k2)
1768 {
1769 	const prof_bt_t *bt1 = (prof_bt_t *)k1;
1770 	const prof_bt_t *bt2 = (prof_bt_t *)k2;
1771 
1772 	cassert(config_prof);
1773 
1774 	if (bt1->len != bt2->len)
1775 		return (false);
1776 	return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1777 }
1778 
1779 JEMALLOC_INLINE_C uint64_t
prof_thr_uid_alloc(tsdn_t * tsdn)1780 prof_thr_uid_alloc(tsdn_t *tsdn)
1781 {
1782 	uint64_t thr_uid;
1783 
1784 	malloc_mutex_lock(tsdn, &next_thr_uid_mtx);
1785 	thr_uid = next_thr_uid;
1786 	next_thr_uid++;
1787 	malloc_mutex_unlock(tsdn, &next_thr_uid_mtx);
1788 
1789 	return (thr_uid);
1790 }
1791 
1792 static prof_tdata_t *
prof_tdata_init_impl(tsd_t * tsd,uint64_t thr_uid,uint64_t thr_discrim,char * thread_name,bool active)1793 prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1794     char *thread_name, bool active)
1795 {
1796 	prof_tdata_t *tdata;
1797 
1798 	cassert(config_prof);
1799 
1800 	/* Initialize an empty cache for this thread. */
1801 	tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
1802 	    size2index(sizeof(prof_tdata_t)), false, NULL, true,
1803 	    arena_get(TSDN_NULL, 0, true), true);
1804 	if (tdata == NULL)
1805 		return (NULL);
1806 
1807 	tdata->lock = prof_tdata_mutex_choose(thr_uid);
1808 	tdata->thr_uid = thr_uid;
1809 	tdata->thr_discrim = thr_discrim;
1810 	tdata->thread_name = thread_name;
1811 	tdata->attached = true;
1812 	tdata->expired = false;
1813 	tdata->tctx_uid_next = 0;
1814 
1815 	if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
1816 	    prof_bt_keycomp)) {
1817 		idalloctm(tsd_tsdn(tsd), tdata, NULL, true, true);
1818 		return (NULL);
1819 	}
1820 
1821 	tdata->prng_state = (uint64_t)(uintptr_t)tdata;
1822 	prof_sample_threshold_update(tdata);
1823 
1824 	tdata->enq = false;
1825 	tdata->enq_idump = false;
1826 	tdata->enq_gdump = false;
1827 
1828 	tdata->dumping = false;
1829 	tdata->active = active;
1830 
1831 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1832 	tdata_tree_insert(&tdatas, tdata);
1833 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1834 
1835 	return (tdata);
1836 }
1837 
1838 prof_tdata_t *
prof_tdata_init(tsd_t * tsd)1839 prof_tdata_init(tsd_t *tsd)
1840 {
1841 
1842 	return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0,
1843 	    NULL, prof_thread_active_init_get(tsd_tsdn(tsd))));
1844 }
1845 
1846 static bool
prof_tdata_should_destroy_unlocked(prof_tdata_t * tdata,bool even_if_attached)1847 prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached)
1848 {
1849 
1850 	if (tdata->attached && !even_if_attached)
1851 		return (false);
1852 	if (ckh_count(&tdata->bt2tctx) != 0)
1853 		return (false);
1854 	return (true);
1855 }
1856 
1857 static bool
prof_tdata_should_destroy(tsdn_t * tsdn,prof_tdata_t * tdata,bool even_if_attached)1858 prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
1859     bool even_if_attached)
1860 {
1861 
1862 	malloc_mutex_assert_owner(tsdn, tdata->lock);
1863 
1864 	return (prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
1865 }
1866 
1867 static void
prof_tdata_destroy_locked(tsd_t * tsd,prof_tdata_t * tdata,bool even_if_attached)1868 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1869     bool even_if_attached)
1870 {
1871 
1872 	malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
1873 
1874 	tdata_tree_remove(&tdatas, tdata);
1875 
1876 	assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
1877 
1878 	if (tdata->thread_name != NULL)
1879 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, true, true);
1880 	ckh_delete(tsd, &tdata->bt2tctx);
1881 	idalloctm(tsd_tsdn(tsd), tdata, NULL, true, true);
1882 }
1883 
1884 static void
prof_tdata_destroy(tsd_t * tsd,prof_tdata_t * tdata,bool even_if_attached)1885 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached)
1886 {
1887 
1888 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1889 	prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1890 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1891 }
1892 
1893 static void
prof_tdata_detach(tsd_t * tsd,prof_tdata_t * tdata)1894 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
1895 {
1896 	bool destroy_tdata;
1897 
1898 	malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
1899 	if (tdata->attached) {
1900 		destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
1901 		    true);
1902 		/*
1903 		 * Only detach if !destroy_tdata, because detaching would allow
1904 		 * another thread to win the race to destroy tdata.
1905 		 */
1906 		if (!destroy_tdata)
1907 			tdata->attached = false;
1908 		tsd_prof_tdata_set(tsd, NULL);
1909 	} else
1910 		destroy_tdata = false;
1911 	malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
1912 	if (destroy_tdata)
1913 		prof_tdata_destroy(tsd, tdata, true);
1914 }
1915 
1916 prof_tdata_t *
prof_tdata_reinit(tsd_t * tsd,prof_tdata_t * tdata)1917 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
1918 {
1919 	uint64_t thr_uid = tdata->thr_uid;
1920 	uint64_t thr_discrim = tdata->thr_discrim + 1;
1921 	char *thread_name = (tdata->thread_name != NULL) ?
1922 	    prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL;
1923 	bool active = tdata->active;
1924 
1925 	prof_tdata_detach(tsd, tdata);
1926 	return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
1927 	    active));
1928 }
1929 
1930 static bool
prof_tdata_expire(tsdn_t * tsdn,prof_tdata_t * tdata)1931 prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata)
1932 {
1933 	bool destroy_tdata;
1934 
1935 	malloc_mutex_lock(tsdn, tdata->lock);
1936 	if (!tdata->expired) {
1937 		tdata->expired = true;
1938 		destroy_tdata = tdata->attached ? false :
1939 		    prof_tdata_should_destroy(tsdn, tdata, false);
1940 	} else
1941 		destroy_tdata = false;
1942 	malloc_mutex_unlock(tsdn, tdata->lock);
1943 
1944 	return (destroy_tdata);
1945 }
1946 
1947 static prof_tdata_t *
prof_tdata_reset_iter(prof_tdata_tree_t * tdatas,prof_tdata_t * tdata,void * arg)1948 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1949 {
1950 	tsdn_t *tsdn = (tsdn_t *)arg;
1951 
1952 	return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
1953 }
1954 
1955 void
prof_reset(tsd_t * tsd,size_t lg_sample)1956 prof_reset(tsd_t *tsd, size_t lg_sample)
1957 {
1958 	prof_tdata_t *next;
1959 
1960 	assert(lg_sample < (sizeof(uint64_t) << 3));
1961 
1962 	malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
1963 	malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1964 
1965 	lg_prof_sample = lg_sample;
1966 
1967 	next = NULL;
1968 	do {
1969 		prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
1970 		    prof_tdata_reset_iter, (void *)tsd);
1971 		if (to_destroy != NULL) {
1972 			next = tdata_tree_next(&tdatas, to_destroy);
1973 			prof_tdata_destroy_locked(tsd, to_destroy, false);
1974 		} else
1975 			next = NULL;
1976 	} while (next != NULL);
1977 
1978 	malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1979 	malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
1980 }
1981 
1982 void
prof_tdata_cleanup(tsd_t * tsd)1983 prof_tdata_cleanup(tsd_t *tsd)
1984 {
1985 	prof_tdata_t *tdata;
1986 
1987 	if (!config_prof)
1988 		return;
1989 
1990 	tdata = tsd_prof_tdata_get(tsd);
1991 	if (tdata != NULL)
1992 		prof_tdata_detach(tsd, tdata);
1993 }
1994 
1995 bool
prof_active_get(tsdn_t * tsdn)1996 prof_active_get(tsdn_t *tsdn)
1997 {
1998 	bool prof_active_current;
1999 
2000 	malloc_mutex_lock(tsdn, &prof_active_mtx);
2001 	prof_active_current = prof_active;
2002 	malloc_mutex_unlock(tsdn, &prof_active_mtx);
2003 	return (prof_active_current);
2004 }
2005 
2006 bool
prof_active_set(tsdn_t * tsdn,bool active)2007 prof_active_set(tsdn_t *tsdn, bool active)
2008 {
2009 	bool prof_active_old;
2010 
2011 	malloc_mutex_lock(tsdn, &prof_active_mtx);
2012 	prof_active_old = prof_active;
2013 	prof_active = active;
2014 	malloc_mutex_unlock(tsdn, &prof_active_mtx);
2015 	return (prof_active_old);
2016 }
2017 
2018 const char *
prof_thread_name_get(tsd_t * tsd)2019 prof_thread_name_get(tsd_t *tsd)
2020 {
2021 	prof_tdata_t *tdata;
2022 
2023 	tdata = prof_tdata_get(tsd, true);
2024 	if (tdata == NULL)
2025 		return ("");
2026 	return (tdata->thread_name != NULL ? tdata->thread_name : "");
2027 }
2028 
2029 static char *
prof_thread_name_alloc(tsdn_t * tsdn,const char * thread_name)2030 prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name)
2031 {
2032 	char *ret;
2033 	size_t size;
2034 
2035 	if (thread_name == NULL)
2036 		return (NULL);
2037 
2038 	size = strlen(thread_name) + 1;
2039 	if (size == 1)
2040 		return ("");
2041 
2042 	ret = iallocztm(tsdn, size, size2index(size), false, NULL, true,
2043 	    arena_get(TSDN_NULL, 0, true), true);
2044 	if (ret == NULL)
2045 		return (NULL);
2046 	memcpy(ret, thread_name, size);
2047 	return (ret);
2048 }
2049 
2050 int
prof_thread_name_set(tsd_t * tsd,const char * thread_name)2051 prof_thread_name_set(tsd_t *tsd, const char *thread_name)
2052 {
2053 	prof_tdata_t *tdata;
2054 	unsigned i;
2055 	char *s;
2056 
2057 	tdata = prof_tdata_get(tsd, true);
2058 	if (tdata == NULL)
2059 		return (EAGAIN);
2060 
2061 	/* Validate input. */
2062 	if (thread_name == NULL)
2063 		return (EFAULT);
2064 	for (i = 0; thread_name[i] != '\0'; i++) {
2065 		char c = thread_name[i];
2066 		if (!isgraph(c) && !isblank(c))
2067 			return (EFAULT);
2068 	}
2069 
2070 	s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name);
2071 	if (s == NULL)
2072 		return (EAGAIN);
2073 
2074 	if (tdata->thread_name != NULL) {
2075 		idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, true, true);
2076 		tdata->thread_name = NULL;
2077 	}
2078 	if (strlen(s) > 0)
2079 		tdata->thread_name = s;
2080 	return (0);
2081 }
2082 
2083 bool
prof_thread_active_get(tsd_t * tsd)2084 prof_thread_active_get(tsd_t *tsd)
2085 {
2086 	prof_tdata_t *tdata;
2087 
2088 	tdata = prof_tdata_get(tsd, true);
2089 	if (tdata == NULL)
2090 		return (false);
2091 	return (tdata->active);
2092 }
2093 
2094 bool
prof_thread_active_set(tsd_t * tsd,bool active)2095 prof_thread_active_set(tsd_t *tsd, bool active)
2096 {
2097 	prof_tdata_t *tdata;
2098 
2099 	tdata = prof_tdata_get(tsd, true);
2100 	if (tdata == NULL)
2101 		return (true);
2102 	tdata->active = active;
2103 	return (false);
2104 }
2105 
2106 bool
prof_thread_active_init_get(tsdn_t * tsdn)2107 prof_thread_active_init_get(tsdn_t *tsdn)
2108 {
2109 	bool active_init;
2110 
2111 	malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx);
2112 	active_init = prof_thread_active_init;
2113 	malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx);
2114 	return (active_init);
2115 }
2116 
2117 bool
prof_thread_active_init_set(tsdn_t * tsdn,bool active_init)2118 prof_thread_active_init_set(tsdn_t *tsdn, bool active_init)
2119 {
2120 	bool active_init_old;
2121 
2122 	malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx);
2123 	active_init_old = prof_thread_active_init;
2124 	prof_thread_active_init = active_init;
2125 	malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx);
2126 	return (active_init_old);
2127 }
2128 
2129 bool
prof_gdump_get(tsdn_t * tsdn)2130 prof_gdump_get(tsdn_t *tsdn)
2131 {
2132 	bool prof_gdump_current;
2133 
2134 	malloc_mutex_lock(tsdn, &prof_gdump_mtx);
2135 	prof_gdump_current = prof_gdump_val;
2136 	malloc_mutex_unlock(tsdn, &prof_gdump_mtx);
2137 	return (prof_gdump_current);
2138 }
2139 
2140 bool
prof_gdump_set(tsdn_t * tsdn,bool gdump)2141 prof_gdump_set(tsdn_t *tsdn, bool gdump)
2142 {
2143 	bool prof_gdump_old;
2144 
2145 	malloc_mutex_lock(tsdn, &prof_gdump_mtx);
2146 	prof_gdump_old = prof_gdump_val;
2147 	prof_gdump_val = gdump;
2148 	malloc_mutex_unlock(tsdn, &prof_gdump_mtx);
2149 	return (prof_gdump_old);
2150 }
2151 
2152 void
prof_boot0(void)2153 prof_boot0(void)
2154 {
2155 
2156 	cassert(config_prof);
2157 
2158 	memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
2159 	    sizeof(PROF_PREFIX_DEFAULT));
2160 }
2161 
2162 void
prof_boot1(void)2163 prof_boot1(void)
2164 {
2165 
2166 	cassert(config_prof);
2167 
2168 	/*
2169 	 * opt_prof must be in its final state before any arenas are
2170 	 * initialized, so this function must be executed early.
2171 	 */
2172 
2173 	if (opt_prof_leak && !opt_prof) {
2174 		/*
2175 		 * Enable opt_prof, but in such a way that profiles are never
2176 		 * automatically dumped.
2177 		 */
2178 		opt_prof = true;
2179 		opt_prof_gdump = false;
2180 	} else if (opt_prof) {
2181 		if (opt_lg_prof_interval >= 0) {
2182 			prof_interval = (((uint64_t)1U) <<
2183 			    opt_lg_prof_interval);
2184 		}
2185 	}
2186 }
2187 
2188 bool
prof_boot2(tsd_t * tsd)2189 prof_boot2(tsd_t *tsd)
2190 {
2191 
2192 	cassert(config_prof);
2193 
2194 	if (opt_prof) {
2195 		unsigned i;
2196 
2197 		lg_prof_sample = opt_lg_prof_sample;
2198 
2199 		prof_active = opt_prof_active;
2200 		if (malloc_mutex_init(&prof_active_mtx, "prof_active",
2201 		    WITNESS_RANK_PROF_ACTIVE))
2202 			return (true);
2203 
2204 		prof_gdump_val = opt_prof_gdump;
2205 		if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump",
2206 		    WITNESS_RANK_PROF_GDUMP))
2207 			return (true);
2208 
2209 		prof_thread_active_init = opt_prof_thread_active_init;
2210 		if (malloc_mutex_init(&prof_thread_active_init_mtx,
2211 		    "prof_thread_active_init",
2212 		    WITNESS_RANK_PROF_THREAD_ACTIVE_INIT))
2213 			return (true);
2214 
2215 		if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
2216 		    prof_bt_keycomp))
2217 			return (true);
2218 		if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx",
2219 		    WITNESS_RANK_PROF_BT2GCTX))
2220 			return (true);
2221 
2222 		tdata_tree_new(&tdatas);
2223 		if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas",
2224 		    WITNESS_RANK_PROF_TDATAS))
2225 			return (true);
2226 
2227 		next_thr_uid = 0;
2228 		if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid",
2229 		    WITNESS_RANK_PROF_NEXT_THR_UID))
2230 			return (true);
2231 
2232 		if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq",
2233 		    WITNESS_RANK_PROF_DUMP_SEQ))
2234 			return (true);
2235 		if (malloc_mutex_init(&prof_dump_mtx, "prof_dump",
2236 		    WITNESS_RANK_PROF_DUMP))
2237 			return (true);
2238 
2239 		if (opt_prof_final && opt_prof_prefix[0] != '\0' &&
2240 		    atexit(prof_fdump) != 0) {
2241 			malloc_write("<jemalloc>: Error in atexit()\n");
2242 			if (opt_abort)
2243 				abort();
2244 		}
2245 
2246 		gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
2247 		    PROF_NCTX_LOCKS * sizeof(malloc_mutex_t));
2248 		if (gctx_locks == NULL)
2249 			return (true);
2250 		for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2251 			if (malloc_mutex_init(&gctx_locks[i], "prof_gctx",
2252 			    WITNESS_RANK_PROF_GCTX))
2253 				return (true);
2254 		}
2255 
2256 		tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
2257 		    PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t));
2258 		if (tdata_locks == NULL)
2259 			return (true);
2260 		for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2261 			if (malloc_mutex_init(&tdata_locks[i], "prof_tdata",
2262 			    WITNESS_RANK_PROF_TDATA))
2263 				return (true);
2264 		}
2265 	}
2266 
2267 #ifdef JEMALLOC_PROF_LIBGCC
2268 	/*
2269 	 * Cause the backtracing machinery to allocate its internal state
2270 	 * before enabling profiling.
2271 	 */
2272 	_Unwind_Backtrace(prof_unwind_init_callback, NULL);
2273 #endif
2274 
2275 	prof_booted = true;
2276 
2277 	return (false);
2278 }
2279 
2280 void
prof_prefork0(tsdn_t * tsdn)2281 prof_prefork0(tsdn_t *tsdn)
2282 {
2283 
2284 	if (opt_prof) {
2285 		unsigned i;
2286 
2287 		malloc_mutex_prefork(tsdn, &prof_dump_mtx);
2288 		malloc_mutex_prefork(tsdn, &bt2gctx_mtx);
2289 		malloc_mutex_prefork(tsdn, &tdatas_mtx);
2290 		for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2291 			malloc_mutex_prefork(tsdn, &tdata_locks[i]);
2292 		for (i = 0; i < PROF_NCTX_LOCKS; i++)
2293 			malloc_mutex_prefork(tsdn, &gctx_locks[i]);
2294 	}
2295 }
2296 
2297 void
prof_prefork1(tsdn_t * tsdn)2298 prof_prefork1(tsdn_t *tsdn)
2299 {
2300 
2301 	if (opt_prof) {
2302 		malloc_mutex_prefork(tsdn, &prof_active_mtx);
2303 		malloc_mutex_prefork(tsdn, &prof_dump_seq_mtx);
2304 		malloc_mutex_prefork(tsdn, &prof_gdump_mtx);
2305 		malloc_mutex_prefork(tsdn, &next_thr_uid_mtx);
2306 		malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx);
2307 	}
2308 }
2309 
2310 void
prof_postfork_parent(tsdn_t * tsdn)2311 prof_postfork_parent(tsdn_t *tsdn)
2312 {
2313 
2314 	if (opt_prof) {
2315 		unsigned i;
2316 
2317 		malloc_mutex_postfork_parent(tsdn,
2318 		    &prof_thread_active_init_mtx);
2319 		malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx);
2320 		malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx);
2321 		malloc_mutex_postfork_parent(tsdn, &prof_dump_seq_mtx);
2322 		malloc_mutex_postfork_parent(tsdn, &prof_active_mtx);
2323 		for (i = 0; i < PROF_NCTX_LOCKS; i++)
2324 			malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]);
2325 		for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2326 			malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]);
2327 		malloc_mutex_postfork_parent(tsdn, &tdatas_mtx);
2328 		malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx);
2329 		malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx);
2330 	}
2331 }
2332 
2333 void
prof_postfork_child(tsdn_t * tsdn)2334 prof_postfork_child(tsdn_t *tsdn)
2335 {
2336 
2337 	if (opt_prof) {
2338 		unsigned i;
2339 
2340 		malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx);
2341 		malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx);
2342 		malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx);
2343 		malloc_mutex_postfork_child(tsdn, &prof_dump_seq_mtx);
2344 		malloc_mutex_postfork_child(tsdn, &prof_active_mtx);
2345 		for (i = 0; i < PROF_NCTX_LOCKS; i++)
2346 			malloc_mutex_postfork_child(tsdn, &gctx_locks[i]);
2347 		for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2348 			malloc_mutex_postfork_child(tsdn, &tdata_locks[i]);
2349 		malloc_mutex_postfork_child(tsdn, &tdatas_mtx);
2350 		malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx);
2351 		malloc_mutex_postfork_child(tsdn, &prof_dump_mtx);
2352 	}
2353 }
2354 
2355 /******************************************************************************/
2356