1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) 5 6 /* Maximum number of regions in one run. */ 7 #define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) 8 #define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 9 10 /* 11 * Minimum redzone size. Redzones may be larger than this if necessary to 12 * preserve region alignment. 13 */ 14 #define REDZONE_MINSIZE 16 15 16 /* 17 * The minimum ratio of active:dirty pages per arena is computed as: 18 * 19 * (nactive >> lg_dirty_mult) >= ndirty 20 * 21 * So, supposing that lg_dirty_mult is 3, there can be no less than 8 times as 22 * many active pages as dirty pages. 23 */ 24 #define LG_DIRTY_MULT_DEFAULT 3 25 26 typedef enum { 27 purge_mode_ratio = 0, 28 purge_mode_decay = 1, 29 30 purge_mode_limit = 2 31 } purge_mode_t; 32 #if defined(__ANDROID__) 33 /* ANDROID change */ 34 /* Use the decay mode purge method. 35 * Setting this value to zero results in performance issues because it 36 * causes purges at every free. Leave the default at zero, but zygote 37 * processes will set this to one using mallopt. This allows apps which 38 * tend to be active to benefit from the extra performance, but allow system 39 * servers to free PSS while they are sitting idle. 40 */ 41 #define PURGE_DEFAULT purge_mode_decay 42 /* Default decay time in seconds. */ 43 #define DECAY_TIME_DEFAULT 0 44 /* End ANDROID change */ 45 #else 46 #define PURGE_DEFAULT purge_mode_ratio 47 /* Default decay time in seconds. */ 48 #define DECAY_TIME_DEFAULT 10 49 #endif 50 /* Number of event ticks between time checks. */ 51 #define DECAY_NTICKS_PER_UPDATE 1000 52 53 typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; 54 typedef struct arena_avail_links_s arena_avail_links_t; 55 typedef struct arena_run_s arena_run_t; 56 typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; 57 typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; 58 typedef struct arena_chunk_s arena_chunk_t; 59 typedef struct arena_bin_info_s arena_bin_info_t; 60 typedef struct arena_decay_s arena_decay_t; 61 typedef struct arena_bin_s arena_bin_t; 62 typedef struct arena_s arena_t; 63 typedef struct arena_tdata_s arena_tdata_t; 64 65 #endif /* JEMALLOC_H_TYPES */ 66 /******************************************************************************/ 67 #ifdef JEMALLOC_H_STRUCTS 68 69 #ifdef JEMALLOC_ARENA_STRUCTS_A 70 struct arena_run_s { 71 /* Index of bin this run is associated with. */ 72 szind_t binind; 73 74 /* Number of free regions in run. */ 75 unsigned nfree; 76 77 /* Per region allocated/deallocated bitmap. */ 78 bitmap_t bitmap[BITMAP_GROUPS_MAX]; 79 }; 80 81 /* Each element of the chunk map corresponds to one page within the chunk. */ 82 struct arena_chunk_map_bits_s { 83 /* 84 * Run address (or size) and various flags are stored together. The bit 85 * layout looks like (assuming 32-bit system): 86 * 87 * ???????? ???????? ???nnnnn nnndumla 88 * 89 * ? : Unallocated: Run address for first/last pages, unset for internal 90 * pages. 91 * Small: Run page offset. 92 * Large: Run page count for first page, unset for trailing pages. 93 * n : binind for small size class, BININD_INVALID for large size class. 94 * d : dirty? 95 * u : unzeroed? 96 * m : decommitted? 97 * l : large? 98 * a : allocated? 99 * 100 * Following are example bit patterns for the three types of runs. 101 * 102 * p : run page offset 103 * s : run size 104 * n : binind for size class; large objects set these to BININD_INVALID 105 * x : don't care 106 * - : 0 107 * + : 1 108 * [DUMLA] : bit set 109 * [dumla] : bit unset 110 * 111 * Unallocated (clean): 112 * ssssssss ssssssss sss+++++ +++dum-a 113 * xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx 114 * ssssssss ssssssss sss+++++ +++dUm-a 115 * 116 * Unallocated (dirty): 117 * ssssssss ssssssss sss+++++ +++D-m-a 118 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 119 * ssssssss ssssssss sss+++++ +++D-m-a 120 * 121 * Small: 122 * pppppppp pppppppp pppnnnnn nnnd---A 123 * pppppppp pppppppp pppnnnnn nnn----A 124 * pppppppp pppppppp pppnnnnn nnnd---A 125 * 126 * Large: 127 * ssssssss ssssssss sss+++++ +++D--LA 128 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 129 * -------- -------- ---+++++ +++D--LA 130 * 131 * Large (sampled, size <= LARGE_MINCLASS): 132 * ssssssss ssssssss sssnnnnn nnnD--LA 133 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 134 * -------- -------- ---+++++ +++D--LA 135 * 136 * Large (not sampled, size == LARGE_MINCLASS): 137 * ssssssss ssssssss sss+++++ +++D--LA 138 * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 139 * -------- -------- ---+++++ +++D--LA 140 */ 141 size_t bits; 142 #define CHUNK_MAP_ALLOCATED ((size_t)0x01U) 143 #define CHUNK_MAP_LARGE ((size_t)0x02U) 144 #define CHUNK_MAP_STATE_MASK ((size_t)0x3U) 145 146 #define CHUNK_MAP_DECOMMITTED ((size_t)0x04U) 147 #define CHUNK_MAP_UNZEROED ((size_t)0x08U) 148 #define CHUNK_MAP_DIRTY ((size_t)0x10U) 149 #define CHUNK_MAP_FLAGS_MASK ((size_t)0x1cU) 150 151 #define CHUNK_MAP_BININD_SHIFT 5 152 #define BININD_INVALID ((size_t)0xffU) 153 #define CHUNK_MAP_BININD_MASK (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) 154 #define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 155 156 #define CHUNK_MAP_RUNIND_SHIFT (CHUNK_MAP_BININD_SHIFT + 8) 157 #define CHUNK_MAP_SIZE_SHIFT (CHUNK_MAP_RUNIND_SHIFT - LG_PAGE) 158 #define CHUNK_MAP_SIZE_MASK \ 159 (~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK)) 160 }; 161 162 struct arena_runs_dirty_link_s { 163 qr(arena_runs_dirty_link_t) rd_link; 164 }; 165 166 /* 167 * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just 168 * like arena_chunk_map_bits_t. Two separate arrays are stored within each 169 * chunk header in order to improve cache locality. 170 */ 171 struct arena_chunk_map_misc_s { 172 /* 173 * Linkage for run heaps. There are two disjoint uses: 174 * 175 * 1) arena_t's runs_avail heaps. 176 * 2) arena_run_t conceptually uses this linkage for in-use non-full 177 * runs, rather than directly embedding linkage. 178 */ 179 phn(arena_chunk_map_misc_t) ph_link; 180 181 union { 182 /* Linkage for list of dirty runs. */ 183 arena_runs_dirty_link_t rd; 184 185 /* Profile counters, used for large object runs. */ 186 union { 187 void *prof_tctx_pun; 188 prof_tctx_t *prof_tctx; 189 }; 190 191 /* Small region run metadata. */ 192 arena_run_t run; 193 }; 194 }; 195 typedef ph(arena_chunk_map_misc_t) arena_run_heap_t; 196 #endif /* JEMALLOC_ARENA_STRUCTS_A */ 197 198 #ifdef JEMALLOC_ARENA_STRUCTS_B 199 /* Arena chunk header. */ 200 struct arena_chunk_s { 201 /* 202 * A pointer to the arena that owns the chunk is stored within the node. 203 * This field as a whole is used by chunks_rtree to support both 204 * ivsalloc() and core-based debugging. 205 */ 206 extent_node_t node; 207 208 /* 209 * True if memory could be backed by transparent huge pages. This is 210 * only directly relevant to Linux, since it is the only supported 211 * platform on which jemalloc interacts with explicit transparent huge 212 * page controls. 213 */ 214 bool hugepage; 215 216 /* 217 * Map of pages within chunk that keeps track of free/large/small. The 218 * first map_bias entries are omitted, since the chunk header does not 219 * need to be tracked in the map. This omission saves a header page 220 * for common chunk sizes (e.g. 4 MiB). 221 */ 222 arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ 223 }; 224 225 /* 226 * Read-only information associated with each element of arena_t's bins array 227 * is stored separately, partly to reduce memory usage (only one copy, rather 228 * than one per arena), but mainly to avoid false cacheline sharing. 229 * 230 * Each run has the following layout: 231 * 232 * /--------------------\ 233 * | pad? | 234 * |--------------------| 235 * | redzone | 236 * reg0_offset | region 0 | 237 * | redzone | 238 * |--------------------| \ 239 * | redzone | | 240 * | region 1 | > reg_interval 241 * | redzone | / 242 * |--------------------| 243 * | ... | 244 * | ... | 245 * | ... | 246 * |--------------------| 247 * | redzone | 248 * | region nregs-1 | 249 * | redzone | 250 * |--------------------| 251 * | alignment pad? | 252 * \--------------------/ 253 * 254 * reg_interval has at least the same minimum alignment as reg_size; this 255 * preserves the alignment constraint that sa2u() depends on. Alignment pad is 256 * either 0 or redzone_size; it is present only if needed to align reg0_offset. 257 */ 258 struct arena_bin_info_s { 259 /* Size of regions in a run for this bin's size class. */ 260 size_t reg_size; 261 262 /* Redzone size. */ 263 size_t redzone_size; 264 265 /* Interval between regions (reg_size + (redzone_size << 1)). */ 266 size_t reg_interval; 267 268 /* Total size of a run for this bin's size class. */ 269 size_t run_size; 270 271 /* Total number of regions in a run for this bin's size class. */ 272 uint32_t nregs; 273 274 /* 275 * Metadata used to manipulate bitmaps for runs associated with this 276 * bin. 277 */ 278 bitmap_info_t bitmap_info; 279 280 /* Offset of first region in a run for this bin's size class. */ 281 uint32_t reg0_offset; 282 }; 283 284 struct arena_decay_s { 285 /* 286 * Approximate time in seconds from the creation of a set of unused 287 * dirty pages until an equivalent set of unused dirty pages is purged 288 * and/or reused. 289 */ 290 ssize_t time; 291 /* time / SMOOTHSTEP_NSTEPS. */ 292 nstime_t interval; 293 /* 294 * Time at which the current decay interval logically started. We do 295 * not actually advance to a new epoch until sometime after it starts 296 * because of scheduling and computation delays, and it is even possible 297 * to completely skip epochs. In all cases, during epoch advancement we 298 * merge all relevant activity into the most recently recorded epoch. 299 */ 300 nstime_t epoch; 301 /* Deadline randomness generator. */ 302 uint64_t jitter_state; 303 /* 304 * Deadline for current epoch. This is the sum of interval and per 305 * epoch jitter which is a uniform random variable in [0..interval). 306 * Epochs always advance by precise multiples of interval, but we 307 * randomize the deadline to reduce the likelihood of arenas purging in 308 * lockstep. 309 */ 310 nstime_t deadline; 311 /* 312 * Number of dirty pages at beginning of current epoch. During epoch 313 * advancement we use the delta between arena->decay.ndirty and 314 * arena->ndirty to determine how many dirty pages, if any, were 315 * generated. 316 */ 317 size_t ndirty; 318 /* 319 * Trailing log of how many unused dirty pages were generated during 320 * each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last 321 * element is the most recent epoch. Corresponding epoch times are 322 * relative to epoch. 323 */ 324 size_t backlog[SMOOTHSTEP_NSTEPS]; 325 }; 326 327 struct arena_bin_s { 328 /* 329 * All operations on runcur, runs, and stats require that lock be 330 * locked. Run allocation/deallocation are protected by the arena lock, 331 * which may be acquired while holding one or more bin locks, but not 332 * vise versa. 333 */ 334 malloc_mutex_t lock; 335 336 /* 337 * Current run being used to service allocations of this bin's size 338 * class. 339 */ 340 arena_run_t *runcur; 341 342 /* 343 * Heap of non-full runs. This heap is used when looking for an 344 * existing run when runcur is no longer usable. We choose the 345 * non-full run that is lowest in memory; this policy tends to keep 346 * objects packed well, and it can also help reduce the number of 347 * almost-empty chunks. 348 */ 349 arena_run_heap_t runs; 350 351 /* Bin statistics. */ 352 malloc_bin_stats_t stats; 353 }; 354 355 struct arena_s { 356 /* This arena's index within the arenas array. */ 357 unsigned ind; 358 359 /* 360 * Number of threads currently assigned to this arena, synchronized via 361 * atomic operations. Each thread has two distinct assignments, one for 362 * application-serving allocation, and the other for internal metadata 363 * allocation. Internal metadata must not be allocated from arenas 364 * created via the arenas.extend mallctl, because the arena.<i>.reset 365 * mallctl indiscriminately discards all allocations for the affected 366 * arena. 367 * 368 * 0: Application allocation. 369 * 1: Internal metadata allocation. 370 */ 371 unsigned nthreads[2]; 372 373 /* 374 * There are three classes of arena operations from a locking 375 * perspective: 376 * 1) Thread assignment (modifies nthreads) is synchronized via atomics. 377 * 2) Bin-related operations are protected by bin locks. 378 * 3) Chunk- and run-related operations are protected by this mutex. 379 */ 380 malloc_mutex_t lock; 381 382 arena_stats_t stats; 383 /* 384 * List of tcaches for extant threads associated with this arena. 385 * Stats from these are merged incrementally, and at exit if 386 * opt_stats_print is enabled. 387 */ 388 ql_head(tcache_t) tcache_ql; 389 390 uint64_t prof_accumbytes; 391 392 /* 393 * PRNG state for cache index randomization of large allocation base 394 * pointers. 395 */ 396 size_t offset_state; 397 398 dss_prec_t dss_prec; 399 400 /* Extant arena chunks. */ 401 ql_head(extent_node_t) achunks; 402 403 /* Extent serial number generator state. */ 404 size_t extent_sn_next; 405 406 /* 407 * In order to avoid rapid chunk allocation/deallocation when an arena 408 * oscillates right on the cusp of needing a new chunk, cache the most 409 * recently freed chunk. The spare is left in the arena's chunk trees 410 * until it is deleted. 411 * 412 * There is one spare chunk per arena, rather than one spare total, in 413 * order to avoid interactions between multiple threads that could make 414 * a single spare inadequate. 415 */ 416 arena_chunk_t *spare; 417 418 /* Minimum ratio (log base 2) of nactive:ndirty. */ 419 ssize_t lg_dirty_mult; 420 421 /* True if a thread is currently executing arena_purge_to_limit(). */ 422 bool purging; 423 424 /* Number of pages in active runs and huge regions. */ 425 size_t nactive; 426 427 /* 428 * Current count of pages within unused runs that are potentially 429 * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 430 * By tracking this, we can institute a limit on how much dirty unused 431 * memory is mapped for each arena. 432 */ 433 size_t ndirty; 434 435 /* 436 * Unused dirty memory this arena manages. Dirty memory is conceptually 437 * tracked as an arbitrarily interleaved LRU of dirty runs and cached 438 * chunks, but the list linkage is actually semi-duplicated in order to 439 * avoid extra arena_chunk_map_misc_t space overhead. 440 * 441 * LRU-----------------------------------------------------------MRU 442 * 443 * /-- arena ---\ 444 * | | 445 * | | 446 * |------------| /- chunk -\ 447 * ...->|chunks_cache|<--------------------------->| /----\ |<--... 448 * |------------| | |node| | 449 * | | | | | | 450 * | | /- run -\ /- run -\ | | | | 451 * | | | | | | | | | | 452 * | | | | | | | | | | 453 * |------------| |-------| |-------| | |----| | 454 * ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----... 455 * |------------| |-------| |-------| | |----| | 456 * | | | | | | | | | | 457 * | | | | | | | \----/ | 458 * | | \-------/ \-------/ | | 459 * | | | | 460 * | | | | 461 * \------------/ \---------/ 462 */ 463 arena_runs_dirty_link_t runs_dirty; 464 extent_node_t chunks_cache; 465 466 /* Decay-based purging state. */ 467 arena_decay_t decay; 468 469 /* Extant huge allocations. */ 470 ql_head(extent_node_t) huge; 471 /* Synchronizes all huge allocation/update/deallocation. */ 472 malloc_mutex_t huge_mtx; 473 474 /* 475 * Trees of chunks that were previously allocated (trees differ only in 476 * node ordering). These are used when allocating chunks, in an attempt 477 * to re-use address space. Depending on function, different tree 478 * orderings are needed, which is why there are two trees with the same 479 * contents. 480 */ 481 extent_tree_t chunks_szsnad_cached; 482 extent_tree_t chunks_ad_cached; 483 extent_tree_t chunks_szsnad_retained; 484 extent_tree_t chunks_ad_retained; 485 486 malloc_mutex_t chunks_mtx; 487 /* Cache of nodes that were allocated via base_alloc(). */ 488 ql_head(extent_node_t) node_cache; 489 malloc_mutex_t node_cache_mtx; 490 491 /* User-configurable chunk hook functions. */ 492 chunk_hooks_t chunk_hooks; 493 494 /* bins is used to store trees of free regions. */ 495 arena_bin_t bins[NBINS]; 496 497 /* 498 * Size-segregated address-ordered heaps of this arena's available runs, 499 * used for first-best-fit run allocation. Runs are quantized, i.e. 500 * they reside in the last heap which corresponds to a size class less 501 * than or equal to the run size. 502 */ 503 arena_run_heap_t runs_avail[NPSIZES]; 504 }; 505 506 /* Used in conjunction with tsd for fast arena-related context lookup. */ 507 struct arena_tdata_s { 508 ticker_t decay_ticker; 509 }; 510 #endif /* JEMALLOC_ARENA_STRUCTS_B */ 511 512 #endif /* JEMALLOC_H_STRUCTS */ 513 /******************************************************************************/ 514 #ifdef JEMALLOC_H_EXTERNS 515 516 static const size_t large_pad = 517 #ifdef JEMALLOC_CACHE_OBLIVIOUS 518 PAGE 519 #else 520 0 521 #endif 522 ; 523 524 extern purge_mode_t opt_purge; 525 extern const char *purge_mode_names[]; 526 extern ssize_t opt_lg_dirty_mult; 527 extern ssize_t opt_decay_time; 528 529 extern arena_bin_info_t arena_bin_info[NBINS]; 530 531 extern size_t map_bias; /* Number of arena chunk header pages. */ 532 extern size_t map_misc_offset; 533 extern size_t arena_maxrun; /* Max run size for arenas. */ 534 extern size_t large_maxclass; /* Max large size class. */ 535 extern unsigned nlclasses; /* Number of large size classes. */ 536 extern unsigned nhclasses; /* Number of huge size classes. */ 537 538 #ifdef JEMALLOC_JET 539 typedef size_t (run_quantize_t)(size_t); 540 extern run_quantize_t *run_quantize_floor; 541 extern run_quantize_t *run_quantize_ceil; 542 #endif 543 void arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, 544 bool cache); 545 void arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, 546 bool cache); 547 extent_node_t *arena_node_alloc(tsdn_t *tsdn, arena_t *arena); 548 void arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node); 549 void *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, 550 size_t alignment, size_t *sn, bool *zero); 551 void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, 552 size_t usize, size_t sn); 553 void arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, 554 void *chunk, size_t oldsize, size_t usize); 555 void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, 556 void *chunk, size_t oldsize, size_t usize, size_t sn); 557 bool arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, 558 void *chunk, size_t oldsize, size_t usize, bool *zero); 559 ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena); 560 bool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena, 561 ssize_t lg_dirty_mult); 562 ssize_t arena_decay_time_get(tsdn_t *tsdn, arena_t *arena); 563 bool arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time); 564 void arena_purge(tsdn_t *tsdn, arena_t *arena, bool all); 565 void arena_maybe_purge(tsdn_t *tsdn, arena_t *arena); 566 void arena_reset(tsd_t *tsd, arena_t *arena); 567 void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, 568 tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes); 569 void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 570 bool zero); 571 #ifdef JEMALLOC_JET 572 typedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, 573 uint8_t); 574 extern arena_redzone_corruption_t *arena_redzone_corruption; 575 typedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); 576 extern arena_dalloc_junk_small_t *arena_dalloc_junk_small; 577 #else 578 void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 579 #endif 580 void arena_quarantine_junk_small(void *ptr, size_t usize); 581 void *arena_malloc_large(tsdn_t *tsdn, arena_t *arena, szind_t ind, 582 bool zero); 583 void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, 584 szind_t ind, bool zero); 585 void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, 586 size_t alignment, bool zero, tcache_t *tcache); 587 void arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size); 588 void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, 589 arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm); 590 void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 591 void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm); 592 void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 593 void *ptr, size_t pageind); 594 #ifdef JEMALLOC_JET 595 typedef void (arena_dalloc_junk_large_t)(void *, size_t); 596 extern arena_dalloc_junk_large_t *arena_dalloc_junk_large; 597 #else 598 void arena_dalloc_junk_large(void *ptr, size_t usize); 599 #endif 600 void arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena, 601 arena_chunk_t *chunk, void *ptr); 602 void arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 603 void *ptr); 604 #ifdef JEMALLOC_JET 605 typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); 606 extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; 607 #endif 608 bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, 609 size_t size, size_t extra, bool zero); 610 void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, 611 size_t size, size_t alignment, bool zero, tcache_t *tcache); 612 dss_prec_t arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena); 613 bool arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec); 614 ssize_t arena_lg_dirty_mult_default_get(void); 615 bool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult); 616 ssize_t arena_decay_time_default_get(void); 617 bool arena_decay_time_default_set(ssize_t decay_time); 618 void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, 619 unsigned *nthreads, const char **dss, ssize_t *lg_dirty_mult, 620 ssize_t *decay_time, size_t *nactive, size_t *ndirty); 621 void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, 622 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 623 size_t *nactive, size_t *ndirty, arena_stats_t *astats, 624 malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, 625 malloc_huge_stats_t *hstats); 626 unsigned arena_nthreads_get(arena_t *arena, bool internal); 627 void arena_nthreads_inc(arena_t *arena, bool internal); 628 void arena_nthreads_dec(arena_t *arena, bool internal); 629 size_t arena_extent_sn_next(arena_t *arena); 630 arena_t *arena_new(tsdn_t *tsdn, unsigned ind); 631 void arena_boot(void); 632 void arena_prefork0(tsdn_t *tsdn, arena_t *arena); 633 void arena_prefork1(tsdn_t *tsdn, arena_t *arena); 634 void arena_prefork2(tsdn_t *tsdn, arena_t *arena); 635 void arena_prefork3(tsdn_t *tsdn, arena_t *arena); 636 void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena); 637 void arena_postfork_child(tsdn_t *tsdn, arena_t *arena); 638 639 #endif /* JEMALLOC_H_EXTERNS */ 640 /******************************************************************************/ 641 #ifdef JEMALLOC_H_INLINES 642 643 #ifndef JEMALLOC_ENABLE_INLINE 644 arena_chunk_map_bits_t *arena_bitselm_get_mutable(arena_chunk_t *chunk, 645 size_t pageind); 646 const arena_chunk_map_bits_t *arena_bitselm_get_const( 647 const arena_chunk_t *chunk, size_t pageind); 648 arena_chunk_map_misc_t *arena_miscelm_get_mutable(arena_chunk_t *chunk, 649 size_t pageind); 650 const arena_chunk_map_misc_t *arena_miscelm_get_const( 651 const arena_chunk_t *chunk, size_t pageind); 652 size_t arena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm); 653 void *arena_miscelm_to_rpages(const arena_chunk_map_misc_t *miscelm); 654 arena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd); 655 arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); 656 size_t *arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind); 657 const size_t *arena_mapbitsp_get_const(const arena_chunk_t *chunk, 658 size_t pageind); 659 size_t arena_mapbitsp_read(const size_t *mapbitsp); 660 size_t arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind); 661 size_t arena_mapbits_size_decode(size_t mapbits); 662 size_t arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, 663 size_t pageind); 664 size_t arena_mapbits_large_size_get(const arena_chunk_t *chunk, 665 size_t pageind); 666 size_t arena_mapbits_small_runind_get(const arena_chunk_t *chunk, 667 size_t pageind); 668 szind_t arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind); 669 size_t arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind); 670 size_t arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind); 671 size_t arena_mapbits_decommitted_get(const arena_chunk_t *chunk, 672 size_t pageind); 673 size_t arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind); 674 size_t arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind); 675 void arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); 676 size_t arena_mapbits_size_encode(size_t size); 677 void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 678 size_t size, size_t flags); 679 void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 680 size_t size); 681 void arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, 682 size_t flags); 683 void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 684 size_t size, size_t flags); 685 void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 686 szind_t binind); 687 void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 688 size_t runind, szind_t binind, size_t flags); 689 void arena_metadata_allocated_add(arena_t *arena, size_t size); 690 void arena_metadata_allocated_sub(arena_t *arena, size_t size); 691 size_t arena_metadata_allocated_get(arena_t *arena); 692 bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 693 bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 694 bool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes); 695 szind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 696 szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 697 size_t arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 698 const void *ptr); 699 prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr); 700 void arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, 701 prof_tctx_t *tctx); 702 void arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, 703 const void *old_ptr, prof_tctx_t *old_tctx); 704 void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks); 705 void arena_decay_tick(tsdn_t *tsdn, arena_t *arena); 706 void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, 707 bool zero, tcache_t *tcache, bool slow_path); 708 arena_t *arena_aalloc(const void *ptr); 709 size_t arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote); 710 void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); 711 void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, 712 bool slow_path); 713 #endif 714 715 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 716 # ifdef JEMALLOC_ARENA_INLINE_A 717 JEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * arena_bitselm_get_mutable(arena_chunk_t * chunk,size_t pageind)718 arena_bitselm_get_mutable(arena_chunk_t *chunk, size_t pageind) 719 { 720 721 assert(pageind >= map_bias); 722 assert(pageind < chunk_npages); 723 724 return (&chunk->map_bits[pageind-map_bias]); 725 } 726 727 JEMALLOC_ALWAYS_INLINE const arena_chunk_map_bits_t * arena_bitselm_get_const(const arena_chunk_t * chunk,size_t pageind)728 arena_bitselm_get_const(const arena_chunk_t *chunk, size_t pageind) 729 { 730 731 return (arena_bitselm_get_mutable((arena_chunk_t *)chunk, pageind)); 732 } 733 734 JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * arena_miscelm_get_mutable(arena_chunk_t * chunk,size_t pageind)735 arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind) 736 { 737 738 assert(pageind >= map_bias); 739 assert(pageind < chunk_npages); 740 741 return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + 742 (uintptr_t)map_misc_offset) + pageind-map_bias); 743 } 744 745 JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t * arena_miscelm_get_const(const arena_chunk_t * chunk,size_t pageind)746 arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind) 747 { 748 749 return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind)); 750 } 751 752 JEMALLOC_ALWAYS_INLINE size_t arena_miscelm_to_pageind(const arena_chunk_map_misc_t * miscelm)753 arena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm) 754 { 755 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 756 size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + 757 map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; 758 759 assert(pageind >= map_bias); 760 assert(pageind < chunk_npages); 761 762 return (pageind); 763 } 764 765 JEMALLOC_ALWAYS_INLINE void * arena_miscelm_to_rpages(const arena_chunk_map_misc_t * miscelm)766 arena_miscelm_to_rpages(const arena_chunk_map_misc_t *miscelm) 767 { 768 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 769 size_t pageind = arena_miscelm_to_pageind(miscelm); 770 771 return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); 772 } 773 774 JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * arena_rd_to_miscelm(arena_runs_dirty_link_t * rd)775 arena_rd_to_miscelm(arena_runs_dirty_link_t *rd) 776 { 777 arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 778 *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); 779 780 assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 781 assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 782 783 return (miscelm); 784 } 785 786 JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * arena_run_to_miscelm(arena_run_t * run)787 arena_run_to_miscelm(arena_run_t *run) 788 { 789 arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 790 *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); 791 792 assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 793 assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 794 795 return (miscelm); 796 } 797 798 JEMALLOC_ALWAYS_INLINE size_t * arena_mapbitsp_get_mutable(arena_chunk_t * chunk,size_t pageind)799 arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind) 800 { 801 802 return (&arena_bitselm_get_mutable(chunk, pageind)->bits); 803 } 804 805 JEMALLOC_ALWAYS_INLINE const size_t * arena_mapbitsp_get_const(const arena_chunk_t * chunk,size_t pageind)806 arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind) 807 { 808 809 return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind)); 810 } 811 812 JEMALLOC_ALWAYS_INLINE size_t arena_mapbitsp_read(const size_t * mapbitsp)813 arena_mapbitsp_read(const size_t *mapbitsp) 814 { 815 816 return (*mapbitsp); 817 } 818 819 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_get(const arena_chunk_t * chunk,size_t pageind)820 arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind) 821 { 822 823 return (arena_mapbitsp_read(arena_mapbitsp_get_const(chunk, pageind))); 824 } 825 826 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_size_decode(size_t mapbits)827 arena_mapbits_size_decode(size_t mapbits) 828 { 829 size_t size; 830 831 #if CHUNK_MAP_SIZE_SHIFT > 0 832 size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; 833 #elif CHUNK_MAP_SIZE_SHIFT == 0 834 size = mapbits & CHUNK_MAP_SIZE_MASK; 835 #else 836 size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; 837 #endif 838 839 return (size); 840 } 841 842 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_unallocated_size_get(const arena_chunk_t * chunk,size_t pageind)843 arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, size_t pageind) 844 { 845 size_t mapbits; 846 847 mapbits = arena_mapbits_get(chunk, pageind); 848 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 849 return (arena_mapbits_size_decode(mapbits)); 850 } 851 852 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_large_size_get(const arena_chunk_t * chunk,size_t pageind)853 arena_mapbits_large_size_get(const arena_chunk_t *chunk, size_t pageind) 854 { 855 size_t mapbits; 856 857 mapbits = arena_mapbits_get(chunk, pageind); 858 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 859 (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 860 return (arena_mapbits_size_decode(mapbits)); 861 } 862 863 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_small_runind_get(const arena_chunk_t * chunk,size_t pageind)864 arena_mapbits_small_runind_get(const arena_chunk_t *chunk, size_t pageind) 865 { 866 size_t mapbits; 867 868 mapbits = arena_mapbits_get(chunk, pageind); 869 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 870 CHUNK_MAP_ALLOCATED); 871 return (mapbits >> CHUNK_MAP_RUNIND_SHIFT); 872 } 873 874 JEMALLOC_ALWAYS_INLINE szind_t arena_mapbits_binind_get(const arena_chunk_t * chunk,size_t pageind)875 arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind) 876 { 877 size_t mapbits; 878 szind_t binind; 879 880 mapbits = arena_mapbits_get(chunk, pageind); 881 binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 882 assert(binind < NBINS || binind == BININD_INVALID); 883 return (binind); 884 } 885 886 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_dirty_get(const arena_chunk_t * chunk,size_t pageind)887 arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind) 888 { 889 size_t mapbits; 890 891 mapbits = arena_mapbits_get(chunk, pageind); 892 assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 893 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 894 return (mapbits & CHUNK_MAP_DIRTY); 895 } 896 897 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_unzeroed_get(const arena_chunk_t * chunk,size_t pageind)898 arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind) 899 { 900 size_t mapbits; 901 902 mapbits = arena_mapbits_get(chunk, pageind); 903 assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 904 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 905 return (mapbits & CHUNK_MAP_UNZEROED); 906 } 907 908 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_decommitted_get(const arena_chunk_t * chunk,size_t pageind)909 arena_mapbits_decommitted_get(const arena_chunk_t *chunk, size_t pageind) 910 { 911 size_t mapbits; 912 913 mapbits = arena_mapbits_get(chunk, pageind); 914 assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 915 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 916 return (mapbits & CHUNK_MAP_DECOMMITTED); 917 } 918 919 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_large_get(const arena_chunk_t * chunk,size_t pageind)920 arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind) 921 { 922 size_t mapbits; 923 924 mapbits = arena_mapbits_get(chunk, pageind); 925 return (mapbits & CHUNK_MAP_LARGE); 926 } 927 928 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_allocated_get(const arena_chunk_t * chunk,size_t pageind)929 arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind) 930 { 931 size_t mapbits; 932 933 mapbits = arena_mapbits_get(chunk, pageind); 934 return (mapbits & CHUNK_MAP_ALLOCATED); 935 } 936 937 JEMALLOC_ALWAYS_INLINE void arena_mapbitsp_write(size_t * mapbitsp,size_t mapbits)938 arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) 939 { 940 941 *mapbitsp = mapbits; 942 } 943 944 JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_size_encode(size_t size)945 arena_mapbits_size_encode(size_t size) 946 { 947 size_t mapbits; 948 949 #if CHUNK_MAP_SIZE_SHIFT > 0 950 mapbits = size << CHUNK_MAP_SIZE_SHIFT; 951 #elif CHUNK_MAP_SIZE_SHIFT == 0 952 mapbits = size; 953 #else 954 mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; 955 #endif 956 957 assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0); 958 return (mapbits); 959 } 960 961 JEMALLOC_ALWAYS_INLINE void arena_mapbits_unallocated_set(arena_chunk_t * chunk,size_t pageind,size_t size,size_t flags)962 arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 963 size_t flags) 964 { 965 size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 966 967 assert((size & PAGE_MASK) == 0); 968 assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 969 assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 970 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 971 arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 972 CHUNK_MAP_BININD_INVALID | flags); 973 } 974 975 JEMALLOC_ALWAYS_INLINE void arena_mapbits_unallocated_size_set(arena_chunk_t * chunk,size_t pageind,size_t size)976 arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 977 size_t size) 978 { 979 size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 980 size_t mapbits = arena_mapbitsp_read(mapbitsp); 981 982 assert((size & PAGE_MASK) == 0); 983 assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 984 arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 985 (mapbits & ~CHUNK_MAP_SIZE_MASK)); 986 } 987 988 JEMALLOC_ALWAYS_INLINE void arena_mapbits_internal_set(arena_chunk_t * chunk,size_t pageind,size_t flags)989 arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, size_t flags) 990 { 991 size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 992 993 assert((flags & CHUNK_MAP_UNZEROED) == flags); 994 arena_mapbitsp_write(mapbitsp, flags); 995 } 996 997 JEMALLOC_ALWAYS_INLINE void arena_mapbits_large_set(arena_chunk_t * chunk,size_t pageind,size_t size,size_t flags)998 arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 999 size_t flags) 1000 { 1001 size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 1002 1003 assert((size & PAGE_MASK) == 0); 1004 assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 1005 assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 1006 (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 1007 arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 1008 CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE | 1009 CHUNK_MAP_ALLOCATED); 1010 } 1011 1012 JEMALLOC_ALWAYS_INLINE void arena_mapbits_large_binind_set(arena_chunk_t * chunk,size_t pageind,szind_t binind)1013 arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 1014 szind_t binind) 1015 { 1016 size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 1017 size_t mapbits = arena_mapbitsp_read(mapbitsp); 1018 1019 assert(binind <= BININD_INVALID); 1020 assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS + 1021 large_pad); 1022 arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | 1023 (binind << CHUNK_MAP_BININD_SHIFT)); 1024 } 1025 1026 JEMALLOC_ALWAYS_INLINE void arena_mapbits_small_set(arena_chunk_t * chunk,size_t pageind,size_t runind,szind_t binind,size_t flags)1027 arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 1028 szind_t binind, size_t flags) 1029 { 1030 size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 1031 1032 assert(binind < BININD_INVALID); 1033 assert(pageind - runind >= map_bias); 1034 assert((flags & CHUNK_MAP_UNZEROED) == flags); 1035 arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) | 1036 (binind << CHUNK_MAP_BININD_SHIFT) | flags | CHUNK_MAP_ALLOCATED); 1037 } 1038 1039 JEMALLOC_INLINE void arena_metadata_allocated_add(arena_t * arena,size_t size)1040 arena_metadata_allocated_add(arena_t *arena, size_t size) 1041 { 1042 1043 atomic_add_z(&arena->stats.metadata_allocated, size); 1044 } 1045 1046 JEMALLOC_INLINE void arena_metadata_allocated_sub(arena_t * arena,size_t size)1047 arena_metadata_allocated_sub(arena_t *arena, size_t size) 1048 { 1049 1050 atomic_sub_z(&arena->stats.metadata_allocated, size); 1051 } 1052 1053 JEMALLOC_INLINE size_t arena_metadata_allocated_get(arena_t * arena)1054 arena_metadata_allocated_get(arena_t *arena) 1055 { 1056 1057 return (atomic_read_z(&arena->stats.metadata_allocated)); 1058 } 1059 1060 JEMALLOC_INLINE bool arena_prof_accum_impl(arena_t * arena,uint64_t accumbytes)1061 arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 1062 { 1063 1064 cassert(config_prof); 1065 assert(prof_interval != 0); 1066 1067 arena->prof_accumbytes += accumbytes; 1068 if (arena->prof_accumbytes >= prof_interval) { 1069 arena->prof_accumbytes -= prof_interval; 1070 return (true); 1071 } 1072 return (false); 1073 } 1074 1075 JEMALLOC_INLINE bool arena_prof_accum_locked(arena_t * arena,uint64_t accumbytes)1076 arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 1077 { 1078 1079 cassert(config_prof); 1080 1081 if (likely(prof_interval == 0)) 1082 return (false); 1083 return (arena_prof_accum_impl(arena, accumbytes)); 1084 } 1085 1086 JEMALLOC_INLINE bool arena_prof_accum(tsdn_t * tsdn,arena_t * arena,uint64_t accumbytes)1087 arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) 1088 { 1089 1090 cassert(config_prof); 1091 1092 if (likely(prof_interval == 0)) 1093 return (false); 1094 1095 { 1096 bool ret; 1097 1098 malloc_mutex_lock(tsdn, &arena->lock); 1099 ret = arena_prof_accum_impl(arena, accumbytes); 1100 malloc_mutex_unlock(tsdn, &arena->lock); 1101 return (ret); 1102 } 1103 } 1104 1105 JEMALLOC_ALWAYS_INLINE szind_t arena_ptr_small_binind_get(const void * ptr,size_t mapbits)1106 arena_ptr_small_binind_get(const void *ptr, size_t mapbits) 1107 { 1108 szind_t binind; 1109 1110 binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 1111 1112 if (config_debug) { 1113 arena_chunk_t *chunk; 1114 arena_t *arena; 1115 size_t pageind; 1116 size_t actual_mapbits; 1117 size_t rpages_ind; 1118 const arena_run_t *run; 1119 arena_bin_t *bin; 1120 szind_t run_binind, actual_binind; 1121 arena_bin_info_t *bin_info; 1122 const arena_chunk_map_misc_t *miscelm; 1123 const void *rpages; 1124 1125 assert(binind != BININD_INVALID); 1126 assert(binind < NBINS); 1127 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1128 arena = extent_node_arena_get(&chunk->node); 1129 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1130 actual_mapbits = arena_mapbits_get(chunk, pageind); 1131 assert(mapbits == actual_mapbits); 1132 assert(arena_mapbits_large_get(chunk, pageind) == 0); 1133 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1134 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, 1135 pageind); 1136 miscelm = arena_miscelm_get_const(chunk, rpages_ind); 1137 run = &miscelm->run; 1138 run_binind = run->binind; 1139 bin = &arena->bins[run_binind]; 1140 actual_binind = (szind_t)(bin - arena->bins); 1141 assert(run_binind == actual_binind); 1142 bin_info = &arena_bin_info[actual_binind]; 1143 rpages = arena_miscelm_to_rpages(miscelm); 1144 assert(((uintptr_t)ptr - ((uintptr_t)rpages + 1145 (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 1146 == 0); 1147 } 1148 1149 return (binind); 1150 } 1151 # endif /* JEMALLOC_ARENA_INLINE_A */ 1152 1153 # ifdef JEMALLOC_ARENA_INLINE_B 1154 JEMALLOC_INLINE szind_t arena_bin_index(arena_t * arena,arena_bin_t * bin)1155 arena_bin_index(arena_t *arena, arena_bin_t *bin) 1156 { 1157 szind_t binind = (szind_t)(bin - arena->bins); 1158 assert(binind < NBINS); 1159 return (binind); 1160 } 1161 1162 JEMALLOC_INLINE size_t arena_run_regind(arena_run_t * run,arena_bin_info_t * bin_info,const void * ptr)1163 arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 1164 { 1165 size_t diff, interval, shift, regind; 1166 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1167 void *rpages = arena_miscelm_to_rpages(miscelm); 1168 1169 /* 1170 * Freeing a pointer lower than region zero can cause assertion 1171 * failure. 1172 */ 1173 assert((uintptr_t)ptr >= (uintptr_t)rpages + 1174 (uintptr_t)bin_info->reg0_offset); 1175 1176 /* 1177 * Avoid doing division with a variable divisor if possible. Using 1178 * actual division here can reduce allocator throughput by over 20%! 1179 */ 1180 diff = (size_t)((uintptr_t)ptr - (uintptr_t)rpages - 1181 bin_info->reg0_offset); 1182 1183 /* Rescale (factor powers of 2 out of the numerator and denominator). */ 1184 interval = bin_info->reg_interval; 1185 shift = ffs_zu(interval) - 1; 1186 diff >>= shift; 1187 interval >>= shift; 1188 1189 if (interval == 1) { 1190 /* The divisor was a power of 2. */ 1191 regind = diff; 1192 } else { 1193 /* 1194 * To divide by a number D that is not a power of two we 1195 * multiply by (2^21 / D) and then right shift by 21 positions. 1196 * 1197 * X / D 1198 * 1199 * becomes 1200 * 1201 * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 1202 * 1203 * We can omit the first three elements, because we never 1204 * divide by 0, and 1 and 2 are both powers of two, which are 1205 * handled above. 1206 */ 1207 #define SIZE_INV_SHIFT ((sizeof(size_t) << 3) - LG_RUN_MAXREGS) 1208 #define SIZE_INV(s) (((ZU(1) << SIZE_INV_SHIFT) / (s)) + 1) 1209 static const size_t interval_invs[] = { 1210 SIZE_INV(3), 1211 SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 1212 SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 1213 SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 1214 SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 1215 SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 1216 SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 1217 SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 1218 }; 1219 1220 if (likely(interval <= ((sizeof(interval_invs) / sizeof(size_t)) 1221 + 2))) { 1222 regind = (diff * interval_invs[interval - 3]) >> 1223 SIZE_INV_SHIFT; 1224 } else 1225 regind = diff / interval; 1226 #undef SIZE_INV 1227 #undef SIZE_INV_SHIFT 1228 } 1229 assert(diff == regind * interval); 1230 assert(regind < bin_info->nregs); 1231 1232 return (regind); 1233 } 1234 1235 JEMALLOC_INLINE prof_tctx_t * arena_prof_tctx_get(tsdn_t * tsdn,const void * ptr)1236 arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr) 1237 { 1238 prof_tctx_t *ret; 1239 arena_chunk_t *chunk; 1240 1241 cassert(config_prof); 1242 assert(ptr != NULL); 1243 1244 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1245 if (likely(chunk != ptr)) { 1246 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1247 size_t mapbits = arena_mapbits_get(chunk, pageind); 1248 assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1249 if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) 1250 ret = (prof_tctx_t *)(uintptr_t)1U; 1251 else { 1252 arena_chunk_map_misc_t *elm = 1253 arena_miscelm_get_mutable(chunk, pageind); 1254 ret = atomic_read_p(&elm->prof_tctx_pun); 1255 } 1256 } else 1257 ret = huge_prof_tctx_get(tsdn, ptr); 1258 1259 return (ret); 1260 } 1261 1262 JEMALLOC_INLINE void arena_prof_tctx_set(tsdn_t * tsdn,const void * ptr,size_t usize,prof_tctx_t * tctx)1263 arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, 1264 prof_tctx_t *tctx) 1265 { 1266 arena_chunk_t *chunk; 1267 1268 cassert(config_prof); 1269 assert(ptr != NULL); 1270 1271 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1272 if (likely(chunk != ptr)) { 1273 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1274 1275 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1276 1277 if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx > 1278 (uintptr_t)1U)) { 1279 arena_chunk_map_misc_t *elm; 1280 1281 assert(arena_mapbits_large_get(chunk, pageind) != 0); 1282 1283 elm = arena_miscelm_get_mutable(chunk, pageind); 1284 atomic_write_p(&elm->prof_tctx_pun, tctx); 1285 } else { 1286 /* 1287 * tctx must always be initialized for large runs. 1288 * Assert that the surrounding conditional logic is 1289 * equivalent to checking whether ptr refers to a large 1290 * run. 1291 */ 1292 assert(arena_mapbits_large_get(chunk, pageind) == 0); 1293 } 1294 } else 1295 huge_prof_tctx_set(tsdn, ptr, tctx); 1296 } 1297 1298 JEMALLOC_INLINE void arena_prof_tctx_reset(tsdn_t * tsdn,const void * ptr,size_t usize,const void * old_ptr,prof_tctx_t * old_tctx)1299 arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, 1300 const void *old_ptr, prof_tctx_t *old_tctx) 1301 { 1302 1303 cassert(config_prof); 1304 assert(ptr != NULL); 1305 1306 if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr && 1307 (uintptr_t)old_tctx > (uintptr_t)1U))) { 1308 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1309 if (likely(chunk != ptr)) { 1310 size_t pageind; 1311 arena_chunk_map_misc_t *elm; 1312 1313 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1314 LG_PAGE; 1315 assert(arena_mapbits_allocated_get(chunk, pageind) != 1316 0); 1317 assert(arena_mapbits_large_get(chunk, pageind) != 0); 1318 1319 elm = arena_miscelm_get_mutable(chunk, pageind); 1320 atomic_write_p(&elm->prof_tctx_pun, 1321 (prof_tctx_t *)(uintptr_t)1U); 1322 } else 1323 huge_prof_tctx_reset(tsdn, ptr); 1324 } 1325 } 1326 1327 JEMALLOC_ALWAYS_INLINE void arena_decay_ticks(tsdn_t * tsdn,arena_t * arena,unsigned nticks)1328 arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) 1329 { 1330 tsd_t *tsd; 1331 ticker_t *decay_ticker; 1332 1333 if (unlikely(tsdn_null(tsdn))) 1334 return; 1335 tsd = tsdn_tsd(tsdn); 1336 decay_ticker = decay_ticker_get(tsd, arena->ind); 1337 if (unlikely(decay_ticker == NULL)) 1338 return; 1339 if (unlikely(ticker_ticks(decay_ticker, nticks))) 1340 arena_purge(tsdn, arena, false); 1341 } 1342 1343 JEMALLOC_ALWAYS_INLINE void arena_decay_tick(tsdn_t * tsdn,arena_t * arena)1344 arena_decay_tick(tsdn_t *tsdn, arena_t *arena) 1345 { 1346 1347 arena_decay_ticks(tsdn, arena, 1); 1348 } 1349 1350 JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsdn_t * tsdn,arena_t * arena,size_t size,szind_t ind,bool zero,tcache_t * tcache,bool slow_path)1351 arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, 1352 tcache_t *tcache, bool slow_path) 1353 { 1354 1355 assert(!tsdn_null(tsdn) || tcache == NULL); 1356 assert(size != 0); 1357 1358 if (likely(tcache != NULL)) { 1359 if (likely(size <= SMALL_MAXCLASS)) { 1360 return (tcache_alloc_small(tsdn_tsd(tsdn), arena, 1361 tcache, size, ind, zero, slow_path)); 1362 } 1363 if (likely(size <= tcache_maxclass)) { 1364 return (tcache_alloc_large(tsdn_tsd(tsdn), arena, 1365 tcache, size, ind, zero, slow_path)); 1366 } 1367 /* (size > tcache_maxclass) case falls through. */ 1368 assert(size > tcache_maxclass); 1369 } 1370 1371 return (arena_malloc_hard(tsdn, arena, size, ind, zero)); 1372 } 1373 1374 JEMALLOC_ALWAYS_INLINE arena_t * arena_aalloc(const void * ptr)1375 arena_aalloc(const void *ptr) 1376 { 1377 arena_chunk_t *chunk; 1378 1379 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1380 if (likely(chunk != ptr)) 1381 return (extent_node_arena_get(&chunk->node)); 1382 else 1383 return (huge_aalloc(ptr)); 1384 } 1385 1386 /* Return the size of the allocation pointed to by ptr. */ 1387 JEMALLOC_ALWAYS_INLINE size_t arena_salloc(tsdn_t * tsdn,const void * ptr,bool demote)1388 arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote) 1389 { 1390 size_t ret; 1391 arena_chunk_t *chunk; 1392 size_t pageind; 1393 szind_t binind; 1394 1395 assert(ptr != NULL); 1396 1397 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1398 if (likely(chunk != ptr)) { 1399 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1400 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1401 binind = arena_mapbits_binind_get(chunk, pageind); 1402 if (unlikely(binind == BININD_INVALID || (config_prof && !demote 1403 && arena_mapbits_large_get(chunk, pageind) != 0))) { 1404 /* 1405 * Large allocation. In the common case (demote), and 1406 * as this is an inline function, most callers will only 1407 * end up looking at binind to determine that ptr is a 1408 * small allocation. 1409 */ 1410 assert(config_cache_oblivious || ((uintptr_t)ptr & 1411 PAGE_MASK) == 0); 1412 ret = arena_mapbits_large_size_get(chunk, pageind) - 1413 large_pad; 1414 assert(ret != 0); 1415 assert(pageind + ((ret+large_pad)>>LG_PAGE) <= 1416 chunk_npages); 1417 assert(arena_mapbits_dirty_get(chunk, pageind) == 1418 arena_mapbits_dirty_get(chunk, 1419 pageind+((ret+large_pad)>>LG_PAGE)-1)); 1420 } else { 1421 /* 1422 * Small allocation (possibly promoted to a large 1423 * object). 1424 */ 1425 assert(arena_mapbits_large_get(chunk, pageind) != 0 || 1426 arena_ptr_small_binind_get(ptr, 1427 arena_mapbits_get(chunk, pageind)) == binind); 1428 ret = index2size(binind); 1429 } 1430 } else 1431 ret = huge_salloc(tsdn, ptr); 1432 1433 return (ret); 1434 } 1435 1436 JEMALLOC_ALWAYS_INLINE void arena_dalloc(tsdn_t * tsdn,void * ptr,tcache_t * tcache,bool slow_path)1437 arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) 1438 { 1439 arena_chunk_t *chunk; 1440 size_t pageind, mapbits; 1441 1442 assert(!tsdn_null(tsdn) || tcache == NULL); 1443 assert(ptr != NULL); 1444 1445 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1446 if (likely(chunk != ptr)) { 1447 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1448 #if defined(__ANDROID__) 1449 /* Verify the ptr is actually in the chunk. */ 1450 if (unlikely(pageind < map_bias || pageind >= chunk_npages)) { 1451 async_safe_fatal("Invalid address %p passed to free: invalid page index", ptr); 1452 } 1453 #endif 1454 mapbits = arena_mapbits_get(chunk, pageind); 1455 assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1456 #if defined(__ANDROID__) 1457 /* Verify the ptr has been allocated. */ 1458 if (unlikely((mapbits & CHUNK_MAP_ALLOCATED) == 0)) { 1459 async_safe_fatal("Invalid address %p passed to free: value not allocated", ptr); 1460 } 1461 #endif 1462 if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { 1463 /* Small allocation. */ 1464 if (likely(tcache != NULL)) { 1465 szind_t binind = arena_ptr_small_binind_get(ptr, 1466 mapbits); 1467 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, 1468 binind, slow_path); 1469 } else { 1470 arena_dalloc_small(tsdn, 1471 extent_node_arena_get(&chunk->node), chunk, 1472 ptr, pageind); 1473 } 1474 } else { 1475 size_t size = arena_mapbits_large_size_get(chunk, 1476 pageind); 1477 1478 assert(config_cache_oblivious || ((uintptr_t)ptr & 1479 PAGE_MASK) == 0); 1480 1481 if (likely(tcache != NULL) && size - large_pad <= 1482 tcache_maxclass) { 1483 tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, 1484 size - large_pad, slow_path); 1485 } else { 1486 arena_dalloc_large(tsdn, 1487 extent_node_arena_get(&chunk->node), chunk, 1488 ptr); 1489 } 1490 } 1491 } else 1492 huge_dalloc(tsdn, ptr); 1493 } 1494 1495 JEMALLOC_ALWAYS_INLINE void arena_sdalloc(tsdn_t * tsdn,void * ptr,size_t size,tcache_t * tcache,bool slow_path)1496 arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, 1497 bool slow_path) 1498 { 1499 arena_chunk_t *chunk; 1500 1501 assert(!tsdn_null(tsdn) || tcache == NULL); 1502 1503 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1504 if (likely(chunk != ptr)) { 1505 if (config_prof && opt_prof) { 1506 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1507 LG_PAGE; 1508 assert(arena_mapbits_allocated_get(chunk, pageind) != 1509 0); 1510 if (arena_mapbits_large_get(chunk, pageind) != 0) { 1511 /* 1512 * Make sure to use promoted size, not request 1513 * size. 1514 */ 1515 size = arena_mapbits_large_size_get(chunk, 1516 pageind) - large_pad; 1517 } 1518 } 1519 assert(s2u(size) == s2u(arena_salloc(tsdn, ptr, false))); 1520 1521 if (likely(size <= SMALL_MAXCLASS)) { 1522 /* Small allocation. */ 1523 if (likely(tcache != NULL)) { 1524 szind_t binind = size2index(size); 1525 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, 1526 binind, slow_path); 1527 } else { 1528 size_t pageind = ((uintptr_t)ptr - 1529 (uintptr_t)chunk) >> LG_PAGE; 1530 arena_dalloc_small(tsdn, 1531 extent_node_arena_get(&chunk->node), chunk, 1532 ptr, pageind); 1533 } 1534 } else { 1535 assert(config_cache_oblivious || ((uintptr_t)ptr & 1536 PAGE_MASK) == 0); 1537 1538 if (likely(tcache != NULL) && size <= tcache_maxclass) { 1539 tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, 1540 size, slow_path); 1541 } else { 1542 arena_dalloc_large(tsdn, 1543 extent_node_arena_get(&chunk->node), chunk, 1544 ptr); 1545 } 1546 } 1547 } else 1548 huge_dalloc(tsdn, ptr); 1549 } 1550 # endif /* JEMALLOC_ARENA_INLINE_B */ 1551 #endif 1552 1553 #endif /* JEMALLOC_H_INLINES */ 1554 /******************************************************************************/ 1555