310 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)  in arena_run_zero()  argument
314 	    (run_ind << LG_PAGE)), (npages << LG_PAGE));  in arena_run_zero()
315 	memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0,  in arena_run_zero()
320 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind)  in arena_run_page_mark_zeroed()  argument
323 	JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind  in arena_run_page_mark_zeroed()
328 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)  in arena_run_page_validate_zeroed()  argument
331 	UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));  in arena_run_page_validate_zeroed()
333 	arena_run_page_mark_zeroed(chunk, run_ind);  in arena_run_page_validate_zeroed()
352 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,  in arena_run_split_remove()  argument
357 	total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>  in arena_run_split_remove()
359 	assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) ==  in arena_run_split_remove()
364 	arena_avail_remove(arena, chunk, run_ind, total_pages);  in arena_run_split_remove()
366 		arena_run_dirty_remove(arena, chunk, run_ind, total_pages);  in arena_run_split_remove()
374 			    run_ind+need_pages, (rem_pages << LG_PAGE),  in arena_run_split_remove()
377 			    run_ind+total_pages-1, (rem_pages << LG_PAGE),  in arena_run_split_remove()
379 			arena_run_dirty_insert(arena, chunk, run_ind+need_pages,  in arena_run_split_remove()
382 			arena_mapbits_unallocated_set(chunk, run_ind+need_pages,  in arena_run_split_remove()
385 			    run_ind+need_pages));  in arena_run_split_remove()
387 			    run_ind+total_pages-1, (rem_pages << LG_PAGE),  in arena_run_split_remove()
389 			    run_ind+total_pages-1));  in arena_run_split_remove()
391 		arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages);  in arena_run_split_remove()
401 	size_t flag_dirty, run_ind, need_pages, i;  in arena_run_split_large_helper()  local
405 	run_ind = arena_miscelm_to_pageind(miscelm);  in arena_run_split_large_helper()
406 	flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);  in arena_run_split_large_helper()
411 		arena_run_split_remove(arena, chunk, run_ind, flag_dirty,  in arena_run_split_large_helper()
422 				if (arena_mapbits_unzeroed_get(chunk, run_ind+i)  in arena_run_split_large_helper()
424 					arena_run_zero(chunk, run_ind+i, 1);  in arena_run_split_large_helper()
427 					    run_ind+i);  in arena_run_split_large_helper()
430 					    run_ind+i);  in arena_run_split_large_helper()
435 			arena_run_zero(chunk, run_ind, need_pages);  in arena_run_split_large_helper()
439 		    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));  in arena_run_split_large_helper()
446 	arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty);  in arena_run_split_large_helper()
447 	arena_mapbits_large_set(chunk, run_ind, size, flag_dirty);  in arena_run_split_large_helper()
470 	size_t flag_dirty, run_ind, need_pages, i;  in arena_run_split_small()  local
476 	run_ind = arena_miscelm_to_pageind(miscelm);  in arena_run_split_small()
477 	flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);  in arena_run_split_small()
481 	arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages);  in arena_run_split_small()
484 		arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0);  in arena_run_split_small()
486 		    arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0)  in arena_run_split_small()
487 			arena_run_page_validate_zeroed(chunk, run_ind+i);  in arena_run_split_small()
490 	    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));  in arena_run_split_small()
1432 	size_t run_ind = *p_run_ind;  in arena_run_coalesce()  local
1436 	if (run_ind + run_pages < chunk_npages &&  in arena_run_coalesce()
1437 	    arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 &&  in arena_run_coalesce()
1438 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) {  in arena_run_coalesce()
1440 		    run_ind+run_pages);  in arena_run_coalesce()
1448 		    run_ind+run_pages+nrun_pages-1) == nrun_size);  in arena_run_coalesce()
1450 		    run_ind+run_pages+nrun_pages-1) == flag_dirty);  in arena_run_coalesce()
1451 		arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages);  in arena_run_coalesce()
1458 			arena_run_dirty_remove(arena, chunk, run_ind+run_pages,  in arena_run_coalesce()
1465 		arena_mapbits_unallocated_size_set(chunk, run_ind, size);  in arena_run_coalesce()
1466 		arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,  in arena_run_coalesce()
1471 	if (run_ind > map_bias && arena_mapbits_allocated_get(chunk,  in arena_run_coalesce()
1472 	    run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) ==  in arena_run_coalesce()
1475 		    run_ind-1);  in arena_run_coalesce()
1478 		run_ind -= prun_pages;  in arena_run_coalesce()
1484 		assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==  in arena_run_coalesce()
1486 		assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty);  in arena_run_coalesce()
1487 		arena_avail_remove(arena, chunk, run_ind, prun_pages);  in arena_run_coalesce()
1494 			arena_run_dirty_remove(arena, chunk, run_ind,  in arena_run_coalesce()
1501 		arena_mapbits_unallocated_size_set(chunk, run_ind, size);  in arena_run_coalesce()
1502 		arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,  in arena_run_coalesce()
1507 	*p_run_ind = run_ind;  in arena_run_coalesce()
1516 	size_t size, run_ind, run_pages, flag_dirty;  in arena_run_dalloc()  local
1520 	run_ind = arena_miscelm_to_pageind(miscelm);  in arena_run_dalloc()
1521 	assert(run_ind >= map_bias);  in arena_run_dalloc()
1522 	assert(run_ind < chunk_npages);  in arena_run_dalloc()
1523 	if (arena_mapbits_large_get(chunk, run_ind) != 0) {  in arena_run_dalloc()
1524 		size = arena_mapbits_large_size_get(chunk, run_ind);  in arena_run_dalloc()
1527 		    run_ind+(size>>LG_PAGE)-1) == 0);  in arena_run_dalloc()
1541 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==  in arena_run_dalloc()
1542 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1543 	if (!cleaned && arena_mapbits_dirty_get(chunk, run_ind) != 0)  in arena_run_dalloc()
1549 		arena_mapbits_unallocated_set(chunk, run_ind, size,  in arena_run_dalloc()
1551 		arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,  in arena_run_dalloc()
1554 		arena_mapbits_unallocated_set(chunk, run_ind, size,  in arena_run_dalloc()
1555 		    arena_mapbits_unzeroed_get(chunk, run_ind));  in arena_run_dalloc()
1556 		arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,  in arena_run_dalloc()
1557 		    arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1560 	arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, flag_dirty);  in arena_run_dalloc()
1563 	assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==  in arena_run_dalloc()
1564 	    arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1565 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==  in arena_run_dalloc()
1566 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1567 	arena_avail_insert(arena, chunk, run_ind, run_pages);  in arena_run_dalloc()
1570 		arena_run_dirty_insert(arena, chunk, run_ind, run_pages);  in arena_run_dalloc()
1574 		assert(run_ind == map_bias);  in arena_run_dalloc()