Lines Matching refs:run_ind
347 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) in arena_run_zero() argument
351 (run_ind << LG_PAGE)), (npages << LG_PAGE)); in arena_run_zero()
352 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, in arena_run_zero()
357 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_mark_zeroed() argument
360 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind in arena_run_page_mark_zeroed()
365 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_validate_zeroed() argument
368 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); in arena_run_page_validate_zeroed()
370 arena_run_page_mark_zeroed(chunk, run_ind); in arena_run_page_validate_zeroed()
403 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, in arena_run_split_remove() argument
410 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> in arena_run_split_remove()
412 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == in arena_run_split_remove()
417 arena_avail_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
419 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
428 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, in arena_run_split_remove()
430 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & in arena_run_split_remove()
432 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, in arena_run_split_remove()
434 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & in arena_run_split_remove()
437 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, in arena_run_split_remove()
440 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); in arena_run_split_remove()
450 size_t flag_dirty, flag_decommitted, run_ind, need_pages; in arena_run_split_large_helper() local
455 run_ind = arena_miscelm_to_pageind(miscelm); in arena_run_split_large_helper()
456 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_large_helper()
457 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_large_helper()
462 run_ind << LG_PAGE, size, arena->ind)) in arena_run_split_large_helper()
466 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_large_helper()
474 *)((uintptr_t)chunk + (run_ind << LG_PAGE)), in arena_run_split_large_helper()
478 arena_run_zero(chunk, run_ind, need_pages); in arena_run_split_large_helper()
486 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) in arena_run_split_large_helper()
488 arena_run_zero(chunk, run_ind+i, 1); in arena_run_split_large_helper()
491 run_ind+i); in arena_run_split_large_helper()
494 run_ind+i); in arena_run_split_large_helper()
500 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); in arena_run_split_large_helper()
509 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | in arena_run_split_large_helper()
511 run_ind+need_pages-1))); in arena_run_split_large_helper()
512 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | in arena_run_split_large_helper()
513 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); in arena_run_split_large_helper()
537 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; in arena_run_split_small() local
543 run_ind = arena_miscelm_to_pageind(miscelm); in arena_run_split_small()
544 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_small()
545 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_small()
550 run_ind << LG_PAGE, size, arena->ind)) in arena_run_split_small()
553 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_small()
558 run_ind+i); in arena_run_split_small()
559 arena_mapbits_small_set(chunk, run_ind+i, i, binind, in arena_run_split_small()
562 arena_run_page_validate_zeroed(chunk, run_ind+i); in arena_run_split_small()
565 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); in arena_run_split_small()
1795 size_t run_ind = *p_run_ind; in arena_run_coalesce() local
1799 if (run_ind + run_pages < chunk_npages && in arena_run_coalesce()
1800 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && in arena_run_coalesce()
1801 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && in arena_run_coalesce()
1802 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == in arena_run_coalesce()
1805 run_ind+run_pages); in arena_run_coalesce()
1813 run_ind+run_pages+nrun_pages-1) == nrun_size); in arena_run_coalesce()
1815 run_ind+run_pages+nrun_pages-1) == flag_dirty); in arena_run_coalesce()
1817 run_ind+run_pages+nrun_pages-1) == flag_decommitted); in arena_run_coalesce()
1818 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); in arena_run_coalesce()
1825 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, in arena_run_coalesce()
1832 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
1833 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
1838 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, in arena_run_coalesce()
1839 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == in arena_run_coalesce()
1840 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == in arena_run_coalesce()
1843 run_ind-1); in arena_run_coalesce()
1846 run_ind -= prun_pages; in arena_run_coalesce()
1852 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_coalesce()
1854 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); in arena_run_coalesce()
1855 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_coalesce()
1857 arena_avail_remove(arena, chunk, run_ind, prun_pages); in arena_run_coalesce()
1864 arena_run_dirty_remove(arena, chunk, run_ind, in arena_run_coalesce()
1871 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
1872 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
1877 *p_run_ind = run_ind; in arena_run_coalesce()
1883 size_t run_ind) in arena_run_size_get() argument
1887 assert(run_ind >= map_bias); in arena_run_size_get()
1888 assert(run_ind < chunk_npages); in arena_run_size_get()
1890 if (arena_mapbits_large_get(chunk, run_ind) != 0) { in arena_run_size_get()
1891 size = arena_mapbits_large_size_get(chunk, run_ind); in arena_run_size_get()
1893 run_ind+(size>>LG_PAGE)-1) == 0); in arena_run_size_get()
1908 size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; in arena_run_dalloc() local
1912 run_ind = arena_miscelm_to_pageind(miscelm); in arena_run_dalloc()
1913 assert(run_ind >= map_bias); in arena_run_dalloc()
1914 assert(run_ind < chunk_npages); in arena_run_dalloc()
1915 size = arena_run_size_get(arena, chunk, run, run_ind); in arena_run_dalloc()
1924 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
1925 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1926 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) in arena_run_dalloc()
1935 arena_mapbits_unallocated_set(chunk, run_ind, size, flags); in arena_run_dalloc()
1936 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
1939 arena_mapbits_unallocated_set(chunk, run_ind, size, in arena_run_dalloc()
1940 arena_mapbits_unzeroed_get(chunk, run_ind)); in arena_run_dalloc()
1941 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
1942 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1945 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, in arena_run_dalloc()
1949 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_dalloc()
1950 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1951 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
1952 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1953 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_dalloc()
1954 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1955 arena_avail_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
1958 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
1962 assert(run_ind == map_bias); in arena_run_dalloc()
2548 size_t run_ind = in arena_palloc_large() local
2550 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); in arena_palloc_large()
2552 run_ind) != 0); in arena_palloc_large()