37 struct pool {  struct
54 static struct pool mp[MAX_POOLS];  argument
59 static inline void pool_lock(struct pool *pool)  in pool_lock()  argument
61 	fio_mutex_down(pool->lock);  in pool_lock()
64 static inline void pool_unlock(struct pool *pool)  in pool_unlock()  argument
66 	fio_mutex_up(pool->lock);  in pool_unlock()
89 static inline int ptr_valid(struct pool *pool, void *ptr)  in ptr_valid()  argument
91 	unsigned int pool_size = pool->nr_blocks * SMALLOC_BPL;  in ptr_valid()
93 	return (ptr >= pool->map) && (ptr < pool->map + pool_size);  in ptr_valid()
101 static int blocks_iter(struct pool *pool, unsigned int pool_idx,  in blocks_iter()  argument
110 		if (pool_idx >= pool->nr_blocks)  in blocks_iter()
113 		map = &pool->bitmap[pool_idx];  in blocks_iter()
156 static int blocks_free(struct pool *pool, unsigned int pool_idx,  in blocks_free()  argument
159 	return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);  in blocks_free()
162 static void set_blocks(struct pool *pool, unsigned int pool_idx,  in set_blocks()  argument
165 	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);  in set_blocks()
168 static void clear_blocks(struct pool *pool, unsigned int pool_idx,  in clear_blocks()  argument
171 	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);  in clear_blocks()
181 static int add_pool(struct pool *pool, unsigned int alloc_size)  in add_pool()  argument
198 	pool->mmap_size = alloc_size;  in add_pool()
200 	pool->nr_blocks = bitmap_blocks;  in add_pool()
201 	pool->free_blocks = bitmap_blocks * SMALLOC_BPB;  in add_pool()
215 	pool->map = ptr;  in add_pool()
216 	pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL);  in add_pool()
218 	pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);  in add_pool()
219 	if (!pool->lock)  in add_pool()
226 	if (pool->map)  in add_pool()
227 		munmap(pool->map, pool->mmap_size);  in add_pool()
250 static void cleanup_pool(struct pool *pool)  in cleanup_pool()  argument
256 	munmap(pool->map, pool->mmap_size);  in cleanup_pool()
258 	if (pool->lock)  in cleanup_pool()
259 		fio_mutex_remove(pool->lock);  in cleanup_pool()
319 static void sfree_pool(struct pool *pool, void *ptr)  in sfree_pool()  argument
331 	assert(ptr_valid(pool, ptr));  in sfree_pool()
335 	offset = ptr - pool->map;  in sfree_pool()
339 	pool_lock(pool);  in sfree_pool()
340 	clear_blocks(pool, i, idx, size_to_blocks(hdr->size));  in sfree_pool()
341 	if (i < pool->next_non_full)  in sfree_pool()
342 		pool->next_non_full = i;  in sfree_pool()
343 	pool->free_blocks += size_to_blocks(hdr->size);  in sfree_pool()
344 	pool_unlock(pool);  in sfree_pool()
349 	struct pool *pool = NULL;  in sfree()  local
359 			pool = &mp[i];  in sfree()
366 	assert(pool);  in sfree()
367 	sfree_pool(pool, ptr);  in sfree()
370 static void *__smalloc_pool(struct pool *pool, size_t size)  in __smalloc_pool()  argument
378 	pool_lock(pool);  in __smalloc_pool()
381 	if (nr_blocks > pool->free_blocks)  in __smalloc_pool()
384 	i = pool->next_non_full;  in __smalloc_pool()
387 	while (i < pool->nr_blocks) {  in __smalloc_pool()
390 		if (pool->bitmap[i] == -1U) {  in __smalloc_pool()
392 			pool->next_non_full = i;  in __smalloc_pool()
397 		idx = find_next_zero(pool->bitmap[i], last_idx);  in __smalloc_pool()
398 		if (!blocks_free(pool, i, idx, nr_blocks)) {  in __smalloc_pool()
411 		set_blocks(pool, i, idx, nr_blocks);  in __smalloc_pool()
416 	if (i < pool->nr_blocks) {  in __smalloc_pool()
417 		pool->free_blocks -= nr_blocks;  in __smalloc_pool()
418 		ret = pool->map + offset;  in __smalloc_pool()
421 	pool_unlock(pool);  in __smalloc_pool()
425 static void *smalloc_pool(struct pool *pool, size_t size)  in smalloc_pool()  argument
439 	ptr = __smalloc_pool(pool, alloc_size);  in smalloc_pool()