59 static inline void ring_inc(struct libaio_data *ld, unsigned int *val,  in ring_inc()  argument
62 	if (ld->is_pow2)  in ring_inc()
63 		*val = (*val + add) & (ld->entries - 1);  in ring_inc()
65 		*val = (*val + add) % ld->entries;  in ring_inc()
84 	struct libaio_data *ld = td->io_ops->data;  in fio_libaio_event()  local
88 	ev = ld->aio_events + event;  in fio_libaio_event()
146 	struct libaio_data *ld = td->io_ops->data;  in fio_libaio_getevents()  local
160 		    && ((struct aio_ring *)(ld->aio_ctx))->magic  in fio_libaio_getevents()
162 			r = user_io_getevents(ld->aio_ctx, max,  in fio_libaio_getevents()
163 				ld->aio_events + events);  in fio_libaio_getevents()
165 			r = io_getevents(ld->aio_ctx, actual_min,  in fio_libaio_getevents()
166 				max, ld->aio_events + events, lt);  in fio_libaio_getevents()
182 	struct libaio_data *ld = td->io_ops->data;  in fio_libaio_queue()  local
186 	if (ld->queued == td->o.iodepth)  in fio_libaio_queue()
196 		if (ld->queued)  in fio_libaio_queue()
204 		if (ld->queued)  in fio_libaio_queue()
211 	ld->iocbs[ld->head] = &io_u->iocb;  in fio_libaio_queue()
212 	ld->io_us[ld->head] = io_u;  in fio_libaio_queue()
213 	ring_inc(ld, &ld->head, 1);  in fio_libaio_queue()
214 	ld->queued++;  in fio_libaio_queue()
239 	struct libaio_data *ld = td->io_ops->data;  in fio_libaio_commit()  local
245 	if (!ld->queued)  in fio_libaio_commit()
249 		long nr = ld->queued;  in fio_libaio_commit()
251 		nr = min((unsigned int) nr, ld->entries - ld->tail);  in fio_libaio_commit()
252 		io_us = ld->io_us + ld->tail;  in fio_libaio_commit()
253 		iocbs = ld->iocbs + ld->tail;  in fio_libaio_commit()
255 		ret = io_submit(ld->aio_ctx, nr, iocbs);  in fio_libaio_commit()
260 			ld->queued -= ret;  in fio_libaio_commit()
261 			ring_inc(ld, &ld->tail, ret);  in fio_libaio_commit()
278 			if (ld->queued) {  in fio_libaio_commit()
297 			if (ld->queued)  in fio_libaio_commit()
302 	} while (ld->queued);  in fio_libaio_commit()
309 	struct libaio_data *ld = td->io_ops->data;  in fio_libaio_cancel()  local
311 	return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events);  in fio_libaio_cancel()
316 	struct libaio_data *ld = td->io_ops->data;  in fio_libaio_cleanup()  local
318 	if (ld) {  in fio_libaio_cleanup()
319 		io_destroy(ld->aio_ctx);  in fio_libaio_cleanup()
320 		free(ld->aio_events);  in fio_libaio_cleanup()
321 		free(ld->iocbs);  in fio_libaio_cleanup()
322 		free(ld->io_us);  in fio_libaio_cleanup()
323 		free(ld);  in fio_libaio_cleanup()
330 	struct libaio_data *ld;  in fio_libaio_init()  local
333 	ld = calloc(1, sizeof(*ld));  in fio_libaio_init()
341 		err = io_queue_init(INT_MAX, &ld->aio_ctx);  in fio_libaio_init()
343 		err = io_queue_init(td->o.iodepth, &ld->aio_ctx);  in fio_libaio_init()
347 		free(ld);  in fio_libaio_init()
351 	ld->entries = td->o.iodepth;  in fio_libaio_init()
352 	ld->is_pow2 = is_power_of_2(ld->entries);  in fio_libaio_init()
353 	ld->aio_events = calloc(ld->entries, sizeof(struct io_event));  in fio_libaio_init()
354 	ld->iocbs = calloc(ld->entries, sizeof(struct iocb *));  in fio_libaio_init()
355 	ld->io_us = calloc(ld->entries, sizeof(struct io_u *));  in fio_libaio_init()
357 	td->io_ops->data = ld;  in fio_libaio_init()