88 static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,  in dlopen_ioengine()  argument
99 		td_vmsg(td, -1, dlerror(), "dlopen");  in dlopen_ioengine()
125 		td_vmsg(td, -1, dlerror(), "dlsym");  in dlopen_ioengine()
134 struct ioengine_ops *load_ioengine(struct thread_data *td, const char *name)  in load_ioengine()  argument
151 		ops = dlopen_ioengine(td, name);  in load_ioengine()
174 void free_ioengine(struct thread_data *td)  in free_ioengine()  argument
176 	dprint(FD_IO, "free ioengine %s\n", td->io_ops->name);  in free_ioengine()
178 	if (td->eo && td->io_ops->options) {  in free_ioengine()
179 		options_free(td->io_ops->options, td->eo);  in free_ioengine()
180 		free(td->eo);  in free_ioengine()
181 		td->eo = NULL;  in free_ioengine()
184 	if (td->io_ops->dlhandle)  in free_ioengine()
185 		dlclose(td->io_ops->dlhandle);  in free_ioengine()
187 	free(td->io_ops);  in free_ioengine()
188 	td->io_ops = NULL;  in free_ioengine()
191 void close_ioengine(struct thread_data *td)  in close_ioengine()  argument
193 	dprint(FD_IO, "close ioengine %s\n", td->io_ops->name);  in close_ioengine()
195 	if (td->io_ops->cleanup) {  in close_ioengine()
196 		td->io_ops->cleanup(td);  in close_ioengine()
197 		td->io_ops->data = NULL;  in close_ioengine()
200 	free_ioengine(td);  in close_ioengine()
203 int td_io_prep(struct thread_data *td, struct io_u *io_u)  in td_io_prep()  argument
206 	fio_ro_check(td, io_u);  in td_io_prep()
208 	lock_file(td, io_u->file, io_u->ddir);  in td_io_prep()
210 	if (td->io_ops->prep) {  in td_io_prep()
211 		int ret = td->io_ops->prep(td, io_u);  in td_io_prep()
215 			unlock_file(td, io_u->file);  in td_io_prep()
222 int td_io_getevents(struct thread_data *td, unsigned int min, unsigned int max,  in td_io_getevents()  argument
234 	if (td->done)  in td_io_getevents()
237 	if (min > 0 && td->io_ops->commit) {  in td_io_getevents()
238 		r = td->io_ops->commit(td);  in td_io_getevents()
242 	if (max > td->cur_depth)  in td_io_getevents()
243 		max = td->cur_depth;  in td_io_getevents()
248 	if (max && td->io_ops->getevents)  in td_io_getevents()
249 		r = td->io_ops->getevents(td, min, max, t);  in td_io_getevents()
256 		td->io_u_in_flight -= r;  in td_io_getevents()
257 		io_u_mark_complete(td, r);  in td_io_getevents()
259 		td_verror(td, r, "get_events");  in td_io_getevents()
265 int td_io_queue(struct thread_data *td, struct io_u *io_u)  in td_io_queue()  argument
270 	fio_ro_check(td, io_u);  in td_io_queue()
280 	log_io_u(td, io_u);  in td_io_queue()
285 	if (td->io_ops->flags & FIO_SYNCIO) {  in td_io_queue()
286 		if (fio_fill_issue_time(td))  in td_io_queue()
292 		if (td->o.read_iolog_file)  in td_io_queue()
293 			memcpy(&td->last_issue, &io_u->issue_time,  in td_io_queue()
298 		td->io_issues[acct_ddir(io_u)]++;  in td_io_queue()
299 		td->io_issue_bytes[acct_ddir(io_u)] += io_u->xfer_buflen;  in td_io_queue()
302 	ret = td->io_ops->queue(td, io_u);  in td_io_queue()
304 	unlock_file(td, io_u->file);  in td_io_queue()
307 		td->io_issues[acct_ddir(io_u)]--;  in td_io_queue()
308 		td->io_issue_bytes[acct_ddir(io_u)] -= io_u->xfer_buflen;  in td_io_queue()
315 	if (io_u->error && !td->error)  in td_io_queue()
316 		td_verror(td, io_u->error, "td_io_queue");  in td_io_queue()
324 	if (io_u->error == EINVAL && td->io_issues[io_u->ddir & 1] == 1 &&  in td_io_queue()
325 	    td->o.odirect) {  in td_io_queue()
331 	if (!td->io_ops->commit || io_u->ddir == DDIR_TRIM) {  in td_io_queue()
332 		io_u_mark_submit(td, 1);  in td_io_queue()
333 		io_u_mark_complete(td, 1);  in td_io_queue()
338 			io_u_mark_depth(td, 1);  in td_io_queue()
339 			td->ts.total_io_u[io_u->ddir]++;  in td_io_queue()
345 			td->io_u_queued++;  in td_io_queue()
346 			td->ts.total_io_u[io_u->ddir]++;  in td_io_queue()
349 		if (td->io_u_queued >= td->o.iodepth_batch) {  in td_io_queue()
350 			r = td_io_commit(td);  in td_io_queue()
356 	if ((td->io_ops->flags & FIO_SYNCIO) == 0) {  in td_io_queue()
357 		if (fio_fill_issue_time(td))  in td_io_queue()
363 		if (td->o.read_iolog_file)  in td_io_queue()
364 			memcpy(&td->last_issue, &io_u->issue_time,  in td_io_queue()
371 int td_io_init(struct thread_data *td)  in td_io_init()  argument
375 	if (td->io_ops->init) {  in td_io_init()
376 		ret = td->io_ops->init(td);  in td_io_init()
377 		if (ret && td->o.iodepth > 1) {  in td_io_init()
381 		if (!td->error)  in td_io_init()
382 			td->error = ret;  in td_io_init()
385 	if (!ret && (td->io_ops->flags & FIO_NOIO))  in td_io_init()
386 		td->flags |= TD_F_NOIO;  in td_io_init()
391 int td_io_commit(struct thread_data *td)  in td_io_commit()  argument
395 	dprint(FD_IO, "calling ->commit(), depth %d\n", td->cur_depth);  in td_io_commit()
397 	if (!td->cur_depth || !td->io_u_queued)  in td_io_commit()
400 	io_u_mark_depth(td, td->io_u_queued);  in td_io_commit()
402 	if (td->io_ops->commit) {  in td_io_commit()
403 		ret = td->io_ops->commit(td);  in td_io_commit()
405 			td_verror(td, -ret, "io commit");  in td_io_commit()
411 	td->io_u_in_flight += td->io_u_queued;  in td_io_commit()
412 	td->io_u_queued = 0;  in td_io_commit()
417 int td_io_open_file(struct thread_data *td, struct fio_file *f)  in td_io_open_file()  argument
422 	if (td->io_ops->open_file(td, f)) {  in td_io_open_file()
423 		if (td->error == EINVAL && td->o.odirect)  in td_io_open_file()
425 		if (td->error == EMFILE) {  in td_io_open_file()
427 				" at %u of %u)\n", td->nr_open_files,  in td_io_open_file()
428 							td->o.nr_files);  in td_io_open_file()
436 	fio_file_reset(td, f);  in td_io_open_file()
441 	td->nr_open_files++;  in td_io_open_file()
445 		if (td_random(td)) {  in td_io_open_file()
451 	if (td->io_ops->flags & FIO_DISKLESSIO)  in td_io_open_file()
454 	if (td->o.invalidate_cache && file_invalidate_cache(td, f))  in td_io_open_file()
457 	if (td->o.fadvise_hint &&  in td_io_open_file()
461 		if (td_random(td))  in td_io_open_file()
467 			td_verror(td, errno, "fadvise");  in td_io_open_file()
477 	if (td->o.odirect) {  in td_io_open_file()
481 			td_verror(td, ret, "fio_set_odirect");  in td_io_open_file()
489 	log_file(td, f, FIO_LOG_OPEN_FILE);  in td_io_open_file()
493 	if (td->io_ops->close_file)  in td_io_open_file()
494 		td->io_ops->close_file(td, f);  in td_io_open_file()
498 int td_io_close_file(struct thread_data *td, struct fio_file *f)  in td_io_close_file()  argument
501 		log_file(td, f, FIO_LOG_CLOSE_FILE);  in td_io_close_file()
510 	if (td->o.file_lock_mode != FILE_LOCK_NONE)  in td_io_close_file()
511 		unlock_file_all(td, f);  in td_io_close_file()
513 	return put_file(td, f);  in td_io_close_file()
516 int td_io_unlink_file(struct thread_data *td, struct fio_file *f)  in td_io_unlink_file()  argument
518 	if (td->io_ops->unlink_file)  in td_io_unlink_file()
519 		return td->io_ops->unlink_file(td, f);  in td_io_unlink_file()
524 int td_io_get_file_size(struct thread_data *td, struct fio_file *f)  in td_io_get_file_size()  argument
526 	if (!td->io_ops->get_file_size)  in td_io_get_file_size()
529 	return td->io_ops->get_file_size(td, f);  in td_io_get_file_size()
532 static int do_sync_file_range(const struct thread_data *td,  in do_sync_file_range()  argument
543 	return sync_file_range(f->fd, offset, nbytes, td->o.sync_file_range);  in do_sync_file_range()
546 int do_io_u_sync(const struct thread_data *td, struct io_u *io_u)  in do_io_u_sync()  argument
560 		ret = do_sync_file_range(td, io_u->file);  in do_io_u_sync()
572 int do_io_u_trim(const struct thread_data *td, struct io_u *io_u)  in do_io_u_trim()  argument
593 	struct thread_data td;  in fio_show_ioengine_help()  local
600 			td.io_ops = flist_entry(entry, struct ioengine_ops,  in fio_show_ioengine_help()
602 			log_info("\t%s\n", td.io_ops->name);  in fio_show_ioengine_help()
612 	memset(&td, 0, sizeof(td));  in fio_show_ioengine_help()
614 	td.io_ops = load_ioengine(&td, engine);  in fio_show_ioengine_help()
615 	if (!td.io_ops) {  in fio_show_ioengine_help()
620 	if (td.io_ops->options)  in fio_show_ioengine_help()
621 		ret = show_cmd_help(td.io_ops->options, sep);  in fio_show_ioengine_help()
623 		log_info("IO engine %s has no options\n", td.io_ops->name);  in fio_show_ioengine_help()
625 	free_ioengine(&td);  in fio_show_ioengine_help()