27 static inline void clear_error(struct thread_data *td)  in clear_error()  argument
29 	td->error = 0;  in clear_error()
30 	td->verror[0] = '\0';  in clear_error()
36 static int extend_file(struct thread_data *td, struct fio_file *f)  in extend_file()  argument
53 	if (td_read(td) ||  in extend_file()
54 	   (td_write(td) && td->o.overwrite && !td->o.file_append) ||  in extend_file()
55 	    (td_write(td) && td->io_ops->flags & FIO_NOEXTEND))  in extend_file()
57 	if (td_write(td) && !td->o.overwrite && !td->o.file_append)  in extend_file()
62 		if ((td_io_unlink_file(td, f) < 0) && (errno != ENOENT)) {  in extend_file()
63 			td_verror(td, errno, "unlink");  in extend_file()
79 		td_verror(td, errno, "open");  in extend_file()
84 	if (!td->o.fill_device) {  in extend_file()
85 		switch (td->o.fallocate_mode) {  in extend_file()
109 				td_verror(td, errno, "fallocate");  in extend_file()
115 				td->o.fallocate_mode);  in extend_file()
128 	if (!td->o.fill_device) {  in extend_file()
133 				td_verror(td, errno, "ftruncate");  in extend_file()
139 	b = malloc(td->o.max_bs[DDIR_WRITE]);  in extend_file()
142 	while (left && !td->terminate) {  in extend_file()
143 		bs = td->o.max_bs[DDIR_WRITE];  in extend_file()
147 		fill_io_buffer(td, b, bs, bs);  in extend_file()
159 					if (td->o.fill_device)  in extend_file()
165 				td_verror(td, errno, "write");  in extend_file()
167 				td_verror(td, EIO, "write");  in extend_file()
173 	if (td->terminate) {  in extend_file()
175 		td_io_unlink_file(td, f);  in extend_file()
176 	} else if (td->o.create_fsync) {  in extend_file()
178 			td_verror(td, errno, "fsync");  in extend_file()
182 	if (td->o.fill_device && !td_write(td)) {  in extend_file()
184 		if (td_io_get_file_size(td, f))  in extend_file()
201 static int pre_read_file(struct thread_data *td, struct fio_file *f)  in pre_read_file()  argument
208 	if (td->io_ops->flags & FIO_PIPEIO)  in pre_read_file()
212 		if (td->io_ops->open_file(td, f)) {  in pre_read_file()
219 	old_runstate = td_bump_runstate(td, TD_PRE_READING);  in pre_read_file()
221 	bs = td->o.max_bs[DDIR_READ];  in pre_read_file()
226 		td_verror(td, errno, "lseek");  in pre_read_file()
234 	while (left && !td->terminate) {  in pre_read_file()
244 			td_verror(td, EIO, "pre_read");  in pre_read_file()
250 	td_restore_runstate(td, old_runstate);  in pre_read_file()
253 		td->io_ops->close_file(td, f);  in pre_read_file()
259 static unsigned long long get_rand_file_size(struct thread_data *td)  in get_rand_file_size()  argument
264 	r = __rand(&td->file_size_state);  in get_rand_file_size()
265 	sized = td->o.file_size_high - td->o.file_size_low;  in get_rand_file_size()
267 	ret += td->o.file_size_low;  in get_rand_file_size()
268 	ret -= (ret % td->o.rw_min_bs);  in get_rand_file_size()
272 static int file_size(struct thread_data *td, struct fio_file *f)  in file_size()  argument
277 		td_verror(td, errno, "fstat");  in file_size()
285 static int bdev_size(struct thread_data *td, struct fio_file *f)  in bdev_size()  argument
290 	if (td->io_ops->open_file(td, f)) {  in bdev_size()
298 		td_verror(td, r, "blockdev_size");  in bdev_size()
308 	td->io_ops->close_file(td, f);  in bdev_size()
311 	td->io_ops->close_file(td, f);  in bdev_size()
315 static int char_size(struct thread_data *td, struct fio_file *f)  in char_size()  argument
321 	if (td->io_ops->open_file(td, f)) {  in char_size()
329 		td_verror(td, r, "chardev_size");  in char_size()
339 	td->io_ops->close_file(td, f);  in char_size()
342 	td->io_ops->close_file(td, f);  in char_size()
350 static int get_file_size(struct thread_data *td, struct fio_file *f)  in get_file_size()  argument
358 		ret = file_size(td, f);  in get_file_size()
360 		ret = bdev_size(td, f);  in get_file_size()
362 		ret = char_size(td, f);  in get_file_size()
370 		log_err("%s: offset extends end (%llu > %llu)\n", td->o.name,  in get_file_size()
380 static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f,  in __file_invalidate_cache()  argument
401 	if (td->io_ops->invalidate)  in __file_invalidate_cache()
402 		ret = td->io_ops->invalidate(td, f);  in __file_invalidate_cache()
433 int file_invalidate_cache(struct thread_data *td, struct fio_file *f)  in file_invalidate_cache()  argument
438 	return __file_invalidate_cache(td, f, -1ULL, -1ULL);  in file_invalidate_cache()
441 int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)  in generic_close_file()  argument
489 static int file_close_shadow_fds(struct thread_data *td)  in file_close_shadow_fds()  argument
495 	for_each_file(td, f, i) {  in file_close_shadow_fds()
507 int generic_open_file(struct thread_data *td, struct fio_file *f)  in generic_open_file()  argument
515 	if (td_trim(td) && f->filetype != FIO_TYPE_BD) {  in generic_open_file()
521 		if (td_rw(td)) {  in generic_open_file()
530 		if (td_write(td))  in generic_open_file()
534 	if (td_trim(td))  in generic_open_file()
536 	if (td->o.odirect)  in generic_open_file()
538 	if (td->o.oatomic) {  in generic_open_file()
540 			td_verror(td, EINVAL, "OS does not support atomic IO");  in generic_open_file()
545 	if (td->o.sync_io)  in generic_open_file()
547 	if (td->o.create_on_open)  in generic_open_file()
554 	if (td_write(td)) {  in generic_open_file()
565 	} else if (td_read(td)) {  in generic_open_file()
588 		if (__e == EMFILE && file_close_shadow_fds(td))  in generic_open_file()
598 		td_verror(td, __e, buf);  in generic_open_file()
621 				ret = generic_close_file(td, f);  in generic_open_file()
630 int generic_get_file_size(struct thread_data *td, struct fio_file *f)  in generic_get_file_size()  argument
632 	return get_file_size(td, f);  in generic_get_file_size()
638 static int get_file_sizes(struct thread_data *td)  in get_file_sizes()  argument
644 	for_each_file(td, f, i) {  in get_file_sizes()
648 		if (td_io_get_file_size(td, f)) {  in get_file_sizes()
649 			if (td->error != ENOENT) {  in get_file_sizes()
650 				log_err("%s\n", td->verror);  in get_file_sizes()
654 			clear_error(td);  in get_file_sizes()
657 		if (f->real_file_size == -1ULL && td->o.size)  in get_file_sizes()
658 			f->real_file_size = td->o.size / td->o.nr_files;  in get_file_sizes()
674 static unsigned long long get_fs_free_counts(struct thread_data *td)  in get_fs_free_counts()  argument
683 	for_each_file(td, f, i) {  in get_fs_free_counts()
740 uint64_t get_start_offset(struct thread_data *td, struct fio_file *f)  in get_start_offset()  argument
742 	struct thread_options *o = &td->o;  in get_start_offset()
747 	return td->o.start_offset +  in get_start_offset()
748 		td->subjob_number * td->o.offset_increment;  in get_start_offset()
754 int setup_files(struct thread_data *td)  in setup_files()  argument
757 	struct thread_options *o = &td->o;  in setup_files()
762 	const unsigned int bs = td_min_bs(td);  in setup_files()
767 	old_state = td_bump_runstate(td, TD_SETTING_UP);  in setup_files()
777 	if (td->io_ops->setup)  in setup_files()
778 		err = td->io_ops->setup(td);  in setup_files()
780 		err = get_file_sizes(td);  in setup_files()
790 	for_each_file(td, f, i) {  in setup_files()
798 		td->fill_device_size = get_fs_free_counts(td);  in setup_files()
804 	    !(td->io_ops->flags & FIO_NOIO) && !o->fill_device &&  in setup_files()
807 		td_verror(td, EINVAL, "total_file_size");  in setup_files()
832 	for_each_file(td, f, i) {  in setup_files()
833 		f->file_offset = get_start_offset(td, f);  in setup_files()
862 				f->io_size = get_rand_file_size(td)  in setup_files()
878 		    !(td->io_ops->flags & FIO_DISKLESSIO)) {  in setup_files()
891 	if (o->size < td_min_bs(td)) {  in setup_files()
906 		for_each_file(td, f, i) {  in setup_files()
920 			err = extend_file(td, f);  in setup_files()
924 			err = __file_invalidate_cache(td, f, old_len,  in setup_files()
952 			td->total_io_size = o->io_limit * o->loops;  in setup_files()
954 			td->total_io_size = o->size * o->loops;  in setup_files()
959 		td->done = 1;  in setup_files()
961 	td_restore_runstate(td, old_state);  in setup_files()
966 	td_restore_runstate(td, old_state);  in setup_files()
970 int pre_read_files(struct thread_data *td)  in pre_read_files()  argument
977 	for_each_file(td, f, i) {  in pre_read_files()
978 		pre_read_file(td, f);  in pre_read_files()
984 static int __init_rand_distribution(struct thread_data *td, struct fio_file *f)  in __init_rand_distribution()  argument
990 	range_size = min(td->o.min_bs[DDIR_READ], td->o.min_bs[DDIR_WRITE]);  in __init_rand_distribution()
995 	seed = jhash(f->file_name, strlen(f->file_name), 0) * td->thread_number;  in __init_rand_distribution()
996 	if (!td->o.rand_repeatable)  in __init_rand_distribution()
997 		seed = td->rand_seeds[4];  in __init_rand_distribution()
999 	if (td->o.random_distribution == FIO_RAND_DIST_ZIPF)  in __init_rand_distribution()
1000 		zipf_init(&f->zipf, nranges, td->o.zipf_theta.u.f, seed);  in __init_rand_distribution()
1002 		pareto_init(&f->zipf, nranges, td->o.pareto_h.u.f, seed);  in __init_rand_distribution()
1007 static int init_rand_distribution(struct thread_data *td)  in init_rand_distribution()  argument
1013 	if (td->o.random_distribution == FIO_RAND_DIST_RANDOM)  in init_rand_distribution()
1016 	state = td_bump_runstate(td, TD_SETTING_UP);  in init_rand_distribution()
1018 	for_each_file(td, f, i)  in init_rand_distribution()
1019 		__init_rand_distribution(td, f);  in init_rand_distribution()
1021 	td_restore_runstate(td, state);  in init_rand_distribution()
1026 int init_random_map(struct thread_data *td)  in init_random_map()  argument
1032 	if (init_rand_distribution(td))  in init_random_map()
1034 	if (!td_random(td))  in init_random_map()
1037 	for_each_file(td, f, i) {  in init_random_map()
1040 		blocks = fsize / (unsigned long long) td->o.rw_min_bs;  in init_random_map()
1042 		if (td->o.random_generator == FIO_RAND_GEN_LFSR) {  in init_random_map()
1045 			seed = td->rand_seeds[FIO_RAND_BLOCK_OFF];  in init_random_map()
1051 		} else if (!td->o.norandommap) {  in init_random_map()
1057 		} else if (td->o.norandommap)  in init_random_map()
1060 		if (!td->o.softrandommap) {  in init_random_map()
1075 void close_files(struct thread_data *td)  in close_files()  argument
1080 	for_each_file(td, f, i) {  in close_files()
1082 			td_io_close_file(td, f);  in close_files()
1086 void close_and_free_files(struct thread_data *td)  in close_and_free_files()  argument
1093 	for_each_file(td, f, i) {  in close_and_free_files()
1094 		if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {  in close_and_free_files()
1096 			td_io_unlink_file(td, f);  in close_and_free_files()
1100 			td_io_close_file(td, f);  in close_and_free_files()
1104 		if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {  in close_and_free_files()
1106 			td_io_unlink_file(td, f);  in close_and_free_files()
1118 	td->o.filename = NULL;  in close_and_free_files()
1119 	free(td->files);  in close_and_free_files()
1120 	free(td->file_locks);  in close_and_free_files()
1121 	td->files_index = 0;  in close_and_free_files()
1122 	td->files = NULL;  in close_and_free_files()
1123 	td->file_locks = NULL;  in close_and_free_files()
1124 	td->o.file_lock_mode = FILE_LOCK_NONE;  in close_and_free_files()
1125 	td->o.nr_files = 0;  in close_and_free_files()
1220 static struct fio_file *alloc_new_file(struct thread_data *td)  in alloc_new_file()  argument
1233 	fio_file_reset(td, f);  in alloc_new_file()
1237 int add_file(struct thread_data *td, const char *fname, int numjob, int inc)  in add_file()  argument
1239 	int cur_files = td->files_index;  in add_file()
1246 	if (td->o.directory)  in add_file()
1247 		len = set_name_idx(file_name, td->o.directory, numjob);  in add_file()
1255 	f = alloc_new_file(td);  in add_file()
1257 	if (td->files_size <= td->files_index) {  in add_file()
1258 		unsigned int new_size = td->o.nr_files + 1;  in add_file()
1262 		td->files = realloc(td->files, new_size * sizeof(f));  in add_file()
1263 		if (td->files == NULL) {  in add_file()
1267 		if (td->o.file_lock_mode != FILE_LOCK_NONE) {  in add_file()
1268 			td->file_locks = realloc(td->file_locks, new_size);  in add_file()
1269 			if (!td->file_locks) {  in add_file()
1273 			td->file_locks[cur_files] = FILE_LOCK_NONE;  in add_file()
1275 		td->files_size = new_size;  in add_file()
1277 	td->files[cur_files] = f;  in add_file()
1283 	if (td->io_ops && (td->io_ops->flags & FIO_DISKLESSIO))  in add_file()
1294 	switch (td->o.file_lock_mode) {  in add_file()
1304 		log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode);  in add_file()
1308 	td->files_index++;  in add_file()
1310 		td->nr_normal_files++;  in add_file()
1315 		td->o.nr_files++;  in add_file()
1323 int add_file_exclusive(struct thread_data *td, const char *fname)  in add_file_exclusive()  argument
1328 	for_each_file(td, f, i) {  in add_file_exclusive()
1333 	return add_file(td, fname, 0, 1);  in add_file_exclusive()
1343 int put_file(struct thread_data *td, struct fio_file *f)  in put_file()  argument
1358 	if (should_fsync(td) && td->o.fsync_on_close) {  in put_file()
1364 	if (td->io_ops->close_file)  in put_file()
1365 		ret = td->io_ops->close_file(td, f);  in put_file()
1370 	td->nr_open_files--;  in put_file()
1376 void lock_file(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir)  in lock_file()  argument
1378 	if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)  in lock_file()
1381 	if (td->o.file_lock_mode == FILE_LOCK_READWRITE) {  in lock_file()
1386 	} else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)  in lock_file()
1389 	td->file_locks[f->fileno] = td->o.file_lock_mode;  in lock_file()
1392 void unlock_file(struct thread_data *td, struct fio_file *f)  in unlock_file()  argument
1394 	if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)  in unlock_file()
1397 	if (td->o.file_lock_mode == FILE_LOCK_READWRITE)  in unlock_file()
1399 	else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)  in unlock_file()
1402 	td->file_locks[f->fileno] = FILE_LOCK_NONE;  in unlock_file()
1405 void unlock_file_all(struct thread_data *td, struct fio_file *f)  in unlock_file_all()  argument
1407 	if (td->o.file_lock_mode == FILE_LOCK_NONE || !td->file_locks)  in unlock_file_all()
1409 	if (td->file_locks[f->fileno] != FILE_LOCK_NONE)  in unlock_file_all()
1410 		unlock_file(td, f);  in unlock_file_all()
1413 static int recurse_dir(struct thread_data *td, const char *dirname)  in recurse_dir()  argument
1424 		td_verror(td, errno, buf);  in recurse_dir()
1439 				td_verror(td, errno, "stat");  in recurse_dir()
1446 			add_file(td, full_path, 0, 1);  in recurse_dir()
1452 		ret = recurse_dir(td, full_path);  in recurse_dir()
1461 int add_dir_files(struct thread_data *td, const char *path)  in add_dir_files()  argument
1463 	int ret = recurse_dir(td, path);  in add_dir_files()
1466 		log_info("fio: opendir added %d files\n", td->o.nr_files);  in add_dir_files()
1471 void dup_files(struct thread_data *td, struct thread_data *org)  in dup_files()  argument
1481 	td->files = malloc(org->files_index * sizeof(f));  in dup_files()
1483 	if (td->o.file_lock_mode != FILE_LOCK_NONE)  in dup_files()
1484 		td->file_locks = malloc(org->files_index);  in dup_files()
1489 		__f = alloc_new_file(td);  in dup_files()
1501 		if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)  in dup_files()
1503 		else if (td->o.file_lock_mode == FILE_LOCK_READWRITE)  in dup_files()
1506 		td->files[i] = __f;  in dup_files()
1513 int get_fileno(struct thread_data *td, const char *fname)  in get_fileno()  argument
1518 	for_each_file(td, f, i)  in get_fileno()
1528 void free_release_files(struct thread_data *td)  in free_release_files()  argument
1530 	close_files(td);  in free_release_files()
1531 	td->o.nr_files = 0;  in free_release_files()
1532 	td->o.open_files = 0;  in free_release_files()
1533 	td->files_index = 0;  in free_release_files()
1534 	td->nr_normal_files = 0;  in free_release_files()
1537 void fio_file_reset(struct thread_data *td, struct fio_file *f)  in fio_file_reset()  argument
1549 		lfsr_reset(&f->lfsr, td->rand_seeds[FIO_RAND_BLOCK_OFF]);  in fio_file_reset()
1552 int fio_files_done(struct thread_data *td)  in fio_files_done()  argument
1557 	for_each_file(td, f, i)  in fio_files_done()