54 			   enum fio_ddir ddir)  in last_block()  argument
59 	assert(ddir_rw(ddir));  in last_block()
71 	if (td->o.min_bs[ddir] > td->o.ba[ddir])  in last_block()
72 		max_size -= td->o.min_bs[ddir] - td->o.ba[ddir];  in last_block()
74 	max_blocks = max_size / (uint64_t) td->o.ba[ddir];  in last_block()
87 				  enum fio_ddir ddir, uint64_t *b)  in __get_next_rand_offset()  argument
94 		lastb = last_block(td, f, ddir);  in __get_next_rand_offset()
137 				       struct fio_file *f, enum fio_ddir ddir,  in __get_next_rand_offset_zipf()  argument
145 					 struct fio_file *f, enum fio_ddir ddir,  in __get_next_rand_offset_pareto()  argument
161 			       enum fio_ddir ddir, uint64_t *b)  in get_off_from_method()  argument
164 		return __get_next_rand_offset(td, f, ddir, b);  in get_off_from_method()
166 		return __get_next_rand_offset_zipf(td, f, ddir, b);  in get_off_from_method()
168 		return __get_next_rand_offset_pareto(td, f, ddir, b);  in get_off_from_method()
192 static int should_do_random(struct thread_data *td, enum fio_ddir ddir)  in should_do_random()  argument
197 	if (td->o.perc_rand[ddir] == 100)  in should_do_random()
200 	r = __rand(&td->seq_rand_state[ddir]);  in should_do_random()
203 	return v <= td->o.perc_rand[ddir];  in should_do_random()
207 				enum fio_ddir ddir, uint64_t *b)  in get_next_rand_offset()  argument
213 		return get_off_from_method(td, f, ddir, b);  in get_next_rand_offset()
227 		ret = get_off_from_method(td, f, ddir, &r->off);  in get_next_rand_offset()
245 			       enum fio_ddir ddir, uint64_t *b)  in get_next_rand_block()  argument
247 	if (!get_next_rand_offset(td, f, ddir, b))  in get_next_rand_block()
252 		if (!get_next_rand_offset(td, f, ddir, b))  in get_next_rand_block()
257 			f->file_name, (unsigned long long) f->last_pos[ddir],  in get_next_rand_block()
263 			       enum fio_ddir ddir, uint64_t *offset)  in get_next_seq_offset()  argument
267 	assert(ddir_rw(ddir));  in get_next_seq_offset()
269 	if (f->last_pos[ddir] >= f->io_size + get_start_offset(td, f) &&  in get_next_seq_offset()
271 		f->last_pos[ddir] = f->last_pos[ddir] - f->io_size;  in get_next_seq_offset()
273 	if (f->last_pos[ddir] < f->real_file_size) {  in get_next_seq_offset()
276 		if (f->last_pos[ddir] == f->file_offset && o->ddir_seq_add < 0)  in get_next_seq_offset()
277 			f->last_pos[ddir] = f->real_file_size;  in get_next_seq_offset()
279 		pos = f->last_pos[ddir] - f->file_offset;  in get_next_seq_offset()
300 			  enum fio_ddir ddir, int rw_seq,  in get_next_block()  argument
307 	assert(ddir_rw(ddir));  in get_next_block()
313 			if (should_do_random(td, ddir)) {  in get_next_block()
314 				ret = get_next_rand_block(td, f, ddir, &b);  in get_next_block()
319 				ret = get_next_seq_offset(td, f, ddir, &offset);  in get_next_block()
321 					ret = get_next_rand_block(td, f, ddir, &b);  in get_next_block()
325 			ret = get_next_seq_offset(td, f, ddir, &offset);  in get_next_block()
332 			ret = get_next_seq_offset(td, f, ddir, &offset);  in get_next_block()
334 				ret = get_next_rand_block(td, f, ddir, &b);  in get_next_block()
338 			if (f->last_start[ddir] != -1ULL)  in get_next_block()
339 				offset = f->last_start[ddir] - f->file_offset;  in get_next_block()
353 			io_u->offset = b * td->o.ba[ddir];  in get_next_block()
372 	enum fio_ddir ddir = io_u->ddir;  in __get_next_offset()  local
375 	assert(ddir_rw(ddir));  in __get_next_offset()
382 	if (get_next_block(td, io_u, ddir, rw_seq_hit, is_random))  in __get_next_offset()
427 	int ddir = io_u->ddir;  in __get_next_buflen()  local
432 	assert(ddir_rw(ddir));  in __get_next_buflen()
435 		ddir = is_random ? DDIR_WRITE: DDIR_READ;  in __get_next_buflen()
437 	minbs = td->o.min_bs[ddir];  in __get_next_buflen()
438 	maxbs = td->o.max_bs[ddir];  in __get_next_buflen()
452 		if (!td->o.bssplit_nr[ddir]) {  in __get_next_buflen()
461 			for (i = 0; i < td->o.bssplit_nr[ddir]; i++) {  in __get_next_buflen()
462 				struct bssplit *bsp = &td->o.bssplit[ddir][i];  in __get_next_buflen()
548 static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir)  in rate_ddir()  argument
550 	enum fio_ddir odir = ddir ^ 1;  in rate_ddir()
553 	assert(ddir_rw(ddir));  in rate_ddir()
555 	if (td->rate_pending_usleep[ddir] <= 0)  in rate_ddir()
556 		return ddir;  in rate_ddir()
573 		if (td->rate_pending_usleep[ddir] <=  in rate_ddir()
575 			usec = td->rate_pending_usleep[ddir];  in rate_ddir()
578 			ddir = odir;  in rate_ddir()
581 		usec = td->rate_pending_usleep[ddir];  in rate_ddir()
587 	td->rate_pending_usleep[ddir] -= usec;  in rate_ddir()
589 	odir = ddir ^ 1;  in rate_ddir()
593 	if (ddir == DDIR_TRIM)  in rate_ddir()
596 	return ddir;  in rate_ddir()
606 	enum fio_ddir ddir;  in get_rw_ddir()  local
642 			ddir = get_rand_ddir(td);  in get_rw_ddir()
644 			if (ddir != td->rwmix_ddir)  in get_rw_ddir()
647 			td->rwmix_ddir = ddir;  in get_rw_ddir()
649 		ddir = td->rwmix_ddir;  in get_rw_ddir()
651 		ddir = DDIR_READ;  in get_rw_ddir()
653 		ddir = DDIR_WRITE;  in get_rw_ddir()
655 		ddir = DDIR_TRIM;  in get_rw_ddir()
657 	td->rwmix_ddir = rate_ddir(td, ddir);  in get_rw_ddir()
663 	io_u->ddir = io_u->acct_ddir = get_rw_ddir(td);  in set_rw_ddir()
665 	if (io_u->ddir == DDIR_WRITE && (td->io_ops->flags & FIO_BARRIER) &&  in set_rw_ddir()
706 	enum fio_ddir ddir = acct_ddir(__io_u);  in requeue_io_u()  local
713 	if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(ddir))  in requeue_io_u()
714 		td->io_issues[ddir]--;  in requeue_io_u()
737 	if (!ddir_rw(io_u->ddir))  in fill_io_u()
754 		f->last_pos[io_u->ddir] = f->file_offset;  in fill_io_u()
1476 	if (ddir_rw(io_u->ddir)) {  in get_io_u()
1482 		f->last_start[io_u->ddir] = io_u->offset;  in get_io_u()
1483 		f->last_pos[io_u->ddir] = io_u->offset + io_u->buflen;  in get_io_u()
1485 		if (io_u->ddir == DDIR_WRITE) {  in get_io_u()
1497 		} else if (io_u->ddir == DDIR_READ) {  in get_io_u()
1529 	enum error_type_bit eb = td_error_type(io_u->ddir, io_u->error);  in io_u_log_error()
1538 		io_ddir_name(io_u->ddir),  in io_u_log_error()
1593 static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir)  in usec_for_io()  argument
1597 	bytes = td->this_io_bytes[ddir];  in usec_for_io()
1598 	bps = td->rate_bps[ddir];  in usec_for_io()
1608 	enum fio_ddir ddir = io_u->ddir;  in io_completed()  local
1634 	if (ddir_sync(ddir)) {  in io_completed()
1644 	td->last_ddir = ddir;  in io_completed()
1646 	if (!io_u->error && ddir_rw(ddir)) {  in io_completed()
1648 		const enum fio_ddir oddir = ddir ^ 1;  in io_completed()
1651 		td->io_blocks[ddir]++;  in io_completed()
1652 		td->this_io_blocks[ddir]++;  in io_completed()
1653 		td->io_bytes[ddir] += bytes;  in io_completed()
1656 			td->this_io_bytes[ddir] += bytes;  in io_completed()
1658 		if (ddir == DDIR_WRITE) {  in io_completed()
1678 			account_io_completion(td, io_u, icd, ddir, bytes);  in io_completed()
1680 			if (__should_check_rate(td, ddir)) {  in io_completed()
1681 				td->rate_pending_usleep[ddir] =  in io_completed()
1682 					(usec_for_io(td, ddir) -  in io_completed()
1685 			if (ddir != DDIR_TRIM &&  in io_completed()
1693 		icd->bytes_done[ddir] += bytes;  in io_completed()
1706 		enum error_type_bit eb = td_error_type(ddir, icd->error);  in io_completed()
1726 	int ddir;  in init_icd()  local
1734 	for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++)  in init_icd()
1735 		icd->bytes_done[ddir] = 0;  in init_icd()
1774 		int ddir;  in io_u_sync_complete()  local
1776 		for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++)  in io_u_sync_complete()
1777 			bytes[ddir] += icd.bytes_done[ddir];  in io_u_sync_complete()
1816 		int ddir;  in io_u_queued_complete()  local
1818 		for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++)  in io_u_queued_complete()
1819 			bytes[ddir] += icd.bytes_done[ddir];  in io_u_queued_complete()
1834 		add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen,  in io_u_queued()