308 static void oper_list_add(struct io_oper *oper, struct io_oper **list)  in oper_list_add()  argument
311 		*list = oper;  in oper_list_add()
312 		oper->prev = oper->next = oper;  in oper_list_add()
315 	oper->prev = (*list)->prev;  in oper_list_add()
316 	oper->next = *list;  in oper_list_add()
317 	(*list)->prev->next = oper;  in oper_list_add()
318 	(*list)->prev = oper;  in oper_list_add()
322 static void oper_list_del(struct io_oper *oper, struct io_oper **list)  in oper_list_del()  argument
328 	oper->prev->next = oper->next;  in oper_list_del()
329 	oper->next->prev = oper->prev;  in oper_list_del()
330 	if (*list == oper)  in oper_list_del()
331 		*list = oper->next;  in oper_list_del()
380 static int grab_iou(struct io_unit *io, struct io_oper *oper)  in grab_iou()  argument
387 	io->io_oper = oper;  in grab_iou()
406 static inline double oper_mb_trans(struct io_oper *oper)  in oper_mb_trans()  argument
408 	return ((double)oper->started_ios * (double)oper->reclen) /  in oper_mb_trans()
412 static void print_time(struct io_oper *oper)  in print_time()  argument
418 	runtime = time_since_now(&oper->start_time);  in print_time()
419 	mb = oper_mb_trans(oper);  in print_time()
422 		stage_name(oper->rw), oper->file_name, tput, mb, runtime);  in print_time()
463 	struct io_oper *oper = io->io_oper;  in finish_io()  local
470 	oper->num_pending--;  in finish_io()
473 	if (oper->num_pending == 0 &&  in finish_io()
474 	    (oper->started_ios == oper->total_ios || oper->stonewalled)) {  in finish_io()
475 		print_time(oper);  in finish_io()
513 static struct io_unit *find_iou(struct thread_info *t, struct io_oper *oper)  in find_iou()  argument
522 		if (grab_iou(event_io, oper)) {  in find_iou()
539 static int io_oper_wait(struct thread_info *t, struct io_oper *oper)  argument
544 	if (oper == NULL) {
548 	if (oper->num_pending == 0)
565 		if (oper->num_pending == 0)
569 	if (oper->num_err) {
571 			oper->num_err, oper->last_err);
576 off_t random_byte_offset(struct io_oper * oper)  argument
579 	off_t rand_byte = oper->start;
583 	range = (oper->end - oper->start) / (1024 * 1024);
602 	if (rand_byte + oper->reclen > oper->end) {
603 		rand_byte -= oper->reclen;
616 static struct io_unit *build_iocb(struct thread_info *t, struct io_oper *oper)  argument
621 	io = find_iou(t, oper);
627 	switch (oper->rw) {
629 		io_prep_pwrite(&io->iocb, oper->fd, io->buf, oper->reclen,
630 			       oper->last_offset);
631 		oper->last_offset += oper->reclen;
634 		io_prep_pread(&io->iocb, oper->fd, io->buf, oper->reclen,
635 			      oper->last_offset);
636 		oper->last_offset += oper->reclen;
639 		rand_byte = random_byte_offset(oper);
640 		oper->last_offset = rand_byte;
641 		io_prep_pread(&io->iocb, oper->fd, io->buf, oper->reclen,
645 		rand_byte = random_byte_offset(oper);
646 		oper->last_offset = rand_byte;
647 		io_prep_pwrite(&io->iocb, oper->fd, io->buf, oper->reclen,
660 static int finish_oper(struct thread_info *t, struct io_oper *oper)  argument
664 	io_oper_wait(t, oper);
665 	last_err = oper->last_err;
666 	if (oper->num_pending > 0) {
667 		fprintf(stderr, "oper num_pending is %d\n", oper->num_pending);
669 	close(oper->fd);
670 	free(oper);
682 	struct io_oper *oper;  local
684 	oper = malloc(sizeof(*oper));
685 	if (!oper) {
689 	memset(oper, 0, sizeof(*oper));
691 	oper->depth = depth;
692 	oper->start = start;
693 	oper->end = end;
694 	oper->last_offset = oper->start;
695 	oper->fd = fd;
696 	oper->reclen = reclen;
697 	oper->rw = rw;
698 	oper->total_ios = (oper->end - oper->start) / oper->reclen;
699 	oper->file_name = file_name;
701 	return oper;
708 int build_oper(struct thread_info *t, struct io_oper *oper, int num_ios,  argument
714 	if (oper->started_ios == 0)
715 		gettimeofday(&oper->start_time, NULL);
718 		num_ios = oper->total_ios;
720 	if ((oper->started_ios + num_ios) > oper->total_ios)
721 		num_ios = oper->total_ios - oper->started_ios;
724 		io = build_iocb(t, oper);
799 static int restart_oper(struct io_oper *oper)  argument
802 	if (oper->last_err)
806 	switch (oper->rw) {
819 		oper->started_ios = 0;
820 		oper->last_offset = oper->start;
821 		oper->stonewalled = 0;
827 		if (oper->num_pending)
828 			print_time(oper);
830 		oper->rw = new_rw;
836 static int oper_runnable(struct io_oper *oper)  argument
844 	if (oper->started_ios || oper->start == 0)
848 	if (oper->rw >= RWRITE)
850 	ret = fstat(oper->fd, &buf);
855 	if (S_ISREG(buf.st_mode) && buf.st_size < oper->start)
874 	struct io_oper *oper;  local
880 	oper = t->active_opers;
881 	while (oper) {
882 		if (!oper_runnable(oper)) {
883 			oper = oper->next;
884 			if (oper == t->active_opers)
888 		ret = build_oper(t, oper, io_iter, my_iocbs);
892 			oper_list_del(oper, &t->active_opers);
893 			oper_list_add(oper, &built_opers);
894 			oper = t->active_opers;
907 			oper = built_opers;
908 			oper_list_del(oper, &built_opers);
909 			oper_list_add(oper, &t->active_opers);
910 			if (oper->started_ios == oper->total_ios) {
911 				oper_list_del(oper, &t->active_opers);
912 				oper_list_add(oper, &t->finished_opers);
1112 	struct io_oper *oper;  local
1145 			oper = t->active_opers;
1146 			oper->stonewalled = 1;
1147 			oper_list_del(oper, &t->active_opers);
1148 			oper_list_add(oper, &t->finished_opers);
1161 	oper = t->finished_opers;
1163 		if (!oper)
1165 		io_oper_wait(t, oper);
1166 		oper = oper->next;
1167 	} while (oper != t->finished_opers);
1172 	oper = t->finished_opers;
1173 	while (oper) {
1175 			fsync(oper->fd);
1176 		t->stage_mb_trans += oper_mb_trans(oper);
1177 		if (restart_oper(oper)) {
1178 			oper_list_del(oper, &t->finished_opers);
1179 			oper_list_add(oper, &t->active_opers);
1180 			oper = t->finished_opers;
1183 		oper = oper->next;
1184 		if (oper == t->finished_opers)
1218 		oper = t->finished_opers;
1219 		oper_list_del(oper, &t->finished_opers);
1220 		status = finish_oper(t, oper);
1339 	struct io_oper *oper;  local
1518 			oper =
1522 			if (!oper) {
1526 			oper_list_add(oper, &t[thread_index].active_opers);