15 void fio_unpin_memory(struct thread_data *td)  in fio_unpin_memory()  argument
17 	if (td->pinned_mem) {  in fio_unpin_memory()
18 		dprint(FD_MEM, "unpinning %llu bytes\n", td->o.lockmem);  in fio_unpin_memory()
19 		if (munlock(td->pinned_mem, td->o.lockmem) < 0)  in fio_unpin_memory()
21 		munmap(td->pinned_mem, td->o.lockmem);  in fio_unpin_memory()
22 		td->pinned_mem = NULL;  in fio_unpin_memory()
26 int fio_pin_memory(struct thread_data *td)  in fio_pin_memory()  argument
30 	if (!td->o.lockmem)  in fio_pin_memory()
33 	dprint(FD_MEM, "pinning %llu bytes\n", td->o.lockmem);  in fio_pin_memory()
40 		if ((td->o.lockmem + 128 * 1024 * 1024) > phys_mem) {  in fio_pin_memory()
41 			td->o.lockmem = phys_mem - 128 * 1024 * 1024;  in fio_pin_memory()
43 							td->o.lockmem >> 20);  in fio_pin_memory()
47 	td->pinned_mem = mmap(NULL, td->o.lockmem, PROT_READ | PROT_WRITE,  in fio_pin_memory()
49 	if (td->pinned_mem == MAP_FAILED) {  in fio_pin_memory()
51 		td->pinned_mem = NULL;  in fio_pin_memory()
54 	if (mlock(td->pinned_mem, td->o.lockmem) < 0) {  in fio_pin_memory()
56 		munmap(td->pinned_mem, td->o.lockmem);  in fio_pin_memory()
57 		td->pinned_mem = NULL;  in fio_pin_memory()
64 static int alloc_mem_shm(struct thread_data *td, unsigned int total_mem)  in alloc_mem_shm()  argument
69 	if (td->o.mem_type == MEM_SHMHUGE) {  in alloc_mem_shm()
70 		unsigned long mask = td->o.hugepage_size - 1;  in alloc_mem_shm()
76 	td->shm_id = shmget(IPC_PRIVATE, total_mem, flags);  in alloc_mem_shm()
77 	dprint(FD_MEM, "shmget %u, %d\n", total_mem, td->shm_id);  in alloc_mem_shm()
78 	if (td->shm_id < 0) {  in alloc_mem_shm()
79 		td_verror(td, errno, "shmget");  in alloc_mem_shm()
82 		if (td->o.mem_type == MEM_SHMHUGE) {  in alloc_mem_shm()
99 	td->orig_buffer = shmat(td->shm_id, NULL, 0);  in alloc_mem_shm()
100 	dprint(FD_MEM, "shmat %d, %p\n", td->shm_id, td->orig_buffer);  in alloc_mem_shm()
101 	if (td->orig_buffer == (void *) -1) {  in alloc_mem_shm()
102 		td_verror(td, errno, "shmat");  in alloc_mem_shm()
103 		td->orig_buffer = NULL;  in alloc_mem_shm()
114 static void free_mem_shm(struct thread_data *td)  in free_mem_shm()  argument
119 	dprint(FD_MEM, "shmdt/ctl %d %p\n", td->shm_id, td->orig_buffer);  in free_mem_shm()
120 	shmdt(td->orig_buffer);  in free_mem_shm()
121 	shmctl(td->shm_id, IPC_RMID, &sbuf);  in free_mem_shm()
125 static int alloc_mem_mmap(struct thread_data *td, size_t total_mem)  in alloc_mem_mmap()  argument
129 	td->mmapfd = -1;  in alloc_mem_mmap()
131 	if (td->o.mem_type == MEM_MMAPHUGE) {  in alloc_mem_mmap()
132 		unsigned long mask = td->o.hugepage_size - 1;  in alloc_mem_mmap()
135 		if (!td->o.mmapfile)  in alloc_mem_mmap()
140 	if (td->o.mmapfile) {  in alloc_mem_mmap()
141 		td->mmapfd = open(td->o.mmapfile, O_RDWR|O_CREAT, 0644);  in alloc_mem_mmap()
143 		if (td->mmapfd < 0) {  in alloc_mem_mmap()
144 			td_verror(td, errno, "open mmap file");  in alloc_mem_mmap()
145 			td->orig_buffer = NULL;  in alloc_mem_mmap()
148 		if (td->o.mem_type != MEM_MMAPHUGE &&  in alloc_mem_mmap()
149 		    ftruncate(td->mmapfd, total_mem) < 0) {  in alloc_mem_mmap()
150 			td_verror(td, errno, "truncate mmap file");  in alloc_mem_mmap()
151 			td->orig_buffer = NULL;  in alloc_mem_mmap()
154 		if (td->o.mem_type == MEM_MMAPHUGE)  in alloc_mem_mmap()
161 	td->orig_buffer = mmap(NULL, total_mem, PROT_READ | PROT_WRITE, flags,  in alloc_mem_mmap()
162 				td->mmapfd, 0);  in alloc_mem_mmap()
164 						td->mmapfd, td->orig_buffer);  in alloc_mem_mmap()
165 	if (td->orig_buffer == MAP_FAILED) {  in alloc_mem_mmap()
166 		td_verror(td, errno, "mmap");  in alloc_mem_mmap()
167 		td->orig_buffer = NULL;  in alloc_mem_mmap()
168 		if (td->mmapfd != 1 && td->mmapfd != -1) {  in alloc_mem_mmap()
169 			close(td->mmapfd);  in alloc_mem_mmap()
170 			if (td->o.mmapfile)  in alloc_mem_mmap()
171 				unlink(td->o.mmapfile);  in alloc_mem_mmap()
180 static void free_mem_mmap(struct thread_data *td, size_t total_mem)  in free_mem_mmap()  argument
183 						td->orig_buffer);  in free_mem_mmap()
184 	munmap(td->orig_buffer, td->orig_buffer_size);  in free_mem_mmap()
185 	if (td->o.mmapfile) {  in free_mem_mmap()
186 		if (td->mmapfd != -1)  in free_mem_mmap()
187 			close(td->mmapfd);  in free_mem_mmap()
188 		unlink(td->o.mmapfile);  in free_mem_mmap()
189 		free(td->o.mmapfile);  in free_mem_mmap()
193 static int alloc_mem_malloc(struct thread_data *td, size_t total_mem)  in alloc_mem_malloc()  argument
195 	td->orig_buffer = malloc(total_mem);  in alloc_mem_malloc()
197 							td->orig_buffer);  in alloc_mem_malloc()
199 	return td->orig_buffer == NULL;  in alloc_mem_malloc()
202 static void free_mem_malloc(struct thread_data *td)  in free_mem_malloc()  argument
204 	dprint(FD_MEM, "free malloc mem %p\n", td->orig_buffer);  in free_mem_malloc()
205 	free(td->orig_buffer);  in free_mem_malloc()
211 int allocate_io_mem(struct thread_data *td)  in allocate_io_mem()  argument
216 	if (td->io_ops->flags & FIO_NOIO)  in allocate_io_mem()
219 	total_mem = td->orig_buffer_size;  in allocate_io_mem()
221 	if (td->o.odirect || td->o.mem_align || td->o.oatomic ||  in allocate_io_mem()
222 	    (td->io_ops->flags & FIO_MEMALIGN)) {  in allocate_io_mem()
224 		if (td->o.mem_align && td->o.mem_align > page_size)  in allocate_io_mem()
225 			total_mem += td->o.mem_align - page_size;  in allocate_io_mem()
230 	if (td->o.mem_type == MEM_MALLOC)  in allocate_io_mem()
231 		ret = alloc_mem_malloc(td, total_mem);  in allocate_io_mem()
232 	else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)  in allocate_io_mem()
233 		ret = alloc_mem_shm(td, total_mem);  in allocate_io_mem()
234 	else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)  in allocate_io_mem()
235 		ret = alloc_mem_mmap(td, total_mem);  in allocate_io_mem()
237 		log_err("fio: bad mem type: %d\n", td->o.mem_type);  in allocate_io_mem()
242 		td_verror(td, ENOMEM, "iomem allocation");  in allocate_io_mem()
247 void free_io_mem(struct thread_data *td)  in free_io_mem()  argument
251 	total_mem = td->orig_buffer_size;  in free_io_mem()
252 	if (td->o.odirect || td->o.oatomic)  in free_io_mem()
255 	if (td->o.mem_type == MEM_MALLOC)  in free_io_mem()
256 		free_mem_malloc(td);  in free_io_mem()
257 	else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)  in free_io_mem()
258 		free_mem_shm(td);  in free_io_mem()
259 	else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)  in free_io_mem()
260 		free_mem_mmap(td, total_mem);  in free_io_mem()
262 		log_err("Bad memory type %u\n", td->o.mem_type);  in free_io_mem()
264 	td->orig_buffer = NULL;  in free_io_mem()
265 	td->orig_buffer_size = 0;  in free_io_mem()