1 /*
2  * Copyright © 2016 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 #include <time.h>
25 
26 #include "igt.h"
27 #include "igt_x86.h"
28 
29 IGT_TEST_DESCRIPTION("Basic check of flushing after batches");
30 
31 #define UNCACHED 0
32 #define COHERENT 1
33 #define WC 2
34 #define WRITE 4
35 #define KERNEL 8
36 #define SET_DOMAIN 16
37 #define BEFORE 32
38 #define INTERRUPTIBLE 64
39 #define CMDPARSER 128
40 #define BASIC 256
41 #define MOVNT 512
42 
43 #if defined(__x86_64__) && !defined(__clang__)
44 #pragma GCC push_options
45 #pragma GCC target("sse4.1")
46 #include <smmintrin.h>
47 __attribute__((noinline))
movnt(uint32_t * map,int i)48 static uint32_t movnt(uint32_t *map, int i)
49 {
50 	__m128i tmp;
51 
52 	tmp = _mm_stream_load_si128((__m128i *)map + i/4);
53 	switch (i%4) { /* gcc! */
54 	default:
55 	case 0: return _mm_extract_epi32(tmp, 0);
56 	case 1: return _mm_extract_epi32(tmp, 1);
57 	case 2: return _mm_extract_epi32(tmp, 2);
58 	case 3: return _mm_extract_epi32(tmp, 3);
59 	}
60 }
x86_64_features(void)61 static inline unsigned x86_64_features(void)
62 {
63 	return igt_x86_features();
64 }
65 #pragma GCC pop_options
66 #else
x86_64_features(void)67 static inline unsigned x86_64_features(void)
68 {
69 	return 0;
70 }
movnt(uint32_t * map,int i)71 static uint32_t movnt(uint32_t *map, int i)
72 {
73 	igt_assert(!"reached");
74 }
75 #endif
76 
run(int fd,unsigned ring,int nchild,int timeout,unsigned flags)77 static void run(int fd, unsigned ring, int nchild, int timeout,
78 		unsigned flags)
79 {
80 	const int gen = intel_gen(intel_get_drm_devid(fd));
81 
82 	/* The crux of this testing is whether writes by the GPU are coherent
83 	 * from the CPU.
84 	 *
85 	 * For example, using plain clflush (the simplest and most visible
86 	 * in terms of function calls / syscalls) we have two tests which
87 	 * perform:
88 	 *
89 	 * USER (0):
90 	 *	execbuf(map[i] = i);
91 	 *	sync();
92 	 *	clflush(&map[i]);
93 	 *	assert(map[i] == i);
94 	 *
95 	 *	execbuf(map[i] = i ^ ~0);
96 	 *	sync();
97 	 *	clflush(&map[i]);
98 	 *	assert(map[i] == i ^ ~0);
99 	 *
100 	 * BEFORE:
101 	 *	clflush(&map[i]);
102 	 *	execbuf(map[i] = i);
103 	 *	sync();
104 	 *	assert(map[i] == i);
105 	 *
106 	 *	clflush(&map[i]);
107 	 *	execbuf(map[i] = i ^ ~0);
108 	 *	sync();
109 	 *	assert(map[i] == i ^ ~0);
110 	 *
111 	 * The assertion here is that the cacheline invalidations are precise
112 	 * and we have no speculative prefetch that can see the future map[i]
113 	 * access and bring it ahead of the execution, or accidental cache
114 	 * pollution by the kernel.
115 	 */
116 
117 	igt_fork(child, nchild) {
118 		const uint32_t bbe = MI_BATCH_BUFFER_END;
119 		struct drm_i915_gem_exec_object2 obj[3];
120 		struct drm_i915_gem_relocation_entry reloc0[1024];
121 		struct drm_i915_gem_relocation_entry reloc1[1024];
122 		struct drm_i915_gem_execbuffer2 execbuf;
123 		unsigned long cycles = 0;
124 		bool snoop = false;
125 		uint32_t *ptr;
126 		uint32_t *map;
127 		int i;
128 
129 		memset(obj, 0, sizeof(obj));
130 		obj[0].handle = gem_create(fd, 4096);
131 		obj[0].flags |= EXEC_OBJECT_WRITE;
132 
133 		if (flags & WC) {
134 			igt_assert(flags & COHERENT);
135 			map = gem_mmap__wc(fd, obj[0].handle, 0, 4096, PROT_WRITE);
136 			gem_set_domain(fd, obj[0].handle,
137 				       I915_GEM_DOMAIN_WC,
138 				       I915_GEM_DOMAIN_WC);
139 		} else {
140 			snoop = flags & COHERENT;
141 			gem_set_caching(fd, obj[0].handle, snoop);
142 			map = gem_mmap__cpu(fd, obj[0].handle, 0, 4096, PROT_WRITE);
143 			gem_set_domain(fd, obj[0].handle,
144 				       I915_GEM_DOMAIN_CPU,
145 				       I915_GEM_DOMAIN_CPU);
146 		}
147 
148 		for (i = 0; i < 1024; i++)
149 			map[i] = 0xabcdabcd;
150 
151 		gem_set_domain(fd, obj[0].handle,
152 			       I915_GEM_DOMAIN_WC,
153 			       I915_GEM_DOMAIN_WC);
154 
155 		/* Prepara a mappable binding to prevent pread mighrating */
156 		if (!snoop) {
157 			ptr = gem_mmap__gtt(fd, obj[0].handle, 4096, PROT_READ);
158 			igt_assert_eq_u32(ptr[0], 0xabcdabcd);
159 			munmap(ptr, 4096);
160 		}
161 
162 		memset(&execbuf, 0, sizeof(execbuf));
163 		execbuf.buffers_ptr = to_user_pointer(obj);
164 		execbuf.buffer_count = 3;
165 		execbuf.flags = ring | (1 << 11) | (1<<12);
166 		if (gen < 6)
167 			execbuf.flags |= I915_EXEC_SECURE;
168 
169 		obj[1].handle = gem_create(fd, 1024*64);
170 		obj[2].handle = gem_create(fd, 1024*64);
171 		gem_write(fd, obj[2].handle, 0, &bbe, sizeof(bbe));
172 		igt_require(__gem_execbuf(fd, &execbuf) == 0);
173 
174 		obj[1].relocation_count = 1;
175 		obj[2].relocation_count = 1;
176 
177 		ptr = gem_mmap__wc(fd, obj[1].handle, 0, 64*1024,
178 				   PROT_WRITE | PROT_READ);
179 		gem_set_domain(fd, obj[1].handle,
180 			       I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC);
181 
182 		memset(reloc0, 0, sizeof(reloc0));
183 		for (i = 0; i < 1024; i++) {
184 			uint64_t offset;
185 			uint32_t *b = &ptr[16 * i];
186 
187 			reloc0[i].presumed_offset = obj[0].offset;
188 			reloc0[i].offset = (b - ptr + 1) * sizeof(*ptr);
189 			reloc0[i].delta = i * sizeof(uint32_t);
190 			reloc0[i].read_domains = I915_GEM_DOMAIN_INSTRUCTION;
191 			reloc0[i].write_domain = I915_GEM_DOMAIN_INSTRUCTION;
192 
193 			offset = obj[0].offset + reloc0[i].delta;
194 			*b++ = MI_STORE_DWORD_IMM | (gen < 6 ? 1 << 22 : 0);
195 			if (gen >= 8) {
196 				*b++ = offset;
197 				*b++ = offset >> 32;
198 			} else if (gen >= 4) {
199 				*b++ = 0;
200 				*b++ = offset;
201 				reloc0[i].offset += sizeof(*ptr);
202 			} else {
203 				b[-1] -= 1;
204 				*b++ = offset;
205 			}
206 			*b++ = i;
207 			*b++ = MI_BATCH_BUFFER_END;
208 		}
209 		munmap(ptr, 64*1024);
210 
211 		ptr = gem_mmap__wc(fd, obj[2].handle, 0, 64*1024,
212 				   PROT_WRITE | PROT_READ);
213 		gem_set_domain(fd, obj[2].handle,
214 			       I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC);
215 
216 		memset(reloc1, 0, sizeof(reloc1));
217 		for (i = 0; i < 1024; i++) {
218 			uint64_t offset;
219 			uint32_t *b = &ptr[16 * i];
220 
221 			reloc1[i].presumed_offset = obj[0].offset;
222 			reloc1[i].offset = (b - ptr + 1) * sizeof(*ptr);
223 			reloc1[i].delta = i * sizeof(uint32_t);
224 			reloc1[i].read_domains = I915_GEM_DOMAIN_INSTRUCTION;
225 			reloc1[i].write_domain = I915_GEM_DOMAIN_INSTRUCTION;
226 
227 			offset = obj[0].offset + reloc1[i].delta;
228 			*b++ = MI_STORE_DWORD_IMM | (gen < 6 ? 1 << 22 : 0);
229 			if (gen >= 8) {
230 				*b++ = offset;
231 				*b++ = offset >> 32;
232 			} else if (gen >= 4) {
233 				*b++ = 0;
234 				*b++ = offset;
235 				reloc1[i].offset += sizeof(*ptr);
236 			} else {
237 				b[-1] -= 1;
238 				*b++ = offset;
239 			}
240 			*b++ = i ^ 0xffffffff;
241 			*b++ = MI_BATCH_BUFFER_END;
242 		}
243 		munmap(ptr, 64*1024);
244 
245 		igt_until_timeout(timeout) {
246 			bool xor = false;
247 			int idx = cycles++ % 1024;
248 
249 			/* Inspect a different cacheline each iteration */
250 			i = 16 * (idx % 64) + (idx / 64);
251 			obj[1].relocs_ptr = to_user_pointer(&reloc0[i]);
252 			obj[2].relocs_ptr = to_user_pointer(&reloc1[i]);
253 			igt_assert_eq_u64(reloc0[i].presumed_offset, obj[0].offset);
254 			igt_assert_eq_u64(reloc1[i].presumed_offset, obj[0].offset);
255 			execbuf.batch_start_offset =  64*i;
256 
257 overwrite:
258 			if ((flags & BEFORE) &&
259 			    !((flags & COHERENT) || gem_has_llc(fd)))
260 				igt_clflush_range(&map[i], sizeof(map[i]));
261 
262 			execbuf.buffer_count = 2 + xor;
263 			gem_execbuf(fd, &execbuf);
264 
265 			if (flags & SET_DOMAIN) {
266 				unsigned domain = flags & WC ? I915_GEM_DOMAIN_WC : I915_GEM_DOMAIN_CPU;
267 				igt_while_interruptible(flags & INTERRUPTIBLE)
268 					gem_set_domain(fd, obj[0].handle,
269 						       domain, (flags & WRITE) ? domain : 0);
270 
271 				if (xor)
272 					igt_assert_eq_u32(map[i], i ^ 0xffffffff);
273 				else
274 					igt_assert_eq_u32(map[i], i);
275 
276 				if (flags & WRITE)
277 					map[i] = 0xdeadbeef;
278 			} else if (flags & KERNEL) {
279 				uint32_t val;
280 
281 				igt_while_interruptible(flags & INTERRUPTIBLE)
282 					gem_read(fd, obj[0].handle,
283 						 i*sizeof(uint32_t),
284 						 &val, sizeof(val));
285 
286 				if (xor)
287 					igt_assert_eq_u32(val, i ^ 0xffffffff);
288 				else
289 					igt_assert_eq_u32(val, i);
290 
291 				if (flags & WRITE) {
292 					val = 0xdeadbeef;
293 					igt_while_interruptible(flags & INTERRUPTIBLE)
294 						gem_write(fd, obj[0].handle,
295 							  i*sizeof(uint32_t),
296 							  &val, sizeof(val));
297 				}
298 			} else if (flags & MOVNT) {
299 				uint32_t x;
300 
301 				igt_while_interruptible(flags & INTERRUPTIBLE)
302 					gem_sync(fd, obj[0].handle);
303 
304 				x = movnt(map, i);
305 				if (xor)
306 					igt_assert_eq_u32(x, i ^ 0xffffffff);
307 				else
308 					igt_assert_eq_u32(x, i);
309 
310 				if (flags & WRITE)
311 					map[i] = 0xdeadbeef;
312 			} else {
313 				igt_while_interruptible(flags & INTERRUPTIBLE)
314 					gem_sync(fd, obj[0].handle);
315 
316 				if (!(flags & (BEFORE | COHERENT)) &&
317 				    !gem_has_llc(fd))
318 					igt_clflush_range(&map[i], sizeof(map[i]));
319 
320 				if (xor)
321 					igt_assert_eq_u32(map[i], i ^ 0xffffffff);
322 				else
323 					igt_assert_eq_u32(map[i], i);
324 
325 				if (flags & WRITE) {
326 					map[i] = 0xdeadbeef;
327 					if (!(flags & (COHERENT | BEFORE)))
328 						igt_clflush_range(&map[i], sizeof(map[i]));
329 				}
330 			}
331 
332 			if (!xor) {
333 				xor= true;
334 				goto overwrite;
335 			}
336 		}
337 		igt_info("Child[%d]: %lu cycles\n", child, cycles);
338 
339 		gem_close(fd, obj[2].handle);
340 		gem_close(fd, obj[1].handle);
341 
342 		munmap(map, 4096);
343 		gem_close(fd, obj[0].handle);
344 	}
345 	igt_waitchildren();
346 }
347 
348 enum batch_mode {
349 	BATCH_KERNEL,
350 	BATCH_USER,
351 	BATCH_CPU,
352 	BATCH_GTT,
353 	BATCH_WC,
354 };
batch(int fd,unsigned ring,int nchild,int timeout,enum batch_mode mode,unsigned flags)355 static void batch(int fd, unsigned ring, int nchild, int timeout,
356 		  enum batch_mode mode, unsigned flags)
357 {
358 	const int gen = intel_gen(intel_get_drm_devid(fd));
359 
360 	if (flags & CMDPARSER) {
361 		int cmdparser = -1;
362                 drm_i915_getparam_t gp;
363 
364 		gp.param = I915_PARAM_CMD_PARSER_VERSION;
365 		gp.value = &cmdparser;
366 		drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
367 		igt_require(cmdparser > 0);
368 	}
369 
370 	intel_detect_and_clear_missed_interrupts(fd);
371 	igt_fork(child, nchild) {
372 		const uint32_t bbe = MI_BATCH_BUFFER_END;
373 		struct drm_i915_gem_exec_object2 obj[2];
374 		struct drm_i915_gem_relocation_entry reloc;
375 		struct drm_i915_gem_execbuffer2 execbuf;
376 		unsigned long cycles = 0;
377 		uint32_t *ptr;
378 		uint32_t *map;
379 		int i;
380 
381 		memset(obj, 0, sizeof(obj));
382 		obj[0].handle = gem_create(fd, 4096);
383 		obj[0].flags |= EXEC_OBJECT_WRITE;
384 
385 		gem_set_caching(fd, obj[0].handle, !!(flags & COHERENT));
386 		map = gem_mmap__cpu(fd, obj[0].handle, 0, 4096, PROT_WRITE);
387 
388 		gem_set_domain(fd, obj[0].handle,
389 				I915_GEM_DOMAIN_CPU,
390 				I915_GEM_DOMAIN_CPU);
391 		for (i = 0; i < 1024; i++)
392 			map[i] = 0xabcdabcd;
393 
394 		memset(&execbuf, 0, sizeof(execbuf));
395 		execbuf.buffers_ptr = to_user_pointer(obj);
396 		execbuf.buffer_count = 2;
397 		execbuf.flags = ring | (1 << 11) | (1<<12);
398 		if (gen < 6)
399 			execbuf.flags |= I915_EXEC_SECURE;
400 
401 		obj[1].handle = gem_create(fd, 64<<10);
402 		gem_write(fd, obj[1].handle, 0, &bbe, sizeof(bbe));
403 		igt_require(__gem_execbuf(fd, &execbuf) == 0);
404 
405 		obj[1].relocation_count = 1;
406 		obj[1].relocs_ptr = to_user_pointer(&reloc);
407 
408 		switch (mode) {
409 		case BATCH_CPU:
410 		case BATCH_USER:
411 			ptr = gem_mmap__cpu(fd, obj[1].handle, 0, 64<<10,
412 					    PROT_WRITE);
413 			break;
414 
415 		case BATCH_WC:
416 			ptr = gem_mmap__wc(fd, obj[1].handle, 0, 64<<10,
417 					    PROT_WRITE);
418 			break;
419 
420 		case BATCH_GTT:
421 			ptr = gem_mmap__gtt(fd, obj[1].handle, 64<<10,
422 					    PROT_WRITE);
423 			break;
424 
425 		case BATCH_KERNEL:
426 			ptr = mmap(0, 64<<10, PROT_WRITE,
427 				   MAP_PRIVATE | MAP_ANON, -1, 0);
428 			break;
429 
430 		default:
431 			igt_assert(!"reachable");
432 			ptr = NULL;
433 			break;
434 		}
435 
436 		memset(&reloc, 0, sizeof(reloc));
437 		reloc.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
438 		reloc.write_domain = I915_GEM_DOMAIN_INSTRUCTION;
439 
440 		igt_until_timeout(timeout) {
441 			execbuf.batch_start_offset = 0;
442 			reloc.offset = sizeof(uint32_t);
443 			if (gen >= 4 && gen < 8)
444 				reloc.offset += sizeof(uint32_t);
445 
446 			for (i = 0; i < 1024; i++) {
447 				uint64_t offset;
448 				uint32_t *start = &ptr[execbuf.batch_start_offset/sizeof(*start)];
449 				uint32_t *b = start;
450 
451 				switch (mode) {
452 				case BATCH_CPU:
453 					gem_set_domain(fd, obj[1].handle,
454 						       I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
455 					break;
456 
457 				case BATCH_WC:
458 					gem_set_domain(fd, obj[1].handle,
459 						       I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC);
460 					break;
461 
462 				case BATCH_GTT:
463 					gem_set_domain(fd, obj[1].handle,
464 						       I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
465 					break;
466 
467 				case BATCH_USER:
468 				case BATCH_KERNEL:
469 					break;
470 				}
471 
472 				reloc.presumed_offset = obj[0].offset;
473 				reloc.delta = i * sizeof(uint32_t);
474 
475 				offset = reloc.presumed_offset + reloc.delta;
476 				*b++ = MI_STORE_DWORD_IMM | (gen < 6 ? 1 << 22 : 0);
477 				if (gen >= 8) {
478 					*b++ = offset;
479 					*b++ = offset >> 32;
480 				} else if (gen >= 4) {
481 					*b++ = 0;
482 					*b++ = offset;
483 				} else {
484 					b[-1] -= 1;
485 					*b++ = offset;
486 				}
487 				*b++ = cycles + i;
488 				*b++ = MI_BATCH_BUFFER_END;
489 
490 				if (flags & CMDPARSER) {
491 					execbuf.batch_len =
492 						(b - start) * sizeof(uint32_t);
493 					if (execbuf.batch_len & 4)
494 						execbuf.batch_len += 4;
495 				}
496 
497 				switch (mode) {
498 				case BATCH_KERNEL:
499 					gem_write(fd, obj[1].handle,
500 						  execbuf.batch_start_offset,
501 						  start, (b - start) * sizeof(uint32_t));
502 					break;
503 
504 				case BATCH_USER:
505 					if (!gem_has_llc(fd))
506 						igt_clflush_range(start,
507 								  (b - start) * sizeof(uint32_t));
508 					break;
509 
510 				case BATCH_CPU:
511 				case BATCH_GTT:
512 				case BATCH_WC:
513 					break;
514 				}
515 				gem_execbuf(fd, &execbuf);
516 
517 				execbuf.batch_start_offset += 64;
518 				reloc.offset += 64;
519 			}
520 
521 			if (!(flags & COHERENT)) {
522 				gem_set_domain(fd, obj[0].handle,
523 					       I915_GEM_DOMAIN_CPU,
524 					       I915_GEM_DOMAIN_CPU);
525 			} else
526 				gem_sync(fd, obj[0].handle);
527 			for (i = 0; i < 1024; i++) {
528 				igt_assert_eq_u32(map[i], cycles + i);
529 				map[i] = 0xabcdabcd ^ cycles;
530 			}
531 			cycles += 1024;
532 
533 			if (mode == BATCH_USER)
534 				gem_sync(fd, obj[1].handle);
535 		}
536 		igt_info("Child[%d]: %lu cycles\n", child, cycles);
537 
538 		munmap(ptr, 64<<10);
539 		gem_close(fd, obj[1].handle);
540 
541 		munmap(map, 4096);
542 		gem_close(fd, obj[0].handle);
543 	}
544 	igt_waitchildren();
545 	igt_assert_eq(intel_detect_and_clear_missed_interrupts(fd), 0);
546 }
547 
yesno(bool x)548 static const char *yesno(bool x)
549 {
550 	return x ? "yes" : "no";
551 }
552 
553 igt_main
554 {
555 	const struct intel_execution_engine *e;
556 	const int ncpus = sysconf(_SC_NPROCESSORS_ONLN);
557 	const struct batch {
558 		const char *name;
559 		unsigned mode;
560 	} batches[] = {
561 		{ "kernel", BATCH_KERNEL },
562 		{ "user", BATCH_USER },
563 		{ "cpu", BATCH_CPU },
564 		{ "gtt", BATCH_GTT },
565 		{ "wc", BATCH_WC },
566 		{ NULL }
567 	};
568 	const struct mode {
569 		const char *name;
570 		unsigned flags;
571 	} modes[] = {
572 		{ "ro", BASIC },
573 		{ "rw", BASIC | WRITE },
574 		{ "ro-before", BEFORE },
575 		{ "rw-before", BEFORE | WRITE },
576 		{ "pro", BASIC | KERNEL },
577 		{ "prw", BASIC | KERNEL | WRITE },
578 		{ "set", BASIC | SET_DOMAIN | WRITE },
579 		{ NULL }
580 	};
581 	unsigned cpu = x86_64_features();
582 	int fd = -1;
583 
584 	igt_fixture {
585 		igt_require(igt_setup_clflush());
586 		fd = drm_open_driver(DRIVER_INTEL);
587 		igt_require_gem(fd);
588 		gem_require_mmap_wc(fd);
589 		igt_require(gem_can_store_dword(fd, 0));
590 		igt_info("Has LLC? %s\n", yesno(gem_has_llc(fd)));
591 
592 		if (cpu) {
593 			char str[1024];
594 
595 			igt_info("CPU features: %s\n",
596 				 igt_x86_features_to_string(cpu, str));
597 		}
598 
599 		igt_fork_hang_detector(fd);
600 	}
601 
602 	for (e = intel_execution_engines; e->name; e++) igt_subtest_group {
603 		unsigned ring = e->exec_id | e->flags;
604 		unsigned timeout = 5 + 120*!!e->exec_id;
605 
606 		igt_fixture {
607 			gem_require_ring(fd, ring);
608 			igt_require(gem_can_store_dword(fd, ring));
609 		}
610 
611 		for (const struct batch *b = batches; b->name; b++) {
612 			igt_subtest_f("%sbatch-%s-%s-uc",
613 				      b == batches && e->exec_id == 0 ? "basic-" : "",
614 				      b->name,
615 				      e->name)
616 				batch(fd, ring, ncpus, timeout, b->mode, 0);
617 			igt_subtest_f("%sbatch-%s-%s-wb",
618 				      b == batches && e->exec_id == 0 ? "basic-" : "",
619 				      b->name,
620 				      e->name)
621 				batch(fd, ring, ncpus, timeout, b->mode, COHERENT);
622 			igt_subtest_f("%sbatch-%s-%s-cmd",
623 				      b == batches && e->exec_id == 0 ? "basic-" : "",
624 				      b->name,
625 				      e->name)
626 				batch(fd, ring, ncpus, timeout, b->mode,
627 				      COHERENT | CMDPARSER);
628 		}
629 
630 		for (const struct mode *m = modes; m->name; m++) {
631 			igt_subtest_f("%suc-%s-%s",
632 				      (m->flags & BASIC && e->exec_id == 0) ? "basic-" : "",
633 				      m->name,
634 				      e->name)
635 				run(fd, ring, ncpus, timeout,
636 				    UNCACHED | m->flags);
637 
638 			igt_subtest_f("uc-%s-%s-interruptible",
639 				      m->name,
640 				      e->name)
641 				run(fd, ring, ncpus, timeout,
642 				    UNCACHED | m->flags | INTERRUPTIBLE);
643 
644 			igt_subtest_f("%swb-%s-%s",
645 				      e->exec_id == 0 ? "basic-" : "",
646 				      m->name,
647 				      e->name)
648 				run(fd, ring, ncpus, timeout,
649 				    COHERENT | m->flags);
650 
651 			igt_subtest_f("wb-%s-%s-interruptible",
652 				      m->name,
653 				      e->name)
654 				run(fd, ring, ncpus, timeout,
655 				    COHERENT | m->flags | INTERRUPTIBLE);
656 
657 			igt_subtest_f("wc-%s-%s",
658 				      m->name,
659 				      e->name)
660 				run(fd, ring, ncpus, timeout,
661 				    COHERENT | WC | m->flags);
662 
663 			igt_subtest_f("wc-%s-%s-interruptible",
664 				      m->name,
665 				      e->name)
666 				run(fd, ring, ncpus, timeout,
667 				    COHERENT | WC | m->flags | INTERRUPTIBLE);
668 
669 			igt_subtest_f("stream-%s-%s",
670 				      m->name,
671 				      e->name) {
672 				igt_require(cpu & SSE4_1);
673 				run(fd, ring, ncpus, timeout,
674 				    MOVNT | COHERENT | WC | m->flags);
675 			}
676 
677 			igt_subtest_f("stream-%s-%s-interruptible",
678 				      m->name,
679 				      e->name) {
680 				igt_require(cpu & SSE4_1);
681 				run(fd, ring, ncpus, timeout,
682 				    MOVNT | COHERENT | WC | m->flags | INTERRUPTIBLE);
683 			}
684 		}
685 	}
686 
687 	igt_fixture {
688 		igt_stop_hang_detector();
689 		close(fd);
690 	}
691 }
692