1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2018 Intel Corporation
5  */
6 
7 #include <linux/prime_numbers.h>
8 
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
12 
13 #include "i915_selftest.h"
14 #include "selftests/i915_random.h"
15 #include "selftests/igt_flush_test.h"
16 #include "selftests/igt_live_test.h"
17 #include "selftests/igt_spinner.h"
18 #include "selftests/lib_sw_fence.h"
19 
20 #include "gem/selftests/igt_gem_utils.h"
21 #include "gem/selftests/mock_context.h"
22 
23 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
24 #define NUM_GPR_DW (16 * 2) /* each GPR is 2 dwords */
25 
26 static struct i915_vma *create_scratch(struct intel_gt *gt)
27 {
28 	struct drm_i915_gem_object *obj;
29 	struct i915_vma *vma;
30 	int err;
31 
32 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
33 	if (IS_ERR(obj))
34 		return ERR_CAST(obj);
35 
36 	i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
37 
38 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
39 	if (IS_ERR(vma)) {
40 		i915_gem_object_put(obj);
41 		return vma;
42 	}
43 
44 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
45 	if (err) {
46 		i915_gem_object_put(obj);
47 		return ERR_PTR(err);
48 	}
49 
50 	return vma;
51 }
52 
53 static int live_sanitycheck(void *arg)
54 {
55 	struct intel_gt *gt = arg;
56 	struct i915_gem_engines_iter it;
57 	struct i915_gem_context *ctx;
58 	struct intel_context *ce;
59 	struct igt_spinner spin;
60 	int err = -ENOMEM;
61 
62 	if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
63 		return 0;
64 
65 	if (igt_spinner_init(&spin, gt))
66 		return -ENOMEM;
67 
68 	ctx = kernel_context(gt->i915);
69 	if (!ctx)
70 		goto err_spin;
71 
72 	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
73 		struct i915_request *rq;
74 
75 		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
76 		if (IS_ERR(rq)) {
77 			err = PTR_ERR(rq);
78 			goto err_ctx;
79 		}
80 
81 		i915_request_add(rq);
82 		if (!igt_wait_for_spinner(&spin, rq)) {
83 			GEM_TRACE("spinner failed to start\n");
84 			GEM_TRACE_DUMP();
85 			intel_gt_set_wedged(gt);
86 			err = -EIO;
87 			goto err_ctx;
88 		}
89 
90 		igt_spinner_end(&spin);
91 		if (igt_flush_test(gt->i915)) {
92 			err = -EIO;
93 			goto err_ctx;
94 		}
95 	}
96 
97 	err = 0;
98 err_ctx:
99 	i915_gem_context_unlock_engines(ctx);
100 	kernel_context_close(ctx);
101 err_spin:
102 	igt_spinner_fini(&spin);
103 	return err;
104 }
105 
106 static int live_unlite_restore(struct intel_gt *gt, int prio)
107 {
108 	struct intel_engine_cs *engine;
109 	struct i915_gem_context *ctx;
110 	enum intel_engine_id id;
111 	struct igt_spinner spin;
112 	int err = -ENOMEM;
113 
114 	/*
115 	 * Check that we can correctly context switch between 2 instances
116 	 * on the same engine from the same parent context.
117 	 */
118 
119 	if (igt_spinner_init(&spin, gt))
120 		return err;
121 
122 	ctx = kernel_context(gt->i915);
123 	if (!ctx)
124 		goto err_spin;
125 
126 	err = 0;
127 	for_each_engine(engine, gt, id) {
128 		struct intel_context *ce[2] = {};
129 		struct i915_request *rq[2];
130 		struct igt_live_test t;
131 		int n;
132 
133 		if (prio && !intel_engine_has_preemption(engine))
134 			continue;
135 
136 		if (!intel_engine_can_store_dword(engine))
137 			continue;
138 
139 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
140 			err = -EIO;
141 			break;
142 		}
143 
144 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
145 			struct intel_context *tmp;
146 
147 			tmp = intel_context_create(ctx, engine);
148 			if (IS_ERR(tmp)) {
149 				err = PTR_ERR(tmp);
150 				goto err_ce;
151 			}
152 
153 			err = intel_context_pin(tmp);
154 			if (err) {
155 				intel_context_put(tmp);
156 				goto err_ce;
157 			}
158 
159 			/*
160 			 * Setup the pair of contexts such that if we
161 			 * lite-restore using the RING_TAIL from ce[1] it
162 			 * will execute garbage from ce[0]->ring.
163 			 */
164 			memset(tmp->ring->vaddr,
165 			       POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
166 			       tmp->ring->vma->size);
167 
168 			ce[n] = tmp;
169 		}
170 		GEM_BUG_ON(!ce[1]->ring->size);
171 		intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
172 		__execlists_update_reg_state(ce[1], engine);
173 
174 		rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
175 		if (IS_ERR(rq[0])) {
176 			err = PTR_ERR(rq[0]);
177 			goto err_ce;
178 		}
179 
180 		i915_request_get(rq[0]);
181 		i915_request_add(rq[0]);
182 		GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
183 
184 		if (!igt_wait_for_spinner(&spin, rq[0])) {
185 			i915_request_put(rq[0]);
186 			goto err_ce;
187 		}
188 
189 		rq[1] = i915_request_create(ce[1]);
190 		if (IS_ERR(rq[1])) {
191 			err = PTR_ERR(rq[1]);
192 			i915_request_put(rq[0]);
193 			goto err_ce;
194 		}
195 
196 		if (!prio) {
197 			/*
198 			 * Ensure we do the switch to ce[1] on completion.
199 			 *
200 			 * rq[0] is already submitted, so this should reduce
201 			 * to a no-op (a wait on a request on the same engine
202 			 * uses the submit fence, not the completion fence),
203 			 * but it will install a dependency on rq[1] for rq[0]
204 			 * that will prevent the pair being reordered by
205 			 * timeslicing.
206 			 */
207 			i915_request_await_dma_fence(rq[1], &rq[0]->fence);
208 		}
209 
210 		i915_request_get(rq[1]);
211 		i915_request_add(rq[1]);
212 		GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
213 		i915_request_put(rq[0]);
214 
215 		if (prio) {
216 			struct i915_sched_attr attr = {
217 				.priority = prio,
218 			};
219 
220 			/* Alternatively preempt the spinner with ce[1] */
221 			engine->schedule(rq[1], &attr);
222 		}
223 
224 		/* And switch back to ce[0] for good measure */
225 		rq[0] = i915_request_create(ce[0]);
226 		if (IS_ERR(rq[0])) {
227 			err = PTR_ERR(rq[0]);
228 			i915_request_put(rq[1]);
229 			goto err_ce;
230 		}
231 
232 		i915_request_await_dma_fence(rq[0], &rq[1]->fence);
233 		i915_request_get(rq[0]);
234 		i915_request_add(rq[0]);
235 		GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
236 		i915_request_put(rq[1]);
237 		i915_request_put(rq[0]);
238 
239 err_ce:
240 		tasklet_kill(&engine->execlists.tasklet); /* flush submission */
241 		igt_spinner_end(&spin);
242 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
243 			if (IS_ERR_OR_NULL(ce[n]))
244 				break;
245 
246 			intel_context_unpin(ce[n]);
247 			intel_context_put(ce[n]);
248 		}
249 
250 		if (igt_live_test_end(&t))
251 			err = -EIO;
252 		if (err)
253 			break;
254 	}
255 
256 	kernel_context_close(ctx);
257 err_spin:
258 	igt_spinner_fini(&spin);
259 	return err;
260 }
261 
262 static int live_unlite_switch(void *arg)
263 {
264 	return live_unlite_restore(arg, 0);
265 }
266 
267 static int live_unlite_preempt(void *arg)
268 {
269 	return live_unlite_restore(arg, I915_USER_PRIORITY(I915_PRIORITY_MAX));
270 }
271 
272 static int
273 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
274 {
275 	u32 *cs;
276 
277 	cs = intel_ring_begin(rq, 10);
278 	if (IS_ERR(cs))
279 		return PTR_ERR(cs);
280 
281 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
282 
283 	*cs++ = MI_SEMAPHORE_WAIT |
284 		MI_SEMAPHORE_GLOBAL_GTT |
285 		MI_SEMAPHORE_POLL |
286 		MI_SEMAPHORE_SAD_NEQ_SDD;
287 	*cs++ = 0;
288 	*cs++ = i915_ggtt_offset(vma) + 4 * idx;
289 	*cs++ = 0;
290 
291 	if (idx > 0) {
292 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
293 		*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
294 		*cs++ = 0;
295 		*cs++ = 1;
296 	} else {
297 		*cs++ = MI_NOOP;
298 		*cs++ = MI_NOOP;
299 		*cs++ = MI_NOOP;
300 		*cs++ = MI_NOOP;
301 	}
302 
303 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
304 
305 	intel_ring_advance(rq, cs);
306 	return 0;
307 }
308 
309 static struct i915_request *
310 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
311 {
312 	struct i915_gem_context *ctx;
313 	struct i915_request *rq;
314 	int err;
315 
316 	ctx = kernel_context(engine->i915);
317 	if (!ctx)
318 		return ERR_PTR(-ENOMEM);
319 
320 	rq = igt_request_alloc(ctx, engine);
321 	if (IS_ERR(rq))
322 		goto out_ctx;
323 
324 	err = 0;
325 	if (rq->engine->emit_init_breadcrumb)
326 		err = rq->engine->emit_init_breadcrumb(rq);
327 	if (err == 0)
328 		err = emit_semaphore_chain(rq, vma, idx);
329 	if (err == 0)
330 		i915_request_get(rq);
331 	i915_request_add(rq);
332 	if (err)
333 		rq = ERR_PTR(err);
334 
335 out_ctx:
336 	kernel_context_close(ctx);
337 	return rq;
338 }
339 
340 static int
341 release_queue(struct intel_engine_cs *engine,
342 	      struct i915_vma *vma,
343 	      int idx, int prio)
344 {
345 	struct i915_sched_attr attr = {
346 		.priority = prio,
347 	};
348 	struct i915_request *rq;
349 	u32 *cs;
350 
351 	rq = i915_request_create(engine->kernel_context);
352 	if (IS_ERR(rq))
353 		return PTR_ERR(rq);
354 
355 	cs = intel_ring_begin(rq, 4);
356 	if (IS_ERR(cs)) {
357 		i915_request_add(rq);
358 		return PTR_ERR(cs);
359 	}
360 
361 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
362 	*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
363 	*cs++ = 0;
364 	*cs++ = 1;
365 
366 	intel_ring_advance(rq, cs);
367 
368 	i915_request_get(rq);
369 	i915_request_add(rq);
370 
371 	local_bh_disable();
372 	engine->schedule(rq, &attr);
373 	local_bh_enable(); /* kick tasklet */
374 
375 	i915_request_put(rq);
376 
377 	return 0;
378 }
379 
380 static int
381 slice_semaphore_queue(struct intel_engine_cs *outer,
382 		      struct i915_vma *vma,
383 		      int count)
384 {
385 	struct intel_engine_cs *engine;
386 	struct i915_request *head;
387 	enum intel_engine_id id;
388 	int err, i, n = 0;
389 
390 	head = semaphore_queue(outer, vma, n++);
391 	if (IS_ERR(head))
392 		return PTR_ERR(head);
393 
394 	for_each_engine(engine, outer->gt, id) {
395 		for (i = 0; i < count; i++) {
396 			struct i915_request *rq;
397 
398 			rq = semaphore_queue(engine, vma, n++);
399 			if (IS_ERR(rq)) {
400 				err = PTR_ERR(rq);
401 				goto out;
402 			}
403 
404 			i915_request_put(rq);
405 		}
406 	}
407 
408 	err = release_queue(outer, vma, n, INT_MAX);
409 	if (err)
410 		goto out;
411 
412 	if (i915_request_wait(head, 0,
413 			      2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
414 		pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
415 		       count, n);
416 		GEM_TRACE_DUMP();
417 		intel_gt_set_wedged(outer->gt);
418 		err = -EIO;
419 	}
420 
421 out:
422 	i915_request_put(head);
423 	return err;
424 }
425 
426 static int live_timeslice_preempt(void *arg)
427 {
428 	struct intel_gt *gt = arg;
429 	struct drm_i915_gem_object *obj;
430 	struct i915_vma *vma;
431 	void *vaddr;
432 	int err = 0;
433 	int count;
434 
435 	/*
436 	 * If a request takes too long, we would like to give other users
437 	 * a fair go on the GPU. In particular, users may create batches
438 	 * that wait upon external input, where that input may even be
439 	 * supplied by another GPU job. To avoid blocking forever, we
440 	 * need to preempt the current task and replace it with another
441 	 * ready task.
442 	 */
443 	if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
444 		return 0;
445 
446 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
447 	if (IS_ERR(obj))
448 		return PTR_ERR(obj);
449 
450 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
451 	if (IS_ERR(vma)) {
452 		err = PTR_ERR(vma);
453 		goto err_obj;
454 	}
455 
456 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
457 	if (IS_ERR(vaddr)) {
458 		err = PTR_ERR(vaddr);
459 		goto err_obj;
460 	}
461 
462 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
463 	if (err)
464 		goto err_map;
465 
466 	for_each_prime_number_from(count, 1, 16) {
467 		struct intel_engine_cs *engine;
468 		enum intel_engine_id id;
469 
470 		for_each_engine(engine, gt, id) {
471 			if (!intel_engine_has_preemption(engine))
472 				continue;
473 
474 			memset(vaddr, 0, PAGE_SIZE);
475 
476 			err = slice_semaphore_queue(engine, vma, count);
477 			if (err)
478 				goto err_pin;
479 
480 			if (igt_flush_test(gt->i915)) {
481 				err = -EIO;
482 				goto err_pin;
483 			}
484 		}
485 	}
486 
487 err_pin:
488 	i915_vma_unpin(vma);
489 err_map:
490 	i915_gem_object_unpin_map(obj);
491 err_obj:
492 	i915_gem_object_put(obj);
493 	return err;
494 }
495 
496 static struct i915_request *nop_request(struct intel_engine_cs *engine)
497 {
498 	struct i915_request *rq;
499 
500 	rq = i915_request_create(engine->kernel_context);
501 	if (IS_ERR(rq))
502 		return rq;
503 
504 	i915_request_get(rq);
505 	i915_request_add(rq);
506 
507 	return rq;
508 }
509 
510 static void wait_for_submit(struct intel_engine_cs *engine,
511 			    struct i915_request *rq)
512 {
513 	do {
514 		cond_resched();
515 		intel_engine_flush_submission(engine);
516 	} while (!i915_request_is_active(rq));
517 }
518 
519 static long timeslice_threshold(const struct intel_engine_cs *engine)
520 {
521 	return 2 * msecs_to_jiffies_timeout(timeslice(engine)) + 1;
522 }
523 
524 static int live_timeslice_queue(void *arg)
525 {
526 	struct intel_gt *gt = arg;
527 	struct drm_i915_gem_object *obj;
528 	struct intel_engine_cs *engine;
529 	enum intel_engine_id id;
530 	struct i915_vma *vma;
531 	void *vaddr;
532 	int err = 0;
533 
534 	/*
535 	 * Make sure that even if ELSP[0] and ELSP[1] are filled with
536 	 * timeslicing between them disabled, we *do* enable timeslicing
537 	 * if the queue demands it. (Normally, we do not submit if
538 	 * ELSP[1] is already occupied, so must rely on timeslicing to
539 	 * eject ELSP[0] in favour of the queue.)
540 	 */
541 	if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
542 		return 0;
543 
544 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
545 	if (IS_ERR(obj))
546 		return PTR_ERR(obj);
547 
548 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
549 	if (IS_ERR(vma)) {
550 		err = PTR_ERR(vma);
551 		goto err_obj;
552 	}
553 
554 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
555 	if (IS_ERR(vaddr)) {
556 		err = PTR_ERR(vaddr);
557 		goto err_obj;
558 	}
559 
560 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
561 	if (err)
562 		goto err_map;
563 
564 	for_each_engine(engine, gt, id) {
565 		struct i915_sched_attr attr = {
566 			.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
567 		};
568 		struct i915_request *rq, *nop;
569 
570 		if (!intel_engine_has_preemption(engine))
571 			continue;
572 
573 		memset(vaddr, 0, PAGE_SIZE);
574 
575 		/* ELSP[0]: semaphore wait */
576 		rq = semaphore_queue(engine, vma, 0);
577 		if (IS_ERR(rq)) {
578 			err = PTR_ERR(rq);
579 			goto err_pin;
580 		}
581 		engine->schedule(rq, &attr);
582 		wait_for_submit(engine, rq);
583 
584 		/* ELSP[1]: nop request */
585 		nop = nop_request(engine);
586 		if (IS_ERR(nop)) {
587 			err = PTR_ERR(nop);
588 			i915_request_put(rq);
589 			goto err_pin;
590 		}
591 		wait_for_submit(engine, nop);
592 		i915_request_put(nop);
593 
594 		GEM_BUG_ON(i915_request_completed(rq));
595 		GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
596 
597 		/* Queue: semaphore signal, matching priority as semaphore */
598 		err = release_queue(engine, vma, 1, effective_prio(rq));
599 		if (err) {
600 			i915_request_put(rq);
601 			goto err_pin;
602 		}
603 
604 		intel_engine_flush_submission(engine);
605 		if (!READ_ONCE(engine->execlists.timer.expires) &&
606 		    !i915_request_completed(rq)) {
607 			struct drm_printer p =
608 				drm_info_printer(gt->i915->drm.dev);
609 
610 			GEM_TRACE_ERR("%s: Failed to enable timeslicing!\n",
611 				      engine->name);
612 			intel_engine_dump(engine, &p,
613 					  "%s\n", engine->name);
614 			GEM_TRACE_DUMP();
615 
616 			memset(vaddr, 0xff, PAGE_SIZE);
617 			err = -EINVAL;
618 		}
619 
620 		/* Timeslice every jiffy, so within 2 we should signal */
621 		if (i915_request_wait(rq, 0, timeslice_threshold(engine)) < 0) {
622 			struct drm_printer p =
623 				drm_info_printer(gt->i915->drm.dev);
624 
625 			pr_err("%s: Failed to timeslice into queue\n",
626 			       engine->name);
627 			intel_engine_dump(engine, &p,
628 					  "%s\n", engine->name);
629 
630 			memset(vaddr, 0xff, PAGE_SIZE);
631 			err = -EIO;
632 		}
633 		i915_request_put(rq);
634 		if (err)
635 			break;
636 	}
637 
638 err_pin:
639 	i915_vma_unpin(vma);
640 err_map:
641 	i915_gem_object_unpin_map(obj);
642 err_obj:
643 	i915_gem_object_put(obj);
644 	return err;
645 }
646 
647 static int live_busywait_preempt(void *arg)
648 {
649 	struct intel_gt *gt = arg;
650 	struct i915_gem_context *ctx_hi, *ctx_lo;
651 	struct intel_engine_cs *engine;
652 	struct drm_i915_gem_object *obj;
653 	struct i915_vma *vma;
654 	enum intel_engine_id id;
655 	int err = -ENOMEM;
656 	u32 *map;
657 
658 	/*
659 	 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
660 	 * preempt the busywaits used to synchronise between rings.
661 	 */
662 
663 	ctx_hi = kernel_context(gt->i915);
664 	if (!ctx_hi)
665 		return -ENOMEM;
666 	ctx_hi->sched.priority =
667 		I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
668 
669 	ctx_lo = kernel_context(gt->i915);
670 	if (!ctx_lo)
671 		goto err_ctx_hi;
672 	ctx_lo->sched.priority =
673 		I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
674 
675 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
676 	if (IS_ERR(obj)) {
677 		err = PTR_ERR(obj);
678 		goto err_ctx_lo;
679 	}
680 
681 	map = i915_gem_object_pin_map(obj, I915_MAP_WC);
682 	if (IS_ERR(map)) {
683 		err = PTR_ERR(map);
684 		goto err_obj;
685 	}
686 
687 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
688 	if (IS_ERR(vma)) {
689 		err = PTR_ERR(vma);
690 		goto err_map;
691 	}
692 
693 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
694 	if (err)
695 		goto err_map;
696 
697 	for_each_engine(engine, gt, id) {
698 		struct i915_request *lo, *hi;
699 		struct igt_live_test t;
700 		u32 *cs;
701 
702 		if (!intel_engine_has_preemption(engine))
703 			continue;
704 
705 		if (!intel_engine_can_store_dword(engine))
706 			continue;
707 
708 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
709 			err = -EIO;
710 			goto err_vma;
711 		}
712 
713 		/*
714 		 * We create two requests. The low priority request
715 		 * busywaits on a semaphore (inside the ringbuffer where
716 		 * is should be preemptible) and the high priority requests
717 		 * uses a MI_STORE_DWORD_IMM to update the semaphore value
718 		 * allowing the first request to complete. If preemption
719 		 * fails, we hang instead.
720 		 */
721 
722 		lo = igt_request_alloc(ctx_lo, engine);
723 		if (IS_ERR(lo)) {
724 			err = PTR_ERR(lo);
725 			goto err_vma;
726 		}
727 
728 		cs = intel_ring_begin(lo, 8);
729 		if (IS_ERR(cs)) {
730 			err = PTR_ERR(cs);
731 			i915_request_add(lo);
732 			goto err_vma;
733 		}
734 
735 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
736 		*cs++ = i915_ggtt_offset(vma);
737 		*cs++ = 0;
738 		*cs++ = 1;
739 
740 		/* XXX Do we need a flush + invalidate here? */
741 
742 		*cs++ = MI_SEMAPHORE_WAIT |
743 			MI_SEMAPHORE_GLOBAL_GTT |
744 			MI_SEMAPHORE_POLL |
745 			MI_SEMAPHORE_SAD_EQ_SDD;
746 		*cs++ = 0;
747 		*cs++ = i915_ggtt_offset(vma);
748 		*cs++ = 0;
749 
750 		intel_ring_advance(lo, cs);
751 		i915_request_add(lo);
752 
753 		if (wait_for(READ_ONCE(*map), 10)) {
754 			err = -ETIMEDOUT;
755 			goto err_vma;
756 		}
757 
758 		/* Low priority request should be busywaiting now */
759 		if (i915_request_wait(lo, 0, 1) != -ETIME) {
760 			pr_err("%s: Busywaiting request did not!\n",
761 			       engine->name);
762 			err = -EIO;
763 			goto err_vma;
764 		}
765 
766 		hi = igt_request_alloc(ctx_hi, engine);
767 		if (IS_ERR(hi)) {
768 			err = PTR_ERR(hi);
769 			goto err_vma;
770 		}
771 
772 		cs = intel_ring_begin(hi, 4);
773 		if (IS_ERR(cs)) {
774 			err = PTR_ERR(cs);
775 			i915_request_add(hi);
776 			goto err_vma;
777 		}
778 
779 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
780 		*cs++ = i915_ggtt_offset(vma);
781 		*cs++ = 0;
782 		*cs++ = 0;
783 
784 		intel_ring_advance(hi, cs);
785 		i915_request_add(hi);
786 
787 		if (i915_request_wait(lo, 0, HZ / 5) < 0) {
788 			struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
789 
790 			pr_err("%s: Failed to preempt semaphore busywait!\n",
791 			       engine->name);
792 
793 			intel_engine_dump(engine, &p, "%s\n", engine->name);
794 			GEM_TRACE_DUMP();
795 
796 			intel_gt_set_wedged(gt);
797 			err = -EIO;
798 			goto err_vma;
799 		}
800 		GEM_BUG_ON(READ_ONCE(*map));
801 
802 		if (igt_live_test_end(&t)) {
803 			err = -EIO;
804 			goto err_vma;
805 		}
806 	}
807 
808 	err = 0;
809 err_vma:
810 	i915_vma_unpin(vma);
811 err_map:
812 	i915_gem_object_unpin_map(obj);
813 err_obj:
814 	i915_gem_object_put(obj);
815 err_ctx_lo:
816 	kernel_context_close(ctx_lo);
817 err_ctx_hi:
818 	kernel_context_close(ctx_hi);
819 	return err;
820 }
821 
822 static struct i915_request *
823 spinner_create_request(struct igt_spinner *spin,
824 		       struct i915_gem_context *ctx,
825 		       struct intel_engine_cs *engine,
826 		       u32 arb)
827 {
828 	struct intel_context *ce;
829 	struct i915_request *rq;
830 
831 	ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
832 	if (IS_ERR(ce))
833 		return ERR_CAST(ce);
834 
835 	rq = igt_spinner_create_request(spin, ce, arb);
836 	intel_context_put(ce);
837 	return rq;
838 }
839 
840 static int live_preempt(void *arg)
841 {
842 	struct intel_gt *gt = arg;
843 	struct i915_gem_context *ctx_hi, *ctx_lo;
844 	struct igt_spinner spin_hi, spin_lo;
845 	struct intel_engine_cs *engine;
846 	enum intel_engine_id id;
847 	int err = -ENOMEM;
848 
849 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
850 		return 0;
851 
852 	if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
853 		pr_err("Logical preemption supported, but not exposed\n");
854 
855 	if (igt_spinner_init(&spin_hi, gt))
856 		return -ENOMEM;
857 
858 	if (igt_spinner_init(&spin_lo, gt))
859 		goto err_spin_hi;
860 
861 	ctx_hi = kernel_context(gt->i915);
862 	if (!ctx_hi)
863 		goto err_spin_lo;
864 	ctx_hi->sched.priority =
865 		I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
866 
867 	ctx_lo = kernel_context(gt->i915);
868 	if (!ctx_lo)
869 		goto err_ctx_hi;
870 	ctx_lo->sched.priority =
871 		I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
872 
873 	for_each_engine(engine, gt, id) {
874 		struct igt_live_test t;
875 		struct i915_request *rq;
876 
877 		if (!intel_engine_has_preemption(engine))
878 			continue;
879 
880 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
881 			err = -EIO;
882 			goto err_ctx_lo;
883 		}
884 
885 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
886 					    MI_ARB_CHECK);
887 		if (IS_ERR(rq)) {
888 			err = PTR_ERR(rq);
889 			goto err_ctx_lo;
890 		}
891 
892 		i915_request_add(rq);
893 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
894 			GEM_TRACE("lo spinner failed to start\n");
895 			GEM_TRACE_DUMP();
896 			intel_gt_set_wedged(gt);
897 			err = -EIO;
898 			goto err_ctx_lo;
899 		}
900 
901 		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
902 					    MI_ARB_CHECK);
903 		if (IS_ERR(rq)) {
904 			igt_spinner_end(&spin_lo);
905 			err = PTR_ERR(rq);
906 			goto err_ctx_lo;
907 		}
908 
909 		i915_request_add(rq);
910 		if (!igt_wait_for_spinner(&spin_hi, rq)) {
911 			GEM_TRACE("hi spinner failed to start\n");
912 			GEM_TRACE_DUMP();
913 			intel_gt_set_wedged(gt);
914 			err = -EIO;
915 			goto err_ctx_lo;
916 		}
917 
918 		igt_spinner_end(&spin_hi);
919 		igt_spinner_end(&spin_lo);
920 
921 		if (igt_live_test_end(&t)) {
922 			err = -EIO;
923 			goto err_ctx_lo;
924 		}
925 	}
926 
927 	err = 0;
928 err_ctx_lo:
929 	kernel_context_close(ctx_lo);
930 err_ctx_hi:
931 	kernel_context_close(ctx_hi);
932 err_spin_lo:
933 	igt_spinner_fini(&spin_lo);
934 err_spin_hi:
935 	igt_spinner_fini(&spin_hi);
936 	return err;
937 }
938 
939 static int live_late_preempt(void *arg)
940 {
941 	struct intel_gt *gt = arg;
942 	struct i915_gem_context *ctx_hi, *ctx_lo;
943 	struct igt_spinner spin_hi, spin_lo;
944 	struct intel_engine_cs *engine;
945 	struct i915_sched_attr attr = {};
946 	enum intel_engine_id id;
947 	int err = -ENOMEM;
948 
949 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
950 		return 0;
951 
952 	if (igt_spinner_init(&spin_hi, gt))
953 		return -ENOMEM;
954 
955 	if (igt_spinner_init(&spin_lo, gt))
956 		goto err_spin_hi;
957 
958 	ctx_hi = kernel_context(gt->i915);
959 	if (!ctx_hi)
960 		goto err_spin_lo;
961 
962 	ctx_lo = kernel_context(gt->i915);
963 	if (!ctx_lo)
964 		goto err_ctx_hi;
965 
966 	/* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
967 	ctx_lo->sched.priority = I915_USER_PRIORITY(1);
968 
969 	for_each_engine(engine, gt, id) {
970 		struct igt_live_test t;
971 		struct i915_request *rq;
972 
973 		if (!intel_engine_has_preemption(engine))
974 			continue;
975 
976 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
977 			err = -EIO;
978 			goto err_ctx_lo;
979 		}
980 
981 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
982 					    MI_ARB_CHECK);
983 		if (IS_ERR(rq)) {
984 			err = PTR_ERR(rq);
985 			goto err_ctx_lo;
986 		}
987 
988 		i915_request_add(rq);
989 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
990 			pr_err("First context failed to start\n");
991 			goto err_wedged;
992 		}
993 
994 		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
995 					    MI_NOOP);
996 		if (IS_ERR(rq)) {
997 			igt_spinner_end(&spin_lo);
998 			err = PTR_ERR(rq);
999 			goto err_ctx_lo;
1000 		}
1001 
1002 		i915_request_add(rq);
1003 		if (igt_wait_for_spinner(&spin_hi, rq)) {
1004 			pr_err("Second context overtook first?\n");
1005 			goto err_wedged;
1006 		}
1007 
1008 		attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1009 		engine->schedule(rq, &attr);
1010 
1011 		if (!igt_wait_for_spinner(&spin_hi, rq)) {
1012 			pr_err("High priority context failed to preempt the low priority context\n");
1013 			GEM_TRACE_DUMP();
1014 			goto err_wedged;
1015 		}
1016 
1017 		igt_spinner_end(&spin_hi);
1018 		igt_spinner_end(&spin_lo);
1019 
1020 		if (igt_live_test_end(&t)) {
1021 			err = -EIO;
1022 			goto err_ctx_lo;
1023 		}
1024 	}
1025 
1026 	err = 0;
1027 err_ctx_lo:
1028 	kernel_context_close(ctx_lo);
1029 err_ctx_hi:
1030 	kernel_context_close(ctx_hi);
1031 err_spin_lo:
1032 	igt_spinner_fini(&spin_lo);
1033 err_spin_hi:
1034 	igt_spinner_fini(&spin_hi);
1035 	return err;
1036 
1037 err_wedged:
1038 	igt_spinner_end(&spin_hi);
1039 	igt_spinner_end(&spin_lo);
1040 	intel_gt_set_wedged(gt);
1041 	err = -EIO;
1042 	goto err_ctx_lo;
1043 }
1044 
1045 struct preempt_client {
1046 	struct igt_spinner spin;
1047 	struct i915_gem_context *ctx;
1048 };
1049 
1050 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1051 {
1052 	c->ctx = kernel_context(gt->i915);
1053 	if (!c->ctx)
1054 		return -ENOMEM;
1055 
1056 	if (igt_spinner_init(&c->spin, gt))
1057 		goto err_ctx;
1058 
1059 	return 0;
1060 
1061 err_ctx:
1062 	kernel_context_close(c->ctx);
1063 	return -ENOMEM;
1064 }
1065 
1066 static void preempt_client_fini(struct preempt_client *c)
1067 {
1068 	igt_spinner_fini(&c->spin);
1069 	kernel_context_close(c->ctx);
1070 }
1071 
1072 static int live_nopreempt(void *arg)
1073 {
1074 	struct intel_gt *gt = arg;
1075 	struct intel_engine_cs *engine;
1076 	struct preempt_client a, b;
1077 	enum intel_engine_id id;
1078 	int err = -ENOMEM;
1079 
1080 	/*
1081 	 * Verify that we can disable preemption for an individual request
1082 	 * that may be being observed and not want to be interrupted.
1083 	 */
1084 
1085 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1086 		return 0;
1087 
1088 	if (preempt_client_init(gt, &a))
1089 		return -ENOMEM;
1090 	if (preempt_client_init(gt, &b))
1091 		goto err_client_a;
1092 	b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1093 
1094 	for_each_engine(engine, gt, id) {
1095 		struct i915_request *rq_a, *rq_b;
1096 
1097 		if (!intel_engine_has_preemption(engine))
1098 			continue;
1099 
1100 		engine->execlists.preempt_hang.count = 0;
1101 
1102 		rq_a = spinner_create_request(&a.spin,
1103 					      a.ctx, engine,
1104 					      MI_ARB_CHECK);
1105 		if (IS_ERR(rq_a)) {
1106 			err = PTR_ERR(rq_a);
1107 			goto err_client_b;
1108 		}
1109 
1110 		/* Low priority client, but unpreemptable! */
1111 		rq_a->flags |= I915_REQUEST_NOPREEMPT;
1112 
1113 		i915_request_add(rq_a);
1114 		if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1115 			pr_err("First client failed to start\n");
1116 			goto err_wedged;
1117 		}
1118 
1119 		rq_b = spinner_create_request(&b.spin,
1120 					      b.ctx, engine,
1121 					      MI_ARB_CHECK);
1122 		if (IS_ERR(rq_b)) {
1123 			err = PTR_ERR(rq_b);
1124 			goto err_client_b;
1125 		}
1126 
1127 		i915_request_add(rq_b);
1128 
1129 		/* B is much more important than A! (But A is unpreemptable.) */
1130 		GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
1131 
1132 		/* Wait long enough for preemption and timeslicing */
1133 		if (igt_wait_for_spinner(&b.spin, rq_b)) {
1134 			pr_err("Second client started too early!\n");
1135 			goto err_wedged;
1136 		}
1137 
1138 		igt_spinner_end(&a.spin);
1139 
1140 		if (!igt_wait_for_spinner(&b.spin, rq_b)) {
1141 			pr_err("Second client failed to start\n");
1142 			goto err_wedged;
1143 		}
1144 
1145 		igt_spinner_end(&b.spin);
1146 
1147 		if (engine->execlists.preempt_hang.count) {
1148 			pr_err("Preemption recorded x%d; should have been suppressed!\n",
1149 			       engine->execlists.preempt_hang.count);
1150 			err = -EINVAL;
1151 			goto err_wedged;
1152 		}
1153 
1154 		if (igt_flush_test(gt->i915))
1155 			goto err_wedged;
1156 	}
1157 
1158 	err = 0;
1159 err_client_b:
1160 	preempt_client_fini(&b);
1161 err_client_a:
1162 	preempt_client_fini(&a);
1163 	return err;
1164 
1165 err_wedged:
1166 	igt_spinner_end(&b.spin);
1167 	igt_spinner_end(&a.spin);
1168 	intel_gt_set_wedged(gt);
1169 	err = -EIO;
1170 	goto err_client_b;
1171 }
1172 
1173 struct live_preempt_cancel {
1174 	struct intel_engine_cs *engine;
1175 	struct preempt_client a, b;
1176 };
1177 
1178 static int __cancel_active0(struct live_preempt_cancel *arg)
1179 {
1180 	struct i915_request *rq;
1181 	struct igt_live_test t;
1182 	int err;
1183 
1184 	/* Preempt cancel of ELSP0 */
1185 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1186 	if (igt_live_test_begin(&t, arg->engine->i915,
1187 				__func__, arg->engine->name))
1188 		return -EIO;
1189 
1190 	clear_bit(CONTEXT_BANNED, &arg->a.ctx->flags);
1191 	rq = spinner_create_request(&arg->a.spin,
1192 				    arg->a.ctx, arg->engine,
1193 				    MI_ARB_CHECK);
1194 	if (IS_ERR(rq))
1195 		return PTR_ERR(rq);
1196 
1197 	i915_request_get(rq);
1198 	i915_request_add(rq);
1199 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
1200 		err = -EIO;
1201 		goto out;
1202 	}
1203 
1204 	i915_gem_context_set_banned(arg->a.ctx);
1205 	err = intel_engine_pulse(arg->engine);
1206 	if (err)
1207 		goto out;
1208 
1209 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1210 		err = -EIO;
1211 		goto out;
1212 	}
1213 
1214 	if (rq->fence.error != -EIO) {
1215 		pr_err("Cancelled inflight0 request did not report -EIO\n");
1216 		err = -EINVAL;
1217 		goto out;
1218 	}
1219 
1220 out:
1221 	i915_request_put(rq);
1222 	if (igt_live_test_end(&t))
1223 		err = -EIO;
1224 	return err;
1225 }
1226 
1227 static int __cancel_active1(struct live_preempt_cancel *arg)
1228 {
1229 	struct i915_request *rq[2] = {};
1230 	struct igt_live_test t;
1231 	int err;
1232 
1233 	/* Preempt cancel of ELSP1 */
1234 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1235 	if (igt_live_test_begin(&t, arg->engine->i915,
1236 				__func__, arg->engine->name))
1237 		return -EIO;
1238 
1239 	clear_bit(CONTEXT_BANNED, &arg->a.ctx->flags);
1240 	rq[0] = spinner_create_request(&arg->a.spin,
1241 				       arg->a.ctx, arg->engine,
1242 				       MI_NOOP); /* no preemption */
1243 	if (IS_ERR(rq[0]))
1244 		return PTR_ERR(rq[0]);
1245 
1246 	i915_request_get(rq[0]);
1247 	i915_request_add(rq[0]);
1248 	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1249 		err = -EIO;
1250 		goto out;
1251 	}
1252 
1253 	clear_bit(CONTEXT_BANNED, &arg->b.ctx->flags);
1254 	rq[1] = spinner_create_request(&arg->b.spin,
1255 				       arg->b.ctx, arg->engine,
1256 				       MI_ARB_CHECK);
1257 	if (IS_ERR(rq[1])) {
1258 		err = PTR_ERR(rq[1]);
1259 		goto out;
1260 	}
1261 
1262 	i915_request_get(rq[1]);
1263 	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1264 	i915_request_add(rq[1]);
1265 	if (err)
1266 		goto out;
1267 
1268 	i915_gem_context_set_banned(arg->b.ctx);
1269 	err = intel_engine_pulse(arg->engine);
1270 	if (err)
1271 		goto out;
1272 
1273 	igt_spinner_end(&arg->a.spin);
1274 	if (i915_request_wait(rq[1], 0, HZ / 5) < 0) {
1275 		err = -EIO;
1276 		goto out;
1277 	}
1278 
1279 	if (rq[0]->fence.error != 0) {
1280 		pr_err("Normal inflight0 request did not complete\n");
1281 		err = -EINVAL;
1282 		goto out;
1283 	}
1284 
1285 	if (rq[1]->fence.error != -EIO) {
1286 		pr_err("Cancelled inflight1 request did not report -EIO\n");
1287 		err = -EINVAL;
1288 		goto out;
1289 	}
1290 
1291 out:
1292 	i915_request_put(rq[1]);
1293 	i915_request_put(rq[0]);
1294 	if (igt_live_test_end(&t))
1295 		err = -EIO;
1296 	return err;
1297 }
1298 
1299 static int __cancel_queued(struct live_preempt_cancel *arg)
1300 {
1301 	struct i915_request *rq[3] = {};
1302 	struct igt_live_test t;
1303 	int err;
1304 
1305 	/* Full ELSP and one in the wings */
1306 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1307 	if (igt_live_test_begin(&t, arg->engine->i915,
1308 				__func__, arg->engine->name))
1309 		return -EIO;
1310 
1311 	clear_bit(CONTEXT_BANNED, &arg->a.ctx->flags);
1312 	rq[0] = spinner_create_request(&arg->a.spin,
1313 				       arg->a.ctx, arg->engine,
1314 				       MI_ARB_CHECK);
1315 	if (IS_ERR(rq[0]))
1316 		return PTR_ERR(rq[0]);
1317 
1318 	i915_request_get(rq[0]);
1319 	i915_request_add(rq[0]);
1320 	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1321 		err = -EIO;
1322 		goto out;
1323 	}
1324 
1325 	clear_bit(CONTEXT_BANNED, &arg->b.ctx->flags);
1326 	rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
1327 	if (IS_ERR(rq[1])) {
1328 		err = PTR_ERR(rq[1]);
1329 		goto out;
1330 	}
1331 
1332 	i915_request_get(rq[1]);
1333 	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1334 	i915_request_add(rq[1]);
1335 	if (err)
1336 		goto out;
1337 
1338 	rq[2] = spinner_create_request(&arg->b.spin,
1339 				       arg->a.ctx, arg->engine,
1340 				       MI_ARB_CHECK);
1341 	if (IS_ERR(rq[2])) {
1342 		err = PTR_ERR(rq[2]);
1343 		goto out;
1344 	}
1345 
1346 	i915_request_get(rq[2]);
1347 	err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
1348 	i915_request_add(rq[2]);
1349 	if (err)
1350 		goto out;
1351 
1352 	i915_gem_context_set_banned(arg->a.ctx);
1353 	err = intel_engine_pulse(arg->engine);
1354 	if (err)
1355 		goto out;
1356 
1357 	if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {
1358 		err = -EIO;
1359 		goto out;
1360 	}
1361 
1362 	if (rq[0]->fence.error != -EIO) {
1363 		pr_err("Cancelled inflight0 request did not report -EIO\n");
1364 		err = -EINVAL;
1365 		goto out;
1366 	}
1367 
1368 	if (rq[1]->fence.error != 0) {
1369 		pr_err("Normal inflight1 request did not complete\n");
1370 		err = -EINVAL;
1371 		goto out;
1372 	}
1373 
1374 	if (rq[2]->fence.error != -EIO) {
1375 		pr_err("Cancelled queued request did not report -EIO\n");
1376 		err = -EINVAL;
1377 		goto out;
1378 	}
1379 
1380 out:
1381 	i915_request_put(rq[2]);
1382 	i915_request_put(rq[1]);
1383 	i915_request_put(rq[0]);
1384 	if (igt_live_test_end(&t))
1385 		err = -EIO;
1386 	return err;
1387 }
1388 
1389 static int __cancel_hostile(struct live_preempt_cancel *arg)
1390 {
1391 	struct i915_request *rq;
1392 	int err;
1393 
1394 	/* Preempt cancel non-preemptible spinner in ELSP0 */
1395 	if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
1396 		return 0;
1397 
1398 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1399 	clear_bit(CONTEXT_BANNED, &arg->a.ctx->flags);
1400 	rq = spinner_create_request(&arg->a.spin,
1401 				    arg->a.ctx, arg->engine,
1402 				    MI_NOOP); /* preemption disabled */
1403 	if (IS_ERR(rq))
1404 		return PTR_ERR(rq);
1405 
1406 	i915_request_get(rq);
1407 	i915_request_add(rq);
1408 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
1409 		err = -EIO;
1410 		goto out;
1411 	}
1412 
1413 	i915_gem_context_set_banned(arg->a.ctx);
1414 	err = intel_engine_pulse(arg->engine); /* force reset */
1415 	if (err)
1416 		goto out;
1417 
1418 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1419 		err = -EIO;
1420 		goto out;
1421 	}
1422 
1423 	if (rq->fence.error != -EIO) {
1424 		pr_err("Cancelled inflight0 request did not report -EIO\n");
1425 		err = -EINVAL;
1426 		goto out;
1427 	}
1428 
1429 out:
1430 	i915_request_put(rq);
1431 	if (igt_flush_test(arg->engine->i915))
1432 		err = -EIO;
1433 	return err;
1434 }
1435 
1436 static int live_preempt_cancel(void *arg)
1437 {
1438 	struct intel_gt *gt = arg;
1439 	struct live_preempt_cancel data;
1440 	enum intel_engine_id id;
1441 	int err = -ENOMEM;
1442 
1443 	/*
1444 	 * To cancel an inflight context, we need to first remove it from the
1445 	 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
1446 	 */
1447 
1448 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1449 		return 0;
1450 
1451 	if (preempt_client_init(gt, &data.a))
1452 		return -ENOMEM;
1453 	if (preempt_client_init(gt, &data.b))
1454 		goto err_client_a;
1455 
1456 	for_each_engine(data.engine, gt, id) {
1457 		if (!intel_engine_has_preemption(data.engine))
1458 			continue;
1459 
1460 		err = __cancel_active0(&data);
1461 		if (err)
1462 			goto err_wedged;
1463 
1464 		err = __cancel_active1(&data);
1465 		if (err)
1466 			goto err_wedged;
1467 
1468 		err = __cancel_queued(&data);
1469 		if (err)
1470 			goto err_wedged;
1471 
1472 		err = __cancel_hostile(&data);
1473 		if (err)
1474 			goto err_wedged;
1475 	}
1476 
1477 	err = 0;
1478 err_client_b:
1479 	preempt_client_fini(&data.b);
1480 err_client_a:
1481 	preempt_client_fini(&data.a);
1482 	return err;
1483 
1484 err_wedged:
1485 	GEM_TRACE_DUMP();
1486 	igt_spinner_end(&data.b.spin);
1487 	igt_spinner_end(&data.a.spin);
1488 	intel_gt_set_wedged(gt);
1489 	goto err_client_b;
1490 }
1491 
1492 static int live_suppress_self_preempt(void *arg)
1493 {
1494 	struct intel_gt *gt = arg;
1495 	struct intel_engine_cs *engine;
1496 	struct i915_sched_attr attr = {
1497 		.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX)
1498 	};
1499 	struct preempt_client a, b;
1500 	enum intel_engine_id id;
1501 	int err = -ENOMEM;
1502 
1503 	/*
1504 	 * Verify that if a preemption request does not cause a change in
1505 	 * the current execution order, the preempt-to-idle injection is
1506 	 * skipped and that we do not accidentally apply it after the CS
1507 	 * completion event.
1508 	 */
1509 
1510 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1511 		return 0;
1512 
1513 	if (USES_GUC_SUBMISSION(gt->i915))
1514 		return 0; /* presume black blox */
1515 
1516 	if (intel_vgpu_active(gt->i915))
1517 		return 0; /* GVT forces single port & request submission */
1518 
1519 	if (preempt_client_init(gt, &a))
1520 		return -ENOMEM;
1521 	if (preempt_client_init(gt, &b))
1522 		goto err_client_a;
1523 
1524 	for_each_engine(engine, gt, id) {
1525 		struct i915_request *rq_a, *rq_b;
1526 		int depth;
1527 
1528 		if (!intel_engine_has_preemption(engine))
1529 			continue;
1530 
1531 		if (igt_flush_test(gt->i915))
1532 			goto err_wedged;
1533 
1534 		intel_engine_pm_get(engine);
1535 		engine->execlists.preempt_hang.count = 0;
1536 
1537 		rq_a = spinner_create_request(&a.spin,
1538 					      a.ctx, engine,
1539 					      MI_NOOP);
1540 		if (IS_ERR(rq_a)) {
1541 			err = PTR_ERR(rq_a);
1542 			intel_engine_pm_put(engine);
1543 			goto err_client_b;
1544 		}
1545 
1546 		i915_request_add(rq_a);
1547 		if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1548 			pr_err("First client failed to start\n");
1549 			intel_engine_pm_put(engine);
1550 			goto err_wedged;
1551 		}
1552 
1553 		/* Keep postponing the timer to avoid premature slicing */
1554 		mod_timer(&engine->execlists.timer, jiffies + HZ);
1555 		for (depth = 0; depth < 8; depth++) {
1556 			rq_b = spinner_create_request(&b.spin,
1557 						      b.ctx, engine,
1558 						      MI_NOOP);
1559 			if (IS_ERR(rq_b)) {
1560 				err = PTR_ERR(rq_b);
1561 				intel_engine_pm_put(engine);
1562 				goto err_client_b;
1563 			}
1564 			i915_request_add(rq_b);
1565 
1566 			GEM_BUG_ON(i915_request_completed(rq_a));
1567 			engine->schedule(rq_a, &attr);
1568 			igt_spinner_end(&a.spin);
1569 
1570 			if (!igt_wait_for_spinner(&b.spin, rq_b)) {
1571 				pr_err("Second client failed to start\n");
1572 				intel_engine_pm_put(engine);
1573 				goto err_wedged;
1574 			}
1575 
1576 			swap(a, b);
1577 			rq_a = rq_b;
1578 		}
1579 		igt_spinner_end(&a.spin);
1580 
1581 		if (engine->execlists.preempt_hang.count) {
1582 			pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
1583 			       engine->name,
1584 			       engine->execlists.preempt_hang.count,
1585 			       depth);
1586 			intel_engine_pm_put(engine);
1587 			err = -EINVAL;
1588 			goto err_client_b;
1589 		}
1590 
1591 		intel_engine_pm_put(engine);
1592 		if (igt_flush_test(gt->i915))
1593 			goto err_wedged;
1594 	}
1595 
1596 	err = 0;
1597 err_client_b:
1598 	preempt_client_fini(&b);
1599 err_client_a:
1600 	preempt_client_fini(&a);
1601 	return err;
1602 
1603 err_wedged:
1604 	igt_spinner_end(&b.spin);
1605 	igt_spinner_end(&a.spin);
1606 	intel_gt_set_wedged(gt);
1607 	err = -EIO;
1608 	goto err_client_b;
1609 }
1610 
1611 static int __i915_sw_fence_call
1612 dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
1613 {
1614 	return NOTIFY_DONE;
1615 }
1616 
1617 static struct i915_request *dummy_request(struct intel_engine_cs *engine)
1618 {
1619 	struct i915_request *rq;
1620 
1621 	rq = kzalloc(sizeof(*rq), GFP_KERNEL);
1622 	if (!rq)
1623 		return NULL;
1624 
1625 	rq->engine = engine;
1626 
1627 	spin_lock_init(&rq->lock);
1628 	INIT_LIST_HEAD(&rq->fence.cb_list);
1629 	rq->fence.lock = &rq->lock;
1630 	rq->fence.ops = &i915_fence_ops;
1631 
1632 	i915_sched_node_init(&rq->sched);
1633 
1634 	/* mark this request as permanently incomplete */
1635 	rq->fence.seqno = 1;
1636 	BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */
1637 	rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1;
1638 	GEM_BUG_ON(i915_request_completed(rq));
1639 
1640 	i915_sw_fence_init(&rq->submit, dummy_notify);
1641 	set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
1642 
1643 	spin_lock_init(&rq->lock);
1644 	rq->fence.lock = &rq->lock;
1645 	INIT_LIST_HEAD(&rq->fence.cb_list);
1646 
1647 	return rq;
1648 }
1649 
1650 static void dummy_request_free(struct i915_request *dummy)
1651 {
1652 	/* We have to fake the CS interrupt to kick the next request */
1653 	i915_sw_fence_commit(&dummy->submit);
1654 
1655 	i915_request_mark_complete(dummy);
1656 	dma_fence_signal(&dummy->fence);
1657 
1658 	i915_sched_node_fini(&dummy->sched);
1659 	i915_sw_fence_fini(&dummy->submit);
1660 
1661 	dma_fence_free(&dummy->fence);
1662 }
1663 
1664 static int live_suppress_wait_preempt(void *arg)
1665 {
1666 	struct intel_gt *gt = arg;
1667 	struct preempt_client client[4];
1668 	struct intel_engine_cs *engine;
1669 	enum intel_engine_id id;
1670 	int err = -ENOMEM;
1671 	int i;
1672 
1673 	/*
1674 	 * Waiters are given a little priority nudge, but not enough
1675 	 * to actually cause any preemption. Double check that we do
1676 	 * not needlessly generate preempt-to-idle cycles.
1677 	 */
1678 
1679 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1680 		return 0;
1681 
1682 	if (preempt_client_init(gt, &client[0])) /* ELSP[0] */
1683 		return -ENOMEM;
1684 	if (preempt_client_init(gt, &client[1])) /* ELSP[1] */
1685 		goto err_client_0;
1686 	if (preempt_client_init(gt, &client[2])) /* head of queue */
1687 		goto err_client_1;
1688 	if (preempt_client_init(gt, &client[3])) /* bystander */
1689 		goto err_client_2;
1690 
1691 	for_each_engine(engine, gt, id) {
1692 		int depth;
1693 
1694 		if (!intel_engine_has_preemption(engine))
1695 			continue;
1696 
1697 		if (!engine->emit_init_breadcrumb)
1698 			continue;
1699 
1700 		for (depth = 0; depth < ARRAY_SIZE(client); depth++) {
1701 			struct i915_request *rq[ARRAY_SIZE(client)];
1702 			struct i915_request *dummy;
1703 
1704 			engine->execlists.preempt_hang.count = 0;
1705 
1706 			dummy = dummy_request(engine);
1707 			if (!dummy)
1708 				goto err_client_3;
1709 
1710 			for (i = 0; i < ARRAY_SIZE(client); i++) {
1711 				rq[i] = spinner_create_request(&client[i].spin,
1712 							       client[i].ctx, engine,
1713 							       MI_NOOP);
1714 				if (IS_ERR(rq[i])) {
1715 					err = PTR_ERR(rq[i]);
1716 					goto err_wedged;
1717 				}
1718 
1719 				/* Disable NEWCLIENT promotion */
1720 				__i915_active_fence_set(&i915_request_timeline(rq[i])->last_request,
1721 							&dummy->fence);
1722 				i915_request_add(rq[i]);
1723 			}
1724 
1725 			dummy_request_free(dummy);
1726 
1727 			GEM_BUG_ON(i915_request_completed(rq[0]));
1728 			if (!igt_wait_for_spinner(&client[0].spin, rq[0])) {
1729 				pr_err("%s: First client failed to start\n",
1730 				       engine->name);
1731 				goto err_wedged;
1732 			}
1733 			GEM_BUG_ON(!i915_request_started(rq[0]));
1734 
1735 			if (i915_request_wait(rq[depth],
1736 					      I915_WAIT_PRIORITY,
1737 					      1) != -ETIME) {
1738 				pr_err("%s: Waiter depth:%d completed!\n",
1739 				       engine->name, depth);
1740 				goto err_wedged;
1741 			}
1742 
1743 			for (i = 0; i < ARRAY_SIZE(client); i++)
1744 				igt_spinner_end(&client[i].spin);
1745 
1746 			if (igt_flush_test(gt->i915))
1747 				goto err_wedged;
1748 
1749 			if (engine->execlists.preempt_hang.count) {
1750 				pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n",
1751 				       engine->name,
1752 				       engine->execlists.preempt_hang.count,
1753 				       depth);
1754 				err = -EINVAL;
1755 				goto err_client_3;
1756 			}
1757 		}
1758 	}
1759 
1760 	err = 0;
1761 err_client_3:
1762 	preempt_client_fini(&client[3]);
1763 err_client_2:
1764 	preempt_client_fini(&client[2]);
1765 err_client_1:
1766 	preempt_client_fini(&client[1]);
1767 err_client_0:
1768 	preempt_client_fini(&client[0]);
1769 	return err;
1770 
1771 err_wedged:
1772 	for (i = 0; i < ARRAY_SIZE(client); i++)
1773 		igt_spinner_end(&client[i].spin);
1774 	intel_gt_set_wedged(gt);
1775 	err = -EIO;
1776 	goto err_client_3;
1777 }
1778 
1779 static int live_chain_preempt(void *arg)
1780 {
1781 	struct intel_gt *gt = arg;
1782 	struct intel_engine_cs *engine;
1783 	struct preempt_client hi, lo;
1784 	enum intel_engine_id id;
1785 	int err = -ENOMEM;
1786 
1787 	/*
1788 	 * Build a chain AB...BA between two contexts (A, B) and request
1789 	 * preemption of the last request. It should then complete before
1790 	 * the previously submitted spinner in B.
1791 	 */
1792 
1793 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1794 		return 0;
1795 
1796 	if (preempt_client_init(gt, &hi))
1797 		return -ENOMEM;
1798 
1799 	if (preempt_client_init(gt, &lo))
1800 		goto err_client_hi;
1801 
1802 	for_each_engine(engine, gt, id) {
1803 		struct i915_sched_attr attr = {
1804 			.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
1805 		};
1806 		struct igt_live_test t;
1807 		struct i915_request *rq;
1808 		int ring_size, count, i;
1809 
1810 		if (!intel_engine_has_preemption(engine))
1811 			continue;
1812 
1813 		rq = spinner_create_request(&lo.spin,
1814 					    lo.ctx, engine,
1815 					    MI_ARB_CHECK);
1816 		if (IS_ERR(rq))
1817 			goto err_wedged;
1818 		i915_request_add(rq);
1819 
1820 		ring_size = rq->wa_tail - rq->head;
1821 		if (ring_size < 0)
1822 			ring_size += rq->ring->size;
1823 		ring_size = rq->ring->size / ring_size;
1824 		pr_debug("%s(%s): Using maximum of %d requests\n",
1825 			 __func__, engine->name, ring_size);
1826 
1827 		igt_spinner_end(&lo.spin);
1828 		if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1829 			pr_err("Timed out waiting to flush %s\n", engine->name);
1830 			goto err_wedged;
1831 		}
1832 
1833 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1834 			err = -EIO;
1835 			goto err_wedged;
1836 		}
1837 
1838 		for_each_prime_number_from(count, 1, ring_size) {
1839 			rq = spinner_create_request(&hi.spin,
1840 						    hi.ctx, engine,
1841 						    MI_ARB_CHECK);
1842 			if (IS_ERR(rq))
1843 				goto err_wedged;
1844 			i915_request_add(rq);
1845 			if (!igt_wait_for_spinner(&hi.spin, rq))
1846 				goto err_wedged;
1847 
1848 			rq = spinner_create_request(&lo.spin,
1849 						    lo.ctx, engine,
1850 						    MI_ARB_CHECK);
1851 			if (IS_ERR(rq))
1852 				goto err_wedged;
1853 			i915_request_add(rq);
1854 
1855 			for (i = 0; i < count; i++) {
1856 				rq = igt_request_alloc(lo.ctx, engine);
1857 				if (IS_ERR(rq))
1858 					goto err_wedged;
1859 				i915_request_add(rq);
1860 			}
1861 
1862 			rq = igt_request_alloc(hi.ctx, engine);
1863 			if (IS_ERR(rq))
1864 				goto err_wedged;
1865 			i915_request_add(rq);
1866 			engine->schedule(rq, &attr);
1867 
1868 			igt_spinner_end(&hi.spin);
1869 			if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1870 				struct drm_printer p =
1871 					drm_info_printer(gt->i915->drm.dev);
1872 
1873 				pr_err("Failed to preempt over chain of %d\n",
1874 				       count);
1875 				intel_engine_dump(engine, &p,
1876 						  "%s\n", engine->name);
1877 				goto err_wedged;
1878 			}
1879 			igt_spinner_end(&lo.spin);
1880 
1881 			rq = igt_request_alloc(lo.ctx, engine);
1882 			if (IS_ERR(rq))
1883 				goto err_wedged;
1884 			i915_request_add(rq);
1885 			if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1886 				struct drm_printer p =
1887 					drm_info_printer(gt->i915->drm.dev);
1888 
1889 				pr_err("Failed to flush low priority chain of %d requests\n",
1890 				       count);
1891 				intel_engine_dump(engine, &p,
1892 						  "%s\n", engine->name);
1893 				goto err_wedged;
1894 			}
1895 		}
1896 
1897 		if (igt_live_test_end(&t)) {
1898 			err = -EIO;
1899 			goto err_wedged;
1900 		}
1901 	}
1902 
1903 	err = 0;
1904 err_client_lo:
1905 	preempt_client_fini(&lo);
1906 err_client_hi:
1907 	preempt_client_fini(&hi);
1908 	return err;
1909 
1910 err_wedged:
1911 	igt_spinner_end(&hi.spin);
1912 	igt_spinner_end(&lo.spin);
1913 	intel_gt_set_wedged(gt);
1914 	err = -EIO;
1915 	goto err_client_lo;
1916 }
1917 
1918 static int live_preempt_hang(void *arg)
1919 {
1920 	struct intel_gt *gt = arg;
1921 	struct i915_gem_context *ctx_hi, *ctx_lo;
1922 	struct igt_spinner spin_hi, spin_lo;
1923 	struct intel_engine_cs *engine;
1924 	enum intel_engine_id id;
1925 	int err = -ENOMEM;
1926 
1927 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
1928 		return 0;
1929 
1930 	if (!intel_has_reset_engine(gt))
1931 		return 0;
1932 
1933 	if (igt_spinner_init(&spin_hi, gt))
1934 		return -ENOMEM;
1935 
1936 	if (igt_spinner_init(&spin_lo, gt))
1937 		goto err_spin_hi;
1938 
1939 	ctx_hi = kernel_context(gt->i915);
1940 	if (!ctx_hi)
1941 		goto err_spin_lo;
1942 	ctx_hi->sched.priority =
1943 		I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
1944 
1945 	ctx_lo = kernel_context(gt->i915);
1946 	if (!ctx_lo)
1947 		goto err_ctx_hi;
1948 	ctx_lo->sched.priority =
1949 		I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
1950 
1951 	for_each_engine(engine, gt, id) {
1952 		struct i915_request *rq;
1953 
1954 		if (!intel_engine_has_preemption(engine))
1955 			continue;
1956 
1957 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1958 					    MI_ARB_CHECK);
1959 		if (IS_ERR(rq)) {
1960 			err = PTR_ERR(rq);
1961 			goto err_ctx_lo;
1962 		}
1963 
1964 		i915_request_add(rq);
1965 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
1966 			GEM_TRACE("lo spinner failed to start\n");
1967 			GEM_TRACE_DUMP();
1968 			intel_gt_set_wedged(gt);
1969 			err = -EIO;
1970 			goto err_ctx_lo;
1971 		}
1972 
1973 		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1974 					    MI_ARB_CHECK);
1975 		if (IS_ERR(rq)) {
1976 			igt_spinner_end(&spin_lo);
1977 			err = PTR_ERR(rq);
1978 			goto err_ctx_lo;
1979 		}
1980 
1981 		init_completion(&engine->execlists.preempt_hang.completion);
1982 		engine->execlists.preempt_hang.inject_hang = true;
1983 
1984 		i915_request_add(rq);
1985 
1986 		if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion,
1987 						 HZ / 10)) {
1988 			pr_err("Preemption did not occur within timeout!");
1989 			GEM_TRACE_DUMP();
1990 			intel_gt_set_wedged(gt);
1991 			err = -EIO;
1992 			goto err_ctx_lo;
1993 		}
1994 
1995 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
1996 		intel_engine_reset(engine, NULL);
1997 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
1998 
1999 		engine->execlists.preempt_hang.inject_hang = false;
2000 
2001 		if (!igt_wait_for_spinner(&spin_hi, rq)) {
2002 			GEM_TRACE("hi spinner failed to start\n");
2003 			GEM_TRACE_DUMP();
2004 			intel_gt_set_wedged(gt);
2005 			err = -EIO;
2006 			goto err_ctx_lo;
2007 		}
2008 
2009 		igt_spinner_end(&spin_hi);
2010 		igt_spinner_end(&spin_lo);
2011 		if (igt_flush_test(gt->i915)) {
2012 			err = -EIO;
2013 			goto err_ctx_lo;
2014 		}
2015 	}
2016 
2017 	err = 0;
2018 err_ctx_lo:
2019 	kernel_context_close(ctx_lo);
2020 err_ctx_hi:
2021 	kernel_context_close(ctx_hi);
2022 err_spin_lo:
2023 	igt_spinner_fini(&spin_lo);
2024 err_spin_hi:
2025 	igt_spinner_fini(&spin_hi);
2026 	return err;
2027 }
2028 
2029 static int live_preempt_timeout(void *arg)
2030 {
2031 	struct intel_gt *gt = arg;
2032 	struct i915_gem_context *ctx_hi, *ctx_lo;
2033 	struct igt_spinner spin_lo;
2034 	struct intel_engine_cs *engine;
2035 	enum intel_engine_id id;
2036 	int err = -ENOMEM;
2037 
2038 	/*
2039 	 * Check that we force preemption to occur by cancelling the previous
2040 	 * context if it refuses to yield the GPU.
2041 	 */
2042 	if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
2043 		return 0;
2044 
2045 	if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2046 		return 0;
2047 
2048 	if (!intel_has_reset_engine(gt))
2049 		return 0;
2050 
2051 	if (igt_spinner_init(&spin_lo, gt))
2052 		return -ENOMEM;
2053 
2054 	ctx_hi = kernel_context(gt->i915);
2055 	if (!ctx_hi)
2056 		goto err_spin_lo;
2057 	ctx_hi->sched.priority =
2058 		I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
2059 
2060 	ctx_lo = kernel_context(gt->i915);
2061 	if (!ctx_lo)
2062 		goto err_ctx_hi;
2063 	ctx_lo->sched.priority =
2064 		I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
2065 
2066 	for_each_engine(engine, gt, id) {
2067 		unsigned long saved_timeout;
2068 		struct i915_request *rq;
2069 
2070 		if (!intel_engine_has_preemption(engine))
2071 			continue;
2072 
2073 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
2074 					    MI_NOOP); /* preemption disabled */
2075 		if (IS_ERR(rq)) {
2076 			err = PTR_ERR(rq);
2077 			goto err_ctx_lo;
2078 		}
2079 
2080 		i915_request_add(rq);
2081 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
2082 			intel_gt_set_wedged(gt);
2083 			err = -EIO;
2084 			goto err_ctx_lo;
2085 		}
2086 
2087 		rq = igt_request_alloc(ctx_hi, engine);
2088 		if (IS_ERR(rq)) {
2089 			igt_spinner_end(&spin_lo);
2090 			err = PTR_ERR(rq);
2091 			goto err_ctx_lo;
2092 		}
2093 
2094 		/* Flush the previous CS ack before changing timeouts */
2095 		while (READ_ONCE(engine->execlists.pending[0]))
2096 			cpu_relax();
2097 
2098 		saved_timeout = engine->props.preempt_timeout_ms;
2099 		engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
2100 
2101 		i915_request_get(rq);
2102 		i915_request_add(rq);
2103 
2104 		intel_engine_flush_submission(engine);
2105 		engine->props.preempt_timeout_ms = saved_timeout;
2106 
2107 		if (i915_request_wait(rq, 0, HZ / 10) < 0) {
2108 			intel_gt_set_wedged(gt);
2109 			i915_request_put(rq);
2110 			err = -ETIME;
2111 			goto err_ctx_lo;
2112 		}
2113 
2114 		igt_spinner_end(&spin_lo);
2115 		i915_request_put(rq);
2116 	}
2117 
2118 	err = 0;
2119 err_ctx_lo:
2120 	kernel_context_close(ctx_lo);
2121 err_ctx_hi:
2122 	kernel_context_close(ctx_hi);
2123 err_spin_lo:
2124 	igt_spinner_fini(&spin_lo);
2125 	return err;
2126 }
2127 
2128 static int random_range(struct rnd_state *rnd, int min, int max)
2129 {
2130 	return i915_prandom_u32_max_state(max - min, rnd) + min;
2131 }
2132 
2133 static int random_priority(struct rnd_state *rnd)
2134 {
2135 	return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
2136 }
2137 
2138 struct preempt_smoke {
2139 	struct intel_gt *gt;
2140 	struct i915_gem_context **contexts;
2141 	struct intel_engine_cs *engine;
2142 	struct drm_i915_gem_object *batch;
2143 	unsigned int ncontext;
2144 	struct rnd_state prng;
2145 	unsigned long count;
2146 };
2147 
2148 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
2149 {
2150 	return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
2151 							  &smoke->prng)];
2152 }
2153 
2154 static int smoke_submit(struct preempt_smoke *smoke,
2155 			struct i915_gem_context *ctx, int prio,
2156 			struct drm_i915_gem_object *batch)
2157 {
2158 	struct i915_request *rq;
2159 	struct i915_vma *vma = NULL;
2160 	int err = 0;
2161 
2162 	if (batch) {
2163 		struct i915_address_space *vm;
2164 
2165 		vm = i915_gem_context_get_vm_rcu(ctx);
2166 		vma = i915_vma_instance(batch, vm, NULL);
2167 		i915_vm_put(vm);
2168 		if (IS_ERR(vma))
2169 			return PTR_ERR(vma);
2170 
2171 		err = i915_vma_pin(vma, 0, 0, PIN_USER);
2172 		if (err)
2173 			return err;
2174 	}
2175 
2176 	ctx->sched.priority = prio;
2177 
2178 	rq = igt_request_alloc(ctx, smoke->engine);
2179 	if (IS_ERR(rq)) {
2180 		err = PTR_ERR(rq);
2181 		goto unpin;
2182 	}
2183 
2184 	if (vma) {
2185 		i915_vma_lock(vma);
2186 		err = i915_request_await_object(rq, vma->obj, false);
2187 		if (!err)
2188 			err = i915_vma_move_to_active(vma, rq, 0);
2189 		if (!err)
2190 			err = rq->engine->emit_bb_start(rq,
2191 							vma->node.start,
2192 							PAGE_SIZE, 0);
2193 		i915_vma_unlock(vma);
2194 	}
2195 
2196 	i915_request_add(rq);
2197 
2198 unpin:
2199 	if (vma)
2200 		i915_vma_unpin(vma);
2201 
2202 	return err;
2203 }
2204 
2205 static int smoke_crescendo_thread(void *arg)
2206 {
2207 	struct preempt_smoke *smoke = arg;
2208 	IGT_TIMEOUT(end_time);
2209 	unsigned long count;
2210 
2211 	count = 0;
2212 	do {
2213 		struct i915_gem_context *ctx = smoke_context(smoke);
2214 		int err;
2215 
2216 		err = smoke_submit(smoke,
2217 				   ctx, count % I915_PRIORITY_MAX,
2218 				   smoke->batch);
2219 		if (err)
2220 			return err;
2221 
2222 		count++;
2223 	} while (!__igt_timeout(end_time, NULL));
2224 
2225 	smoke->count = count;
2226 	return 0;
2227 }
2228 
2229 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
2230 #define BATCH BIT(0)
2231 {
2232 	struct task_struct *tsk[I915_NUM_ENGINES] = {};
2233 	struct preempt_smoke arg[I915_NUM_ENGINES];
2234 	struct intel_engine_cs *engine;
2235 	enum intel_engine_id id;
2236 	unsigned long count;
2237 	int err = 0;
2238 
2239 	for_each_engine(engine, smoke->gt, id) {
2240 		arg[id] = *smoke;
2241 		arg[id].engine = engine;
2242 		if (!(flags & BATCH))
2243 			arg[id].batch = NULL;
2244 		arg[id].count = 0;
2245 
2246 		tsk[id] = kthread_run(smoke_crescendo_thread, &arg,
2247 				      "igt/smoke:%d", id);
2248 		if (IS_ERR(tsk[id])) {
2249 			err = PTR_ERR(tsk[id]);
2250 			break;
2251 		}
2252 		get_task_struct(tsk[id]);
2253 	}
2254 
2255 	yield(); /* start all threads before we kthread_stop() */
2256 
2257 	count = 0;
2258 	for_each_engine(engine, smoke->gt, id) {
2259 		int status;
2260 
2261 		if (IS_ERR_OR_NULL(tsk[id]))
2262 			continue;
2263 
2264 		status = kthread_stop(tsk[id]);
2265 		if (status && !err)
2266 			err = status;
2267 
2268 		count += arg[id].count;
2269 
2270 		put_task_struct(tsk[id]);
2271 	}
2272 
2273 	pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
2274 		count, flags,
2275 		RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
2276 	return 0;
2277 }
2278 
2279 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
2280 {
2281 	enum intel_engine_id id;
2282 	IGT_TIMEOUT(end_time);
2283 	unsigned long count;
2284 
2285 	count = 0;
2286 	do {
2287 		for_each_engine(smoke->engine, smoke->gt, id) {
2288 			struct i915_gem_context *ctx = smoke_context(smoke);
2289 			int err;
2290 
2291 			err = smoke_submit(smoke,
2292 					   ctx, random_priority(&smoke->prng),
2293 					   flags & BATCH ? smoke->batch : NULL);
2294 			if (err)
2295 				return err;
2296 
2297 			count++;
2298 		}
2299 	} while (!__igt_timeout(end_time, NULL));
2300 
2301 	pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
2302 		count, flags,
2303 		RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
2304 	return 0;
2305 }
2306 
2307 static int live_preempt_smoke(void *arg)
2308 {
2309 	struct preempt_smoke smoke = {
2310 		.gt = arg,
2311 		.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
2312 		.ncontext = 1024,
2313 	};
2314 	const unsigned int phase[] = { 0, BATCH };
2315 	struct igt_live_test t;
2316 	int err = -ENOMEM;
2317 	u32 *cs;
2318 	int n;
2319 
2320 	if (!HAS_LOGICAL_RING_PREEMPTION(smoke.gt->i915))
2321 		return 0;
2322 
2323 	smoke.contexts = kmalloc_array(smoke.ncontext,
2324 				       sizeof(*smoke.contexts),
2325 				       GFP_KERNEL);
2326 	if (!smoke.contexts)
2327 		return -ENOMEM;
2328 
2329 	smoke.batch =
2330 		i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
2331 	if (IS_ERR(smoke.batch)) {
2332 		err = PTR_ERR(smoke.batch);
2333 		goto err_free;
2334 	}
2335 
2336 	cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
2337 	if (IS_ERR(cs)) {
2338 		err = PTR_ERR(cs);
2339 		goto err_batch;
2340 	}
2341 	for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
2342 		cs[n] = MI_ARB_CHECK;
2343 	cs[n] = MI_BATCH_BUFFER_END;
2344 	i915_gem_object_flush_map(smoke.batch);
2345 	i915_gem_object_unpin_map(smoke.batch);
2346 
2347 	if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
2348 		err = -EIO;
2349 		goto err_batch;
2350 	}
2351 
2352 	for (n = 0; n < smoke.ncontext; n++) {
2353 		smoke.contexts[n] = kernel_context(smoke.gt->i915);
2354 		if (!smoke.contexts[n])
2355 			goto err_ctx;
2356 	}
2357 
2358 	for (n = 0; n < ARRAY_SIZE(phase); n++) {
2359 		err = smoke_crescendo(&smoke, phase[n]);
2360 		if (err)
2361 			goto err_ctx;
2362 
2363 		err = smoke_random(&smoke, phase[n]);
2364 		if (err)
2365 			goto err_ctx;
2366 	}
2367 
2368 err_ctx:
2369 	if (igt_live_test_end(&t))
2370 		err = -EIO;
2371 
2372 	for (n = 0; n < smoke.ncontext; n++) {
2373 		if (!smoke.contexts[n])
2374 			break;
2375 		kernel_context_close(smoke.contexts[n]);
2376 	}
2377 
2378 err_batch:
2379 	i915_gem_object_put(smoke.batch);
2380 err_free:
2381 	kfree(smoke.contexts);
2382 
2383 	return err;
2384 }
2385 
2386 static int nop_virtual_engine(struct intel_gt *gt,
2387 			      struct intel_engine_cs **siblings,
2388 			      unsigned int nsibling,
2389 			      unsigned int nctx,
2390 			      unsigned int flags)
2391 #define CHAIN BIT(0)
2392 {
2393 	IGT_TIMEOUT(end_time);
2394 	struct i915_request *request[16];
2395 	struct i915_gem_context *ctx[16];
2396 	struct intel_context *ve[16];
2397 	unsigned long n, prime, nc;
2398 	struct igt_live_test t;
2399 	ktime_t times[2] = {};
2400 	int err;
2401 
2402 	GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ctx));
2403 
2404 	for (n = 0; n < nctx; n++) {
2405 		ctx[n] = kernel_context(gt->i915);
2406 		if (!ctx[n]) {
2407 			err = -ENOMEM;
2408 			nctx = n;
2409 			goto out;
2410 		}
2411 
2412 		ve[n] = intel_execlists_create_virtual(ctx[n],
2413 						       siblings, nsibling);
2414 		if (IS_ERR(ve[n])) {
2415 			kernel_context_close(ctx[n]);
2416 			err = PTR_ERR(ve[n]);
2417 			nctx = n;
2418 			goto out;
2419 		}
2420 
2421 		err = intel_context_pin(ve[n]);
2422 		if (err) {
2423 			intel_context_put(ve[n]);
2424 			kernel_context_close(ctx[n]);
2425 			nctx = n;
2426 			goto out;
2427 		}
2428 	}
2429 
2430 	err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
2431 	if (err)
2432 		goto out;
2433 
2434 	for_each_prime_number_from(prime, 1, 8192) {
2435 		times[1] = ktime_get_raw();
2436 
2437 		if (flags & CHAIN) {
2438 			for (nc = 0; nc < nctx; nc++) {
2439 				for (n = 0; n < prime; n++) {
2440 					request[nc] =
2441 						i915_request_create(ve[nc]);
2442 					if (IS_ERR(request[nc])) {
2443 						err = PTR_ERR(request[nc]);
2444 						goto out;
2445 					}
2446 
2447 					i915_request_add(request[nc]);
2448 				}
2449 			}
2450 		} else {
2451 			for (n = 0; n < prime; n++) {
2452 				for (nc = 0; nc < nctx; nc++) {
2453 					request[nc] =
2454 						i915_request_create(ve[nc]);
2455 					if (IS_ERR(request[nc])) {
2456 						err = PTR_ERR(request[nc]);
2457 						goto out;
2458 					}
2459 
2460 					i915_request_add(request[nc]);
2461 				}
2462 			}
2463 		}
2464 
2465 		for (nc = 0; nc < nctx; nc++) {
2466 			if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
2467 				pr_err("%s(%s): wait for %llx:%lld timed out\n",
2468 				       __func__, ve[0]->engine->name,
2469 				       request[nc]->fence.context,
2470 				       request[nc]->fence.seqno);
2471 
2472 				GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
2473 					  __func__, ve[0]->engine->name,
2474 					  request[nc]->fence.context,
2475 					  request[nc]->fence.seqno);
2476 				GEM_TRACE_DUMP();
2477 				intel_gt_set_wedged(gt);
2478 				break;
2479 			}
2480 		}
2481 
2482 		times[1] = ktime_sub(ktime_get_raw(), times[1]);
2483 		if (prime == 1)
2484 			times[0] = times[1];
2485 
2486 		if (__igt_timeout(end_time, NULL))
2487 			break;
2488 	}
2489 
2490 	err = igt_live_test_end(&t);
2491 	if (err)
2492 		goto out;
2493 
2494 	pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
2495 		nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
2496 		prime, div64_u64(ktime_to_ns(times[1]), prime));
2497 
2498 out:
2499 	if (igt_flush_test(gt->i915))
2500 		err = -EIO;
2501 
2502 	for (nc = 0; nc < nctx; nc++) {
2503 		intel_context_unpin(ve[nc]);
2504 		intel_context_put(ve[nc]);
2505 		kernel_context_close(ctx[nc]);
2506 	}
2507 	return err;
2508 }
2509 
2510 static int live_virtual_engine(void *arg)
2511 {
2512 	struct intel_gt *gt = arg;
2513 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
2514 	struct intel_engine_cs *engine;
2515 	enum intel_engine_id id;
2516 	unsigned int class, inst;
2517 	int err;
2518 
2519 	if (USES_GUC_SUBMISSION(gt->i915))
2520 		return 0;
2521 
2522 	for_each_engine(engine, gt, id) {
2523 		err = nop_virtual_engine(gt, &engine, 1, 1, 0);
2524 		if (err) {
2525 			pr_err("Failed to wrap engine %s: err=%d\n",
2526 			       engine->name, err);
2527 			return err;
2528 		}
2529 	}
2530 
2531 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
2532 		int nsibling, n;
2533 
2534 		nsibling = 0;
2535 		for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
2536 			if (!gt->engine_class[class][inst])
2537 				continue;
2538 
2539 			siblings[nsibling++] = gt->engine_class[class][inst];
2540 		}
2541 		if (nsibling < 2)
2542 			continue;
2543 
2544 		for (n = 1; n <= nsibling + 1; n++) {
2545 			err = nop_virtual_engine(gt, siblings, nsibling,
2546 						 n, 0);
2547 			if (err)
2548 				return err;
2549 		}
2550 
2551 		err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
2552 		if (err)
2553 			return err;
2554 	}
2555 
2556 	return 0;
2557 }
2558 
2559 static int mask_virtual_engine(struct intel_gt *gt,
2560 			       struct intel_engine_cs **siblings,
2561 			       unsigned int nsibling)
2562 {
2563 	struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
2564 	struct i915_gem_context *ctx;
2565 	struct intel_context *ve;
2566 	struct igt_live_test t;
2567 	unsigned int n;
2568 	int err;
2569 
2570 	/*
2571 	 * Check that by setting the execution mask on a request, we can
2572 	 * restrict it to our desired engine within the virtual engine.
2573 	 */
2574 
2575 	ctx = kernel_context(gt->i915);
2576 	if (!ctx)
2577 		return -ENOMEM;
2578 
2579 	ve = intel_execlists_create_virtual(ctx, siblings, nsibling);
2580 	if (IS_ERR(ve)) {
2581 		err = PTR_ERR(ve);
2582 		goto out_close;
2583 	}
2584 
2585 	err = intel_context_pin(ve);
2586 	if (err)
2587 		goto out_put;
2588 
2589 	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
2590 	if (err)
2591 		goto out_unpin;
2592 
2593 	for (n = 0; n < nsibling; n++) {
2594 		request[n] = i915_request_create(ve);
2595 		if (IS_ERR(request[n])) {
2596 			err = PTR_ERR(request[n]);
2597 			nsibling = n;
2598 			goto out;
2599 		}
2600 
2601 		/* Reverse order as it's more likely to be unnatural */
2602 		request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
2603 
2604 		i915_request_get(request[n]);
2605 		i915_request_add(request[n]);
2606 	}
2607 
2608 	for (n = 0; n < nsibling; n++) {
2609 		if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
2610 			pr_err("%s(%s): wait for %llx:%lld timed out\n",
2611 			       __func__, ve->engine->name,
2612 			       request[n]->fence.context,
2613 			       request[n]->fence.seqno);
2614 
2615 			GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
2616 				  __func__, ve->engine->name,
2617 				  request[n]->fence.context,
2618 				  request[n]->fence.seqno);
2619 			GEM_TRACE_DUMP();
2620 			intel_gt_set_wedged(gt);
2621 			err = -EIO;
2622 			goto out;
2623 		}
2624 
2625 		if (request[n]->engine != siblings[nsibling - n - 1]) {
2626 			pr_err("Executed on wrong sibling '%s', expected '%s'\n",
2627 			       request[n]->engine->name,
2628 			       siblings[nsibling - n - 1]->name);
2629 			err = -EINVAL;
2630 			goto out;
2631 		}
2632 	}
2633 
2634 	err = igt_live_test_end(&t);
2635 out:
2636 	if (igt_flush_test(gt->i915))
2637 		err = -EIO;
2638 
2639 	for (n = 0; n < nsibling; n++)
2640 		i915_request_put(request[n]);
2641 
2642 out_unpin:
2643 	intel_context_unpin(ve);
2644 out_put:
2645 	intel_context_put(ve);
2646 out_close:
2647 	kernel_context_close(ctx);
2648 	return err;
2649 }
2650 
2651 static int live_virtual_mask(void *arg)
2652 {
2653 	struct intel_gt *gt = arg;
2654 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
2655 	unsigned int class, inst;
2656 	int err;
2657 
2658 	if (USES_GUC_SUBMISSION(gt->i915))
2659 		return 0;
2660 
2661 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
2662 		unsigned int nsibling;
2663 
2664 		nsibling = 0;
2665 		for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
2666 			if (!gt->engine_class[class][inst])
2667 				break;
2668 
2669 			siblings[nsibling++] = gt->engine_class[class][inst];
2670 		}
2671 		if (nsibling < 2)
2672 			continue;
2673 
2674 		err = mask_virtual_engine(gt, siblings, nsibling);
2675 		if (err)
2676 			return err;
2677 	}
2678 
2679 	return 0;
2680 }
2681 
2682 static int preserved_virtual_engine(struct intel_gt *gt,
2683 				    struct intel_engine_cs **siblings,
2684 				    unsigned int nsibling)
2685 {
2686 	struct i915_request *last = NULL;
2687 	struct i915_gem_context *ctx;
2688 	struct intel_context *ve;
2689 	struct i915_vma *scratch;
2690 	struct igt_live_test t;
2691 	unsigned int n;
2692 	int err = 0;
2693 	u32 *cs;
2694 
2695 	ctx = kernel_context(gt->i915);
2696 	if (!ctx)
2697 		return -ENOMEM;
2698 
2699 	scratch = create_scratch(siblings[0]->gt);
2700 	if (IS_ERR(scratch)) {
2701 		err = PTR_ERR(scratch);
2702 		goto out_close;
2703 	}
2704 
2705 	ve = intel_execlists_create_virtual(ctx, siblings, nsibling);
2706 	if (IS_ERR(ve)) {
2707 		err = PTR_ERR(ve);
2708 		goto out_scratch;
2709 	}
2710 
2711 	err = intel_context_pin(ve);
2712 	if (err)
2713 		goto out_put;
2714 
2715 	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
2716 	if (err)
2717 		goto out_unpin;
2718 
2719 	for (n = 0; n < NUM_GPR_DW; n++) {
2720 		struct intel_engine_cs *engine = siblings[n % nsibling];
2721 		struct i915_request *rq;
2722 
2723 		rq = i915_request_create(ve);
2724 		if (IS_ERR(rq)) {
2725 			err = PTR_ERR(rq);
2726 			goto out_end;
2727 		}
2728 
2729 		i915_request_put(last);
2730 		last = i915_request_get(rq);
2731 
2732 		cs = intel_ring_begin(rq, 8);
2733 		if (IS_ERR(cs)) {
2734 			i915_request_add(rq);
2735 			err = PTR_ERR(cs);
2736 			goto out_end;
2737 		}
2738 
2739 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
2740 		*cs++ = CS_GPR(engine, n);
2741 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
2742 		*cs++ = 0;
2743 
2744 		*cs++ = MI_LOAD_REGISTER_IMM(1);
2745 		*cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
2746 		*cs++ = n + 1;
2747 
2748 		*cs++ = MI_NOOP;
2749 		intel_ring_advance(rq, cs);
2750 
2751 		/* Restrict this request to run on a particular engine */
2752 		rq->execution_mask = engine->mask;
2753 		i915_request_add(rq);
2754 	}
2755 
2756 	if (i915_request_wait(last, 0, HZ / 5) < 0) {
2757 		err = -ETIME;
2758 		goto out_end;
2759 	}
2760 
2761 	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
2762 	if (IS_ERR(cs)) {
2763 		err = PTR_ERR(cs);
2764 		goto out_end;
2765 	}
2766 
2767 	for (n = 0; n < NUM_GPR_DW; n++) {
2768 		if (cs[n] != n) {
2769 			pr_err("Incorrect value[%d] found for GPR[%d]\n",
2770 			       cs[n], n);
2771 			err = -EINVAL;
2772 			break;
2773 		}
2774 	}
2775 
2776 	i915_gem_object_unpin_map(scratch->obj);
2777 
2778 out_end:
2779 	if (igt_live_test_end(&t))
2780 		err = -EIO;
2781 	i915_request_put(last);
2782 out_unpin:
2783 	intel_context_unpin(ve);
2784 out_put:
2785 	intel_context_put(ve);
2786 out_scratch:
2787 	i915_vma_unpin_and_release(&scratch, 0);
2788 out_close:
2789 	kernel_context_close(ctx);
2790 	return err;
2791 }
2792 
2793 static int live_virtual_preserved(void *arg)
2794 {
2795 	struct intel_gt *gt = arg;
2796 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
2797 	unsigned int class, inst;
2798 
2799 	/*
2800 	 * Check that the context image retains non-privileged (user) registers
2801 	 * from one engine to the next. For this we check that the CS_GPR
2802 	 * are preserved.
2803 	 */
2804 
2805 	if (USES_GUC_SUBMISSION(gt->i915))
2806 		return 0;
2807 
2808 	/* As we use CS_GPR we cannot run before they existed on all engines. */
2809 	if (INTEL_GEN(gt->i915) < 9)
2810 		return 0;
2811 
2812 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
2813 		int nsibling, err;
2814 
2815 		nsibling = 0;
2816 		for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
2817 			if (!gt->engine_class[class][inst])
2818 				continue;
2819 
2820 			siblings[nsibling++] = gt->engine_class[class][inst];
2821 		}
2822 		if (nsibling < 2)
2823 			continue;
2824 
2825 		err = preserved_virtual_engine(gt, siblings, nsibling);
2826 		if (err)
2827 			return err;
2828 	}
2829 
2830 	return 0;
2831 }
2832 
2833 static int bond_virtual_engine(struct intel_gt *gt,
2834 			       unsigned int class,
2835 			       struct intel_engine_cs **siblings,
2836 			       unsigned int nsibling,
2837 			       unsigned int flags)
2838 #define BOND_SCHEDULE BIT(0)
2839 {
2840 	struct intel_engine_cs *master;
2841 	struct i915_gem_context *ctx;
2842 	struct i915_request *rq[16];
2843 	enum intel_engine_id id;
2844 	unsigned long n;
2845 	int err;
2846 
2847 	GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1);
2848 
2849 	ctx = kernel_context(gt->i915);
2850 	if (!ctx)
2851 		return -ENOMEM;
2852 
2853 	err = 0;
2854 	rq[0] = ERR_PTR(-ENOMEM);
2855 	for_each_engine(master, gt, id) {
2856 		struct i915_sw_fence fence = {};
2857 
2858 		if (master->class == class)
2859 			continue;
2860 
2861 		memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq));
2862 
2863 		rq[0] = igt_request_alloc(ctx, master);
2864 		if (IS_ERR(rq[0])) {
2865 			err = PTR_ERR(rq[0]);
2866 			goto out;
2867 		}
2868 		i915_request_get(rq[0]);
2869 
2870 		if (flags & BOND_SCHEDULE) {
2871 			onstack_fence_init(&fence);
2872 			err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit,
2873 							       &fence,
2874 							       GFP_KERNEL);
2875 		}
2876 		i915_request_add(rq[0]);
2877 		if (err < 0)
2878 			goto out;
2879 
2880 		for (n = 0; n < nsibling; n++) {
2881 			struct intel_context *ve;
2882 
2883 			ve = intel_execlists_create_virtual(ctx,
2884 							    siblings,
2885 							    nsibling);
2886 			if (IS_ERR(ve)) {
2887 				err = PTR_ERR(ve);
2888 				onstack_fence_fini(&fence);
2889 				goto out;
2890 			}
2891 
2892 			err = intel_virtual_engine_attach_bond(ve->engine,
2893 							       master,
2894 							       siblings[n]);
2895 			if (err) {
2896 				intel_context_put(ve);
2897 				onstack_fence_fini(&fence);
2898 				goto out;
2899 			}
2900 
2901 			err = intel_context_pin(ve);
2902 			intel_context_put(ve);
2903 			if (err) {
2904 				onstack_fence_fini(&fence);
2905 				goto out;
2906 			}
2907 
2908 			rq[n + 1] = i915_request_create(ve);
2909 			intel_context_unpin(ve);
2910 			if (IS_ERR(rq[n + 1])) {
2911 				err = PTR_ERR(rq[n + 1]);
2912 				onstack_fence_fini(&fence);
2913 				goto out;
2914 			}
2915 			i915_request_get(rq[n + 1]);
2916 
2917 			err = i915_request_await_execution(rq[n + 1],
2918 							   &rq[0]->fence,
2919 							   ve->engine->bond_execute);
2920 			i915_request_add(rq[n + 1]);
2921 			if (err < 0) {
2922 				onstack_fence_fini(&fence);
2923 				goto out;
2924 			}
2925 		}
2926 		onstack_fence_fini(&fence);
2927 
2928 		if (i915_request_wait(rq[0], 0, HZ / 10) < 0) {
2929 			pr_err("Master request did not execute (on %s)!\n",
2930 			       rq[0]->engine->name);
2931 			err = -EIO;
2932 			goto out;
2933 		}
2934 
2935 		for (n = 0; n < nsibling; n++) {
2936 			if (i915_request_wait(rq[n + 1], 0,
2937 					      MAX_SCHEDULE_TIMEOUT) < 0) {
2938 				err = -EIO;
2939 				goto out;
2940 			}
2941 
2942 			if (rq[n + 1]->engine != siblings[n]) {
2943 				pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n",
2944 				       siblings[n]->name,
2945 				       rq[n + 1]->engine->name,
2946 				       rq[0]->engine->name);
2947 				err = -EINVAL;
2948 				goto out;
2949 			}
2950 		}
2951 
2952 		for (n = 0; !IS_ERR(rq[n]); n++)
2953 			i915_request_put(rq[n]);
2954 		rq[0] = ERR_PTR(-ENOMEM);
2955 	}
2956 
2957 out:
2958 	for (n = 0; !IS_ERR(rq[n]); n++)
2959 		i915_request_put(rq[n]);
2960 	if (igt_flush_test(gt->i915))
2961 		err = -EIO;
2962 
2963 	kernel_context_close(ctx);
2964 	return err;
2965 }
2966 
2967 static int live_virtual_bond(void *arg)
2968 {
2969 	static const struct phase {
2970 		const char *name;
2971 		unsigned int flags;
2972 	} phases[] = {
2973 		{ "", 0 },
2974 		{ "schedule", BOND_SCHEDULE },
2975 		{ },
2976 	};
2977 	struct intel_gt *gt = arg;
2978 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
2979 	unsigned int class, inst;
2980 	int err;
2981 
2982 	if (USES_GUC_SUBMISSION(gt->i915))
2983 		return 0;
2984 
2985 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
2986 		const struct phase *p;
2987 		int nsibling;
2988 
2989 		nsibling = 0;
2990 		for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
2991 			if (!gt->engine_class[class][inst])
2992 				break;
2993 
2994 			GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings));
2995 			siblings[nsibling++] = gt->engine_class[class][inst];
2996 		}
2997 		if (nsibling < 2)
2998 			continue;
2999 
3000 		for (p = phases; p->name; p++) {
3001 			err = bond_virtual_engine(gt,
3002 						  class, siblings, nsibling,
3003 						  p->flags);
3004 			if (err) {
3005 				pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n",
3006 				       __func__, p->name, class, nsibling, err);
3007 				return err;
3008 			}
3009 		}
3010 	}
3011 
3012 	return 0;
3013 }
3014 
3015 int intel_execlists_live_selftests(struct drm_i915_private *i915)
3016 {
3017 	static const struct i915_subtest tests[] = {
3018 		SUBTEST(live_sanitycheck),
3019 		SUBTEST(live_unlite_switch),
3020 		SUBTEST(live_unlite_preempt),
3021 		SUBTEST(live_timeslice_preempt),
3022 		SUBTEST(live_timeslice_queue),
3023 		SUBTEST(live_busywait_preempt),
3024 		SUBTEST(live_preempt),
3025 		SUBTEST(live_late_preempt),
3026 		SUBTEST(live_nopreempt),
3027 		SUBTEST(live_preempt_cancel),
3028 		SUBTEST(live_suppress_self_preempt),
3029 		SUBTEST(live_suppress_wait_preempt),
3030 		SUBTEST(live_chain_preempt),
3031 		SUBTEST(live_preempt_hang),
3032 		SUBTEST(live_preempt_timeout),
3033 		SUBTEST(live_preempt_smoke),
3034 		SUBTEST(live_virtual_engine),
3035 		SUBTEST(live_virtual_mask),
3036 		SUBTEST(live_virtual_preserved),
3037 		SUBTEST(live_virtual_bond),
3038 	};
3039 
3040 	if (!HAS_EXECLISTS(i915))
3041 		return 0;
3042 
3043 	if (intel_gt_is_wedged(&i915->gt))
3044 		return 0;
3045 
3046 	return intel_gt_live_subtests(tests, &i915->gt);
3047 }
3048 
3049 static void hexdump(const void *buf, size_t len)
3050 {
3051 	const size_t rowsize = 8 * sizeof(u32);
3052 	const void *prev = NULL;
3053 	bool skip = false;
3054 	size_t pos;
3055 
3056 	for (pos = 0; pos < len; pos += rowsize) {
3057 		char line[128];
3058 
3059 		if (prev && !memcmp(prev, buf + pos, rowsize)) {
3060 			if (!skip) {
3061 				pr_info("*\n");
3062 				skip = true;
3063 			}
3064 			continue;
3065 		}
3066 
3067 		WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
3068 						rowsize, sizeof(u32),
3069 						line, sizeof(line),
3070 						false) >= sizeof(line));
3071 		pr_info("[%04zx] %s\n", pos, line);
3072 
3073 		prev = buf + pos;
3074 		skip = false;
3075 	}
3076 }
3077 
3078 static int live_lrc_layout(void *arg)
3079 {
3080 	struct intel_gt *gt = arg;
3081 	struct intel_engine_cs *engine;
3082 	enum intel_engine_id id;
3083 	u32 *mem;
3084 	int err;
3085 
3086 	/*
3087 	 * Check the registers offsets we use to create the initial reg state
3088 	 * match the layout saved by HW.
3089 	 */
3090 
3091 	mem = kmalloc(PAGE_SIZE, GFP_KERNEL);
3092 	if (!mem)
3093 		return -ENOMEM;
3094 
3095 	err = 0;
3096 	for_each_engine(engine, gt, id) {
3097 		u32 *hw, *lrc;
3098 		int dw;
3099 
3100 		if (!engine->default_state)
3101 			continue;
3102 
3103 		hw = i915_gem_object_pin_map(engine->default_state,
3104 					     I915_MAP_WB);
3105 		if (IS_ERR(hw)) {
3106 			err = PTR_ERR(hw);
3107 			break;
3108 		}
3109 		hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
3110 
3111 		lrc = memset(mem, 0, PAGE_SIZE);
3112 		execlists_init_reg_state(lrc,
3113 					 engine->kernel_context,
3114 					 engine,
3115 					 engine->kernel_context->ring,
3116 					 true);
3117 
3118 		dw = 0;
3119 		do {
3120 			u32 lri = hw[dw];
3121 
3122 			if (lri == 0) {
3123 				dw++;
3124 				continue;
3125 			}
3126 
3127 			if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
3128 				pr_err("%s: Expected LRI command at dword %d, found %08x\n",
3129 				       engine->name, dw, lri);
3130 				err = -EINVAL;
3131 				break;
3132 			}
3133 
3134 			if (lrc[dw] != lri) {
3135 				pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
3136 				       engine->name, dw, lri, lrc[dw]);
3137 				err = -EINVAL;
3138 				break;
3139 			}
3140 
3141 			lri &= 0x7f;
3142 			lri++;
3143 			dw++;
3144 
3145 			while (lri) {
3146 				if (hw[dw] != lrc[dw]) {
3147 					pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
3148 					       engine->name, dw, hw[dw], lrc[dw]);
3149 					err = -EINVAL;
3150 					break;
3151 				}
3152 
3153 				/*
3154 				 * Skip over the actual register value as we
3155 				 * expect that to differ.
3156 				 */
3157 				dw += 2;
3158 				lri -= 2;
3159 			}
3160 		} while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
3161 
3162 		if (err) {
3163 			pr_info("%s: HW register image:\n", engine->name);
3164 			hexdump(hw, PAGE_SIZE);
3165 
3166 			pr_info("%s: SW register image:\n", engine->name);
3167 			hexdump(lrc, PAGE_SIZE);
3168 		}
3169 
3170 		i915_gem_object_unpin_map(engine->default_state);
3171 		if (err)
3172 			break;
3173 	}
3174 
3175 	kfree(mem);
3176 	return err;
3177 }
3178 
3179 static int find_offset(const u32 *lri, u32 offset)
3180 {
3181 	int i;
3182 
3183 	for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
3184 		if (lri[i] == offset)
3185 			return i;
3186 
3187 	return -1;
3188 }
3189 
3190 static int live_lrc_fixed(void *arg)
3191 {
3192 	struct intel_gt *gt = arg;
3193 	struct intel_engine_cs *engine;
3194 	enum intel_engine_id id;
3195 	int err = 0;
3196 
3197 	/*
3198 	 * Check the assumed register offsets match the actual locations in
3199 	 * the context image.
3200 	 */
3201 
3202 	for_each_engine(engine, gt, id) {
3203 		const struct {
3204 			u32 reg;
3205 			u32 offset;
3206 			const char *name;
3207 		} tbl[] = {
3208 			{
3209 				i915_mmio_reg_offset(RING_START(engine->mmio_base)),
3210 				CTX_RING_BUFFER_START - 1,
3211 				"RING_START"
3212 			},
3213 			{
3214 				i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
3215 				CTX_RING_BUFFER_CONTROL - 1,
3216 				"RING_CTL"
3217 			},
3218 			{
3219 				i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
3220 				CTX_RING_HEAD - 1,
3221 				"RING_HEAD"
3222 			},
3223 			{
3224 				i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
3225 				CTX_RING_TAIL - 1,
3226 				"RING_TAIL"
3227 			},
3228 			{
3229 				i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
3230 				lrc_ring_mi_mode(engine),
3231 				"RING_MI_MODE"
3232 			},
3233 			{
3234 				engine->mmio_base + 0x110,
3235 				CTX_BB_STATE - 1,
3236 				"BB_STATE"
3237 			},
3238 			{ },
3239 		}, *t;
3240 		u32 *hw;
3241 
3242 		if (!engine->default_state)
3243 			continue;
3244 
3245 		hw = i915_gem_object_pin_map(engine->default_state,
3246 					     I915_MAP_WB);
3247 		if (IS_ERR(hw)) {
3248 			err = PTR_ERR(hw);
3249 			break;
3250 		}
3251 		hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
3252 
3253 		for (t = tbl; t->name; t++) {
3254 			int dw = find_offset(hw, t->reg);
3255 
3256 			if (dw != t->offset) {
3257 				pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
3258 				       engine->name,
3259 				       t->name,
3260 				       t->reg,
3261 				       dw,
3262 				       t->offset);
3263 				err = -EINVAL;
3264 			}
3265 		}
3266 
3267 		i915_gem_object_unpin_map(engine->default_state);
3268 	}
3269 
3270 	return err;
3271 }
3272 
3273 static int __live_lrc_state(struct i915_gem_context *fixme,
3274 			    struct intel_engine_cs *engine,
3275 			    struct i915_vma *scratch)
3276 {
3277 	struct intel_context *ce;
3278 	struct i915_request *rq;
3279 	enum {
3280 		RING_START_IDX = 0,
3281 		RING_TAIL_IDX,
3282 		MAX_IDX
3283 	};
3284 	u32 expected[MAX_IDX];
3285 	u32 *cs;
3286 	int err;
3287 	int n;
3288 
3289 	ce = intel_context_create(fixme, engine);
3290 	if (IS_ERR(ce))
3291 		return PTR_ERR(ce);
3292 
3293 	err = intel_context_pin(ce);
3294 	if (err)
3295 		goto err_put;
3296 
3297 	rq = i915_request_create(ce);
3298 	if (IS_ERR(rq)) {
3299 		err = PTR_ERR(rq);
3300 		goto err_unpin;
3301 	}
3302 
3303 	cs = intel_ring_begin(rq, 4 * MAX_IDX);
3304 	if (IS_ERR(cs)) {
3305 		err = PTR_ERR(cs);
3306 		i915_request_add(rq);
3307 		goto err_unpin;
3308 	}
3309 
3310 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3311 	*cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
3312 	*cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
3313 	*cs++ = 0;
3314 
3315 	expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
3316 
3317 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3318 	*cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
3319 	*cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
3320 	*cs++ = 0;
3321 
3322 	i915_request_get(rq);
3323 	i915_request_add(rq);
3324 
3325 	intel_engine_flush_submission(engine);
3326 	expected[RING_TAIL_IDX] = ce->ring->tail;
3327 
3328 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
3329 		err = -ETIME;
3330 		goto err_rq;
3331 	}
3332 
3333 	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
3334 	if (IS_ERR(cs)) {
3335 		err = PTR_ERR(cs);
3336 		goto err_rq;
3337 	}
3338 
3339 	for (n = 0; n < MAX_IDX; n++) {
3340 		if (cs[n] != expected[n]) {
3341 			pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
3342 			       engine->name, n, cs[n], expected[n]);
3343 			err = -EINVAL;
3344 			break;
3345 		}
3346 	}
3347 
3348 	i915_gem_object_unpin_map(scratch->obj);
3349 
3350 err_rq:
3351 	i915_request_put(rq);
3352 err_unpin:
3353 	intel_context_unpin(ce);
3354 err_put:
3355 	intel_context_put(ce);
3356 	return err;
3357 }
3358 
3359 static int live_lrc_state(void *arg)
3360 {
3361 	struct intel_gt *gt = arg;
3362 	struct intel_engine_cs *engine;
3363 	struct i915_gem_context *fixme;
3364 	struct i915_vma *scratch;
3365 	enum intel_engine_id id;
3366 	int err = 0;
3367 
3368 	/*
3369 	 * Check the live register state matches what we expect for this
3370 	 * intel_context.
3371 	 */
3372 
3373 	fixme = kernel_context(gt->i915);
3374 	if (!fixme)
3375 		return -ENOMEM;
3376 
3377 	scratch = create_scratch(gt);
3378 	if (IS_ERR(scratch)) {
3379 		err = PTR_ERR(scratch);
3380 		goto out_close;
3381 	}
3382 
3383 	for_each_engine(engine, gt, id) {
3384 		err = __live_lrc_state(fixme, engine, scratch);
3385 		if (err)
3386 			break;
3387 	}
3388 
3389 	if (igt_flush_test(gt->i915))
3390 		err = -EIO;
3391 
3392 	i915_vma_unpin_and_release(&scratch, 0);
3393 out_close:
3394 	kernel_context_close(fixme);
3395 	return err;
3396 }
3397 
3398 static int gpr_make_dirty(struct intel_engine_cs *engine)
3399 {
3400 	struct i915_request *rq;
3401 	u32 *cs;
3402 	int n;
3403 
3404 	rq = i915_request_create(engine->kernel_context);
3405 	if (IS_ERR(rq))
3406 		return PTR_ERR(rq);
3407 
3408 	cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
3409 	if (IS_ERR(cs)) {
3410 		i915_request_add(rq);
3411 		return PTR_ERR(cs);
3412 	}
3413 
3414 	*cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
3415 	for (n = 0; n < NUM_GPR_DW; n++) {
3416 		*cs++ = CS_GPR(engine, n);
3417 		*cs++ = STACK_MAGIC;
3418 	}
3419 	*cs++ = MI_NOOP;
3420 
3421 	intel_ring_advance(rq, cs);
3422 	i915_request_add(rq);
3423 
3424 	return 0;
3425 }
3426 
3427 static int __live_gpr_clear(struct i915_gem_context *fixme,
3428 			    struct intel_engine_cs *engine,
3429 			    struct i915_vma *scratch)
3430 {
3431 	struct intel_context *ce;
3432 	struct i915_request *rq;
3433 	u32 *cs;
3434 	int err;
3435 	int n;
3436 
3437 	if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
3438 		return 0; /* GPR only on rcs0 for gen8 */
3439 
3440 	err = gpr_make_dirty(engine);
3441 	if (err)
3442 		return err;
3443 
3444 	ce = intel_context_create(fixme, engine);
3445 	if (IS_ERR(ce))
3446 		return PTR_ERR(ce);
3447 
3448 	rq = intel_context_create_request(ce);
3449 	if (IS_ERR(rq)) {
3450 		err = PTR_ERR(rq);
3451 		goto err_put;
3452 	}
3453 
3454 	cs = intel_ring_begin(rq, 4 * NUM_GPR_DW);
3455 	if (IS_ERR(cs)) {
3456 		err = PTR_ERR(cs);
3457 		i915_request_add(rq);
3458 		goto err_put;
3459 	}
3460 
3461 	for (n = 0; n < NUM_GPR_DW; n++) {
3462 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3463 		*cs++ = CS_GPR(engine, n);
3464 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
3465 		*cs++ = 0;
3466 	}
3467 
3468 	i915_request_get(rq);
3469 	i915_request_add(rq);
3470 
3471 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
3472 		err = -ETIME;
3473 		goto err_rq;
3474 	}
3475 
3476 	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
3477 	if (IS_ERR(cs)) {
3478 		err = PTR_ERR(cs);
3479 		goto err_rq;
3480 	}
3481 
3482 	for (n = 0; n < NUM_GPR_DW; n++) {
3483 		if (cs[n]) {
3484 			pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
3485 			       engine->name,
3486 			       n / 2, n & 1 ? "udw" : "ldw",
3487 			       cs[n]);
3488 			err = -EINVAL;
3489 			break;
3490 		}
3491 	}
3492 
3493 	i915_gem_object_unpin_map(scratch->obj);
3494 
3495 err_rq:
3496 	i915_request_put(rq);
3497 err_put:
3498 	intel_context_put(ce);
3499 	return err;
3500 }
3501 
3502 static int live_gpr_clear(void *arg)
3503 {
3504 	struct intel_gt *gt = arg;
3505 	struct intel_engine_cs *engine;
3506 	struct i915_gem_context *fixme;
3507 	struct i915_vma *scratch;
3508 	enum intel_engine_id id;
3509 	int err = 0;
3510 
3511 	/*
3512 	 * Check that GPR registers are cleared in new contexts as we need
3513 	 * to avoid leaking any information from previous contexts.
3514 	 */
3515 
3516 	fixme = kernel_context(gt->i915);
3517 	if (!fixme)
3518 		return -ENOMEM;
3519 
3520 	scratch = create_scratch(gt);
3521 	if (IS_ERR(scratch)) {
3522 		err = PTR_ERR(scratch);
3523 		goto out_close;
3524 	}
3525 
3526 	for_each_engine(engine, gt, id) {
3527 		err = __live_gpr_clear(fixme, engine, scratch);
3528 		if (err)
3529 			break;
3530 	}
3531 
3532 	if (igt_flush_test(gt->i915))
3533 		err = -EIO;
3534 
3535 	i915_vma_unpin_and_release(&scratch, 0);
3536 out_close:
3537 	kernel_context_close(fixme);
3538 	return err;
3539 }
3540 
3541 int intel_lrc_live_selftests(struct drm_i915_private *i915)
3542 {
3543 	static const struct i915_subtest tests[] = {
3544 		SUBTEST(live_lrc_layout),
3545 		SUBTEST(live_lrc_fixed),
3546 		SUBTEST(live_lrc_state),
3547 		SUBTEST(live_gpr_clear),
3548 	};
3549 
3550 	if (!HAS_LOGICAL_RING_CONTEXTS(i915))
3551 		return 0;
3552 
3553 	return intel_gt_live_subtests(tests, &i915->gt);
3554 }
3555