1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include <linux/prime_numbers.h>
7 
8 #include "gem/i915_gem_internal.h"
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
12 #include "gt/selftest_engine_heartbeat.h"
13 
14 #include "i915_selftest.h"
15 #include "selftests/i915_random.h"
16 #include "selftests/igt_flush_test.h"
17 #include "selftests/igt_live_test.h"
18 #include "selftests/igt_spinner.h"
19 #include "selftests/lib_sw_fence.h"
20 
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
23 
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR 16
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27 
28 static bool is_active(struct i915_request *rq)
29 {
30 	if (i915_request_is_active(rq))
31 		return true;
32 
33 	if (i915_request_on_hold(rq))
34 		return true;
35 
36 	if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
37 		return true;
38 
39 	return false;
40 }
41 
42 static int wait_for_submit(struct intel_engine_cs *engine,
43 			   struct i915_request *rq,
44 			   unsigned long timeout)
45 {
46 	/* Ignore our own attempts to suppress excess tasklets */
47 	tasklet_hi_schedule(&engine->sched_engine->tasklet);
48 
49 	timeout += jiffies;
50 	do {
51 		bool done = time_after(jiffies, timeout);
52 
53 		if (i915_request_completed(rq)) /* that was quick! */
54 			return 0;
55 
56 		/* Wait until the HW has acknowleged the submission (or err) */
57 		intel_engine_flush_submission(engine);
58 		if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
59 			return 0;
60 
61 		if (done)
62 			return -ETIME;
63 
64 		cond_resched();
65 	} while (1);
66 }
67 
68 static int wait_for_reset(struct intel_engine_cs *engine,
69 			  struct i915_request *rq,
70 			  unsigned long timeout)
71 {
72 	timeout += jiffies;
73 
74 	do {
75 		cond_resched();
76 		intel_engine_flush_submission(engine);
77 
78 		if (READ_ONCE(engine->execlists.pending[0]))
79 			continue;
80 
81 		if (i915_request_completed(rq))
82 			break;
83 
84 		if (READ_ONCE(rq->fence.error))
85 			break;
86 	} while (time_before(jiffies, timeout));
87 
88 	flush_scheduled_work();
89 
90 	if (rq->fence.error != -EIO) {
91 		pr_err("%s: hanging request %llx:%lld not reset\n",
92 		       engine->name,
93 		       rq->fence.context,
94 		       rq->fence.seqno);
95 		return -EINVAL;
96 	}
97 
98 	/* Give the request a jiffie to complete after flushing the worker */
99 	if (i915_request_wait(rq, 0,
100 			      max(0l, (long)(timeout - jiffies)) + 1) < 0) {
101 		pr_err("%s: hanging request %llx:%lld did not complete\n",
102 		       engine->name,
103 		       rq->fence.context,
104 		       rq->fence.seqno);
105 		return -ETIME;
106 	}
107 
108 	return 0;
109 }
110 
111 static int live_sanitycheck(void *arg)
112 {
113 	struct intel_gt *gt = arg;
114 	struct intel_engine_cs *engine;
115 	enum intel_engine_id id;
116 	struct igt_spinner spin;
117 	int err = 0;
118 
119 	if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
120 		return 0;
121 
122 	if (igt_spinner_init(&spin, gt))
123 		return -ENOMEM;
124 
125 	for_each_engine(engine, gt, id) {
126 		struct intel_context *ce;
127 		struct i915_request *rq;
128 
129 		ce = intel_context_create(engine);
130 		if (IS_ERR(ce)) {
131 			err = PTR_ERR(ce);
132 			break;
133 		}
134 
135 		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
136 		if (IS_ERR(rq)) {
137 			err = PTR_ERR(rq);
138 			goto out_ctx;
139 		}
140 
141 		i915_request_add(rq);
142 		if (!igt_wait_for_spinner(&spin, rq)) {
143 			GEM_TRACE("spinner failed to start\n");
144 			GEM_TRACE_DUMP();
145 			intel_gt_set_wedged(gt);
146 			err = -EIO;
147 			goto out_ctx;
148 		}
149 
150 		igt_spinner_end(&spin);
151 		if (igt_flush_test(gt->i915)) {
152 			err = -EIO;
153 			goto out_ctx;
154 		}
155 
156 out_ctx:
157 		intel_context_put(ce);
158 		if (err)
159 			break;
160 	}
161 
162 	igt_spinner_fini(&spin);
163 	return err;
164 }
165 
166 static int live_unlite_restore(struct intel_gt *gt, int prio)
167 {
168 	struct intel_engine_cs *engine;
169 	enum intel_engine_id id;
170 	struct igt_spinner spin;
171 	int err = -ENOMEM;
172 
173 	/*
174 	 * Check that we can correctly context switch between 2 instances
175 	 * on the same engine from the same parent context.
176 	 */
177 
178 	if (igt_spinner_init(&spin, gt))
179 		return err;
180 
181 	err = 0;
182 	for_each_engine(engine, gt, id) {
183 		struct intel_context *ce[2] = {};
184 		struct i915_request *rq[2];
185 		struct igt_live_test t;
186 		int n;
187 
188 		if (prio && !intel_engine_has_preemption(engine))
189 			continue;
190 
191 		if (!intel_engine_can_store_dword(engine))
192 			continue;
193 
194 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
195 			err = -EIO;
196 			break;
197 		}
198 		st_engine_heartbeat_disable(engine);
199 
200 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
201 			struct intel_context *tmp;
202 
203 			tmp = intel_context_create(engine);
204 			if (IS_ERR(tmp)) {
205 				err = PTR_ERR(tmp);
206 				goto err_ce;
207 			}
208 
209 			err = intel_context_pin(tmp);
210 			if (err) {
211 				intel_context_put(tmp);
212 				goto err_ce;
213 			}
214 
215 			/*
216 			 * Setup the pair of contexts such that if we
217 			 * lite-restore using the RING_TAIL from ce[1] it
218 			 * will execute garbage from ce[0]->ring.
219 			 */
220 			memset(tmp->ring->vaddr,
221 			       POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
222 			       tmp->ring->vma->size);
223 
224 			ce[n] = tmp;
225 		}
226 		GEM_BUG_ON(!ce[1]->ring->size);
227 		intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
228 		lrc_update_regs(ce[1], engine, ce[1]->ring->head);
229 
230 		rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
231 		if (IS_ERR(rq[0])) {
232 			err = PTR_ERR(rq[0]);
233 			goto err_ce;
234 		}
235 
236 		i915_request_get(rq[0]);
237 		i915_request_add(rq[0]);
238 		GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
239 
240 		if (!igt_wait_for_spinner(&spin, rq[0])) {
241 			i915_request_put(rq[0]);
242 			goto err_ce;
243 		}
244 
245 		rq[1] = i915_request_create(ce[1]);
246 		if (IS_ERR(rq[1])) {
247 			err = PTR_ERR(rq[1]);
248 			i915_request_put(rq[0]);
249 			goto err_ce;
250 		}
251 
252 		if (!prio) {
253 			/*
254 			 * Ensure we do the switch to ce[1] on completion.
255 			 *
256 			 * rq[0] is already submitted, so this should reduce
257 			 * to a no-op (a wait on a request on the same engine
258 			 * uses the submit fence, not the completion fence),
259 			 * but it will install a dependency on rq[1] for rq[0]
260 			 * that will prevent the pair being reordered by
261 			 * timeslicing.
262 			 */
263 			i915_request_await_dma_fence(rq[1], &rq[0]->fence);
264 		}
265 
266 		i915_request_get(rq[1]);
267 		i915_request_add(rq[1]);
268 		GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
269 		i915_request_put(rq[0]);
270 
271 		if (prio) {
272 			struct i915_sched_attr attr = {
273 				.priority = prio,
274 			};
275 
276 			/* Alternatively preempt the spinner with ce[1] */
277 			engine->sched_engine->schedule(rq[1], &attr);
278 		}
279 
280 		/* And switch back to ce[0] for good measure */
281 		rq[0] = i915_request_create(ce[0]);
282 		if (IS_ERR(rq[0])) {
283 			err = PTR_ERR(rq[0]);
284 			i915_request_put(rq[1]);
285 			goto err_ce;
286 		}
287 
288 		i915_request_await_dma_fence(rq[0], &rq[1]->fence);
289 		i915_request_get(rq[0]);
290 		i915_request_add(rq[0]);
291 		GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
292 		i915_request_put(rq[1]);
293 		i915_request_put(rq[0]);
294 
295 err_ce:
296 		intel_engine_flush_submission(engine);
297 		igt_spinner_end(&spin);
298 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
299 			if (IS_ERR_OR_NULL(ce[n]))
300 				break;
301 
302 			intel_context_unpin(ce[n]);
303 			intel_context_put(ce[n]);
304 		}
305 
306 		st_engine_heartbeat_enable(engine);
307 		if (igt_live_test_end(&t))
308 			err = -EIO;
309 		if (err)
310 			break;
311 	}
312 
313 	igt_spinner_fini(&spin);
314 	return err;
315 }
316 
317 static int live_unlite_switch(void *arg)
318 {
319 	return live_unlite_restore(arg, 0);
320 }
321 
322 static int live_unlite_preempt(void *arg)
323 {
324 	return live_unlite_restore(arg, I915_PRIORITY_MAX);
325 }
326 
327 static int live_unlite_ring(void *arg)
328 {
329 	struct intel_gt *gt = arg;
330 	struct intel_engine_cs *engine;
331 	struct igt_spinner spin;
332 	enum intel_engine_id id;
333 	int err = 0;
334 
335 	/*
336 	 * Setup a preemption event that will cause almost the entire ring
337 	 * to be unwound, potentially fooling our intel_ring_direction()
338 	 * into emitting a forward lite-restore instead of the rollback.
339 	 */
340 
341 	if (igt_spinner_init(&spin, gt))
342 		return -ENOMEM;
343 
344 	for_each_engine(engine, gt, id) {
345 		struct intel_context *ce[2] = {};
346 		struct i915_request *rq;
347 		struct igt_live_test t;
348 		int n;
349 
350 		if (!intel_engine_has_preemption(engine))
351 			continue;
352 
353 		if (!intel_engine_can_store_dword(engine))
354 			continue;
355 
356 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
357 			err = -EIO;
358 			break;
359 		}
360 		st_engine_heartbeat_disable(engine);
361 
362 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
363 			struct intel_context *tmp;
364 
365 			tmp = intel_context_create(engine);
366 			if (IS_ERR(tmp)) {
367 				err = PTR_ERR(tmp);
368 				goto err_ce;
369 			}
370 
371 			err = intel_context_pin(tmp);
372 			if (err) {
373 				intel_context_put(tmp);
374 				goto err_ce;
375 			}
376 
377 			memset32(tmp->ring->vaddr,
378 				 0xdeadbeef, /* trigger a hang if executed */
379 				 tmp->ring->vma->size / sizeof(u32));
380 
381 			ce[n] = tmp;
382 		}
383 
384 		/* Create max prio spinner, followed by N low prio nops */
385 		rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
386 		if (IS_ERR(rq)) {
387 			err = PTR_ERR(rq);
388 			goto err_ce;
389 		}
390 
391 		i915_request_get(rq);
392 		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
393 		i915_request_add(rq);
394 
395 		if (!igt_wait_for_spinner(&spin, rq)) {
396 			intel_gt_set_wedged(gt);
397 			i915_request_put(rq);
398 			err = -ETIME;
399 			goto err_ce;
400 		}
401 
402 		/* Fill the ring, until we will cause a wrap */
403 		n = 0;
404 		while (intel_ring_direction(ce[0]->ring,
405 					    rq->wa_tail,
406 					    ce[0]->ring->tail) <= 0) {
407 			struct i915_request *tmp;
408 
409 			tmp = intel_context_create_request(ce[0]);
410 			if (IS_ERR(tmp)) {
411 				err = PTR_ERR(tmp);
412 				i915_request_put(rq);
413 				goto err_ce;
414 			}
415 
416 			i915_request_add(tmp);
417 			intel_engine_flush_submission(engine);
418 			n++;
419 		}
420 		intel_engine_flush_submission(engine);
421 		pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
422 			 engine->name, n,
423 			 ce[0]->ring->size,
424 			 ce[0]->ring->tail,
425 			 ce[0]->ring->emit,
426 			 rq->tail);
427 		GEM_BUG_ON(intel_ring_direction(ce[0]->ring,
428 						rq->tail,
429 						ce[0]->ring->tail) <= 0);
430 		i915_request_put(rq);
431 
432 		/* Create a second ring to preempt the first ring after rq[0] */
433 		rq = intel_context_create_request(ce[1]);
434 		if (IS_ERR(rq)) {
435 			err = PTR_ERR(rq);
436 			goto err_ce;
437 		}
438 
439 		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
440 		i915_request_get(rq);
441 		i915_request_add(rq);
442 
443 		err = wait_for_submit(engine, rq, HZ / 2);
444 		i915_request_put(rq);
445 		if (err) {
446 			pr_err("%s: preemption request was not submitted\n",
447 			       engine->name);
448 			err = -ETIME;
449 		}
450 
451 		pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
452 			 engine->name,
453 			 ce[0]->ring->tail, ce[0]->ring->emit,
454 			 ce[1]->ring->tail, ce[1]->ring->emit);
455 
456 err_ce:
457 		intel_engine_flush_submission(engine);
458 		igt_spinner_end(&spin);
459 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
460 			if (IS_ERR_OR_NULL(ce[n]))
461 				break;
462 
463 			intel_context_unpin(ce[n]);
464 			intel_context_put(ce[n]);
465 		}
466 		st_engine_heartbeat_enable(engine);
467 		if (igt_live_test_end(&t))
468 			err = -EIO;
469 		if (err)
470 			break;
471 	}
472 
473 	igt_spinner_fini(&spin);
474 	return err;
475 }
476 
477 static int live_pin_rewind(void *arg)
478 {
479 	struct intel_gt *gt = arg;
480 	struct intel_engine_cs *engine;
481 	enum intel_engine_id id;
482 	int err = 0;
483 
484 	/*
485 	 * We have to be careful not to trust intel_ring too much, for example
486 	 * ring->head is updated upon retire which is out of sync with pinning
487 	 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,
488 	 * or else we risk writing an older, stale value.
489 	 *
490 	 * To simulate this, let's apply a bit of deliberate sabotague.
491 	 */
492 
493 	for_each_engine(engine, gt, id) {
494 		struct intel_context *ce;
495 		struct i915_request *rq;
496 		struct intel_ring *ring;
497 		struct igt_live_test t;
498 
499 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
500 			err = -EIO;
501 			break;
502 		}
503 
504 		ce = intel_context_create(engine);
505 		if (IS_ERR(ce)) {
506 			err = PTR_ERR(ce);
507 			break;
508 		}
509 
510 		err = intel_context_pin(ce);
511 		if (err) {
512 			intel_context_put(ce);
513 			break;
514 		}
515 
516 		/* Keep the context awake while we play games */
517 		err = i915_active_acquire(&ce->active);
518 		if (err) {
519 			intel_context_unpin(ce);
520 			intel_context_put(ce);
521 			break;
522 		}
523 		ring = ce->ring;
524 
525 		/* Poison the ring, and offset the next request from HEAD */
526 		memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
527 		ring->emit = ring->size / 2;
528 		ring->tail = ring->emit;
529 		GEM_BUG_ON(ring->head);
530 
531 		intel_context_unpin(ce);
532 
533 		/* Submit a simple nop request */
534 		GEM_BUG_ON(intel_context_is_pinned(ce));
535 		rq = intel_context_create_request(ce);
536 		i915_active_release(&ce->active); /* e.g. async retire */
537 		intel_context_put(ce);
538 		if (IS_ERR(rq)) {
539 			err = PTR_ERR(rq);
540 			break;
541 		}
542 		GEM_BUG_ON(!rq->head);
543 		i915_request_add(rq);
544 
545 		/* Expect not to hang! */
546 		if (igt_live_test_end(&t)) {
547 			err = -EIO;
548 			break;
549 		}
550 	}
551 
552 	return err;
553 }
554 
555 static int engine_lock_reset_tasklet(struct intel_engine_cs *engine)
556 {
557 	tasklet_disable(&engine->sched_engine->tasklet);
558 	local_bh_disable();
559 
560 	if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
561 			     &engine->gt->reset.flags)) {
562 		local_bh_enable();
563 		tasklet_enable(&engine->sched_engine->tasklet);
564 
565 		intel_gt_set_wedged(engine->gt);
566 		return -EBUSY;
567 	}
568 
569 	return 0;
570 }
571 
572 static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine)
573 {
574 	clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,
575 			      &engine->gt->reset.flags);
576 
577 	local_bh_enable();
578 	tasklet_enable(&engine->sched_engine->tasklet);
579 }
580 
581 static int live_hold_reset(void *arg)
582 {
583 	struct intel_gt *gt = arg;
584 	struct intel_engine_cs *engine;
585 	enum intel_engine_id id;
586 	struct igt_spinner spin;
587 	int err = 0;
588 
589 	/*
590 	 * In order to support offline error capture for fast preempt reset,
591 	 * we need to decouple the guilty request and ensure that it and its
592 	 * descendents are not executed while the capture is in progress.
593 	 */
594 
595 	if (!intel_has_reset_engine(gt))
596 		return 0;
597 
598 	if (igt_spinner_init(&spin, gt))
599 		return -ENOMEM;
600 
601 	for_each_engine(engine, gt, id) {
602 		struct intel_context *ce;
603 		struct i915_request *rq;
604 
605 		ce = intel_context_create(engine);
606 		if (IS_ERR(ce)) {
607 			err = PTR_ERR(ce);
608 			break;
609 		}
610 
611 		st_engine_heartbeat_disable(engine);
612 
613 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
614 		if (IS_ERR(rq)) {
615 			err = PTR_ERR(rq);
616 			goto out;
617 		}
618 		i915_request_add(rq);
619 
620 		if (!igt_wait_for_spinner(&spin, rq)) {
621 			intel_gt_set_wedged(gt);
622 			err = -ETIME;
623 			goto out;
624 		}
625 
626 		/* We have our request executing, now remove it and reset */
627 
628 		err = engine_lock_reset_tasklet(engine);
629 		if (err)
630 			goto out;
631 
632 		engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
633 		GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
634 
635 		i915_request_get(rq);
636 		execlists_hold(engine, rq);
637 		GEM_BUG_ON(!i915_request_on_hold(rq));
638 
639 		__intel_engine_reset_bh(engine, NULL);
640 		GEM_BUG_ON(rq->fence.error != -EIO);
641 
642 		engine_unlock_reset_tasklet(engine);
643 
644 		/* Check that we do not resubmit the held request */
645 		if (!i915_request_wait(rq, 0, HZ / 5)) {
646 			pr_err("%s: on hold request completed!\n",
647 			       engine->name);
648 			i915_request_put(rq);
649 			err = -EIO;
650 			goto out;
651 		}
652 		GEM_BUG_ON(!i915_request_on_hold(rq));
653 
654 		/* But is resubmitted on release */
655 		execlists_unhold(engine, rq);
656 		if (i915_request_wait(rq, 0, HZ / 5) < 0) {
657 			pr_err("%s: held request did not complete!\n",
658 			       engine->name);
659 			intel_gt_set_wedged(gt);
660 			err = -ETIME;
661 		}
662 		i915_request_put(rq);
663 
664 out:
665 		st_engine_heartbeat_enable(engine);
666 		intel_context_put(ce);
667 		if (err)
668 			break;
669 	}
670 
671 	igt_spinner_fini(&spin);
672 	return err;
673 }
674 
675 static const char *error_repr(int err)
676 {
677 	return err ? "bad" : "good";
678 }
679 
680 static int live_error_interrupt(void *arg)
681 {
682 	static const struct error_phase {
683 		enum { GOOD = 0, BAD = -EIO } error[2];
684 	} phases[] = {
685 		{ { BAD,  GOOD } },
686 		{ { BAD,  BAD  } },
687 		{ { BAD,  GOOD } },
688 		{ { GOOD, GOOD } }, /* sentinel */
689 	};
690 	struct intel_gt *gt = arg;
691 	struct intel_engine_cs *engine;
692 	enum intel_engine_id id;
693 
694 	/*
695 	 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning
696 	 * of invalid commands in user batches that will cause a GPU hang.
697 	 * This is a faster mechanism than using hangcheck/heartbeats, but
698 	 * only detects problems the HW knows about -- it will not warn when
699 	 * we kill the HW!
700 	 *
701 	 * To verify our detection and reset, we throw some invalid commands
702 	 * at the HW and wait for the interrupt.
703 	 */
704 
705 	if (!intel_has_reset_engine(gt))
706 		return 0;
707 
708 	for_each_engine(engine, gt, id) {
709 		const struct error_phase *p;
710 		int err = 0;
711 
712 		st_engine_heartbeat_disable(engine);
713 
714 		for (p = phases; p->error[0] != GOOD; p++) {
715 			struct i915_request *client[ARRAY_SIZE(phases->error)];
716 			u32 *cs;
717 			int i;
718 
719 			memset(client, 0, sizeof(*client));
720 			for (i = 0; i < ARRAY_SIZE(client); i++) {
721 				struct intel_context *ce;
722 				struct i915_request *rq;
723 
724 				ce = intel_context_create(engine);
725 				if (IS_ERR(ce)) {
726 					err = PTR_ERR(ce);
727 					goto out;
728 				}
729 
730 				rq = intel_context_create_request(ce);
731 				intel_context_put(ce);
732 				if (IS_ERR(rq)) {
733 					err = PTR_ERR(rq);
734 					goto out;
735 				}
736 
737 				if (rq->engine->emit_init_breadcrumb) {
738 					err = rq->engine->emit_init_breadcrumb(rq);
739 					if (err) {
740 						i915_request_add(rq);
741 						goto out;
742 					}
743 				}
744 
745 				cs = intel_ring_begin(rq, 2);
746 				if (IS_ERR(cs)) {
747 					i915_request_add(rq);
748 					err = PTR_ERR(cs);
749 					goto out;
750 				}
751 
752 				if (p->error[i]) {
753 					*cs++ = 0xdeadbeef;
754 					*cs++ = 0xdeadbeef;
755 				} else {
756 					*cs++ = MI_NOOP;
757 					*cs++ = MI_NOOP;
758 				}
759 
760 				client[i] = i915_request_get(rq);
761 				i915_request_add(rq);
762 			}
763 
764 			err = wait_for_submit(engine, client[0], HZ / 2);
765 			if (err) {
766 				pr_err("%s: first request did not start within time!\n",
767 				       engine->name);
768 				err = -ETIME;
769 				goto out;
770 			}
771 
772 			for (i = 0; i < ARRAY_SIZE(client); i++) {
773 				if (i915_request_wait(client[i], 0, HZ / 5) < 0)
774 					pr_debug("%s: %s request incomplete!\n",
775 						 engine->name,
776 						 error_repr(p->error[i]));
777 
778 				if (!i915_request_started(client[i])) {
779 					pr_err("%s: %s request not started!\n",
780 					       engine->name,
781 					       error_repr(p->error[i]));
782 					err = -ETIME;
783 					goto out;
784 				}
785 
786 				/* Kick the tasklet to process the error */
787 				intel_engine_flush_submission(engine);
788 				if (client[i]->fence.error != p->error[i]) {
789 					pr_err("%s: %s request (%s) with wrong error code: %d\n",
790 					       engine->name,
791 					       error_repr(p->error[i]),
792 					       i915_request_completed(client[i]) ? "completed" : "running",
793 					       client[i]->fence.error);
794 					err = -EINVAL;
795 					goto out;
796 				}
797 			}
798 
799 out:
800 			for (i = 0; i < ARRAY_SIZE(client); i++)
801 				if (client[i])
802 					i915_request_put(client[i]);
803 			if (err) {
804 				pr_err("%s: failed at phase[%zd] { %d, %d }\n",
805 				       engine->name, p - phases,
806 				       p->error[0], p->error[1]);
807 				break;
808 			}
809 		}
810 
811 		st_engine_heartbeat_enable(engine);
812 		if (err) {
813 			intel_gt_set_wedged(gt);
814 			return err;
815 		}
816 	}
817 
818 	return 0;
819 }
820 
821 static int
822 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
823 {
824 	u32 *cs;
825 
826 	cs = intel_ring_begin(rq, 10);
827 	if (IS_ERR(cs))
828 		return PTR_ERR(cs);
829 
830 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
831 
832 	*cs++ = MI_SEMAPHORE_WAIT |
833 		MI_SEMAPHORE_GLOBAL_GTT |
834 		MI_SEMAPHORE_POLL |
835 		MI_SEMAPHORE_SAD_NEQ_SDD;
836 	*cs++ = 0;
837 	*cs++ = i915_ggtt_offset(vma) + 4 * idx;
838 	*cs++ = 0;
839 
840 	if (idx > 0) {
841 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
842 		*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
843 		*cs++ = 0;
844 		*cs++ = 1;
845 	} else {
846 		*cs++ = MI_NOOP;
847 		*cs++ = MI_NOOP;
848 		*cs++ = MI_NOOP;
849 		*cs++ = MI_NOOP;
850 	}
851 
852 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
853 
854 	intel_ring_advance(rq, cs);
855 	return 0;
856 }
857 
858 static struct i915_request *
859 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
860 {
861 	struct intel_context *ce;
862 	struct i915_request *rq;
863 	int err;
864 
865 	ce = intel_context_create(engine);
866 	if (IS_ERR(ce))
867 		return ERR_CAST(ce);
868 
869 	rq = intel_context_create_request(ce);
870 	if (IS_ERR(rq))
871 		goto out_ce;
872 
873 	err = 0;
874 	if (rq->engine->emit_init_breadcrumb)
875 		err = rq->engine->emit_init_breadcrumb(rq);
876 	if (err == 0)
877 		err = emit_semaphore_chain(rq, vma, idx);
878 	if (err == 0)
879 		i915_request_get(rq);
880 	i915_request_add(rq);
881 	if (err)
882 		rq = ERR_PTR(err);
883 
884 out_ce:
885 	intel_context_put(ce);
886 	return rq;
887 }
888 
889 static int
890 release_queue(struct intel_engine_cs *engine,
891 	      struct i915_vma *vma,
892 	      int idx, int prio)
893 {
894 	struct i915_sched_attr attr = {
895 		.priority = prio,
896 	};
897 	struct i915_request *rq;
898 	u32 *cs;
899 
900 	rq = intel_engine_create_kernel_request(engine);
901 	if (IS_ERR(rq))
902 		return PTR_ERR(rq);
903 
904 	cs = intel_ring_begin(rq, 4);
905 	if (IS_ERR(cs)) {
906 		i915_request_add(rq);
907 		return PTR_ERR(cs);
908 	}
909 
910 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
911 	*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
912 	*cs++ = 0;
913 	*cs++ = 1;
914 
915 	intel_ring_advance(rq, cs);
916 
917 	i915_request_get(rq);
918 	i915_request_add(rq);
919 
920 	local_bh_disable();
921 	engine->sched_engine->schedule(rq, &attr);
922 	local_bh_enable(); /* kick tasklet */
923 
924 	i915_request_put(rq);
925 
926 	return 0;
927 }
928 
929 static int
930 slice_semaphore_queue(struct intel_engine_cs *outer,
931 		      struct i915_vma *vma,
932 		      int count)
933 {
934 	struct intel_engine_cs *engine;
935 	struct i915_request *head;
936 	enum intel_engine_id id;
937 	int err, i, n = 0;
938 
939 	head = semaphore_queue(outer, vma, n++);
940 	if (IS_ERR(head))
941 		return PTR_ERR(head);
942 
943 	for_each_engine(engine, outer->gt, id) {
944 		if (!intel_engine_has_preemption(engine))
945 			continue;
946 
947 		for (i = 0; i < count; i++) {
948 			struct i915_request *rq;
949 
950 			rq = semaphore_queue(engine, vma, n++);
951 			if (IS_ERR(rq)) {
952 				err = PTR_ERR(rq);
953 				goto out;
954 			}
955 
956 			i915_request_put(rq);
957 		}
958 	}
959 
960 	err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
961 	if (err)
962 		goto out;
963 
964 	if (i915_request_wait(head, 0,
965 			      2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) {
966 		pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n",
967 		       outer->name, count, n);
968 		GEM_TRACE_DUMP();
969 		intel_gt_set_wedged(outer->gt);
970 		err = -EIO;
971 	}
972 
973 out:
974 	i915_request_put(head);
975 	return err;
976 }
977 
978 static int live_timeslice_preempt(void *arg)
979 {
980 	struct intel_gt *gt = arg;
981 	struct drm_i915_gem_object *obj;
982 	struct intel_engine_cs *engine;
983 	enum intel_engine_id id;
984 	struct i915_vma *vma;
985 	void *vaddr;
986 	int err = 0;
987 
988 	/*
989 	 * If a request takes too long, we would like to give other users
990 	 * a fair go on the GPU. In particular, users may create batches
991 	 * that wait upon external input, where that input may even be
992 	 * supplied by another GPU job. To avoid blocking forever, we
993 	 * need to preempt the current task and replace it with another
994 	 * ready task.
995 	 */
996 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
997 		return 0;
998 
999 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1000 	if (IS_ERR(obj))
1001 		return PTR_ERR(obj);
1002 
1003 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1004 	if (IS_ERR(vma)) {
1005 		err = PTR_ERR(vma);
1006 		goto err_obj;
1007 	}
1008 
1009 	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1010 	if (IS_ERR(vaddr)) {
1011 		err = PTR_ERR(vaddr);
1012 		goto err_obj;
1013 	}
1014 
1015 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1016 	if (err)
1017 		goto err_map;
1018 
1019 	err = i915_vma_sync(vma);
1020 	if (err)
1021 		goto err_pin;
1022 
1023 	for_each_engine(engine, gt, id) {
1024 		if (!intel_engine_has_preemption(engine))
1025 			continue;
1026 
1027 		memset(vaddr, 0, PAGE_SIZE);
1028 
1029 		st_engine_heartbeat_disable(engine);
1030 		err = slice_semaphore_queue(engine, vma, 5);
1031 		st_engine_heartbeat_enable(engine);
1032 		if (err)
1033 			goto err_pin;
1034 
1035 		if (igt_flush_test(gt->i915)) {
1036 			err = -EIO;
1037 			goto err_pin;
1038 		}
1039 	}
1040 
1041 err_pin:
1042 	i915_vma_unpin(vma);
1043 err_map:
1044 	i915_gem_object_unpin_map(obj);
1045 err_obj:
1046 	i915_gem_object_put(obj);
1047 	return err;
1048 }
1049 
1050 static struct i915_request *
1051 create_rewinder(struct intel_context *ce,
1052 		struct i915_request *wait,
1053 		void *slot, int idx)
1054 {
1055 	const u32 offset =
1056 		i915_ggtt_offset(ce->engine->status_page.vma) +
1057 		offset_in_page(slot);
1058 	struct i915_request *rq;
1059 	u32 *cs;
1060 	int err;
1061 
1062 	rq = intel_context_create_request(ce);
1063 	if (IS_ERR(rq))
1064 		return rq;
1065 
1066 	if (wait) {
1067 		err = i915_request_await_dma_fence(rq, &wait->fence);
1068 		if (err)
1069 			goto err;
1070 	}
1071 
1072 	cs = intel_ring_begin(rq, 14);
1073 	if (IS_ERR(cs)) {
1074 		err = PTR_ERR(cs);
1075 		goto err;
1076 	}
1077 
1078 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1079 	*cs++ = MI_NOOP;
1080 
1081 	*cs++ = MI_SEMAPHORE_WAIT |
1082 		MI_SEMAPHORE_GLOBAL_GTT |
1083 		MI_SEMAPHORE_POLL |
1084 		MI_SEMAPHORE_SAD_GTE_SDD;
1085 	*cs++ = idx;
1086 	*cs++ = offset;
1087 	*cs++ = 0;
1088 
1089 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1090 	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
1091 	*cs++ = offset + idx * sizeof(u32);
1092 	*cs++ = 0;
1093 
1094 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1095 	*cs++ = offset;
1096 	*cs++ = 0;
1097 	*cs++ = idx + 1;
1098 
1099 	intel_ring_advance(rq, cs);
1100 
1101 	err = 0;
1102 err:
1103 	i915_request_get(rq);
1104 	i915_request_add(rq);
1105 	if (err) {
1106 		i915_request_put(rq);
1107 		return ERR_PTR(err);
1108 	}
1109 
1110 	return rq;
1111 }
1112 
1113 static int live_timeslice_rewind(void *arg)
1114 {
1115 	struct intel_gt *gt = arg;
1116 	struct intel_engine_cs *engine;
1117 	enum intel_engine_id id;
1118 
1119 	/*
1120 	 * The usual presumption on timeslice expiration is that we replace
1121 	 * the active context with another. However, given a chain of
1122 	 * dependencies we may end up with replacing the context with itself,
1123 	 * but only a few of those requests, forcing us to rewind the
1124 	 * RING_TAIL of the original request.
1125 	 */
1126 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1127 		return 0;
1128 
1129 	for_each_engine(engine, gt, id) {
1130 		enum { A1, A2, B1 };
1131 		enum { X = 1, Z, Y };
1132 		struct i915_request *rq[3] = {};
1133 		struct intel_context *ce;
1134 		unsigned long timeslice;
1135 		int i, err = 0;
1136 		u32 *slot;
1137 
1138 		if (!intel_engine_has_timeslices(engine))
1139 			continue;
1140 
1141 		/*
1142 		 * A:rq1 -- semaphore wait, timestamp X
1143 		 * A:rq2 -- write timestamp Y
1144 		 *
1145 		 * B:rq1 [await A:rq1] -- write timestamp Z
1146 		 *
1147 		 * Force timeslice, release semaphore.
1148 		 *
1149 		 * Expect execution/evaluation order XZY
1150 		 */
1151 
1152 		st_engine_heartbeat_disable(engine);
1153 		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1154 
1155 		slot = memset32(engine->status_page.addr + 1000, 0, 4);
1156 
1157 		ce = intel_context_create(engine);
1158 		if (IS_ERR(ce)) {
1159 			err = PTR_ERR(ce);
1160 			goto err;
1161 		}
1162 
1163 		rq[A1] = create_rewinder(ce, NULL, slot, X);
1164 		if (IS_ERR(rq[A1])) {
1165 			intel_context_put(ce);
1166 			goto err;
1167 		}
1168 
1169 		rq[A2] = create_rewinder(ce, NULL, slot, Y);
1170 		intel_context_put(ce);
1171 		if (IS_ERR(rq[A2]))
1172 			goto err;
1173 
1174 		err = wait_for_submit(engine, rq[A2], HZ / 2);
1175 		if (err) {
1176 			pr_err("%s: failed to submit first context\n",
1177 			       engine->name);
1178 			goto err;
1179 		}
1180 
1181 		ce = intel_context_create(engine);
1182 		if (IS_ERR(ce)) {
1183 			err = PTR_ERR(ce);
1184 			goto err;
1185 		}
1186 
1187 		rq[B1] = create_rewinder(ce, rq[A1], slot, Z);
1188 		intel_context_put(ce);
1189 		if (IS_ERR(rq[2]))
1190 			goto err;
1191 
1192 		err = wait_for_submit(engine, rq[B1], HZ / 2);
1193 		if (err) {
1194 			pr_err("%s: failed to submit second context\n",
1195 			       engine->name);
1196 			goto err;
1197 		}
1198 
1199 		/* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */
1200 		ENGINE_TRACE(engine, "forcing tasklet for rewind\n");
1201 		while (i915_request_is_active(rq[A2])) { /* semaphore yield! */
1202 			/* Wait for the timeslice to kick in */
1203 			del_timer(&engine->execlists.timer);
1204 			tasklet_hi_schedule(&engine->sched_engine->tasklet);
1205 			intel_engine_flush_submission(engine);
1206 		}
1207 		/* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */
1208 		GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1209 		GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1210 		GEM_BUG_ON(i915_request_is_active(rq[A2]));
1211 
1212 		/* Release the hounds! */
1213 		slot[0] = 1;
1214 		wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */
1215 
1216 		for (i = 1; i <= 3; i++) {
1217 			unsigned long timeout = jiffies + HZ / 2;
1218 
1219 			while (!READ_ONCE(slot[i]) &&
1220 			       time_before(jiffies, timeout))
1221 				;
1222 
1223 			if (!time_before(jiffies, timeout)) {
1224 				pr_err("%s: rq[%d] timed out\n",
1225 				       engine->name, i - 1);
1226 				err = -ETIME;
1227 				goto err;
1228 			}
1229 
1230 			pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1231 		}
1232 
1233 		/* XZY: XZ < XY */
1234 		if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1235 			pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1236 			       engine->name,
1237 			       slot[Z] - slot[X],
1238 			       slot[Y] - slot[X]);
1239 			err = -EINVAL;
1240 		}
1241 
1242 err:
1243 		memset32(&slot[0], -1, 4);
1244 		wmb();
1245 
1246 		engine->props.timeslice_duration_ms = timeslice;
1247 		st_engine_heartbeat_enable(engine);
1248 		for (i = 0; i < 3; i++)
1249 			i915_request_put(rq[i]);
1250 		if (igt_flush_test(gt->i915))
1251 			err = -EIO;
1252 		if (err)
1253 			return err;
1254 	}
1255 
1256 	return 0;
1257 }
1258 
1259 static struct i915_request *nop_request(struct intel_engine_cs *engine)
1260 {
1261 	struct i915_request *rq;
1262 
1263 	rq = intel_engine_create_kernel_request(engine);
1264 	if (IS_ERR(rq))
1265 		return rq;
1266 
1267 	i915_request_get(rq);
1268 	i915_request_add(rq);
1269 
1270 	return rq;
1271 }
1272 
1273 static long slice_timeout(struct intel_engine_cs *engine)
1274 {
1275 	long timeout;
1276 
1277 	/* Enough time for a timeslice to kick in, and kick out */
1278 	timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine));
1279 
1280 	/* Enough time for the nop request to complete */
1281 	timeout += HZ / 5;
1282 
1283 	return timeout + 1;
1284 }
1285 
1286 static int live_timeslice_queue(void *arg)
1287 {
1288 	struct intel_gt *gt = arg;
1289 	struct drm_i915_gem_object *obj;
1290 	struct intel_engine_cs *engine;
1291 	enum intel_engine_id id;
1292 	struct i915_vma *vma;
1293 	void *vaddr;
1294 	int err = 0;
1295 
1296 	/*
1297 	 * Make sure that even if ELSP[0] and ELSP[1] are filled with
1298 	 * timeslicing between them disabled, we *do* enable timeslicing
1299 	 * if the queue demands it. (Normally, we do not submit if
1300 	 * ELSP[1] is already occupied, so must rely on timeslicing to
1301 	 * eject ELSP[0] in favour of the queue.)
1302 	 */
1303 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1304 		return 0;
1305 
1306 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1307 	if (IS_ERR(obj))
1308 		return PTR_ERR(obj);
1309 
1310 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1311 	if (IS_ERR(vma)) {
1312 		err = PTR_ERR(vma);
1313 		goto err_obj;
1314 	}
1315 
1316 	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1317 	if (IS_ERR(vaddr)) {
1318 		err = PTR_ERR(vaddr);
1319 		goto err_obj;
1320 	}
1321 
1322 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1323 	if (err)
1324 		goto err_map;
1325 
1326 	err = i915_vma_sync(vma);
1327 	if (err)
1328 		goto err_pin;
1329 
1330 	for_each_engine(engine, gt, id) {
1331 		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
1332 		struct i915_request *rq, *nop;
1333 
1334 		if (!intel_engine_has_preemption(engine))
1335 			continue;
1336 
1337 		st_engine_heartbeat_disable(engine);
1338 		memset(vaddr, 0, PAGE_SIZE);
1339 
1340 		/* ELSP[0]: semaphore wait */
1341 		rq = semaphore_queue(engine, vma, 0);
1342 		if (IS_ERR(rq)) {
1343 			err = PTR_ERR(rq);
1344 			goto err_heartbeat;
1345 		}
1346 		engine->sched_engine->schedule(rq, &attr);
1347 		err = wait_for_submit(engine, rq, HZ / 2);
1348 		if (err) {
1349 			pr_err("%s: Timed out trying to submit semaphores\n",
1350 			       engine->name);
1351 			goto err_rq;
1352 		}
1353 
1354 		/* ELSP[1]: nop request */
1355 		nop = nop_request(engine);
1356 		if (IS_ERR(nop)) {
1357 			err = PTR_ERR(nop);
1358 			goto err_rq;
1359 		}
1360 		err = wait_for_submit(engine, nop, HZ / 2);
1361 		i915_request_put(nop);
1362 		if (err) {
1363 			pr_err("%s: Timed out trying to submit nop\n",
1364 			       engine->name);
1365 			goto err_rq;
1366 		}
1367 
1368 		GEM_BUG_ON(i915_request_completed(rq));
1369 		GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1370 
1371 		/* Queue: semaphore signal, matching priority as semaphore */
1372 		err = release_queue(engine, vma, 1, effective_prio(rq));
1373 		if (err)
1374 			goto err_rq;
1375 
1376 		/* Wait until we ack the release_queue and start timeslicing */
1377 		do {
1378 			cond_resched();
1379 			intel_engine_flush_submission(engine);
1380 		} while (READ_ONCE(engine->execlists.pending[0]));
1381 
1382 		/* Timeslice every jiffy, so within 2 we should signal */
1383 		if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) {
1384 			struct drm_printer p =
1385 				drm_info_printer(gt->i915->drm.dev);
1386 
1387 			pr_err("%s: Failed to timeslice into queue\n",
1388 			       engine->name);
1389 			intel_engine_dump(engine, &p,
1390 					  "%s\n", engine->name);
1391 
1392 			memset(vaddr, 0xff, PAGE_SIZE);
1393 			err = -EIO;
1394 		}
1395 err_rq:
1396 		i915_request_put(rq);
1397 err_heartbeat:
1398 		st_engine_heartbeat_enable(engine);
1399 		if (err)
1400 			break;
1401 	}
1402 
1403 err_pin:
1404 	i915_vma_unpin(vma);
1405 err_map:
1406 	i915_gem_object_unpin_map(obj);
1407 err_obj:
1408 	i915_gem_object_put(obj);
1409 	return err;
1410 }
1411 
1412 static int live_timeslice_nopreempt(void *arg)
1413 {
1414 	struct intel_gt *gt = arg;
1415 	struct intel_engine_cs *engine;
1416 	enum intel_engine_id id;
1417 	struct igt_spinner spin;
1418 	int err = 0;
1419 
1420 	/*
1421 	 * We should not timeslice into a request that is marked with
1422 	 * I915_REQUEST_NOPREEMPT.
1423 	 */
1424 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1425 		return 0;
1426 
1427 	if (igt_spinner_init(&spin, gt))
1428 		return -ENOMEM;
1429 
1430 	for_each_engine(engine, gt, id) {
1431 		struct intel_context *ce;
1432 		struct i915_request *rq;
1433 		unsigned long timeslice;
1434 
1435 		if (!intel_engine_has_preemption(engine))
1436 			continue;
1437 
1438 		ce = intel_context_create(engine);
1439 		if (IS_ERR(ce)) {
1440 			err = PTR_ERR(ce);
1441 			break;
1442 		}
1443 
1444 		st_engine_heartbeat_disable(engine);
1445 		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1446 
1447 		/* Create an unpreemptible spinner */
1448 
1449 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
1450 		intel_context_put(ce);
1451 		if (IS_ERR(rq)) {
1452 			err = PTR_ERR(rq);
1453 			goto out_heartbeat;
1454 		}
1455 
1456 		i915_request_get(rq);
1457 		i915_request_add(rq);
1458 
1459 		if (!igt_wait_for_spinner(&spin, rq)) {
1460 			i915_request_put(rq);
1461 			err = -ETIME;
1462 			goto out_spin;
1463 		}
1464 
1465 		set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
1466 		i915_request_put(rq);
1467 
1468 		/* Followed by a maximum priority barrier (heartbeat) */
1469 
1470 		ce = intel_context_create(engine);
1471 		if (IS_ERR(ce)) {
1472 			err = PTR_ERR(ce);
1473 			goto out_spin;
1474 		}
1475 
1476 		rq = intel_context_create_request(ce);
1477 		intel_context_put(ce);
1478 		if (IS_ERR(rq)) {
1479 			err = PTR_ERR(rq);
1480 			goto out_spin;
1481 		}
1482 
1483 		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1484 		i915_request_get(rq);
1485 		i915_request_add(rq);
1486 
1487 		/*
1488 		 * Wait until the barrier is in ELSP, and we know timeslicing
1489 		 * will have been activated.
1490 		 */
1491 		if (wait_for_submit(engine, rq, HZ / 2)) {
1492 			i915_request_put(rq);
1493 			err = -ETIME;
1494 			goto out_spin;
1495 		}
1496 
1497 		/*
1498 		 * Since the ELSP[0] request is unpreemptible, it should not
1499 		 * allow the maximum priority barrier through. Wait long
1500 		 * enough to see if it is timesliced in by mistake.
1501 		 */
1502 		if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) {
1503 			pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",
1504 			       engine->name);
1505 			err = -EINVAL;
1506 		}
1507 		i915_request_put(rq);
1508 
1509 out_spin:
1510 		igt_spinner_end(&spin);
1511 out_heartbeat:
1512 		xchg(&engine->props.timeslice_duration_ms, timeslice);
1513 		st_engine_heartbeat_enable(engine);
1514 		if (err)
1515 			break;
1516 
1517 		if (igt_flush_test(gt->i915)) {
1518 			err = -EIO;
1519 			break;
1520 		}
1521 	}
1522 
1523 	igt_spinner_fini(&spin);
1524 	return err;
1525 }
1526 
1527 static int live_busywait_preempt(void *arg)
1528 {
1529 	struct intel_gt *gt = arg;
1530 	struct i915_gem_context *ctx_hi, *ctx_lo;
1531 	struct intel_engine_cs *engine;
1532 	struct drm_i915_gem_object *obj;
1533 	struct i915_vma *vma;
1534 	enum intel_engine_id id;
1535 	int err = -ENOMEM;
1536 	u32 *map;
1537 
1538 	/*
1539 	 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
1540 	 * preempt the busywaits used to synchronise between rings.
1541 	 */
1542 
1543 	ctx_hi = kernel_context(gt->i915, NULL);
1544 	if (!ctx_hi)
1545 		return -ENOMEM;
1546 	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1547 
1548 	ctx_lo = kernel_context(gt->i915, NULL);
1549 	if (!ctx_lo)
1550 		goto err_ctx_hi;
1551 	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1552 
1553 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1554 	if (IS_ERR(obj)) {
1555 		err = PTR_ERR(obj);
1556 		goto err_ctx_lo;
1557 	}
1558 
1559 	map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1560 	if (IS_ERR(map)) {
1561 		err = PTR_ERR(map);
1562 		goto err_obj;
1563 	}
1564 
1565 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1566 	if (IS_ERR(vma)) {
1567 		err = PTR_ERR(vma);
1568 		goto err_map;
1569 	}
1570 
1571 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1572 	if (err)
1573 		goto err_map;
1574 
1575 	err = i915_vma_sync(vma);
1576 	if (err)
1577 		goto err_vma;
1578 
1579 	for_each_engine(engine, gt, id) {
1580 		struct i915_request *lo, *hi;
1581 		struct igt_live_test t;
1582 		u32 *cs;
1583 
1584 		if (!intel_engine_has_preemption(engine))
1585 			continue;
1586 
1587 		if (!intel_engine_can_store_dword(engine))
1588 			continue;
1589 
1590 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1591 			err = -EIO;
1592 			goto err_vma;
1593 		}
1594 
1595 		/*
1596 		 * We create two requests. The low priority request
1597 		 * busywaits on a semaphore (inside the ringbuffer where
1598 		 * is should be preemptible) and the high priority requests
1599 		 * uses a MI_STORE_DWORD_IMM to update the semaphore value
1600 		 * allowing the first request to complete. If preemption
1601 		 * fails, we hang instead.
1602 		 */
1603 
1604 		lo = igt_request_alloc(ctx_lo, engine);
1605 		if (IS_ERR(lo)) {
1606 			err = PTR_ERR(lo);
1607 			goto err_vma;
1608 		}
1609 
1610 		cs = intel_ring_begin(lo, 8);
1611 		if (IS_ERR(cs)) {
1612 			err = PTR_ERR(cs);
1613 			i915_request_add(lo);
1614 			goto err_vma;
1615 		}
1616 
1617 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1618 		*cs++ = i915_ggtt_offset(vma);
1619 		*cs++ = 0;
1620 		*cs++ = 1;
1621 
1622 		/* XXX Do we need a flush + invalidate here? */
1623 
1624 		*cs++ = MI_SEMAPHORE_WAIT |
1625 			MI_SEMAPHORE_GLOBAL_GTT |
1626 			MI_SEMAPHORE_POLL |
1627 			MI_SEMAPHORE_SAD_EQ_SDD;
1628 		*cs++ = 0;
1629 		*cs++ = i915_ggtt_offset(vma);
1630 		*cs++ = 0;
1631 
1632 		intel_ring_advance(lo, cs);
1633 
1634 		i915_request_get(lo);
1635 		i915_request_add(lo);
1636 
1637 		if (wait_for(READ_ONCE(*map), 10)) {
1638 			i915_request_put(lo);
1639 			err = -ETIMEDOUT;
1640 			goto err_vma;
1641 		}
1642 
1643 		/* Low priority request should be busywaiting now */
1644 		if (i915_request_wait(lo, 0, 1) != -ETIME) {
1645 			i915_request_put(lo);
1646 			pr_err("%s: Busywaiting request did not!\n",
1647 			       engine->name);
1648 			err = -EIO;
1649 			goto err_vma;
1650 		}
1651 
1652 		hi = igt_request_alloc(ctx_hi, engine);
1653 		if (IS_ERR(hi)) {
1654 			err = PTR_ERR(hi);
1655 			i915_request_put(lo);
1656 			goto err_vma;
1657 		}
1658 
1659 		cs = intel_ring_begin(hi, 4);
1660 		if (IS_ERR(cs)) {
1661 			err = PTR_ERR(cs);
1662 			i915_request_add(hi);
1663 			i915_request_put(lo);
1664 			goto err_vma;
1665 		}
1666 
1667 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1668 		*cs++ = i915_ggtt_offset(vma);
1669 		*cs++ = 0;
1670 		*cs++ = 0;
1671 
1672 		intel_ring_advance(hi, cs);
1673 		i915_request_add(hi);
1674 
1675 		if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1676 			struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1677 
1678 			pr_err("%s: Failed to preempt semaphore busywait!\n",
1679 			       engine->name);
1680 
1681 			intel_engine_dump(engine, &p, "%s\n", engine->name);
1682 			GEM_TRACE_DUMP();
1683 
1684 			i915_request_put(lo);
1685 			intel_gt_set_wedged(gt);
1686 			err = -EIO;
1687 			goto err_vma;
1688 		}
1689 		GEM_BUG_ON(READ_ONCE(*map));
1690 		i915_request_put(lo);
1691 
1692 		if (igt_live_test_end(&t)) {
1693 			err = -EIO;
1694 			goto err_vma;
1695 		}
1696 	}
1697 
1698 	err = 0;
1699 err_vma:
1700 	i915_vma_unpin(vma);
1701 err_map:
1702 	i915_gem_object_unpin_map(obj);
1703 err_obj:
1704 	i915_gem_object_put(obj);
1705 err_ctx_lo:
1706 	kernel_context_close(ctx_lo);
1707 err_ctx_hi:
1708 	kernel_context_close(ctx_hi);
1709 	return err;
1710 }
1711 
1712 static struct i915_request *
1713 spinner_create_request(struct igt_spinner *spin,
1714 		       struct i915_gem_context *ctx,
1715 		       struct intel_engine_cs *engine,
1716 		       u32 arb)
1717 {
1718 	struct intel_context *ce;
1719 	struct i915_request *rq;
1720 
1721 	ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
1722 	if (IS_ERR(ce))
1723 		return ERR_CAST(ce);
1724 
1725 	rq = igt_spinner_create_request(spin, ce, arb);
1726 	intel_context_put(ce);
1727 	return rq;
1728 }
1729 
1730 static int live_preempt(void *arg)
1731 {
1732 	struct intel_gt *gt = arg;
1733 	struct i915_gem_context *ctx_hi, *ctx_lo;
1734 	struct igt_spinner spin_hi, spin_lo;
1735 	struct intel_engine_cs *engine;
1736 	enum intel_engine_id id;
1737 	int err = -ENOMEM;
1738 
1739 	ctx_hi = kernel_context(gt->i915, NULL);
1740 	if (!ctx_hi)
1741 		return -ENOMEM;
1742 	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1743 
1744 	ctx_lo = kernel_context(gt->i915, NULL);
1745 	if (!ctx_lo)
1746 		goto err_ctx_hi;
1747 	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1748 
1749 	if (igt_spinner_init(&spin_hi, gt))
1750 		goto err_ctx_lo;
1751 
1752 	if (igt_spinner_init(&spin_lo, gt))
1753 		goto err_spin_hi;
1754 
1755 	for_each_engine(engine, gt, id) {
1756 		struct igt_live_test t;
1757 		struct i915_request *rq;
1758 
1759 		if (!intel_engine_has_preemption(engine))
1760 			continue;
1761 
1762 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1763 			err = -EIO;
1764 			goto err_spin_lo;
1765 		}
1766 
1767 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1768 					    MI_ARB_CHECK);
1769 		if (IS_ERR(rq)) {
1770 			err = PTR_ERR(rq);
1771 			goto err_spin_lo;
1772 		}
1773 
1774 		i915_request_add(rq);
1775 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
1776 			GEM_TRACE("lo spinner failed to start\n");
1777 			GEM_TRACE_DUMP();
1778 			intel_gt_set_wedged(gt);
1779 			err = -EIO;
1780 			goto err_spin_lo;
1781 		}
1782 
1783 		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1784 					    MI_ARB_CHECK);
1785 		if (IS_ERR(rq)) {
1786 			igt_spinner_end(&spin_lo);
1787 			err = PTR_ERR(rq);
1788 			goto err_spin_lo;
1789 		}
1790 
1791 		i915_request_add(rq);
1792 		if (!igt_wait_for_spinner(&spin_hi, rq)) {
1793 			GEM_TRACE("hi spinner failed to start\n");
1794 			GEM_TRACE_DUMP();
1795 			intel_gt_set_wedged(gt);
1796 			err = -EIO;
1797 			goto err_spin_lo;
1798 		}
1799 
1800 		igt_spinner_end(&spin_hi);
1801 		igt_spinner_end(&spin_lo);
1802 
1803 		if (igt_live_test_end(&t)) {
1804 			err = -EIO;
1805 			goto err_spin_lo;
1806 		}
1807 	}
1808 
1809 	err = 0;
1810 err_spin_lo:
1811 	igt_spinner_fini(&spin_lo);
1812 err_spin_hi:
1813 	igt_spinner_fini(&spin_hi);
1814 err_ctx_lo:
1815 	kernel_context_close(ctx_lo);
1816 err_ctx_hi:
1817 	kernel_context_close(ctx_hi);
1818 	return err;
1819 }
1820 
1821 static int live_late_preempt(void *arg)
1822 {
1823 	struct intel_gt *gt = arg;
1824 	struct i915_gem_context *ctx_hi, *ctx_lo;
1825 	struct igt_spinner spin_hi, spin_lo;
1826 	struct intel_engine_cs *engine;
1827 	struct i915_sched_attr attr = {};
1828 	enum intel_engine_id id;
1829 	int err = -ENOMEM;
1830 
1831 	ctx_hi = kernel_context(gt->i915, NULL);
1832 	if (!ctx_hi)
1833 		return -ENOMEM;
1834 
1835 	ctx_lo = kernel_context(gt->i915, NULL);
1836 	if (!ctx_lo)
1837 		goto err_ctx_hi;
1838 
1839 	if (igt_spinner_init(&spin_hi, gt))
1840 		goto err_ctx_lo;
1841 
1842 	if (igt_spinner_init(&spin_lo, gt))
1843 		goto err_spin_hi;
1844 
1845 	/* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1846 	ctx_lo->sched.priority = 1;
1847 
1848 	for_each_engine(engine, gt, id) {
1849 		struct igt_live_test t;
1850 		struct i915_request *rq;
1851 
1852 		if (!intel_engine_has_preemption(engine))
1853 			continue;
1854 
1855 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1856 			err = -EIO;
1857 			goto err_spin_lo;
1858 		}
1859 
1860 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1861 					    MI_ARB_CHECK);
1862 		if (IS_ERR(rq)) {
1863 			err = PTR_ERR(rq);
1864 			goto err_spin_lo;
1865 		}
1866 
1867 		i915_request_add(rq);
1868 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
1869 			pr_err("First context failed to start\n");
1870 			goto err_wedged;
1871 		}
1872 
1873 		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1874 					    MI_NOOP);
1875 		if (IS_ERR(rq)) {
1876 			igt_spinner_end(&spin_lo);
1877 			err = PTR_ERR(rq);
1878 			goto err_spin_lo;
1879 		}
1880 
1881 		i915_request_add(rq);
1882 		if (igt_wait_for_spinner(&spin_hi, rq)) {
1883 			pr_err("Second context overtook first?\n");
1884 			goto err_wedged;
1885 		}
1886 
1887 		attr.priority = I915_PRIORITY_MAX;
1888 		engine->sched_engine->schedule(rq, &attr);
1889 
1890 		if (!igt_wait_for_spinner(&spin_hi, rq)) {
1891 			pr_err("High priority context failed to preempt the low priority context\n");
1892 			GEM_TRACE_DUMP();
1893 			goto err_wedged;
1894 		}
1895 
1896 		igt_spinner_end(&spin_hi);
1897 		igt_spinner_end(&spin_lo);
1898 
1899 		if (igt_live_test_end(&t)) {
1900 			err = -EIO;
1901 			goto err_spin_lo;
1902 		}
1903 	}
1904 
1905 	err = 0;
1906 err_spin_lo:
1907 	igt_spinner_fini(&spin_lo);
1908 err_spin_hi:
1909 	igt_spinner_fini(&spin_hi);
1910 err_ctx_lo:
1911 	kernel_context_close(ctx_lo);
1912 err_ctx_hi:
1913 	kernel_context_close(ctx_hi);
1914 	return err;
1915 
1916 err_wedged:
1917 	igt_spinner_end(&spin_hi);
1918 	igt_spinner_end(&spin_lo);
1919 	intel_gt_set_wedged(gt);
1920 	err = -EIO;
1921 	goto err_spin_lo;
1922 }
1923 
1924 struct preempt_client {
1925 	struct igt_spinner spin;
1926 	struct i915_gem_context *ctx;
1927 };
1928 
1929 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1930 {
1931 	c->ctx = kernel_context(gt->i915, NULL);
1932 	if (!c->ctx)
1933 		return -ENOMEM;
1934 
1935 	if (igt_spinner_init(&c->spin, gt))
1936 		goto err_ctx;
1937 
1938 	return 0;
1939 
1940 err_ctx:
1941 	kernel_context_close(c->ctx);
1942 	return -ENOMEM;
1943 }
1944 
1945 static void preempt_client_fini(struct preempt_client *c)
1946 {
1947 	igt_spinner_fini(&c->spin);
1948 	kernel_context_close(c->ctx);
1949 }
1950 
1951 static int live_nopreempt(void *arg)
1952 {
1953 	struct intel_gt *gt = arg;
1954 	struct intel_engine_cs *engine;
1955 	struct preempt_client a, b;
1956 	enum intel_engine_id id;
1957 	int err = -ENOMEM;
1958 
1959 	/*
1960 	 * Verify that we can disable preemption for an individual request
1961 	 * that may be being observed and not want to be interrupted.
1962 	 */
1963 
1964 	if (preempt_client_init(gt, &a))
1965 		return -ENOMEM;
1966 	if (preempt_client_init(gt, &b))
1967 		goto err_client_a;
1968 	b.ctx->sched.priority = I915_PRIORITY_MAX;
1969 
1970 	for_each_engine(engine, gt, id) {
1971 		struct i915_request *rq_a, *rq_b;
1972 
1973 		if (!intel_engine_has_preemption(engine))
1974 			continue;
1975 
1976 		engine->execlists.preempt_hang.count = 0;
1977 
1978 		rq_a = spinner_create_request(&a.spin,
1979 					      a.ctx, engine,
1980 					      MI_ARB_CHECK);
1981 		if (IS_ERR(rq_a)) {
1982 			err = PTR_ERR(rq_a);
1983 			goto err_client_b;
1984 		}
1985 
1986 		/* Low priority client, but unpreemptable! */
1987 		__set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
1988 
1989 		i915_request_add(rq_a);
1990 		if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1991 			pr_err("First client failed to start\n");
1992 			goto err_wedged;
1993 		}
1994 
1995 		rq_b = spinner_create_request(&b.spin,
1996 					      b.ctx, engine,
1997 					      MI_ARB_CHECK);
1998 		if (IS_ERR(rq_b)) {
1999 			err = PTR_ERR(rq_b);
2000 			goto err_client_b;
2001 		}
2002 
2003 		i915_request_add(rq_b);
2004 
2005 		/* B is much more important than A! (But A is unpreemptable.) */
2006 		GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
2007 
2008 		/* Wait long enough for preemption and timeslicing */
2009 		if (igt_wait_for_spinner(&b.spin, rq_b)) {
2010 			pr_err("Second client started too early!\n");
2011 			goto err_wedged;
2012 		}
2013 
2014 		igt_spinner_end(&a.spin);
2015 
2016 		if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2017 			pr_err("Second client failed to start\n");
2018 			goto err_wedged;
2019 		}
2020 
2021 		igt_spinner_end(&b.spin);
2022 
2023 		if (engine->execlists.preempt_hang.count) {
2024 			pr_err("Preemption recorded x%d; should have been suppressed!\n",
2025 			       engine->execlists.preempt_hang.count);
2026 			err = -EINVAL;
2027 			goto err_wedged;
2028 		}
2029 
2030 		if (igt_flush_test(gt->i915))
2031 			goto err_wedged;
2032 	}
2033 
2034 	err = 0;
2035 err_client_b:
2036 	preempt_client_fini(&b);
2037 err_client_a:
2038 	preempt_client_fini(&a);
2039 	return err;
2040 
2041 err_wedged:
2042 	igt_spinner_end(&b.spin);
2043 	igt_spinner_end(&a.spin);
2044 	intel_gt_set_wedged(gt);
2045 	err = -EIO;
2046 	goto err_client_b;
2047 }
2048 
2049 struct live_preempt_cancel {
2050 	struct intel_engine_cs *engine;
2051 	struct preempt_client a, b;
2052 };
2053 
2054 static int __cancel_active0(struct live_preempt_cancel *arg)
2055 {
2056 	struct i915_request *rq;
2057 	struct igt_live_test t;
2058 	int err;
2059 
2060 	/* Preempt cancel of ELSP0 */
2061 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2062 	if (igt_live_test_begin(&t, arg->engine->i915,
2063 				__func__, arg->engine->name))
2064 		return -EIO;
2065 
2066 	rq = spinner_create_request(&arg->a.spin,
2067 				    arg->a.ctx, arg->engine,
2068 				    MI_ARB_CHECK);
2069 	if (IS_ERR(rq))
2070 		return PTR_ERR(rq);
2071 
2072 	clear_bit(CONTEXT_BANNED, &rq->context->flags);
2073 	i915_request_get(rq);
2074 	i915_request_add(rq);
2075 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2076 		err = -EIO;
2077 		goto out;
2078 	}
2079 
2080 	intel_context_ban(rq->context, rq);
2081 	err = intel_engine_pulse(arg->engine);
2082 	if (err)
2083 		goto out;
2084 
2085 	err = wait_for_reset(arg->engine, rq, HZ / 2);
2086 	if (err) {
2087 		pr_err("Cancelled inflight0 request did not reset\n");
2088 		goto out;
2089 	}
2090 
2091 out:
2092 	i915_request_put(rq);
2093 	if (igt_live_test_end(&t))
2094 		err = -EIO;
2095 	return err;
2096 }
2097 
2098 static int __cancel_active1(struct live_preempt_cancel *arg)
2099 {
2100 	struct i915_request *rq[2] = {};
2101 	struct igt_live_test t;
2102 	int err;
2103 
2104 	/* Preempt cancel of ELSP1 */
2105 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2106 	if (igt_live_test_begin(&t, arg->engine->i915,
2107 				__func__, arg->engine->name))
2108 		return -EIO;
2109 
2110 	rq[0] = spinner_create_request(&arg->a.spin,
2111 				       arg->a.ctx, arg->engine,
2112 				       MI_NOOP); /* no preemption */
2113 	if (IS_ERR(rq[0]))
2114 		return PTR_ERR(rq[0]);
2115 
2116 	clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2117 	i915_request_get(rq[0]);
2118 	i915_request_add(rq[0]);
2119 	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2120 		err = -EIO;
2121 		goto out;
2122 	}
2123 
2124 	rq[1] = spinner_create_request(&arg->b.spin,
2125 				       arg->b.ctx, arg->engine,
2126 				       MI_ARB_CHECK);
2127 	if (IS_ERR(rq[1])) {
2128 		err = PTR_ERR(rq[1]);
2129 		goto out;
2130 	}
2131 
2132 	clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2133 	i915_request_get(rq[1]);
2134 	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2135 	i915_request_add(rq[1]);
2136 	if (err)
2137 		goto out;
2138 
2139 	intel_context_ban(rq[1]->context, rq[1]);
2140 	err = intel_engine_pulse(arg->engine);
2141 	if (err)
2142 		goto out;
2143 
2144 	igt_spinner_end(&arg->a.spin);
2145 	err = wait_for_reset(arg->engine, rq[1], HZ / 2);
2146 	if (err)
2147 		goto out;
2148 
2149 	if (rq[0]->fence.error != 0) {
2150 		pr_err("Normal inflight0 request did not complete\n");
2151 		err = -EINVAL;
2152 		goto out;
2153 	}
2154 
2155 	if (rq[1]->fence.error != -EIO) {
2156 		pr_err("Cancelled inflight1 request did not report -EIO\n");
2157 		err = -EINVAL;
2158 		goto out;
2159 	}
2160 
2161 out:
2162 	i915_request_put(rq[1]);
2163 	i915_request_put(rq[0]);
2164 	if (igt_live_test_end(&t))
2165 		err = -EIO;
2166 	return err;
2167 }
2168 
2169 static int __cancel_queued(struct live_preempt_cancel *arg)
2170 {
2171 	struct i915_request *rq[3] = {};
2172 	struct igt_live_test t;
2173 	int err;
2174 
2175 	/* Full ELSP and one in the wings */
2176 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2177 	if (igt_live_test_begin(&t, arg->engine->i915,
2178 				__func__, arg->engine->name))
2179 		return -EIO;
2180 
2181 	rq[0] = spinner_create_request(&arg->a.spin,
2182 				       arg->a.ctx, arg->engine,
2183 				       MI_ARB_CHECK);
2184 	if (IS_ERR(rq[0]))
2185 		return PTR_ERR(rq[0]);
2186 
2187 	clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2188 	i915_request_get(rq[0]);
2189 	i915_request_add(rq[0]);
2190 	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2191 		err = -EIO;
2192 		goto out;
2193 	}
2194 
2195 	rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
2196 	if (IS_ERR(rq[1])) {
2197 		err = PTR_ERR(rq[1]);
2198 		goto out;
2199 	}
2200 
2201 	clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2202 	i915_request_get(rq[1]);
2203 	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2204 	i915_request_add(rq[1]);
2205 	if (err)
2206 		goto out;
2207 
2208 	rq[2] = spinner_create_request(&arg->b.spin,
2209 				       arg->a.ctx, arg->engine,
2210 				       MI_ARB_CHECK);
2211 	if (IS_ERR(rq[2])) {
2212 		err = PTR_ERR(rq[2]);
2213 		goto out;
2214 	}
2215 
2216 	i915_request_get(rq[2]);
2217 	err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
2218 	i915_request_add(rq[2]);
2219 	if (err)
2220 		goto out;
2221 
2222 	intel_context_ban(rq[2]->context, rq[2]);
2223 	err = intel_engine_pulse(arg->engine);
2224 	if (err)
2225 		goto out;
2226 
2227 	err = wait_for_reset(arg->engine, rq[2], HZ / 2);
2228 	if (err)
2229 		goto out;
2230 
2231 	if (rq[0]->fence.error != -EIO) {
2232 		pr_err("Cancelled inflight0 request did not report -EIO\n");
2233 		err = -EINVAL;
2234 		goto out;
2235 	}
2236 
2237 	/*
2238 	 * The behavior between having semaphores and not is different. With
2239 	 * semaphores the subsequent request is on the hardware and not cancelled
2240 	 * while without the request is held in the driver and cancelled.
2241 	 */
2242 	if (intel_engine_has_semaphores(rq[1]->engine) &&
2243 	    rq[1]->fence.error != 0) {
2244 		pr_err("Normal inflight1 request did not complete\n");
2245 		err = -EINVAL;
2246 		goto out;
2247 	}
2248 
2249 	if (rq[2]->fence.error != -EIO) {
2250 		pr_err("Cancelled queued request did not report -EIO\n");
2251 		err = -EINVAL;
2252 		goto out;
2253 	}
2254 
2255 out:
2256 	i915_request_put(rq[2]);
2257 	i915_request_put(rq[1]);
2258 	i915_request_put(rq[0]);
2259 	if (igt_live_test_end(&t))
2260 		err = -EIO;
2261 	return err;
2262 }
2263 
2264 static int __cancel_hostile(struct live_preempt_cancel *arg)
2265 {
2266 	struct i915_request *rq;
2267 	int err;
2268 
2269 	/* Preempt cancel non-preemptible spinner in ELSP0 */
2270 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2271 		return 0;
2272 
2273 	if (!intel_has_reset_engine(arg->engine->gt))
2274 		return 0;
2275 
2276 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2277 	rq = spinner_create_request(&arg->a.spin,
2278 				    arg->a.ctx, arg->engine,
2279 				    MI_NOOP); /* preemption disabled */
2280 	if (IS_ERR(rq))
2281 		return PTR_ERR(rq);
2282 
2283 	clear_bit(CONTEXT_BANNED, &rq->context->flags);
2284 	i915_request_get(rq);
2285 	i915_request_add(rq);
2286 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2287 		err = -EIO;
2288 		goto out;
2289 	}
2290 
2291 	intel_context_ban(rq->context, rq);
2292 	err = intel_engine_pulse(arg->engine); /* force reset */
2293 	if (err)
2294 		goto out;
2295 
2296 	err = wait_for_reset(arg->engine, rq, HZ / 2);
2297 	if (err) {
2298 		pr_err("Cancelled inflight0 request did not reset\n");
2299 		goto out;
2300 	}
2301 
2302 out:
2303 	i915_request_put(rq);
2304 	if (igt_flush_test(arg->engine->i915))
2305 		err = -EIO;
2306 	return err;
2307 }
2308 
2309 static void force_reset_timeout(struct intel_engine_cs *engine)
2310 {
2311 	engine->reset_timeout.probability = 999;
2312 	atomic_set(&engine->reset_timeout.times, -1);
2313 }
2314 
2315 static void cancel_reset_timeout(struct intel_engine_cs *engine)
2316 {
2317 	memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
2318 }
2319 
2320 static int __cancel_fail(struct live_preempt_cancel *arg)
2321 {
2322 	struct intel_engine_cs *engine = arg->engine;
2323 	struct i915_request *rq;
2324 	int err;
2325 
2326 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2327 		return 0;
2328 
2329 	if (!intel_has_reset_engine(engine->gt))
2330 		return 0;
2331 
2332 	GEM_TRACE("%s(%s)\n", __func__, engine->name);
2333 	rq = spinner_create_request(&arg->a.spin,
2334 				    arg->a.ctx, engine,
2335 				    MI_NOOP); /* preemption disabled */
2336 	if (IS_ERR(rq))
2337 		return PTR_ERR(rq);
2338 
2339 	clear_bit(CONTEXT_BANNED, &rq->context->flags);
2340 	i915_request_get(rq);
2341 	i915_request_add(rq);
2342 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2343 		err = -EIO;
2344 		goto out;
2345 	}
2346 
2347 	intel_context_set_banned(rq->context);
2348 
2349 	err = intel_engine_pulse(engine);
2350 	if (err)
2351 		goto out;
2352 
2353 	force_reset_timeout(engine);
2354 
2355 	/* force preempt reset [failure] */
2356 	while (!engine->execlists.pending[0])
2357 		intel_engine_flush_submission(engine);
2358 	del_timer_sync(&engine->execlists.preempt);
2359 	intel_engine_flush_submission(engine);
2360 
2361 	cancel_reset_timeout(engine);
2362 
2363 	/* after failure, require heartbeats to reset device */
2364 	intel_engine_set_heartbeat(engine, 1);
2365 	err = wait_for_reset(engine, rq, HZ / 2);
2366 	intel_engine_set_heartbeat(engine,
2367 				   engine->defaults.heartbeat_interval_ms);
2368 	if (err) {
2369 		pr_err("Cancelled inflight0 request did not reset\n");
2370 		goto out;
2371 	}
2372 
2373 out:
2374 	i915_request_put(rq);
2375 	if (igt_flush_test(engine->i915))
2376 		err = -EIO;
2377 	return err;
2378 }
2379 
2380 static int live_preempt_cancel(void *arg)
2381 {
2382 	struct intel_gt *gt = arg;
2383 	struct live_preempt_cancel data;
2384 	enum intel_engine_id id;
2385 	int err = -ENOMEM;
2386 
2387 	/*
2388 	 * To cancel an inflight context, we need to first remove it from the
2389 	 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
2390 	 */
2391 
2392 	if (preempt_client_init(gt, &data.a))
2393 		return -ENOMEM;
2394 	if (preempt_client_init(gt, &data.b))
2395 		goto err_client_a;
2396 
2397 	for_each_engine(data.engine, gt, id) {
2398 		if (!intel_engine_has_preemption(data.engine))
2399 			continue;
2400 
2401 		err = __cancel_active0(&data);
2402 		if (err)
2403 			goto err_wedged;
2404 
2405 		err = __cancel_active1(&data);
2406 		if (err)
2407 			goto err_wedged;
2408 
2409 		err = __cancel_queued(&data);
2410 		if (err)
2411 			goto err_wedged;
2412 
2413 		err = __cancel_hostile(&data);
2414 		if (err)
2415 			goto err_wedged;
2416 
2417 		err = __cancel_fail(&data);
2418 		if (err)
2419 			goto err_wedged;
2420 	}
2421 
2422 	err = 0;
2423 err_client_b:
2424 	preempt_client_fini(&data.b);
2425 err_client_a:
2426 	preempt_client_fini(&data.a);
2427 	return err;
2428 
2429 err_wedged:
2430 	GEM_TRACE_DUMP();
2431 	igt_spinner_end(&data.b.spin);
2432 	igt_spinner_end(&data.a.spin);
2433 	intel_gt_set_wedged(gt);
2434 	goto err_client_b;
2435 }
2436 
2437 static int live_suppress_self_preempt(void *arg)
2438 {
2439 	struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2440 	struct intel_gt *gt = arg;
2441 	struct intel_engine_cs *engine;
2442 	struct preempt_client a, b;
2443 	enum intel_engine_id id;
2444 	int err = -ENOMEM;
2445 
2446 	/*
2447 	 * Verify that if a preemption request does not cause a change in
2448 	 * the current execution order, the preempt-to-idle injection is
2449 	 * skipped and that we do not accidentally apply it after the CS
2450 	 * completion event.
2451 	 */
2452 
2453 	if (intel_uc_uses_guc_submission(&gt->uc))
2454 		return 0; /* presume black blox */
2455 
2456 	if (intel_vgpu_active(gt->i915))
2457 		return 0; /* GVT forces single port & request submission */
2458 
2459 	if (preempt_client_init(gt, &a))
2460 		return -ENOMEM;
2461 	if (preempt_client_init(gt, &b))
2462 		goto err_client_a;
2463 
2464 	for_each_engine(engine, gt, id) {
2465 		struct i915_request *rq_a, *rq_b;
2466 		int depth;
2467 
2468 		if (!intel_engine_has_preemption(engine))
2469 			continue;
2470 
2471 		if (igt_flush_test(gt->i915))
2472 			goto err_wedged;
2473 
2474 		st_engine_heartbeat_disable(engine);
2475 		engine->execlists.preempt_hang.count = 0;
2476 
2477 		rq_a = spinner_create_request(&a.spin,
2478 					      a.ctx, engine,
2479 					      MI_NOOP);
2480 		if (IS_ERR(rq_a)) {
2481 			err = PTR_ERR(rq_a);
2482 			st_engine_heartbeat_enable(engine);
2483 			goto err_client_b;
2484 		}
2485 
2486 		i915_request_add(rq_a);
2487 		if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2488 			pr_err("First client failed to start\n");
2489 			st_engine_heartbeat_enable(engine);
2490 			goto err_wedged;
2491 		}
2492 
2493 		/* Keep postponing the timer to avoid premature slicing */
2494 		mod_timer(&engine->execlists.timer, jiffies + HZ);
2495 		for (depth = 0; depth < 8; depth++) {
2496 			rq_b = spinner_create_request(&b.spin,
2497 						      b.ctx, engine,
2498 						      MI_NOOP);
2499 			if (IS_ERR(rq_b)) {
2500 				err = PTR_ERR(rq_b);
2501 				st_engine_heartbeat_enable(engine);
2502 				goto err_client_b;
2503 			}
2504 			i915_request_add(rq_b);
2505 
2506 			GEM_BUG_ON(i915_request_completed(rq_a));
2507 			engine->sched_engine->schedule(rq_a, &attr);
2508 			igt_spinner_end(&a.spin);
2509 
2510 			if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2511 				pr_err("Second client failed to start\n");
2512 				st_engine_heartbeat_enable(engine);
2513 				goto err_wedged;
2514 			}
2515 
2516 			swap(a, b);
2517 			rq_a = rq_b;
2518 		}
2519 		igt_spinner_end(&a.spin);
2520 
2521 		if (engine->execlists.preempt_hang.count) {
2522 			pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2523 			       engine->name,
2524 			       engine->execlists.preempt_hang.count,
2525 			       depth);
2526 			st_engine_heartbeat_enable(engine);
2527 			err = -EINVAL;
2528 			goto err_client_b;
2529 		}
2530 
2531 		st_engine_heartbeat_enable(engine);
2532 		if (igt_flush_test(gt->i915))
2533 			goto err_wedged;
2534 	}
2535 
2536 	err = 0;
2537 err_client_b:
2538 	preempt_client_fini(&b);
2539 err_client_a:
2540 	preempt_client_fini(&a);
2541 	return err;
2542 
2543 err_wedged:
2544 	igt_spinner_end(&b.spin);
2545 	igt_spinner_end(&a.spin);
2546 	intel_gt_set_wedged(gt);
2547 	err = -EIO;
2548 	goto err_client_b;
2549 }
2550 
2551 static int live_chain_preempt(void *arg)
2552 {
2553 	struct intel_gt *gt = arg;
2554 	struct intel_engine_cs *engine;
2555 	struct preempt_client hi, lo;
2556 	enum intel_engine_id id;
2557 	int err = -ENOMEM;
2558 
2559 	/*
2560 	 * Build a chain AB...BA between two contexts (A, B) and request
2561 	 * preemption of the last request. It should then complete before
2562 	 * the previously submitted spinner in B.
2563 	 */
2564 
2565 	if (preempt_client_init(gt, &hi))
2566 		return -ENOMEM;
2567 
2568 	if (preempt_client_init(gt, &lo))
2569 		goto err_client_hi;
2570 
2571 	for_each_engine(engine, gt, id) {
2572 		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2573 		struct igt_live_test t;
2574 		struct i915_request *rq;
2575 		int ring_size, count, i;
2576 
2577 		if (!intel_engine_has_preemption(engine))
2578 			continue;
2579 
2580 		rq = spinner_create_request(&lo.spin,
2581 					    lo.ctx, engine,
2582 					    MI_ARB_CHECK);
2583 		if (IS_ERR(rq))
2584 			goto err_wedged;
2585 
2586 		i915_request_get(rq);
2587 		i915_request_add(rq);
2588 
2589 		ring_size = rq->wa_tail - rq->head;
2590 		if (ring_size < 0)
2591 			ring_size += rq->ring->size;
2592 		ring_size = rq->ring->size / ring_size;
2593 		pr_debug("%s(%s): Using maximum of %d requests\n",
2594 			 __func__, engine->name, ring_size);
2595 
2596 		igt_spinner_end(&lo.spin);
2597 		if (i915_request_wait(rq, 0, HZ / 2) < 0) {
2598 			pr_err("Timed out waiting to flush %s\n", engine->name);
2599 			i915_request_put(rq);
2600 			goto err_wedged;
2601 		}
2602 		i915_request_put(rq);
2603 
2604 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
2605 			err = -EIO;
2606 			goto err_wedged;
2607 		}
2608 
2609 		for_each_prime_number_from(count, 1, ring_size) {
2610 			rq = spinner_create_request(&hi.spin,
2611 						    hi.ctx, engine,
2612 						    MI_ARB_CHECK);
2613 			if (IS_ERR(rq))
2614 				goto err_wedged;
2615 			i915_request_add(rq);
2616 			if (!igt_wait_for_spinner(&hi.spin, rq))
2617 				goto err_wedged;
2618 
2619 			rq = spinner_create_request(&lo.spin,
2620 						    lo.ctx, engine,
2621 						    MI_ARB_CHECK);
2622 			if (IS_ERR(rq))
2623 				goto err_wedged;
2624 			i915_request_add(rq);
2625 
2626 			for (i = 0; i < count; i++) {
2627 				rq = igt_request_alloc(lo.ctx, engine);
2628 				if (IS_ERR(rq))
2629 					goto err_wedged;
2630 				i915_request_add(rq);
2631 			}
2632 
2633 			rq = igt_request_alloc(hi.ctx, engine);
2634 			if (IS_ERR(rq))
2635 				goto err_wedged;
2636 
2637 			i915_request_get(rq);
2638 			i915_request_add(rq);
2639 			engine->sched_engine->schedule(rq, &attr);
2640 
2641 			igt_spinner_end(&hi.spin);
2642 			if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2643 				struct drm_printer p =
2644 					drm_info_printer(gt->i915->drm.dev);
2645 
2646 				pr_err("Failed to preempt over chain of %d\n",
2647 				       count);
2648 				intel_engine_dump(engine, &p,
2649 						  "%s\n", engine->name);
2650 				i915_request_put(rq);
2651 				goto err_wedged;
2652 			}
2653 			igt_spinner_end(&lo.spin);
2654 			i915_request_put(rq);
2655 
2656 			rq = igt_request_alloc(lo.ctx, engine);
2657 			if (IS_ERR(rq))
2658 				goto err_wedged;
2659 
2660 			i915_request_get(rq);
2661 			i915_request_add(rq);
2662 
2663 			if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2664 				struct drm_printer p =
2665 					drm_info_printer(gt->i915->drm.dev);
2666 
2667 				pr_err("Failed to flush low priority chain of %d requests\n",
2668 				       count);
2669 				intel_engine_dump(engine, &p,
2670 						  "%s\n", engine->name);
2671 
2672 				i915_request_put(rq);
2673 				goto err_wedged;
2674 			}
2675 			i915_request_put(rq);
2676 		}
2677 
2678 		if (igt_live_test_end(&t)) {
2679 			err = -EIO;
2680 			goto err_wedged;
2681 		}
2682 	}
2683 
2684 	err = 0;
2685 err_client_lo:
2686 	preempt_client_fini(&lo);
2687 err_client_hi:
2688 	preempt_client_fini(&hi);
2689 	return err;
2690 
2691 err_wedged:
2692 	igt_spinner_end(&hi.spin);
2693 	igt_spinner_end(&lo.spin);
2694 	intel_gt_set_wedged(gt);
2695 	err = -EIO;
2696 	goto err_client_lo;
2697 }
2698 
2699 static int create_gang(struct intel_engine_cs *engine,
2700 		       struct i915_request **prev)
2701 {
2702 	struct drm_i915_gem_object *obj;
2703 	struct intel_context *ce;
2704 	struct i915_request *rq;
2705 	struct i915_vma *vma;
2706 	u32 *cs;
2707 	int err;
2708 
2709 	ce = intel_context_create(engine);
2710 	if (IS_ERR(ce))
2711 		return PTR_ERR(ce);
2712 
2713 	obj = i915_gem_object_create_internal(engine->i915, 4096);
2714 	if (IS_ERR(obj)) {
2715 		err = PTR_ERR(obj);
2716 		goto err_ce;
2717 	}
2718 
2719 	vma = i915_vma_instance(obj, ce->vm, NULL);
2720 	if (IS_ERR(vma)) {
2721 		err = PTR_ERR(vma);
2722 		goto err_obj;
2723 	}
2724 
2725 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
2726 	if (err)
2727 		goto err_obj;
2728 
2729 	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
2730 	if (IS_ERR(cs)) {
2731 		err = PTR_ERR(cs);
2732 		goto err_obj;
2733 	}
2734 
2735 	/* Semaphore target: spin until zero */
2736 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2737 
2738 	*cs++ = MI_SEMAPHORE_WAIT |
2739 		MI_SEMAPHORE_POLL |
2740 		MI_SEMAPHORE_SAD_EQ_SDD;
2741 	*cs++ = 0;
2742 	*cs++ = lower_32_bits(vma->node.start);
2743 	*cs++ = upper_32_bits(vma->node.start);
2744 
2745 	if (*prev) {
2746 		u64 offset = (*prev)->batch->node.start;
2747 
2748 		/* Terminate the spinner in the next lower priority batch. */
2749 		*cs++ = MI_STORE_DWORD_IMM_GEN4;
2750 		*cs++ = lower_32_bits(offset);
2751 		*cs++ = upper_32_bits(offset);
2752 		*cs++ = 0;
2753 	}
2754 
2755 	*cs++ = MI_BATCH_BUFFER_END;
2756 	i915_gem_object_flush_map(obj);
2757 	i915_gem_object_unpin_map(obj);
2758 
2759 	rq = intel_context_create_request(ce);
2760 	if (IS_ERR(rq)) {
2761 		err = PTR_ERR(rq);
2762 		goto err_obj;
2763 	}
2764 
2765 	rq->batch = i915_vma_get(vma);
2766 	i915_request_get(rq);
2767 
2768 	i915_vma_lock(vma);
2769 	err = i915_request_await_object(rq, vma->obj, false);
2770 	if (!err)
2771 		err = i915_vma_move_to_active(vma, rq, 0);
2772 	if (!err)
2773 		err = rq->engine->emit_bb_start(rq,
2774 						vma->node.start,
2775 						PAGE_SIZE, 0);
2776 	i915_vma_unlock(vma);
2777 	i915_request_add(rq);
2778 	if (err)
2779 		goto err_rq;
2780 
2781 	i915_gem_object_put(obj);
2782 	intel_context_put(ce);
2783 
2784 	rq->mock.link.next = &(*prev)->mock.link;
2785 	*prev = rq;
2786 	return 0;
2787 
2788 err_rq:
2789 	i915_vma_put(rq->batch);
2790 	i915_request_put(rq);
2791 err_obj:
2792 	i915_gem_object_put(obj);
2793 err_ce:
2794 	intel_context_put(ce);
2795 	return err;
2796 }
2797 
2798 static int __live_preempt_ring(struct intel_engine_cs *engine,
2799 			       struct igt_spinner *spin,
2800 			       int queue_sz, int ring_sz)
2801 {
2802 	struct intel_context *ce[2] = {};
2803 	struct i915_request *rq;
2804 	struct igt_live_test t;
2805 	int err = 0;
2806 	int n;
2807 
2808 	if (igt_live_test_begin(&t, engine->i915, __func__, engine->name))
2809 		return -EIO;
2810 
2811 	for (n = 0; n < ARRAY_SIZE(ce); n++) {
2812 		struct intel_context *tmp;
2813 
2814 		tmp = intel_context_create(engine);
2815 		if (IS_ERR(tmp)) {
2816 			err = PTR_ERR(tmp);
2817 			goto err_ce;
2818 		}
2819 
2820 		tmp->ring_size = ring_sz;
2821 
2822 		err = intel_context_pin(tmp);
2823 		if (err) {
2824 			intel_context_put(tmp);
2825 			goto err_ce;
2826 		}
2827 
2828 		memset32(tmp->ring->vaddr,
2829 			 0xdeadbeef, /* trigger a hang if executed */
2830 			 tmp->ring->vma->size / sizeof(u32));
2831 
2832 		ce[n] = tmp;
2833 	}
2834 
2835 	rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK);
2836 	if (IS_ERR(rq)) {
2837 		err = PTR_ERR(rq);
2838 		goto err_ce;
2839 	}
2840 
2841 	i915_request_get(rq);
2842 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2843 	i915_request_add(rq);
2844 
2845 	if (!igt_wait_for_spinner(spin, rq)) {
2846 		intel_gt_set_wedged(engine->gt);
2847 		i915_request_put(rq);
2848 		err = -ETIME;
2849 		goto err_ce;
2850 	}
2851 
2852 	/* Fill the ring, until we will cause a wrap */
2853 	n = 0;
2854 	while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) {
2855 		struct i915_request *tmp;
2856 
2857 		tmp = intel_context_create_request(ce[0]);
2858 		if (IS_ERR(tmp)) {
2859 			err = PTR_ERR(tmp);
2860 			i915_request_put(rq);
2861 			goto err_ce;
2862 		}
2863 
2864 		i915_request_add(tmp);
2865 		intel_engine_flush_submission(engine);
2866 		n++;
2867 	}
2868 	intel_engine_flush_submission(engine);
2869 	pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
2870 		 engine->name, queue_sz, n,
2871 		 ce[0]->ring->size,
2872 		 ce[0]->ring->tail,
2873 		 ce[0]->ring->emit,
2874 		 rq->tail);
2875 	i915_request_put(rq);
2876 
2877 	/* Create a second request to preempt the first ring */
2878 	rq = intel_context_create_request(ce[1]);
2879 	if (IS_ERR(rq)) {
2880 		err = PTR_ERR(rq);
2881 		goto err_ce;
2882 	}
2883 
2884 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2885 	i915_request_get(rq);
2886 	i915_request_add(rq);
2887 
2888 	err = wait_for_submit(engine, rq, HZ / 2);
2889 	i915_request_put(rq);
2890 	if (err) {
2891 		pr_err("%s: preemption request was not submitted\n",
2892 		       engine->name);
2893 		err = -ETIME;
2894 	}
2895 
2896 	pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
2897 		 engine->name,
2898 		 ce[0]->ring->tail, ce[0]->ring->emit,
2899 		 ce[1]->ring->tail, ce[1]->ring->emit);
2900 
2901 err_ce:
2902 	intel_engine_flush_submission(engine);
2903 	igt_spinner_end(spin);
2904 	for (n = 0; n < ARRAY_SIZE(ce); n++) {
2905 		if (IS_ERR_OR_NULL(ce[n]))
2906 			break;
2907 
2908 		intel_context_unpin(ce[n]);
2909 		intel_context_put(ce[n]);
2910 	}
2911 	if (igt_live_test_end(&t))
2912 		err = -EIO;
2913 	return err;
2914 }
2915 
2916 static int live_preempt_ring(void *arg)
2917 {
2918 	struct intel_gt *gt = arg;
2919 	struct intel_engine_cs *engine;
2920 	struct igt_spinner spin;
2921 	enum intel_engine_id id;
2922 	int err = 0;
2923 
2924 	/*
2925 	 * Check that we rollback large chunks of a ring in order to do a
2926 	 * preemption event. Similar to live_unlite_ring, but looking at
2927 	 * ring size rather than the impact of intel_ring_direction().
2928 	 */
2929 
2930 	if (igt_spinner_init(&spin, gt))
2931 		return -ENOMEM;
2932 
2933 	for_each_engine(engine, gt, id) {
2934 		int n;
2935 
2936 		if (!intel_engine_has_preemption(engine))
2937 			continue;
2938 
2939 		if (!intel_engine_can_store_dword(engine))
2940 			continue;
2941 
2942 		st_engine_heartbeat_disable(engine);
2943 
2944 		for (n = 0; n <= 3; n++) {
2945 			err = __live_preempt_ring(engine, &spin,
2946 						  n * SZ_4K / 4, SZ_4K);
2947 			if (err)
2948 				break;
2949 		}
2950 
2951 		st_engine_heartbeat_enable(engine);
2952 		if (err)
2953 			break;
2954 	}
2955 
2956 	igt_spinner_fini(&spin);
2957 	return err;
2958 }
2959 
2960 static int live_preempt_gang(void *arg)
2961 {
2962 	struct intel_gt *gt = arg;
2963 	struct intel_engine_cs *engine;
2964 	enum intel_engine_id id;
2965 
2966 	/*
2967 	 * Build as long a chain of preempters as we can, with each
2968 	 * request higher priority than the last. Once we are ready, we release
2969 	 * the last batch which then precolates down the chain, each releasing
2970 	 * the next oldest in turn. The intent is to simply push as hard as we
2971 	 * can with the number of preemptions, trying to exceed narrow HW
2972 	 * limits. At a minimum, we insist that we can sort all the user
2973 	 * high priority levels into execution order.
2974 	 */
2975 
2976 	for_each_engine(engine, gt, id) {
2977 		struct i915_request *rq = NULL;
2978 		struct igt_live_test t;
2979 		IGT_TIMEOUT(end_time);
2980 		int prio = 0;
2981 		int err = 0;
2982 		u32 *cs;
2983 
2984 		if (!intel_engine_has_preemption(engine))
2985 			continue;
2986 
2987 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2988 			return -EIO;
2989 
2990 		do {
2991 			struct i915_sched_attr attr = { .priority = prio++ };
2992 
2993 			err = create_gang(engine, &rq);
2994 			if (err)
2995 				break;
2996 
2997 			/* Submit each spinner at increasing priority */
2998 			engine->sched_engine->schedule(rq, &attr);
2999 		} while (prio <= I915_PRIORITY_MAX &&
3000 			 !__igt_timeout(end_time, NULL));
3001 		pr_debug("%s: Preempt chain of %d requests\n",
3002 			 engine->name, prio);
3003 
3004 		/*
3005 		 * Such that the last spinner is the highest priority and
3006 		 * should execute first. When that spinner completes,
3007 		 * it will terminate the next lowest spinner until there
3008 		 * are no more spinners and the gang is complete.
3009 		 */
3010 		cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);
3011 		if (!IS_ERR(cs)) {
3012 			*cs = 0;
3013 			i915_gem_object_unpin_map(rq->batch->obj);
3014 		} else {
3015 			err = PTR_ERR(cs);
3016 			intel_gt_set_wedged(gt);
3017 		}
3018 
3019 		while (rq) { /* wait for each rq from highest to lowest prio */
3020 			struct i915_request *n = list_next_entry(rq, mock.link);
3021 
3022 			if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
3023 				struct drm_printer p =
3024 					drm_info_printer(engine->i915->drm.dev);
3025 
3026 				pr_err("Failed to flush chain of %d requests, at %d\n",
3027 				       prio, rq_prio(rq));
3028 				intel_engine_dump(engine, &p,
3029 						  "%s\n", engine->name);
3030 
3031 				err = -ETIME;
3032 			}
3033 
3034 			i915_vma_put(rq->batch);
3035 			i915_request_put(rq);
3036 			rq = n;
3037 		}
3038 
3039 		if (igt_live_test_end(&t))
3040 			err = -EIO;
3041 		if (err)
3042 			return err;
3043 	}
3044 
3045 	return 0;
3046 }
3047 
3048 static struct i915_vma *
3049 create_gpr_user(struct intel_engine_cs *engine,
3050 		struct i915_vma *result,
3051 		unsigned int offset)
3052 {
3053 	struct drm_i915_gem_object *obj;
3054 	struct i915_vma *vma;
3055 	u32 *cs;
3056 	int err;
3057 	int i;
3058 
3059 	obj = i915_gem_object_create_internal(engine->i915, 4096);
3060 	if (IS_ERR(obj))
3061 		return ERR_CAST(obj);
3062 
3063 	vma = i915_vma_instance(obj, result->vm, NULL);
3064 	if (IS_ERR(vma)) {
3065 		i915_gem_object_put(obj);
3066 		return vma;
3067 	}
3068 
3069 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
3070 	if (err) {
3071 		i915_vma_put(vma);
3072 		return ERR_PTR(err);
3073 	}
3074 
3075 	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
3076 	if (IS_ERR(cs)) {
3077 		i915_vma_put(vma);
3078 		return ERR_CAST(cs);
3079 	}
3080 
3081 	/* All GPR are clear for new contexts. We use GPR(0) as a constant */
3082 	*cs++ = MI_LOAD_REGISTER_IMM(1);
3083 	*cs++ = CS_GPR(engine, 0);
3084 	*cs++ = 1;
3085 
3086 	for (i = 1; i < NUM_GPR; i++) {
3087 		u64 addr;
3088 
3089 		/*
3090 		 * Perform: GPR[i]++
3091 		 *
3092 		 * As we read and write into the context saved GPR[i], if
3093 		 * we restart this batch buffer from an earlier point, we
3094 		 * will repeat the increment and store a value > 1.
3095 		 */
3096 		*cs++ = MI_MATH(4);
3097 		*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));
3098 		*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));
3099 		*cs++ = MI_MATH_ADD;
3100 		*cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);
3101 
3102 		addr = result->node.start + offset + i * sizeof(*cs);
3103 		*cs++ = MI_STORE_REGISTER_MEM_GEN8;
3104 		*cs++ = CS_GPR(engine, 2 * i);
3105 		*cs++ = lower_32_bits(addr);
3106 		*cs++ = upper_32_bits(addr);
3107 
3108 		*cs++ = MI_SEMAPHORE_WAIT |
3109 			MI_SEMAPHORE_POLL |
3110 			MI_SEMAPHORE_SAD_GTE_SDD;
3111 		*cs++ = i;
3112 		*cs++ = lower_32_bits(result->node.start);
3113 		*cs++ = upper_32_bits(result->node.start);
3114 	}
3115 
3116 	*cs++ = MI_BATCH_BUFFER_END;
3117 	i915_gem_object_flush_map(obj);
3118 	i915_gem_object_unpin_map(obj);
3119 
3120 	return vma;
3121 }
3122 
3123 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
3124 {
3125 	struct drm_i915_gem_object *obj;
3126 	struct i915_vma *vma;
3127 	int err;
3128 
3129 	obj = i915_gem_object_create_internal(gt->i915, sz);
3130 	if (IS_ERR(obj))
3131 		return ERR_CAST(obj);
3132 
3133 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
3134 	if (IS_ERR(vma)) {
3135 		i915_gem_object_put(obj);
3136 		return vma;
3137 	}
3138 
3139 	err = i915_ggtt_pin(vma, NULL, 0, 0);
3140 	if (err) {
3141 		i915_vma_put(vma);
3142 		return ERR_PTR(err);
3143 	}
3144 
3145 	return vma;
3146 }
3147 
3148 static struct i915_request *
3149 create_gpr_client(struct intel_engine_cs *engine,
3150 		  struct i915_vma *global,
3151 		  unsigned int offset)
3152 {
3153 	struct i915_vma *batch, *vma;
3154 	struct intel_context *ce;
3155 	struct i915_request *rq;
3156 	int err;
3157 
3158 	ce = intel_context_create(engine);
3159 	if (IS_ERR(ce))
3160 		return ERR_CAST(ce);
3161 
3162 	vma = i915_vma_instance(global->obj, ce->vm, NULL);
3163 	if (IS_ERR(vma)) {
3164 		err = PTR_ERR(vma);
3165 		goto out_ce;
3166 	}
3167 
3168 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
3169 	if (err)
3170 		goto out_ce;
3171 
3172 	batch = create_gpr_user(engine, vma, offset);
3173 	if (IS_ERR(batch)) {
3174 		err = PTR_ERR(batch);
3175 		goto out_vma;
3176 	}
3177 
3178 	rq = intel_context_create_request(ce);
3179 	if (IS_ERR(rq)) {
3180 		err = PTR_ERR(rq);
3181 		goto out_batch;
3182 	}
3183 
3184 	i915_vma_lock(vma);
3185 	err = i915_request_await_object(rq, vma->obj, false);
3186 	if (!err)
3187 		err = i915_vma_move_to_active(vma, rq, 0);
3188 	i915_vma_unlock(vma);
3189 
3190 	i915_vma_lock(batch);
3191 	if (!err)
3192 		err = i915_request_await_object(rq, batch->obj, false);
3193 	if (!err)
3194 		err = i915_vma_move_to_active(batch, rq, 0);
3195 	if (!err)
3196 		err = rq->engine->emit_bb_start(rq,
3197 						batch->node.start,
3198 						PAGE_SIZE, 0);
3199 	i915_vma_unlock(batch);
3200 	i915_vma_unpin(batch);
3201 
3202 	if (!err)
3203 		i915_request_get(rq);
3204 	i915_request_add(rq);
3205 
3206 out_batch:
3207 	i915_vma_put(batch);
3208 out_vma:
3209 	i915_vma_unpin(vma);
3210 out_ce:
3211 	intel_context_put(ce);
3212 	return err ? ERR_PTR(err) : rq;
3213 }
3214 
3215 static int preempt_user(struct intel_engine_cs *engine,
3216 			struct i915_vma *global,
3217 			int id)
3218 {
3219 	struct i915_sched_attr attr = {
3220 		.priority = I915_PRIORITY_MAX
3221 	};
3222 	struct i915_request *rq;
3223 	int err = 0;
3224 	u32 *cs;
3225 
3226 	rq = intel_engine_create_kernel_request(engine);
3227 	if (IS_ERR(rq))
3228 		return PTR_ERR(rq);
3229 
3230 	cs = intel_ring_begin(rq, 4);
3231 	if (IS_ERR(cs)) {
3232 		i915_request_add(rq);
3233 		return PTR_ERR(cs);
3234 	}
3235 
3236 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
3237 	*cs++ = i915_ggtt_offset(global);
3238 	*cs++ = 0;
3239 	*cs++ = id;
3240 
3241 	intel_ring_advance(rq, cs);
3242 
3243 	i915_request_get(rq);
3244 	i915_request_add(rq);
3245 
3246 	engine->sched_engine->schedule(rq, &attr);
3247 
3248 	if (i915_request_wait(rq, 0, HZ / 2) < 0)
3249 		err = -ETIME;
3250 	i915_request_put(rq);
3251 
3252 	return err;
3253 }
3254 
3255 static int live_preempt_user(void *arg)
3256 {
3257 	struct intel_gt *gt = arg;
3258 	struct intel_engine_cs *engine;
3259 	struct i915_vma *global;
3260 	enum intel_engine_id id;
3261 	u32 *result;
3262 	int err = 0;
3263 
3264 	/*
3265 	 * In our other tests, we look at preemption in carefully
3266 	 * controlled conditions in the ringbuffer. Since most of the
3267 	 * time is spent in user batches, most of our preemptions naturally
3268 	 * occur there. We want to verify that when we preempt inside a batch
3269 	 * we continue on from the current instruction and do not roll back
3270 	 * to the start, or another earlier arbitration point.
3271 	 *
3272 	 * To verify this, we create a batch which is a mixture of
3273 	 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with
3274 	 * a few preempting contexts thrown into the mix, we look for any
3275 	 * repeated instructions (which show up as incorrect values).
3276 	 */
3277 
3278 	global = create_global(gt, 4096);
3279 	if (IS_ERR(global))
3280 		return PTR_ERR(global);
3281 
3282 	result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);
3283 	if (IS_ERR(result)) {
3284 		i915_vma_unpin_and_release(&global, 0);
3285 		return PTR_ERR(result);
3286 	}
3287 
3288 	for_each_engine(engine, gt, id) {
3289 		struct i915_request *client[3] = {};
3290 		struct igt_live_test t;
3291 		int i;
3292 
3293 		if (!intel_engine_has_preemption(engine))
3294 			continue;
3295 
3296 		if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS)
3297 			continue; /* we need per-context GPR */
3298 
3299 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
3300 			err = -EIO;
3301 			break;
3302 		}
3303 
3304 		memset(result, 0, 4096);
3305 
3306 		for (i = 0; i < ARRAY_SIZE(client); i++) {
3307 			struct i915_request *rq;
3308 
3309 			rq = create_gpr_client(engine, global,
3310 					       NUM_GPR * i * sizeof(u32));
3311 			if (IS_ERR(rq)) {
3312 				err = PTR_ERR(rq);
3313 				goto end_test;
3314 			}
3315 
3316 			client[i] = rq;
3317 		}
3318 
3319 		/* Continuously preempt the set of 3 running contexts */
3320 		for (i = 1; i <= NUM_GPR; i++) {
3321 			err = preempt_user(engine, global, i);
3322 			if (err)
3323 				goto end_test;
3324 		}
3325 
3326 		if (READ_ONCE(result[0]) != NUM_GPR) {
3327 			pr_err("%s: Failed to release semaphore\n",
3328 			       engine->name);
3329 			err = -EIO;
3330 			goto end_test;
3331 		}
3332 
3333 		for (i = 0; i < ARRAY_SIZE(client); i++) {
3334 			int gpr;
3335 
3336 			if (i915_request_wait(client[i], 0, HZ / 2) < 0) {
3337 				err = -ETIME;
3338 				goto end_test;
3339 			}
3340 
3341 			for (gpr = 1; gpr < NUM_GPR; gpr++) {
3342 				if (result[NUM_GPR * i + gpr] != 1) {
3343 					pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",
3344 					       engine->name,
3345 					       i, gpr, result[NUM_GPR * i + gpr]);
3346 					err = -EINVAL;
3347 					goto end_test;
3348 				}
3349 			}
3350 		}
3351 
3352 end_test:
3353 		for (i = 0; i < ARRAY_SIZE(client); i++) {
3354 			if (!client[i])
3355 				break;
3356 
3357 			i915_request_put(client[i]);
3358 		}
3359 
3360 		/* Flush the semaphores on error */
3361 		smp_store_mb(result[0], -1);
3362 		if (igt_live_test_end(&t))
3363 			err = -EIO;
3364 		if (err)
3365 			break;
3366 	}
3367 
3368 	i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);
3369 	return err;
3370 }
3371 
3372 static int live_preempt_timeout(void *arg)
3373 {
3374 	struct intel_gt *gt = arg;
3375 	struct i915_gem_context *ctx_hi, *ctx_lo;
3376 	struct igt_spinner spin_lo;
3377 	struct intel_engine_cs *engine;
3378 	enum intel_engine_id id;
3379 	int err = -ENOMEM;
3380 
3381 	/*
3382 	 * Check that we force preemption to occur by cancelling the previous
3383 	 * context if it refuses to yield the GPU.
3384 	 */
3385 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
3386 		return 0;
3387 
3388 	if (!intel_has_reset_engine(gt))
3389 		return 0;
3390 
3391 	ctx_hi = kernel_context(gt->i915, NULL);
3392 	if (!ctx_hi)
3393 		return -ENOMEM;
3394 	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
3395 
3396 	ctx_lo = kernel_context(gt->i915, NULL);
3397 	if (!ctx_lo)
3398 		goto err_ctx_hi;
3399 	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
3400 
3401 	if (igt_spinner_init(&spin_lo, gt))
3402 		goto err_ctx_lo;
3403 
3404 	for_each_engine(engine, gt, id) {
3405 		unsigned long saved_timeout;
3406 		struct i915_request *rq;
3407 
3408 		if (!intel_engine_has_preemption(engine))
3409 			continue;
3410 
3411 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
3412 					    MI_NOOP); /* preemption disabled */
3413 		if (IS_ERR(rq)) {
3414 			err = PTR_ERR(rq);
3415 			goto err_spin_lo;
3416 		}
3417 
3418 		i915_request_add(rq);
3419 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
3420 			intel_gt_set_wedged(gt);
3421 			err = -EIO;
3422 			goto err_spin_lo;
3423 		}
3424 
3425 		rq = igt_request_alloc(ctx_hi, engine);
3426 		if (IS_ERR(rq)) {
3427 			igt_spinner_end(&spin_lo);
3428 			err = PTR_ERR(rq);
3429 			goto err_spin_lo;
3430 		}
3431 
3432 		/* Flush the previous CS ack before changing timeouts */
3433 		while (READ_ONCE(engine->execlists.pending[0]))
3434 			cpu_relax();
3435 
3436 		saved_timeout = engine->props.preempt_timeout_ms;
3437 		engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
3438 
3439 		i915_request_get(rq);
3440 		i915_request_add(rq);
3441 
3442 		intel_engine_flush_submission(engine);
3443 		engine->props.preempt_timeout_ms = saved_timeout;
3444 
3445 		if (i915_request_wait(rq, 0, HZ / 10) < 0) {
3446 			intel_gt_set_wedged(gt);
3447 			i915_request_put(rq);
3448 			err = -ETIME;
3449 			goto err_spin_lo;
3450 		}
3451 
3452 		igt_spinner_end(&spin_lo);
3453 		i915_request_put(rq);
3454 	}
3455 
3456 	err = 0;
3457 err_spin_lo:
3458 	igt_spinner_fini(&spin_lo);
3459 err_ctx_lo:
3460 	kernel_context_close(ctx_lo);
3461 err_ctx_hi:
3462 	kernel_context_close(ctx_hi);
3463 	return err;
3464 }
3465 
3466 static int random_range(struct rnd_state *rnd, int min, int max)
3467 {
3468 	return i915_prandom_u32_max_state(max - min, rnd) + min;
3469 }
3470 
3471 static int random_priority(struct rnd_state *rnd)
3472 {
3473 	return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
3474 }
3475 
3476 struct preempt_smoke {
3477 	struct intel_gt *gt;
3478 	struct i915_gem_context **contexts;
3479 	struct intel_engine_cs *engine;
3480 	struct drm_i915_gem_object *batch;
3481 	unsigned int ncontext;
3482 	struct rnd_state prng;
3483 	unsigned long count;
3484 };
3485 
3486 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
3487 {
3488 	return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
3489 							  &smoke->prng)];
3490 }
3491 
3492 static int smoke_submit(struct preempt_smoke *smoke,
3493 			struct i915_gem_context *ctx, int prio,
3494 			struct drm_i915_gem_object *batch)
3495 {
3496 	struct i915_request *rq;
3497 	struct i915_vma *vma = NULL;
3498 	int err = 0;
3499 
3500 	if (batch) {
3501 		struct i915_address_space *vm;
3502 
3503 		vm = i915_gem_context_get_eb_vm(ctx);
3504 		vma = i915_vma_instance(batch, vm, NULL);
3505 		i915_vm_put(vm);
3506 		if (IS_ERR(vma))
3507 			return PTR_ERR(vma);
3508 
3509 		err = i915_vma_pin(vma, 0, 0, PIN_USER);
3510 		if (err)
3511 			return err;
3512 	}
3513 
3514 	ctx->sched.priority = prio;
3515 
3516 	rq = igt_request_alloc(ctx, smoke->engine);
3517 	if (IS_ERR(rq)) {
3518 		err = PTR_ERR(rq);
3519 		goto unpin;
3520 	}
3521 
3522 	if (vma) {
3523 		i915_vma_lock(vma);
3524 		err = i915_request_await_object(rq, vma->obj, false);
3525 		if (!err)
3526 			err = i915_vma_move_to_active(vma, rq, 0);
3527 		if (!err)
3528 			err = rq->engine->emit_bb_start(rq,
3529 							vma->node.start,
3530 							PAGE_SIZE, 0);
3531 		i915_vma_unlock(vma);
3532 	}
3533 
3534 	i915_request_add(rq);
3535 
3536 unpin:
3537 	if (vma)
3538 		i915_vma_unpin(vma);
3539 
3540 	return err;
3541 }
3542 
3543 static int smoke_crescendo_thread(void *arg)
3544 {
3545 	struct preempt_smoke *smoke = arg;
3546 	IGT_TIMEOUT(end_time);
3547 	unsigned long count;
3548 
3549 	count = 0;
3550 	do {
3551 		struct i915_gem_context *ctx = smoke_context(smoke);
3552 		int err;
3553 
3554 		err = smoke_submit(smoke,
3555 				   ctx, count % I915_PRIORITY_MAX,
3556 				   smoke->batch);
3557 		if (err)
3558 			return err;
3559 
3560 		count++;
3561 	} while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3562 
3563 	smoke->count = count;
3564 	return 0;
3565 }
3566 
3567 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
3568 #define BATCH BIT(0)
3569 {
3570 	struct task_struct *tsk[I915_NUM_ENGINES] = {};
3571 	struct preempt_smoke *arg;
3572 	struct intel_engine_cs *engine;
3573 	enum intel_engine_id id;
3574 	unsigned long count;
3575 	int err = 0;
3576 
3577 	arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL);
3578 	if (!arg)
3579 		return -ENOMEM;
3580 
3581 	for_each_engine(engine, smoke->gt, id) {
3582 		arg[id] = *smoke;
3583 		arg[id].engine = engine;
3584 		if (!(flags & BATCH))
3585 			arg[id].batch = NULL;
3586 		arg[id].count = 0;
3587 
3588 		tsk[id] = kthread_run(smoke_crescendo_thread, arg,
3589 				      "igt/smoke:%d", id);
3590 		if (IS_ERR(tsk[id])) {
3591 			err = PTR_ERR(tsk[id]);
3592 			break;
3593 		}
3594 		get_task_struct(tsk[id]);
3595 	}
3596 
3597 	yield(); /* start all threads before we kthread_stop() */
3598 
3599 	count = 0;
3600 	for_each_engine(engine, smoke->gt, id) {
3601 		int status;
3602 
3603 		if (IS_ERR_OR_NULL(tsk[id]))
3604 			continue;
3605 
3606 		status = kthread_stop(tsk[id]);
3607 		if (status && !err)
3608 			err = status;
3609 
3610 		count += arg[id].count;
3611 
3612 		put_task_struct(tsk[id]);
3613 	}
3614 
3615 	pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3616 		count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3617 
3618 	kfree(arg);
3619 	return 0;
3620 }
3621 
3622 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
3623 {
3624 	enum intel_engine_id id;
3625 	IGT_TIMEOUT(end_time);
3626 	unsigned long count;
3627 
3628 	count = 0;
3629 	do {
3630 		for_each_engine(smoke->engine, smoke->gt, id) {
3631 			struct i915_gem_context *ctx = smoke_context(smoke);
3632 			int err;
3633 
3634 			err = smoke_submit(smoke,
3635 					   ctx, random_priority(&smoke->prng),
3636 					   flags & BATCH ? smoke->batch : NULL);
3637 			if (err)
3638 				return err;
3639 
3640 			count++;
3641 		}
3642 	} while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3643 
3644 	pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3645 		count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3646 	return 0;
3647 }
3648 
3649 static int live_preempt_smoke(void *arg)
3650 {
3651 	struct preempt_smoke smoke = {
3652 		.gt = arg,
3653 		.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3654 		.ncontext = 256,
3655 	};
3656 	const unsigned int phase[] = { 0, BATCH };
3657 	struct igt_live_test t;
3658 	int err = -ENOMEM;
3659 	u32 *cs;
3660 	int n;
3661 
3662 	smoke.contexts = kmalloc_array(smoke.ncontext,
3663 				       sizeof(*smoke.contexts),
3664 				       GFP_KERNEL);
3665 	if (!smoke.contexts)
3666 		return -ENOMEM;
3667 
3668 	smoke.batch =
3669 		i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
3670 	if (IS_ERR(smoke.batch)) {
3671 		err = PTR_ERR(smoke.batch);
3672 		goto err_free;
3673 	}
3674 
3675 	cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);
3676 	if (IS_ERR(cs)) {
3677 		err = PTR_ERR(cs);
3678 		goto err_batch;
3679 	}
3680 	for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3681 		cs[n] = MI_ARB_CHECK;
3682 	cs[n] = MI_BATCH_BUFFER_END;
3683 	i915_gem_object_flush_map(smoke.batch);
3684 	i915_gem_object_unpin_map(smoke.batch);
3685 
3686 	if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
3687 		err = -EIO;
3688 		goto err_batch;
3689 	}
3690 
3691 	for (n = 0; n < smoke.ncontext; n++) {
3692 		smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL);
3693 		if (!smoke.contexts[n])
3694 			goto err_ctx;
3695 	}
3696 
3697 	for (n = 0; n < ARRAY_SIZE(phase); n++) {
3698 		err = smoke_crescendo(&smoke, phase[n]);
3699 		if (err)
3700 			goto err_ctx;
3701 
3702 		err = smoke_random(&smoke, phase[n]);
3703 		if (err)
3704 			goto err_ctx;
3705 	}
3706 
3707 err_ctx:
3708 	if (igt_live_test_end(&t))
3709 		err = -EIO;
3710 
3711 	for (n = 0; n < smoke.ncontext; n++) {
3712 		if (!smoke.contexts[n])
3713 			break;
3714 		kernel_context_close(smoke.contexts[n]);
3715 	}
3716 
3717 err_batch:
3718 	i915_gem_object_put(smoke.batch);
3719 err_free:
3720 	kfree(smoke.contexts);
3721 
3722 	return err;
3723 }
3724 
3725 static int nop_virtual_engine(struct intel_gt *gt,
3726 			      struct intel_engine_cs **siblings,
3727 			      unsigned int nsibling,
3728 			      unsigned int nctx,
3729 			      unsigned int flags)
3730 #define CHAIN BIT(0)
3731 {
3732 	IGT_TIMEOUT(end_time);
3733 	struct i915_request *request[16] = {};
3734 	struct intel_context *ve[16];
3735 	unsigned long n, prime, nc;
3736 	struct igt_live_test t;
3737 	ktime_t times[2] = {};
3738 	int err;
3739 
3740 	GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
3741 
3742 	for (n = 0; n < nctx; n++) {
3743 		ve[n] = intel_engine_create_virtual(siblings, nsibling, 0);
3744 		if (IS_ERR(ve[n])) {
3745 			err = PTR_ERR(ve[n]);
3746 			nctx = n;
3747 			goto out;
3748 		}
3749 
3750 		err = intel_context_pin(ve[n]);
3751 		if (err) {
3752 			intel_context_put(ve[n]);
3753 			nctx = n;
3754 			goto out;
3755 		}
3756 	}
3757 
3758 	err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
3759 	if (err)
3760 		goto out;
3761 
3762 	for_each_prime_number_from(prime, 1, 8192) {
3763 		times[1] = ktime_get_raw();
3764 
3765 		if (flags & CHAIN) {
3766 			for (nc = 0; nc < nctx; nc++) {
3767 				for (n = 0; n < prime; n++) {
3768 					struct i915_request *rq;
3769 
3770 					rq = i915_request_create(ve[nc]);
3771 					if (IS_ERR(rq)) {
3772 						err = PTR_ERR(rq);
3773 						goto out;
3774 					}
3775 
3776 					if (request[nc])
3777 						i915_request_put(request[nc]);
3778 					request[nc] = i915_request_get(rq);
3779 					i915_request_add(rq);
3780 				}
3781 			}
3782 		} else {
3783 			for (n = 0; n < prime; n++) {
3784 				for (nc = 0; nc < nctx; nc++) {
3785 					struct i915_request *rq;
3786 
3787 					rq = i915_request_create(ve[nc]);
3788 					if (IS_ERR(rq)) {
3789 						err = PTR_ERR(rq);
3790 						goto out;
3791 					}
3792 
3793 					if (request[nc])
3794 						i915_request_put(request[nc]);
3795 					request[nc] = i915_request_get(rq);
3796 					i915_request_add(rq);
3797 				}
3798 			}
3799 		}
3800 
3801 		for (nc = 0; nc < nctx; nc++) {
3802 			if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
3803 				pr_err("%s(%s): wait for %llx:%lld timed out\n",
3804 				       __func__, ve[0]->engine->name,
3805 				       request[nc]->fence.context,
3806 				       request[nc]->fence.seqno);
3807 
3808 				GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3809 					  __func__, ve[0]->engine->name,
3810 					  request[nc]->fence.context,
3811 					  request[nc]->fence.seqno);
3812 				GEM_TRACE_DUMP();
3813 				intel_gt_set_wedged(gt);
3814 				break;
3815 			}
3816 		}
3817 
3818 		times[1] = ktime_sub(ktime_get_raw(), times[1]);
3819 		if (prime == 1)
3820 			times[0] = times[1];
3821 
3822 		for (nc = 0; nc < nctx; nc++) {
3823 			i915_request_put(request[nc]);
3824 			request[nc] = NULL;
3825 		}
3826 
3827 		if (__igt_timeout(end_time, NULL))
3828 			break;
3829 	}
3830 
3831 	err = igt_live_test_end(&t);
3832 	if (err)
3833 		goto out;
3834 
3835 	pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3836 		nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3837 		prime, div64_u64(ktime_to_ns(times[1]), prime));
3838 
3839 out:
3840 	if (igt_flush_test(gt->i915))
3841 		err = -EIO;
3842 
3843 	for (nc = 0; nc < nctx; nc++) {
3844 		i915_request_put(request[nc]);
3845 		intel_context_unpin(ve[nc]);
3846 		intel_context_put(ve[nc]);
3847 	}
3848 	return err;
3849 }
3850 
3851 static unsigned int
3852 __select_siblings(struct intel_gt *gt,
3853 		  unsigned int class,
3854 		  struct intel_engine_cs **siblings,
3855 		  bool (*filter)(const struct intel_engine_cs *))
3856 {
3857 	unsigned int n = 0;
3858 	unsigned int inst;
3859 
3860 	for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3861 		if (!gt->engine_class[class][inst])
3862 			continue;
3863 
3864 		if (filter && !filter(gt->engine_class[class][inst]))
3865 			continue;
3866 
3867 		siblings[n++] = gt->engine_class[class][inst];
3868 	}
3869 
3870 	return n;
3871 }
3872 
3873 static unsigned int
3874 select_siblings(struct intel_gt *gt,
3875 		unsigned int class,
3876 		struct intel_engine_cs **siblings)
3877 {
3878 	return __select_siblings(gt, class, siblings, NULL);
3879 }
3880 
3881 static int live_virtual_engine(void *arg)
3882 {
3883 	struct intel_gt *gt = arg;
3884 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3885 	struct intel_engine_cs *engine;
3886 	enum intel_engine_id id;
3887 	unsigned int class;
3888 	int err;
3889 
3890 	if (intel_uc_uses_guc_submission(&gt->uc))
3891 		return 0;
3892 
3893 	for_each_engine(engine, gt, id) {
3894 		err = nop_virtual_engine(gt, &engine, 1, 1, 0);
3895 		if (err) {
3896 			pr_err("Failed to wrap engine %s: err=%d\n",
3897 			       engine->name, err);
3898 			return err;
3899 		}
3900 	}
3901 
3902 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3903 		int nsibling, n;
3904 
3905 		nsibling = select_siblings(gt, class, siblings);
3906 		if (nsibling < 2)
3907 			continue;
3908 
3909 		for (n = 1; n <= nsibling + 1; n++) {
3910 			err = nop_virtual_engine(gt, siblings, nsibling,
3911 						 n, 0);
3912 			if (err)
3913 				return err;
3914 		}
3915 
3916 		err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
3917 		if (err)
3918 			return err;
3919 	}
3920 
3921 	return 0;
3922 }
3923 
3924 static int mask_virtual_engine(struct intel_gt *gt,
3925 			       struct intel_engine_cs **siblings,
3926 			       unsigned int nsibling)
3927 {
3928 	struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
3929 	struct intel_context *ve;
3930 	struct igt_live_test t;
3931 	unsigned int n;
3932 	int err;
3933 
3934 	/*
3935 	 * Check that by setting the execution mask on a request, we can
3936 	 * restrict it to our desired engine within the virtual engine.
3937 	 */
3938 
3939 	ve = intel_engine_create_virtual(siblings, nsibling, 0);
3940 	if (IS_ERR(ve)) {
3941 		err = PTR_ERR(ve);
3942 		goto out_close;
3943 	}
3944 
3945 	err = intel_context_pin(ve);
3946 	if (err)
3947 		goto out_put;
3948 
3949 	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3950 	if (err)
3951 		goto out_unpin;
3952 
3953 	for (n = 0; n < nsibling; n++) {
3954 		request[n] = i915_request_create(ve);
3955 		if (IS_ERR(request[n])) {
3956 			err = PTR_ERR(request[n]);
3957 			nsibling = n;
3958 			goto out;
3959 		}
3960 
3961 		/* Reverse order as it's more likely to be unnatural */
3962 		request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3963 
3964 		i915_request_get(request[n]);
3965 		i915_request_add(request[n]);
3966 	}
3967 
3968 	for (n = 0; n < nsibling; n++) {
3969 		if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
3970 			pr_err("%s(%s): wait for %llx:%lld timed out\n",
3971 			       __func__, ve->engine->name,
3972 			       request[n]->fence.context,
3973 			       request[n]->fence.seqno);
3974 
3975 			GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3976 				  __func__, ve->engine->name,
3977 				  request[n]->fence.context,
3978 				  request[n]->fence.seqno);
3979 			GEM_TRACE_DUMP();
3980 			intel_gt_set_wedged(gt);
3981 			err = -EIO;
3982 			goto out;
3983 		}
3984 
3985 		if (request[n]->engine != siblings[nsibling - n - 1]) {
3986 			pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3987 			       request[n]->engine->name,
3988 			       siblings[nsibling - n - 1]->name);
3989 			err = -EINVAL;
3990 			goto out;
3991 		}
3992 	}
3993 
3994 	err = igt_live_test_end(&t);
3995 out:
3996 	if (igt_flush_test(gt->i915))
3997 		err = -EIO;
3998 
3999 	for (n = 0; n < nsibling; n++)
4000 		i915_request_put(request[n]);
4001 
4002 out_unpin:
4003 	intel_context_unpin(ve);
4004 out_put:
4005 	intel_context_put(ve);
4006 out_close:
4007 	return err;
4008 }
4009 
4010 static int live_virtual_mask(void *arg)
4011 {
4012 	struct intel_gt *gt = arg;
4013 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4014 	unsigned int class;
4015 	int err;
4016 
4017 	if (intel_uc_uses_guc_submission(&gt->uc))
4018 		return 0;
4019 
4020 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4021 		unsigned int nsibling;
4022 
4023 		nsibling = select_siblings(gt, class, siblings);
4024 		if (nsibling < 2)
4025 			continue;
4026 
4027 		err = mask_virtual_engine(gt, siblings, nsibling);
4028 		if (err)
4029 			return err;
4030 	}
4031 
4032 	return 0;
4033 }
4034 
4035 static int slicein_virtual_engine(struct intel_gt *gt,
4036 				  struct intel_engine_cs **siblings,
4037 				  unsigned int nsibling)
4038 {
4039 	const long timeout = slice_timeout(siblings[0]);
4040 	struct intel_context *ce;
4041 	struct i915_request *rq;
4042 	struct igt_spinner spin;
4043 	unsigned int n;
4044 	int err = 0;
4045 
4046 	/*
4047 	 * Virtual requests must take part in timeslicing on the target engines.
4048 	 */
4049 
4050 	if (igt_spinner_init(&spin, gt))
4051 		return -ENOMEM;
4052 
4053 	for (n = 0; n < nsibling; n++) {
4054 		ce = intel_context_create(siblings[n]);
4055 		if (IS_ERR(ce)) {
4056 			err = PTR_ERR(ce);
4057 			goto out;
4058 		}
4059 
4060 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4061 		intel_context_put(ce);
4062 		if (IS_ERR(rq)) {
4063 			err = PTR_ERR(rq);
4064 			goto out;
4065 		}
4066 
4067 		i915_request_add(rq);
4068 	}
4069 
4070 	ce = intel_engine_create_virtual(siblings, nsibling, 0);
4071 	if (IS_ERR(ce)) {
4072 		err = PTR_ERR(ce);
4073 		goto out;
4074 	}
4075 
4076 	rq = intel_context_create_request(ce);
4077 	intel_context_put(ce);
4078 	if (IS_ERR(rq)) {
4079 		err = PTR_ERR(rq);
4080 		goto out;
4081 	}
4082 
4083 	i915_request_get(rq);
4084 	i915_request_add(rq);
4085 	if (i915_request_wait(rq, 0, timeout) < 0) {
4086 		GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n",
4087 			      __func__, rq->engine->name);
4088 		GEM_TRACE_DUMP();
4089 		intel_gt_set_wedged(gt);
4090 		err = -EIO;
4091 	}
4092 	i915_request_put(rq);
4093 
4094 out:
4095 	igt_spinner_end(&spin);
4096 	if (igt_flush_test(gt->i915))
4097 		err = -EIO;
4098 	igt_spinner_fini(&spin);
4099 	return err;
4100 }
4101 
4102 static int sliceout_virtual_engine(struct intel_gt *gt,
4103 				   struct intel_engine_cs **siblings,
4104 				   unsigned int nsibling)
4105 {
4106 	const long timeout = slice_timeout(siblings[0]);
4107 	struct intel_context *ce;
4108 	struct i915_request *rq;
4109 	struct igt_spinner spin;
4110 	unsigned int n;
4111 	int err = 0;
4112 
4113 	/*
4114 	 * Virtual requests must allow others a fair timeslice.
4115 	 */
4116 
4117 	if (igt_spinner_init(&spin, gt))
4118 		return -ENOMEM;
4119 
4120 	/* XXX We do not handle oversubscription and fairness with normal rq */
4121 	for (n = 0; n < nsibling; n++) {
4122 		ce = intel_engine_create_virtual(siblings, nsibling, 0);
4123 		if (IS_ERR(ce)) {
4124 			err = PTR_ERR(ce);
4125 			goto out;
4126 		}
4127 
4128 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4129 		intel_context_put(ce);
4130 		if (IS_ERR(rq)) {
4131 			err = PTR_ERR(rq);
4132 			goto out;
4133 		}
4134 
4135 		i915_request_add(rq);
4136 	}
4137 
4138 	for (n = 0; !err && n < nsibling; n++) {
4139 		ce = intel_context_create(siblings[n]);
4140 		if (IS_ERR(ce)) {
4141 			err = PTR_ERR(ce);
4142 			goto out;
4143 		}
4144 
4145 		rq = intel_context_create_request(ce);
4146 		intel_context_put(ce);
4147 		if (IS_ERR(rq)) {
4148 			err = PTR_ERR(rq);
4149 			goto out;
4150 		}
4151 
4152 		i915_request_get(rq);
4153 		i915_request_add(rq);
4154 		if (i915_request_wait(rq, 0, timeout) < 0) {
4155 			GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n",
4156 				      __func__, siblings[n]->name);
4157 			GEM_TRACE_DUMP();
4158 			intel_gt_set_wedged(gt);
4159 			err = -EIO;
4160 		}
4161 		i915_request_put(rq);
4162 	}
4163 
4164 out:
4165 	igt_spinner_end(&spin);
4166 	if (igt_flush_test(gt->i915))
4167 		err = -EIO;
4168 	igt_spinner_fini(&spin);
4169 	return err;
4170 }
4171 
4172 static int live_virtual_slice(void *arg)
4173 {
4174 	struct intel_gt *gt = arg;
4175 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4176 	unsigned int class;
4177 	int err;
4178 
4179 	if (intel_uc_uses_guc_submission(&gt->uc))
4180 		return 0;
4181 
4182 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4183 		unsigned int nsibling;
4184 
4185 		nsibling = __select_siblings(gt, class, siblings,
4186 					     intel_engine_has_timeslices);
4187 		if (nsibling < 2)
4188 			continue;
4189 
4190 		err = slicein_virtual_engine(gt, siblings, nsibling);
4191 		if (err)
4192 			return err;
4193 
4194 		err = sliceout_virtual_engine(gt, siblings, nsibling);
4195 		if (err)
4196 			return err;
4197 	}
4198 
4199 	return 0;
4200 }
4201 
4202 static int preserved_virtual_engine(struct intel_gt *gt,
4203 				    struct intel_engine_cs **siblings,
4204 				    unsigned int nsibling)
4205 {
4206 	struct i915_request *last = NULL;
4207 	struct intel_context *ve;
4208 	struct i915_vma *scratch;
4209 	struct igt_live_test t;
4210 	unsigned int n;
4211 	int err = 0;
4212 	u32 *cs;
4213 
4214 	scratch =
4215 		__vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,
4216 						    PAGE_SIZE);
4217 	if (IS_ERR(scratch))
4218 		return PTR_ERR(scratch);
4219 
4220 	err = i915_vma_sync(scratch);
4221 	if (err)
4222 		goto out_scratch;
4223 
4224 	ve = intel_engine_create_virtual(siblings, nsibling, 0);
4225 	if (IS_ERR(ve)) {
4226 		err = PTR_ERR(ve);
4227 		goto out_scratch;
4228 	}
4229 
4230 	err = intel_context_pin(ve);
4231 	if (err)
4232 		goto out_put;
4233 
4234 	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
4235 	if (err)
4236 		goto out_unpin;
4237 
4238 	for (n = 0; n < NUM_GPR_DW; n++) {
4239 		struct intel_engine_cs *engine = siblings[n % nsibling];
4240 		struct i915_request *rq;
4241 
4242 		rq = i915_request_create(ve);
4243 		if (IS_ERR(rq)) {
4244 			err = PTR_ERR(rq);
4245 			goto out_end;
4246 		}
4247 
4248 		i915_request_put(last);
4249 		last = i915_request_get(rq);
4250 
4251 		cs = intel_ring_begin(rq, 8);
4252 		if (IS_ERR(cs)) {
4253 			i915_request_add(rq);
4254 			err = PTR_ERR(cs);
4255 			goto out_end;
4256 		}
4257 
4258 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4259 		*cs++ = CS_GPR(engine, n);
4260 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4261 		*cs++ = 0;
4262 
4263 		*cs++ = MI_LOAD_REGISTER_IMM(1);
4264 		*cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
4265 		*cs++ = n + 1;
4266 
4267 		*cs++ = MI_NOOP;
4268 		intel_ring_advance(rq, cs);
4269 
4270 		/* Restrict this request to run on a particular engine */
4271 		rq->execution_mask = engine->mask;
4272 		i915_request_add(rq);
4273 	}
4274 
4275 	if (i915_request_wait(last, 0, HZ / 5) < 0) {
4276 		err = -ETIME;
4277 		goto out_end;
4278 	}
4279 
4280 	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
4281 	if (IS_ERR(cs)) {
4282 		err = PTR_ERR(cs);
4283 		goto out_end;
4284 	}
4285 
4286 	for (n = 0; n < NUM_GPR_DW; n++) {
4287 		if (cs[n] != n) {
4288 			pr_err("Incorrect value[%d] found for GPR[%d]\n",
4289 			       cs[n], n);
4290 			err = -EINVAL;
4291 			break;
4292 		}
4293 	}
4294 
4295 	i915_gem_object_unpin_map(scratch->obj);
4296 
4297 out_end:
4298 	if (igt_live_test_end(&t))
4299 		err = -EIO;
4300 	i915_request_put(last);
4301 out_unpin:
4302 	intel_context_unpin(ve);
4303 out_put:
4304 	intel_context_put(ve);
4305 out_scratch:
4306 	i915_vma_unpin_and_release(&scratch, 0);
4307 	return err;
4308 }
4309 
4310 static int live_virtual_preserved(void *arg)
4311 {
4312 	struct intel_gt *gt = arg;
4313 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4314 	unsigned int class;
4315 
4316 	/*
4317 	 * Check that the context image retains non-privileged (user) registers
4318 	 * from one engine to the next. For this we check that the CS_GPR
4319 	 * are preserved.
4320 	 */
4321 
4322 	if (intel_uc_uses_guc_submission(&gt->uc))
4323 		return 0;
4324 
4325 	/* As we use CS_GPR we cannot run before they existed on all engines. */
4326 	if (GRAPHICS_VER(gt->i915) < 9)
4327 		return 0;
4328 
4329 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4330 		int nsibling, err;
4331 
4332 		nsibling = select_siblings(gt, class, siblings);
4333 		if (nsibling < 2)
4334 			continue;
4335 
4336 		err = preserved_virtual_engine(gt, siblings, nsibling);
4337 		if (err)
4338 			return err;
4339 	}
4340 
4341 	return 0;
4342 }
4343 
4344 static int reset_virtual_engine(struct intel_gt *gt,
4345 				struct intel_engine_cs **siblings,
4346 				unsigned int nsibling)
4347 {
4348 	struct intel_engine_cs *engine;
4349 	struct intel_context *ve;
4350 	struct igt_spinner spin;
4351 	struct i915_request *rq;
4352 	unsigned int n;
4353 	int err = 0;
4354 
4355 	/*
4356 	 * In order to support offline error capture for fast preempt reset,
4357 	 * we need to decouple the guilty request and ensure that it and its
4358 	 * descendents are not executed while the capture is in progress.
4359 	 */
4360 
4361 	if (igt_spinner_init(&spin, gt))
4362 		return -ENOMEM;
4363 
4364 	ve = intel_engine_create_virtual(siblings, nsibling, 0);
4365 	if (IS_ERR(ve)) {
4366 		err = PTR_ERR(ve);
4367 		goto out_spin;
4368 	}
4369 
4370 	for (n = 0; n < nsibling; n++)
4371 		st_engine_heartbeat_disable(siblings[n]);
4372 
4373 	rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
4374 	if (IS_ERR(rq)) {
4375 		err = PTR_ERR(rq);
4376 		goto out_heartbeat;
4377 	}
4378 	i915_request_add(rq);
4379 
4380 	if (!igt_wait_for_spinner(&spin, rq)) {
4381 		intel_gt_set_wedged(gt);
4382 		err = -ETIME;
4383 		goto out_heartbeat;
4384 	}
4385 
4386 	engine = rq->engine;
4387 	GEM_BUG_ON(engine == ve->engine);
4388 
4389 	/* Take ownership of the reset and tasklet */
4390 	err = engine_lock_reset_tasklet(engine);
4391 	if (err)
4392 		goto out_heartbeat;
4393 
4394 	engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
4395 	GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
4396 
4397 	/* Fake a preemption event; failed of course */
4398 	spin_lock_irq(&engine->sched_engine->lock);
4399 	__unwind_incomplete_requests(engine);
4400 	spin_unlock_irq(&engine->sched_engine->lock);
4401 	GEM_BUG_ON(rq->engine != engine);
4402 
4403 	/* Reset the engine while keeping our active request on hold */
4404 	execlists_hold(engine, rq);
4405 	GEM_BUG_ON(!i915_request_on_hold(rq));
4406 
4407 	__intel_engine_reset_bh(engine, NULL);
4408 	GEM_BUG_ON(rq->fence.error != -EIO);
4409 
4410 	/* Release our grasp on the engine, letting CS flow again */
4411 	engine_unlock_reset_tasklet(engine);
4412 
4413 	/* Check that we do not resubmit the held request */
4414 	i915_request_get(rq);
4415 	if (!i915_request_wait(rq, 0, HZ / 5)) {
4416 		pr_err("%s: on hold request completed!\n",
4417 		       engine->name);
4418 		intel_gt_set_wedged(gt);
4419 		err = -EIO;
4420 		goto out_rq;
4421 	}
4422 	GEM_BUG_ON(!i915_request_on_hold(rq));
4423 
4424 	/* But is resubmitted on release */
4425 	execlists_unhold(engine, rq);
4426 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4427 		pr_err("%s: held request did not complete!\n",
4428 		       engine->name);
4429 		intel_gt_set_wedged(gt);
4430 		err = -ETIME;
4431 	}
4432 
4433 out_rq:
4434 	i915_request_put(rq);
4435 out_heartbeat:
4436 	for (n = 0; n < nsibling; n++)
4437 		st_engine_heartbeat_enable(siblings[n]);
4438 
4439 	intel_context_put(ve);
4440 out_spin:
4441 	igt_spinner_fini(&spin);
4442 	return err;
4443 }
4444 
4445 static int live_virtual_reset(void *arg)
4446 {
4447 	struct intel_gt *gt = arg;
4448 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4449 	unsigned int class;
4450 
4451 	/*
4452 	 * Check that we handle a reset event within a virtual engine.
4453 	 * Only the physical engine is reset, but we have to check the flow
4454 	 * of the virtual requests around the reset, and make sure it is not
4455 	 * forgotten.
4456 	 */
4457 
4458 	if (intel_uc_uses_guc_submission(&gt->uc))
4459 		return 0;
4460 
4461 	if (!intel_has_reset_engine(gt))
4462 		return 0;
4463 
4464 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4465 		int nsibling, err;
4466 
4467 		nsibling = select_siblings(gt, class, siblings);
4468 		if (nsibling < 2)
4469 			continue;
4470 
4471 		err = reset_virtual_engine(gt, siblings, nsibling);
4472 		if (err)
4473 			return err;
4474 	}
4475 
4476 	return 0;
4477 }
4478 
4479 int intel_execlists_live_selftests(struct drm_i915_private *i915)
4480 {
4481 	static const struct i915_subtest tests[] = {
4482 		SUBTEST(live_sanitycheck),
4483 		SUBTEST(live_unlite_switch),
4484 		SUBTEST(live_unlite_preempt),
4485 		SUBTEST(live_unlite_ring),
4486 		SUBTEST(live_pin_rewind),
4487 		SUBTEST(live_hold_reset),
4488 		SUBTEST(live_error_interrupt),
4489 		SUBTEST(live_timeslice_preempt),
4490 		SUBTEST(live_timeslice_rewind),
4491 		SUBTEST(live_timeslice_queue),
4492 		SUBTEST(live_timeslice_nopreempt),
4493 		SUBTEST(live_busywait_preempt),
4494 		SUBTEST(live_preempt),
4495 		SUBTEST(live_late_preempt),
4496 		SUBTEST(live_nopreempt),
4497 		SUBTEST(live_preempt_cancel),
4498 		SUBTEST(live_suppress_self_preempt),
4499 		SUBTEST(live_chain_preempt),
4500 		SUBTEST(live_preempt_ring),
4501 		SUBTEST(live_preempt_gang),
4502 		SUBTEST(live_preempt_timeout),
4503 		SUBTEST(live_preempt_user),
4504 		SUBTEST(live_preempt_smoke),
4505 		SUBTEST(live_virtual_engine),
4506 		SUBTEST(live_virtual_mask),
4507 		SUBTEST(live_virtual_preserved),
4508 		SUBTEST(live_virtual_slice),
4509 		SUBTEST(live_virtual_reset),
4510 	};
4511 
4512 	if (to_gt(i915)->submission_method != INTEL_SUBMISSION_ELSP)
4513 		return 0;
4514 
4515 	if (intel_gt_is_wedged(to_gt(i915)))
4516 		return 0;
4517 
4518 	return intel_gt_live_subtests(tests, to_gt(i915));
4519 }
4520