1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include <linux/prime_numbers.h>
7 
8 #include "gem/i915_gem_internal.h"
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
12 #include "gt/selftest_engine_heartbeat.h"
13 
14 #include "i915_selftest.h"
15 #include "selftests/i915_random.h"
16 #include "selftests/igt_flush_test.h"
17 #include "selftests/igt_live_test.h"
18 #include "selftests/igt_spinner.h"
19 #include "selftests/lib_sw_fence.h"
20 
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
23 
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR 16
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27 
28 static bool is_active(struct i915_request *rq)
29 {
30 	if (i915_request_is_active(rq))
31 		return true;
32 
33 	if (i915_request_on_hold(rq))
34 		return true;
35 
36 	if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
37 		return true;
38 
39 	return false;
40 }
41 
42 static int wait_for_submit(struct intel_engine_cs *engine,
43 			   struct i915_request *rq,
44 			   unsigned long timeout)
45 {
46 	/* Ignore our own attempts to suppress excess tasklets */
47 	tasklet_hi_schedule(&engine->sched_engine->tasklet);
48 
49 	timeout += jiffies;
50 	do {
51 		bool done = time_after(jiffies, timeout);
52 
53 		if (i915_request_completed(rq)) /* that was quick! */
54 			return 0;
55 
56 		/* Wait until the HW has acknowleged the submission (or err) */
57 		intel_engine_flush_submission(engine);
58 		if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
59 			return 0;
60 
61 		if (done)
62 			return -ETIME;
63 
64 		cond_resched();
65 	} while (1);
66 }
67 
68 static int wait_for_reset(struct intel_engine_cs *engine,
69 			  struct i915_request *rq,
70 			  unsigned long timeout)
71 {
72 	timeout += jiffies;
73 
74 	do {
75 		cond_resched();
76 		intel_engine_flush_submission(engine);
77 
78 		if (READ_ONCE(engine->execlists.pending[0]))
79 			continue;
80 
81 		if (i915_request_completed(rq))
82 			break;
83 
84 		if (READ_ONCE(rq->fence.error))
85 			break;
86 	} while (time_before(jiffies, timeout));
87 
88 	if (rq->fence.error != -EIO) {
89 		pr_err("%s: hanging request %llx:%lld not reset\n",
90 		       engine->name,
91 		       rq->fence.context,
92 		       rq->fence.seqno);
93 		return -EINVAL;
94 	}
95 
96 	/* Give the request a jiffie to complete after flushing the worker */
97 	if (i915_request_wait(rq, 0,
98 			      max(0l, (long)(timeout - jiffies)) + 1) < 0) {
99 		pr_err("%s: hanging request %llx:%lld did not complete\n",
100 		       engine->name,
101 		       rq->fence.context,
102 		       rq->fence.seqno);
103 		return -ETIME;
104 	}
105 
106 	return 0;
107 }
108 
109 static int live_sanitycheck(void *arg)
110 {
111 	struct intel_gt *gt = arg;
112 	struct intel_engine_cs *engine;
113 	enum intel_engine_id id;
114 	struct igt_spinner spin;
115 	int err = 0;
116 
117 	if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
118 		return 0;
119 
120 	if (igt_spinner_init(&spin, gt))
121 		return -ENOMEM;
122 
123 	for_each_engine(engine, gt, id) {
124 		struct intel_context *ce;
125 		struct i915_request *rq;
126 
127 		ce = intel_context_create(engine);
128 		if (IS_ERR(ce)) {
129 			err = PTR_ERR(ce);
130 			break;
131 		}
132 
133 		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
134 		if (IS_ERR(rq)) {
135 			err = PTR_ERR(rq);
136 			goto out_ctx;
137 		}
138 
139 		i915_request_add(rq);
140 		if (!igt_wait_for_spinner(&spin, rq)) {
141 			GEM_TRACE("spinner failed to start\n");
142 			GEM_TRACE_DUMP();
143 			intel_gt_set_wedged(gt);
144 			err = -EIO;
145 			goto out_ctx;
146 		}
147 
148 		igt_spinner_end(&spin);
149 		if (igt_flush_test(gt->i915)) {
150 			err = -EIO;
151 			goto out_ctx;
152 		}
153 
154 out_ctx:
155 		intel_context_put(ce);
156 		if (err)
157 			break;
158 	}
159 
160 	igt_spinner_fini(&spin);
161 	return err;
162 }
163 
164 static int live_unlite_restore(struct intel_gt *gt, int prio)
165 {
166 	struct intel_engine_cs *engine;
167 	enum intel_engine_id id;
168 	struct igt_spinner spin;
169 	int err = -ENOMEM;
170 
171 	/*
172 	 * Check that we can correctly context switch between 2 instances
173 	 * on the same engine from the same parent context.
174 	 */
175 
176 	if (igt_spinner_init(&spin, gt))
177 		return err;
178 
179 	err = 0;
180 	for_each_engine(engine, gt, id) {
181 		struct intel_context *ce[2] = {};
182 		struct i915_request *rq[2];
183 		struct igt_live_test t;
184 		int n;
185 
186 		if (prio && !intel_engine_has_preemption(engine))
187 			continue;
188 
189 		if (!intel_engine_can_store_dword(engine))
190 			continue;
191 
192 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
193 			err = -EIO;
194 			break;
195 		}
196 		st_engine_heartbeat_disable(engine);
197 
198 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
199 			struct intel_context *tmp;
200 
201 			tmp = intel_context_create(engine);
202 			if (IS_ERR(tmp)) {
203 				err = PTR_ERR(tmp);
204 				goto err_ce;
205 			}
206 
207 			err = intel_context_pin(tmp);
208 			if (err) {
209 				intel_context_put(tmp);
210 				goto err_ce;
211 			}
212 
213 			/*
214 			 * Setup the pair of contexts such that if we
215 			 * lite-restore using the RING_TAIL from ce[1] it
216 			 * will execute garbage from ce[0]->ring.
217 			 */
218 			memset(tmp->ring->vaddr,
219 			       POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
220 			       tmp->ring->vma->size);
221 
222 			ce[n] = tmp;
223 		}
224 		GEM_BUG_ON(!ce[1]->ring->size);
225 		intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
226 		lrc_update_regs(ce[1], engine, ce[1]->ring->head);
227 
228 		rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
229 		if (IS_ERR(rq[0])) {
230 			err = PTR_ERR(rq[0]);
231 			goto err_ce;
232 		}
233 
234 		i915_request_get(rq[0]);
235 		i915_request_add(rq[0]);
236 		GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
237 
238 		if (!igt_wait_for_spinner(&spin, rq[0])) {
239 			i915_request_put(rq[0]);
240 			goto err_ce;
241 		}
242 
243 		rq[1] = i915_request_create(ce[1]);
244 		if (IS_ERR(rq[1])) {
245 			err = PTR_ERR(rq[1]);
246 			i915_request_put(rq[0]);
247 			goto err_ce;
248 		}
249 
250 		if (!prio) {
251 			/*
252 			 * Ensure we do the switch to ce[1] on completion.
253 			 *
254 			 * rq[0] is already submitted, so this should reduce
255 			 * to a no-op (a wait on a request on the same engine
256 			 * uses the submit fence, not the completion fence),
257 			 * but it will install a dependency on rq[1] for rq[0]
258 			 * that will prevent the pair being reordered by
259 			 * timeslicing.
260 			 */
261 			i915_request_await_dma_fence(rq[1], &rq[0]->fence);
262 		}
263 
264 		i915_request_get(rq[1]);
265 		i915_request_add(rq[1]);
266 		GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
267 		i915_request_put(rq[0]);
268 
269 		if (prio) {
270 			struct i915_sched_attr attr = {
271 				.priority = prio,
272 			};
273 
274 			/* Alternatively preempt the spinner with ce[1] */
275 			engine->sched_engine->schedule(rq[1], &attr);
276 		}
277 
278 		/* And switch back to ce[0] for good measure */
279 		rq[0] = i915_request_create(ce[0]);
280 		if (IS_ERR(rq[0])) {
281 			err = PTR_ERR(rq[0]);
282 			i915_request_put(rq[1]);
283 			goto err_ce;
284 		}
285 
286 		i915_request_await_dma_fence(rq[0], &rq[1]->fence);
287 		i915_request_get(rq[0]);
288 		i915_request_add(rq[0]);
289 		GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
290 		i915_request_put(rq[1]);
291 		i915_request_put(rq[0]);
292 
293 err_ce:
294 		intel_engine_flush_submission(engine);
295 		igt_spinner_end(&spin);
296 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
297 			if (IS_ERR_OR_NULL(ce[n]))
298 				break;
299 
300 			intel_context_unpin(ce[n]);
301 			intel_context_put(ce[n]);
302 		}
303 
304 		st_engine_heartbeat_enable(engine);
305 		if (igt_live_test_end(&t))
306 			err = -EIO;
307 		if (err)
308 			break;
309 	}
310 
311 	igt_spinner_fini(&spin);
312 	return err;
313 }
314 
315 static int live_unlite_switch(void *arg)
316 {
317 	return live_unlite_restore(arg, 0);
318 }
319 
320 static int live_unlite_preempt(void *arg)
321 {
322 	return live_unlite_restore(arg, I915_PRIORITY_MAX);
323 }
324 
325 static int live_unlite_ring(void *arg)
326 {
327 	struct intel_gt *gt = arg;
328 	struct intel_engine_cs *engine;
329 	struct igt_spinner spin;
330 	enum intel_engine_id id;
331 	int err = 0;
332 
333 	/*
334 	 * Setup a preemption event that will cause almost the entire ring
335 	 * to be unwound, potentially fooling our intel_ring_direction()
336 	 * into emitting a forward lite-restore instead of the rollback.
337 	 */
338 
339 	if (igt_spinner_init(&spin, gt))
340 		return -ENOMEM;
341 
342 	for_each_engine(engine, gt, id) {
343 		struct intel_context *ce[2] = {};
344 		struct i915_request *rq;
345 		struct igt_live_test t;
346 		int n;
347 
348 		if (!intel_engine_has_preemption(engine))
349 			continue;
350 
351 		if (!intel_engine_can_store_dword(engine))
352 			continue;
353 
354 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
355 			err = -EIO;
356 			break;
357 		}
358 		st_engine_heartbeat_disable(engine);
359 
360 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
361 			struct intel_context *tmp;
362 
363 			tmp = intel_context_create(engine);
364 			if (IS_ERR(tmp)) {
365 				err = PTR_ERR(tmp);
366 				goto err_ce;
367 			}
368 
369 			err = intel_context_pin(tmp);
370 			if (err) {
371 				intel_context_put(tmp);
372 				goto err_ce;
373 			}
374 
375 			memset32(tmp->ring->vaddr,
376 				 0xdeadbeef, /* trigger a hang if executed */
377 				 tmp->ring->vma->size / sizeof(u32));
378 
379 			ce[n] = tmp;
380 		}
381 
382 		/* Create max prio spinner, followed by N low prio nops */
383 		rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
384 		if (IS_ERR(rq)) {
385 			err = PTR_ERR(rq);
386 			goto err_ce;
387 		}
388 
389 		i915_request_get(rq);
390 		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
391 		i915_request_add(rq);
392 
393 		if (!igt_wait_for_spinner(&spin, rq)) {
394 			intel_gt_set_wedged(gt);
395 			i915_request_put(rq);
396 			err = -ETIME;
397 			goto err_ce;
398 		}
399 
400 		/* Fill the ring, until we will cause a wrap */
401 		n = 0;
402 		while (intel_ring_direction(ce[0]->ring,
403 					    rq->wa_tail,
404 					    ce[0]->ring->tail) <= 0) {
405 			struct i915_request *tmp;
406 
407 			tmp = intel_context_create_request(ce[0]);
408 			if (IS_ERR(tmp)) {
409 				err = PTR_ERR(tmp);
410 				i915_request_put(rq);
411 				goto err_ce;
412 			}
413 
414 			i915_request_add(tmp);
415 			intel_engine_flush_submission(engine);
416 			n++;
417 		}
418 		intel_engine_flush_submission(engine);
419 		pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
420 			 engine->name, n,
421 			 ce[0]->ring->size,
422 			 ce[0]->ring->tail,
423 			 ce[0]->ring->emit,
424 			 rq->tail);
425 		GEM_BUG_ON(intel_ring_direction(ce[0]->ring,
426 						rq->tail,
427 						ce[0]->ring->tail) <= 0);
428 		i915_request_put(rq);
429 
430 		/* Create a second ring to preempt the first ring after rq[0] */
431 		rq = intel_context_create_request(ce[1]);
432 		if (IS_ERR(rq)) {
433 			err = PTR_ERR(rq);
434 			goto err_ce;
435 		}
436 
437 		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
438 		i915_request_get(rq);
439 		i915_request_add(rq);
440 
441 		err = wait_for_submit(engine, rq, HZ / 2);
442 		i915_request_put(rq);
443 		if (err) {
444 			pr_err("%s: preemption request was not submitted\n",
445 			       engine->name);
446 			err = -ETIME;
447 		}
448 
449 		pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
450 			 engine->name,
451 			 ce[0]->ring->tail, ce[0]->ring->emit,
452 			 ce[1]->ring->tail, ce[1]->ring->emit);
453 
454 err_ce:
455 		intel_engine_flush_submission(engine);
456 		igt_spinner_end(&spin);
457 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
458 			if (IS_ERR_OR_NULL(ce[n]))
459 				break;
460 
461 			intel_context_unpin(ce[n]);
462 			intel_context_put(ce[n]);
463 		}
464 		st_engine_heartbeat_enable(engine);
465 		if (igt_live_test_end(&t))
466 			err = -EIO;
467 		if (err)
468 			break;
469 	}
470 
471 	igt_spinner_fini(&spin);
472 	return err;
473 }
474 
475 static int live_pin_rewind(void *arg)
476 {
477 	struct intel_gt *gt = arg;
478 	struct intel_engine_cs *engine;
479 	enum intel_engine_id id;
480 	int err = 0;
481 
482 	/*
483 	 * We have to be careful not to trust intel_ring too much, for example
484 	 * ring->head is updated upon retire which is out of sync with pinning
485 	 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,
486 	 * or else we risk writing an older, stale value.
487 	 *
488 	 * To simulate this, let's apply a bit of deliberate sabotague.
489 	 */
490 
491 	for_each_engine(engine, gt, id) {
492 		struct intel_context *ce;
493 		struct i915_request *rq;
494 		struct intel_ring *ring;
495 		struct igt_live_test t;
496 
497 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
498 			err = -EIO;
499 			break;
500 		}
501 
502 		ce = intel_context_create(engine);
503 		if (IS_ERR(ce)) {
504 			err = PTR_ERR(ce);
505 			break;
506 		}
507 
508 		err = intel_context_pin(ce);
509 		if (err) {
510 			intel_context_put(ce);
511 			break;
512 		}
513 
514 		/* Keep the context awake while we play games */
515 		err = i915_active_acquire(&ce->active);
516 		if (err) {
517 			intel_context_unpin(ce);
518 			intel_context_put(ce);
519 			break;
520 		}
521 		ring = ce->ring;
522 
523 		/* Poison the ring, and offset the next request from HEAD */
524 		memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
525 		ring->emit = ring->size / 2;
526 		ring->tail = ring->emit;
527 		GEM_BUG_ON(ring->head);
528 
529 		intel_context_unpin(ce);
530 
531 		/* Submit a simple nop request */
532 		GEM_BUG_ON(intel_context_is_pinned(ce));
533 		rq = intel_context_create_request(ce);
534 		i915_active_release(&ce->active); /* e.g. async retire */
535 		intel_context_put(ce);
536 		if (IS_ERR(rq)) {
537 			err = PTR_ERR(rq);
538 			break;
539 		}
540 		GEM_BUG_ON(!rq->head);
541 		i915_request_add(rq);
542 
543 		/* Expect not to hang! */
544 		if (igt_live_test_end(&t)) {
545 			err = -EIO;
546 			break;
547 		}
548 	}
549 
550 	return err;
551 }
552 
553 static int engine_lock_reset_tasklet(struct intel_engine_cs *engine)
554 {
555 	tasklet_disable(&engine->sched_engine->tasklet);
556 	local_bh_disable();
557 
558 	if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
559 			     &engine->gt->reset.flags)) {
560 		local_bh_enable();
561 		tasklet_enable(&engine->sched_engine->tasklet);
562 
563 		intel_gt_set_wedged(engine->gt);
564 		return -EBUSY;
565 	}
566 
567 	return 0;
568 }
569 
570 static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine)
571 {
572 	clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,
573 			      &engine->gt->reset.flags);
574 
575 	local_bh_enable();
576 	tasklet_enable(&engine->sched_engine->tasklet);
577 }
578 
579 static int live_hold_reset(void *arg)
580 {
581 	struct intel_gt *gt = arg;
582 	struct intel_engine_cs *engine;
583 	enum intel_engine_id id;
584 	struct igt_spinner spin;
585 	int err = 0;
586 
587 	/*
588 	 * In order to support offline error capture for fast preempt reset,
589 	 * we need to decouple the guilty request and ensure that it and its
590 	 * descendents are not executed while the capture is in progress.
591 	 */
592 
593 	if (!intel_has_reset_engine(gt))
594 		return 0;
595 
596 	if (igt_spinner_init(&spin, gt))
597 		return -ENOMEM;
598 
599 	for_each_engine(engine, gt, id) {
600 		struct intel_context *ce;
601 		struct i915_request *rq;
602 
603 		ce = intel_context_create(engine);
604 		if (IS_ERR(ce)) {
605 			err = PTR_ERR(ce);
606 			break;
607 		}
608 
609 		st_engine_heartbeat_disable(engine);
610 
611 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
612 		if (IS_ERR(rq)) {
613 			err = PTR_ERR(rq);
614 			goto out;
615 		}
616 		i915_request_add(rq);
617 
618 		if (!igt_wait_for_spinner(&spin, rq)) {
619 			intel_gt_set_wedged(gt);
620 			err = -ETIME;
621 			goto out;
622 		}
623 
624 		/* We have our request executing, now remove it and reset */
625 
626 		err = engine_lock_reset_tasklet(engine);
627 		if (err)
628 			goto out;
629 
630 		engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
631 		GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
632 
633 		i915_request_get(rq);
634 		execlists_hold(engine, rq);
635 		GEM_BUG_ON(!i915_request_on_hold(rq));
636 
637 		__intel_engine_reset_bh(engine, NULL);
638 		GEM_BUG_ON(rq->fence.error != -EIO);
639 
640 		engine_unlock_reset_tasklet(engine);
641 
642 		/* Check that we do not resubmit the held request */
643 		if (!i915_request_wait(rq, 0, HZ / 5)) {
644 			pr_err("%s: on hold request completed!\n",
645 			       engine->name);
646 			i915_request_put(rq);
647 			err = -EIO;
648 			goto out;
649 		}
650 		GEM_BUG_ON(!i915_request_on_hold(rq));
651 
652 		/* But is resubmitted on release */
653 		execlists_unhold(engine, rq);
654 		if (i915_request_wait(rq, 0, HZ / 5) < 0) {
655 			pr_err("%s: held request did not complete!\n",
656 			       engine->name);
657 			intel_gt_set_wedged(gt);
658 			err = -ETIME;
659 		}
660 		i915_request_put(rq);
661 
662 out:
663 		st_engine_heartbeat_enable(engine);
664 		intel_context_put(ce);
665 		if (err)
666 			break;
667 	}
668 
669 	igt_spinner_fini(&spin);
670 	return err;
671 }
672 
673 static const char *error_repr(int err)
674 {
675 	return err ? "bad" : "good";
676 }
677 
678 static int live_error_interrupt(void *arg)
679 {
680 	static const struct error_phase {
681 		enum { GOOD = 0, BAD = -EIO } error[2];
682 	} phases[] = {
683 		{ { BAD,  GOOD } },
684 		{ { BAD,  BAD  } },
685 		{ { BAD,  GOOD } },
686 		{ { GOOD, GOOD } }, /* sentinel */
687 	};
688 	struct intel_gt *gt = arg;
689 	struct intel_engine_cs *engine;
690 	enum intel_engine_id id;
691 
692 	/*
693 	 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning
694 	 * of invalid commands in user batches that will cause a GPU hang.
695 	 * This is a faster mechanism than using hangcheck/heartbeats, but
696 	 * only detects problems the HW knows about -- it will not warn when
697 	 * we kill the HW!
698 	 *
699 	 * To verify our detection and reset, we throw some invalid commands
700 	 * at the HW and wait for the interrupt.
701 	 */
702 
703 	if (!intel_has_reset_engine(gt))
704 		return 0;
705 
706 	for_each_engine(engine, gt, id) {
707 		const struct error_phase *p;
708 		int err = 0;
709 
710 		st_engine_heartbeat_disable(engine);
711 
712 		for (p = phases; p->error[0] != GOOD; p++) {
713 			struct i915_request *client[ARRAY_SIZE(phases->error)];
714 			u32 *cs;
715 			int i;
716 
717 			memset(client, 0, sizeof(*client));
718 			for (i = 0; i < ARRAY_SIZE(client); i++) {
719 				struct intel_context *ce;
720 				struct i915_request *rq;
721 
722 				ce = intel_context_create(engine);
723 				if (IS_ERR(ce)) {
724 					err = PTR_ERR(ce);
725 					goto out;
726 				}
727 
728 				rq = intel_context_create_request(ce);
729 				intel_context_put(ce);
730 				if (IS_ERR(rq)) {
731 					err = PTR_ERR(rq);
732 					goto out;
733 				}
734 
735 				if (rq->engine->emit_init_breadcrumb) {
736 					err = rq->engine->emit_init_breadcrumb(rq);
737 					if (err) {
738 						i915_request_add(rq);
739 						goto out;
740 					}
741 				}
742 
743 				cs = intel_ring_begin(rq, 2);
744 				if (IS_ERR(cs)) {
745 					i915_request_add(rq);
746 					err = PTR_ERR(cs);
747 					goto out;
748 				}
749 
750 				if (p->error[i]) {
751 					*cs++ = 0xdeadbeef;
752 					*cs++ = 0xdeadbeef;
753 				} else {
754 					*cs++ = MI_NOOP;
755 					*cs++ = MI_NOOP;
756 				}
757 
758 				client[i] = i915_request_get(rq);
759 				i915_request_add(rq);
760 			}
761 
762 			err = wait_for_submit(engine, client[0], HZ / 2);
763 			if (err) {
764 				pr_err("%s: first request did not start within time!\n",
765 				       engine->name);
766 				err = -ETIME;
767 				goto out;
768 			}
769 
770 			for (i = 0; i < ARRAY_SIZE(client); i++) {
771 				if (i915_request_wait(client[i], 0, HZ / 5) < 0)
772 					pr_debug("%s: %s request incomplete!\n",
773 						 engine->name,
774 						 error_repr(p->error[i]));
775 
776 				if (!i915_request_started(client[i])) {
777 					pr_err("%s: %s request not started!\n",
778 					       engine->name,
779 					       error_repr(p->error[i]));
780 					err = -ETIME;
781 					goto out;
782 				}
783 
784 				/* Kick the tasklet to process the error */
785 				intel_engine_flush_submission(engine);
786 				if (client[i]->fence.error != p->error[i]) {
787 					pr_err("%s: %s request (%s) with wrong error code: %d\n",
788 					       engine->name,
789 					       error_repr(p->error[i]),
790 					       i915_request_completed(client[i]) ? "completed" : "running",
791 					       client[i]->fence.error);
792 					err = -EINVAL;
793 					goto out;
794 				}
795 			}
796 
797 out:
798 			for (i = 0; i < ARRAY_SIZE(client); i++)
799 				if (client[i])
800 					i915_request_put(client[i]);
801 			if (err) {
802 				pr_err("%s: failed at phase[%zd] { %d, %d }\n",
803 				       engine->name, p - phases,
804 				       p->error[0], p->error[1]);
805 				break;
806 			}
807 		}
808 
809 		st_engine_heartbeat_enable(engine);
810 		if (err) {
811 			intel_gt_set_wedged(gt);
812 			return err;
813 		}
814 	}
815 
816 	return 0;
817 }
818 
819 static int
820 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
821 {
822 	u32 *cs;
823 
824 	cs = intel_ring_begin(rq, 10);
825 	if (IS_ERR(cs))
826 		return PTR_ERR(cs);
827 
828 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
829 
830 	*cs++ = MI_SEMAPHORE_WAIT |
831 		MI_SEMAPHORE_GLOBAL_GTT |
832 		MI_SEMAPHORE_POLL |
833 		MI_SEMAPHORE_SAD_NEQ_SDD;
834 	*cs++ = 0;
835 	*cs++ = i915_ggtt_offset(vma) + 4 * idx;
836 	*cs++ = 0;
837 
838 	if (idx > 0) {
839 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
840 		*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
841 		*cs++ = 0;
842 		*cs++ = 1;
843 	} else {
844 		*cs++ = MI_NOOP;
845 		*cs++ = MI_NOOP;
846 		*cs++ = MI_NOOP;
847 		*cs++ = MI_NOOP;
848 	}
849 
850 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
851 
852 	intel_ring_advance(rq, cs);
853 	return 0;
854 }
855 
856 static struct i915_request *
857 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
858 {
859 	struct intel_context *ce;
860 	struct i915_request *rq;
861 	int err;
862 
863 	ce = intel_context_create(engine);
864 	if (IS_ERR(ce))
865 		return ERR_CAST(ce);
866 
867 	rq = intel_context_create_request(ce);
868 	if (IS_ERR(rq))
869 		goto out_ce;
870 
871 	err = 0;
872 	if (rq->engine->emit_init_breadcrumb)
873 		err = rq->engine->emit_init_breadcrumb(rq);
874 	if (err == 0)
875 		err = emit_semaphore_chain(rq, vma, idx);
876 	if (err == 0)
877 		i915_request_get(rq);
878 	i915_request_add(rq);
879 	if (err)
880 		rq = ERR_PTR(err);
881 
882 out_ce:
883 	intel_context_put(ce);
884 	return rq;
885 }
886 
887 static int
888 release_queue(struct intel_engine_cs *engine,
889 	      struct i915_vma *vma,
890 	      int idx, int prio)
891 {
892 	struct i915_sched_attr attr = {
893 		.priority = prio,
894 	};
895 	struct i915_request *rq;
896 	u32 *cs;
897 
898 	rq = intel_engine_create_kernel_request(engine);
899 	if (IS_ERR(rq))
900 		return PTR_ERR(rq);
901 
902 	cs = intel_ring_begin(rq, 4);
903 	if (IS_ERR(cs)) {
904 		i915_request_add(rq);
905 		return PTR_ERR(cs);
906 	}
907 
908 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
909 	*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
910 	*cs++ = 0;
911 	*cs++ = 1;
912 
913 	intel_ring_advance(rq, cs);
914 
915 	i915_request_get(rq);
916 	i915_request_add(rq);
917 
918 	local_bh_disable();
919 	engine->sched_engine->schedule(rq, &attr);
920 	local_bh_enable(); /* kick tasklet */
921 
922 	i915_request_put(rq);
923 
924 	return 0;
925 }
926 
927 static int
928 slice_semaphore_queue(struct intel_engine_cs *outer,
929 		      struct i915_vma *vma,
930 		      int count)
931 {
932 	struct intel_engine_cs *engine;
933 	struct i915_request *head;
934 	enum intel_engine_id id;
935 	int err, i, n = 0;
936 
937 	head = semaphore_queue(outer, vma, n++);
938 	if (IS_ERR(head))
939 		return PTR_ERR(head);
940 
941 	for_each_engine(engine, outer->gt, id) {
942 		if (!intel_engine_has_preemption(engine))
943 			continue;
944 
945 		for (i = 0; i < count; i++) {
946 			struct i915_request *rq;
947 
948 			rq = semaphore_queue(engine, vma, n++);
949 			if (IS_ERR(rq)) {
950 				err = PTR_ERR(rq);
951 				goto out;
952 			}
953 
954 			i915_request_put(rq);
955 		}
956 	}
957 
958 	err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
959 	if (err)
960 		goto out;
961 
962 	if (i915_request_wait(head, 0,
963 			      2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) {
964 		pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n",
965 		       outer->name, count, n);
966 		GEM_TRACE_DUMP();
967 		intel_gt_set_wedged(outer->gt);
968 		err = -EIO;
969 	}
970 
971 out:
972 	i915_request_put(head);
973 	return err;
974 }
975 
976 static int live_timeslice_preempt(void *arg)
977 {
978 	struct intel_gt *gt = arg;
979 	struct drm_i915_gem_object *obj;
980 	struct intel_engine_cs *engine;
981 	enum intel_engine_id id;
982 	struct i915_vma *vma;
983 	void *vaddr;
984 	int err = 0;
985 
986 	/*
987 	 * If a request takes too long, we would like to give other users
988 	 * a fair go on the GPU. In particular, users may create batches
989 	 * that wait upon external input, where that input may even be
990 	 * supplied by another GPU job. To avoid blocking forever, we
991 	 * need to preempt the current task and replace it with another
992 	 * ready task.
993 	 */
994 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
995 		return 0;
996 
997 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
998 	if (IS_ERR(obj))
999 		return PTR_ERR(obj);
1000 
1001 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1002 	if (IS_ERR(vma)) {
1003 		err = PTR_ERR(vma);
1004 		goto err_obj;
1005 	}
1006 
1007 	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1008 	if (IS_ERR(vaddr)) {
1009 		err = PTR_ERR(vaddr);
1010 		goto err_obj;
1011 	}
1012 
1013 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1014 	if (err)
1015 		goto err_map;
1016 
1017 	err = i915_vma_sync(vma);
1018 	if (err)
1019 		goto err_pin;
1020 
1021 	for_each_engine(engine, gt, id) {
1022 		if (!intel_engine_has_preemption(engine))
1023 			continue;
1024 
1025 		memset(vaddr, 0, PAGE_SIZE);
1026 
1027 		st_engine_heartbeat_disable(engine);
1028 		err = slice_semaphore_queue(engine, vma, 5);
1029 		st_engine_heartbeat_enable(engine);
1030 		if (err)
1031 			goto err_pin;
1032 
1033 		if (igt_flush_test(gt->i915)) {
1034 			err = -EIO;
1035 			goto err_pin;
1036 		}
1037 	}
1038 
1039 err_pin:
1040 	i915_vma_unpin(vma);
1041 err_map:
1042 	i915_gem_object_unpin_map(obj);
1043 err_obj:
1044 	i915_gem_object_put(obj);
1045 	return err;
1046 }
1047 
1048 static struct i915_request *
1049 create_rewinder(struct intel_context *ce,
1050 		struct i915_request *wait,
1051 		void *slot, int idx)
1052 {
1053 	const u32 offset =
1054 		i915_ggtt_offset(ce->engine->status_page.vma) +
1055 		offset_in_page(slot);
1056 	struct i915_request *rq;
1057 	u32 *cs;
1058 	int err;
1059 
1060 	rq = intel_context_create_request(ce);
1061 	if (IS_ERR(rq))
1062 		return rq;
1063 
1064 	if (wait) {
1065 		err = i915_request_await_dma_fence(rq, &wait->fence);
1066 		if (err)
1067 			goto err;
1068 	}
1069 
1070 	cs = intel_ring_begin(rq, 14);
1071 	if (IS_ERR(cs)) {
1072 		err = PTR_ERR(cs);
1073 		goto err;
1074 	}
1075 
1076 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1077 	*cs++ = MI_NOOP;
1078 
1079 	*cs++ = MI_SEMAPHORE_WAIT |
1080 		MI_SEMAPHORE_GLOBAL_GTT |
1081 		MI_SEMAPHORE_POLL |
1082 		MI_SEMAPHORE_SAD_GTE_SDD;
1083 	*cs++ = idx;
1084 	*cs++ = offset;
1085 	*cs++ = 0;
1086 
1087 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1088 	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
1089 	*cs++ = offset + idx * sizeof(u32);
1090 	*cs++ = 0;
1091 
1092 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1093 	*cs++ = offset;
1094 	*cs++ = 0;
1095 	*cs++ = idx + 1;
1096 
1097 	intel_ring_advance(rq, cs);
1098 
1099 	err = 0;
1100 err:
1101 	i915_request_get(rq);
1102 	i915_request_add(rq);
1103 	if (err) {
1104 		i915_request_put(rq);
1105 		return ERR_PTR(err);
1106 	}
1107 
1108 	return rq;
1109 }
1110 
1111 static int live_timeslice_rewind(void *arg)
1112 {
1113 	struct intel_gt *gt = arg;
1114 	struct intel_engine_cs *engine;
1115 	enum intel_engine_id id;
1116 
1117 	/*
1118 	 * The usual presumption on timeslice expiration is that we replace
1119 	 * the active context with another. However, given a chain of
1120 	 * dependencies we may end up with replacing the context with itself,
1121 	 * but only a few of those requests, forcing us to rewind the
1122 	 * RING_TAIL of the original request.
1123 	 */
1124 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1125 		return 0;
1126 
1127 	for_each_engine(engine, gt, id) {
1128 		enum { A1, A2, B1 };
1129 		enum { X = 1, Z, Y };
1130 		struct i915_request *rq[3] = {};
1131 		struct intel_context *ce;
1132 		unsigned long timeslice;
1133 		int i, err = 0;
1134 		u32 *slot;
1135 
1136 		if (!intel_engine_has_timeslices(engine))
1137 			continue;
1138 
1139 		/*
1140 		 * A:rq1 -- semaphore wait, timestamp X
1141 		 * A:rq2 -- write timestamp Y
1142 		 *
1143 		 * B:rq1 [await A:rq1] -- write timestamp Z
1144 		 *
1145 		 * Force timeslice, release semaphore.
1146 		 *
1147 		 * Expect execution/evaluation order XZY
1148 		 */
1149 
1150 		st_engine_heartbeat_disable(engine);
1151 		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1152 
1153 		slot = memset32(engine->status_page.addr + 1000, 0, 4);
1154 
1155 		ce = intel_context_create(engine);
1156 		if (IS_ERR(ce)) {
1157 			err = PTR_ERR(ce);
1158 			goto err;
1159 		}
1160 
1161 		rq[A1] = create_rewinder(ce, NULL, slot, X);
1162 		if (IS_ERR(rq[A1])) {
1163 			intel_context_put(ce);
1164 			goto err;
1165 		}
1166 
1167 		rq[A2] = create_rewinder(ce, NULL, slot, Y);
1168 		intel_context_put(ce);
1169 		if (IS_ERR(rq[A2]))
1170 			goto err;
1171 
1172 		err = wait_for_submit(engine, rq[A2], HZ / 2);
1173 		if (err) {
1174 			pr_err("%s: failed to submit first context\n",
1175 			       engine->name);
1176 			goto err;
1177 		}
1178 
1179 		ce = intel_context_create(engine);
1180 		if (IS_ERR(ce)) {
1181 			err = PTR_ERR(ce);
1182 			goto err;
1183 		}
1184 
1185 		rq[B1] = create_rewinder(ce, rq[A1], slot, Z);
1186 		intel_context_put(ce);
1187 		if (IS_ERR(rq[2]))
1188 			goto err;
1189 
1190 		err = wait_for_submit(engine, rq[B1], HZ / 2);
1191 		if (err) {
1192 			pr_err("%s: failed to submit second context\n",
1193 			       engine->name);
1194 			goto err;
1195 		}
1196 
1197 		/* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */
1198 		ENGINE_TRACE(engine, "forcing tasklet for rewind\n");
1199 		while (i915_request_is_active(rq[A2])) { /* semaphore yield! */
1200 			/* Wait for the timeslice to kick in */
1201 			del_timer(&engine->execlists.timer);
1202 			tasklet_hi_schedule(&engine->sched_engine->tasklet);
1203 			intel_engine_flush_submission(engine);
1204 		}
1205 		/* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */
1206 		GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1207 		GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1208 		GEM_BUG_ON(i915_request_is_active(rq[A2]));
1209 
1210 		/* Release the hounds! */
1211 		slot[0] = 1;
1212 		wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */
1213 
1214 		for (i = 1; i <= 3; i++) {
1215 			unsigned long timeout = jiffies + HZ / 2;
1216 
1217 			while (!READ_ONCE(slot[i]) &&
1218 			       time_before(jiffies, timeout))
1219 				;
1220 
1221 			if (!time_before(jiffies, timeout)) {
1222 				pr_err("%s: rq[%d] timed out\n",
1223 				       engine->name, i - 1);
1224 				err = -ETIME;
1225 				goto err;
1226 			}
1227 
1228 			pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1229 		}
1230 
1231 		/* XZY: XZ < XY */
1232 		if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1233 			pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1234 			       engine->name,
1235 			       slot[Z] - slot[X],
1236 			       slot[Y] - slot[X]);
1237 			err = -EINVAL;
1238 		}
1239 
1240 err:
1241 		memset32(&slot[0], -1, 4);
1242 		wmb();
1243 
1244 		engine->props.timeslice_duration_ms = timeslice;
1245 		st_engine_heartbeat_enable(engine);
1246 		for (i = 0; i < 3; i++)
1247 			i915_request_put(rq[i]);
1248 		if (igt_flush_test(gt->i915))
1249 			err = -EIO;
1250 		if (err)
1251 			return err;
1252 	}
1253 
1254 	return 0;
1255 }
1256 
1257 static struct i915_request *nop_request(struct intel_engine_cs *engine)
1258 {
1259 	struct i915_request *rq;
1260 
1261 	rq = intel_engine_create_kernel_request(engine);
1262 	if (IS_ERR(rq))
1263 		return rq;
1264 
1265 	i915_request_get(rq);
1266 	i915_request_add(rq);
1267 
1268 	return rq;
1269 }
1270 
1271 static long slice_timeout(struct intel_engine_cs *engine)
1272 {
1273 	long timeout;
1274 
1275 	/* Enough time for a timeslice to kick in, and kick out */
1276 	timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine));
1277 
1278 	/* Enough time for the nop request to complete */
1279 	timeout += HZ / 5;
1280 
1281 	return timeout + 1;
1282 }
1283 
1284 static int live_timeslice_queue(void *arg)
1285 {
1286 	struct intel_gt *gt = arg;
1287 	struct drm_i915_gem_object *obj;
1288 	struct intel_engine_cs *engine;
1289 	enum intel_engine_id id;
1290 	struct i915_vma *vma;
1291 	void *vaddr;
1292 	int err = 0;
1293 
1294 	/*
1295 	 * Make sure that even if ELSP[0] and ELSP[1] are filled with
1296 	 * timeslicing between them disabled, we *do* enable timeslicing
1297 	 * if the queue demands it. (Normally, we do not submit if
1298 	 * ELSP[1] is already occupied, so must rely on timeslicing to
1299 	 * eject ELSP[0] in favour of the queue.)
1300 	 */
1301 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1302 		return 0;
1303 
1304 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1305 	if (IS_ERR(obj))
1306 		return PTR_ERR(obj);
1307 
1308 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1309 	if (IS_ERR(vma)) {
1310 		err = PTR_ERR(vma);
1311 		goto err_obj;
1312 	}
1313 
1314 	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1315 	if (IS_ERR(vaddr)) {
1316 		err = PTR_ERR(vaddr);
1317 		goto err_obj;
1318 	}
1319 
1320 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1321 	if (err)
1322 		goto err_map;
1323 
1324 	err = i915_vma_sync(vma);
1325 	if (err)
1326 		goto err_pin;
1327 
1328 	for_each_engine(engine, gt, id) {
1329 		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
1330 		struct i915_request *rq, *nop;
1331 
1332 		if (!intel_engine_has_preemption(engine))
1333 			continue;
1334 
1335 		st_engine_heartbeat_disable(engine);
1336 		memset(vaddr, 0, PAGE_SIZE);
1337 
1338 		/* ELSP[0]: semaphore wait */
1339 		rq = semaphore_queue(engine, vma, 0);
1340 		if (IS_ERR(rq)) {
1341 			err = PTR_ERR(rq);
1342 			goto err_heartbeat;
1343 		}
1344 		engine->sched_engine->schedule(rq, &attr);
1345 		err = wait_for_submit(engine, rq, HZ / 2);
1346 		if (err) {
1347 			pr_err("%s: Timed out trying to submit semaphores\n",
1348 			       engine->name);
1349 			goto err_rq;
1350 		}
1351 
1352 		/* ELSP[1]: nop request */
1353 		nop = nop_request(engine);
1354 		if (IS_ERR(nop)) {
1355 			err = PTR_ERR(nop);
1356 			goto err_rq;
1357 		}
1358 		err = wait_for_submit(engine, nop, HZ / 2);
1359 		i915_request_put(nop);
1360 		if (err) {
1361 			pr_err("%s: Timed out trying to submit nop\n",
1362 			       engine->name);
1363 			goto err_rq;
1364 		}
1365 
1366 		GEM_BUG_ON(i915_request_completed(rq));
1367 		GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1368 
1369 		/* Queue: semaphore signal, matching priority as semaphore */
1370 		err = release_queue(engine, vma, 1, effective_prio(rq));
1371 		if (err)
1372 			goto err_rq;
1373 
1374 		/* Wait until we ack the release_queue and start timeslicing */
1375 		do {
1376 			cond_resched();
1377 			intel_engine_flush_submission(engine);
1378 		} while (READ_ONCE(engine->execlists.pending[0]));
1379 
1380 		/* Timeslice every jiffy, so within 2 we should signal */
1381 		if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) {
1382 			struct drm_printer p =
1383 				drm_info_printer(gt->i915->drm.dev);
1384 
1385 			pr_err("%s: Failed to timeslice into queue\n",
1386 			       engine->name);
1387 			intel_engine_dump(engine, &p,
1388 					  "%s\n", engine->name);
1389 
1390 			memset(vaddr, 0xff, PAGE_SIZE);
1391 			err = -EIO;
1392 		}
1393 err_rq:
1394 		i915_request_put(rq);
1395 err_heartbeat:
1396 		st_engine_heartbeat_enable(engine);
1397 		if (err)
1398 			break;
1399 	}
1400 
1401 err_pin:
1402 	i915_vma_unpin(vma);
1403 err_map:
1404 	i915_gem_object_unpin_map(obj);
1405 err_obj:
1406 	i915_gem_object_put(obj);
1407 	return err;
1408 }
1409 
1410 static int live_timeslice_nopreempt(void *arg)
1411 {
1412 	struct intel_gt *gt = arg;
1413 	struct intel_engine_cs *engine;
1414 	enum intel_engine_id id;
1415 	struct igt_spinner spin;
1416 	int err = 0;
1417 
1418 	/*
1419 	 * We should not timeslice into a request that is marked with
1420 	 * I915_REQUEST_NOPREEMPT.
1421 	 */
1422 	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1423 		return 0;
1424 
1425 	if (igt_spinner_init(&spin, gt))
1426 		return -ENOMEM;
1427 
1428 	for_each_engine(engine, gt, id) {
1429 		struct intel_context *ce;
1430 		struct i915_request *rq;
1431 		unsigned long timeslice;
1432 
1433 		if (!intel_engine_has_preemption(engine))
1434 			continue;
1435 
1436 		ce = intel_context_create(engine);
1437 		if (IS_ERR(ce)) {
1438 			err = PTR_ERR(ce);
1439 			break;
1440 		}
1441 
1442 		st_engine_heartbeat_disable(engine);
1443 		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1444 
1445 		/* Create an unpreemptible spinner */
1446 
1447 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
1448 		intel_context_put(ce);
1449 		if (IS_ERR(rq)) {
1450 			err = PTR_ERR(rq);
1451 			goto out_heartbeat;
1452 		}
1453 
1454 		i915_request_get(rq);
1455 		i915_request_add(rq);
1456 
1457 		if (!igt_wait_for_spinner(&spin, rq)) {
1458 			i915_request_put(rq);
1459 			err = -ETIME;
1460 			goto out_spin;
1461 		}
1462 
1463 		set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
1464 		i915_request_put(rq);
1465 
1466 		/* Followed by a maximum priority barrier (heartbeat) */
1467 
1468 		ce = intel_context_create(engine);
1469 		if (IS_ERR(ce)) {
1470 			err = PTR_ERR(ce);
1471 			goto out_spin;
1472 		}
1473 
1474 		rq = intel_context_create_request(ce);
1475 		intel_context_put(ce);
1476 		if (IS_ERR(rq)) {
1477 			err = PTR_ERR(rq);
1478 			goto out_spin;
1479 		}
1480 
1481 		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1482 		i915_request_get(rq);
1483 		i915_request_add(rq);
1484 
1485 		/*
1486 		 * Wait until the barrier is in ELSP, and we know timeslicing
1487 		 * will have been activated.
1488 		 */
1489 		if (wait_for_submit(engine, rq, HZ / 2)) {
1490 			i915_request_put(rq);
1491 			err = -ETIME;
1492 			goto out_spin;
1493 		}
1494 
1495 		/*
1496 		 * Since the ELSP[0] request is unpreemptible, it should not
1497 		 * allow the maximum priority barrier through. Wait long
1498 		 * enough to see if it is timesliced in by mistake.
1499 		 */
1500 		if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) {
1501 			pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",
1502 			       engine->name);
1503 			err = -EINVAL;
1504 		}
1505 		i915_request_put(rq);
1506 
1507 out_spin:
1508 		igt_spinner_end(&spin);
1509 out_heartbeat:
1510 		xchg(&engine->props.timeslice_duration_ms, timeslice);
1511 		st_engine_heartbeat_enable(engine);
1512 		if (err)
1513 			break;
1514 
1515 		if (igt_flush_test(gt->i915)) {
1516 			err = -EIO;
1517 			break;
1518 		}
1519 	}
1520 
1521 	igt_spinner_fini(&spin);
1522 	return err;
1523 }
1524 
1525 static int live_busywait_preempt(void *arg)
1526 {
1527 	struct intel_gt *gt = arg;
1528 	struct i915_gem_context *ctx_hi, *ctx_lo;
1529 	struct intel_engine_cs *engine;
1530 	struct drm_i915_gem_object *obj;
1531 	struct i915_vma *vma;
1532 	enum intel_engine_id id;
1533 	u32 *map;
1534 	int err;
1535 
1536 	/*
1537 	 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
1538 	 * preempt the busywaits used to synchronise between rings.
1539 	 */
1540 
1541 	ctx_hi = kernel_context(gt->i915, NULL);
1542 	if (IS_ERR(ctx_hi))
1543 		return PTR_ERR(ctx_hi);
1544 
1545 	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1546 
1547 	ctx_lo = kernel_context(gt->i915, NULL);
1548 	if (IS_ERR(ctx_lo)) {
1549 		err = PTR_ERR(ctx_lo);
1550 		goto err_ctx_hi;
1551 	}
1552 
1553 	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1554 
1555 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1556 	if (IS_ERR(obj)) {
1557 		err = PTR_ERR(obj);
1558 		goto err_ctx_lo;
1559 	}
1560 
1561 	map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1562 	if (IS_ERR(map)) {
1563 		err = PTR_ERR(map);
1564 		goto err_obj;
1565 	}
1566 
1567 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1568 	if (IS_ERR(vma)) {
1569 		err = PTR_ERR(vma);
1570 		goto err_map;
1571 	}
1572 
1573 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1574 	if (err)
1575 		goto err_map;
1576 
1577 	err = i915_vma_sync(vma);
1578 	if (err)
1579 		goto err_vma;
1580 
1581 	for_each_engine(engine, gt, id) {
1582 		struct i915_request *lo, *hi;
1583 		struct igt_live_test t;
1584 		u32 *cs;
1585 
1586 		if (!intel_engine_has_preemption(engine))
1587 			continue;
1588 
1589 		if (!intel_engine_can_store_dword(engine))
1590 			continue;
1591 
1592 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1593 			err = -EIO;
1594 			goto err_vma;
1595 		}
1596 
1597 		/*
1598 		 * We create two requests. The low priority request
1599 		 * busywaits on a semaphore (inside the ringbuffer where
1600 		 * is should be preemptible) and the high priority requests
1601 		 * uses a MI_STORE_DWORD_IMM to update the semaphore value
1602 		 * allowing the first request to complete. If preemption
1603 		 * fails, we hang instead.
1604 		 */
1605 
1606 		lo = igt_request_alloc(ctx_lo, engine);
1607 		if (IS_ERR(lo)) {
1608 			err = PTR_ERR(lo);
1609 			goto err_vma;
1610 		}
1611 
1612 		cs = intel_ring_begin(lo, 8);
1613 		if (IS_ERR(cs)) {
1614 			err = PTR_ERR(cs);
1615 			i915_request_add(lo);
1616 			goto err_vma;
1617 		}
1618 
1619 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1620 		*cs++ = i915_ggtt_offset(vma);
1621 		*cs++ = 0;
1622 		*cs++ = 1;
1623 
1624 		/* XXX Do we need a flush + invalidate here? */
1625 
1626 		*cs++ = MI_SEMAPHORE_WAIT |
1627 			MI_SEMAPHORE_GLOBAL_GTT |
1628 			MI_SEMAPHORE_POLL |
1629 			MI_SEMAPHORE_SAD_EQ_SDD;
1630 		*cs++ = 0;
1631 		*cs++ = i915_ggtt_offset(vma);
1632 		*cs++ = 0;
1633 
1634 		intel_ring_advance(lo, cs);
1635 
1636 		i915_request_get(lo);
1637 		i915_request_add(lo);
1638 
1639 		if (wait_for(READ_ONCE(*map), 10)) {
1640 			i915_request_put(lo);
1641 			err = -ETIMEDOUT;
1642 			goto err_vma;
1643 		}
1644 
1645 		/* Low priority request should be busywaiting now */
1646 		if (i915_request_wait(lo, 0, 1) != -ETIME) {
1647 			i915_request_put(lo);
1648 			pr_err("%s: Busywaiting request did not!\n",
1649 			       engine->name);
1650 			err = -EIO;
1651 			goto err_vma;
1652 		}
1653 
1654 		hi = igt_request_alloc(ctx_hi, engine);
1655 		if (IS_ERR(hi)) {
1656 			err = PTR_ERR(hi);
1657 			i915_request_put(lo);
1658 			goto err_vma;
1659 		}
1660 
1661 		cs = intel_ring_begin(hi, 4);
1662 		if (IS_ERR(cs)) {
1663 			err = PTR_ERR(cs);
1664 			i915_request_add(hi);
1665 			i915_request_put(lo);
1666 			goto err_vma;
1667 		}
1668 
1669 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1670 		*cs++ = i915_ggtt_offset(vma);
1671 		*cs++ = 0;
1672 		*cs++ = 0;
1673 
1674 		intel_ring_advance(hi, cs);
1675 		i915_request_add(hi);
1676 
1677 		if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1678 			struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1679 
1680 			pr_err("%s: Failed to preempt semaphore busywait!\n",
1681 			       engine->name);
1682 
1683 			intel_engine_dump(engine, &p, "%s\n", engine->name);
1684 			GEM_TRACE_DUMP();
1685 
1686 			i915_request_put(lo);
1687 			intel_gt_set_wedged(gt);
1688 			err = -EIO;
1689 			goto err_vma;
1690 		}
1691 		GEM_BUG_ON(READ_ONCE(*map));
1692 		i915_request_put(lo);
1693 
1694 		if (igt_live_test_end(&t)) {
1695 			err = -EIO;
1696 			goto err_vma;
1697 		}
1698 	}
1699 
1700 	err = 0;
1701 err_vma:
1702 	i915_vma_unpin(vma);
1703 err_map:
1704 	i915_gem_object_unpin_map(obj);
1705 err_obj:
1706 	i915_gem_object_put(obj);
1707 err_ctx_lo:
1708 	kernel_context_close(ctx_lo);
1709 err_ctx_hi:
1710 	kernel_context_close(ctx_hi);
1711 	return err;
1712 }
1713 
1714 static struct i915_request *
1715 spinner_create_request(struct igt_spinner *spin,
1716 		       struct i915_gem_context *ctx,
1717 		       struct intel_engine_cs *engine,
1718 		       u32 arb)
1719 {
1720 	struct intel_context *ce;
1721 	struct i915_request *rq;
1722 
1723 	ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
1724 	if (IS_ERR(ce))
1725 		return ERR_CAST(ce);
1726 
1727 	rq = igt_spinner_create_request(spin, ce, arb);
1728 	intel_context_put(ce);
1729 	return rq;
1730 }
1731 
1732 static int live_preempt(void *arg)
1733 {
1734 	struct intel_gt *gt = arg;
1735 	struct i915_gem_context *ctx_hi, *ctx_lo;
1736 	struct igt_spinner spin_hi, spin_lo;
1737 	struct intel_engine_cs *engine;
1738 	enum intel_engine_id id;
1739 	int err = -ENOMEM;
1740 
1741 	ctx_hi = kernel_context(gt->i915, NULL);
1742 	if (!ctx_hi)
1743 		return -ENOMEM;
1744 	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1745 
1746 	ctx_lo = kernel_context(gt->i915, NULL);
1747 	if (!ctx_lo)
1748 		goto err_ctx_hi;
1749 	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1750 
1751 	if (igt_spinner_init(&spin_hi, gt))
1752 		goto err_ctx_lo;
1753 
1754 	if (igt_spinner_init(&spin_lo, gt))
1755 		goto err_spin_hi;
1756 
1757 	for_each_engine(engine, gt, id) {
1758 		struct igt_live_test t;
1759 		struct i915_request *rq;
1760 
1761 		if (!intel_engine_has_preemption(engine))
1762 			continue;
1763 
1764 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1765 			err = -EIO;
1766 			goto err_spin_lo;
1767 		}
1768 
1769 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1770 					    MI_ARB_CHECK);
1771 		if (IS_ERR(rq)) {
1772 			err = PTR_ERR(rq);
1773 			goto err_spin_lo;
1774 		}
1775 
1776 		i915_request_add(rq);
1777 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
1778 			GEM_TRACE("lo spinner failed to start\n");
1779 			GEM_TRACE_DUMP();
1780 			intel_gt_set_wedged(gt);
1781 			err = -EIO;
1782 			goto err_spin_lo;
1783 		}
1784 
1785 		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1786 					    MI_ARB_CHECK);
1787 		if (IS_ERR(rq)) {
1788 			igt_spinner_end(&spin_lo);
1789 			err = PTR_ERR(rq);
1790 			goto err_spin_lo;
1791 		}
1792 
1793 		i915_request_add(rq);
1794 		if (!igt_wait_for_spinner(&spin_hi, rq)) {
1795 			GEM_TRACE("hi spinner failed to start\n");
1796 			GEM_TRACE_DUMP();
1797 			intel_gt_set_wedged(gt);
1798 			err = -EIO;
1799 			goto err_spin_lo;
1800 		}
1801 
1802 		igt_spinner_end(&spin_hi);
1803 		igt_spinner_end(&spin_lo);
1804 
1805 		if (igt_live_test_end(&t)) {
1806 			err = -EIO;
1807 			goto err_spin_lo;
1808 		}
1809 	}
1810 
1811 	err = 0;
1812 err_spin_lo:
1813 	igt_spinner_fini(&spin_lo);
1814 err_spin_hi:
1815 	igt_spinner_fini(&spin_hi);
1816 err_ctx_lo:
1817 	kernel_context_close(ctx_lo);
1818 err_ctx_hi:
1819 	kernel_context_close(ctx_hi);
1820 	return err;
1821 }
1822 
1823 static int live_late_preempt(void *arg)
1824 {
1825 	struct intel_gt *gt = arg;
1826 	struct i915_gem_context *ctx_hi, *ctx_lo;
1827 	struct igt_spinner spin_hi, spin_lo;
1828 	struct intel_engine_cs *engine;
1829 	struct i915_sched_attr attr = {};
1830 	enum intel_engine_id id;
1831 	int err = -ENOMEM;
1832 
1833 	ctx_hi = kernel_context(gt->i915, NULL);
1834 	if (!ctx_hi)
1835 		return -ENOMEM;
1836 
1837 	ctx_lo = kernel_context(gt->i915, NULL);
1838 	if (!ctx_lo)
1839 		goto err_ctx_hi;
1840 
1841 	if (igt_spinner_init(&spin_hi, gt))
1842 		goto err_ctx_lo;
1843 
1844 	if (igt_spinner_init(&spin_lo, gt))
1845 		goto err_spin_hi;
1846 
1847 	/* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1848 	ctx_lo->sched.priority = 1;
1849 
1850 	for_each_engine(engine, gt, id) {
1851 		struct igt_live_test t;
1852 		struct i915_request *rq;
1853 
1854 		if (!intel_engine_has_preemption(engine))
1855 			continue;
1856 
1857 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1858 			err = -EIO;
1859 			goto err_spin_lo;
1860 		}
1861 
1862 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1863 					    MI_ARB_CHECK);
1864 		if (IS_ERR(rq)) {
1865 			err = PTR_ERR(rq);
1866 			goto err_spin_lo;
1867 		}
1868 
1869 		i915_request_add(rq);
1870 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
1871 			pr_err("First context failed to start\n");
1872 			goto err_wedged;
1873 		}
1874 
1875 		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1876 					    MI_NOOP);
1877 		if (IS_ERR(rq)) {
1878 			igt_spinner_end(&spin_lo);
1879 			err = PTR_ERR(rq);
1880 			goto err_spin_lo;
1881 		}
1882 
1883 		i915_request_add(rq);
1884 		if (igt_wait_for_spinner(&spin_hi, rq)) {
1885 			pr_err("Second context overtook first?\n");
1886 			goto err_wedged;
1887 		}
1888 
1889 		attr.priority = I915_PRIORITY_MAX;
1890 		engine->sched_engine->schedule(rq, &attr);
1891 
1892 		if (!igt_wait_for_spinner(&spin_hi, rq)) {
1893 			pr_err("High priority context failed to preempt the low priority context\n");
1894 			GEM_TRACE_DUMP();
1895 			goto err_wedged;
1896 		}
1897 
1898 		igt_spinner_end(&spin_hi);
1899 		igt_spinner_end(&spin_lo);
1900 
1901 		if (igt_live_test_end(&t)) {
1902 			err = -EIO;
1903 			goto err_spin_lo;
1904 		}
1905 	}
1906 
1907 	err = 0;
1908 err_spin_lo:
1909 	igt_spinner_fini(&spin_lo);
1910 err_spin_hi:
1911 	igt_spinner_fini(&spin_hi);
1912 err_ctx_lo:
1913 	kernel_context_close(ctx_lo);
1914 err_ctx_hi:
1915 	kernel_context_close(ctx_hi);
1916 	return err;
1917 
1918 err_wedged:
1919 	igt_spinner_end(&spin_hi);
1920 	igt_spinner_end(&spin_lo);
1921 	intel_gt_set_wedged(gt);
1922 	err = -EIO;
1923 	goto err_spin_lo;
1924 }
1925 
1926 struct preempt_client {
1927 	struct igt_spinner spin;
1928 	struct i915_gem_context *ctx;
1929 };
1930 
1931 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1932 {
1933 	c->ctx = kernel_context(gt->i915, NULL);
1934 	if (!c->ctx)
1935 		return -ENOMEM;
1936 
1937 	if (igt_spinner_init(&c->spin, gt))
1938 		goto err_ctx;
1939 
1940 	return 0;
1941 
1942 err_ctx:
1943 	kernel_context_close(c->ctx);
1944 	return -ENOMEM;
1945 }
1946 
1947 static void preempt_client_fini(struct preempt_client *c)
1948 {
1949 	igt_spinner_fini(&c->spin);
1950 	kernel_context_close(c->ctx);
1951 }
1952 
1953 static int live_nopreempt(void *arg)
1954 {
1955 	struct intel_gt *gt = arg;
1956 	struct intel_engine_cs *engine;
1957 	struct preempt_client a, b;
1958 	enum intel_engine_id id;
1959 	int err = -ENOMEM;
1960 
1961 	/*
1962 	 * Verify that we can disable preemption for an individual request
1963 	 * that may be being observed and not want to be interrupted.
1964 	 */
1965 
1966 	if (preempt_client_init(gt, &a))
1967 		return -ENOMEM;
1968 	if (preempt_client_init(gt, &b))
1969 		goto err_client_a;
1970 	b.ctx->sched.priority = I915_PRIORITY_MAX;
1971 
1972 	for_each_engine(engine, gt, id) {
1973 		struct i915_request *rq_a, *rq_b;
1974 
1975 		if (!intel_engine_has_preemption(engine))
1976 			continue;
1977 
1978 		engine->execlists.preempt_hang.count = 0;
1979 
1980 		rq_a = spinner_create_request(&a.spin,
1981 					      a.ctx, engine,
1982 					      MI_ARB_CHECK);
1983 		if (IS_ERR(rq_a)) {
1984 			err = PTR_ERR(rq_a);
1985 			goto err_client_b;
1986 		}
1987 
1988 		/* Low priority client, but unpreemptable! */
1989 		__set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
1990 
1991 		i915_request_add(rq_a);
1992 		if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1993 			pr_err("First client failed to start\n");
1994 			goto err_wedged;
1995 		}
1996 
1997 		rq_b = spinner_create_request(&b.spin,
1998 					      b.ctx, engine,
1999 					      MI_ARB_CHECK);
2000 		if (IS_ERR(rq_b)) {
2001 			err = PTR_ERR(rq_b);
2002 			goto err_client_b;
2003 		}
2004 
2005 		i915_request_add(rq_b);
2006 
2007 		/* B is much more important than A! (But A is unpreemptable.) */
2008 		GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
2009 
2010 		/* Wait long enough for preemption and timeslicing */
2011 		if (igt_wait_for_spinner(&b.spin, rq_b)) {
2012 			pr_err("Second client started too early!\n");
2013 			goto err_wedged;
2014 		}
2015 
2016 		igt_spinner_end(&a.spin);
2017 
2018 		if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2019 			pr_err("Second client failed to start\n");
2020 			goto err_wedged;
2021 		}
2022 
2023 		igt_spinner_end(&b.spin);
2024 
2025 		if (engine->execlists.preempt_hang.count) {
2026 			pr_err("Preemption recorded x%d; should have been suppressed!\n",
2027 			       engine->execlists.preempt_hang.count);
2028 			err = -EINVAL;
2029 			goto err_wedged;
2030 		}
2031 
2032 		if (igt_flush_test(gt->i915))
2033 			goto err_wedged;
2034 	}
2035 
2036 	err = 0;
2037 err_client_b:
2038 	preempt_client_fini(&b);
2039 err_client_a:
2040 	preempt_client_fini(&a);
2041 	return err;
2042 
2043 err_wedged:
2044 	igt_spinner_end(&b.spin);
2045 	igt_spinner_end(&a.spin);
2046 	intel_gt_set_wedged(gt);
2047 	err = -EIO;
2048 	goto err_client_b;
2049 }
2050 
2051 struct live_preempt_cancel {
2052 	struct intel_engine_cs *engine;
2053 	struct preempt_client a, b;
2054 };
2055 
2056 static int __cancel_active0(struct live_preempt_cancel *arg)
2057 {
2058 	struct i915_request *rq;
2059 	struct igt_live_test t;
2060 	int err;
2061 
2062 	/* Preempt cancel of ELSP0 */
2063 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2064 	if (igt_live_test_begin(&t, arg->engine->i915,
2065 				__func__, arg->engine->name))
2066 		return -EIO;
2067 
2068 	rq = spinner_create_request(&arg->a.spin,
2069 				    arg->a.ctx, arg->engine,
2070 				    MI_ARB_CHECK);
2071 	if (IS_ERR(rq))
2072 		return PTR_ERR(rq);
2073 
2074 	clear_bit(CONTEXT_BANNED, &rq->context->flags);
2075 	i915_request_get(rq);
2076 	i915_request_add(rq);
2077 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2078 		err = -EIO;
2079 		goto out;
2080 	}
2081 
2082 	intel_context_ban(rq->context, rq);
2083 	err = intel_engine_pulse(arg->engine);
2084 	if (err)
2085 		goto out;
2086 
2087 	err = wait_for_reset(arg->engine, rq, HZ / 2);
2088 	if (err) {
2089 		pr_err("Cancelled inflight0 request did not reset\n");
2090 		goto out;
2091 	}
2092 
2093 out:
2094 	i915_request_put(rq);
2095 	if (igt_live_test_end(&t))
2096 		err = -EIO;
2097 	return err;
2098 }
2099 
2100 static int __cancel_active1(struct live_preempt_cancel *arg)
2101 {
2102 	struct i915_request *rq[2] = {};
2103 	struct igt_live_test t;
2104 	int err;
2105 
2106 	/* Preempt cancel of ELSP1 */
2107 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2108 	if (igt_live_test_begin(&t, arg->engine->i915,
2109 				__func__, arg->engine->name))
2110 		return -EIO;
2111 
2112 	rq[0] = spinner_create_request(&arg->a.spin,
2113 				       arg->a.ctx, arg->engine,
2114 				       MI_NOOP); /* no preemption */
2115 	if (IS_ERR(rq[0]))
2116 		return PTR_ERR(rq[0]);
2117 
2118 	clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2119 	i915_request_get(rq[0]);
2120 	i915_request_add(rq[0]);
2121 	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2122 		err = -EIO;
2123 		goto out;
2124 	}
2125 
2126 	rq[1] = spinner_create_request(&arg->b.spin,
2127 				       arg->b.ctx, arg->engine,
2128 				       MI_ARB_CHECK);
2129 	if (IS_ERR(rq[1])) {
2130 		err = PTR_ERR(rq[1]);
2131 		goto out;
2132 	}
2133 
2134 	clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2135 	i915_request_get(rq[1]);
2136 	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2137 	i915_request_add(rq[1]);
2138 	if (err)
2139 		goto out;
2140 
2141 	intel_context_ban(rq[1]->context, rq[1]);
2142 	err = intel_engine_pulse(arg->engine);
2143 	if (err)
2144 		goto out;
2145 
2146 	igt_spinner_end(&arg->a.spin);
2147 	err = wait_for_reset(arg->engine, rq[1], HZ / 2);
2148 	if (err)
2149 		goto out;
2150 
2151 	if (rq[0]->fence.error != 0) {
2152 		pr_err("Normal inflight0 request did not complete\n");
2153 		err = -EINVAL;
2154 		goto out;
2155 	}
2156 
2157 	if (rq[1]->fence.error != -EIO) {
2158 		pr_err("Cancelled inflight1 request did not report -EIO\n");
2159 		err = -EINVAL;
2160 		goto out;
2161 	}
2162 
2163 out:
2164 	i915_request_put(rq[1]);
2165 	i915_request_put(rq[0]);
2166 	if (igt_live_test_end(&t))
2167 		err = -EIO;
2168 	return err;
2169 }
2170 
2171 static int __cancel_queued(struct live_preempt_cancel *arg)
2172 {
2173 	struct i915_request *rq[3] = {};
2174 	struct igt_live_test t;
2175 	int err;
2176 
2177 	/* Full ELSP and one in the wings */
2178 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2179 	if (igt_live_test_begin(&t, arg->engine->i915,
2180 				__func__, arg->engine->name))
2181 		return -EIO;
2182 
2183 	rq[0] = spinner_create_request(&arg->a.spin,
2184 				       arg->a.ctx, arg->engine,
2185 				       MI_ARB_CHECK);
2186 	if (IS_ERR(rq[0]))
2187 		return PTR_ERR(rq[0]);
2188 
2189 	clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2190 	i915_request_get(rq[0]);
2191 	i915_request_add(rq[0]);
2192 	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2193 		err = -EIO;
2194 		goto out;
2195 	}
2196 
2197 	rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
2198 	if (IS_ERR(rq[1])) {
2199 		err = PTR_ERR(rq[1]);
2200 		goto out;
2201 	}
2202 
2203 	clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2204 	i915_request_get(rq[1]);
2205 	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2206 	i915_request_add(rq[1]);
2207 	if (err)
2208 		goto out;
2209 
2210 	rq[2] = spinner_create_request(&arg->b.spin,
2211 				       arg->a.ctx, arg->engine,
2212 				       MI_ARB_CHECK);
2213 	if (IS_ERR(rq[2])) {
2214 		err = PTR_ERR(rq[2]);
2215 		goto out;
2216 	}
2217 
2218 	i915_request_get(rq[2]);
2219 	err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
2220 	i915_request_add(rq[2]);
2221 	if (err)
2222 		goto out;
2223 
2224 	intel_context_ban(rq[2]->context, rq[2]);
2225 	err = intel_engine_pulse(arg->engine);
2226 	if (err)
2227 		goto out;
2228 
2229 	err = wait_for_reset(arg->engine, rq[2], HZ / 2);
2230 	if (err)
2231 		goto out;
2232 
2233 	if (rq[0]->fence.error != -EIO) {
2234 		pr_err("Cancelled inflight0 request did not report -EIO\n");
2235 		err = -EINVAL;
2236 		goto out;
2237 	}
2238 
2239 	/*
2240 	 * The behavior between having semaphores and not is different. With
2241 	 * semaphores the subsequent request is on the hardware and not cancelled
2242 	 * while without the request is held in the driver and cancelled.
2243 	 */
2244 	if (intel_engine_has_semaphores(rq[1]->engine) &&
2245 	    rq[1]->fence.error != 0) {
2246 		pr_err("Normal inflight1 request did not complete\n");
2247 		err = -EINVAL;
2248 		goto out;
2249 	}
2250 
2251 	if (rq[2]->fence.error != -EIO) {
2252 		pr_err("Cancelled queued request did not report -EIO\n");
2253 		err = -EINVAL;
2254 		goto out;
2255 	}
2256 
2257 out:
2258 	i915_request_put(rq[2]);
2259 	i915_request_put(rq[1]);
2260 	i915_request_put(rq[0]);
2261 	if (igt_live_test_end(&t))
2262 		err = -EIO;
2263 	return err;
2264 }
2265 
2266 static int __cancel_hostile(struct live_preempt_cancel *arg)
2267 {
2268 	struct i915_request *rq;
2269 	int err;
2270 
2271 	/* Preempt cancel non-preemptible spinner in ELSP0 */
2272 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2273 		return 0;
2274 
2275 	if (!intel_has_reset_engine(arg->engine->gt))
2276 		return 0;
2277 
2278 	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2279 	rq = spinner_create_request(&arg->a.spin,
2280 				    arg->a.ctx, arg->engine,
2281 				    MI_NOOP); /* preemption disabled */
2282 	if (IS_ERR(rq))
2283 		return PTR_ERR(rq);
2284 
2285 	clear_bit(CONTEXT_BANNED, &rq->context->flags);
2286 	i915_request_get(rq);
2287 	i915_request_add(rq);
2288 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2289 		err = -EIO;
2290 		goto out;
2291 	}
2292 
2293 	intel_context_ban(rq->context, rq);
2294 	err = intel_engine_pulse(arg->engine); /* force reset */
2295 	if (err)
2296 		goto out;
2297 
2298 	err = wait_for_reset(arg->engine, rq, HZ / 2);
2299 	if (err) {
2300 		pr_err("Cancelled inflight0 request did not reset\n");
2301 		goto out;
2302 	}
2303 
2304 out:
2305 	i915_request_put(rq);
2306 	if (igt_flush_test(arg->engine->i915))
2307 		err = -EIO;
2308 	return err;
2309 }
2310 
2311 static void force_reset_timeout(struct intel_engine_cs *engine)
2312 {
2313 	engine->reset_timeout.probability = 999;
2314 	atomic_set(&engine->reset_timeout.times, -1);
2315 }
2316 
2317 static void cancel_reset_timeout(struct intel_engine_cs *engine)
2318 {
2319 	memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
2320 }
2321 
2322 static int __cancel_fail(struct live_preempt_cancel *arg)
2323 {
2324 	struct intel_engine_cs *engine = arg->engine;
2325 	struct i915_request *rq;
2326 	int err;
2327 
2328 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2329 		return 0;
2330 
2331 	if (!intel_has_reset_engine(engine->gt))
2332 		return 0;
2333 
2334 	GEM_TRACE("%s(%s)\n", __func__, engine->name);
2335 	rq = spinner_create_request(&arg->a.spin,
2336 				    arg->a.ctx, engine,
2337 				    MI_NOOP); /* preemption disabled */
2338 	if (IS_ERR(rq))
2339 		return PTR_ERR(rq);
2340 
2341 	clear_bit(CONTEXT_BANNED, &rq->context->flags);
2342 	i915_request_get(rq);
2343 	i915_request_add(rq);
2344 	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2345 		err = -EIO;
2346 		goto out;
2347 	}
2348 
2349 	intel_context_set_banned(rq->context);
2350 
2351 	err = intel_engine_pulse(engine);
2352 	if (err)
2353 		goto out;
2354 
2355 	force_reset_timeout(engine);
2356 
2357 	/* force preempt reset [failure] */
2358 	while (!engine->execlists.pending[0])
2359 		intel_engine_flush_submission(engine);
2360 	del_timer_sync(&engine->execlists.preempt);
2361 	intel_engine_flush_submission(engine);
2362 
2363 	cancel_reset_timeout(engine);
2364 
2365 	/* after failure, require heartbeats to reset device */
2366 	intel_engine_set_heartbeat(engine, 1);
2367 	err = wait_for_reset(engine, rq, HZ / 2);
2368 	intel_engine_set_heartbeat(engine,
2369 				   engine->defaults.heartbeat_interval_ms);
2370 	if (err) {
2371 		pr_err("Cancelled inflight0 request did not reset\n");
2372 		goto out;
2373 	}
2374 
2375 out:
2376 	i915_request_put(rq);
2377 	if (igt_flush_test(engine->i915))
2378 		err = -EIO;
2379 	return err;
2380 }
2381 
2382 static int live_preempt_cancel(void *arg)
2383 {
2384 	struct intel_gt *gt = arg;
2385 	struct live_preempt_cancel data;
2386 	enum intel_engine_id id;
2387 	int err = -ENOMEM;
2388 
2389 	/*
2390 	 * To cancel an inflight context, we need to first remove it from the
2391 	 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
2392 	 */
2393 
2394 	if (preempt_client_init(gt, &data.a))
2395 		return -ENOMEM;
2396 	if (preempt_client_init(gt, &data.b))
2397 		goto err_client_a;
2398 
2399 	for_each_engine(data.engine, gt, id) {
2400 		if (!intel_engine_has_preemption(data.engine))
2401 			continue;
2402 
2403 		err = __cancel_active0(&data);
2404 		if (err)
2405 			goto err_wedged;
2406 
2407 		err = __cancel_active1(&data);
2408 		if (err)
2409 			goto err_wedged;
2410 
2411 		err = __cancel_queued(&data);
2412 		if (err)
2413 			goto err_wedged;
2414 
2415 		err = __cancel_hostile(&data);
2416 		if (err)
2417 			goto err_wedged;
2418 
2419 		err = __cancel_fail(&data);
2420 		if (err)
2421 			goto err_wedged;
2422 	}
2423 
2424 	err = 0;
2425 err_client_b:
2426 	preempt_client_fini(&data.b);
2427 err_client_a:
2428 	preempt_client_fini(&data.a);
2429 	return err;
2430 
2431 err_wedged:
2432 	GEM_TRACE_DUMP();
2433 	igt_spinner_end(&data.b.spin);
2434 	igt_spinner_end(&data.a.spin);
2435 	intel_gt_set_wedged(gt);
2436 	goto err_client_b;
2437 }
2438 
2439 static int live_suppress_self_preempt(void *arg)
2440 {
2441 	struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2442 	struct intel_gt *gt = arg;
2443 	struct intel_engine_cs *engine;
2444 	struct preempt_client a, b;
2445 	enum intel_engine_id id;
2446 	int err = -ENOMEM;
2447 
2448 	/*
2449 	 * Verify that if a preemption request does not cause a change in
2450 	 * the current execution order, the preempt-to-idle injection is
2451 	 * skipped and that we do not accidentally apply it after the CS
2452 	 * completion event.
2453 	 */
2454 
2455 	if (intel_uc_uses_guc_submission(&gt->uc))
2456 		return 0; /* presume black blox */
2457 
2458 	if (intel_vgpu_active(gt->i915))
2459 		return 0; /* GVT forces single port & request submission */
2460 
2461 	if (preempt_client_init(gt, &a))
2462 		return -ENOMEM;
2463 	if (preempt_client_init(gt, &b))
2464 		goto err_client_a;
2465 
2466 	for_each_engine(engine, gt, id) {
2467 		struct i915_request *rq_a, *rq_b;
2468 		int depth;
2469 
2470 		if (!intel_engine_has_preemption(engine))
2471 			continue;
2472 
2473 		if (igt_flush_test(gt->i915))
2474 			goto err_wedged;
2475 
2476 		st_engine_heartbeat_disable(engine);
2477 		engine->execlists.preempt_hang.count = 0;
2478 
2479 		rq_a = spinner_create_request(&a.spin,
2480 					      a.ctx, engine,
2481 					      MI_NOOP);
2482 		if (IS_ERR(rq_a)) {
2483 			err = PTR_ERR(rq_a);
2484 			st_engine_heartbeat_enable(engine);
2485 			goto err_client_b;
2486 		}
2487 
2488 		i915_request_add(rq_a);
2489 		if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2490 			pr_err("First client failed to start\n");
2491 			st_engine_heartbeat_enable(engine);
2492 			goto err_wedged;
2493 		}
2494 
2495 		/* Keep postponing the timer to avoid premature slicing */
2496 		mod_timer(&engine->execlists.timer, jiffies + HZ);
2497 		for (depth = 0; depth < 8; depth++) {
2498 			rq_b = spinner_create_request(&b.spin,
2499 						      b.ctx, engine,
2500 						      MI_NOOP);
2501 			if (IS_ERR(rq_b)) {
2502 				err = PTR_ERR(rq_b);
2503 				st_engine_heartbeat_enable(engine);
2504 				goto err_client_b;
2505 			}
2506 			i915_request_add(rq_b);
2507 
2508 			GEM_BUG_ON(i915_request_completed(rq_a));
2509 			engine->sched_engine->schedule(rq_a, &attr);
2510 			igt_spinner_end(&a.spin);
2511 
2512 			if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2513 				pr_err("Second client failed to start\n");
2514 				st_engine_heartbeat_enable(engine);
2515 				goto err_wedged;
2516 			}
2517 
2518 			swap(a, b);
2519 			rq_a = rq_b;
2520 		}
2521 		igt_spinner_end(&a.spin);
2522 
2523 		if (engine->execlists.preempt_hang.count) {
2524 			pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2525 			       engine->name,
2526 			       engine->execlists.preempt_hang.count,
2527 			       depth);
2528 			st_engine_heartbeat_enable(engine);
2529 			err = -EINVAL;
2530 			goto err_client_b;
2531 		}
2532 
2533 		st_engine_heartbeat_enable(engine);
2534 		if (igt_flush_test(gt->i915))
2535 			goto err_wedged;
2536 	}
2537 
2538 	err = 0;
2539 err_client_b:
2540 	preempt_client_fini(&b);
2541 err_client_a:
2542 	preempt_client_fini(&a);
2543 	return err;
2544 
2545 err_wedged:
2546 	igt_spinner_end(&b.spin);
2547 	igt_spinner_end(&a.spin);
2548 	intel_gt_set_wedged(gt);
2549 	err = -EIO;
2550 	goto err_client_b;
2551 }
2552 
2553 static int live_chain_preempt(void *arg)
2554 {
2555 	struct intel_gt *gt = arg;
2556 	struct intel_engine_cs *engine;
2557 	struct preempt_client hi, lo;
2558 	enum intel_engine_id id;
2559 	int err = -ENOMEM;
2560 
2561 	/*
2562 	 * Build a chain AB...BA between two contexts (A, B) and request
2563 	 * preemption of the last request. It should then complete before
2564 	 * the previously submitted spinner in B.
2565 	 */
2566 
2567 	if (preempt_client_init(gt, &hi))
2568 		return -ENOMEM;
2569 
2570 	if (preempt_client_init(gt, &lo))
2571 		goto err_client_hi;
2572 
2573 	for_each_engine(engine, gt, id) {
2574 		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2575 		struct igt_live_test t;
2576 		struct i915_request *rq;
2577 		int ring_size, count, i;
2578 
2579 		if (!intel_engine_has_preemption(engine))
2580 			continue;
2581 
2582 		rq = spinner_create_request(&lo.spin,
2583 					    lo.ctx, engine,
2584 					    MI_ARB_CHECK);
2585 		if (IS_ERR(rq))
2586 			goto err_wedged;
2587 
2588 		i915_request_get(rq);
2589 		i915_request_add(rq);
2590 
2591 		ring_size = rq->wa_tail - rq->head;
2592 		if (ring_size < 0)
2593 			ring_size += rq->ring->size;
2594 		ring_size = rq->ring->size / ring_size;
2595 		pr_debug("%s(%s): Using maximum of %d requests\n",
2596 			 __func__, engine->name, ring_size);
2597 
2598 		igt_spinner_end(&lo.spin);
2599 		if (i915_request_wait(rq, 0, HZ / 2) < 0) {
2600 			pr_err("Timed out waiting to flush %s\n", engine->name);
2601 			i915_request_put(rq);
2602 			goto err_wedged;
2603 		}
2604 		i915_request_put(rq);
2605 
2606 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
2607 			err = -EIO;
2608 			goto err_wedged;
2609 		}
2610 
2611 		for_each_prime_number_from(count, 1, ring_size) {
2612 			rq = spinner_create_request(&hi.spin,
2613 						    hi.ctx, engine,
2614 						    MI_ARB_CHECK);
2615 			if (IS_ERR(rq))
2616 				goto err_wedged;
2617 			i915_request_add(rq);
2618 			if (!igt_wait_for_spinner(&hi.spin, rq))
2619 				goto err_wedged;
2620 
2621 			rq = spinner_create_request(&lo.spin,
2622 						    lo.ctx, engine,
2623 						    MI_ARB_CHECK);
2624 			if (IS_ERR(rq))
2625 				goto err_wedged;
2626 			i915_request_add(rq);
2627 
2628 			for (i = 0; i < count; i++) {
2629 				rq = igt_request_alloc(lo.ctx, engine);
2630 				if (IS_ERR(rq))
2631 					goto err_wedged;
2632 				i915_request_add(rq);
2633 			}
2634 
2635 			rq = igt_request_alloc(hi.ctx, engine);
2636 			if (IS_ERR(rq))
2637 				goto err_wedged;
2638 
2639 			i915_request_get(rq);
2640 			i915_request_add(rq);
2641 			engine->sched_engine->schedule(rq, &attr);
2642 
2643 			igt_spinner_end(&hi.spin);
2644 			if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2645 				struct drm_printer p =
2646 					drm_info_printer(gt->i915->drm.dev);
2647 
2648 				pr_err("Failed to preempt over chain of %d\n",
2649 				       count);
2650 				intel_engine_dump(engine, &p,
2651 						  "%s\n", engine->name);
2652 				i915_request_put(rq);
2653 				goto err_wedged;
2654 			}
2655 			igt_spinner_end(&lo.spin);
2656 			i915_request_put(rq);
2657 
2658 			rq = igt_request_alloc(lo.ctx, engine);
2659 			if (IS_ERR(rq))
2660 				goto err_wedged;
2661 
2662 			i915_request_get(rq);
2663 			i915_request_add(rq);
2664 
2665 			if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2666 				struct drm_printer p =
2667 					drm_info_printer(gt->i915->drm.dev);
2668 
2669 				pr_err("Failed to flush low priority chain of %d requests\n",
2670 				       count);
2671 				intel_engine_dump(engine, &p,
2672 						  "%s\n", engine->name);
2673 
2674 				i915_request_put(rq);
2675 				goto err_wedged;
2676 			}
2677 			i915_request_put(rq);
2678 		}
2679 
2680 		if (igt_live_test_end(&t)) {
2681 			err = -EIO;
2682 			goto err_wedged;
2683 		}
2684 	}
2685 
2686 	err = 0;
2687 err_client_lo:
2688 	preempt_client_fini(&lo);
2689 err_client_hi:
2690 	preempt_client_fini(&hi);
2691 	return err;
2692 
2693 err_wedged:
2694 	igt_spinner_end(&hi.spin);
2695 	igt_spinner_end(&lo.spin);
2696 	intel_gt_set_wedged(gt);
2697 	err = -EIO;
2698 	goto err_client_lo;
2699 }
2700 
2701 static int create_gang(struct intel_engine_cs *engine,
2702 		       struct i915_request **prev)
2703 {
2704 	struct drm_i915_gem_object *obj;
2705 	struct intel_context *ce;
2706 	struct i915_request *rq;
2707 	struct i915_vma *vma;
2708 	u32 *cs;
2709 	int err;
2710 
2711 	ce = intel_context_create(engine);
2712 	if (IS_ERR(ce))
2713 		return PTR_ERR(ce);
2714 
2715 	obj = i915_gem_object_create_internal(engine->i915, 4096);
2716 	if (IS_ERR(obj)) {
2717 		err = PTR_ERR(obj);
2718 		goto err_ce;
2719 	}
2720 
2721 	vma = i915_vma_instance(obj, ce->vm, NULL);
2722 	if (IS_ERR(vma)) {
2723 		err = PTR_ERR(vma);
2724 		goto err_obj;
2725 	}
2726 
2727 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
2728 	if (err)
2729 		goto err_obj;
2730 
2731 	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
2732 	if (IS_ERR(cs)) {
2733 		err = PTR_ERR(cs);
2734 		goto err_obj;
2735 	}
2736 
2737 	/* Semaphore target: spin until zero */
2738 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2739 
2740 	*cs++ = MI_SEMAPHORE_WAIT |
2741 		MI_SEMAPHORE_POLL |
2742 		MI_SEMAPHORE_SAD_EQ_SDD;
2743 	*cs++ = 0;
2744 	*cs++ = lower_32_bits(i915_vma_offset(vma));
2745 	*cs++ = upper_32_bits(i915_vma_offset(vma));
2746 
2747 	if (*prev) {
2748 		u64 offset = i915_vma_offset((*prev)->batch);
2749 
2750 		/* Terminate the spinner in the next lower priority batch. */
2751 		*cs++ = MI_STORE_DWORD_IMM_GEN4;
2752 		*cs++ = lower_32_bits(offset);
2753 		*cs++ = upper_32_bits(offset);
2754 		*cs++ = 0;
2755 	}
2756 
2757 	*cs++ = MI_BATCH_BUFFER_END;
2758 	i915_gem_object_flush_map(obj);
2759 	i915_gem_object_unpin_map(obj);
2760 
2761 	rq = intel_context_create_request(ce);
2762 	if (IS_ERR(rq)) {
2763 		err = PTR_ERR(rq);
2764 		goto err_obj;
2765 	}
2766 
2767 	rq->batch = i915_vma_get(vma);
2768 	i915_request_get(rq);
2769 
2770 	err = igt_vma_move_to_active_unlocked(vma, rq, 0);
2771 	if (!err)
2772 		err = rq->engine->emit_bb_start(rq,
2773 						i915_vma_offset(vma),
2774 						PAGE_SIZE, 0);
2775 	i915_request_add(rq);
2776 	if (err)
2777 		goto err_rq;
2778 
2779 	i915_gem_object_put(obj);
2780 	intel_context_put(ce);
2781 
2782 	rq->mock.link.next = &(*prev)->mock.link;
2783 	*prev = rq;
2784 	return 0;
2785 
2786 err_rq:
2787 	i915_vma_put(rq->batch);
2788 	i915_request_put(rq);
2789 err_obj:
2790 	i915_gem_object_put(obj);
2791 err_ce:
2792 	intel_context_put(ce);
2793 	return err;
2794 }
2795 
2796 static int __live_preempt_ring(struct intel_engine_cs *engine,
2797 			       struct igt_spinner *spin,
2798 			       int queue_sz, int ring_sz)
2799 {
2800 	struct intel_context *ce[2] = {};
2801 	struct i915_request *rq;
2802 	struct igt_live_test t;
2803 	int err = 0;
2804 	int n;
2805 
2806 	if (igt_live_test_begin(&t, engine->i915, __func__, engine->name))
2807 		return -EIO;
2808 
2809 	for (n = 0; n < ARRAY_SIZE(ce); n++) {
2810 		struct intel_context *tmp;
2811 
2812 		tmp = intel_context_create(engine);
2813 		if (IS_ERR(tmp)) {
2814 			err = PTR_ERR(tmp);
2815 			goto err_ce;
2816 		}
2817 
2818 		tmp->ring_size = ring_sz;
2819 
2820 		err = intel_context_pin(tmp);
2821 		if (err) {
2822 			intel_context_put(tmp);
2823 			goto err_ce;
2824 		}
2825 
2826 		memset32(tmp->ring->vaddr,
2827 			 0xdeadbeef, /* trigger a hang if executed */
2828 			 tmp->ring->vma->size / sizeof(u32));
2829 
2830 		ce[n] = tmp;
2831 	}
2832 
2833 	rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK);
2834 	if (IS_ERR(rq)) {
2835 		err = PTR_ERR(rq);
2836 		goto err_ce;
2837 	}
2838 
2839 	i915_request_get(rq);
2840 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2841 	i915_request_add(rq);
2842 
2843 	if (!igt_wait_for_spinner(spin, rq)) {
2844 		intel_gt_set_wedged(engine->gt);
2845 		i915_request_put(rq);
2846 		err = -ETIME;
2847 		goto err_ce;
2848 	}
2849 
2850 	/* Fill the ring, until we will cause a wrap */
2851 	n = 0;
2852 	while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) {
2853 		struct i915_request *tmp;
2854 
2855 		tmp = intel_context_create_request(ce[0]);
2856 		if (IS_ERR(tmp)) {
2857 			err = PTR_ERR(tmp);
2858 			i915_request_put(rq);
2859 			goto err_ce;
2860 		}
2861 
2862 		i915_request_add(tmp);
2863 		intel_engine_flush_submission(engine);
2864 		n++;
2865 	}
2866 	intel_engine_flush_submission(engine);
2867 	pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
2868 		 engine->name, queue_sz, n,
2869 		 ce[0]->ring->size,
2870 		 ce[0]->ring->tail,
2871 		 ce[0]->ring->emit,
2872 		 rq->tail);
2873 	i915_request_put(rq);
2874 
2875 	/* Create a second request to preempt the first ring */
2876 	rq = intel_context_create_request(ce[1]);
2877 	if (IS_ERR(rq)) {
2878 		err = PTR_ERR(rq);
2879 		goto err_ce;
2880 	}
2881 
2882 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2883 	i915_request_get(rq);
2884 	i915_request_add(rq);
2885 
2886 	err = wait_for_submit(engine, rq, HZ / 2);
2887 	i915_request_put(rq);
2888 	if (err) {
2889 		pr_err("%s: preemption request was not submitted\n",
2890 		       engine->name);
2891 		err = -ETIME;
2892 	}
2893 
2894 	pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
2895 		 engine->name,
2896 		 ce[0]->ring->tail, ce[0]->ring->emit,
2897 		 ce[1]->ring->tail, ce[1]->ring->emit);
2898 
2899 err_ce:
2900 	intel_engine_flush_submission(engine);
2901 	igt_spinner_end(spin);
2902 	for (n = 0; n < ARRAY_SIZE(ce); n++) {
2903 		if (IS_ERR_OR_NULL(ce[n]))
2904 			break;
2905 
2906 		intel_context_unpin(ce[n]);
2907 		intel_context_put(ce[n]);
2908 	}
2909 	if (igt_live_test_end(&t))
2910 		err = -EIO;
2911 	return err;
2912 }
2913 
2914 static int live_preempt_ring(void *arg)
2915 {
2916 	struct intel_gt *gt = arg;
2917 	struct intel_engine_cs *engine;
2918 	struct igt_spinner spin;
2919 	enum intel_engine_id id;
2920 	int err = 0;
2921 
2922 	/*
2923 	 * Check that we rollback large chunks of a ring in order to do a
2924 	 * preemption event. Similar to live_unlite_ring, but looking at
2925 	 * ring size rather than the impact of intel_ring_direction().
2926 	 */
2927 
2928 	if (igt_spinner_init(&spin, gt))
2929 		return -ENOMEM;
2930 
2931 	for_each_engine(engine, gt, id) {
2932 		int n;
2933 
2934 		if (!intel_engine_has_preemption(engine))
2935 			continue;
2936 
2937 		if (!intel_engine_can_store_dword(engine))
2938 			continue;
2939 
2940 		st_engine_heartbeat_disable(engine);
2941 
2942 		for (n = 0; n <= 3; n++) {
2943 			err = __live_preempt_ring(engine, &spin,
2944 						  n * SZ_4K / 4, SZ_4K);
2945 			if (err)
2946 				break;
2947 		}
2948 
2949 		st_engine_heartbeat_enable(engine);
2950 		if (err)
2951 			break;
2952 	}
2953 
2954 	igt_spinner_fini(&spin);
2955 	return err;
2956 }
2957 
2958 static int live_preempt_gang(void *arg)
2959 {
2960 	struct intel_gt *gt = arg;
2961 	struct intel_engine_cs *engine;
2962 	enum intel_engine_id id;
2963 
2964 	/*
2965 	 * Build as long a chain of preempters as we can, with each
2966 	 * request higher priority than the last. Once we are ready, we release
2967 	 * the last batch which then precolates down the chain, each releasing
2968 	 * the next oldest in turn. The intent is to simply push as hard as we
2969 	 * can with the number of preemptions, trying to exceed narrow HW
2970 	 * limits. At a minimum, we insist that we can sort all the user
2971 	 * high priority levels into execution order.
2972 	 */
2973 
2974 	for_each_engine(engine, gt, id) {
2975 		struct i915_request *rq = NULL;
2976 		struct igt_live_test t;
2977 		IGT_TIMEOUT(end_time);
2978 		int prio = 0;
2979 		int err = 0;
2980 		u32 *cs;
2981 
2982 		if (!intel_engine_has_preemption(engine))
2983 			continue;
2984 
2985 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2986 			return -EIO;
2987 
2988 		do {
2989 			struct i915_sched_attr attr = { .priority = prio++ };
2990 
2991 			err = create_gang(engine, &rq);
2992 			if (err)
2993 				break;
2994 
2995 			/* Submit each spinner at increasing priority */
2996 			engine->sched_engine->schedule(rq, &attr);
2997 		} while (prio <= I915_PRIORITY_MAX &&
2998 			 !__igt_timeout(end_time, NULL));
2999 		pr_debug("%s: Preempt chain of %d requests\n",
3000 			 engine->name, prio);
3001 
3002 		/*
3003 		 * Such that the last spinner is the highest priority and
3004 		 * should execute first. When that spinner completes,
3005 		 * it will terminate the next lowest spinner until there
3006 		 * are no more spinners and the gang is complete.
3007 		 */
3008 		cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);
3009 		if (!IS_ERR(cs)) {
3010 			*cs = 0;
3011 			i915_gem_object_unpin_map(rq->batch->obj);
3012 		} else {
3013 			err = PTR_ERR(cs);
3014 			intel_gt_set_wedged(gt);
3015 		}
3016 
3017 		while (rq) { /* wait for each rq from highest to lowest prio */
3018 			struct i915_request *n = list_next_entry(rq, mock.link);
3019 
3020 			if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
3021 				struct drm_printer p =
3022 					drm_info_printer(engine->i915->drm.dev);
3023 
3024 				pr_err("Failed to flush chain of %d requests, at %d\n",
3025 				       prio, rq_prio(rq));
3026 				intel_engine_dump(engine, &p,
3027 						  "%s\n", engine->name);
3028 
3029 				err = -ETIME;
3030 			}
3031 
3032 			i915_vma_put(rq->batch);
3033 			i915_request_put(rq);
3034 			rq = n;
3035 		}
3036 
3037 		if (igt_live_test_end(&t))
3038 			err = -EIO;
3039 		if (err)
3040 			return err;
3041 	}
3042 
3043 	return 0;
3044 }
3045 
3046 static struct i915_vma *
3047 create_gpr_user(struct intel_engine_cs *engine,
3048 		struct i915_vma *result,
3049 		unsigned int offset)
3050 {
3051 	struct drm_i915_gem_object *obj;
3052 	struct i915_vma *vma;
3053 	u32 *cs;
3054 	int err;
3055 	int i;
3056 
3057 	obj = i915_gem_object_create_internal(engine->i915, 4096);
3058 	if (IS_ERR(obj))
3059 		return ERR_CAST(obj);
3060 
3061 	vma = i915_vma_instance(obj, result->vm, NULL);
3062 	if (IS_ERR(vma)) {
3063 		i915_gem_object_put(obj);
3064 		return vma;
3065 	}
3066 
3067 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
3068 	if (err) {
3069 		i915_vma_put(vma);
3070 		return ERR_PTR(err);
3071 	}
3072 
3073 	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
3074 	if (IS_ERR(cs)) {
3075 		i915_vma_put(vma);
3076 		return ERR_CAST(cs);
3077 	}
3078 
3079 	/* All GPR are clear for new contexts. We use GPR(0) as a constant */
3080 	*cs++ = MI_LOAD_REGISTER_IMM(1);
3081 	*cs++ = CS_GPR(engine, 0);
3082 	*cs++ = 1;
3083 
3084 	for (i = 1; i < NUM_GPR; i++) {
3085 		u64 addr;
3086 
3087 		/*
3088 		 * Perform: GPR[i]++
3089 		 *
3090 		 * As we read and write into the context saved GPR[i], if
3091 		 * we restart this batch buffer from an earlier point, we
3092 		 * will repeat the increment and store a value > 1.
3093 		 */
3094 		*cs++ = MI_MATH(4);
3095 		*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));
3096 		*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));
3097 		*cs++ = MI_MATH_ADD;
3098 		*cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);
3099 
3100 		addr = i915_vma_offset(result) + offset + i * sizeof(*cs);
3101 		*cs++ = MI_STORE_REGISTER_MEM_GEN8;
3102 		*cs++ = CS_GPR(engine, 2 * i);
3103 		*cs++ = lower_32_bits(addr);
3104 		*cs++ = upper_32_bits(addr);
3105 
3106 		*cs++ = MI_SEMAPHORE_WAIT |
3107 			MI_SEMAPHORE_POLL |
3108 			MI_SEMAPHORE_SAD_GTE_SDD;
3109 		*cs++ = i;
3110 		*cs++ = lower_32_bits(i915_vma_offset(result));
3111 		*cs++ = upper_32_bits(i915_vma_offset(result));
3112 	}
3113 
3114 	*cs++ = MI_BATCH_BUFFER_END;
3115 	i915_gem_object_flush_map(obj);
3116 	i915_gem_object_unpin_map(obj);
3117 
3118 	return vma;
3119 }
3120 
3121 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
3122 {
3123 	struct drm_i915_gem_object *obj;
3124 	struct i915_vma *vma;
3125 	int err;
3126 
3127 	obj = i915_gem_object_create_internal(gt->i915, sz);
3128 	if (IS_ERR(obj))
3129 		return ERR_CAST(obj);
3130 
3131 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
3132 	if (IS_ERR(vma)) {
3133 		i915_gem_object_put(obj);
3134 		return vma;
3135 	}
3136 
3137 	err = i915_ggtt_pin(vma, NULL, 0, 0);
3138 	if (err) {
3139 		i915_vma_put(vma);
3140 		return ERR_PTR(err);
3141 	}
3142 
3143 	return vma;
3144 }
3145 
3146 static struct i915_request *
3147 create_gpr_client(struct intel_engine_cs *engine,
3148 		  struct i915_vma *global,
3149 		  unsigned int offset)
3150 {
3151 	struct i915_vma *batch, *vma;
3152 	struct intel_context *ce;
3153 	struct i915_request *rq;
3154 	int err;
3155 
3156 	ce = intel_context_create(engine);
3157 	if (IS_ERR(ce))
3158 		return ERR_CAST(ce);
3159 
3160 	vma = i915_vma_instance(global->obj, ce->vm, NULL);
3161 	if (IS_ERR(vma)) {
3162 		err = PTR_ERR(vma);
3163 		goto out_ce;
3164 	}
3165 
3166 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
3167 	if (err)
3168 		goto out_ce;
3169 
3170 	batch = create_gpr_user(engine, vma, offset);
3171 	if (IS_ERR(batch)) {
3172 		err = PTR_ERR(batch);
3173 		goto out_vma;
3174 	}
3175 
3176 	rq = intel_context_create_request(ce);
3177 	if (IS_ERR(rq)) {
3178 		err = PTR_ERR(rq);
3179 		goto out_batch;
3180 	}
3181 
3182 	err = igt_vma_move_to_active_unlocked(vma, rq, 0);
3183 
3184 	i915_vma_lock(batch);
3185 	if (!err)
3186 		err = i915_vma_move_to_active(batch, rq, 0);
3187 	if (!err)
3188 		err = rq->engine->emit_bb_start(rq,
3189 						i915_vma_offset(batch),
3190 						PAGE_SIZE, 0);
3191 	i915_vma_unlock(batch);
3192 	i915_vma_unpin(batch);
3193 
3194 	if (!err)
3195 		i915_request_get(rq);
3196 	i915_request_add(rq);
3197 
3198 out_batch:
3199 	i915_vma_put(batch);
3200 out_vma:
3201 	i915_vma_unpin(vma);
3202 out_ce:
3203 	intel_context_put(ce);
3204 	return err ? ERR_PTR(err) : rq;
3205 }
3206 
3207 static int preempt_user(struct intel_engine_cs *engine,
3208 			struct i915_vma *global,
3209 			int id)
3210 {
3211 	struct i915_sched_attr attr = {
3212 		.priority = I915_PRIORITY_MAX
3213 	};
3214 	struct i915_request *rq;
3215 	int err = 0;
3216 	u32 *cs;
3217 
3218 	rq = intel_engine_create_kernel_request(engine);
3219 	if (IS_ERR(rq))
3220 		return PTR_ERR(rq);
3221 
3222 	cs = intel_ring_begin(rq, 4);
3223 	if (IS_ERR(cs)) {
3224 		i915_request_add(rq);
3225 		return PTR_ERR(cs);
3226 	}
3227 
3228 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
3229 	*cs++ = i915_ggtt_offset(global);
3230 	*cs++ = 0;
3231 	*cs++ = id;
3232 
3233 	intel_ring_advance(rq, cs);
3234 
3235 	i915_request_get(rq);
3236 	i915_request_add(rq);
3237 
3238 	engine->sched_engine->schedule(rq, &attr);
3239 
3240 	if (i915_request_wait(rq, 0, HZ / 2) < 0)
3241 		err = -ETIME;
3242 	i915_request_put(rq);
3243 
3244 	return err;
3245 }
3246 
3247 static int live_preempt_user(void *arg)
3248 {
3249 	struct intel_gt *gt = arg;
3250 	struct intel_engine_cs *engine;
3251 	struct i915_vma *global;
3252 	enum intel_engine_id id;
3253 	u32 *result;
3254 	int err = 0;
3255 
3256 	/*
3257 	 * In our other tests, we look at preemption in carefully
3258 	 * controlled conditions in the ringbuffer. Since most of the
3259 	 * time is spent in user batches, most of our preemptions naturally
3260 	 * occur there. We want to verify that when we preempt inside a batch
3261 	 * we continue on from the current instruction and do not roll back
3262 	 * to the start, or another earlier arbitration point.
3263 	 *
3264 	 * To verify this, we create a batch which is a mixture of
3265 	 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with
3266 	 * a few preempting contexts thrown into the mix, we look for any
3267 	 * repeated instructions (which show up as incorrect values).
3268 	 */
3269 
3270 	global = create_global(gt, 4096);
3271 	if (IS_ERR(global))
3272 		return PTR_ERR(global);
3273 
3274 	result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);
3275 	if (IS_ERR(result)) {
3276 		i915_vma_unpin_and_release(&global, 0);
3277 		return PTR_ERR(result);
3278 	}
3279 
3280 	for_each_engine(engine, gt, id) {
3281 		struct i915_request *client[3] = {};
3282 		struct igt_live_test t;
3283 		int i;
3284 
3285 		if (!intel_engine_has_preemption(engine))
3286 			continue;
3287 
3288 		if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS)
3289 			continue; /* we need per-context GPR */
3290 
3291 		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
3292 			err = -EIO;
3293 			break;
3294 		}
3295 
3296 		memset(result, 0, 4096);
3297 
3298 		for (i = 0; i < ARRAY_SIZE(client); i++) {
3299 			struct i915_request *rq;
3300 
3301 			rq = create_gpr_client(engine, global,
3302 					       NUM_GPR * i * sizeof(u32));
3303 			if (IS_ERR(rq)) {
3304 				err = PTR_ERR(rq);
3305 				goto end_test;
3306 			}
3307 
3308 			client[i] = rq;
3309 		}
3310 
3311 		/* Continuously preempt the set of 3 running contexts */
3312 		for (i = 1; i <= NUM_GPR; i++) {
3313 			err = preempt_user(engine, global, i);
3314 			if (err)
3315 				goto end_test;
3316 		}
3317 
3318 		if (READ_ONCE(result[0]) != NUM_GPR) {
3319 			pr_err("%s: Failed to release semaphore\n",
3320 			       engine->name);
3321 			err = -EIO;
3322 			goto end_test;
3323 		}
3324 
3325 		for (i = 0; i < ARRAY_SIZE(client); i++) {
3326 			int gpr;
3327 
3328 			if (i915_request_wait(client[i], 0, HZ / 2) < 0) {
3329 				err = -ETIME;
3330 				goto end_test;
3331 			}
3332 
3333 			for (gpr = 1; gpr < NUM_GPR; gpr++) {
3334 				if (result[NUM_GPR * i + gpr] != 1) {
3335 					pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",
3336 					       engine->name,
3337 					       i, gpr, result[NUM_GPR * i + gpr]);
3338 					err = -EINVAL;
3339 					goto end_test;
3340 				}
3341 			}
3342 		}
3343 
3344 end_test:
3345 		for (i = 0; i < ARRAY_SIZE(client); i++) {
3346 			if (!client[i])
3347 				break;
3348 
3349 			i915_request_put(client[i]);
3350 		}
3351 
3352 		/* Flush the semaphores on error */
3353 		smp_store_mb(result[0], -1);
3354 		if (igt_live_test_end(&t))
3355 			err = -EIO;
3356 		if (err)
3357 			break;
3358 	}
3359 
3360 	i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);
3361 	return err;
3362 }
3363 
3364 static int live_preempt_timeout(void *arg)
3365 {
3366 	struct intel_gt *gt = arg;
3367 	struct i915_gem_context *ctx_hi, *ctx_lo;
3368 	struct igt_spinner spin_lo;
3369 	struct intel_engine_cs *engine;
3370 	enum intel_engine_id id;
3371 	int err = -ENOMEM;
3372 
3373 	/*
3374 	 * Check that we force preemption to occur by cancelling the previous
3375 	 * context if it refuses to yield the GPU.
3376 	 */
3377 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
3378 		return 0;
3379 
3380 	if (!intel_has_reset_engine(gt))
3381 		return 0;
3382 
3383 	ctx_hi = kernel_context(gt->i915, NULL);
3384 	if (!ctx_hi)
3385 		return -ENOMEM;
3386 	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
3387 
3388 	ctx_lo = kernel_context(gt->i915, NULL);
3389 	if (!ctx_lo)
3390 		goto err_ctx_hi;
3391 	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
3392 
3393 	if (igt_spinner_init(&spin_lo, gt))
3394 		goto err_ctx_lo;
3395 
3396 	for_each_engine(engine, gt, id) {
3397 		unsigned long saved_timeout;
3398 		struct i915_request *rq;
3399 
3400 		if (!intel_engine_has_preemption(engine))
3401 			continue;
3402 
3403 		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
3404 					    MI_NOOP); /* preemption disabled */
3405 		if (IS_ERR(rq)) {
3406 			err = PTR_ERR(rq);
3407 			goto err_spin_lo;
3408 		}
3409 
3410 		i915_request_add(rq);
3411 		if (!igt_wait_for_spinner(&spin_lo, rq)) {
3412 			intel_gt_set_wedged(gt);
3413 			err = -EIO;
3414 			goto err_spin_lo;
3415 		}
3416 
3417 		rq = igt_request_alloc(ctx_hi, engine);
3418 		if (IS_ERR(rq)) {
3419 			igt_spinner_end(&spin_lo);
3420 			err = PTR_ERR(rq);
3421 			goto err_spin_lo;
3422 		}
3423 
3424 		/* Flush the previous CS ack before changing timeouts */
3425 		while (READ_ONCE(engine->execlists.pending[0]))
3426 			cpu_relax();
3427 
3428 		saved_timeout = engine->props.preempt_timeout_ms;
3429 		engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
3430 
3431 		i915_request_get(rq);
3432 		i915_request_add(rq);
3433 
3434 		intel_engine_flush_submission(engine);
3435 		engine->props.preempt_timeout_ms = saved_timeout;
3436 
3437 		if (i915_request_wait(rq, 0, HZ / 10) < 0) {
3438 			intel_gt_set_wedged(gt);
3439 			i915_request_put(rq);
3440 			err = -ETIME;
3441 			goto err_spin_lo;
3442 		}
3443 
3444 		igt_spinner_end(&spin_lo);
3445 		i915_request_put(rq);
3446 	}
3447 
3448 	err = 0;
3449 err_spin_lo:
3450 	igt_spinner_fini(&spin_lo);
3451 err_ctx_lo:
3452 	kernel_context_close(ctx_lo);
3453 err_ctx_hi:
3454 	kernel_context_close(ctx_hi);
3455 	return err;
3456 }
3457 
3458 static int random_range(struct rnd_state *rnd, int min, int max)
3459 {
3460 	return i915_prandom_u32_max_state(max - min, rnd) + min;
3461 }
3462 
3463 static int random_priority(struct rnd_state *rnd)
3464 {
3465 	return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
3466 }
3467 
3468 struct preempt_smoke {
3469 	struct intel_gt *gt;
3470 	struct kthread_work work;
3471 	struct i915_gem_context **contexts;
3472 	struct intel_engine_cs *engine;
3473 	struct drm_i915_gem_object *batch;
3474 	unsigned int ncontext;
3475 	struct rnd_state prng;
3476 	unsigned long count;
3477 	int result;
3478 };
3479 
3480 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
3481 {
3482 	return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
3483 							  &smoke->prng)];
3484 }
3485 
3486 static int smoke_submit(struct preempt_smoke *smoke,
3487 			struct i915_gem_context *ctx, int prio,
3488 			struct drm_i915_gem_object *batch)
3489 {
3490 	struct i915_request *rq;
3491 	struct i915_vma *vma = NULL;
3492 	int err = 0;
3493 
3494 	if (batch) {
3495 		struct i915_address_space *vm;
3496 
3497 		vm = i915_gem_context_get_eb_vm(ctx);
3498 		vma = i915_vma_instance(batch, vm, NULL);
3499 		i915_vm_put(vm);
3500 		if (IS_ERR(vma))
3501 			return PTR_ERR(vma);
3502 
3503 		err = i915_vma_pin(vma, 0, 0, PIN_USER);
3504 		if (err)
3505 			return err;
3506 	}
3507 
3508 	ctx->sched.priority = prio;
3509 
3510 	rq = igt_request_alloc(ctx, smoke->engine);
3511 	if (IS_ERR(rq)) {
3512 		err = PTR_ERR(rq);
3513 		goto unpin;
3514 	}
3515 
3516 	if (vma) {
3517 		err = igt_vma_move_to_active_unlocked(vma, rq, 0);
3518 		if (!err)
3519 			err = rq->engine->emit_bb_start(rq,
3520 							i915_vma_offset(vma),
3521 							PAGE_SIZE, 0);
3522 	}
3523 
3524 	i915_request_add(rq);
3525 
3526 unpin:
3527 	if (vma)
3528 		i915_vma_unpin(vma);
3529 
3530 	return err;
3531 }
3532 
3533 static void smoke_crescendo_work(struct kthread_work *work)
3534 {
3535 	struct preempt_smoke *smoke = container_of(work, typeof(*smoke), work);
3536 	IGT_TIMEOUT(end_time);
3537 	unsigned long count;
3538 
3539 	count = 0;
3540 	do {
3541 		struct i915_gem_context *ctx = smoke_context(smoke);
3542 
3543 		smoke->result = smoke_submit(smoke, ctx,
3544 					     count % I915_PRIORITY_MAX,
3545 					     smoke->batch);
3546 
3547 		count++;
3548 	} while (!smoke->result && count < smoke->ncontext &&
3549 		 !__igt_timeout(end_time, NULL));
3550 
3551 	smoke->count = count;
3552 }
3553 
3554 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
3555 #define BATCH BIT(0)
3556 {
3557 	struct kthread_worker *worker[I915_NUM_ENGINES] = {};
3558 	struct preempt_smoke *arg;
3559 	struct intel_engine_cs *engine;
3560 	enum intel_engine_id id;
3561 	unsigned long count;
3562 	int err = 0;
3563 
3564 	arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL);
3565 	if (!arg)
3566 		return -ENOMEM;
3567 
3568 	memset(arg, 0, I915_NUM_ENGINES * sizeof(*arg));
3569 
3570 	for_each_engine(engine, smoke->gt, id) {
3571 		arg[id] = *smoke;
3572 		arg[id].engine = engine;
3573 		if (!(flags & BATCH))
3574 			arg[id].batch = NULL;
3575 		arg[id].count = 0;
3576 
3577 		worker[id] = kthread_create_worker(0, "igt/smoke:%d", id);
3578 		if (IS_ERR(worker[id])) {
3579 			err = PTR_ERR(worker[id]);
3580 			break;
3581 		}
3582 
3583 		kthread_init_work(&arg[id].work, smoke_crescendo_work);
3584 		kthread_queue_work(worker[id], &arg[id].work);
3585 	}
3586 
3587 	count = 0;
3588 	for_each_engine(engine, smoke->gt, id) {
3589 		if (IS_ERR_OR_NULL(worker[id]))
3590 			continue;
3591 
3592 		kthread_flush_work(&arg[id].work);
3593 		if (arg[id].result && !err)
3594 			err = arg[id].result;
3595 
3596 		count += arg[id].count;
3597 
3598 		kthread_destroy_worker(worker[id]);
3599 	}
3600 
3601 	pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3602 		count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3603 
3604 	kfree(arg);
3605 	return 0;
3606 }
3607 
3608 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
3609 {
3610 	enum intel_engine_id id;
3611 	IGT_TIMEOUT(end_time);
3612 	unsigned long count;
3613 
3614 	count = 0;
3615 	do {
3616 		for_each_engine(smoke->engine, smoke->gt, id) {
3617 			struct i915_gem_context *ctx = smoke_context(smoke);
3618 			int err;
3619 
3620 			err = smoke_submit(smoke,
3621 					   ctx, random_priority(&smoke->prng),
3622 					   flags & BATCH ? smoke->batch : NULL);
3623 			if (err)
3624 				return err;
3625 
3626 			count++;
3627 		}
3628 	} while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3629 
3630 	pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3631 		count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3632 	return 0;
3633 }
3634 
3635 static int live_preempt_smoke(void *arg)
3636 {
3637 	struct preempt_smoke smoke = {
3638 		.gt = arg,
3639 		.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3640 		.ncontext = 256,
3641 	};
3642 	const unsigned int phase[] = { 0, BATCH };
3643 	struct igt_live_test t;
3644 	int err = -ENOMEM;
3645 	u32 *cs;
3646 	int n;
3647 
3648 	smoke.contexts = kmalloc_array(smoke.ncontext,
3649 				       sizeof(*smoke.contexts),
3650 				       GFP_KERNEL);
3651 	if (!smoke.contexts)
3652 		return -ENOMEM;
3653 
3654 	smoke.batch =
3655 		i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
3656 	if (IS_ERR(smoke.batch)) {
3657 		err = PTR_ERR(smoke.batch);
3658 		goto err_free;
3659 	}
3660 
3661 	cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);
3662 	if (IS_ERR(cs)) {
3663 		err = PTR_ERR(cs);
3664 		goto err_batch;
3665 	}
3666 	for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3667 		cs[n] = MI_ARB_CHECK;
3668 	cs[n] = MI_BATCH_BUFFER_END;
3669 	i915_gem_object_flush_map(smoke.batch);
3670 	i915_gem_object_unpin_map(smoke.batch);
3671 
3672 	if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
3673 		err = -EIO;
3674 		goto err_batch;
3675 	}
3676 
3677 	for (n = 0; n < smoke.ncontext; n++) {
3678 		smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL);
3679 		if (!smoke.contexts[n])
3680 			goto err_ctx;
3681 	}
3682 
3683 	for (n = 0; n < ARRAY_SIZE(phase); n++) {
3684 		err = smoke_crescendo(&smoke, phase[n]);
3685 		if (err)
3686 			goto err_ctx;
3687 
3688 		err = smoke_random(&smoke, phase[n]);
3689 		if (err)
3690 			goto err_ctx;
3691 	}
3692 
3693 err_ctx:
3694 	if (igt_live_test_end(&t))
3695 		err = -EIO;
3696 
3697 	for (n = 0; n < smoke.ncontext; n++) {
3698 		if (!smoke.contexts[n])
3699 			break;
3700 		kernel_context_close(smoke.contexts[n]);
3701 	}
3702 
3703 err_batch:
3704 	i915_gem_object_put(smoke.batch);
3705 err_free:
3706 	kfree(smoke.contexts);
3707 
3708 	return err;
3709 }
3710 
3711 static int nop_virtual_engine(struct intel_gt *gt,
3712 			      struct intel_engine_cs **siblings,
3713 			      unsigned int nsibling,
3714 			      unsigned int nctx,
3715 			      unsigned int flags)
3716 #define CHAIN BIT(0)
3717 {
3718 	IGT_TIMEOUT(end_time);
3719 	struct i915_request *request[16] = {};
3720 	struct intel_context *ve[16];
3721 	unsigned long n, prime, nc;
3722 	struct igt_live_test t;
3723 	ktime_t times[2] = {};
3724 	int err;
3725 
3726 	GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
3727 
3728 	for (n = 0; n < nctx; n++) {
3729 		ve[n] = intel_engine_create_virtual(siblings, nsibling, 0);
3730 		if (IS_ERR(ve[n])) {
3731 			err = PTR_ERR(ve[n]);
3732 			nctx = n;
3733 			goto out;
3734 		}
3735 
3736 		err = intel_context_pin(ve[n]);
3737 		if (err) {
3738 			intel_context_put(ve[n]);
3739 			nctx = n;
3740 			goto out;
3741 		}
3742 	}
3743 
3744 	err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
3745 	if (err)
3746 		goto out;
3747 
3748 	for_each_prime_number_from(prime, 1, 8192) {
3749 		times[1] = ktime_get_raw();
3750 
3751 		if (flags & CHAIN) {
3752 			for (nc = 0; nc < nctx; nc++) {
3753 				for (n = 0; n < prime; n++) {
3754 					struct i915_request *rq;
3755 
3756 					rq = i915_request_create(ve[nc]);
3757 					if (IS_ERR(rq)) {
3758 						err = PTR_ERR(rq);
3759 						goto out;
3760 					}
3761 
3762 					if (request[nc])
3763 						i915_request_put(request[nc]);
3764 					request[nc] = i915_request_get(rq);
3765 					i915_request_add(rq);
3766 				}
3767 			}
3768 		} else {
3769 			for (n = 0; n < prime; n++) {
3770 				for (nc = 0; nc < nctx; nc++) {
3771 					struct i915_request *rq;
3772 
3773 					rq = i915_request_create(ve[nc]);
3774 					if (IS_ERR(rq)) {
3775 						err = PTR_ERR(rq);
3776 						goto out;
3777 					}
3778 
3779 					if (request[nc])
3780 						i915_request_put(request[nc]);
3781 					request[nc] = i915_request_get(rq);
3782 					i915_request_add(rq);
3783 				}
3784 			}
3785 		}
3786 
3787 		for (nc = 0; nc < nctx; nc++) {
3788 			if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
3789 				pr_err("%s(%s): wait for %llx:%lld timed out\n",
3790 				       __func__, ve[0]->engine->name,
3791 				       request[nc]->fence.context,
3792 				       request[nc]->fence.seqno);
3793 
3794 				GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3795 					  __func__, ve[0]->engine->name,
3796 					  request[nc]->fence.context,
3797 					  request[nc]->fence.seqno);
3798 				GEM_TRACE_DUMP();
3799 				intel_gt_set_wedged(gt);
3800 				break;
3801 			}
3802 		}
3803 
3804 		times[1] = ktime_sub(ktime_get_raw(), times[1]);
3805 		if (prime == 1)
3806 			times[0] = times[1];
3807 
3808 		for (nc = 0; nc < nctx; nc++) {
3809 			i915_request_put(request[nc]);
3810 			request[nc] = NULL;
3811 		}
3812 
3813 		if (__igt_timeout(end_time, NULL))
3814 			break;
3815 	}
3816 
3817 	err = igt_live_test_end(&t);
3818 	if (err)
3819 		goto out;
3820 
3821 	pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3822 		nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3823 		prime, div64_u64(ktime_to_ns(times[1]), prime));
3824 
3825 out:
3826 	if (igt_flush_test(gt->i915))
3827 		err = -EIO;
3828 
3829 	for (nc = 0; nc < nctx; nc++) {
3830 		i915_request_put(request[nc]);
3831 		intel_context_unpin(ve[nc]);
3832 		intel_context_put(ve[nc]);
3833 	}
3834 	return err;
3835 }
3836 
3837 static unsigned int
3838 __select_siblings(struct intel_gt *gt,
3839 		  unsigned int class,
3840 		  struct intel_engine_cs **siblings,
3841 		  bool (*filter)(const struct intel_engine_cs *))
3842 {
3843 	unsigned int n = 0;
3844 	unsigned int inst;
3845 
3846 	for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3847 		if (!gt->engine_class[class][inst])
3848 			continue;
3849 
3850 		if (filter && !filter(gt->engine_class[class][inst]))
3851 			continue;
3852 
3853 		siblings[n++] = gt->engine_class[class][inst];
3854 	}
3855 
3856 	return n;
3857 }
3858 
3859 static unsigned int
3860 select_siblings(struct intel_gt *gt,
3861 		unsigned int class,
3862 		struct intel_engine_cs **siblings)
3863 {
3864 	return __select_siblings(gt, class, siblings, NULL);
3865 }
3866 
3867 static int live_virtual_engine(void *arg)
3868 {
3869 	struct intel_gt *gt = arg;
3870 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3871 	struct intel_engine_cs *engine;
3872 	enum intel_engine_id id;
3873 	unsigned int class;
3874 	int err;
3875 
3876 	if (intel_uc_uses_guc_submission(&gt->uc))
3877 		return 0;
3878 
3879 	for_each_engine(engine, gt, id) {
3880 		err = nop_virtual_engine(gt, &engine, 1, 1, 0);
3881 		if (err) {
3882 			pr_err("Failed to wrap engine %s: err=%d\n",
3883 			       engine->name, err);
3884 			return err;
3885 		}
3886 	}
3887 
3888 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3889 		int nsibling, n;
3890 
3891 		nsibling = select_siblings(gt, class, siblings);
3892 		if (nsibling < 2)
3893 			continue;
3894 
3895 		for (n = 1; n <= nsibling + 1; n++) {
3896 			err = nop_virtual_engine(gt, siblings, nsibling,
3897 						 n, 0);
3898 			if (err)
3899 				return err;
3900 		}
3901 
3902 		err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
3903 		if (err)
3904 			return err;
3905 	}
3906 
3907 	return 0;
3908 }
3909 
3910 static int mask_virtual_engine(struct intel_gt *gt,
3911 			       struct intel_engine_cs **siblings,
3912 			       unsigned int nsibling)
3913 {
3914 	struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
3915 	struct intel_context *ve;
3916 	struct igt_live_test t;
3917 	unsigned int n;
3918 	int err;
3919 
3920 	/*
3921 	 * Check that by setting the execution mask on a request, we can
3922 	 * restrict it to our desired engine within the virtual engine.
3923 	 */
3924 
3925 	ve = intel_engine_create_virtual(siblings, nsibling, 0);
3926 	if (IS_ERR(ve)) {
3927 		err = PTR_ERR(ve);
3928 		goto out_close;
3929 	}
3930 
3931 	err = intel_context_pin(ve);
3932 	if (err)
3933 		goto out_put;
3934 
3935 	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3936 	if (err)
3937 		goto out_unpin;
3938 
3939 	for (n = 0; n < nsibling; n++) {
3940 		request[n] = i915_request_create(ve);
3941 		if (IS_ERR(request[n])) {
3942 			err = PTR_ERR(request[n]);
3943 			nsibling = n;
3944 			goto out;
3945 		}
3946 
3947 		/* Reverse order as it's more likely to be unnatural */
3948 		request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3949 
3950 		i915_request_get(request[n]);
3951 		i915_request_add(request[n]);
3952 	}
3953 
3954 	for (n = 0; n < nsibling; n++) {
3955 		if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
3956 			pr_err("%s(%s): wait for %llx:%lld timed out\n",
3957 			       __func__, ve->engine->name,
3958 			       request[n]->fence.context,
3959 			       request[n]->fence.seqno);
3960 
3961 			GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3962 				  __func__, ve->engine->name,
3963 				  request[n]->fence.context,
3964 				  request[n]->fence.seqno);
3965 			GEM_TRACE_DUMP();
3966 			intel_gt_set_wedged(gt);
3967 			err = -EIO;
3968 			goto out;
3969 		}
3970 
3971 		if (request[n]->engine != siblings[nsibling - n - 1]) {
3972 			pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3973 			       request[n]->engine->name,
3974 			       siblings[nsibling - n - 1]->name);
3975 			err = -EINVAL;
3976 			goto out;
3977 		}
3978 	}
3979 
3980 	err = igt_live_test_end(&t);
3981 out:
3982 	if (igt_flush_test(gt->i915))
3983 		err = -EIO;
3984 
3985 	for (n = 0; n < nsibling; n++)
3986 		i915_request_put(request[n]);
3987 
3988 out_unpin:
3989 	intel_context_unpin(ve);
3990 out_put:
3991 	intel_context_put(ve);
3992 out_close:
3993 	return err;
3994 }
3995 
3996 static int live_virtual_mask(void *arg)
3997 {
3998 	struct intel_gt *gt = arg;
3999 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4000 	unsigned int class;
4001 	int err;
4002 
4003 	if (intel_uc_uses_guc_submission(&gt->uc))
4004 		return 0;
4005 
4006 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4007 		unsigned int nsibling;
4008 
4009 		nsibling = select_siblings(gt, class, siblings);
4010 		if (nsibling < 2)
4011 			continue;
4012 
4013 		err = mask_virtual_engine(gt, siblings, nsibling);
4014 		if (err)
4015 			return err;
4016 	}
4017 
4018 	return 0;
4019 }
4020 
4021 static int slicein_virtual_engine(struct intel_gt *gt,
4022 				  struct intel_engine_cs **siblings,
4023 				  unsigned int nsibling)
4024 {
4025 	const long timeout = slice_timeout(siblings[0]);
4026 	struct intel_context *ce;
4027 	struct i915_request *rq;
4028 	struct igt_spinner spin;
4029 	unsigned int n;
4030 	int err = 0;
4031 
4032 	/*
4033 	 * Virtual requests must take part in timeslicing on the target engines.
4034 	 */
4035 
4036 	if (igt_spinner_init(&spin, gt))
4037 		return -ENOMEM;
4038 
4039 	for (n = 0; n < nsibling; n++) {
4040 		ce = intel_context_create(siblings[n]);
4041 		if (IS_ERR(ce)) {
4042 			err = PTR_ERR(ce);
4043 			goto out;
4044 		}
4045 
4046 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4047 		intel_context_put(ce);
4048 		if (IS_ERR(rq)) {
4049 			err = PTR_ERR(rq);
4050 			goto out;
4051 		}
4052 
4053 		i915_request_add(rq);
4054 	}
4055 
4056 	ce = intel_engine_create_virtual(siblings, nsibling, 0);
4057 	if (IS_ERR(ce)) {
4058 		err = PTR_ERR(ce);
4059 		goto out;
4060 	}
4061 
4062 	rq = intel_context_create_request(ce);
4063 	intel_context_put(ce);
4064 	if (IS_ERR(rq)) {
4065 		err = PTR_ERR(rq);
4066 		goto out;
4067 	}
4068 
4069 	i915_request_get(rq);
4070 	i915_request_add(rq);
4071 	if (i915_request_wait(rq, 0, timeout) < 0) {
4072 		GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n",
4073 			      __func__, rq->engine->name);
4074 		GEM_TRACE_DUMP();
4075 		intel_gt_set_wedged(gt);
4076 		err = -EIO;
4077 	}
4078 	i915_request_put(rq);
4079 
4080 out:
4081 	igt_spinner_end(&spin);
4082 	if (igt_flush_test(gt->i915))
4083 		err = -EIO;
4084 	igt_spinner_fini(&spin);
4085 	return err;
4086 }
4087 
4088 static int sliceout_virtual_engine(struct intel_gt *gt,
4089 				   struct intel_engine_cs **siblings,
4090 				   unsigned int nsibling)
4091 {
4092 	const long timeout = slice_timeout(siblings[0]);
4093 	struct intel_context *ce;
4094 	struct i915_request *rq;
4095 	struct igt_spinner spin;
4096 	unsigned int n;
4097 	int err = 0;
4098 
4099 	/*
4100 	 * Virtual requests must allow others a fair timeslice.
4101 	 */
4102 
4103 	if (igt_spinner_init(&spin, gt))
4104 		return -ENOMEM;
4105 
4106 	/* XXX We do not handle oversubscription and fairness with normal rq */
4107 	for (n = 0; n < nsibling; n++) {
4108 		ce = intel_engine_create_virtual(siblings, nsibling, 0);
4109 		if (IS_ERR(ce)) {
4110 			err = PTR_ERR(ce);
4111 			goto out;
4112 		}
4113 
4114 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4115 		intel_context_put(ce);
4116 		if (IS_ERR(rq)) {
4117 			err = PTR_ERR(rq);
4118 			goto out;
4119 		}
4120 
4121 		i915_request_add(rq);
4122 	}
4123 
4124 	for (n = 0; !err && n < nsibling; n++) {
4125 		ce = intel_context_create(siblings[n]);
4126 		if (IS_ERR(ce)) {
4127 			err = PTR_ERR(ce);
4128 			goto out;
4129 		}
4130 
4131 		rq = intel_context_create_request(ce);
4132 		intel_context_put(ce);
4133 		if (IS_ERR(rq)) {
4134 			err = PTR_ERR(rq);
4135 			goto out;
4136 		}
4137 
4138 		i915_request_get(rq);
4139 		i915_request_add(rq);
4140 		if (i915_request_wait(rq, 0, timeout) < 0) {
4141 			GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n",
4142 				      __func__, siblings[n]->name);
4143 			GEM_TRACE_DUMP();
4144 			intel_gt_set_wedged(gt);
4145 			err = -EIO;
4146 		}
4147 		i915_request_put(rq);
4148 	}
4149 
4150 out:
4151 	igt_spinner_end(&spin);
4152 	if (igt_flush_test(gt->i915))
4153 		err = -EIO;
4154 	igt_spinner_fini(&spin);
4155 	return err;
4156 }
4157 
4158 static int live_virtual_slice(void *arg)
4159 {
4160 	struct intel_gt *gt = arg;
4161 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4162 	unsigned int class;
4163 	int err;
4164 
4165 	if (intel_uc_uses_guc_submission(&gt->uc))
4166 		return 0;
4167 
4168 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4169 		unsigned int nsibling;
4170 
4171 		nsibling = __select_siblings(gt, class, siblings,
4172 					     intel_engine_has_timeslices);
4173 		if (nsibling < 2)
4174 			continue;
4175 
4176 		err = slicein_virtual_engine(gt, siblings, nsibling);
4177 		if (err)
4178 			return err;
4179 
4180 		err = sliceout_virtual_engine(gt, siblings, nsibling);
4181 		if (err)
4182 			return err;
4183 	}
4184 
4185 	return 0;
4186 }
4187 
4188 static int preserved_virtual_engine(struct intel_gt *gt,
4189 				    struct intel_engine_cs **siblings,
4190 				    unsigned int nsibling)
4191 {
4192 	struct i915_request *last = NULL;
4193 	struct intel_context *ve;
4194 	struct i915_vma *scratch;
4195 	struct igt_live_test t;
4196 	unsigned int n;
4197 	int err = 0;
4198 	u32 *cs;
4199 
4200 	scratch =
4201 		__vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,
4202 						    PAGE_SIZE);
4203 	if (IS_ERR(scratch))
4204 		return PTR_ERR(scratch);
4205 
4206 	err = i915_vma_sync(scratch);
4207 	if (err)
4208 		goto out_scratch;
4209 
4210 	ve = intel_engine_create_virtual(siblings, nsibling, 0);
4211 	if (IS_ERR(ve)) {
4212 		err = PTR_ERR(ve);
4213 		goto out_scratch;
4214 	}
4215 
4216 	err = intel_context_pin(ve);
4217 	if (err)
4218 		goto out_put;
4219 
4220 	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
4221 	if (err)
4222 		goto out_unpin;
4223 
4224 	for (n = 0; n < NUM_GPR_DW; n++) {
4225 		struct intel_engine_cs *engine = siblings[n % nsibling];
4226 		struct i915_request *rq;
4227 
4228 		rq = i915_request_create(ve);
4229 		if (IS_ERR(rq)) {
4230 			err = PTR_ERR(rq);
4231 			goto out_end;
4232 		}
4233 
4234 		i915_request_put(last);
4235 		last = i915_request_get(rq);
4236 
4237 		cs = intel_ring_begin(rq, 8);
4238 		if (IS_ERR(cs)) {
4239 			i915_request_add(rq);
4240 			err = PTR_ERR(cs);
4241 			goto out_end;
4242 		}
4243 
4244 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4245 		*cs++ = CS_GPR(engine, n);
4246 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4247 		*cs++ = 0;
4248 
4249 		*cs++ = MI_LOAD_REGISTER_IMM(1);
4250 		*cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
4251 		*cs++ = n + 1;
4252 
4253 		*cs++ = MI_NOOP;
4254 		intel_ring_advance(rq, cs);
4255 
4256 		/* Restrict this request to run on a particular engine */
4257 		rq->execution_mask = engine->mask;
4258 		i915_request_add(rq);
4259 	}
4260 
4261 	if (i915_request_wait(last, 0, HZ / 5) < 0) {
4262 		err = -ETIME;
4263 		goto out_end;
4264 	}
4265 
4266 	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
4267 	if (IS_ERR(cs)) {
4268 		err = PTR_ERR(cs);
4269 		goto out_end;
4270 	}
4271 
4272 	for (n = 0; n < NUM_GPR_DW; n++) {
4273 		if (cs[n] != n) {
4274 			pr_err("Incorrect value[%d] found for GPR[%d]\n",
4275 			       cs[n], n);
4276 			err = -EINVAL;
4277 			break;
4278 		}
4279 	}
4280 
4281 	i915_gem_object_unpin_map(scratch->obj);
4282 
4283 out_end:
4284 	if (igt_live_test_end(&t))
4285 		err = -EIO;
4286 	i915_request_put(last);
4287 out_unpin:
4288 	intel_context_unpin(ve);
4289 out_put:
4290 	intel_context_put(ve);
4291 out_scratch:
4292 	i915_vma_unpin_and_release(&scratch, 0);
4293 	return err;
4294 }
4295 
4296 static int live_virtual_preserved(void *arg)
4297 {
4298 	struct intel_gt *gt = arg;
4299 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4300 	unsigned int class;
4301 
4302 	/*
4303 	 * Check that the context image retains non-privileged (user) registers
4304 	 * from one engine to the next. For this we check that the CS_GPR
4305 	 * are preserved.
4306 	 */
4307 
4308 	if (intel_uc_uses_guc_submission(&gt->uc))
4309 		return 0;
4310 
4311 	/* As we use CS_GPR we cannot run before they existed on all engines. */
4312 	if (GRAPHICS_VER(gt->i915) < 9)
4313 		return 0;
4314 
4315 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4316 		int nsibling, err;
4317 
4318 		nsibling = select_siblings(gt, class, siblings);
4319 		if (nsibling < 2)
4320 			continue;
4321 
4322 		err = preserved_virtual_engine(gt, siblings, nsibling);
4323 		if (err)
4324 			return err;
4325 	}
4326 
4327 	return 0;
4328 }
4329 
4330 static int reset_virtual_engine(struct intel_gt *gt,
4331 				struct intel_engine_cs **siblings,
4332 				unsigned int nsibling)
4333 {
4334 	struct intel_engine_cs *engine;
4335 	struct intel_context *ve;
4336 	struct igt_spinner spin;
4337 	struct i915_request *rq;
4338 	unsigned int n;
4339 	int err = 0;
4340 
4341 	/*
4342 	 * In order to support offline error capture for fast preempt reset,
4343 	 * we need to decouple the guilty request and ensure that it and its
4344 	 * descendents are not executed while the capture is in progress.
4345 	 */
4346 
4347 	if (igt_spinner_init(&spin, gt))
4348 		return -ENOMEM;
4349 
4350 	ve = intel_engine_create_virtual(siblings, nsibling, 0);
4351 	if (IS_ERR(ve)) {
4352 		err = PTR_ERR(ve);
4353 		goto out_spin;
4354 	}
4355 
4356 	for (n = 0; n < nsibling; n++)
4357 		st_engine_heartbeat_disable(siblings[n]);
4358 
4359 	rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
4360 	if (IS_ERR(rq)) {
4361 		err = PTR_ERR(rq);
4362 		goto out_heartbeat;
4363 	}
4364 	i915_request_add(rq);
4365 
4366 	if (!igt_wait_for_spinner(&spin, rq)) {
4367 		intel_gt_set_wedged(gt);
4368 		err = -ETIME;
4369 		goto out_heartbeat;
4370 	}
4371 
4372 	engine = rq->engine;
4373 	GEM_BUG_ON(engine == ve->engine);
4374 
4375 	/* Take ownership of the reset and tasklet */
4376 	err = engine_lock_reset_tasklet(engine);
4377 	if (err)
4378 		goto out_heartbeat;
4379 
4380 	engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
4381 	GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
4382 
4383 	/* Fake a preemption event; failed of course */
4384 	spin_lock_irq(&engine->sched_engine->lock);
4385 	__unwind_incomplete_requests(engine);
4386 	spin_unlock_irq(&engine->sched_engine->lock);
4387 	GEM_BUG_ON(rq->engine != engine);
4388 
4389 	/* Reset the engine while keeping our active request on hold */
4390 	execlists_hold(engine, rq);
4391 	GEM_BUG_ON(!i915_request_on_hold(rq));
4392 
4393 	__intel_engine_reset_bh(engine, NULL);
4394 	GEM_BUG_ON(rq->fence.error != -EIO);
4395 
4396 	/* Release our grasp on the engine, letting CS flow again */
4397 	engine_unlock_reset_tasklet(engine);
4398 
4399 	/* Check that we do not resubmit the held request */
4400 	i915_request_get(rq);
4401 	if (!i915_request_wait(rq, 0, HZ / 5)) {
4402 		pr_err("%s: on hold request completed!\n",
4403 		       engine->name);
4404 		intel_gt_set_wedged(gt);
4405 		err = -EIO;
4406 		goto out_rq;
4407 	}
4408 	GEM_BUG_ON(!i915_request_on_hold(rq));
4409 
4410 	/* But is resubmitted on release */
4411 	execlists_unhold(engine, rq);
4412 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4413 		pr_err("%s: held request did not complete!\n",
4414 		       engine->name);
4415 		intel_gt_set_wedged(gt);
4416 		err = -ETIME;
4417 	}
4418 
4419 out_rq:
4420 	i915_request_put(rq);
4421 out_heartbeat:
4422 	for (n = 0; n < nsibling; n++)
4423 		st_engine_heartbeat_enable(siblings[n]);
4424 
4425 	intel_context_put(ve);
4426 out_spin:
4427 	igt_spinner_fini(&spin);
4428 	return err;
4429 }
4430 
4431 static int live_virtual_reset(void *arg)
4432 {
4433 	struct intel_gt *gt = arg;
4434 	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4435 	unsigned int class;
4436 
4437 	/*
4438 	 * Check that we handle a reset event within a virtual engine.
4439 	 * Only the physical engine is reset, but we have to check the flow
4440 	 * of the virtual requests around the reset, and make sure it is not
4441 	 * forgotten.
4442 	 */
4443 
4444 	if (intel_uc_uses_guc_submission(&gt->uc))
4445 		return 0;
4446 
4447 	if (!intel_has_reset_engine(gt))
4448 		return 0;
4449 
4450 	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4451 		int nsibling, err;
4452 
4453 		nsibling = select_siblings(gt, class, siblings);
4454 		if (nsibling < 2)
4455 			continue;
4456 
4457 		err = reset_virtual_engine(gt, siblings, nsibling);
4458 		if (err)
4459 			return err;
4460 	}
4461 
4462 	return 0;
4463 }
4464 
4465 int intel_execlists_live_selftests(struct drm_i915_private *i915)
4466 {
4467 	static const struct i915_subtest tests[] = {
4468 		SUBTEST(live_sanitycheck),
4469 		SUBTEST(live_unlite_switch),
4470 		SUBTEST(live_unlite_preempt),
4471 		SUBTEST(live_unlite_ring),
4472 		SUBTEST(live_pin_rewind),
4473 		SUBTEST(live_hold_reset),
4474 		SUBTEST(live_error_interrupt),
4475 		SUBTEST(live_timeslice_preempt),
4476 		SUBTEST(live_timeslice_rewind),
4477 		SUBTEST(live_timeslice_queue),
4478 		SUBTEST(live_timeslice_nopreempt),
4479 		SUBTEST(live_busywait_preempt),
4480 		SUBTEST(live_preempt),
4481 		SUBTEST(live_late_preempt),
4482 		SUBTEST(live_nopreempt),
4483 		SUBTEST(live_preempt_cancel),
4484 		SUBTEST(live_suppress_self_preempt),
4485 		SUBTEST(live_chain_preempt),
4486 		SUBTEST(live_preempt_ring),
4487 		SUBTEST(live_preempt_gang),
4488 		SUBTEST(live_preempt_timeout),
4489 		SUBTEST(live_preempt_user),
4490 		SUBTEST(live_preempt_smoke),
4491 		SUBTEST(live_virtual_engine),
4492 		SUBTEST(live_virtual_mask),
4493 		SUBTEST(live_virtual_preserved),
4494 		SUBTEST(live_virtual_slice),
4495 		SUBTEST(live_virtual_reset),
4496 	};
4497 
4498 	if (to_gt(i915)->submission_method != INTEL_SUBMISSION_ELSP)
4499 		return 0;
4500 
4501 	if (intel_gt_is_wedged(to_gt(i915)))
4502 		return 0;
4503 
4504 	return intel_gt_live_subtests(tests, to_gt(i915));
4505 }
4506