1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2018 Intel Corporation
4 */
5
6 #include <linux/prime_numbers.h>
7
8 #include "gem/i915_gem_internal.h"
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
12 #include "gt/selftest_engine_heartbeat.h"
13
14 #include "i915_selftest.h"
15 #include "selftests/i915_random.h"
16 #include "selftests/igt_flush_test.h"
17 #include "selftests/igt_live_test.h"
18 #include "selftests/igt_spinner.h"
19 #include "selftests/lib_sw_fence.h"
20
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
23
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR 16
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27
is_active(struct i915_request * rq)28 static bool is_active(struct i915_request *rq)
29 {
30 if (i915_request_is_active(rq))
31 return true;
32
33 if (i915_request_on_hold(rq))
34 return true;
35
36 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
37 return true;
38
39 return false;
40 }
41
wait_for_submit(struct intel_engine_cs * engine,struct i915_request * rq,unsigned long timeout)42 static int wait_for_submit(struct intel_engine_cs *engine,
43 struct i915_request *rq,
44 unsigned long timeout)
45 {
46 /* Ignore our own attempts to suppress excess tasklets */
47 tasklet_hi_schedule(&engine->sched_engine->tasklet);
48
49 timeout += jiffies;
50 do {
51 bool done = time_after(jiffies, timeout);
52
53 if (i915_request_completed(rq)) /* that was quick! */
54 return 0;
55
56 /* Wait until the HW has acknowleged the submission (or err) */
57 intel_engine_flush_submission(engine);
58 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
59 return 0;
60
61 if (done)
62 return -ETIME;
63
64 cond_resched();
65 } while (1);
66 }
67
wait_for_reset(struct intel_engine_cs * engine,struct i915_request * rq,unsigned long timeout)68 static int wait_for_reset(struct intel_engine_cs *engine,
69 struct i915_request *rq,
70 unsigned long timeout)
71 {
72 timeout += jiffies;
73
74 do {
75 cond_resched();
76 intel_engine_flush_submission(engine);
77
78 if (READ_ONCE(engine->execlists.pending[0]))
79 continue;
80
81 if (i915_request_completed(rq))
82 break;
83
84 if (READ_ONCE(rq->fence.error))
85 break;
86 } while (time_before(jiffies, timeout));
87
88 if (rq->fence.error != -EIO) {
89 pr_err("%s: hanging request %llx:%lld not reset\n",
90 engine->name,
91 rq->fence.context,
92 rq->fence.seqno);
93 return -EINVAL;
94 }
95
96 /* Give the request a jiffie to complete after flushing the worker */
97 if (i915_request_wait(rq, 0,
98 max(0l, (long)(timeout - jiffies)) + 1) < 0) {
99 pr_err("%s: hanging request %llx:%lld did not complete\n",
100 engine->name,
101 rq->fence.context,
102 rq->fence.seqno);
103 return -ETIME;
104 }
105
106 return 0;
107 }
108
live_sanitycheck(void * arg)109 static int live_sanitycheck(void *arg)
110 {
111 struct intel_gt *gt = arg;
112 struct intel_engine_cs *engine;
113 enum intel_engine_id id;
114 struct igt_spinner spin;
115 int err = 0;
116
117 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
118 return 0;
119
120 if (igt_spinner_init(&spin, gt))
121 return -ENOMEM;
122
123 for_each_engine(engine, gt, id) {
124 struct intel_context *ce;
125 struct i915_request *rq;
126
127 ce = intel_context_create(engine);
128 if (IS_ERR(ce)) {
129 err = PTR_ERR(ce);
130 break;
131 }
132
133 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
134 if (IS_ERR(rq)) {
135 err = PTR_ERR(rq);
136 goto out_ctx;
137 }
138
139 i915_request_add(rq);
140 if (!igt_wait_for_spinner(&spin, rq)) {
141 GEM_TRACE("spinner failed to start\n");
142 GEM_TRACE_DUMP();
143 intel_gt_set_wedged(gt);
144 err = -EIO;
145 goto out_ctx;
146 }
147
148 igt_spinner_end(&spin);
149 if (igt_flush_test(gt->i915)) {
150 err = -EIO;
151 goto out_ctx;
152 }
153
154 out_ctx:
155 intel_context_put(ce);
156 if (err)
157 break;
158 }
159
160 igt_spinner_fini(&spin);
161 return err;
162 }
163
live_unlite_restore(struct intel_gt * gt,int prio)164 static int live_unlite_restore(struct intel_gt *gt, int prio)
165 {
166 struct intel_engine_cs *engine;
167 enum intel_engine_id id;
168 struct igt_spinner spin;
169 int err = -ENOMEM;
170
171 /*
172 * Check that we can correctly context switch between 2 instances
173 * on the same engine from the same parent context.
174 */
175
176 if (igt_spinner_init(&spin, gt))
177 return err;
178
179 err = 0;
180 for_each_engine(engine, gt, id) {
181 struct intel_context *ce[2] = {};
182 struct i915_request *rq[2];
183 struct igt_live_test t;
184 int n;
185
186 if (prio && !intel_engine_has_preemption(engine))
187 continue;
188
189 if (!intel_engine_can_store_dword(engine))
190 continue;
191
192 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
193 err = -EIO;
194 break;
195 }
196 st_engine_heartbeat_disable(engine);
197
198 for (n = 0; n < ARRAY_SIZE(ce); n++) {
199 struct intel_context *tmp;
200
201 tmp = intel_context_create(engine);
202 if (IS_ERR(tmp)) {
203 err = PTR_ERR(tmp);
204 goto err_ce;
205 }
206
207 err = intel_context_pin(tmp);
208 if (err) {
209 intel_context_put(tmp);
210 goto err_ce;
211 }
212
213 /*
214 * Setup the pair of contexts such that if we
215 * lite-restore using the RING_TAIL from ce[1] it
216 * will execute garbage from ce[0]->ring.
217 */
218 memset(tmp->ring->vaddr,
219 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
220 tmp->ring->vma->size);
221
222 ce[n] = tmp;
223 }
224 GEM_BUG_ON(!ce[1]->ring->size);
225 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
226 lrc_update_regs(ce[1], engine, ce[1]->ring->head);
227
228 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
229 if (IS_ERR(rq[0])) {
230 err = PTR_ERR(rq[0]);
231 goto err_ce;
232 }
233
234 i915_request_get(rq[0]);
235 i915_request_add(rq[0]);
236 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
237
238 if (!igt_wait_for_spinner(&spin, rq[0])) {
239 i915_request_put(rq[0]);
240 goto err_ce;
241 }
242
243 rq[1] = i915_request_create(ce[1]);
244 if (IS_ERR(rq[1])) {
245 err = PTR_ERR(rq[1]);
246 i915_request_put(rq[0]);
247 goto err_ce;
248 }
249
250 if (!prio) {
251 /*
252 * Ensure we do the switch to ce[1] on completion.
253 *
254 * rq[0] is already submitted, so this should reduce
255 * to a no-op (a wait on a request on the same engine
256 * uses the submit fence, not the completion fence),
257 * but it will install a dependency on rq[1] for rq[0]
258 * that will prevent the pair being reordered by
259 * timeslicing.
260 */
261 i915_request_await_dma_fence(rq[1], &rq[0]->fence);
262 }
263
264 i915_request_get(rq[1]);
265 i915_request_add(rq[1]);
266 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
267 i915_request_put(rq[0]);
268
269 if (prio) {
270 struct i915_sched_attr attr = {
271 .priority = prio,
272 };
273
274 /* Alternatively preempt the spinner with ce[1] */
275 engine->sched_engine->schedule(rq[1], &attr);
276 }
277
278 /* And switch back to ce[0] for good measure */
279 rq[0] = i915_request_create(ce[0]);
280 if (IS_ERR(rq[0])) {
281 err = PTR_ERR(rq[0]);
282 i915_request_put(rq[1]);
283 goto err_ce;
284 }
285
286 i915_request_await_dma_fence(rq[0], &rq[1]->fence);
287 i915_request_get(rq[0]);
288 i915_request_add(rq[0]);
289 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
290 i915_request_put(rq[1]);
291 i915_request_put(rq[0]);
292
293 err_ce:
294 intel_engine_flush_submission(engine);
295 igt_spinner_end(&spin);
296 for (n = 0; n < ARRAY_SIZE(ce); n++) {
297 if (IS_ERR_OR_NULL(ce[n]))
298 break;
299
300 intel_context_unpin(ce[n]);
301 intel_context_put(ce[n]);
302 }
303
304 st_engine_heartbeat_enable(engine);
305 if (igt_live_test_end(&t))
306 err = -EIO;
307 if (err)
308 break;
309 }
310
311 igt_spinner_fini(&spin);
312 return err;
313 }
314
live_unlite_switch(void * arg)315 static int live_unlite_switch(void *arg)
316 {
317 return live_unlite_restore(arg, 0);
318 }
319
live_unlite_preempt(void * arg)320 static int live_unlite_preempt(void *arg)
321 {
322 return live_unlite_restore(arg, I915_PRIORITY_MAX);
323 }
324
live_unlite_ring(void * arg)325 static int live_unlite_ring(void *arg)
326 {
327 struct intel_gt *gt = arg;
328 struct intel_engine_cs *engine;
329 struct igt_spinner spin;
330 enum intel_engine_id id;
331 int err = 0;
332
333 /*
334 * Setup a preemption event that will cause almost the entire ring
335 * to be unwound, potentially fooling our intel_ring_direction()
336 * into emitting a forward lite-restore instead of the rollback.
337 */
338
339 if (igt_spinner_init(&spin, gt))
340 return -ENOMEM;
341
342 for_each_engine(engine, gt, id) {
343 struct intel_context *ce[2] = {};
344 struct i915_request *rq;
345 struct igt_live_test t;
346 int n;
347
348 if (!intel_engine_has_preemption(engine))
349 continue;
350
351 if (!intel_engine_can_store_dword(engine))
352 continue;
353
354 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
355 err = -EIO;
356 break;
357 }
358 st_engine_heartbeat_disable(engine);
359
360 for (n = 0; n < ARRAY_SIZE(ce); n++) {
361 struct intel_context *tmp;
362
363 tmp = intel_context_create(engine);
364 if (IS_ERR(tmp)) {
365 err = PTR_ERR(tmp);
366 goto err_ce;
367 }
368
369 err = intel_context_pin(tmp);
370 if (err) {
371 intel_context_put(tmp);
372 goto err_ce;
373 }
374
375 memset32(tmp->ring->vaddr,
376 0xdeadbeef, /* trigger a hang if executed */
377 tmp->ring->vma->size / sizeof(u32));
378
379 ce[n] = tmp;
380 }
381
382 /* Create max prio spinner, followed by N low prio nops */
383 rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
384 if (IS_ERR(rq)) {
385 err = PTR_ERR(rq);
386 goto err_ce;
387 }
388
389 i915_request_get(rq);
390 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
391 i915_request_add(rq);
392
393 if (!igt_wait_for_spinner(&spin, rq)) {
394 intel_gt_set_wedged(gt);
395 i915_request_put(rq);
396 err = -ETIME;
397 goto err_ce;
398 }
399
400 /* Fill the ring, until we will cause a wrap */
401 n = 0;
402 while (intel_ring_direction(ce[0]->ring,
403 rq->wa_tail,
404 ce[0]->ring->tail) <= 0) {
405 struct i915_request *tmp;
406
407 tmp = intel_context_create_request(ce[0]);
408 if (IS_ERR(tmp)) {
409 err = PTR_ERR(tmp);
410 i915_request_put(rq);
411 goto err_ce;
412 }
413
414 i915_request_add(tmp);
415 intel_engine_flush_submission(engine);
416 n++;
417 }
418 intel_engine_flush_submission(engine);
419 pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
420 engine->name, n,
421 ce[0]->ring->size,
422 ce[0]->ring->tail,
423 ce[0]->ring->emit,
424 rq->tail);
425 GEM_BUG_ON(intel_ring_direction(ce[0]->ring,
426 rq->tail,
427 ce[0]->ring->tail) <= 0);
428 i915_request_put(rq);
429
430 /* Create a second ring to preempt the first ring after rq[0] */
431 rq = intel_context_create_request(ce[1]);
432 if (IS_ERR(rq)) {
433 err = PTR_ERR(rq);
434 goto err_ce;
435 }
436
437 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
438 i915_request_get(rq);
439 i915_request_add(rq);
440
441 err = wait_for_submit(engine, rq, HZ / 2);
442 i915_request_put(rq);
443 if (err) {
444 pr_err("%s: preemption request was not submitted\n",
445 engine->name);
446 err = -ETIME;
447 }
448
449 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
450 engine->name,
451 ce[0]->ring->tail, ce[0]->ring->emit,
452 ce[1]->ring->tail, ce[1]->ring->emit);
453
454 err_ce:
455 intel_engine_flush_submission(engine);
456 igt_spinner_end(&spin);
457 for (n = 0; n < ARRAY_SIZE(ce); n++) {
458 if (IS_ERR_OR_NULL(ce[n]))
459 break;
460
461 intel_context_unpin(ce[n]);
462 intel_context_put(ce[n]);
463 }
464 st_engine_heartbeat_enable(engine);
465 if (igt_live_test_end(&t))
466 err = -EIO;
467 if (err)
468 break;
469 }
470
471 igt_spinner_fini(&spin);
472 return err;
473 }
474
live_pin_rewind(void * arg)475 static int live_pin_rewind(void *arg)
476 {
477 struct intel_gt *gt = arg;
478 struct intel_engine_cs *engine;
479 enum intel_engine_id id;
480 int err = 0;
481
482 /*
483 * We have to be careful not to trust intel_ring too much, for example
484 * ring->head is updated upon retire which is out of sync with pinning
485 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,
486 * or else we risk writing an older, stale value.
487 *
488 * To simulate this, let's apply a bit of deliberate sabotague.
489 */
490
491 for_each_engine(engine, gt, id) {
492 struct intel_context *ce;
493 struct i915_request *rq;
494 struct intel_ring *ring;
495 struct igt_live_test t;
496
497 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
498 err = -EIO;
499 break;
500 }
501
502 ce = intel_context_create(engine);
503 if (IS_ERR(ce)) {
504 err = PTR_ERR(ce);
505 break;
506 }
507
508 err = intel_context_pin(ce);
509 if (err) {
510 intel_context_put(ce);
511 break;
512 }
513
514 /* Keep the context awake while we play games */
515 err = i915_active_acquire(&ce->active);
516 if (err) {
517 intel_context_unpin(ce);
518 intel_context_put(ce);
519 break;
520 }
521 ring = ce->ring;
522
523 /* Poison the ring, and offset the next request from HEAD */
524 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
525 ring->emit = ring->size / 2;
526 ring->tail = ring->emit;
527 GEM_BUG_ON(ring->head);
528
529 intel_context_unpin(ce);
530
531 /* Submit a simple nop request */
532 GEM_BUG_ON(intel_context_is_pinned(ce));
533 rq = intel_context_create_request(ce);
534 i915_active_release(&ce->active); /* e.g. async retire */
535 intel_context_put(ce);
536 if (IS_ERR(rq)) {
537 err = PTR_ERR(rq);
538 break;
539 }
540 GEM_BUG_ON(!rq->head);
541 i915_request_add(rq);
542
543 /* Expect not to hang! */
544 if (igt_live_test_end(&t)) {
545 err = -EIO;
546 break;
547 }
548 }
549
550 return err;
551 }
552
engine_lock_reset_tasklet(struct intel_engine_cs * engine)553 static int engine_lock_reset_tasklet(struct intel_engine_cs *engine)
554 {
555 tasklet_disable(&engine->sched_engine->tasklet);
556 local_bh_disable();
557
558 if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
559 &engine->gt->reset.flags)) {
560 local_bh_enable();
561 tasklet_enable(&engine->sched_engine->tasklet);
562
563 intel_gt_set_wedged(engine->gt);
564 return -EBUSY;
565 }
566
567 return 0;
568 }
569
engine_unlock_reset_tasklet(struct intel_engine_cs * engine)570 static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine)
571 {
572 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,
573 &engine->gt->reset.flags);
574
575 local_bh_enable();
576 tasklet_enable(&engine->sched_engine->tasklet);
577 }
578
live_hold_reset(void * arg)579 static int live_hold_reset(void *arg)
580 {
581 struct intel_gt *gt = arg;
582 struct intel_engine_cs *engine;
583 enum intel_engine_id id;
584 struct igt_spinner spin;
585 int err = 0;
586
587 /*
588 * In order to support offline error capture for fast preempt reset,
589 * we need to decouple the guilty request and ensure that it and its
590 * descendents are not executed while the capture is in progress.
591 */
592
593 if (!intel_has_reset_engine(gt))
594 return 0;
595
596 if (igt_spinner_init(&spin, gt))
597 return -ENOMEM;
598
599 for_each_engine(engine, gt, id) {
600 struct intel_context *ce;
601 struct i915_request *rq;
602
603 ce = intel_context_create(engine);
604 if (IS_ERR(ce)) {
605 err = PTR_ERR(ce);
606 break;
607 }
608
609 st_engine_heartbeat_disable(engine);
610
611 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
612 if (IS_ERR(rq)) {
613 err = PTR_ERR(rq);
614 goto out;
615 }
616 i915_request_add(rq);
617
618 if (!igt_wait_for_spinner(&spin, rq)) {
619 intel_gt_set_wedged(gt);
620 err = -ETIME;
621 goto out;
622 }
623
624 /* We have our request executing, now remove it and reset */
625
626 err = engine_lock_reset_tasklet(engine);
627 if (err)
628 goto out;
629
630 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
631 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
632
633 i915_request_get(rq);
634 execlists_hold(engine, rq);
635 GEM_BUG_ON(!i915_request_on_hold(rq));
636
637 __intel_engine_reset_bh(engine, NULL);
638 GEM_BUG_ON(rq->fence.error != -EIO);
639
640 engine_unlock_reset_tasklet(engine);
641
642 /* Check that we do not resubmit the held request */
643 if (!i915_request_wait(rq, 0, HZ / 5)) {
644 pr_err("%s: on hold request completed!\n",
645 engine->name);
646 i915_request_put(rq);
647 err = -EIO;
648 goto out;
649 }
650 GEM_BUG_ON(!i915_request_on_hold(rq));
651
652 /* But is resubmitted on release */
653 execlists_unhold(engine, rq);
654 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
655 pr_err("%s: held request did not complete!\n",
656 engine->name);
657 intel_gt_set_wedged(gt);
658 err = -ETIME;
659 }
660 i915_request_put(rq);
661
662 out:
663 st_engine_heartbeat_enable(engine);
664 intel_context_put(ce);
665 if (err)
666 break;
667 }
668
669 igt_spinner_fini(&spin);
670 return err;
671 }
672
error_repr(int err)673 static const char *error_repr(int err)
674 {
675 return err ? "bad" : "good";
676 }
677
live_error_interrupt(void * arg)678 static int live_error_interrupt(void *arg)
679 {
680 static const struct error_phase {
681 enum { GOOD = 0, BAD = -EIO } error[2];
682 } phases[] = {
683 { { BAD, GOOD } },
684 { { BAD, BAD } },
685 { { BAD, GOOD } },
686 { { GOOD, GOOD } }, /* sentinel */
687 };
688 struct intel_gt *gt = arg;
689 struct intel_engine_cs *engine;
690 enum intel_engine_id id;
691
692 /*
693 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning
694 * of invalid commands in user batches that will cause a GPU hang.
695 * This is a faster mechanism than using hangcheck/heartbeats, but
696 * only detects problems the HW knows about -- it will not warn when
697 * we kill the HW!
698 *
699 * To verify our detection and reset, we throw some invalid commands
700 * at the HW and wait for the interrupt.
701 */
702
703 if (!intel_has_reset_engine(gt))
704 return 0;
705
706 for_each_engine(engine, gt, id) {
707 const struct error_phase *p;
708 int err = 0;
709
710 st_engine_heartbeat_disable(engine);
711
712 for (p = phases; p->error[0] != GOOD; p++) {
713 struct i915_request *client[ARRAY_SIZE(phases->error)];
714 u32 *cs;
715 int i;
716
717 memset(client, 0, sizeof(*client));
718 for (i = 0; i < ARRAY_SIZE(client); i++) {
719 struct intel_context *ce;
720 struct i915_request *rq;
721
722 ce = intel_context_create(engine);
723 if (IS_ERR(ce)) {
724 err = PTR_ERR(ce);
725 goto out;
726 }
727
728 rq = intel_context_create_request(ce);
729 intel_context_put(ce);
730 if (IS_ERR(rq)) {
731 err = PTR_ERR(rq);
732 goto out;
733 }
734
735 if (rq->engine->emit_init_breadcrumb) {
736 err = rq->engine->emit_init_breadcrumb(rq);
737 if (err) {
738 i915_request_add(rq);
739 goto out;
740 }
741 }
742
743 cs = intel_ring_begin(rq, 2);
744 if (IS_ERR(cs)) {
745 i915_request_add(rq);
746 err = PTR_ERR(cs);
747 goto out;
748 }
749
750 if (p->error[i]) {
751 *cs++ = 0xdeadbeef;
752 *cs++ = 0xdeadbeef;
753 } else {
754 *cs++ = MI_NOOP;
755 *cs++ = MI_NOOP;
756 }
757
758 client[i] = i915_request_get(rq);
759 i915_request_add(rq);
760 }
761
762 err = wait_for_submit(engine, client[0], HZ / 2);
763 if (err) {
764 pr_err("%s: first request did not start within time!\n",
765 engine->name);
766 err = -ETIME;
767 goto out;
768 }
769
770 for (i = 0; i < ARRAY_SIZE(client); i++) {
771 if (i915_request_wait(client[i], 0, HZ / 5) < 0)
772 pr_debug("%s: %s request incomplete!\n",
773 engine->name,
774 error_repr(p->error[i]));
775
776 if (!i915_request_started(client[i])) {
777 pr_err("%s: %s request not started!\n",
778 engine->name,
779 error_repr(p->error[i]));
780 err = -ETIME;
781 goto out;
782 }
783
784 /* Kick the tasklet to process the error */
785 intel_engine_flush_submission(engine);
786 if (client[i]->fence.error != p->error[i]) {
787 pr_err("%s: %s request (%s) with wrong error code: %d\n",
788 engine->name,
789 error_repr(p->error[i]),
790 i915_request_completed(client[i]) ? "completed" : "running",
791 client[i]->fence.error);
792 err = -EINVAL;
793 goto out;
794 }
795 }
796
797 out:
798 for (i = 0; i < ARRAY_SIZE(client); i++)
799 if (client[i])
800 i915_request_put(client[i]);
801 if (err) {
802 pr_err("%s: failed at phase[%zd] { %d, %d }\n",
803 engine->name, p - phases,
804 p->error[0], p->error[1]);
805 break;
806 }
807 }
808
809 st_engine_heartbeat_enable(engine);
810 if (err) {
811 intel_gt_set_wedged(gt);
812 return err;
813 }
814 }
815
816 return 0;
817 }
818
819 static int
emit_semaphore_chain(struct i915_request * rq,struct i915_vma * vma,int idx)820 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
821 {
822 u32 *cs;
823
824 cs = intel_ring_begin(rq, 10);
825 if (IS_ERR(cs))
826 return PTR_ERR(cs);
827
828 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
829
830 *cs++ = MI_SEMAPHORE_WAIT |
831 MI_SEMAPHORE_GLOBAL_GTT |
832 MI_SEMAPHORE_POLL |
833 MI_SEMAPHORE_SAD_NEQ_SDD;
834 *cs++ = 0;
835 *cs++ = i915_ggtt_offset(vma) + 4 * idx;
836 *cs++ = 0;
837
838 if (idx > 0) {
839 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
840 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
841 *cs++ = 0;
842 *cs++ = 1;
843 } else {
844 *cs++ = MI_NOOP;
845 *cs++ = MI_NOOP;
846 *cs++ = MI_NOOP;
847 *cs++ = MI_NOOP;
848 }
849
850 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
851
852 intel_ring_advance(rq, cs);
853 return 0;
854 }
855
856 static struct i915_request *
semaphore_queue(struct intel_engine_cs * engine,struct i915_vma * vma,int idx)857 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
858 {
859 struct intel_context *ce;
860 struct i915_request *rq;
861 int err;
862
863 ce = intel_context_create(engine);
864 if (IS_ERR(ce))
865 return ERR_CAST(ce);
866
867 rq = intel_context_create_request(ce);
868 if (IS_ERR(rq))
869 goto out_ce;
870
871 err = 0;
872 if (rq->engine->emit_init_breadcrumb)
873 err = rq->engine->emit_init_breadcrumb(rq);
874 if (err == 0)
875 err = emit_semaphore_chain(rq, vma, idx);
876 if (err == 0)
877 i915_request_get(rq);
878 i915_request_add(rq);
879 if (err)
880 rq = ERR_PTR(err);
881
882 out_ce:
883 intel_context_put(ce);
884 return rq;
885 }
886
887 static int
release_queue(struct intel_engine_cs * engine,struct i915_vma * vma,int idx,int prio)888 release_queue(struct intel_engine_cs *engine,
889 struct i915_vma *vma,
890 int idx, int prio)
891 {
892 struct i915_sched_attr attr = {
893 .priority = prio,
894 };
895 struct i915_request *rq;
896 u32 *cs;
897
898 rq = intel_engine_create_kernel_request(engine);
899 if (IS_ERR(rq))
900 return PTR_ERR(rq);
901
902 cs = intel_ring_begin(rq, 4);
903 if (IS_ERR(cs)) {
904 i915_request_add(rq);
905 return PTR_ERR(cs);
906 }
907
908 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
909 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
910 *cs++ = 0;
911 *cs++ = 1;
912
913 intel_ring_advance(rq, cs);
914
915 i915_request_get(rq);
916 i915_request_add(rq);
917
918 local_bh_disable();
919 engine->sched_engine->schedule(rq, &attr);
920 local_bh_enable(); /* kick tasklet */
921
922 i915_request_put(rq);
923
924 return 0;
925 }
926
927 static int
slice_semaphore_queue(struct intel_engine_cs * outer,struct i915_vma * vma,int count)928 slice_semaphore_queue(struct intel_engine_cs *outer,
929 struct i915_vma *vma,
930 int count)
931 {
932 struct intel_engine_cs *engine;
933 struct i915_request *head;
934 enum intel_engine_id id;
935 int err, i, n = 0;
936
937 head = semaphore_queue(outer, vma, n++);
938 if (IS_ERR(head))
939 return PTR_ERR(head);
940
941 for_each_engine(engine, outer->gt, id) {
942 if (!intel_engine_has_preemption(engine))
943 continue;
944
945 for (i = 0; i < count; i++) {
946 struct i915_request *rq;
947
948 rq = semaphore_queue(engine, vma, n++);
949 if (IS_ERR(rq)) {
950 err = PTR_ERR(rq);
951 goto out;
952 }
953
954 i915_request_put(rq);
955 }
956 }
957
958 err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
959 if (err)
960 goto out;
961
962 if (i915_request_wait(head, 0,
963 2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) {
964 pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n",
965 outer->name, count, n);
966 GEM_TRACE_DUMP();
967 intel_gt_set_wedged(outer->gt);
968 err = -EIO;
969 }
970
971 out:
972 i915_request_put(head);
973 return err;
974 }
975
live_timeslice_preempt(void * arg)976 static int live_timeslice_preempt(void *arg)
977 {
978 struct intel_gt *gt = arg;
979 struct drm_i915_gem_object *obj;
980 struct intel_engine_cs *engine;
981 enum intel_engine_id id;
982 struct i915_vma *vma;
983 void *vaddr;
984 int err = 0;
985
986 /*
987 * If a request takes too long, we would like to give other users
988 * a fair go on the GPU. In particular, users may create batches
989 * that wait upon external input, where that input may even be
990 * supplied by another GPU job. To avoid blocking forever, we
991 * need to preempt the current task and replace it with another
992 * ready task.
993 */
994 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
995 return 0;
996
997 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
998 if (IS_ERR(obj))
999 return PTR_ERR(obj);
1000
1001 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1002 if (IS_ERR(vma)) {
1003 err = PTR_ERR(vma);
1004 goto err_obj;
1005 }
1006
1007 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1008 if (IS_ERR(vaddr)) {
1009 err = PTR_ERR(vaddr);
1010 goto err_obj;
1011 }
1012
1013 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1014 if (err)
1015 goto err_map;
1016
1017 err = i915_vma_sync(vma);
1018 if (err)
1019 goto err_pin;
1020
1021 for_each_engine(engine, gt, id) {
1022 if (!intel_engine_has_preemption(engine))
1023 continue;
1024
1025 memset(vaddr, 0, PAGE_SIZE);
1026
1027 st_engine_heartbeat_disable(engine);
1028 err = slice_semaphore_queue(engine, vma, 5);
1029 st_engine_heartbeat_enable(engine);
1030 if (err)
1031 goto err_pin;
1032
1033 if (igt_flush_test(gt->i915)) {
1034 err = -EIO;
1035 goto err_pin;
1036 }
1037 }
1038
1039 err_pin:
1040 i915_vma_unpin(vma);
1041 err_map:
1042 i915_gem_object_unpin_map(obj);
1043 err_obj:
1044 i915_gem_object_put(obj);
1045 return err;
1046 }
1047
1048 static struct i915_request *
create_rewinder(struct intel_context * ce,struct i915_request * wait,void * slot,int idx)1049 create_rewinder(struct intel_context *ce,
1050 struct i915_request *wait,
1051 void *slot, int idx)
1052 {
1053 const u32 offset =
1054 i915_ggtt_offset(ce->engine->status_page.vma) +
1055 offset_in_page(slot);
1056 struct i915_request *rq;
1057 u32 *cs;
1058 int err;
1059
1060 rq = intel_context_create_request(ce);
1061 if (IS_ERR(rq))
1062 return rq;
1063
1064 if (wait) {
1065 err = i915_request_await_dma_fence(rq, &wait->fence);
1066 if (err)
1067 goto err;
1068 }
1069
1070 cs = intel_ring_begin(rq, 14);
1071 if (IS_ERR(cs)) {
1072 err = PTR_ERR(cs);
1073 goto err;
1074 }
1075
1076 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1077 *cs++ = MI_NOOP;
1078
1079 *cs++ = MI_SEMAPHORE_WAIT |
1080 MI_SEMAPHORE_GLOBAL_GTT |
1081 MI_SEMAPHORE_POLL |
1082 MI_SEMAPHORE_SAD_GTE_SDD;
1083 *cs++ = idx;
1084 *cs++ = offset;
1085 *cs++ = 0;
1086
1087 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1088 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
1089 *cs++ = offset + idx * sizeof(u32);
1090 *cs++ = 0;
1091
1092 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1093 *cs++ = offset;
1094 *cs++ = 0;
1095 *cs++ = idx + 1;
1096
1097 intel_ring_advance(rq, cs);
1098
1099 err = 0;
1100 err:
1101 i915_request_get(rq);
1102 i915_request_add(rq);
1103 if (err) {
1104 i915_request_put(rq);
1105 return ERR_PTR(err);
1106 }
1107
1108 return rq;
1109 }
1110
live_timeslice_rewind(void * arg)1111 static int live_timeslice_rewind(void *arg)
1112 {
1113 struct intel_gt *gt = arg;
1114 struct intel_engine_cs *engine;
1115 enum intel_engine_id id;
1116
1117 /*
1118 * The usual presumption on timeslice expiration is that we replace
1119 * the active context with another. However, given a chain of
1120 * dependencies we may end up with replacing the context with itself,
1121 * but only a few of those requests, forcing us to rewind the
1122 * RING_TAIL of the original request.
1123 */
1124 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1125 return 0;
1126
1127 for_each_engine(engine, gt, id) {
1128 enum { A1, A2, B1 };
1129 enum { X = 1, Z, Y };
1130 struct i915_request *rq[3] = {};
1131 struct intel_context *ce;
1132 unsigned long timeslice;
1133 int i, err = 0;
1134 u32 *slot;
1135
1136 if (!intel_engine_has_timeslices(engine))
1137 continue;
1138
1139 /*
1140 * A:rq1 -- semaphore wait, timestamp X
1141 * A:rq2 -- write timestamp Y
1142 *
1143 * B:rq1 [await A:rq1] -- write timestamp Z
1144 *
1145 * Force timeslice, release semaphore.
1146 *
1147 * Expect execution/evaluation order XZY
1148 */
1149
1150 st_engine_heartbeat_disable(engine);
1151 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1152
1153 slot = memset32(engine->status_page.addr + 1000, 0, 4);
1154
1155 ce = intel_context_create(engine);
1156 if (IS_ERR(ce)) {
1157 err = PTR_ERR(ce);
1158 goto err;
1159 }
1160
1161 rq[A1] = create_rewinder(ce, NULL, slot, X);
1162 if (IS_ERR(rq[A1])) {
1163 intel_context_put(ce);
1164 goto err;
1165 }
1166
1167 rq[A2] = create_rewinder(ce, NULL, slot, Y);
1168 intel_context_put(ce);
1169 if (IS_ERR(rq[A2]))
1170 goto err;
1171
1172 err = wait_for_submit(engine, rq[A2], HZ / 2);
1173 if (err) {
1174 pr_err("%s: failed to submit first context\n",
1175 engine->name);
1176 goto err;
1177 }
1178
1179 ce = intel_context_create(engine);
1180 if (IS_ERR(ce)) {
1181 err = PTR_ERR(ce);
1182 goto err;
1183 }
1184
1185 rq[B1] = create_rewinder(ce, rq[A1], slot, Z);
1186 intel_context_put(ce);
1187 if (IS_ERR(rq[2]))
1188 goto err;
1189
1190 err = wait_for_submit(engine, rq[B1], HZ / 2);
1191 if (err) {
1192 pr_err("%s: failed to submit second context\n",
1193 engine->name);
1194 goto err;
1195 }
1196
1197 /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */
1198 ENGINE_TRACE(engine, "forcing tasklet for rewind\n");
1199 while (i915_request_is_active(rq[A2])) { /* semaphore yield! */
1200 /* Wait for the timeslice to kick in */
1201 del_timer(&engine->execlists.timer);
1202 tasklet_hi_schedule(&engine->sched_engine->tasklet);
1203 intel_engine_flush_submission(engine);
1204 }
1205 /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */
1206 GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1207 GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1208 GEM_BUG_ON(i915_request_is_active(rq[A2]));
1209
1210 /* Release the hounds! */
1211 slot[0] = 1;
1212 wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */
1213
1214 for (i = 1; i <= 3; i++) {
1215 unsigned long timeout = jiffies + HZ / 2;
1216
1217 while (!READ_ONCE(slot[i]) &&
1218 time_before(jiffies, timeout))
1219 ;
1220
1221 if (!time_before(jiffies, timeout)) {
1222 pr_err("%s: rq[%d] timed out\n",
1223 engine->name, i - 1);
1224 err = -ETIME;
1225 goto err;
1226 }
1227
1228 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1229 }
1230
1231 /* XZY: XZ < XY */
1232 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1233 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1234 engine->name,
1235 slot[Z] - slot[X],
1236 slot[Y] - slot[X]);
1237 err = -EINVAL;
1238 }
1239
1240 err:
1241 memset32(&slot[0], -1, 4);
1242 wmb();
1243
1244 engine->props.timeslice_duration_ms = timeslice;
1245 st_engine_heartbeat_enable(engine);
1246 for (i = 0; i < 3; i++)
1247 i915_request_put(rq[i]);
1248 if (igt_flush_test(gt->i915))
1249 err = -EIO;
1250 if (err)
1251 return err;
1252 }
1253
1254 return 0;
1255 }
1256
nop_request(struct intel_engine_cs * engine)1257 static struct i915_request *nop_request(struct intel_engine_cs *engine)
1258 {
1259 struct i915_request *rq;
1260
1261 rq = intel_engine_create_kernel_request(engine);
1262 if (IS_ERR(rq))
1263 return rq;
1264
1265 i915_request_get(rq);
1266 i915_request_add(rq);
1267
1268 return rq;
1269 }
1270
slice_timeout(struct intel_engine_cs * engine)1271 static long slice_timeout(struct intel_engine_cs *engine)
1272 {
1273 long timeout;
1274
1275 /* Enough time for a timeslice to kick in, and kick out */
1276 timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine));
1277
1278 /* Enough time for the nop request to complete */
1279 timeout += HZ / 5;
1280
1281 return timeout + 1;
1282 }
1283
live_timeslice_queue(void * arg)1284 static int live_timeslice_queue(void *arg)
1285 {
1286 struct intel_gt *gt = arg;
1287 struct drm_i915_gem_object *obj;
1288 struct intel_engine_cs *engine;
1289 enum intel_engine_id id;
1290 struct i915_vma *vma;
1291 void *vaddr;
1292 int err = 0;
1293
1294 /*
1295 * Make sure that even if ELSP[0] and ELSP[1] are filled with
1296 * timeslicing between them disabled, we *do* enable timeslicing
1297 * if the queue demands it. (Normally, we do not submit if
1298 * ELSP[1] is already occupied, so must rely on timeslicing to
1299 * eject ELSP[0] in favour of the queue.)
1300 */
1301 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1302 return 0;
1303
1304 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1305 if (IS_ERR(obj))
1306 return PTR_ERR(obj);
1307
1308 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1309 if (IS_ERR(vma)) {
1310 err = PTR_ERR(vma);
1311 goto err_obj;
1312 }
1313
1314 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1315 if (IS_ERR(vaddr)) {
1316 err = PTR_ERR(vaddr);
1317 goto err_obj;
1318 }
1319
1320 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1321 if (err)
1322 goto err_map;
1323
1324 err = i915_vma_sync(vma);
1325 if (err)
1326 goto err_pin;
1327
1328 for_each_engine(engine, gt, id) {
1329 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
1330 struct i915_request *rq, *nop;
1331
1332 if (!intel_engine_has_preemption(engine))
1333 continue;
1334
1335 st_engine_heartbeat_disable(engine);
1336 memset(vaddr, 0, PAGE_SIZE);
1337
1338 /* ELSP[0]: semaphore wait */
1339 rq = semaphore_queue(engine, vma, 0);
1340 if (IS_ERR(rq)) {
1341 err = PTR_ERR(rq);
1342 goto err_heartbeat;
1343 }
1344 engine->sched_engine->schedule(rq, &attr);
1345 err = wait_for_submit(engine, rq, HZ / 2);
1346 if (err) {
1347 pr_err("%s: Timed out trying to submit semaphores\n",
1348 engine->name);
1349 goto err_rq;
1350 }
1351
1352 /* ELSP[1]: nop request */
1353 nop = nop_request(engine);
1354 if (IS_ERR(nop)) {
1355 err = PTR_ERR(nop);
1356 goto err_rq;
1357 }
1358 err = wait_for_submit(engine, nop, HZ / 2);
1359 i915_request_put(nop);
1360 if (err) {
1361 pr_err("%s: Timed out trying to submit nop\n",
1362 engine->name);
1363 goto err_rq;
1364 }
1365
1366 GEM_BUG_ON(i915_request_completed(rq));
1367 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1368
1369 /* Queue: semaphore signal, matching priority as semaphore */
1370 err = release_queue(engine, vma, 1, effective_prio(rq));
1371 if (err)
1372 goto err_rq;
1373
1374 /* Wait until we ack the release_queue and start timeslicing */
1375 do {
1376 cond_resched();
1377 intel_engine_flush_submission(engine);
1378 } while (READ_ONCE(engine->execlists.pending[0]));
1379
1380 /* Timeslice every jiffy, so within 2 we should signal */
1381 if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) {
1382 struct drm_printer p =
1383 drm_info_printer(gt->i915->drm.dev);
1384
1385 pr_err("%s: Failed to timeslice into queue\n",
1386 engine->name);
1387 intel_engine_dump(engine, &p,
1388 "%s\n", engine->name);
1389
1390 memset(vaddr, 0xff, PAGE_SIZE);
1391 err = -EIO;
1392 }
1393 err_rq:
1394 i915_request_put(rq);
1395 err_heartbeat:
1396 st_engine_heartbeat_enable(engine);
1397 if (err)
1398 break;
1399 }
1400
1401 err_pin:
1402 i915_vma_unpin(vma);
1403 err_map:
1404 i915_gem_object_unpin_map(obj);
1405 err_obj:
1406 i915_gem_object_put(obj);
1407 return err;
1408 }
1409
live_timeslice_nopreempt(void * arg)1410 static int live_timeslice_nopreempt(void *arg)
1411 {
1412 struct intel_gt *gt = arg;
1413 struct intel_engine_cs *engine;
1414 enum intel_engine_id id;
1415 struct igt_spinner spin;
1416 int err = 0;
1417
1418 /*
1419 * We should not timeslice into a request that is marked with
1420 * I915_REQUEST_NOPREEMPT.
1421 */
1422 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1423 return 0;
1424
1425 if (igt_spinner_init(&spin, gt))
1426 return -ENOMEM;
1427
1428 for_each_engine(engine, gt, id) {
1429 struct intel_context *ce;
1430 struct i915_request *rq;
1431 unsigned long timeslice;
1432
1433 if (!intel_engine_has_preemption(engine))
1434 continue;
1435
1436 ce = intel_context_create(engine);
1437 if (IS_ERR(ce)) {
1438 err = PTR_ERR(ce);
1439 break;
1440 }
1441
1442 st_engine_heartbeat_disable(engine);
1443 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1444
1445 /* Create an unpreemptible spinner */
1446
1447 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
1448 intel_context_put(ce);
1449 if (IS_ERR(rq)) {
1450 err = PTR_ERR(rq);
1451 goto out_heartbeat;
1452 }
1453
1454 i915_request_get(rq);
1455 i915_request_add(rq);
1456
1457 if (!igt_wait_for_spinner(&spin, rq)) {
1458 i915_request_put(rq);
1459 err = -ETIME;
1460 goto out_spin;
1461 }
1462
1463 set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
1464 i915_request_put(rq);
1465
1466 /* Followed by a maximum priority barrier (heartbeat) */
1467
1468 ce = intel_context_create(engine);
1469 if (IS_ERR(ce)) {
1470 err = PTR_ERR(ce);
1471 goto out_spin;
1472 }
1473
1474 rq = intel_context_create_request(ce);
1475 intel_context_put(ce);
1476 if (IS_ERR(rq)) {
1477 err = PTR_ERR(rq);
1478 goto out_spin;
1479 }
1480
1481 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1482 i915_request_get(rq);
1483 i915_request_add(rq);
1484
1485 /*
1486 * Wait until the barrier is in ELSP, and we know timeslicing
1487 * will have been activated.
1488 */
1489 if (wait_for_submit(engine, rq, HZ / 2)) {
1490 i915_request_put(rq);
1491 err = -ETIME;
1492 goto out_spin;
1493 }
1494
1495 /*
1496 * Since the ELSP[0] request is unpreemptible, it should not
1497 * allow the maximum priority barrier through. Wait long
1498 * enough to see if it is timesliced in by mistake.
1499 */
1500 if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) {
1501 pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",
1502 engine->name);
1503 err = -EINVAL;
1504 }
1505 i915_request_put(rq);
1506
1507 out_spin:
1508 igt_spinner_end(&spin);
1509 out_heartbeat:
1510 xchg(&engine->props.timeslice_duration_ms, timeslice);
1511 st_engine_heartbeat_enable(engine);
1512 if (err)
1513 break;
1514
1515 if (igt_flush_test(gt->i915)) {
1516 err = -EIO;
1517 break;
1518 }
1519 }
1520
1521 igt_spinner_fini(&spin);
1522 return err;
1523 }
1524
live_busywait_preempt(void * arg)1525 static int live_busywait_preempt(void *arg)
1526 {
1527 struct intel_gt *gt = arg;
1528 struct i915_gem_context *ctx_hi, *ctx_lo;
1529 struct intel_engine_cs *engine;
1530 struct drm_i915_gem_object *obj;
1531 struct i915_vma *vma;
1532 enum intel_engine_id id;
1533 u32 *map;
1534 int err;
1535
1536 /*
1537 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
1538 * preempt the busywaits used to synchronise between rings.
1539 */
1540
1541 ctx_hi = kernel_context(gt->i915, NULL);
1542 if (IS_ERR(ctx_hi))
1543 return PTR_ERR(ctx_hi);
1544
1545 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1546
1547 ctx_lo = kernel_context(gt->i915, NULL);
1548 if (IS_ERR(ctx_lo)) {
1549 err = PTR_ERR(ctx_lo);
1550 goto err_ctx_hi;
1551 }
1552
1553 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1554
1555 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1556 if (IS_ERR(obj)) {
1557 err = PTR_ERR(obj);
1558 goto err_ctx_lo;
1559 }
1560
1561 map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1562 if (IS_ERR(map)) {
1563 err = PTR_ERR(map);
1564 goto err_obj;
1565 }
1566
1567 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1568 if (IS_ERR(vma)) {
1569 err = PTR_ERR(vma);
1570 goto err_map;
1571 }
1572
1573 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1574 if (err)
1575 goto err_map;
1576
1577 err = i915_vma_sync(vma);
1578 if (err)
1579 goto err_vma;
1580
1581 for_each_engine(engine, gt, id) {
1582 struct i915_request *lo, *hi;
1583 struct igt_live_test t;
1584 u32 *cs;
1585
1586 if (!intel_engine_has_preemption(engine))
1587 continue;
1588
1589 if (!intel_engine_can_store_dword(engine))
1590 continue;
1591
1592 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1593 err = -EIO;
1594 goto err_vma;
1595 }
1596
1597 /*
1598 * We create two requests. The low priority request
1599 * busywaits on a semaphore (inside the ringbuffer where
1600 * is should be preemptible) and the high priority requests
1601 * uses a MI_STORE_DWORD_IMM to update the semaphore value
1602 * allowing the first request to complete. If preemption
1603 * fails, we hang instead.
1604 */
1605
1606 lo = igt_request_alloc(ctx_lo, engine);
1607 if (IS_ERR(lo)) {
1608 err = PTR_ERR(lo);
1609 goto err_vma;
1610 }
1611
1612 cs = intel_ring_begin(lo, 8);
1613 if (IS_ERR(cs)) {
1614 err = PTR_ERR(cs);
1615 i915_request_add(lo);
1616 goto err_vma;
1617 }
1618
1619 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1620 *cs++ = i915_ggtt_offset(vma);
1621 *cs++ = 0;
1622 *cs++ = 1;
1623
1624 /* XXX Do we need a flush + invalidate here? */
1625
1626 *cs++ = MI_SEMAPHORE_WAIT |
1627 MI_SEMAPHORE_GLOBAL_GTT |
1628 MI_SEMAPHORE_POLL |
1629 MI_SEMAPHORE_SAD_EQ_SDD;
1630 *cs++ = 0;
1631 *cs++ = i915_ggtt_offset(vma);
1632 *cs++ = 0;
1633
1634 intel_ring_advance(lo, cs);
1635
1636 i915_request_get(lo);
1637 i915_request_add(lo);
1638
1639 if (wait_for(READ_ONCE(*map), 10)) {
1640 i915_request_put(lo);
1641 err = -ETIMEDOUT;
1642 goto err_vma;
1643 }
1644
1645 /* Low priority request should be busywaiting now */
1646 if (i915_request_wait(lo, 0, 1) != -ETIME) {
1647 i915_request_put(lo);
1648 pr_err("%s: Busywaiting request did not!\n",
1649 engine->name);
1650 err = -EIO;
1651 goto err_vma;
1652 }
1653
1654 hi = igt_request_alloc(ctx_hi, engine);
1655 if (IS_ERR(hi)) {
1656 err = PTR_ERR(hi);
1657 i915_request_put(lo);
1658 goto err_vma;
1659 }
1660
1661 cs = intel_ring_begin(hi, 4);
1662 if (IS_ERR(cs)) {
1663 err = PTR_ERR(cs);
1664 i915_request_add(hi);
1665 i915_request_put(lo);
1666 goto err_vma;
1667 }
1668
1669 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1670 *cs++ = i915_ggtt_offset(vma);
1671 *cs++ = 0;
1672 *cs++ = 0;
1673
1674 intel_ring_advance(hi, cs);
1675 i915_request_add(hi);
1676
1677 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1678 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1679
1680 pr_err("%s: Failed to preempt semaphore busywait!\n",
1681 engine->name);
1682
1683 intel_engine_dump(engine, &p, "%s\n", engine->name);
1684 GEM_TRACE_DUMP();
1685
1686 i915_request_put(lo);
1687 intel_gt_set_wedged(gt);
1688 err = -EIO;
1689 goto err_vma;
1690 }
1691 GEM_BUG_ON(READ_ONCE(*map));
1692 i915_request_put(lo);
1693
1694 if (igt_live_test_end(&t)) {
1695 err = -EIO;
1696 goto err_vma;
1697 }
1698 }
1699
1700 err = 0;
1701 err_vma:
1702 i915_vma_unpin(vma);
1703 err_map:
1704 i915_gem_object_unpin_map(obj);
1705 err_obj:
1706 i915_gem_object_put(obj);
1707 err_ctx_lo:
1708 kernel_context_close(ctx_lo);
1709 err_ctx_hi:
1710 kernel_context_close(ctx_hi);
1711 return err;
1712 }
1713
1714 static struct i915_request *
spinner_create_request(struct igt_spinner * spin,struct i915_gem_context * ctx,struct intel_engine_cs * engine,u32 arb)1715 spinner_create_request(struct igt_spinner *spin,
1716 struct i915_gem_context *ctx,
1717 struct intel_engine_cs *engine,
1718 u32 arb)
1719 {
1720 struct intel_context *ce;
1721 struct i915_request *rq;
1722
1723 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
1724 if (IS_ERR(ce))
1725 return ERR_CAST(ce);
1726
1727 rq = igt_spinner_create_request(spin, ce, arb);
1728 intel_context_put(ce);
1729 return rq;
1730 }
1731
live_preempt(void * arg)1732 static int live_preempt(void *arg)
1733 {
1734 struct intel_gt *gt = arg;
1735 struct i915_gem_context *ctx_hi, *ctx_lo;
1736 struct igt_spinner spin_hi, spin_lo;
1737 struct intel_engine_cs *engine;
1738 enum intel_engine_id id;
1739 int err = -ENOMEM;
1740
1741 ctx_hi = kernel_context(gt->i915, NULL);
1742 if (!ctx_hi)
1743 return -ENOMEM;
1744 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1745
1746 ctx_lo = kernel_context(gt->i915, NULL);
1747 if (!ctx_lo)
1748 goto err_ctx_hi;
1749 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1750
1751 if (igt_spinner_init(&spin_hi, gt))
1752 goto err_ctx_lo;
1753
1754 if (igt_spinner_init(&spin_lo, gt))
1755 goto err_spin_hi;
1756
1757 for_each_engine(engine, gt, id) {
1758 struct igt_live_test t;
1759 struct i915_request *rq;
1760
1761 if (!intel_engine_has_preemption(engine))
1762 continue;
1763
1764 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1765 err = -EIO;
1766 goto err_spin_lo;
1767 }
1768
1769 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1770 MI_ARB_CHECK);
1771 if (IS_ERR(rq)) {
1772 err = PTR_ERR(rq);
1773 goto err_spin_lo;
1774 }
1775
1776 i915_request_add(rq);
1777 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1778 GEM_TRACE("lo spinner failed to start\n");
1779 GEM_TRACE_DUMP();
1780 intel_gt_set_wedged(gt);
1781 err = -EIO;
1782 goto err_spin_lo;
1783 }
1784
1785 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1786 MI_ARB_CHECK);
1787 if (IS_ERR(rq)) {
1788 igt_spinner_end(&spin_lo);
1789 err = PTR_ERR(rq);
1790 goto err_spin_lo;
1791 }
1792
1793 i915_request_add(rq);
1794 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1795 GEM_TRACE("hi spinner failed to start\n");
1796 GEM_TRACE_DUMP();
1797 intel_gt_set_wedged(gt);
1798 err = -EIO;
1799 goto err_spin_lo;
1800 }
1801
1802 igt_spinner_end(&spin_hi);
1803 igt_spinner_end(&spin_lo);
1804
1805 if (igt_live_test_end(&t)) {
1806 err = -EIO;
1807 goto err_spin_lo;
1808 }
1809 }
1810
1811 err = 0;
1812 err_spin_lo:
1813 igt_spinner_fini(&spin_lo);
1814 err_spin_hi:
1815 igt_spinner_fini(&spin_hi);
1816 err_ctx_lo:
1817 kernel_context_close(ctx_lo);
1818 err_ctx_hi:
1819 kernel_context_close(ctx_hi);
1820 return err;
1821 }
1822
live_late_preempt(void * arg)1823 static int live_late_preempt(void *arg)
1824 {
1825 struct intel_gt *gt = arg;
1826 struct i915_gem_context *ctx_hi, *ctx_lo;
1827 struct igt_spinner spin_hi, spin_lo;
1828 struct intel_engine_cs *engine;
1829 struct i915_sched_attr attr = {};
1830 enum intel_engine_id id;
1831 int err = -ENOMEM;
1832
1833 ctx_hi = kernel_context(gt->i915, NULL);
1834 if (!ctx_hi)
1835 return -ENOMEM;
1836
1837 ctx_lo = kernel_context(gt->i915, NULL);
1838 if (!ctx_lo)
1839 goto err_ctx_hi;
1840
1841 if (igt_spinner_init(&spin_hi, gt))
1842 goto err_ctx_lo;
1843
1844 if (igt_spinner_init(&spin_lo, gt))
1845 goto err_spin_hi;
1846
1847 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1848 ctx_lo->sched.priority = 1;
1849
1850 for_each_engine(engine, gt, id) {
1851 struct igt_live_test t;
1852 struct i915_request *rq;
1853
1854 if (!intel_engine_has_preemption(engine))
1855 continue;
1856
1857 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1858 err = -EIO;
1859 goto err_spin_lo;
1860 }
1861
1862 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1863 MI_ARB_CHECK);
1864 if (IS_ERR(rq)) {
1865 err = PTR_ERR(rq);
1866 goto err_spin_lo;
1867 }
1868
1869 i915_request_add(rq);
1870 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1871 pr_err("First context failed to start\n");
1872 goto err_wedged;
1873 }
1874
1875 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1876 MI_NOOP);
1877 if (IS_ERR(rq)) {
1878 igt_spinner_end(&spin_lo);
1879 err = PTR_ERR(rq);
1880 goto err_spin_lo;
1881 }
1882
1883 i915_request_add(rq);
1884 if (igt_wait_for_spinner(&spin_hi, rq)) {
1885 pr_err("Second context overtook first?\n");
1886 goto err_wedged;
1887 }
1888
1889 attr.priority = I915_PRIORITY_MAX;
1890 engine->sched_engine->schedule(rq, &attr);
1891
1892 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1893 pr_err("High priority context failed to preempt the low priority context\n");
1894 GEM_TRACE_DUMP();
1895 goto err_wedged;
1896 }
1897
1898 igt_spinner_end(&spin_hi);
1899 igt_spinner_end(&spin_lo);
1900
1901 if (igt_live_test_end(&t)) {
1902 err = -EIO;
1903 goto err_spin_lo;
1904 }
1905 }
1906
1907 err = 0;
1908 err_spin_lo:
1909 igt_spinner_fini(&spin_lo);
1910 err_spin_hi:
1911 igt_spinner_fini(&spin_hi);
1912 err_ctx_lo:
1913 kernel_context_close(ctx_lo);
1914 err_ctx_hi:
1915 kernel_context_close(ctx_hi);
1916 return err;
1917
1918 err_wedged:
1919 igt_spinner_end(&spin_hi);
1920 igt_spinner_end(&spin_lo);
1921 intel_gt_set_wedged(gt);
1922 err = -EIO;
1923 goto err_spin_lo;
1924 }
1925
1926 struct preempt_client {
1927 struct igt_spinner spin;
1928 struct i915_gem_context *ctx;
1929 };
1930
preempt_client_init(struct intel_gt * gt,struct preempt_client * c)1931 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1932 {
1933 c->ctx = kernel_context(gt->i915, NULL);
1934 if (!c->ctx)
1935 return -ENOMEM;
1936
1937 if (igt_spinner_init(&c->spin, gt))
1938 goto err_ctx;
1939
1940 return 0;
1941
1942 err_ctx:
1943 kernel_context_close(c->ctx);
1944 return -ENOMEM;
1945 }
1946
preempt_client_fini(struct preempt_client * c)1947 static void preempt_client_fini(struct preempt_client *c)
1948 {
1949 igt_spinner_fini(&c->spin);
1950 kernel_context_close(c->ctx);
1951 }
1952
live_nopreempt(void * arg)1953 static int live_nopreempt(void *arg)
1954 {
1955 struct intel_gt *gt = arg;
1956 struct intel_engine_cs *engine;
1957 struct preempt_client a, b;
1958 enum intel_engine_id id;
1959 int err = -ENOMEM;
1960
1961 /*
1962 * Verify that we can disable preemption for an individual request
1963 * that may be being observed and not want to be interrupted.
1964 */
1965
1966 if (preempt_client_init(gt, &a))
1967 return -ENOMEM;
1968 if (preempt_client_init(gt, &b))
1969 goto err_client_a;
1970 b.ctx->sched.priority = I915_PRIORITY_MAX;
1971
1972 for_each_engine(engine, gt, id) {
1973 struct i915_request *rq_a, *rq_b;
1974
1975 if (!intel_engine_has_preemption(engine))
1976 continue;
1977
1978 engine->execlists.preempt_hang.count = 0;
1979
1980 rq_a = spinner_create_request(&a.spin,
1981 a.ctx, engine,
1982 MI_ARB_CHECK);
1983 if (IS_ERR(rq_a)) {
1984 err = PTR_ERR(rq_a);
1985 goto err_client_b;
1986 }
1987
1988 /* Low priority client, but unpreemptable! */
1989 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
1990
1991 i915_request_add(rq_a);
1992 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1993 pr_err("First client failed to start\n");
1994 goto err_wedged;
1995 }
1996
1997 rq_b = spinner_create_request(&b.spin,
1998 b.ctx, engine,
1999 MI_ARB_CHECK);
2000 if (IS_ERR(rq_b)) {
2001 err = PTR_ERR(rq_b);
2002 goto err_client_b;
2003 }
2004
2005 i915_request_add(rq_b);
2006
2007 /* B is much more important than A! (But A is unpreemptable.) */
2008 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
2009
2010 /* Wait long enough for preemption and timeslicing */
2011 if (igt_wait_for_spinner(&b.spin, rq_b)) {
2012 pr_err("Second client started too early!\n");
2013 goto err_wedged;
2014 }
2015
2016 igt_spinner_end(&a.spin);
2017
2018 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2019 pr_err("Second client failed to start\n");
2020 goto err_wedged;
2021 }
2022
2023 igt_spinner_end(&b.spin);
2024
2025 if (engine->execlists.preempt_hang.count) {
2026 pr_err("Preemption recorded x%d; should have been suppressed!\n",
2027 engine->execlists.preempt_hang.count);
2028 err = -EINVAL;
2029 goto err_wedged;
2030 }
2031
2032 if (igt_flush_test(gt->i915))
2033 goto err_wedged;
2034 }
2035
2036 err = 0;
2037 err_client_b:
2038 preempt_client_fini(&b);
2039 err_client_a:
2040 preempt_client_fini(&a);
2041 return err;
2042
2043 err_wedged:
2044 igt_spinner_end(&b.spin);
2045 igt_spinner_end(&a.spin);
2046 intel_gt_set_wedged(gt);
2047 err = -EIO;
2048 goto err_client_b;
2049 }
2050
2051 struct live_preempt_cancel {
2052 struct intel_engine_cs *engine;
2053 struct preempt_client a, b;
2054 };
2055
__cancel_active0(struct live_preempt_cancel * arg)2056 static int __cancel_active0(struct live_preempt_cancel *arg)
2057 {
2058 struct i915_request *rq;
2059 struct igt_live_test t;
2060 int err;
2061
2062 /* Preempt cancel of ELSP0 */
2063 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2064 if (igt_live_test_begin(&t, arg->engine->i915,
2065 __func__, arg->engine->name))
2066 return -EIO;
2067
2068 rq = spinner_create_request(&arg->a.spin,
2069 arg->a.ctx, arg->engine,
2070 MI_ARB_CHECK);
2071 if (IS_ERR(rq))
2072 return PTR_ERR(rq);
2073
2074 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2075 i915_request_get(rq);
2076 i915_request_add(rq);
2077 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2078 err = -EIO;
2079 goto out;
2080 }
2081
2082 intel_context_ban(rq->context, rq);
2083 err = intel_engine_pulse(arg->engine);
2084 if (err)
2085 goto out;
2086
2087 err = wait_for_reset(arg->engine, rq, HZ / 2);
2088 if (err) {
2089 pr_err("Cancelled inflight0 request did not reset\n");
2090 goto out;
2091 }
2092
2093 out:
2094 i915_request_put(rq);
2095 if (igt_live_test_end(&t))
2096 err = -EIO;
2097 return err;
2098 }
2099
__cancel_active1(struct live_preempt_cancel * arg)2100 static int __cancel_active1(struct live_preempt_cancel *arg)
2101 {
2102 struct i915_request *rq[2] = {};
2103 struct igt_live_test t;
2104 int err;
2105
2106 /* Preempt cancel of ELSP1 */
2107 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2108 if (igt_live_test_begin(&t, arg->engine->i915,
2109 __func__, arg->engine->name))
2110 return -EIO;
2111
2112 rq[0] = spinner_create_request(&arg->a.spin,
2113 arg->a.ctx, arg->engine,
2114 MI_NOOP); /* no preemption */
2115 if (IS_ERR(rq[0]))
2116 return PTR_ERR(rq[0]);
2117
2118 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2119 i915_request_get(rq[0]);
2120 i915_request_add(rq[0]);
2121 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2122 err = -EIO;
2123 goto out;
2124 }
2125
2126 rq[1] = spinner_create_request(&arg->b.spin,
2127 arg->b.ctx, arg->engine,
2128 MI_ARB_CHECK);
2129 if (IS_ERR(rq[1])) {
2130 err = PTR_ERR(rq[1]);
2131 goto out;
2132 }
2133
2134 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2135 i915_request_get(rq[1]);
2136 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2137 i915_request_add(rq[1]);
2138 if (err)
2139 goto out;
2140
2141 intel_context_ban(rq[1]->context, rq[1]);
2142 err = intel_engine_pulse(arg->engine);
2143 if (err)
2144 goto out;
2145
2146 igt_spinner_end(&arg->a.spin);
2147 err = wait_for_reset(arg->engine, rq[1], HZ / 2);
2148 if (err)
2149 goto out;
2150
2151 if (rq[0]->fence.error != 0) {
2152 pr_err("Normal inflight0 request did not complete\n");
2153 err = -EINVAL;
2154 goto out;
2155 }
2156
2157 if (rq[1]->fence.error != -EIO) {
2158 pr_err("Cancelled inflight1 request did not report -EIO\n");
2159 err = -EINVAL;
2160 goto out;
2161 }
2162
2163 out:
2164 i915_request_put(rq[1]);
2165 i915_request_put(rq[0]);
2166 if (igt_live_test_end(&t))
2167 err = -EIO;
2168 return err;
2169 }
2170
__cancel_queued(struct live_preempt_cancel * arg)2171 static int __cancel_queued(struct live_preempt_cancel *arg)
2172 {
2173 struct i915_request *rq[3] = {};
2174 struct igt_live_test t;
2175 int err;
2176
2177 /* Full ELSP and one in the wings */
2178 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2179 if (igt_live_test_begin(&t, arg->engine->i915,
2180 __func__, arg->engine->name))
2181 return -EIO;
2182
2183 rq[0] = spinner_create_request(&arg->a.spin,
2184 arg->a.ctx, arg->engine,
2185 MI_ARB_CHECK);
2186 if (IS_ERR(rq[0]))
2187 return PTR_ERR(rq[0]);
2188
2189 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2190 i915_request_get(rq[0]);
2191 i915_request_add(rq[0]);
2192 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2193 err = -EIO;
2194 goto out;
2195 }
2196
2197 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
2198 if (IS_ERR(rq[1])) {
2199 err = PTR_ERR(rq[1]);
2200 goto out;
2201 }
2202
2203 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2204 i915_request_get(rq[1]);
2205 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2206 i915_request_add(rq[1]);
2207 if (err)
2208 goto out;
2209
2210 rq[2] = spinner_create_request(&arg->b.spin,
2211 arg->a.ctx, arg->engine,
2212 MI_ARB_CHECK);
2213 if (IS_ERR(rq[2])) {
2214 err = PTR_ERR(rq[2]);
2215 goto out;
2216 }
2217
2218 i915_request_get(rq[2]);
2219 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
2220 i915_request_add(rq[2]);
2221 if (err)
2222 goto out;
2223
2224 intel_context_ban(rq[2]->context, rq[2]);
2225 err = intel_engine_pulse(arg->engine);
2226 if (err)
2227 goto out;
2228
2229 err = wait_for_reset(arg->engine, rq[2], HZ / 2);
2230 if (err)
2231 goto out;
2232
2233 if (rq[0]->fence.error != -EIO) {
2234 pr_err("Cancelled inflight0 request did not report -EIO\n");
2235 err = -EINVAL;
2236 goto out;
2237 }
2238
2239 /*
2240 * The behavior between having semaphores and not is different. With
2241 * semaphores the subsequent request is on the hardware and not cancelled
2242 * while without the request is held in the driver and cancelled.
2243 */
2244 if (intel_engine_has_semaphores(rq[1]->engine) &&
2245 rq[1]->fence.error != 0) {
2246 pr_err("Normal inflight1 request did not complete\n");
2247 err = -EINVAL;
2248 goto out;
2249 }
2250
2251 if (rq[2]->fence.error != -EIO) {
2252 pr_err("Cancelled queued request did not report -EIO\n");
2253 err = -EINVAL;
2254 goto out;
2255 }
2256
2257 out:
2258 i915_request_put(rq[2]);
2259 i915_request_put(rq[1]);
2260 i915_request_put(rq[0]);
2261 if (igt_live_test_end(&t))
2262 err = -EIO;
2263 return err;
2264 }
2265
__cancel_hostile(struct live_preempt_cancel * arg)2266 static int __cancel_hostile(struct live_preempt_cancel *arg)
2267 {
2268 struct i915_request *rq;
2269 int err;
2270
2271 /* Preempt cancel non-preemptible spinner in ELSP0 */
2272 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2273 return 0;
2274
2275 if (!intel_has_reset_engine(arg->engine->gt))
2276 return 0;
2277
2278 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2279 rq = spinner_create_request(&arg->a.spin,
2280 arg->a.ctx, arg->engine,
2281 MI_NOOP); /* preemption disabled */
2282 if (IS_ERR(rq))
2283 return PTR_ERR(rq);
2284
2285 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2286 i915_request_get(rq);
2287 i915_request_add(rq);
2288 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2289 err = -EIO;
2290 goto out;
2291 }
2292
2293 intel_context_ban(rq->context, rq);
2294 err = intel_engine_pulse(arg->engine); /* force reset */
2295 if (err)
2296 goto out;
2297
2298 err = wait_for_reset(arg->engine, rq, HZ / 2);
2299 if (err) {
2300 pr_err("Cancelled inflight0 request did not reset\n");
2301 goto out;
2302 }
2303
2304 out:
2305 i915_request_put(rq);
2306 if (igt_flush_test(arg->engine->i915))
2307 err = -EIO;
2308 return err;
2309 }
2310
force_reset_timeout(struct intel_engine_cs * engine)2311 static void force_reset_timeout(struct intel_engine_cs *engine)
2312 {
2313 engine->reset_timeout.probability = 999;
2314 atomic_set(&engine->reset_timeout.times, -1);
2315 }
2316
cancel_reset_timeout(struct intel_engine_cs * engine)2317 static void cancel_reset_timeout(struct intel_engine_cs *engine)
2318 {
2319 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
2320 }
2321
__cancel_fail(struct live_preempt_cancel * arg)2322 static int __cancel_fail(struct live_preempt_cancel *arg)
2323 {
2324 struct intel_engine_cs *engine = arg->engine;
2325 struct i915_request *rq;
2326 int err;
2327
2328 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2329 return 0;
2330
2331 if (!intel_has_reset_engine(engine->gt))
2332 return 0;
2333
2334 GEM_TRACE("%s(%s)\n", __func__, engine->name);
2335 rq = spinner_create_request(&arg->a.spin,
2336 arg->a.ctx, engine,
2337 MI_NOOP); /* preemption disabled */
2338 if (IS_ERR(rq))
2339 return PTR_ERR(rq);
2340
2341 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2342 i915_request_get(rq);
2343 i915_request_add(rq);
2344 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2345 err = -EIO;
2346 goto out;
2347 }
2348
2349 intel_context_set_banned(rq->context);
2350
2351 err = intel_engine_pulse(engine);
2352 if (err)
2353 goto out;
2354
2355 force_reset_timeout(engine);
2356
2357 /* force preempt reset [failure] */
2358 while (!engine->execlists.pending[0])
2359 intel_engine_flush_submission(engine);
2360 del_timer_sync(&engine->execlists.preempt);
2361 intel_engine_flush_submission(engine);
2362
2363 cancel_reset_timeout(engine);
2364
2365 /* after failure, require heartbeats to reset device */
2366 intel_engine_set_heartbeat(engine, 1);
2367 err = wait_for_reset(engine, rq, HZ / 2);
2368 intel_engine_set_heartbeat(engine,
2369 engine->defaults.heartbeat_interval_ms);
2370 if (err) {
2371 pr_err("Cancelled inflight0 request did not reset\n");
2372 goto out;
2373 }
2374
2375 out:
2376 i915_request_put(rq);
2377 if (igt_flush_test(engine->i915))
2378 err = -EIO;
2379 return err;
2380 }
2381
live_preempt_cancel(void * arg)2382 static int live_preempt_cancel(void *arg)
2383 {
2384 struct intel_gt *gt = arg;
2385 struct live_preempt_cancel data;
2386 enum intel_engine_id id;
2387 int err = -ENOMEM;
2388
2389 /*
2390 * To cancel an inflight context, we need to first remove it from the
2391 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
2392 */
2393
2394 if (preempt_client_init(gt, &data.a))
2395 return -ENOMEM;
2396 if (preempt_client_init(gt, &data.b))
2397 goto err_client_a;
2398
2399 for_each_engine(data.engine, gt, id) {
2400 if (!intel_engine_has_preemption(data.engine))
2401 continue;
2402
2403 err = __cancel_active0(&data);
2404 if (err)
2405 goto err_wedged;
2406
2407 err = __cancel_active1(&data);
2408 if (err)
2409 goto err_wedged;
2410
2411 err = __cancel_queued(&data);
2412 if (err)
2413 goto err_wedged;
2414
2415 err = __cancel_hostile(&data);
2416 if (err)
2417 goto err_wedged;
2418
2419 err = __cancel_fail(&data);
2420 if (err)
2421 goto err_wedged;
2422 }
2423
2424 err = 0;
2425 err_client_b:
2426 preempt_client_fini(&data.b);
2427 err_client_a:
2428 preempt_client_fini(&data.a);
2429 return err;
2430
2431 err_wedged:
2432 GEM_TRACE_DUMP();
2433 igt_spinner_end(&data.b.spin);
2434 igt_spinner_end(&data.a.spin);
2435 intel_gt_set_wedged(gt);
2436 goto err_client_b;
2437 }
2438
live_suppress_self_preempt(void * arg)2439 static int live_suppress_self_preempt(void *arg)
2440 {
2441 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2442 struct intel_gt *gt = arg;
2443 struct intel_engine_cs *engine;
2444 struct preempt_client a, b;
2445 enum intel_engine_id id;
2446 int err = -ENOMEM;
2447
2448 /*
2449 * Verify that if a preemption request does not cause a change in
2450 * the current execution order, the preempt-to-idle injection is
2451 * skipped and that we do not accidentally apply it after the CS
2452 * completion event.
2453 */
2454
2455 if (intel_uc_uses_guc_submission(>->uc))
2456 return 0; /* presume black blox */
2457
2458 if (intel_vgpu_active(gt->i915))
2459 return 0; /* GVT forces single port & request submission */
2460
2461 if (preempt_client_init(gt, &a))
2462 return -ENOMEM;
2463 if (preempt_client_init(gt, &b))
2464 goto err_client_a;
2465
2466 for_each_engine(engine, gt, id) {
2467 struct i915_request *rq_a, *rq_b;
2468 int depth;
2469
2470 if (!intel_engine_has_preemption(engine))
2471 continue;
2472
2473 if (igt_flush_test(gt->i915))
2474 goto err_wedged;
2475
2476 st_engine_heartbeat_disable(engine);
2477 engine->execlists.preempt_hang.count = 0;
2478
2479 rq_a = spinner_create_request(&a.spin,
2480 a.ctx, engine,
2481 MI_NOOP);
2482 if (IS_ERR(rq_a)) {
2483 err = PTR_ERR(rq_a);
2484 st_engine_heartbeat_enable(engine);
2485 goto err_client_b;
2486 }
2487
2488 i915_request_add(rq_a);
2489 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2490 pr_err("First client failed to start\n");
2491 st_engine_heartbeat_enable(engine);
2492 goto err_wedged;
2493 }
2494
2495 /* Keep postponing the timer to avoid premature slicing */
2496 mod_timer(&engine->execlists.timer, jiffies + HZ);
2497 for (depth = 0; depth < 8; depth++) {
2498 rq_b = spinner_create_request(&b.spin,
2499 b.ctx, engine,
2500 MI_NOOP);
2501 if (IS_ERR(rq_b)) {
2502 err = PTR_ERR(rq_b);
2503 st_engine_heartbeat_enable(engine);
2504 goto err_client_b;
2505 }
2506 i915_request_add(rq_b);
2507
2508 GEM_BUG_ON(i915_request_completed(rq_a));
2509 engine->sched_engine->schedule(rq_a, &attr);
2510 igt_spinner_end(&a.spin);
2511
2512 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2513 pr_err("Second client failed to start\n");
2514 st_engine_heartbeat_enable(engine);
2515 goto err_wedged;
2516 }
2517
2518 swap(a, b);
2519 rq_a = rq_b;
2520 }
2521 igt_spinner_end(&a.spin);
2522
2523 if (engine->execlists.preempt_hang.count) {
2524 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2525 engine->name,
2526 engine->execlists.preempt_hang.count,
2527 depth);
2528 st_engine_heartbeat_enable(engine);
2529 err = -EINVAL;
2530 goto err_client_b;
2531 }
2532
2533 st_engine_heartbeat_enable(engine);
2534 if (igt_flush_test(gt->i915))
2535 goto err_wedged;
2536 }
2537
2538 err = 0;
2539 err_client_b:
2540 preempt_client_fini(&b);
2541 err_client_a:
2542 preempt_client_fini(&a);
2543 return err;
2544
2545 err_wedged:
2546 igt_spinner_end(&b.spin);
2547 igt_spinner_end(&a.spin);
2548 intel_gt_set_wedged(gt);
2549 err = -EIO;
2550 goto err_client_b;
2551 }
2552
live_chain_preempt(void * arg)2553 static int live_chain_preempt(void *arg)
2554 {
2555 struct intel_gt *gt = arg;
2556 struct intel_engine_cs *engine;
2557 struct preempt_client hi, lo;
2558 enum intel_engine_id id;
2559 int err = -ENOMEM;
2560
2561 /*
2562 * Build a chain AB...BA between two contexts (A, B) and request
2563 * preemption of the last request. It should then complete before
2564 * the previously submitted spinner in B.
2565 */
2566
2567 if (preempt_client_init(gt, &hi))
2568 return -ENOMEM;
2569
2570 if (preempt_client_init(gt, &lo))
2571 goto err_client_hi;
2572
2573 for_each_engine(engine, gt, id) {
2574 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2575 struct igt_live_test t;
2576 struct i915_request *rq;
2577 int ring_size, count, i;
2578
2579 if (!intel_engine_has_preemption(engine))
2580 continue;
2581
2582 rq = spinner_create_request(&lo.spin,
2583 lo.ctx, engine,
2584 MI_ARB_CHECK);
2585 if (IS_ERR(rq))
2586 goto err_wedged;
2587
2588 i915_request_get(rq);
2589 i915_request_add(rq);
2590
2591 ring_size = rq->wa_tail - rq->head;
2592 if (ring_size < 0)
2593 ring_size += rq->ring->size;
2594 ring_size = rq->ring->size / ring_size;
2595 pr_debug("%s(%s): Using maximum of %d requests\n",
2596 __func__, engine->name, ring_size);
2597
2598 igt_spinner_end(&lo.spin);
2599 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
2600 pr_err("Timed out waiting to flush %s\n", engine->name);
2601 i915_request_put(rq);
2602 goto err_wedged;
2603 }
2604 i915_request_put(rq);
2605
2606 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
2607 err = -EIO;
2608 goto err_wedged;
2609 }
2610
2611 for_each_prime_number_from(count, 1, ring_size) {
2612 rq = spinner_create_request(&hi.spin,
2613 hi.ctx, engine,
2614 MI_ARB_CHECK);
2615 if (IS_ERR(rq))
2616 goto err_wedged;
2617 i915_request_add(rq);
2618 if (!igt_wait_for_spinner(&hi.spin, rq))
2619 goto err_wedged;
2620
2621 rq = spinner_create_request(&lo.spin,
2622 lo.ctx, engine,
2623 MI_ARB_CHECK);
2624 if (IS_ERR(rq))
2625 goto err_wedged;
2626 i915_request_add(rq);
2627
2628 for (i = 0; i < count; i++) {
2629 rq = igt_request_alloc(lo.ctx, engine);
2630 if (IS_ERR(rq))
2631 goto err_wedged;
2632 i915_request_add(rq);
2633 }
2634
2635 rq = igt_request_alloc(hi.ctx, engine);
2636 if (IS_ERR(rq))
2637 goto err_wedged;
2638
2639 i915_request_get(rq);
2640 i915_request_add(rq);
2641 engine->sched_engine->schedule(rq, &attr);
2642
2643 igt_spinner_end(&hi.spin);
2644 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2645 struct drm_printer p =
2646 drm_info_printer(gt->i915->drm.dev);
2647
2648 pr_err("Failed to preempt over chain of %d\n",
2649 count);
2650 intel_engine_dump(engine, &p,
2651 "%s\n", engine->name);
2652 i915_request_put(rq);
2653 goto err_wedged;
2654 }
2655 igt_spinner_end(&lo.spin);
2656 i915_request_put(rq);
2657
2658 rq = igt_request_alloc(lo.ctx, engine);
2659 if (IS_ERR(rq))
2660 goto err_wedged;
2661
2662 i915_request_get(rq);
2663 i915_request_add(rq);
2664
2665 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2666 struct drm_printer p =
2667 drm_info_printer(gt->i915->drm.dev);
2668
2669 pr_err("Failed to flush low priority chain of %d requests\n",
2670 count);
2671 intel_engine_dump(engine, &p,
2672 "%s\n", engine->name);
2673
2674 i915_request_put(rq);
2675 goto err_wedged;
2676 }
2677 i915_request_put(rq);
2678 }
2679
2680 if (igt_live_test_end(&t)) {
2681 err = -EIO;
2682 goto err_wedged;
2683 }
2684 }
2685
2686 err = 0;
2687 err_client_lo:
2688 preempt_client_fini(&lo);
2689 err_client_hi:
2690 preempt_client_fini(&hi);
2691 return err;
2692
2693 err_wedged:
2694 igt_spinner_end(&hi.spin);
2695 igt_spinner_end(&lo.spin);
2696 intel_gt_set_wedged(gt);
2697 err = -EIO;
2698 goto err_client_lo;
2699 }
2700
create_gang(struct intel_engine_cs * engine,struct i915_request ** prev)2701 static int create_gang(struct intel_engine_cs *engine,
2702 struct i915_request **prev)
2703 {
2704 struct drm_i915_gem_object *obj;
2705 struct intel_context *ce;
2706 struct i915_request *rq;
2707 struct i915_vma *vma;
2708 u32 *cs;
2709 int err;
2710
2711 ce = intel_context_create(engine);
2712 if (IS_ERR(ce))
2713 return PTR_ERR(ce);
2714
2715 obj = i915_gem_object_create_internal(engine->i915, 4096);
2716 if (IS_ERR(obj)) {
2717 err = PTR_ERR(obj);
2718 goto err_ce;
2719 }
2720
2721 vma = i915_vma_instance(obj, ce->vm, NULL);
2722 if (IS_ERR(vma)) {
2723 err = PTR_ERR(vma);
2724 goto err_obj;
2725 }
2726
2727 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2728 if (err)
2729 goto err_obj;
2730
2731 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
2732 if (IS_ERR(cs)) {
2733 err = PTR_ERR(cs);
2734 goto err_obj;
2735 }
2736
2737 /* Semaphore target: spin until zero */
2738 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2739
2740 *cs++ = MI_SEMAPHORE_WAIT |
2741 MI_SEMAPHORE_POLL |
2742 MI_SEMAPHORE_SAD_EQ_SDD;
2743 *cs++ = 0;
2744 *cs++ = lower_32_bits(i915_vma_offset(vma));
2745 *cs++ = upper_32_bits(i915_vma_offset(vma));
2746
2747 if (*prev) {
2748 u64 offset = i915_vma_offset((*prev)->batch);
2749
2750 /* Terminate the spinner in the next lower priority batch. */
2751 *cs++ = MI_STORE_DWORD_IMM_GEN4;
2752 *cs++ = lower_32_bits(offset);
2753 *cs++ = upper_32_bits(offset);
2754 *cs++ = 0;
2755 }
2756
2757 *cs++ = MI_BATCH_BUFFER_END;
2758 i915_gem_object_flush_map(obj);
2759 i915_gem_object_unpin_map(obj);
2760
2761 rq = intel_context_create_request(ce);
2762 if (IS_ERR(rq)) {
2763 err = PTR_ERR(rq);
2764 goto err_obj;
2765 }
2766
2767 rq->batch = i915_vma_get(vma);
2768 i915_request_get(rq);
2769
2770 err = igt_vma_move_to_active_unlocked(vma, rq, 0);
2771 if (!err)
2772 err = rq->engine->emit_bb_start(rq,
2773 i915_vma_offset(vma),
2774 PAGE_SIZE, 0);
2775 i915_request_add(rq);
2776 if (err)
2777 goto err_rq;
2778
2779 i915_gem_object_put(obj);
2780 intel_context_put(ce);
2781
2782 rq->mock.link.next = &(*prev)->mock.link;
2783 *prev = rq;
2784 return 0;
2785
2786 err_rq:
2787 i915_vma_put(rq->batch);
2788 i915_request_put(rq);
2789 err_obj:
2790 i915_gem_object_put(obj);
2791 err_ce:
2792 intel_context_put(ce);
2793 return err;
2794 }
2795
__live_preempt_ring(struct intel_engine_cs * engine,struct igt_spinner * spin,int queue_sz,int ring_sz)2796 static int __live_preempt_ring(struct intel_engine_cs *engine,
2797 struct igt_spinner *spin,
2798 int queue_sz, int ring_sz)
2799 {
2800 struct intel_context *ce[2] = {};
2801 struct i915_request *rq;
2802 struct igt_live_test t;
2803 int err = 0;
2804 int n;
2805
2806 if (igt_live_test_begin(&t, engine->i915, __func__, engine->name))
2807 return -EIO;
2808
2809 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2810 struct intel_context *tmp;
2811
2812 tmp = intel_context_create(engine);
2813 if (IS_ERR(tmp)) {
2814 err = PTR_ERR(tmp);
2815 goto err_ce;
2816 }
2817
2818 tmp->ring_size = ring_sz;
2819
2820 err = intel_context_pin(tmp);
2821 if (err) {
2822 intel_context_put(tmp);
2823 goto err_ce;
2824 }
2825
2826 memset32(tmp->ring->vaddr,
2827 0xdeadbeef, /* trigger a hang if executed */
2828 tmp->ring->vma->size / sizeof(u32));
2829
2830 ce[n] = tmp;
2831 }
2832
2833 rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK);
2834 if (IS_ERR(rq)) {
2835 err = PTR_ERR(rq);
2836 goto err_ce;
2837 }
2838
2839 i915_request_get(rq);
2840 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2841 i915_request_add(rq);
2842
2843 if (!igt_wait_for_spinner(spin, rq)) {
2844 intel_gt_set_wedged(engine->gt);
2845 i915_request_put(rq);
2846 err = -ETIME;
2847 goto err_ce;
2848 }
2849
2850 /* Fill the ring, until we will cause a wrap */
2851 n = 0;
2852 while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) {
2853 struct i915_request *tmp;
2854
2855 tmp = intel_context_create_request(ce[0]);
2856 if (IS_ERR(tmp)) {
2857 err = PTR_ERR(tmp);
2858 i915_request_put(rq);
2859 goto err_ce;
2860 }
2861
2862 i915_request_add(tmp);
2863 intel_engine_flush_submission(engine);
2864 n++;
2865 }
2866 intel_engine_flush_submission(engine);
2867 pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
2868 engine->name, queue_sz, n,
2869 ce[0]->ring->size,
2870 ce[0]->ring->tail,
2871 ce[0]->ring->emit,
2872 rq->tail);
2873 i915_request_put(rq);
2874
2875 /* Create a second request to preempt the first ring */
2876 rq = intel_context_create_request(ce[1]);
2877 if (IS_ERR(rq)) {
2878 err = PTR_ERR(rq);
2879 goto err_ce;
2880 }
2881
2882 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2883 i915_request_get(rq);
2884 i915_request_add(rq);
2885
2886 err = wait_for_submit(engine, rq, HZ / 2);
2887 i915_request_put(rq);
2888 if (err) {
2889 pr_err("%s: preemption request was not submitted\n",
2890 engine->name);
2891 err = -ETIME;
2892 }
2893
2894 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
2895 engine->name,
2896 ce[0]->ring->tail, ce[0]->ring->emit,
2897 ce[1]->ring->tail, ce[1]->ring->emit);
2898
2899 err_ce:
2900 intel_engine_flush_submission(engine);
2901 igt_spinner_end(spin);
2902 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2903 if (IS_ERR_OR_NULL(ce[n]))
2904 break;
2905
2906 intel_context_unpin(ce[n]);
2907 intel_context_put(ce[n]);
2908 }
2909 if (igt_live_test_end(&t))
2910 err = -EIO;
2911 return err;
2912 }
2913
live_preempt_ring(void * arg)2914 static int live_preempt_ring(void *arg)
2915 {
2916 struct intel_gt *gt = arg;
2917 struct intel_engine_cs *engine;
2918 struct igt_spinner spin;
2919 enum intel_engine_id id;
2920 int err = 0;
2921
2922 /*
2923 * Check that we rollback large chunks of a ring in order to do a
2924 * preemption event. Similar to live_unlite_ring, but looking at
2925 * ring size rather than the impact of intel_ring_direction().
2926 */
2927
2928 if (igt_spinner_init(&spin, gt))
2929 return -ENOMEM;
2930
2931 for_each_engine(engine, gt, id) {
2932 int n;
2933
2934 if (!intel_engine_has_preemption(engine))
2935 continue;
2936
2937 if (!intel_engine_can_store_dword(engine))
2938 continue;
2939
2940 st_engine_heartbeat_disable(engine);
2941
2942 for (n = 0; n <= 3; n++) {
2943 err = __live_preempt_ring(engine, &spin,
2944 n * SZ_4K / 4, SZ_4K);
2945 if (err)
2946 break;
2947 }
2948
2949 st_engine_heartbeat_enable(engine);
2950 if (err)
2951 break;
2952 }
2953
2954 igt_spinner_fini(&spin);
2955 return err;
2956 }
2957
live_preempt_gang(void * arg)2958 static int live_preempt_gang(void *arg)
2959 {
2960 struct intel_gt *gt = arg;
2961 struct intel_engine_cs *engine;
2962 enum intel_engine_id id;
2963
2964 /*
2965 * Build as long a chain of preempters as we can, with each
2966 * request higher priority than the last. Once we are ready, we release
2967 * the last batch which then precolates down the chain, each releasing
2968 * the next oldest in turn. The intent is to simply push as hard as we
2969 * can with the number of preemptions, trying to exceed narrow HW
2970 * limits. At a minimum, we insist that we can sort all the user
2971 * high priority levels into execution order.
2972 */
2973
2974 for_each_engine(engine, gt, id) {
2975 struct i915_request *rq = NULL;
2976 struct igt_live_test t;
2977 IGT_TIMEOUT(end_time);
2978 int prio = 0;
2979 int err = 0;
2980 u32 *cs;
2981
2982 if (!intel_engine_has_preemption(engine))
2983 continue;
2984
2985 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2986 return -EIO;
2987
2988 do {
2989 struct i915_sched_attr attr = { .priority = prio++ };
2990
2991 err = create_gang(engine, &rq);
2992 if (err)
2993 break;
2994
2995 /* Submit each spinner at increasing priority */
2996 engine->sched_engine->schedule(rq, &attr);
2997 } while (prio <= I915_PRIORITY_MAX &&
2998 !__igt_timeout(end_time, NULL));
2999 pr_debug("%s: Preempt chain of %d requests\n",
3000 engine->name, prio);
3001
3002 /*
3003 * Such that the last spinner is the highest priority and
3004 * should execute first. When that spinner completes,
3005 * it will terminate the next lowest spinner until there
3006 * are no more spinners and the gang is complete.
3007 */
3008 cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);
3009 if (!IS_ERR(cs)) {
3010 *cs = 0;
3011 i915_gem_object_unpin_map(rq->batch->obj);
3012 } else {
3013 err = PTR_ERR(cs);
3014 intel_gt_set_wedged(gt);
3015 }
3016
3017 while (rq) { /* wait for each rq from highest to lowest prio */
3018 struct i915_request *n = list_next_entry(rq, mock.link);
3019
3020 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
3021 struct drm_printer p =
3022 drm_info_printer(engine->i915->drm.dev);
3023
3024 pr_err("Failed to flush chain of %d requests, at %d\n",
3025 prio, rq_prio(rq));
3026 intel_engine_dump(engine, &p,
3027 "%s\n", engine->name);
3028
3029 err = -ETIME;
3030 }
3031
3032 i915_vma_put(rq->batch);
3033 i915_request_put(rq);
3034 rq = n;
3035 }
3036
3037 if (igt_live_test_end(&t))
3038 err = -EIO;
3039 if (err)
3040 return err;
3041 }
3042
3043 return 0;
3044 }
3045
3046 static struct i915_vma *
create_gpr_user(struct intel_engine_cs * engine,struct i915_vma * result,unsigned int offset)3047 create_gpr_user(struct intel_engine_cs *engine,
3048 struct i915_vma *result,
3049 unsigned int offset)
3050 {
3051 struct drm_i915_gem_object *obj;
3052 struct i915_vma *vma;
3053 u32 *cs;
3054 int err;
3055 int i;
3056
3057 obj = i915_gem_object_create_internal(engine->i915, 4096);
3058 if (IS_ERR(obj))
3059 return ERR_CAST(obj);
3060
3061 vma = i915_vma_instance(obj, result->vm, NULL);
3062 if (IS_ERR(vma)) {
3063 i915_gem_object_put(obj);
3064 return vma;
3065 }
3066
3067 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3068 if (err) {
3069 i915_vma_put(vma);
3070 return ERR_PTR(err);
3071 }
3072
3073 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
3074 if (IS_ERR(cs)) {
3075 i915_vma_put(vma);
3076 return ERR_CAST(cs);
3077 }
3078
3079 /* All GPR are clear for new contexts. We use GPR(0) as a constant */
3080 *cs++ = MI_LOAD_REGISTER_IMM(1);
3081 *cs++ = CS_GPR(engine, 0);
3082 *cs++ = 1;
3083
3084 for (i = 1; i < NUM_GPR; i++) {
3085 u64 addr;
3086
3087 /*
3088 * Perform: GPR[i]++
3089 *
3090 * As we read and write into the context saved GPR[i], if
3091 * we restart this batch buffer from an earlier point, we
3092 * will repeat the increment and store a value > 1.
3093 */
3094 *cs++ = MI_MATH(4);
3095 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));
3096 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));
3097 *cs++ = MI_MATH_ADD;
3098 *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);
3099
3100 addr = i915_vma_offset(result) + offset + i * sizeof(*cs);
3101 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
3102 *cs++ = CS_GPR(engine, 2 * i);
3103 *cs++ = lower_32_bits(addr);
3104 *cs++ = upper_32_bits(addr);
3105
3106 *cs++ = MI_SEMAPHORE_WAIT |
3107 MI_SEMAPHORE_POLL |
3108 MI_SEMAPHORE_SAD_GTE_SDD;
3109 *cs++ = i;
3110 *cs++ = lower_32_bits(i915_vma_offset(result));
3111 *cs++ = upper_32_bits(i915_vma_offset(result));
3112 }
3113
3114 *cs++ = MI_BATCH_BUFFER_END;
3115 i915_gem_object_flush_map(obj);
3116 i915_gem_object_unpin_map(obj);
3117
3118 return vma;
3119 }
3120
create_global(struct intel_gt * gt,size_t sz)3121 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
3122 {
3123 struct drm_i915_gem_object *obj;
3124 struct i915_vma *vma;
3125 int err;
3126
3127 obj = i915_gem_object_create_internal(gt->i915, sz);
3128 if (IS_ERR(obj))
3129 return ERR_CAST(obj);
3130
3131 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
3132 if (IS_ERR(vma)) {
3133 i915_gem_object_put(obj);
3134 return vma;
3135 }
3136
3137 err = i915_ggtt_pin(vma, NULL, 0, 0);
3138 if (err) {
3139 i915_vma_put(vma);
3140 return ERR_PTR(err);
3141 }
3142
3143 return vma;
3144 }
3145
3146 static struct i915_request *
create_gpr_client(struct intel_engine_cs * engine,struct i915_vma * global,unsigned int offset)3147 create_gpr_client(struct intel_engine_cs *engine,
3148 struct i915_vma *global,
3149 unsigned int offset)
3150 {
3151 struct i915_vma *batch, *vma;
3152 struct intel_context *ce;
3153 struct i915_request *rq;
3154 int err;
3155
3156 ce = intel_context_create(engine);
3157 if (IS_ERR(ce))
3158 return ERR_CAST(ce);
3159
3160 vma = i915_vma_instance(global->obj, ce->vm, NULL);
3161 if (IS_ERR(vma)) {
3162 err = PTR_ERR(vma);
3163 goto out_ce;
3164 }
3165
3166 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3167 if (err)
3168 goto out_ce;
3169
3170 batch = create_gpr_user(engine, vma, offset);
3171 if (IS_ERR(batch)) {
3172 err = PTR_ERR(batch);
3173 goto out_vma;
3174 }
3175
3176 rq = intel_context_create_request(ce);
3177 if (IS_ERR(rq)) {
3178 err = PTR_ERR(rq);
3179 goto out_batch;
3180 }
3181
3182 err = igt_vma_move_to_active_unlocked(vma, rq, 0);
3183
3184 i915_vma_lock(batch);
3185 if (!err)
3186 err = i915_vma_move_to_active(batch, rq, 0);
3187 if (!err)
3188 err = rq->engine->emit_bb_start(rq,
3189 i915_vma_offset(batch),
3190 PAGE_SIZE, 0);
3191 i915_vma_unlock(batch);
3192 i915_vma_unpin(batch);
3193
3194 if (!err)
3195 i915_request_get(rq);
3196 i915_request_add(rq);
3197
3198 out_batch:
3199 i915_vma_put(batch);
3200 out_vma:
3201 i915_vma_unpin(vma);
3202 out_ce:
3203 intel_context_put(ce);
3204 return err ? ERR_PTR(err) : rq;
3205 }
3206
preempt_user(struct intel_engine_cs * engine,struct i915_vma * global,int id)3207 static int preempt_user(struct intel_engine_cs *engine,
3208 struct i915_vma *global,
3209 int id)
3210 {
3211 struct i915_sched_attr attr = {
3212 .priority = I915_PRIORITY_MAX
3213 };
3214 struct i915_request *rq;
3215 int err = 0;
3216 u32 *cs;
3217
3218 rq = intel_engine_create_kernel_request(engine);
3219 if (IS_ERR(rq))
3220 return PTR_ERR(rq);
3221
3222 cs = intel_ring_begin(rq, 4);
3223 if (IS_ERR(cs)) {
3224 i915_request_add(rq);
3225 return PTR_ERR(cs);
3226 }
3227
3228 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
3229 *cs++ = i915_ggtt_offset(global);
3230 *cs++ = 0;
3231 *cs++ = id;
3232
3233 intel_ring_advance(rq, cs);
3234
3235 i915_request_get(rq);
3236 i915_request_add(rq);
3237
3238 engine->sched_engine->schedule(rq, &attr);
3239
3240 if (i915_request_wait(rq, 0, HZ / 2) < 0)
3241 err = -ETIME;
3242 i915_request_put(rq);
3243
3244 return err;
3245 }
3246
live_preempt_user(void * arg)3247 static int live_preempt_user(void *arg)
3248 {
3249 struct intel_gt *gt = arg;
3250 struct intel_engine_cs *engine;
3251 struct i915_vma *global;
3252 enum intel_engine_id id;
3253 u32 *result;
3254 int err = 0;
3255
3256 /*
3257 * In our other tests, we look at preemption in carefully
3258 * controlled conditions in the ringbuffer. Since most of the
3259 * time is spent in user batches, most of our preemptions naturally
3260 * occur there. We want to verify that when we preempt inside a batch
3261 * we continue on from the current instruction and do not roll back
3262 * to the start, or another earlier arbitration point.
3263 *
3264 * To verify this, we create a batch which is a mixture of
3265 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with
3266 * a few preempting contexts thrown into the mix, we look for any
3267 * repeated instructions (which show up as incorrect values).
3268 */
3269
3270 global = create_global(gt, 4096);
3271 if (IS_ERR(global))
3272 return PTR_ERR(global);
3273
3274 result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);
3275 if (IS_ERR(result)) {
3276 i915_vma_unpin_and_release(&global, 0);
3277 return PTR_ERR(result);
3278 }
3279
3280 for_each_engine(engine, gt, id) {
3281 struct i915_request *client[3] = {};
3282 struct igt_live_test t;
3283 int i;
3284
3285 if (!intel_engine_has_preemption(engine))
3286 continue;
3287
3288 if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS)
3289 continue; /* we need per-context GPR */
3290
3291 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
3292 err = -EIO;
3293 break;
3294 }
3295
3296 memset(result, 0, 4096);
3297
3298 for (i = 0; i < ARRAY_SIZE(client); i++) {
3299 struct i915_request *rq;
3300
3301 rq = create_gpr_client(engine, global,
3302 NUM_GPR * i * sizeof(u32));
3303 if (IS_ERR(rq)) {
3304 err = PTR_ERR(rq);
3305 goto end_test;
3306 }
3307
3308 client[i] = rq;
3309 }
3310
3311 /* Continuously preempt the set of 3 running contexts */
3312 for (i = 1; i <= NUM_GPR; i++) {
3313 err = preempt_user(engine, global, i);
3314 if (err)
3315 goto end_test;
3316 }
3317
3318 if (READ_ONCE(result[0]) != NUM_GPR) {
3319 pr_err("%s: Failed to release semaphore\n",
3320 engine->name);
3321 err = -EIO;
3322 goto end_test;
3323 }
3324
3325 for (i = 0; i < ARRAY_SIZE(client); i++) {
3326 int gpr;
3327
3328 if (i915_request_wait(client[i], 0, HZ / 2) < 0) {
3329 err = -ETIME;
3330 goto end_test;
3331 }
3332
3333 for (gpr = 1; gpr < NUM_GPR; gpr++) {
3334 if (result[NUM_GPR * i + gpr] != 1) {
3335 pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",
3336 engine->name,
3337 i, gpr, result[NUM_GPR * i + gpr]);
3338 err = -EINVAL;
3339 goto end_test;
3340 }
3341 }
3342 }
3343
3344 end_test:
3345 for (i = 0; i < ARRAY_SIZE(client); i++) {
3346 if (!client[i])
3347 break;
3348
3349 i915_request_put(client[i]);
3350 }
3351
3352 /* Flush the semaphores on error */
3353 smp_store_mb(result[0], -1);
3354 if (igt_live_test_end(&t))
3355 err = -EIO;
3356 if (err)
3357 break;
3358 }
3359
3360 i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);
3361 return err;
3362 }
3363
live_preempt_timeout(void * arg)3364 static int live_preempt_timeout(void *arg)
3365 {
3366 struct intel_gt *gt = arg;
3367 struct i915_gem_context *ctx_hi, *ctx_lo;
3368 struct igt_spinner spin_lo;
3369 struct intel_engine_cs *engine;
3370 enum intel_engine_id id;
3371 int err = -ENOMEM;
3372
3373 /*
3374 * Check that we force preemption to occur by cancelling the previous
3375 * context if it refuses to yield the GPU.
3376 */
3377 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
3378 return 0;
3379
3380 if (!intel_has_reset_engine(gt))
3381 return 0;
3382
3383 ctx_hi = kernel_context(gt->i915, NULL);
3384 if (!ctx_hi)
3385 return -ENOMEM;
3386 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
3387
3388 ctx_lo = kernel_context(gt->i915, NULL);
3389 if (!ctx_lo)
3390 goto err_ctx_hi;
3391 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
3392
3393 if (igt_spinner_init(&spin_lo, gt))
3394 goto err_ctx_lo;
3395
3396 for_each_engine(engine, gt, id) {
3397 unsigned long saved_timeout;
3398 struct i915_request *rq;
3399
3400 if (!intel_engine_has_preemption(engine))
3401 continue;
3402
3403 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
3404 MI_NOOP); /* preemption disabled */
3405 if (IS_ERR(rq)) {
3406 err = PTR_ERR(rq);
3407 goto err_spin_lo;
3408 }
3409
3410 i915_request_add(rq);
3411 if (!igt_wait_for_spinner(&spin_lo, rq)) {
3412 intel_gt_set_wedged(gt);
3413 err = -EIO;
3414 goto err_spin_lo;
3415 }
3416
3417 rq = igt_request_alloc(ctx_hi, engine);
3418 if (IS_ERR(rq)) {
3419 igt_spinner_end(&spin_lo);
3420 err = PTR_ERR(rq);
3421 goto err_spin_lo;
3422 }
3423
3424 /* Flush the previous CS ack before changing timeouts */
3425 while (READ_ONCE(engine->execlists.pending[0]))
3426 cpu_relax();
3427
3428 saved_timeout = engine->props.preempt_timeout_ms;
3429 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
3430
3431 i915_request_get(rq);
3432 i915_request_add(rq);
3433
3434 intel_engine_flush_submission(engine);
3435 engine->props.preempt_timeout_ms = saved_timeout;
3436
3437 if (i915_request_wait(rq, 0, HZ / 10) < 0) {
3438 intel_gt_set_wedged(gt);
3439 i915_request_put(rq);
3440 err = -ETIME;
3441 goto err_spin_lo;
3442 }
3443
3444 igt_spinner_end(&spin_lo);
3445 i915_request_put(rq);
3446 }
3447
3448 err = 0;
3449 err_spin_lo:
3450 igt_spinner_fini(&spin_lo);
3451 err_ctx_lo:
3452 kernel_context_close(ctx_lo);
3453 err_ctx_hi:
3454 kernel_context_close(ctx_hi);
3455 return err;
3456 }
3457
random_range(struct rnd_state * rnd,int min,int max)3458 static int random_range(struct rnd_state *rnd, int min, int max)
3459 {
3460 return i915_prandom_u32_max_state(max - min, rnd) + min;
3461 }
3462
random_priority(struct rnd_state * rnd)3463 static int random_priority(struct rnd_state *rnd)
3464 {
3465 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
3466 }
3467
3468 struct preempt_smoke {
3469 struct intel_gt *gt;
3470 struct kthread_work work;
3471 struct i915_gem_context **contexts;
3472 struct intel_engine_cs *engine;
3473 struct drm_i915_gem_object *batch;
3474 unsigned int ncontext;
3475 struct rnd_state prng;
3476 unsigned long count;
3477 int result;
3478 };
3479
smoke_context(struct preempt_smoke * smoke)3480 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
3481 {
3482 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
3483 &smoke->prng)];
3484 }
3485
smoke_submit(struct preempt_smoke * smoke,struct i915_gem_context * ctx,int prio,struct drm_i915_gem_object * batch)3486 static int smoke_submit(struct preempt_smoke *smoke,
3487 struct i915_gem_context *ctx, int prio,
3488 struct drm_i915_gem_object *batch)
3489 {
3490 struct i915_request *rq;
3491 struct i915_vma *vma = NULL;
3492 int err = 0;
3493
3494 if (batch) {
3495 struct i915_address_space *vm;
3496
3497 vm = i915_gem_context_get_eb_vm(ctx);
3498 vma = i915_vma_instance(batch, vm, NULL);
3499 i915_vm_put(vm);
3500 if (IS_ERR(vma))
3501 return PTR_ERR(vma);
3502
3503 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3504 if (err)
3505 return err;
3506 }
3507
3508 ctx->sched.priority = prio;
3509
3510 rq = igt_request_alloc(ctx, smoke->engine);
3511 if (IS_ERR(rq)) {
3512 err = PTR_ERR(rq);
3513 goto unpin;
3514 }
3515
3516 if (vma) {
3517 err = igt_vma_move_to_active_unlocked(vma, rq, 0);
3518 if (!err)
3519 err = rq->engine->emit_bb_start(rq,
3520 i915_vma_offset(vma),
3521 PAGE_SIZE, 0);
3522 }
3523
3524 i915_request_add(rq);
3525
3526 unpin:
3527 if (vma)
3528 i915_vma_unpin(vma);
3529
3530 return err;
3531 }
3532
smoke_crescendo_work(struct kthread_work * work)3533 static void smoke_crescendo_work(struct kthread_work *work)
3534 {
3535 struct preempt_smoke *smoke = container_of(work, typeof(*smoke), work);
3536 IGT_TIMEOUT(end_time);
3537 unsigned long count;
3538
3539 count = 0;
3540 do {
3541 struct i915_gem_context *ctx = smoke_context(smoke);
3542
3543 smoke->result = smoke_submit(smoke, ctx,
3544 count % I915_PRIORITY_MAX,
3545 smoke->batch);
3546
3547 count++;
3548 } while (!smoke->result && count < smoke->ncontext &&
3549 !__igt_timeout(end_time, NULL));
3550
3551 smoke->count = count;
3552 }
3553
smoke_crescendo(struct preempt_smoke * smoke,unsigned int flags)3554 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
3555 #define BATCH BIT(0)
3556 {
3557 struct kthread_worker *worker[I915_NUM_ENGINES] = {};
3558 struct preempt_smoke *arg;
3559 struct intel_engine_cs *engine;
3560 enum intel_engine_id id;
3561 unsigned long count;
3562 int err = 0;
3563
3564 arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL);
3565 if (!arg)
3566 return -ENOMEM;
3567
3568 memset(arg, 0, I915_NUM_ENGINES * sizeof(*arg));
3569
3570 for_each_engine(engine, smoke->gt, id) {
3571 arg[id] = *smoke;
3572 arg[id].engine = engine;
3573 if (!(flags & BATCH))
3574 arg[id].batch = NULL;
3575 arg[id].count = 0;
3576
3577 worker[id] = kthread_create_worker(0, "igt/smoke:%d", id);
3578 if (IS_ERR(worker[id])) {
3579 err = PTR_ERR(worker[id]);
3580 break;
3581 }
3582
3583 kthread_init_work(&arg[id].work, smoke_crescendo_work);
3584 kthread_queue_work(worker[id], &arg[id].work);
3585 }
3586
3587 count = 0;
3588 for_each_engine(engine, smoke->gt, id) {
3589 if (IS_ERR_OR_NULL(worker[id]))
3590 continue;
3591
3592 kthread_flush_work(&arg[id].work);
3593 if (arg[id].result && !err)
3594 err = arg[id].result;
3595
3596 count += arg[id].count;
3597
3598 kthread_destroy_worker(worker[id]);
3599 }
3600
3601 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3602 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3603
3604 kfree(arg);
3605 return 0;
3606 }
3607
smoke_random(struct preempt_smoke * smoke,unsigned int flags)3608 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
3609 {
3610 enum intel_engine_id id;
3611 IGT_TIMEOUT(end_time);
3612 unsigned long count;
3613
3614 count = 0;
3615 do {
3616 for_each_engine(smoke->engine, smoke->gt, id) {
3617 struct i915_gem_context *ctx = smoke_context(smoke);
3618 int err;
3619
3620 err = smoke_submit(smoke,
3621 ctx, random_priority(&smoke->prng),
3622 flags & BATCH ? smoke->batch : NULL);
3623 if (err)
3624 return err;
3625
3626 count++;
3627 }
3628 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3629
3630 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3631 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3632 return 0;
3633 }
3634
live_preempt_smoke(void * arg)3635 static int live_preempt_smoke(void *arg)
3636 {
3637 struct preempt_smoke smoke = {
3638 .gt = arg,
3639 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3640 .ncontext = 256,
3641 };
3642 const unsigned int phase[] = { 0, BATCH };
3643 struct igt_live_test t;
3644 int err = -ENOMEM;
3645 u32 *cs;
3646 int n;
3647
3648 smoke.contexts = kmalloc_array(smoke.ncontext,
3649 sizeof(*smoke.contexts),
3650 GFP_KERNEL);
3651 if (!smoke.contexts)
3652 return -ENOMEM;
3653
3654 smoke.batch =
3655 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
3656 if (IS_ERR(smoke.batch)) {
3657 err = PTR_ERR(smoke.batch);
3658 goto err_free;
3659 }
3660
3661 cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);
3662 if (IS_ERR(cs)) {
3663 err = PTR_ERR(cs);
3664 goto err_batch;
3665 }
3666 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3667 cs[n] = MI_ARB_CHECK;
3668 cs[n] = MI_BATCH_BUFFER_END;
3669 i915_gem_object_flush_map(smoke.batch);
3670 i915_gem_object_unpin_map(smoke.batch);
3671
3672 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
3673 err = -EIO;
3674 goto err_batch;
3675 }
3676
3677 for (n = 0; n < smoke.ncontext; n++) {
3678 smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL);
3679 if (!smoke.contexts[n])
3680 goto err_ctx;
3681 }
3682
3683 for (n = 0; n < ARRAY_SIZE(phase); n++) {
3684 err = smoke_crescendo(&smoke, phase[n]);
3685 if (err)
3686 goto err_ctx;
3687
3688 err = smoke_random(&smoke, phase[n]);
3689 if (err)
3690 goto err_ctx;
3691 }
3692
3693 err_ctx:
3694 if (igt_live_test_end(&t))
3695 err = -EIO;
3696
3697 for (n = 0; n < smoke.ncontext; n++) {
3698 if (!smoke.contexts[n])
3699 break;
3700 kernel_context_close(smoke.contexts[n]);
3701 }
3702
3703 err_batch:
3704 i915_gem_object_put(smoke.batch);
3705 err_free:
3706 kfree(smoke.contexts);
3707
3708 return err;
3709 }
3710
nop_virtual_engine(struct intel_gt * gt,struct intel_engine_cs ** siblings,unsigned int nsibling,unsigned int nctx,unsigned int flags)3711 static int nop_virtual_engine(struct intel_gt *gt,
3712 struct intel_engine_cs **siblings,
3713 unsigned int nsibling,
3714 unsigned int nctx,
3715 unsigned int flags)
3716 #define CHAIN BIT(0)
3717 {
3718 IGT_TIMEOUT(end_time);
3719 struct i915_request *request[16] = {};
3720 struct intel_context *ve[16];
3721 unsigned long n, prime, nc;
3722 struct igt_live_test t;
3723 ktime_t times[2] = {};
3724 int err;
3725
3726 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
3727
3728 for (n = 0; n < nctx; n++) {
3729 ve[n] = intel_engine_create_virtual(siblings, nsibling, 0);
3730 if (IS_ERR(ve[n])) {
3731 err = PTR_ERR(ve[n]);
3732 nctx = n;
3733 goto out;
3734 }
3735
3736 err = intel_context_pin(ve[n]);
3737 if (err) {
3738 intel_context_put(ve[n]);
3739 nctx = n;
3740 goto out;
3741 }
3742 }
3743
3744 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
3745 if (err)
3746 goto out;
3747
3748 for_each_prime_number_from(prime, 1, 8192) {
3749 times[1] = ktime_get_raw();
3750
3751 if (flags & CHAIN) {
3752 for (nc = 0; nc < nctx; nc++) {
3753 for (n = 0; n < prime; n++) {
3754 struct i915_request *rq;
3755
3756 rq = i915_request_create(ve[nc]);
3757 if (IS_ERR(rq)) {
3758 err = PTR_ERR(rq);
3759 goto out;
3760 }
3761
3762 if (request[nc])
3763 i915_request_put(request[nc]);
3764 request[nc] = i915_request_get(rq);
3765 i915_request_add(rq);
3766 }
3767 }
3768 } else {
3769 for (n = 0; n < prime; n++) {
3770 for (nc = 0; nc < nctx; nc++) {
3771 struct i915_request *rq;
3772
3773 rq = i915_request_create(ve[nc]);
3774 if (IS_ERR(rq)) {
3775 err = PTR_ERR(rq);
3776 goto out;
3777 }
3778
3779 if (request[nc])
3780 i915_request_put(request[nc]);
3781 request[nc] = i915_request_get(rq);
3782 i915_request_add(rq);
3783 }
3784 }
3785 }
3786
3787 for (nc = 0; nc < nctx; nc++) {
3788 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
3789 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3790 __func__, ve[0]->engine->name,
3791 request[nc]->fence.context,
3792 request[nc]->fence.seqno);
3793
3794 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3795 __func__, ve[0]->engine->name,
3796 request[nc]->fence.context,
3797 request[nc]->fence.seqno);
3798 GEM_TRACE_DUMP();
3799 intel_gt_set_wedged(gt);
3800 break;
3801 }
3802 }
3803
3804 times[1] = ktime_sub(ktime_get_raw(), times[1]);
3805 if (prime == 1)
3806 times[0] = times[1];
3807
3808 for (nc = 0; nc < nctx; nc++) {
3809 i915_request_put(request[nc]);
3810 request[nc] = NULL;
3811 }
3812
3813 if (__igt_timeout(end_time, NULL))
3814 break;
3815 }
3816
3817 err = igt_live_test_end(&t);
3818 if (err)
3819 goto out;
3820
3821 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3822 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3823 prime, div64_u64(ktime_to_ns(times[1]), prime));
3824
3825 out:
3826 if (igt_flush_test(gt->i915))
3827 err = -EIO;
3828
3829 for (nc = 0; nc < nctx; nc++) {
3830 i915_request_put(request[nc]);
3831 intel_context_unpin(ve[nc]);
3832 intel_context_put(ve[nc]);
3833 }
3834 return err;
3835 }
3836
3837 static unsigned int
__select_siblings(struct intel_gt * gt,unsigned int class,struct intel_engine_cs ** siblings,bool (* filter)(const struct intel_engine_cs *))3838 __select_siblings(struct intel_gt *gt,
3839 unsigned int class,
3840 struct intel_engine_cs **siblings,
3841 bool (*filter)(const struct intel_engine_cs *))
3842 {
3843 unsigned int n = 0;
3844 unsigned int inst;
3845
3846 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3847 if (!gt->engine_class[class][inst])
3848 continue;
3849
3850 if (filter && !filter(gt->engine_class[class][inst]))
3851 continue;
3852
3853 siblings[n++] = gt->engine_class[class][inst];
3854 }
3855
3856 return n;
3857 }
3858
3859 static unsigned int
select_siblings(struct intel_gt * gt,unsigned int class,struct intel_engine_cs ** siblings)3860 select_siblings(struct intel_gt *gt,
3861 unsigned int class,
3862 struct intel_engine_cs **siblings)
3863 {
3864 return __select_siblings(gt, class, siblings, NULL);
3865 }
3866
live_virtual_engine(void * arg)3867 static int live_virtual_engine(void *arg)
3868 {
3869 struct intel_gt *gt = arg;
3870 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3871 struct intel_engine_cs *engine;
3872 enum intel_engine_id id;
3873 unsigned int class;
3874 int err;
3875
3876 if (intel_uc_uses_guc_submission(>->uc))
3877 return 0;
3878
3879 for_each_engine(engine, gt, id) {
3880 err = nop_virtual_engine(gt, &engine, 1, 1, 0);
3881 if (err) {
3882 pr_err("Failed to wrap engine %s: err=%d\n",
3883 engine->name, err);
3884 return err;
3885 }
3886 }
3887
3888 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3889 int nsibling, n;
3890
3891 nsibling = select_siblings(gt, class, siblings);
3892 if (nsibling < 2)
3893 continue;
3894
3895 for (n = 1; n <= nsibling + 1; n++) {
3896 err = nop_virtual_engine(gt, siblings, nsibling,
3897 n, 0);
3898 if (err)
3899 return err;
3900 }
3901
3902 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
3903 if (err)
3904 return err;
3905 }
3906
3907 return 0;
3908 }
3909
mask_virtual_engine(struct intel_gt * gt,struct intel_engine_cs ** siblings,unsigned int nsibling)3910 static int mask_virtual_engine(struct intel_gt *gt,
3911 struct intel_engine_cs **siblings,
3912 unsigned int nsibling)
3913 {
3914 struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
3915 struct intel_context *ve;
3916 struct igt_live_test t;
3917 unsigned int n;
3918 int err;
3919
3920 /*
3921 * Check that by setting the execution mask on a request, we can
3922 * restrict it to our desired engine within the virtual engine.
3923 */
3924
3925 ve = intel_engine_create_virtual(siblings, nsibling, 0);
3926 if (IS_ERR(ve)) {
3927 err = PTR_ERR(ve);
3928 goto out_close;
3929 }
3930
3931 err = intel_context_pin(ve);
3932 if (err)
3933 goto out_put;
3934
3935 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3936 if (err)
3937 goto out_unpin;
3938
3939 for (n = 0; n < nsibling; n++) {
3940 request[n] = i915_request_create(ve);
3941 if (IS_ERR(request[n])) {
3942 err = PTR_ERR(request[n]);
3943 nsibling = n;
3944 goto out;
3945 }
3946
3947 /* Reverse order as it's more likely to be unnatural */
3948 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3949
3950 i915_request_get(request[n]);
3951 i915_request_add(request[n]);
3952 }
3953
3954 for (n = 0; n < nsibling; n++) {
3955 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
3956 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3957 __func__, ve->engine->name,
3958 request[n]->fence.context,
3959 request[n]->fence.seqno);
3960
3961 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3962 __func__, ve->engine->name,
3963 request[n]->fence.context,
3964 request[n]->fence.seqno);
3965 GEM_TRACE_DUMP();
3966 intel_gt_set_wedged(gt);
3967 err = -EIO;
3968 goto out;
3969 }
3970
3971 if (request[n]->engine != siblings[nsibling - n - 1]) {
3972 pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3973 request[n]->engine->name,
3974 siblings[nsibling - n - 1]->name);
3975 err = -EINVAL;
3976 goto out;
3977 }
3978 }
3979
3980 err = igt_live_test_end(&t);
3981 out:
3982 if (igt_flush_test(gt->i915))
3983 err = -EIO;
3984
3985 for (n = 0; n < nsibling; n++)
3986 i915_request_put(request[n]);
3987
3988 out_unpin:
3989 intel_context_unpin(ve);
3990 out_put:
3991 intel_context_put(ve);
3992 out_close:
3993 return err;
3994 }
3995
live_virtual_mask(void * arg)3996 static int live_virtual_mask(void *arg)
3997 {
3998 struct intel_gt *gt = arg;
3999 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4000 unsigned int class;
4001 int err;
4002
4003 if (intel_uc_uses_guc_submission(>->uc))
4004 return 0;
4005
4006 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4007 unsigned int nsibling;
4008
4009 nsibling = select_siblings(gt, class, siblings);
4010 if (nsibling < 2)
4011 continue;
4012
4013 err = mask_virtual_engine(gt, siblings, nsibling);
4014 if (err)
4015 return err;
4016 }
4017
4018 return 0;
4019 }
4020
slicein_virtual_engine(struct intel_gt * gt,struct intel_engine_cs ** siblings,unsigned int nsibling)4021 static int slicein_virtual_engine(struct intel_gt *gt,
4022 struct intel_engine_cs **siblings,
4023 unsigned int nsibling)
4024 {
4025 const long timeout = slice_timeout(siblings[0]);
4026 struct intel_context *ce;
4027 struct i915_request *rq;
4028 struct igt_spinner spin;
4029 unsigned int n;
4030 int err = 0;
4031
4032 /*
4033 * Virtual requests must take part in timeslicing on the target engines.
4034 */
4035
4036 if (igt_spinner_init(&spin, gt))
4037 return -ENOMEM;
4038
4039 for (n = 0; n < nsibling; n++) {
4040 ce = intel_context_create(siblings[n]);
4041 if (IS_ERR(ce)) {
4042 err = PTR_ERR(ce);
4043 goto out;
4044 }
4045
4046 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4047 intel_context_put(ce);
4048 if (IS_ERR(rq)) {
4049 err = PTR_ERR(rq);
4050 goto out;
4051 }
4052
4053 i915_request_add(rq);
4054 }
4055
4056 ce = intel_engine_create_virtual(siblings, nsibling, 0);
4057 if (IS_ERR(ce)) {
4058 err = PTR_ERR(ce);
4059 goto out;
4060 }
4061
4062 rq = intel_context_create_request(ce);
4063 intel_context_put(ce);
4064 if (IS_ERR(rq)) {
4065 err = PTR_ERR(rq);
4066 goto out;
4067 }
4068
4069 i915_request_get(rq);
4070 i915_request_add(rq);
4071 if (i915_request_wait(rq, 0, timeout) < 0) {
4072 GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n",
4073 __func__, rq->engine->name);
4074 GEM_TRACE_DUMP();
4075 intel_gt_set_wedged(gt);
4076 err = -EIO;
4077 }
4078 i915_request_put(rq);
4079
4080 out:
4081 igt_spinner_end(&spin);
4082 if (igt_flush_test(gt->i915))
4083 err = -EIO;
4084 igt_spinner_fini(&spin);
4085 return err;
4086 }
4087
sliceout_virtual_engine(struct intel_gt * gt,struct intel_engine_cs ** siblings,unsigned int nsibling)4088 static int sliceout_virtual_engine(struct intel_gt *gt,
4089 struct intel_engine_cs **siblings,
4090 unsigned int nsibling)
4091 {
4092 const long timeout = slice_timeout(siblings[0]);
4093 struct intel_context *ce;
4094 struct i915_request *rq;
4095 struct igt_spinner spin;
4096 unsigned int n;
4097 int err = 0;
4098
4099 /*
4100 * Virtual requests must allow others a fair timeslice.
4101 */
4102
4103 if (igt_spinner_init(&spin, gt))
4104 return -ENOMEM;
4105
4106 /* XXX We do not handle oversubscription and fairness with normal rq */
4107 for (n = 0; n < nsibling; n++) {
4108 ce = intel_engine_create_virtual(siblings, nsibling, 0);
4109 if (IS_ERR(ce)) {
4110 err = PTR_ERR(ce);
4111 goto out;
4112 }
4113
4114 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4115 intel_context_put(ce);
4116 if (IS_ERR(rq)) {
4117 err = PTR_ERR(rq);
4118 goto out;
4119 }
4120
4121 i915_request_add(rq);
4122 }
4123
4124 for (n = 0; !err && n < nsibling; n++) {
4125 ce = intel_context_create(siblings[n]);
4126 if (IS_ERR(ce)) {
4127 err = PTR_ERR(ce);
4128 goto out;
4129 }
4130
4131 rq = intel_context_create_request(ce);
4132 intel_context_put(ce);
4133 if (IS_ERR(rq)) {
4134 err = PTR_ERR(rq);
4135 goto out;
4136 }
4137
4138 i915_request_get(rq);
4139 i915_request_add(rq);
4140 if (i915_request_wait(rq, 0, timeout) < 0) {
4141 GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n",
4142 __func__, siblings[n]->name);
4143 GEM_TRACE_DUMP();
4144 intel_gt_set_wedged(gt);
4145 err = -EIO;
4146 }
4147 i915_request_put(rq);
4148 }
4149
4150 out:
4151 igt_spinner_end(&spin);
4152 if (igt_flush_test(gt->i915))
4153 err = -EIO;
4154 igt_spinner_fini(&spin);
4155 return err;
4156 }
4157
live_virtual_slice(void * arg)4158 static int live_virtual_slice(void *arg)
4159 {
4160 struct intel_gt *gt = arg;
4161 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4162 unsigned int class;
4163 int err;
4164
4165 if (intel_uc_uses_guc_submission(>->uc))
4166 return 0;
4167
4168 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4169 unsigned int nsibling;
4170
4171 nsibling = __select_siblings(gt, class, siblings,
4172 intel_engine_has_timeslices);
4173 if (nsibling < 2)
4174 continue;
4175
4176 err = slicein_virtual_engine(gt, siblings, nsibling);
4177 if (err)
4178 return err;
4179
4180 err = sliceout_virtual_engine(gt, siblings, nsibling);
4181 if (err)
4182 return err;
4183 }
4184
4185 return 0;
4186 }
4187
preserved_virtual_engine(struct intel_gt * gt,struct intel_engine_cs ** siblings,unsigned int nsibling)4188 static int preserved_virtual_engine(struct intel_gt *gt,
4189 struct intel_engine_cs **siblings,
4190 unsigned int nsibling)
4191 {
4192 struct i915_request *last = NULL;
4193 struct intel_context *ve;
4194 struct i915_vma *scratch;
4195 struct igt_live_test t;
4196 unsigned int n;
4197 int err = 0;
4198 u32 *cs;
4199
4200 scratch =
4201 __vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,
4202 PAGE_SIZE);
4203 if (IS_ERR(scratch))
4204 return PTR_ERR(scratch);
4205
4206 err = i915_vma_sync(scratch);
4207 if (err)
4208 goto out_scratch;
4209
4210 ve = intel_engine_create_virtual(siblings, nsibling, 0);
4211 if (IS_ERR(ve)) {
4212 err = PTR_ERR(ve);
4213 goto out_scratch;
4214 }
4215
4216 err = intel_context_pin(ve);
4217 if (err)
4218 goto out_put;
4219
4220 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
4221 if (err)
4222 goto out_unpin;
4223
4224 for (n = 0; n < NUM_GPR_DW; n++) {
4225 struct intel_engine_cs *engine = siblings[n % nsibling];
4226 struct i915_request *rq;
4227
4228 rq = i915_request_create(ve);
4229 if (IS_ERR(rq)) {
4230 err = PTR_ERR(rq);
4231 goto out_end;
4232 }
4233
4234 i915_request_put(last);
4235 last = i915_request_get(rq);
4236
4237 cs = intel_ring_begin(rq, 8);
4238 if (IS_ERR(cs)) {
4239 i915_request_add(rq);
4240 err = PTR_ERR(cs);
4241 goto out_end;
4242 }
4243
4244 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4245 *cs++ = CS_GPR(engine, n);
4246 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4247 *cs++ = 0;
4248
4249 *cs++ = MI_LOAD_REGISTER_IMM(1);
4250 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
4251 *cs++ = n + 1;
4252
4253 *cs++ = MI_NOOP;
4254 intel_ring_advance(rq, cs);
4255
4256 /* Restrict this request to run on a particular engine */
4257 rq->execution_mask = engine->mask;
4258 i915_request_add(rq);
4259 }
4260
4261 if (i915_request_wait(last, 0, HZ / 5) < 0) {
4262 err = -ETIME;
4263 goto out_end;
4264 }
4265
4266 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
4267 if (IS_ERR(cs)) {
4268 err = PTR_ERR(cs);
4269 goto out_end;
4270 }
4271
4272 for (n = 0; n < NUM_GPR_DW; n++) {
4273 if (cs[n] != n) {
4274 pr_err("Incorrect value[%d] found for GPR[%d]\n",
4275 cs[n], n);
4276 err = -EINVAL;
4277 break;
4278 }
4279 }
4280
4281 i915_gem_object_unpin_map(scratch->obj);
4282
4283 out_end:
4284 if (igt_live_test_end(&t))
4285 err = -EIO;
4286 i915_request_put(last);
4287 out_unpin:
4288 intel_context_unpin(ve);
4289 out_put:
4290 intel_context_put(ve);
4291 out_scratch:
4292 i915_vma_unpin_and_release(&scratch, 0);
4293 return err;
4294 }
4295
live_virtual_preserved(void * arg)4296 static int live_virtual_preserved(void *arg)
4297 {
4298 struct intel_gt *gt = arg;
4299 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4300 unsigned int class;
4301
4302 /*
4303 * Check that the context image retains non-privileged (user) registers
4304 * from one engine to the next. For this we check that the CS_GPR
4305 * are preserved.
4306 */
4307
4308 if (intel_uc_uses_guc_submission(>->uc))
4309 return 0;
4310
4311 /* As we use CS_GPR we cannot run before they existed on all engines. */
4312 if (GRAPHICS_VER(gt->i915) < 9)
4313 return 0;
4314
4315 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4316 int nsibling, err;
4317
4318 nsibling = select_siblings(gt, class, siblings);
4319 if (nsibling < 2)
4320 continue;
4321
4322 err = preserved_virtual_engine(gt, siblings, nsibling);
4323 if (err)
4324 return err;
4325 }
4326
4327 return 0;
4328 }
4329
reset_virtual_engine(struct intel_gt * gt,struct intel_engine_cs ** siblings,unsigned int nsibling)4330 static int reset_virtual_engine(struct intel_gt *gt,
4331 struct intel_engine_cs **siblings,
4332 unsigned int nsibling)
4333 {
4334 struct intel_engine_cs *engine;
4335 struct intel_context *ve;
4336 struct igt_spinner spin;
4337 struct i915_request *rq;
4338 unsigned int n;
4339 int err = 0;
4340
4341 /*
4342 * In order to support offline error capture for fast preempt reset,
4343 * we need to decouple the guilty request and ensure that it and its
4344 * descendents are not executed while the capture is in progress.
4345 */
4346
4347 if (igt_spinner_init(&spin, gt))
4348 return -ENOMEM;
4349
4350 ve = intel_engine_create_virtual(siblings, nsibling, 0);
4351 if (IS_ERR(ve)) {
4352 err = PTR_ERR(ve);
4353 goto out_spin;
4354 }
4355
4356 for (n = 0; n < nsibling; n++)
4357 st_engine_heartbeat_disable(siblings[n]);
4358
4359 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
4360 if (IS_ERR(rq)) {
4361 err = PTR_ERR(rq);
4362 goto out_heartbeat;
4363 }
4364 i915_request_add(rq);
4365
4366 if (!igt_wait_for_spinner(&spin, rq)) {
4367 intel_gt_set_wedged(gt);
4368 err = -ETIME;
4369 goto out_heartbeat;
4370 }
4371
4372 engine = rq->engine;
4373 GEM_BUG_ON(engine == ve->engine);
4374
4375 /* Take ownership of the reset and tasklet */
4376 err = engine_lock_reset_tasklet(engine);
4377 if (err)
4378 goto out_heartbeat;
4379
4380 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
4381 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
4382
4383 /* Fake a preemption event; failed of course */
4384 spin_lock_irq(&engine->sched_engine->lock);
4385 __unwind_incomplete_requests(engine);
4386 spin_unlock_irq(&engine->sched_engine->lock);
4387 GEM_BUG_ON(rq->engine != engine);
4388
4389 /* Reset the engine while keeping our active request on hold */
4390 execlists_hold(engine, rq);
4391 GEM_BUG_ON(!i915_request_on_hold(rq));
4392
4393 __intel_engine_reset_bh(engine, NULL);
4394 GEM_BUG_ON(rq->fence.error != -EIO);
4395
4396 /* Release our grasp on the engine, letting CS flow again */
4397 engine_unlock_reset_tasklet(engine);
4398
4399 /* Check that we do not resubmit the held request */
4400 i915_request_get(rq);
4401 if (!i915_request_wait(rq, 0, HZ / 5)) {
4402 pr_err("%s: on hold request completed!\n",
4403 engine->name);
4404 intel_gt_set_wedged(gt);
4405 err = -EIO;
4406 goto out_rq;
4407 }
4408 GEM_BUG_ON(!i915_request_on_hold(rq));
4409
4410 /* But is resubmitted on release */
4411 execlists_unhold(engine, rq);
4412 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4413 pr_err("%s: held request did not complete!\n",
4414 engine->name);
4415 intel_gt_set_wedged(gt);
4416 err = -ETIME;
4417 }
4418
4419 out_rq:
4420 i915_request_put(rq);
4421 out_heartbeat:
4422 for (n = 0; n < nsibling; n++)
4423 st_engine_heartbeat_enable(siblings[n]);
4424
4425 intel_context_put(ve);
4426 out_spin:
4427 igt_spinner_fini(&spin);
4428 return err;
4429 }
4430
live_virtual_reset(void * arg)4431 static int live_virtual_reset(void *arg)
4432 {
4433 struct intel_gt *gt = arg;
4434 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4435 unsigned int class;
4436
4437 /*
4438 * Check that we handle a reset event within a virtual engine.
4439 * Only the physical engine is reset, but we have to check the flow
4440 * of the virtual requests around the reset, and make sure it is not
4441 * forgotten.
4442 */
4443
4444 if (intel_uc_uses_guc_submission(>->uc))
4445 return 0;
4446
4447 if (!intel_has_reset_engine(gt))
4448 return 0;
4449
4450 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4451 int nsibling, err;
4452
4453 nsibling = select_siblings(gt, class, siblings);
4454 if (nsibling < 2)
4455 continue;
4456
4457 err = reset_virtual_engine(gt, siblings, nsibling);
4458 if (err)
4459 return err;
4460 }
4461
4462 return 0;
4463 }
4464
intel_execlists_live_selftests(struct drm_i915_private * i915)4465 int intel_execlists_live_selftests(struct drm_i915_private *i915)
4466 {
4467 static const struct i915_subtest tests[] = {
4468 SUBTEST(live_sanitycheck),
4469 SUBTEST(live_unlite_switch),
4470 SUBTEST(live_unlite_preempt),
4471 SUBTEST(live_unlite_ring),
4472 SUBTEST(live_pin_rewind),
4473 SUBTEST(live_hold_reset),
4474 SUBTEST(live_error_interrupt),
4475 SUBTEST(live_timeslice_preempt),
4476 SUBTEST(live_timeslice_rewind),
4477 SUBTEST(live_timeslice_queue),
4478 SUBTEST(live_timeslice_nopreempt),
4479 SUBTEST(live_busywait_preempt),
4480 SUBTEST(live_preempt),
4481 SUBTEST(live_late_preempt),
4482 SUBTEST(live_nopreempt),
4483 SUBTEST(live_preempt_cancel),
4484 SUBTEST(live_suppress_self_preempt),
4485 SUBTEST(live_chain_preempt),
4486 SUBTEST(live_preempt_ring),
4487 SUBTEST(live_preempt_gang),
4488 SUBTEST(live_preempt_timeout),
4489 SUBTEST(live_preempt_user),
4490 SUBTEST(live_preempt_smoke),
4491 SUBTEST(live_virtual_engine),
4492 SUBTEST(live_virtual_mask),
4493 SUBTEST(live_virtual_preserved),
4494 SUBTEST(live_virtual_slice),
4495 SUBTEST(live_virtual_reset),
4496 };
4497
4498 if (to_gt(i915)->submission_method != INTEL_SUBMISSION_ELSP)
4499 return 0;
4500
4501 if (intel_gt_is_wedged(to_gt(i915)))
4502 return 0;
4503
4504 return intel_gt_live_subtests(tests, to_gt(i915));
4505 }
4506