1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include <linux/prime_numbers.h>
7 
8 #include "i915_selftest.h"
9 #include "intel_engine_heartbeat.h"
10 #include "intel_engine_pm.h"
11 #include "intel_reset.h"
12 #include "intel_ring.h"
13 #include "selftest_engine_heartbeat.h"
14 #include "selftests/i915_random.h"
15 #include "selftests/igt_flush_test.h"
16 #include "selftests/igt_live_test.h"
17 #include "selftests/igt_spinner.h"
18 #include "selftests/lib_sw_fence.h"
19 #include "shmem_utils.h"
20 
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
23 
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR 16
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27 
28 static struct i915_vma *create_scratch(struct intel_gt *gt)
29 {
30 	return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
31 }
32 
33 static bool is_active(struct i915_request *rq)
34 {
35 	if (i915_request_is_active(rq))
36 		return true;
37 
38 	if (i915_request_on_hold(rq))
39 		return true;
40 
41 	if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
42 		return true;
43 
44 	return false;
45 }
46 
47 static int wait_for_submit(struct intel_engine_cs *engine,
48 			   struct i915_request *rq,
49 			   unsigned long timeout)
50 {
51 	/* Ignore our own attempts to suppress excess tasklets */
52 	tasklet_hi_schedule(&engine->execlists.tasklet);
53 
54 	timeout += jiffies;
55 	do {
56 		bool done = time_after(jiffies, timeout);
57 
58 		if (i915_request_completed(rq)) /* that was quick! */
59 			return 0;
60 
61 		/* Wait until the HW has acknowleged the submission (or err) */
62 		intel_engine_flush_submission(engine);
63 		if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
64 			return 0;
65 
66 		if (done)
67 			return -ETIME;
68 
69 		cond_resched();
70 	} while (1);
71 }
72 
73 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
74 {
75 	const u32 offset =
76 		i915_ggtt_offset(ce->engine->status_page.vma) +
77 		offset_in_page(slot);
78 	struct i915_request *rq;
79 	u32 *cs;
80 
81 	rq = intel_context_create_request(ce);
82 	if (IS_ERR(rq))
83 		return PTR_ERR(rq);
84 
85 	cs = intel_ring_begin(rq, 4);
86 	if (IS_ERR(cs)) {
87 		i915_request_add(rq);
88 		return PTR_ERR(cs);
89 	}
90 
91 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
92 	*cs++ = offset;
93 	*cs++ = 0;
94 	*cs++ = 1;
95 
96 	intel_ring_advance(rq, cs);
97 
98 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
99 	i915_request_add(rq);
100 	return 0;
101 }
102 
103 static int context_flush(struct intel_context *ce, long timeout)
104 {
105 	struct i915_request *rq;
106 	struct dma_fence *fence;
107 	int err = 0;
108 
109 	rq = intel_engine_create_kernel_request(ce->engine);
110 	if (IS_ERR(rq))
111 		return PTR_ERR(rq);
112 
113 	fence = i915_active_fence_get(&ce->timeline->last_request);
114 	if (fence) {
115 		i915_request_await_dma_fence(rq, fence);
116 		dma_fence_put(fence);
117 	}
118 
119 	rq = i915_request_get(rq);
120 	i915_request_add(rq);
121 	if (i915_request_wait(rq, 0, timeout) < 0)
122 		err = -ETIME;
123 	i915_request_put(rq);
124 
125 	rmb(); /* We know the request is written, make sure all state is too! */
126 	return err;
127 }
128 
129 static int live_lrc_layout(void *arg)
130 {
131 	struct intel_gt *gt = arg;
132 	struct intel_engine_cs *engine;
133 	enum intel_engine_id id;
134 	u32 *lrc;
135 	int err;
136 
137 	/*
138 	 * Check the registers offsets we use to create the initial reg state
139 	 * match the layout saved by HW.
140 	 */
141 
142 	lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
143 	if (!lrc)
144 		return -ENOMEM;
145 	GEM_BUG_ON(offset_in_page(lrc));
146 
147 	err = 0;
148 	for_each_engine(engine, gt, id) {
149 		u32 *hw;
150 		int dw;
151 
152 		if (!engine->default_state)
153 			continue;
154 
155 		hw = shmem_pin_map(engine->default_state);
156 		if (IS_ERR(hw)) {
157 			err = PTR_ERR(hw);
158 			break;
159 		}
160 		hw += LRC_STATE_OFFSET / sizeof(*hw);
161 
162 		__lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
163 				engine->kernel_context, engine, true);
164 
165 		dw = 0;
166 		do {
167 			u32 lri = READ_ONCE(hw[dw]);
168 
169 			if (lri == 0) {
170 				dw++;
171 				continue;
172 			}
173 
174 			if (lrc[dw] == 0) {
175 				pr_debug("%s: skipped instruction %x at dword %d\n",
176 					 engine->name, lri, dw);
177 				dw++;
178 				continue;
179 			}
180 
181 			if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
182 				pr_err("%s: Expected LRI command at dword %d, found %08x\n",
183 				       engine->name, dw, lri);
184 				err = -EINVAL;
185 				break;
186 			}
187 
188 			if (lrc[dw] != lri) {
189 				pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
190 				       engine->name, dw, lri, lrc[dw]);
191 				err = -EINVAL;
192 				break;
193 			}
194 
195 			lri &= 0x7f;
196 			lri++;
197 			dw++;
198 
199 			while (lri) {
200 				u32 offset = READ_ONCE(hw[dw]);
201 
202 				if (offset != lrc[dw]) {
203 					pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
204 					       engine->name, dw, offset, lrc[dw]);
205 					err = -EINVAL;
206 					break;
207 				}
208 
209 				/*
210 				 * Skip over the actual register value as we
211 				 * expect that to differ.
212 				 */
213 				dw += 2;
214 				lri -= 2;
215 			}
216 		} while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
217 
218 		if (err) {
219 			pr_info("%s: HW register image:\n", engine->name);
220 			igt_hexdump(hw, PAGE_SIZE);
221 
222 			pr_info("%s: SW register image:\n", engine->name);
223 			igt_hexdump(lrc, PAGE_SIZE);
224 		}
225 
226 		shmem_unpin_map(engine->default_state, hw);
227 		if (err)
228 			break;
229 	}
230 
231 	free_page((unsigned long)lrc);
232 	return err;
233 }
234 
235 static int find_offset(const u32 *lri, u32 offset)
236 {
237 	int i;
238 
239 	for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
240 		if (lri[i] == offset)
241 			return i;
242 
243 	return -1;
244 }
245 
246 static int live_lrc_fixed(void *arg)
247 {
248 	struct intel_gt *gt = arg;
249 	struct intel_engine_cs *engine;
250 	enum intel_engine_id id;
251 	int err = 0;
252 
253 	/*
254 	 * Check the assumed register offsets match the actual locations in
255 	 * the context image.
256 	 */
257 
258 	for_each_engine(engine, gt, id) {
259 		const struct {
260 			u32 reg;
261 			u32 offset;
262 			const char *name;
263 		} tbl[] = {
264 			{
265 				i915_mmio_reg_offset(RING_START(engine->mmio_base)),
266 				CTX_RING_START - 1,
267 				"RING_START"
268 			},
269 			{
270 				i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
271 				CTX_RING_CTL - 1,
272 				"RING_CTL"
273 			},
274 			{
275 				i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
276 				CTX_RING_HEAD - 1,
277 				"RING_HEAD"
278 			},
279 			{
280 				i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
281 				CTX_RING_TAIL - 1,
282 				"RING_TAIL"
283 			},
284 			{
285 				i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
286 				lrc_ring_mi_mode(engine),
287 				"RING_MI_MODE"
288 			},
289 			{
290 				i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
291 				CTX_BB_STATE - 1,
292 				"BB_STATE"
293 			},
294 			{
295 				i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
296 				lrc_ring_wa_bb_per_ctx(engine),
297 				"RING_BB_PER_CTX_PTR"
298 			},
299 			{
300 				i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
301 				lrc_ring_indirect_ptr(engine),
302 				"RING_INDIRECT_CTX_PTR"
303 			},
304 			{
305 				i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
306 				lrc_ring_indirect_offset(engine),
307 				"RING_INDIRECT_CTX_OFFSET"
308 			},
309 			{
310 				i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
311 				CTX_TIMESTAMP - 1,
312 				"RING_CTX_TIMESTAMP"
313 			},
314 			{
315 				i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
316 				lrc_ring_gpr0(engine),
317 				"RING_CS_GPR0"
318 			},
319 			{
320 				i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
321 				lrc_ring_cmd_buf_cctl(engine),
322 				"RING_CMD_BUF_CCTL"
323 			},
324 			{ },
325 		}, *t;
326 		u32 *hw;
327 
328 		if (!engine->default_state)
329 			continue;
330 
331 		hw = shmem_pin_map(engine->default_state);
332 		if (IS_ERR(hw)) {
333 			err = PTR_ERR(hw);
334 			break;
335 		}
336 		hw += LRC_STATE_OFFSET / sizeof(*hw);
337 
338 		for (t = tbl; t->name; t++) {
339 			int dw = find_offset(hw, t->reg);
340 
341 			if (dw != t->offset) {
342 				pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
343 				       engine->name,
344 				       t->name,
345 				       t->reg,
346 				       dw,
347 				       t->offset);
348 				err = -EINVAL;
349 			}
350 		}
351 
352 		shmem_unpin_map(engine->default_state, hw);
353 	}
354 
355 	return err;
356 }
357 
358 static int __live_lrc_state(struct intel_engine_cs *engine,
359 			    struct i915_vma *scratch)
360 {
361 	struct intel_context *ce;
362 	struct i915_request *rq;
363 	struct i915_gem_ww_ctx ww;
364 	enum {
365 		RING_START_IDX = 0,
366 		RING_TAIL_IDX,
367 		MAX_IDX
368 	};
369 	u32 expected[MAX_IDX];
370 	u32 *cs;
371 	int err;
372 	int n;
373 
374 	ce = intel_context_create(engine);
375 	if (IS_ERR(ce))
376 		return PTR_ERR(ce);
377 
378 	i915_gem_ww_ctx_init(&ww, false);
379 retry:
380 	err = i915_gem_object_lock(scratch->obj, &ww);
381 	if (!err)
382 		err = intel_context_pin_ww(ce, &ww);
383 	if (err)
384 		goto err_put;
385 
386 	rq = i915_request_create(ce);
387 	if (IS_ERR(rq)) {
388 		err = PTR_ERR(rq);
389 		goto err_unpin;
390 	}
391 
392 	cs = intel_ring_begin(rq, 4 * MAX_IDX);
393 	if (IS_ERR(cs)) {
394 		err = PTR_ERR(cs);
395 		i915_request_add(rq);
396 		goto err_unpin;
397 	}
398 
399 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
400 	*cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
401 	*cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
402 	*cs++ = 0;
403 
404 	expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
405 
406 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
407 	*cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
408 	*cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
409 	*cs++ = 0;
410 
411 	err = i915_request_await_object(rq, scratch->obj, true);
412 	if (!err)
413 		err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
414 
415 	i915_request_get(rq);
416 	i915_request_add(rq);
417 	if (err)
418 		goto err_rq;
419 
420 	intel_engine_flush_submission(engine);
421 	expected[RING_TAIL_IDX] = ce->ring->tail;
422 
423 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
424 		err = -ETIME;
425 		goto err_rq;
426 	}
427 
428 	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
429 	if (IS_ERR(cs)) {
430 		err = PTR_ERR(cs);
431 		goto err_rq;
432 	}
433 
434 	for (n = 0; n < MAX_IDX; n++) {
435 		if (cs[n] != expected[n]) {
436 			pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
437 			       engine->name, n, cs[n], expected[n]);
438 			err = -EINVAL;
439 			break;
440 		}
441 	}
442 
443 	i915_gem_object_unpin_map(scratch->obj);
444 
445 err_rq:
446 	i915_request_put(rq);
447 err_unpin:
448 	intel_context_unpin(ce);
449 err_put:
450 	if (err == -EDEADLK) {
451 		err = i915_gem_ww_ctx_backoff(&ww);
452 		if (!err)
453 			goto retry;
454 	}
455 	i915_gem_ww_ctx_fini(&ww);
456 	intel_context_put(ce);
457 	return err;
458 }
459 
460 static int live_lrc_state(void *arg)
461 {
462 	struct intel_gt *gt = arg;
463 	struct intel_engine_cs *engine;
464 	struct i915_vma *scratch;
465 	enum intel_engine_id id;
466 	int err = 0;
467 
468 	/*
469 	 * Check the live register state matches what we expect for this
470 	 * intel_context.
471 	 */
472 
473 	scratch = create_scratch(gt);
474 	if (IS_ERR(scratch))
475 		return PTR_ERR(scratch);
476 
477 	for_each_engine(engine, gt, id) {
478 		err = __live_lrc_state(engine, scratch);
479 		if (err)
480 			break;
481 	}
482 
483 	if (igt_flush_test(gt->i915))
484 		err = -EIO;
485 
486 	i915_vma_unpin_and_release(&scratch, 0);
487 	return err;
488 }
489 
490 static int gpr_make_dirty(struct intel_context *ce)
491 {
492 	struct i915_request *rq;
493 	u32 *cs;
494 	int n;
495 
496 	rq = intel_context_create_request(ce);
497 	if (IS_ERR(rq))
498 		return PTR_ERR(rq);
499 
500 	cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
501 	if (IS_ERR(cs)) {
502 		i915_request_add(rq);
503 		return PTR_ERR(cs);
504 	}
505 
506 	*cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
507 	for (n = 0; n < NUM_GPR_DW; n++) {
508 		*cs++ = CS_GPR(ce->engine, n);
509 		*cs++ = STACK_MAGIC;
510 	}
511 	*cs++ = MI_NOOP;
512 
513 	intel_ring_advance(rq, cs);
514 
515 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
516 	i915_request_add(rq);
517 
518 	return 0;
519 }
520 
521 static struct i915_request *
522 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
523 {
524 	const u32 offset =
525 		i915_ggtt_offset(ce->engine->status_page.vma) +
526 		offset_in_page(slot);
527 	struct i915_request *rq;
528 	u32 *cs;
529 	int err;
530 	int n;
531 
532 	rq = intel_context_create_request(ce);
533 	if (IS_ERR(rq))
534 		return rq;
535 
536 	cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
537 	if (IS_ERR(cs)) {
538 		i915_request_add(rq);
539 		return ERR_CAST(cs);
540 	}
541 
542 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
543 	*cs++ = MI_NOOP;
544 
545 	*cs++ = MI_SEMAPHORE_WAIT |
546 		MI_SEMAPHORE_GLOBAL_GTT |
547 		MI_SEMAPHORE_POLL |
548 		MI_SEMAPHORE_SAD_NEQ_SDD;
549 	*cs++ = 0;
550 	*cs++ = offset;
551 	*cs++ = 0;
552 
553 	for (n = 0; n < NUM_GPR_DW; n++) {
554 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
555 		*cs++ = CS_GPR(ce->engine, n);
556 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
557 		*cs++ = 0;
558 	}
559 
560 	i915_vma_lock(scratch);
561 	err = i915_request_await_object(rq, scratch->obj, true);
562 	if (!err)
563 		err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
564 	i915_vma_unlock(scratch);
565 
566 	i915_request_get(rq);
567 	i915_request_add(rq);
568 	if (err) {
569 		i915_request_put(rq);
570 		rq = ERR_PTR(err);
571 	}
572 
573 	return rq;
574 }
575 
576 static int __live_lrc_gpr(struct intel_engine_cs *engine,
577 			  struct i915_vma *scratch,
578 			  bool preempt)
579 {
580 	u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
581 	struct intel_context *ce;
582 	struct i915_request *rq;
583 	u32 *cs;
584 	int err;
585 	int n;
586 
587 	if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
588 		return 0; /* GPR only on rcs0 for gen8 */
589 
590 	err = gpr_make_dirty(engine->kernel_context);
591 	if (err)
592 		return err;
593 
594 	ce = intel_context_create(engine);
595 	if (IS_ERR(ce))
596 		return PTR_ERR(ce);
597 
598 	rq = __gpr_read(ce, scratch, slot);
599 	if (IS_ERR(rq)) {
600 		err = PTR_ERR(rq);
601 		goto err_put;
602 	}
603 
604 	err = wait_for_submit(engine, rq, HZ / 2);
605 	if (err)
606 		goto err_rq;
607 
608 	if (preempt) {
609 		err = gpr_make_dirty(engine->kernel_context);
610 		if (err)
611 			goto err_rq;
612 
613 		err = emit_semaphore_signal(engine->kernel_context, slot);
614 		if (err)
615 			goto err_rq;
616 
617 		err = wait_for_submit(engine, rq, HZ / 2);
618 		if (err)
619 			goto err_rq;
620 	} else {
621 		slot[0] = 1;
622 		wmb();
623 	}
624 
625 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
626 		err = -ETIME;
627 		goto err_rq;
628 	}
629 
630 	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
631 	if (IS_ERR(cs)) {
632 		err = PTR_ERR(cs);
633 		goto err_rq;
634 	}
635 
636 	for (n = 0; n < NUM_GPR_DW; n++) {
637 		if (cs[n]) {
638 			pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
639 			       engine->name,
640 			       n / 2, n & 1 ? "udw" : "ldw",
641 			       cs[n]);
642 			err = -EINVAL;
643 			break;
644 		}
645 	}
646 
647 	i915_gem_object_unpin_map(scratch->obj);
648 
649 err_rq:
650 	memset32(&slot[0], -1, 4);
651 	wmb();
652 	i915_request_put(rq);
653 err_put:
654 	intel_context_put(ce);
655 	return err;
656 }
657 
658 static int live_lrc_gpr(void *arg)
659 {
660 	struct intel_gt *gt = arg;
661 	struct intel_engine_cs *engine;
662 	struct i915_vma *scratch;
663 	enum intel_engine_id id;
664 	int err = 0;
665 
666 	/*
667 	 * Check that GPR registers are cleared in new contexts as we need
668 	 * to avoid leaking any information from previous contexts.
669 	 */
670 
671 	scratch = create_scratch(gt);
672 	if (IS_ERR(scratch))
673 		return PTR_ERR(scratch);
674 
675 	for_each_engine(engine, gt, id) {
676 		st_engine_heartbeat_disable(engine);
677 
678 		err = __live_lrc_gpr(engine, scratch, false);
679 		if (err)
680 			goto err;
681 
682 		err = __live_lrc_gpr(engine, scratch, true);
683 		if (err)
684 			goto err;
685 
686 err:
687 		st_engine_heartbeat_enable(engine);
688 		if (igt_flush_test(gt->i915))
689 			err = -EIO;
690 		if (err)
691 			break;
692 	}
693 
694 	i915_vma_unpin_and_release(&scratch, 0);
695 	return err;
696 }
697 
698 static struct i915_request *
699 create_timestamp(struct intel_context *ce, void *slot, int idx)
700 {
701 	const u32 offset =
702 		i915_ggtt_offset(ce->engine->status_page.vma) +
703 		offset_in_page(slot);
704 	struct i915_request *rq;
705 	u32 *cs;
706 	int err;
707 
708 	rq = intel_context_create_request(ce);
709 	if (IS_ERR(rq))
710 		return rq;
711 
712 	cs = intel_ring_begin(rq, 10);
713 	if (IS_ERR(cs)) {
714 		err = PTR_ERR(cs);
715 		goto err;
716 	}
717 
718 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
719 	*cs++ = MI_NOOP;
720 
721 	*cs++ = MI_SEMAPHORE_WAIT |
722 		MI_SEMAPHORE_GLOBAL_GTT |
723 		MI_SEMAPHORE_POLL |
724 		MI_SEMAPHORE_SAD_NEQ_SDD;
725 	*cs++ = 0;
726 	*cs++ = offset;
727 	*cs++ = 0;
728 
729 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
730 	*cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
731 	*cs++ = offset + idx * sizeof(u32);
732 	*cs++ = 0;
733 
734 	intel_ring_advance(rq, cs);
735 
736 	err = 0;
737 err:
738 	i915_request_get(rq);
739 	i915_request_add(rq);
740 	if (err) {
741 		i915_request_put(rq);
742 		return ERR_PTR(err);
743 	}
744 
745 	return rq;
746 }
747 
748 struct lrc_timestamp {
749 	struct intel_engine_cs *engine;
750 	struct intel_context *ce[2];
751 	u32 poison;
752 };
753 
754 static bool timestamp_advanced(u32 start, u32 end)
755 {
756 	return (s32)(end - start) > 0;
757 }
758 
759 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
760 {
761 	u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
762 	struct i915_request *rq;
763 	u32 timestamp;
764 	int err = 0;
765 
766 	arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
767 	rq = create_timestamp(arg->ce[0], slot, 1);
768 	if (IS_ERR(rq))
769 		return PTR_ERR(rq);
770 
771 	err = wait_for_submit(rq->engine, rq, HZ / 2);
772 	if (err)
773 		goto err;
774 
775 	if (preempt) {
776 		arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
777 		err = emit_semaphore_signal(arg->ce[1], slot);
778 		if (err)
779 			goto err;
780 	} else {
781 		slot[0] = 1;
782 		wmb();
783 	}
784 
785 	/* And wait for switch to kernel (to save our context to memory) */
786 	err = context_flush(arg->ce[0], HZ / 2);
787 	if (err)
788 		goto err;
789 
790 	if (!timestamp_advanced(arg->poison, slot[1])) {
791 		pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
792 		       arg->engine->name, preempt ? "preempt" : "simple",
793 		       arg->poison, slot[1]);
794 		err = -EINVAL;
795 	}
796 
797 	timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
798 	if (!timestamp_advanced(slot[1], timestamp)) {
799 		pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
800 		       arg->engine->name, preempt ? "preempt" : "simple",
801 		       slot[1], timestamp);
802 		err = -EINVAL;
803 	}
804 
805 err:
806 	memset32(slot, -1, 4);
807 	i915_request_put(rq);
808 	return err;
809 }
810 
811 static int live_lrc_timestamp(void *arg)
812 {
813 	struct lrc_timestamp data = {};
814 	struct intel_gt *gt = arg;
815 	enum intel_engine_id id;
816 	const u32 poison[] = {
817 		0,
818 		S32_MAX,
819 		(u32)S32_MAX + 1,
820 		U32_MAX,
821 	};
822 
823 	/*
824 	 * We want to verify that the timestamp is saved and restore across
825 	 * context switches and is monotonic.
826 	 *
827 	 * So we do this with a little bit of LRC poisoning to check various
828 	 * boundary conditions, and see what happens if we preempt the context
829 	 * with a second request (carrying more poison into the timestamp).
830 	 */
831 
832 	for_each_engine(data.engine, gt, id) {
833 		int i, err = 0;
834 
835 		st_engine_heartbeat_disable(data.engine);
836 
837 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
838 			struct intel_context *tmp;
839 
840 			tmp = intel_context_create(data.engine);
841 			if (IS_ERR(tmp)) {
842 				err = PTR_ERR(tmp);
843 				goto err;
844 			}
845 
846 			err = intel_context_pin(tmp);
847 			if (err) {
848 				intel_context_put(tmp);
849 				goto err;
850 			}
851 
852 			data.ce[i] = tmp;
853 		}
854 
855 		for (i = 0; i < ARRAY_SIZE(poison); i++) {
856 			data.poison = poison[i];
857 
858 			err = __lrc_timestamp(&data, false);
859 			if (err)
860 				break;
861 
862 			err = __lrc_timestamp(&data, true);
863 			if (err)
864 				break;
865 		}
866 
867 err:
868 		st_engine_heartbeat_enable(data.engine);
869 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
870 			if (!data.ce[i])
871 				break;
872 
873 			intel_context_unpin(data.ce[i]);
874 			intel_context_put(data.ce[i]);
875 		}
876 
877 		if (igt_flush_test(gt->i915))
878 			err = -EIO;
879 		if (err)
880 			return err;
881 	}
882 
883 	return 0;
884 }
885 
886 static struct i915_vma *
887 create_user_vma(struct i915_address_space *vm, unsigned long size)
888 {
889 	struct drm_i915_gem_object *obj;
890 	struct i915_vma *vma;
891 	int err;
892 
893 	obj = i915_gem_object_create_internal(vm->i915, size);
894 	if (IS_ERR(obj))
895 		return ERR_CAST(obj);
896 
897 	vma = i915_vma_instance(obj, vm, NULL);
898 	if (IS_ERR(vma)) {
899 		i915_gem_object_put(obj);
900 		return vma;
901 	}
902 
903 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
904 	if (err) {
905 		i915_gem_object_put(obj);
906 		return ERR_PTR(err);
907 	}
908 
909 	return vma;
910 }
911 
912 static struct i915_vma *
913 store_context(struct intel_context *ce, struct i915_vma *scratch)
914 {
915 	struct i915_vma *batch;
916 	u32 dw, x, *cs, *hw;
917 	u32 *defaults;
918 
919 	batch = create_user_vma(ce->vm, SZ_64K);
920 	if (IS_ERR(batch))
921 		return batch;
922 
923 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
924 	if (IS_ERR(cs)) {
925 		i915_vma_put(batch);
926 		return ERR_CAST(cs);
927 	}
928 
929 	defaults = shmem_pin_map(ce->engine->default_state);
930 	if (!defaults) {
931 		i915_gem_object_unpin_map(batch->obj);
932 		i915_vma_put(batch);
933 		return ERR_PTR(-ENOMEM);
934 	}
935 
936 	x = 0;
937 	dw = 0;
938 	hw = defaults;
939 	hw += LRC_STATE_OFFSET / sizeof(*hw);
940 	do {
941 		u32 len = hw[dw] & 0x7f;
942 
943 		if (hw[dw] == 0) {
944 			dw++;
945 			continue;
946 		}
947 
948 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
949 			dw += len + 2;
950 			continue;
951 		}
952 
953 		dw++;
954 		len = (len + 1) / 2;
955 		while (len--) {
956 			*cs++ = MI_STORE_REGISTER_MEM_GEN8;
957 			*cs++ = hw[dw];
958 			*cs++ = lower_32_bits(scratch->node.start + x);
959 			*cs++ = upper_32_bits(scratch->node.start + x);
960 
961 			dw += 2;
962 			x += 4;
963 		}
964 	} while (dw < PAGE_SIZE / sizeof(u32) &&
965 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
966 
967 	*cs++ = MI_BATCH_BUFFER_END;
968 
969 	shmem_unpin_map(ce->engine->default_state, defaults);
970 
971 	i915_gem_object_flush_map(batch->obj);
972 	i915_gem_object_unpin_map(batch->obj);
973 
974 	return batch;
975 }
976 
977 static int move_to_active(struct i915_request *rq,
978 			  struct i915_vma *vma,
979 			  unsigned int flags)
980 {
981 	int err;
982 
983 	i915_vma_lock(vma);
984 	err = i915_request_await_object(rq, vma->obj, flags);
985 	if (!err)
986 		err = i915_vma_move_to_active(vma, rq, flags);
987 	i915_vma_unlock(vma);
988 
989 	return err;
990 }
991 
992 static struct i915_request *
993 record_registers(struct intel_context *ce,
994 		 struct i915_vma *before,
995 		 struct i915_vma *after,
996 		 u32 *sema)
997 {
998 	struct i915_vma *b_before, *b_after;
999 	struct i915_request *rq;
1000 	u32 *cs;
1001 	int err;
1002 
1003 	b_before = store_context(ce, before);
1004 	if (IS_ERR(b_before))
1005 		return ERR_CAST(b_before);
1006 
1007 	b_after = store_context(ce, after);
1008 	if (IS_ERR(b_after)) {
1009 		rq = ERR_CAST(b_after);
1010 		goto err_before;
1011 	}
1012 
1013 	rq = intel_context_create_request(ce);
1014 	if (IS_ERR(rq))
1015 		goto err_after;
1016 
1017 	err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1018 	if (err)
1019 		goto err_rq;
1020 
1021 	err = move_to_active(rq, b_before, 0);
1022 	if (err)
1023 		goto err_rq;
1024 
1025 	err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1026 	if (err)
1027 		goto err_rq;
1028 
1029 	err = move_to_active(rq, b_after, 0);
1030 	if (err)
1031 		goto err_rq;
1032 
1033 	cs = intel_ring_begin(rq, 14);
1034 	if (IS_ERR(cs)) {
1035 		err = PTR_ERR(cs);
1036 		goto err_rq;
1037 	}
1038 
1039 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1040 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1041 	*cs++ = lower_32_bits(b_before->node.start);
1042 	*cs++ = upper_32_bits(b_before->node.start);
1043 
1044 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1045 	*cs++ = MI_SEMAPHORE_WAIT |
1046 		MI_SEMAPHORE_GLOBAL_GTT |
1047 		MI_SEMAPHORE_POLL |
1048 		MI_SEMAPHORE_SAD_NEQ_SDD;
1049 	*cs++ = 0;
1050 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1051 		offset_in_page(sema);
1052 	*cs++ = 0;
1053 	*cs++ = MI_NOOP;
1054 
1055 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1056 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1057 	*cs++ = lower_32_bits(b_after->node.start);
1058 	*cs++ = upper_32_bits(b_after->node.start);
1059 
1060 	intel_ring_advance(rq, cs);
1061 
1062 	WRITE_ONCE(*sema, 0);
1063 	i915_request_get(rq);
1064 	i915_request_add(rq);
1065 err_after:
1066 	i915_vma_put(b_after);
1067 err_before:
1068 	i915_vma_put(b_before);
1069 	return rq;
1070 
1071 err_rq:
1072 	i915_request_add(rq);
1073 	rq = ERR_PTR(err);
1074 	goto err_after;
1075 }
1076 
1077 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1078 {
1079 	struct i915_vma *batch;
1080 	u32 dw, *cs, *hw;
1081 	u32 *defaults;
1082 
1083 	batch = create_user_vma(ce->vm, SZ_64K);
1084 	if (IS_ERR(batch))
1085 		return batch;
1086 
1087 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1088 	if (IS_ERR(cs)) {
1089 		i915_vma_put(batch);
1090 		return ERR_CAST(cs);
1091 	}
1092 
1093 	defaults = shmem_pin_map(ce->engine->default_state);
1094 	if (!defaults) {
1095 		i915_gem_object_unpin_map(batch->obj);
1096 		i915_vma_put(batch);
1097 		return ERR_PTR(-ENOMEM);
1098 	}
1099 
1100 	dw = 0;
1101 	hw = defaults;
1102 	hw += LRC_STATE_OFFSET / sizeof(*hw);
1103 	do {
1104 		u32 len = hw[dw] & 0x7f;
1105 
1106 		if (hw[dw] == 0) {
1107 			dw++;
1108 			continue;
1109 		}
1110 
1111 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1112 			dw += len + 2;
1113 			continue;
1114 		}
1115 
1116 		dw++;
1117 		len = (len + 1) / 2;
1118 		*cs++ = MI_LOAD_REGISTER_IMM(len);
1119 		while (len--) {
1120 			*cs++ = hw[dw];
1121 			*cs++ = poison;
1122 			dw += 2;
1123 		}
1124 	} while (dw < PAGE_SIZE / sizeof(u32) &&
1125 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1126 
1127 	*cs++ = MI_BATCH_BUFFER_END;
1128 
1129 	shmem_unpin_map(ce->engine->default_state, defaults);
1130 
1131 	i915_gem_object_flush_map(batch->obj);
1132 	i915_gem_object_unpin_map(batch->obj);
1133 
1134 	return batch;
1135 }
1136 
1137 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1138 {
1139 	struct i915_request *rq;
1140 	struct i915_vma *batch;
1141 	u32 *cs;
1142 	int err;
1143 
1144 	batch = load_context(ce, poison);
1145 	if (IS_ERR(batch))
1146 		return PTR_ERR(batch);
1147 
1148 	rq = intel_context_create_request(ce);
1149 	if (IS_ERR(rq)) {
1150 		err = PTR_ERR(rq);
1151 		goto err_batch;
1152 	}
1153 
1154 	err = move_to_active(rq, batch, 0);
1155 	if (err)
1156 		goto err_rq;
1157 
1158 	cs = intel_ring_begin(rq, 8);
1159 	if (IS_ERR(cs)) {
1160 		err = PTR_ERR(cs);
1161 		goto err_rq;
1162 	}
1163 
1164 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1165 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1166 	*cs++ = lower_32_bits(batch->node.start);
1167 	*cs++ = upper_32_bits(batch->node.start);
1168 
1169 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1170 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1171 		offset_in_page(sema);
1172 	*cs++ = 0;
1173 	*cs++ = 1;
1174 
1175 	intel_ring_advance(rq, cs);
1176 
1177 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1178 err_rq:
1179 	i915_request_add(rq);
1180 err_batch:
1181 	i915_vma_put(batch);
1182 	return err;
1183 }
1184 
1185 static bool is_moving(u32 a, u32 b)
1186 {
1187 	return a != b;
1188 }
1189 
1190 static int compare_isolation(struct intel_engine_cs *engine,
1191 			     struct i915_vma *ref[2],
1192 			     struct i915_vma *result[2],
1193 			     struct intel_context *ce,
1194 			     u32 poison)
1195 {
1196 	u32 x, dw, *hw, *lrc;
1197 	u32 *A[2], *B[2];
1198 	u32 *defaults;
1199 	int err = 0;
1200 
1201 	A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1202 	if (IS_ERR(A[0]))
1203 		return PTR_ERR(A[0]);
1204 
1205 	A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1206 	if (IS_ERR(A[1])) {
1207 		err = PTR_ERR(A[1]);
1208 		goto err_A0;
1209 	}
1210 
1211 	B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1212 	if (IS_ERR(B[0])) {
1213 		err = PTR_ERR(B[0]);
1214 		goto err_A1;
1215 	}
1216 
1217 	B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1218 	if (IS_ERR(B[1])) {
1219 		err = PTR_ERR(B[1]);
1220 		goto err_B0;
1221 	}
1222 
1223 	lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1224 				      i915_coherent_map_type(engine->i915));
1225 	if (IS_ERR(lrc)) {
1226 		err = PTR_ERR(lrc);
1227 		goto err_B1;
1228 	}
1229 	lrc += LRC_STATE_OFFSET / sizeof(*hw);
1230 
1231 	defaults = shmem_pin_map(ce->engine->default_state);
1232 	if (!defaults) {
1233 		err = -ENOMEM;
1234 		goto err_lrc;
1235 	}
1236 
1237 	x = 0;
1238 	dw = 0;
1239 	hw = defaults;
1240 	hw += LRC_STATE_OFFSET / sizeof(*hw);
1241 	do {
1242 		u32 len = hw[dw] & 0x7f;
1243 
1244 		if (hw[dw] == 0) {
1245 			dw++;
1246 			continue;
1247 		}
1248 
1249 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1250 			dw += len + 2;
1251 			continue;
1252 		}
1253 
1254 		dw++;
1255 		len = (len + 1) / 2;
1256 		while (len--) {
1257 			if (!is_moving(A[0][x], A[1][x]) &&
1258 			    (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1259 				switch (hw[dw] & 4095) {
1260 				case 0x30: /* RING_HEAD */
1261 				case 0x34: /* RING_TAIL */
1262 					break;
1263 
1264 				default:
1265 					pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1266 					       engine->name, dw,
1267 					       hw[dw], hw[dw + 1],
1268 					       A[0][x], B[0][x], B[1][x],
1269 					       poison, lrc[dw + 1]);
1270 					err = -EINVAL;
1271 				}
1272 			}
1273 			dw += 2;
1274 			x++;
1275 		}
1276 	} while (dw < PAGE_SIZE / sizeof(u32) &&
1277 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1278 
1279 	shmem_unpin_map(ce->engine->default_state, defaults);
1280 err_lrc:
1281 	i915_gem_object_unpin_map(ce->state->obj);
1282 err_B1:
1283 	i915_gem_object_unpin_map(result[1]->obj);
1284 err_B0:
1285 	i915_gem_object_unpin_map(result[0]->obj);
1286 err_A1:
1287 	i915_gem_object_unpin_map(ref[1]->obj);
1288 err_A0:
1289 	i915_gem_object_unpin_map(ref[0]->obj);
1290 	return err;
1291 }
1292 
1293 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1294 {
1295 	u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1296 	struct i915_vma *ref[2], *result[2];
1297 	struct intel_context *A, *B;
1298 	struct i915_request *rq;
1299 	int err;
1300 
1301 	A = intel_context_create(engine);
1302 	if (IS_ERR(A))
1303 		return PTR_ERR(A);
1304 
1305 	B = intel_context_create(engine);
1306 	if (IS_ERR(B)) {
1307 		err = PTR_ERR(B);
1308 		goto err_A;
1309 	}
1310 
1311 	ref[0] = create_user_vma(A->vm, SZ_64K);
1312 	if (IS_ERR(ref[0])) {
1313 		err = PTR_ERR(ref[0]);
1314 		goto err_B;
1315 	}
1316 
1317 	ref[1] = create_user_vma(A->vm, SZ_64K);
1318 	if (IS_ERR(ref[1])) {
1319 		err = PTR_ERR(ref[1]);
1320 		goto err_ref0;
1321 	}
1322 
1323 	rq = record_registers(A, ref[0], ref[1], sema);
1324 	if (IS_ERR(rq)) {
1325 		err = PTR_ERR(rq);
1326 		goto err_ref1;
1327 	}
1328 
1329 	WRITE_ONCE(*sema, 1);
1330 	wmb();
1331 
1332 	if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1333 		i915_request_put(rq);
1334 		err = -ETIME;
1335 		goto err_ref1;
1336 	}
1337 	i915_request_put(rq);
1338 
1339 	result[0] = create_user_vma(A->vm, SZ_64K);
1340 	if (IS_ERR(result[0])) {
1341 		err = PTR_ERR(result[0]);
1342 		goto err_ref1;
1343 	}
1344 
1345 	result[1] = create_user_vma(A->vm, SZ_64K);
1346 	if (IS_ERR(result[1])) {
1347 		err = PTR_ERR(result[1]);
1348 		goto err_result0;
1349 	}
1350 
1351 	rq = record_registers(A, result[0], result[1], sema);
1352 	if (IS_ERR(rq)) {
1353 		err = PTR_ERR(rq);
1354 		goto err_result1;
1355 	}
1356 
1357 	err = poison_registers(B, poison, sema);
1358 	if (err) {
1359 		WRITE_ONCE(*sema, -1);
1360 		i915_request_put(rq);
1361 		goto err_result1;
1362 	}
1363 
1364 	if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1365 		i915_request_put(rq);
1366 		err = -ETIME;
1367 		goto err_result1;
1368 	}
1369 	i915_request_put(rq);
1370 
1371 	err = compare_isolation(engine, ref, result, A, poison);
1372 
1373 err_result1:
1374 	i915_vma_put(result[1]);
1375 err_result0:
1376 	i915_vma_put(result[0]);
1377 err_ref1:
1378 	i915_vma_put(ref[1]);
1379 err_ref0:
1380 	i915_vma_put(ref[0]);
1381 err_B:
1382 	intel_context_put(B);
1383 err_A:
1384 	intel_context_put(A);
1385 	return err;
1386 }
1387 
1388 static bool skip_isolation(const struct intel_engine_cs *engine)
1389 {
1390 	if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9)
1391 		return true;
1392 
1393 	if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11)
1394 		return true;
1395 
1396 	return false;
1397 }
1398 
1399 static int live_lrc_isolation(void *arg)
1400 {
1401 	struct intel_gt *gt = arg;
1402 	struct intel_engine_cs *engine;
1403 	enum intel_engine_id id;
1404 	const u32 poison[] = {
1405 		STACK_MAGIC,
1406 		0x3a3a3a3a,
1407 		0x5c5c5c5c,
1408 		0xffffffff,
1409 		0xffff0000,
1410 	};
1411 	int err = 0;
1412 
1413 	/*
1414 	 * Our goal is try and verify that per-context state cannot be
1415 	 * tampered with by another non-privileged client.
1416 	 *
1417 	 * We take the list of context registers from the LRI in the default
1418 	 * context image and attempt to modify that list from a remote context.
1419 	 */
1420 
1421 	for_each_engine(engine, gt, id) {
1422 		int i;
1423 
1424 		/* Just don't even ask */
1425 		if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1426 		    skip_isolation(engine))
1427 			continue;
1428 
1429 		intel_engine_pm_get(engine);
1430 		for (i = 0; i < ARRAY_SIZE(poison); i++) {
1431 			int result;
1432 
1433 			result = __lrc_isolation(engine, poison[i]);
1434 			if (result && !err)
1435 				err = result;
1436 
1437 			result = __lrc_isolation(engine, ~poison[i]);
1438 			if (result && !err)
1439 				err = result;
1440 		}
1441 		intel_engine_pm_put(engine);
1442 		if (igt_flush_test(gt->i915)) {
1443 			err = -EIO;
1444 			break;
1445 		}
1446 	}
1447 
1448 	return err;
1449 }
1450 
1451 static int indirect_ctx_submit_req(struct intel_context *ce)
1452 {
1453 	struct i915_request *rq;
1454 	int err = 0;
1455 
1456 	rq = intel_context_create_request(ce);
1457 	if (IS_ERR(rq))
1458 		return PTR_ERR(rq);
1459 
1460 	i915_request_get(rq);
1461 	i915_request_add(rq);
1462 
1463 	if (i915_request_wait(rq, 0, HZ / 5) < 0)
1464 		err = -ETIME;
1465 
1466 	i915_request_put(rq);
1467 
1468 	return err;
1469 }
1470 
1471 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1472 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1473 
1474 static u32 *
1475 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1476 {
1477 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1478 		MI_SRM_LRM_GLOBAL_GTT |
1479 		MI_LRI_LRM_CS_MMIO;
1480 	*cs++ = i915_mmio_reg_offset(RING_START(0));
1481 	*cs++ = i915_ggtt_offset(ce->state) +
1482 		context_wa_bb_offset(ce) +
1483 		CTX_BB_CANARY_OFFSET;
1484 	*cs++ = 0;
1485 
1486 	return cs;
1487 }
1488 
1489 static void
1490 indirect_ctx_bb_setup(struct intel_context *ce)
1491 {
1492 	u32 *cs = context_indirect_bb(ce);
1493 
1494 	cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1495 
1496 	setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1497 }
1498 
1499 static bool check_ring_start(struct intel_context *ce)
1500 {
1501 	const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1502 		LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1503 
1504 	if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1505 		return true;
1506 
1507 	pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1508 	       ctx_bb[CTX_BB_CANARY_INDEX],
1509 	       ce->lrc_reg_state[CTX_RING_START]);
1510 
1511 	return false;
1512 }
1513 
1514 static int indirect_ctx_bb_check(struct intel_context *ce)
1515 {
1516 	int err;
1517 
1518 	err = indirect_ctx_submit_req(ce);
1519 	if (err)
1520 		return err;
1521 
1522 	if (!check_ring_start(ce))
1523 		return -EINVAL;
1524 
1525 	return 0;
1526 }
1527 
1528 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1529 {
1530 	struct intel_context *a, *b;
1531 	int err;
1532 
1533 	a = intel_context_create(engine);
1534 	if (IS_ERR(a))
1535 		return PTR_ERR(a);
1536 	err = intel_context_pin(a);
1537 	if (err)
1538 		goto put_a;
1539 
1540 	b = intel_context_create(engine);
1541 	if (IS_ERR(b)) {
1542 		err = PTR_ERR(b);
1543 		goto unpin_a;
1544 	}
1545 	err = intel_context_pin(b);
1546 	if (err)
1547 		goto put_b;
1548 
1549 	/* We use the already reserved extra page in context state */
1550 	if (!a->wa_bb_page) {
1551 		GEM_BUG_ON(b->wa_bb_page);
1552 		GEM_BUG_ON(INTEL_GEN(engine->i915) == 12);
1553 		goto unpin_b;
1554 	}
1555 
1556 	/*
1557 	 * In order to test that our per context bb is truly per context,
1558 	 * and executes at the intended spot on context restoring process,
1559 	 * make the batch store the ring start value to memory.
1560 	 * As ring start is restored apriori of starting the indirect ctx bb and
1561 	 * as it will be different for each context, it fits to this purpose.
1562 	 */
1563 	indirect_ctx_bb_setup(a);
1564 	indirect_ctx_bb_setup(b);
1565 
1566 	err = indirect_ctx_bb_check(a);
1567 	if (err)
1568 		goto unpin_b;
1569 
1570 	err = indirect_ctx_bb_check(b);
1571 
1572 unpin_b:
1573 	intel_context_unpin(b);
1574 put_b:
1575 	intel_context_put(b);
1576 unpin_a:
1577 	intel_context_unpin(a);
1578 put_a:
1579 	intel_context_put(a);
1580 
1581 	return err;
1582 }
1583 
1584 static int live_lrc_indirect_ctx_bb(void *arg)
1585 {
1586 	struct intel_gt *gt = arg;
1587 	struct intel_engine_cs *engine;
1588 	enum intel_engine_id id;
1589 	int err = 0;
1590 
1591 	for_each_engine(engine, gt, id) {
1592 		intel_engine_pm_get(engine);
1593 		err = __live_lrc_indirect_ctx_bb(engine);
1594 		intel_engine_pm_put(engine);
1595 
1596 		if (igt_flush_test(gt->i915))
1597 			err = -EIO;
1598 
1599 		if (err)
1600 			break;
1601 	}
1602 
1603 	return err;
1604 }
1605 
1606 static void garbage_reset(struct intel_engine_cs *engine,
1607 			  struct i915_request *rq)
1608 {
1609 	const unsigned int bit = I915_RESET_ENGINE + engine->id;
1610 	unsigned long *lock = &engine->gt->reset.flags;
1611 
1612 	local_bh_disable();
1613 	if (!test_and_set_bit(bit, lock)) {
1614 		tasklet_disable(&engine->execlists.tasklet);
1615 
1616 		if (!rq->fence.error)
1617 			__intel_engine_reset_bh(engine, NULL);
1618 
1619 		tasklet_enable(&engine->execlists.tasklet);
1620 		clear_and_wake_up_bit(bit, lock);
1621 	}
1622 	local_bh_enable();
1623 }
1624 
1625 static struct i915_request *garbage(struct intel_context *ce,
1626 				    struct rnd_state *prng)
1627 {
1628 	struct i915_request *rq;
1629 	int err;
1630 
1631 	err = intel_context_pin(ce);
1632 	if (err)
1633 		return ERR_PTR(err);
1634 
1635 	prandom_bytes_state(prng,
1636 			    ce->lrc_reg_state,
1637 			    ce->engine->context_size -
1638 			    LRC_STATE_OFFSET);
1639 
1640 	rq = intel_context_create_request(ce);
1641 	if (IS_ERR(rq)) {
1642 		err = PTR_ERR(rq);
1643 		goto err_unpin;
1644 	}
1645 
1646 	i915_request_get(rq);
1647 	i915_request_add(rq);
1648 	return rq;
1649 
1650 err_unpin:
1651 	intel_context_unpin(ce);
1652 	return ERR_PTR(err);
1653 }
1654 
1655 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1656 {
1657 	struct intel_context *ce;
1658 	struct i915_request *hang;
1659 	int err = 0;
1660 
1661 	ce = intel_context_create(engine);
1662 	if (IS_ERR(ce))
1663 		return PTR_ERR(ce);
1664 
1665 	hang = garbage(ce, prng);
1666 	if (IS_ERR(hang)) {
1667 		err = PTR_ERR(hang);
1668 		goto err_ce;
1669 	}
1670 
1671 	if (wait_for_submit(engine, hang, HZ / 2)) {
1672 		i915_request_put(hang);
1673 		err = -ETIME;
1674 		goto err_ce;
1675 	}
1676 
1677 	intel_context_set_banned(ce);
1678 	garbage_reset(engine, hang);
1679 
1680 	intel_engine_flush_submission(engine);
1681 	if (!hang->fence.error) {
1682 		i915_request_put(hang);
1683 		pr_err("%s: corrupted context was not reset\n",
1684 		       engine->name);
1685 		err = -EINVAL;
1686 		goto err_ce;
1687 	}
1688 
1689 	if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1690 		pr_err("%s: corrupted context did not recover\n",
1691 		       engine->name);
1692 		i915_request_put(hang);
1693 		err = -EIO;
1694 		goto err_ce;
1695 	}
1696 	i915_request_put(hang);
1697 
1698 err_ce:
1699 	intel_context_put(ce);
1700 	return err;
1701 }
1702 
1703 static int live_lrc_garbage(void *arg)
1704 {
1705 	struct intel_gt *gt = arg;
1706 	struct intel_engine_cs *engine;
1707 	enum intel_engine_id id;
1708 
1709 	/*
1710 	 * Verify that we can recover if one context state is completely
1711 	 * corrupted.
1712 	 */
1713 
1714 	if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1715 		return 0;
1716 
1717 	for_each_engine(engine, gt, id) {
1718 		I915_RND_STATE(prng);
1719 		int err = 0, i;
1720 
1721 		if (!intel_has_reset_engine(engine->gt))
1722 			continue;
1723 
1724 		intel_engine_pm_get(engine);
1725 		for (i = 0; i < 3; i++) {
1726 			err = __lrc_garbage(engine, &prng);
1727 			if (err)
1728 				break;
1729 		}
1730 		intel_engine_pm_put(engine);
1731 
1732 		if (igt_flush_test(gt->i915))
1733 			err = -EIO;
1734 		if (err)
1735 			return err;
1736 	}
1737 
1738 	return 0;
1739 }
1740 
1741 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1742 {
1743 	struct intel_context *ce;
1744 	struct i915_request *rq;
1745 	IGT_TIMEOUT(end_time);
1746 	int err;
1747 
1748 	ce = intel_context_create(engine);
1749 	if (IS_ERR(ce))
1750 		return PTR_ERR(ce);
1751 
1752 	ce->runtime.num_underflow = 0;
1753 	ce->runtime.max_underflow = 0;
1754 
1755 	do {
1756 		unsigned int loop = 1024;
1757 
1758 		while (loop) {
1759 			rq = intel_context_create_request(ce);
1760 			if (IS_ERR(rq)) {
1761 				err = PTR_ERR(rq);
1762 				goto err_rq;
1763 			}
1764 
1765 			if (--loop == 0)
1766 				i915_request_get(rq);
1767 
1768 			i915_request_add(rq);
1769 		}
1770 
1771 		if (__igt_timeout(end_time, NULL))
1772 			break;
1773 
1774 		i915_request_put(rq);
1775 	} while (1);
1776 
1777 	err = i915_request_wait(rq, 0, HZ / 5);
1778 	if (err < 0) {
1779 		pr_err("%s: request not completed!\n", engine->name);
1780 		goto err_wait;
1781 	}
1782 
1783 	igt_flush_test(engine->i915);
1784 
1785 	pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1786 		engine->name,
1787 		intel_context_get_total_runtime_ns(ce),
1788 		intel_context_get_avg_runtime_ns(ce));
1789 
1790 	err = 0;
1791 	if (ce->runtime.num_underflow) {
1792 		pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1793 		       engine->name,
1794 		       ce->runtime.num_underflow,
1795 		       ce->runtime.max_underflow);
1796 		GEM_TRACE_DUMP();
1797 		err = -EOVERFLOW;
1798 	}
1799 
1800 err_wait:
1801 	i915_request_put(rq);
1802 err_rq:
1803 	intel_context_put(ce);
1804 	return err;
1805 }
1806 
1807 static int live_pphwsp_runtime(void *arg)
1808 {
1809 	struct intel_gt *gt = arg;
1810 	struct intel_engine_cs *engine;
1811 	enum intel_engine_id id;
1812 	int err = 0;
1813 
1814 	/*
1815 	 * Check that cumulative context runtime as stored in the pphwsp[16]
1816 	 * is monotonic.
1817 	 */
1818 
1819 	for_each_engine(engine, gt, id) {
1820 		err = __live_pphwsp_runtime(engine);
1821 		if (err)
1822 			break;
1823 	}
1824 
1825 	if (igt_flush_test(gt->i915))
1826 		err = -EIO;
1827 
1828 	return err;
1829 }
1830 
1831 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1832 {
1833 	static const struct i915_subtest tests[] = {
1834 		SUBTEST(live_lrc_layout),
1835 		SUBTEST(live_lrc_fixed),
1836 		SUBTEST(live_lrc_state),
1837 		SUBTEST(live_lrc_gpr),
1838 		SUBTEST(live_lrc_isolation),
1839 		SUBTEST(live_lrc_timestamp),
1840 		SUBTEST(live_lrc_garbage),
1841 		SUBTEST(live_pphwsp_runtime),
1842 		SUBTEST(live_lrc_indirect_ctx_bb),
1843 	};
1844 
1845 	if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1846 		return 0;
1847 
1848 	return intel_gt_live_subtests(tests, &i915->gt);
1849 }
1850