1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include <linux/prime_numbers.h>
7 
8 #include "i915_selftest.h"
9 #include "intel_engine_heartbeat.h"
10 #include "intel_engine_pm.h"
11 #include "intel_reset.h"
12 #include "intel_ring.h"
13 #include "selftest_engine_heartbeat.h"
14 #include "selftests/i915_random.h"
15 #include "selftests/igt_flush_test.h"
16 #include "selftests/igt_live_test.h"
17 #include "selftests/igt_spinner.h"
18 #include "selftests/lib_sw_fence.h"
19 #include "shmem_utils.h"
20 
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
23 
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR 16
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27 
28 static struct i915_vma *create_scratch(struct intel_gt *gt)
29 {
30 	return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
31 }
32 
33 static bool is_active(struct i915_request *rq)
34 {
35 	if (i915_request_is_active(rq))
36 		return true;
37 
38 	if (i915_request_on_hold(rq))
39 		return true;
40 
41 	if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
42 		return true;
43 
44 	return false;
45 }
46 
47 static int wait_for_submit(struct intel_engine_cs *engine,
48 			   struct i915_request *rq,
49 			   unsigned long timeout)
50 {
51 	/* Ignore our own attempts to suppress excess tasklets */
52 	tasklet_hi_schedule(&engine->sched_engine->tasklet);
53 
54 	timeout += jiffies;
55 	do {
56 		bool done = time_after(jiffies, timeout);
57 
58 		if (i915_request_completed(rq)) /* that was quick! */
59 			return 0;
60 
61 		/* Wait until the HW has acknowleged the submission (or err) */
62 		intel_engine_flush_submission(engine);
63 		if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
64 			return 0;
65 
66 		if (done)
67 			return -ETIME;
68 
69 		cond_resched();
70 	} while (1);
71 }
72 
73 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
74 {
75 	const u32 offset =
76 		i915_ggtt_offset(ce->engine->status_page.vma) +
77 		offset_in_page(slot);
78 	struct i915_request *rq;
79 	u32 *cs;
80 
81 	rq = intel_context_create_request(ce);
82 	if (IS_ERR(rq))
83 		return PTR_ERR(rq);
84 
85 	cs = intel_ring_begin(rq, 4);
86 	if (IS_ERR(cs)) {
87 		i915_request_add(rq);
88 		return PTR_ERR(cs);
89 	}
90 
91 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
92 	*cs++ = offset;
93 	*cs++ = 0;
94 	*cs++ = 1;
95 
96 	intel_ring_advance(rq, cs);
97 
98 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
99 	i915_request_add(rq);
100 	return 0;
101 }
102 
103 static int context_flush(struct intel_context *ce, long timeout)
104 {
105 	struct i915_request *rq;
106 	struct dma_fence *fence;
107 	int err = 0;
108 
109 	rq = intel_engine_create_kernel_request(ce->engine);
110 	if (IS_ERR(rq))
111 		return PTR_ERR(rq);
112 
113 	fence = i915_active_fence_get(&ce->timeline->last_request);
114 	if (fence) {
115 		i915_request_await_dma_fence(rq, fence);
116 		dma_fence_put(fence);
117 	}
118 
119 	rq = i915_request_get(rq);
120 	i915_request_add(rq);
121 	if (i915_request_wait(rq, 0, timeout) < 0)
122 		err = -ETIME;
123 	i915_request_put(rq);
124 
125 	rmb(); /* We know the request is written, make sure all state is too! */
126 	return err;
127 }
128 
129 static int live_lrc_layout(void *arg)
130 {
131 	struct intel_gt *gt = arg;
132 	struct intel_engine_cs *engine;
133 	enum intel_engine_id id;
134 	u32 *lrc;
135 	int err;
136 
137 	/*
138 	 * Check the registers offsets we use to create the initial reg state
139 	 * match the layout saved by HW.
140 	 */
141 
142 	lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
143 	if (!lrc)
144 		return -ENOMEM;
145 	GEM_BUG_ON(offset_in_page(lrc));
146 
147 	err = 0;
148 	for_each_engine(engine, gt, id) {
149 		u32 *hw;
150 		int dw;
151 
152 		if (!engine->default_state)
153 			continue;
154 
155 		hw = shmem_pin_map(engine->default_state);
156 		if (IS_ERR(hw)) {
157 			err = PTR_ERR(hw);
158 			break;
159 		}
160 		hw += LRC_STATE_OFFSET / sizeof(*hw);
161 
162 		__lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
163 				engine->kernel_context, engine, true);
164 
165 		dw = 0;
166 		do {
167 			u32 lri = READ_ONCE(hw[dw]);
168 
169 			if (lri == 0) {
170 				dw++;
171 				continue;
172 			}
173 
174 			if (lrc[dw] == 0) {
175 				pr_debug("%s: skipped instruction %x at dword %d\n",
176 					 engine->name, lri, dw);
177 				dw++;
178 				continue;
179 			}
180 
181 			if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
182 				pr_err("%s: Expected LRI command at dword %d, found %08x\n",
183 				       engine->name, dw, lri);
184 				err = -EINVAL;
185 				break;
186 			}
187 
188 			if (lrc[dw] != lri) {
189 				pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
190 				       engine->name, dw, lri, lrc[dw]);
191 				err = -EINVAL;
192 				break;
193 			}
194 
195 			lri &= 0x7f;
196 			lri++;
197 			dw++;
198 
199 			while (lri) {
200 				u32 offset = READ_ONCE(hw[dw]);
201 
202 				if (offset != lrc[dw]) {
203 					pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
204 					       engine->name, dw, offset, lrc[dw]);
205 					err = -EINVAL;
206 					break;
207 				}
208 
209 				/*
210 				 * Skip over the actual register value as we
211 				 * expect that to differ.
212 				 */
213 				dw += 2;
214 				lri -= 2;
215 			}
216 		} while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
217 
218 		if (err) {
219 			pr_info("%s: HW register image:\n", engine->name);
220 			igt_hexdump(hw, PAGE_SIZE);
221 
222 			pr_info("%s: SW register image:\n", engine->name);
223 			igt_hexdump(lrc, PAGE_SIZE);
224 		}
225 
226 		shmem_unpin_map(engine->default_state, hw);
227 		if (err)
228 			break;
229 	}
230 
231 	free_page((unsigned long)lrc);
232 	return err;
233 }
234 
235 static int find_offset(const u32 *lri, u32 offset)
236 {
237 	int i;
238 
239 	for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
240 		if (lri[i] == offset)
241 			return i;
242 
243 	return -1;
244 }
245 
246 static int live_lrc_fixed(void *arg)
247 {
248 	struct intel_gt *gt = arg;
249 	struct intel_engine_cs *engine;
250 	enum intel_engine_id id;
251 	int err = 0;
252 
253 	/*
254 	 * Check the assumed register offsets match the actual locations in
255 	 * the context image.
256 	 */
257 
258 	for_each_engine(engine, gt, id) {
259 		const struct {
260 			u32 reg;
261 			u32 offset;
262 			const char *name;
263 		} tbl[] = {
264 			{
265 				i915_mmio_reg_offset(RING_START(engine->mmio_base)),
266 				CTX_RING_START - 1,
267 				"RING_START"
268 			},
269 			{
270 				i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
271 				CTX_RING_CTL - 1,
272 				"RING_CTL"
273 			},
274 			{
275 				i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
276 				CTX_RING_HEAD - 1,
277 				"RING_HEAD"
278 			},
279 			{
280 				i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
281 				CTX_RING_TAIL - 1,
282 				"RING_TAIL"
283 			},
284 			{
285 				i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
286 				lrc_ring_mi_mode(engine),
287 				"RING_MI_MODE"
288 			},
289 			{
290 				i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
291 				CTX_BB_STATE - 1,
292 				"BB_STATE"
293 			},
294 			{
295 				i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
296 				lrc_ring_wa_bb_per_ctx(engine),
297 				"RING_BB_PER_CTX_PTR"
298 			},
299 			{
300 				i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
301 				lrc_ring_indirect_ptr(engine),
302 				"RING_INDIRECT_CTX_PTR"
303 			},
304 			{
305 				i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
306 				lrc_ring_indirect_offset(engine),
307 				"RING_INDIRECT_CTX_OFFSET"
308 			},
309 			{
310 				i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
311 				CTX_TIMESTAMP - 1,
312 				"RING_CTX_TIMESTAMP"
313 			},
314 			{
315 				i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
316 				lrc_ring_gpr0(engine),
317 				"RING_CS_GPR0"
318 			},
319 			{
320 				i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
321 				lrc_ring_cmd_buf_cctl(engine),
322 				"RING_CMD_BUF_CCTL"
323 			},
324 			{ },
325 		}, *t;
326 		u32 *hw;
327 
328 		if (!engine->default_state)
329 			continue;
330 
331 		hw = shmem_pin_map(engine->default_state);
332 		if (IS_ERR(hw)) {
333 			err = PTR_ERR(hw);
334 			break;
335 		}
336 		hw += LRC_STATE_OFFSET / sizeof(*hw);
337 
338 		for (t = tbl; t->name; t++) {
339 			int dw = find_offset(hw, t->reg);
340 
341 			if (dw != t->offset) {
342 				pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
343 				       engine->name,
344 				       t->name,
345 				       t->reg,
346 				       dw,
347 				       t->offset);
348 				err = -EINVAL;
349 			}
350 		}
351 
352 		shmem_unpin_map(engine->default_state, hw);
353 	}
354 
355 	return err;
356 }
357 
358 static int __live_lrc_state(struct intel_engine_cs *engine,
359 			    struct i915_vma *scratch)
360 {
361 	struct intel_context *ce;
362 	struct i915_request *rq;
363 	struct i915_gem_ww_ctx ww;
364 	enum {
365 		RING_START_IDX = 0,
366 		RING_TAIL_IDX,
367 		MAX_IDX
368 	};
369 	u32 expected[MAX_IDX];
370 	u32 *cs;
371 	int err;
372 	int n;
373 
374 	ce = intel_context_create(engine);
375 	if (IS_ERR(ce))
376 		return PTR_ERR(ce);
377 
378 	i915_gem_ww_ctx_init(&ww, false);
379 retry:
380 	err = i915_gem_object_lock(scratch->obj, &ww);
381 	if (!err)
382 		err = intel_context_pin_ww(ce, &ww);
383 	if (err)
384 		goto err_put;
385 
386 	rq = i915_request_create(ce);
387 	if (IS_ERR(rq)) {
388 		err = PTR_ERR(rq);
389 		goto err_unpin;
390 	}
391 
392 	cs = intel_ring_begin(rq, 4 * MAX_IDX);
393 	if (IS_ERR(cs)) {
394 		err = PTR_ERR(cs);
395 		i915_request_add(rq);
396 		goto err_unpin;
397 	}
398 
399 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
400 	*cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
401 	*cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
402 	*cs++ = 0;
403 
404 	expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
405 
406 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
407 	*cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
408 	*cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
409 	*cs++ = 0;
410 
411 	err = i915_request_await_object(rq, scratch->obj, true);
412 	if (!err)
413 		err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
414 
415 	i915_request_get(rq);
416 	i915_request_add(rq);
417 	if (err)
418 		goto err_rq;
419 
420 	intel_engine_flush_submission(engine);
421 	expected[RING_TAIL_IDX] = ce->ring->tail;
422 
423 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
424 		err = -ETIME;
425 		goto err_rq;
426 	}
427 
428 	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
429 	if (IS_ERR(cs)) {
430 		err = PTR_ERR(cs);
431 		goto err_rq;
432 	}
433 
434 	for (n = 0; n < MAX_IDX; n++) {
435 		if (cs[n] != expected[n]) {
436 			pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
437 			       engine->name, n, cs[n], expected[n]);
438 			err = -EINVAL;
439 			break;
440 		}
441 	}
442 
443 	i915_gem_object_unpin_map(scratch->obj);
444 
445 err_rq:
446 	i915_request_put(rq);
447 err_unpin:
448 	intel_context_unpin(ce);
449 err_put:
450 	if (err == -EDEADLK) {
451 		err = i915_gem_ww_ctx_backoff(&ww);
452 		if (!err)
453 			goto retry;
454 	}
455 	i915_gem_ww_ctx_fini(&ww);
456 	intel_context_put(ce);
457 	return err;
458 }
459 
460 static int live_lrc_state(void *arg)
461 {
462 	struct intel_gt *gt = arg;
463 	struct intel_engine_cs *engine;
464 	struct i915_vma *scratch;
465 	enum intel_engine_id id;
466 	int err = 0;
467 
468 	/*
469 	 * Check the live register state matches what we expect for this
470 	 * intel_context.
471 	 */
472 
473 	scratch = create_scratch(gt);
474 	if (IS_ERR(scratch))
475 		return PTR_ERR(scratch);
476 
477 	for_each_engine(engine, gt, id) {
478 		err = __live_lrc_state(engine, scratch);
479 		if (err)
480 			break;
481 	}
482 
483 	if (igt_flush_test(gt->i915))
484 		err = -EIO;
485 
486 	i915_vma_unpin_and_release(&scratch, 0);
487 	return err;
488 }
489 
490 static int gpr_make_dirty(struct intel_context *ce)
491 {
492 	struct i915_request *rq;
493 	u32 *cs;
494 	int n;
495 
496 	rq = intel_context_create_request(ce);
497 	if (IS_ERR(rq))
498 		return PTR_ERR(rq);
499 
500 	cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
501 	if (IS_ERR(cs)) {
502 		i915_request_add(rq);
503 		return PTR_ERR(cs);
504 	}
505 
506 	*cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
507 	for (n = 0; n < NUM_GPR_DW; n++) {
508 		*cs++ = CS_GPR(ce->engine, n);
509 		*cs++ = STACK_MAGIC;
510 	}
511 	*cs++ = MI_NOOP;
512 
513 	intel_ring_advance(rq, cs);
514 
515 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
516 	i915_request_add(rq);
517 
518 	return 0;
519 }
520 
521 static struct i915_request *
522 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
523 {
524 	const u32 offset =
525 		i915_ggtt_offset(ce->engine->status_page.vma) +
526 		offset_in_page(slot);
527 	struct i915_request *rq;
528 	u32 *cs;
529 	int err;
530 	int n;
531 
532 	rq = intel_context_create_request(ce);
533 	if (IS_ERR(rq))
534 		return rq;
535 
536 	cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
537 	if (IS_ERR(cs)) {
538 		i915_request_add(rq);
539 		return ERR_CAST(cs);
540 	}
541 
542 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
543 	*cs++ = MI_NOOP;
544 
545 	*cs++ = MI_SEMAPHORE_WAIT |
546 		MI_SEMAPHORE_GLOBAL_GTT |
547 		MI_SEMAPHORE_POLL |
548 		MI_SEMAPHORE_SAD_NEQ_SDD;
549 	*cs++ = 0;
550 	*cs++ = offset;
551 	*cs++ = 0;
552 
553 	for (n = 0; n < NUM_GPR_DW; n++) {
554 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
555 		*cs++ = CS_GPR(ce->engine, n);
556 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
557 		*cs++ = 0;
558 	}
559 
560 	i915_vma_lock(scratch);
561 	err = i915_request_await_object(rq, scratch->obj, true);
562 	if (!err)
563 		err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
564 	i915_vma_unlock(scratch);
565 
566 	i915_request_get(rq);
567 	i915_request_add(rq);
568 	if (err) {
569 		i915_request_put(rq);
570 		rq = ERR_PTR(err);
571 	}
572 
573 	return rq;
574 }
575 
576 static int __live_lrc_gpr(struct intel_engine_cs *engine,
577 			  struct i915_vma *scratch,
578 			  bool preempt)
579 {
580 	u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
581 	struct intel_context *ce;
582 	struct i915_request *rq;
583 	u32 *cs;
584 	int err;
585 	int n;
586 
587 	if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
588 		return 0; /* GPR only on rcs0 for gen8 */
589 
590 	err = gpr_make_dirty(engine->kernel_context);
591 	if (err)
592 		return err;
593 
594 	ce = intel_context_create(engine);
595 	if (IS_ERR(ce))
596 		return PTR_ERR(ce);
597 
598 	rq = __gpr_read(ce, scratch, slot);
599 	if (IS_ERR(rq)) {
600 		err = PTR_ERR(rq);
601 		goto err_put;
602 	}
603 
604 	err = wait_for_submit(engine, rq, HZ / 2);
605 	if (err)
606 		goto err_rq;
607 
608 	if (preempt) {
609 		err = gpr_make_dirty(engine->kernel_context);
610 		if (err)
611 			goto err_rq;
612 
613 		err = emit_semaphore_signal(engine->kernel_context, slot);
614 		if (err)
615 			goto err_rq;
616 
617 		err = wait_for_submit(engine, rq, HZ / 2);
618 		if (err)
619 			goto err_rq;
620 	} else {
621 		slot[0] = 1;
622 		wmb();
623 	}
624 
625 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
626 		err = -ETIME;
627 		goto err_rq;
628 	}
629 
630 	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
631 	if (IS_ERR(cs)) {
632 		err = PTR_ERR(cs);
633 		goto err_rq;
634 	}
635 
636 	for (n = 0; n < NUM_GPR_DW; n++) {
637 		if (cs[n]) {
638 			pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
639 			       engine->name,
640 			       n / 2, n & 1 ? "udw" : "ldw",
641 			       cs[n]);
642 			err = -EINVAL;
643 			break;
644 		}
645 	}
646 
647 	i915_gem_object_unpin_map(scratch->obj);
648 
649 err_rq:
650 	memset32(&slot[0], -1, 4);
651 	wmb();
652 	i915_request_put(rq);
653 err_put:
654 	intel_context_put(ce);
655 	return err;
656 }
657 
658 static int live_lrc_gpr(void *arg)
659 {
660 	struct intel_gt *gt = arg;
661 	struct intel_engine_cs *engine;
662 	struct i915_vma *scratch;
663 	enum intel_engine_id id;
664 	int err = 0;
665 
666 	/*
667 	 * Check that GPR registers are cleared in new contexts as we need
668 	 * to avoid leaking any information from previous contexts.
669 	 */
670 
671 	scratch = create_scratch(gt);
672 	if (IS_ERR(scratch))
673 		return PTR_ERR(scratch);
674 
675 	for_each_engine(engine, gt, id) {
676 		st_engine_heartbeat_disable(engine);
677 
678 		err = __live_lrc_gpr(engine, scratch, false);
679 		if (err)
680 			goto err;
681 
682 		err = __live_lrc_gpr(engine, scratch, true);
683 		if (err)
684 			goto err;
685 
686 err:
687 		st_engine_heartbeat_enable(engine);
688 		if (igt_flush_test(gt->i915))
689 			err = -EIO;
690 		if (err)
691 			break;
692 	}
693 
694 	i915_vma_unpin_and_release(&scratch, 0);
695 	return err;
696 }
697 
698 static struct i915_request *
699 create_timestamp(struct intel_context *ce, void *slot, int idx)
700 {
701 	const u32 offset =
702 		i915_ggtt_offset(ce->engine->status_page.vma) +
703 		offset_in_page(slot);
704 	struct i915_request *rq;
705 	u32 *cs;
706 	int err;
707 
708 	rq = intel_context_create_request(ce);
709 	if (IS_ERR(rq))
710 		return rq;
711 
712 	cs = intel_ring_begin(rq, 10);
713 	if (IS_ERR(cs)) {
714 		err = PTR_ERR(cs);
715 		goto err;
716 	}
717 
718 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
719 	*cs++ = MI_NOOP;
720 
721 	*cs++ = MI_SEMAPHORE_WAIT |
722 		MI_SEMAPHORE_GLOBAL_GTT |
723 		MI_SEMAPHORE_POLL |
724 		MI_SEMAPHORE_SAD_NEQ_SDD;
725 	*cs++ = 0;
726 	*cs++ = offset;
727 	*cs++ = 0;
728 
729 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
730 	*cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
731 	*cs++ = offset + idx * sizeof(u32);
732 	*cs++ = 0;
733 
734 	intel_ring_advance(rq, cs);
735 
736 	err = 0;
737 err:
738 	i915_request_get(rq);
739 	i915_request_add(rq);
740 	if (err) {
741 		i915_request_put(rq);
742 		return ERR_PTR(err);
743 	}
744 
745 	return rq;
746 }
747 
748 struct lrc_timestamp {
749 	struct intel_engine_cs *engine;
750 	struct intel_context *ce[2];
751 	u32 poison;
752 };
753 
754 static bool timestamp_advanced(u32 start, u32 end)
755 {
756 	return (s32)(end - start) > 0;
757 }
758 
759 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
760 {
761 	u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
762 	struct i915_request *rq;
763 	u32 timestamp;
764 	int err = 0;
765 
766 	arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
767 	rq = create_timestamp(arg->ce[0], slot, 1);
768 	if (IS_ERR(rq))
769 		return PTR_ERR(rq);
770 
771 	err = wait_for_submit(rq->engine, rq, HZ / 2);
772 	if (err)
773 		goto err;
774 
775 	if (preempt) {
776 		arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
777 		err = emit_semaphore_signal(arg->ce[1], slot);
778 		if (err)
779 			goto err;
780 	} else {
781 		slot[0] = 1;
782 		wmb();
783 	}
784 
785 	/* And wait for switch to kernel (to save our context to memory) */
786 	err = context_flush(arg->ce[0], HZ / 2);
787 	if (err)
788 		goto err;
789 
790 	if (!timestamp_advanced(arg->poison, slot[1])) {
791 		pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
792 		       arg->engine->name, preempt ? "preempt" : "simple",
793 		       arg->poison, slot[1]);
794 		err = -EINVAL;
795 	}
796 
797 	timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
798 	if (!timestamp_advanced(slot[1], timestamp)) {
799 		pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
800 		       arg->engine->name, preempt ? "preempt" : "simple",
801 		       slot[1], timestamp);
802 		err = -EINVAL;
803 	}
804 
805 err:
806 	memset32(slot, -1, 4);
807 	i915_request_put(rq);
808 	return err;
809 }
810 
811 static int live_lrc_timestamp(void *arg)
812 {
813 	struct lrc_timestamp data = {};
814 	struct intel_gt *gt = arg;
815 	enum intel_engine_id id;
816 	const u32 poison[] = {
817 		0,
818 		S32_MAX,
819 		(u32)S32_MAX + 1,
820 		U32_MAX,
821 	};
822 
823 	/*
824 	 * We want to verify that the timestamp is saved and restore across
825 	 * context switches and is monotonic.
826 	 *
827 	 * So we do this with a little bit of LRC poisoning to check various
828 	 * boundary conditions, and see what happens if we preempt the context
829 	 * with a second request (carrying more poison into the timestamp).
830 	 */
831 
832 	for_each_engine(data.engine, gt, id) {
833 		int i, err = 0;
834 
835 		st_engine_heartbeat_disable(data.engine);
836 
837 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
838 			struct intel_context *tmp;
839 
840 			tmp = intel_context_create(data.engine);
841 			if (IS_ERR(tmp)) {
842 				err = PTR_ERR(tmp);
843 				goto err;
844 			}
845 
846 			err = intel_context_pin(tmp);
847 			if (err) {
848 				intel_context_put(tmp);
849 				goto err;
850 			}
851 
852 			data.ce[i] = tmp;
853 		}
854 
855 		for (i = 0; i < ARRAY_SIZE(poison); i++) {
856 			data.poison = poison[i];
857 
858 			err = __lrc_timestamp(&data, false);
859 			if (err)
860 				break;
861 
862 			err = __lrc_timestamp(&data, true);
863 			if (err)
864 				break;
865 		}
866 
867 err:
868 		st_engine_heartbeat_enable(data.engine);
869 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
870 			if (!data.ce[i])
871 				break;
872 
873 			intel_context_unpin(data.ce[i]);
874 			intel_context_put(data.ce[i]);
875 		}
876 
877 		if (igt_flush_test(gt->i915))
878 			err = -EIO;
879 		if (err)
880 			return err;
881 	}
882 
883 	return 0;
884 }
885 
886 static struct i915_vma *
887 create_user_vma(struct i915_address_space *vm, unsigned long size)
888 {
889 	struct drm_i915_gem_object *obj;
890 	struct i915_vma *vma;
891 	int err;
892 
893 	obj = i915_gem_object_create_internal(vm->i915, size);
894 	if (IS_ERR(obj))
895 		return ERR_CAST(obj);
896 
897 	vma = i915_vma_instance(obj, vm, NULL);
898 	if (IS_ERR(vma)) {
899 		i915_gem_object_put(obj);
900 		return vma;
901 	}
902 
903 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
904 	if (err) {
905 		i915_gem_object_put(obj);
906 		return ERR_PTR(err);
907 	}
908 
909 	return vma;
910 }
911 
912 static struct i915_vma *
913 store_context(struct intel_context *ce, struct i915_vma *scratch)
914 {
915 	struct i915_vma *batch;
916 	u32 dw, x, *cs, *hw;
917 	u32 *defaults;
918 
919 	batch = create_user_vma(ce->vm, SZ_64K);
920 	if (IS_ERR(batch))
921 		return batch;
922 
923 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
924 	if (IS_ERR(cs)) {
925 		i915_vma_put(batch);
926 		return ERR_CAST(cs);
927 	}
928 
929 	defaults = shmem_pin_map(ce->engine->default_state);
930 	if (!defaults) {
931 		i915_gem_object_unpin_map(batch->obj);
932 		i915_vma_put(batch);
933 		return ERR_PTR(-ENOMEM);
934 	}
935 
936 	x = 0;
937 	dw = 0;
938 	hw = defaults;
939 	hw += LRC_STATE_OFFSET / sizeof(*hw);
940 	do {
941 		u32 len = hw[dw] & 0x7f;
942 
943 		if (hw[dw] == 0) {
944 			dw++;
945 			continue;
946 		}
947 
948 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
949 			dw += len + 2;
950 			continue;
951 		}
952 
953 		dw++;
954 		len = (len + 1) / 2;
955 		while (len--) {
956 			*cs++ = MI_STORE_REGISTER_MEM_GEN8;
957 			*cs++ = hw[dw];
958 			*cs++ = lower_32_bits(scratch->node.start + x);
959 			*cs++ = upper_32_bits(scratch->node.start + x);
960 
961 			dw += 2;
962 			x += 4;
963 		}
964 	} while (dw < PAGE_SIZE / sizeof(u32) &&
965 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
966 
967 	*cs++ = MI_BATCH_BUFFER_END;
968 
969 	shmem_unpin_map(ce->engine->default_state, defaults);
970 
971 	i915_gem_object_flush_map(batch->obj);
972 	i915_gem_object_unpin_map(batch->obj);
973 
974 	return batch;
975 }
976 
977 static int move_to_active(struct i915_request *rq,
978 			  struct i915_vma *vma,
979 			  unsigned int flags)
980 {
981 	int err;
982 
983 	i915_vma_lock(vma);
984 	err = i915_request_await_object(rq, vma->obj, flags);
985 	if (!err)
986 		err = i915_vma_move_to_active(vma, rq, flags);
987 	i915_vma_unlock(vma);
988 
989 	return err;
990 }
991 
992 static struct i915_request *
993 record_registers(struct intel_context *ce,
994 		 struct i915_vma *before,
995 		 struct i915_vma *after,
996 		 u32 *sema)
997 {
998 	struct i915_vma *b_before, *b_after;
999 	struct i915_request *rq;
1000 	u32 *cs;
1001 	int err;
1002 
1003 	b_before = store_context(ce, before);
1004 	if (IS_ERR(b_before))
1005 		return ERR_CAST(b_before);
1006 
1007 	b_after = store_context(ce, after);
1008 	if (IS_ERR(b_after)) {
1009 		rq = ERR_CAST(b_after);
1010 		goto err_before;
1011 	}
1012 
1013 	rq = intel_context_create_request(ce);
1014 	if (IS_ERR(rq))
1015 		goto err_after;
1016 
1017 	err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1018 	if (err)
1019 		goto err_rq;
1020 
1021 	err = move_to_active(rq, b_before, 0);
1022 	if (err)
1023 		goto err_rq;
1024 
1025 	err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1026 	if (err)
1027 		goto err_rq;
1028 
1029 	err = move_to_active(rq, b_after, 0);
1030 	if (err)
1031 		goto err_rq;
1032 
1033 	cs = intel_ring_begin(rq, 14);
1034 	if (IS_ERR(cs)) {
1035 		err = PTR_ERR(cs);
1036 		goto err_rq;
1037 	}
1038 
1039 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1040 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1041 	*cs++ = lower_32_bits(b_before->node.start);
1042 	*cs++ = upper_32_bits(b_before->node.start);
1043 
1044 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1045 	*cs++ = MI_SEMAPHORE_WAIT |
1046 		MI_SEMAPHORE_GLOBAL_GTT |
1047 		MI_SEMAPHORE_POLL |
1048 		MI_SEMAPHORE_SAD_NEQ_SDD;
1049 	*cs++ = 0;
1050 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1051 		offset_in_page(sema);
1052 	*cs++ = 0;
1053 	*cs++ = MI_NOOP;
1054 
1055 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1056 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1057 	*cs++ = lower_32_bits(b_after->node.start);
1058 	*cs++ = upper_32_bits(b_after->node.start);
1059 
1060 	intel_ring_advance(rq, cs);
1061 
1062 	WRITE_ONCE(*sema, 0);
1063 	i915_request_get(rq);
1064 	i915_request_add(rq);
1065 err_after:
1066 	i915_vma_put(b_after);
1067 err_before:
1068 	i915_vma_put(b_before);
1069 	return rq;
1070 
1071 err_rq:
1072 	i915_request_add(rq);
1073 	rq = ERR_PTR(err);
1074 	goto err_after;
1075 }
1076 
1077 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1078 {
1079 	struct i915_vma *batch;
1080 	u32 dw, *cs, *hw;
1081 	u32 *defaults;
1082 
1083 	batch = create_user_vma(ce->vm, SZ_64K);
1084 	if (IS_ERR(batch))
1085 		return batch;
1086 
1087 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1088 	if (IS_ERR(cs)) {
1089 		i915_vma_put(batch);
1090 		return ERR_CAST(cs);
1091 	}
1092 
1093 	defaults = shmem_pin_map(ce->engine->default_state);
1094 	if (!defaults) {
1095 		i915_gem_object_unpin_map(batch->obj);
1096 		i915_vma_put(batch);
1097 		return ERR_PTR(-ENOMEM);
1098 	}
1099 
1100 	dw = 0;
1101 	hw = defaults;
1102 	hw += LRC_STATE_OFFSET / sizeof(*hw);
1103 	do {
1104 		u32 len = hw[dw] & 0x7f;
1105 
1106 		if (hw[dw] == 0) {
1107 			dw++;
1108 			continue;
1109 		}
1110 
1111 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1112 			dw += len + 2;
1113 			continue;
1114 		}
1115 
1116 		dw++;
1117 		len = (len + 1) / 2;
1118 		*cs++ = MI_LOAD_REGISTER_IMM(len);
1119 		while (len--) {
1120 			*cs++ = hw[dw];
1121 			*cs++ = poison;
1122 			dw += 2;
1123 		}
1124 	} while (dw < PAGE_SIZE / sizeof(u32) &&
1125 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1126 
1127 	*cs++ = MI_BATCH_BUFFER_END;
1128 
1129 	shmem_unpin_map(ce->engine->default_state, defaults);
1130 
1131 	i915_gem_object_flush_map(batch->obj);
1132 	i915_gem_object_unpin_map(batch->obj);
1133 
1134 	return batch;
1135 }
1136 
1137 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1138 {
1139 	struct i915_request *rq;
1140 	struct i915_vma *batch;
1141 	u32 *cs;
1142 	int err;
1143 
1144 	batch = load_context(ce, poison);
1145 	if (IS_ERR(batch))
1146 		return PTR_ERR(batch);
1147 
1148 	rq = intel_context_create_request(ce);
1149 	if (IS_ERR(rq)) {
1150 		err = PTR_ERR(rq);
1151 		goto err_batch;
1152 	}
1153 
1154 	err = move_to_active(rq, batch, 0);
1155 	if (err)
1156 		goto err_rq;
1157 
1158 	cs = intel_ring_begin(rq, 8);
1159 	if (IS_ERR(cs)) {
1160 		err = PTR_ERR(cs);
1161 		goto err_rq;
1162 	}
1163 
1164 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1165 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1166 	*cs++ = lower_32_bits(batch->node.start);
1167 	*cs++ = upper_32_bits(batch->node.start);
1168 
1169 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1170 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1171 		offset_in_page(sema);
1172 	*cs++ = 0;
1173 	*cs++ = 1;
1174 
1175 	intel_ring_advance(rq, cs);
1176 
1177 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1178 err_rq:
1179 	i915_request_add(rq);
1180 err_batch:
1181 	i915_vma_put(batch);
1182 	return err;
1183 }
1184 
1185 static bool is_moving(u32 a, u32 b)
1186 {
1187 	return a != b;
1188 }
1189 
1190 static int compare_isolation(struct intel_engine_cs *engine,
1191 			     struct i915_vma *ref[2],
1192 			     struct i915_vma *result[2],
1193 			     struct intel_context *ce,
1194 			     u32 poison)
1195 {
1196 	u32 x, dw, *hw, *lrc;
1197 	u32 *A[2], *B[2];
1198 	u32 *defaults;
1199 	int err = 0;
1200 
1201 	A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1202 	if (IS_ERR(A[0]))
1203 		return PTR_ERR(A[0]);
1204 
1205 	A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1206 	if (IS_ERR(A[1])) {
1207 		err = PTR_ERR(A[1]);
1208 		goto err_A0;
1209 	}
1210 
1211 	B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1212 	if (IS_ERR(B[0])) {
1213 		err = PTR_ERR(B[0]);
1214 		goto err_A1;
1215 	}
1216 
1217 	B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1218 	if (IS_ERR(B[1])) {
1219 		err = PTR_ERR(B[1]);
1220 		goto err_B0;
1221 	}
1222 
1223 	lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1224 					       i915_coherent_map_type(engine->i915,
1225 								      ce->state->obj,
1226 								      false));
1227 	if (IS_ERR(lrc)) {
1228 		err = PTR_ERR(lrc);
1229 		goto err_B1;
1230 	}
1231 	lrc += LRC_STATE_OFFSET / sizeof(*hw);
1232 
1233 	defaults = shmem_pin_map(ce->engine->default_state);
1234 	if (!defaults) {
1235 		err = -ENOMEM;
1236 		goto err_lrc;
1237 	}
1238 
1239 	x = 0;
1240 	dw = 0;
1241 	hw = defaults;
1242 	hw += LRC_STATE_OFFSET / sizeof(*hw);
1243 	do {
1244 		u32 len = hw[dw] & 0x7f;
1245 
1246 		if (hw[dw] == 0) {
1247 			dw++;
1248 			continue;
1249 		}
1250 
1251 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1252 			dw += len + 2;
1253 			continue;
1254 		}
1255 
1256 		dw++;
1257 		len = (len + 1) / 2;
1258 		while (len--) {
1259 			if (!is_moving(A[0][x], A[1][x]) &&
1260 			    (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1261 				switch (hw[dw] & 4095) {
1262 				case 0x30: /* RING_HEAD */
1263 				case 0x34: /* RING_TAIL */
1264 					break;
1265 
1266 				default:
1267 					pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1268 					       engine->name, dw,
1269 					       hw[dw], hw[dw + 1],
1270 					       A[0][x], B[0][x], B[1][x],
1271 					       poison, lrc[dw + 1]);
1272 					err = -EINVAL;
1273 				}
1274 			}
1275 			dw += 2;
1276 			x++;
1277 		}
1278 	} while (dw < PAGE_SIZE / sizeof(u32) &&
1279 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1280 
1281 	shmem_unpin_map(ce->engine->default_state, defaults);
1282 err_lrc:
1283 	i915_gem_object_unpin_map(ce->state->obj);
1284 err_B1:
1285 	i915_gem_object_unpin_map(result[1]->obj);
1286 err_B0:
1287 	i915_gem_object_unpin_map(result[0]->obj);
1288 err_A1:
1289 	i915_gem_object_unpin_map(ref[1]->obj);
1290 err_A0:
1291 	i915_gem_object_unpin_map(ref[0]->obj);
1292 	return err;
1293 }
1294 
1295 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1296 {
1297 	u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1298 	struct i915_vma *ref[2], *result[2];
1299 	struct intel_context *A, *B;
1300 	struct i915_request *rq;
1301 	int err;
1302 
1303 	A = intel_context_create(engine);
1304 	if (IS_ERR(A))
1305 		return PTR_ERR(A);
1306 
1307 	B = intel_context_create(engine);
1308 	if (IS_ERR(B)) {
1309 		err = PTR_ERR(B);
1310 		goto err_A;
1311 	}
1312 
1313 	ref[0] = create_user_vma(A->vm, SZ_64K);
1314 	if (IS_ERR(ref[0])) {
1315 		err = PTR_ERR(ref[0]);
1316 		goto err_B;
1317 	}
1318 
1319 	ref[1] = create_user_vma(A->vm, SZ_64K);
1320 	if (IS_ERR(ref[1])) {
1321 		err = PTR_ERR(ref[1]);
1322 		goto err_ref0;
1323 	}
1324 
1325 	rq = record_registers(A, ref[0], ref[1], sema);
1326 	if (IS_ERR(rq)) {
1327 		err = PTR_ERR(rq);
1328 		goto err_ref1;
1329 	}
1330 
1331 	WRITE_ONCE(*sema, 1);
1332 	wmb();
1333 
1334 	if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1335 		i915_request_put(rq);
1336 		err = -ETIME;
1337 		goto err_ref1;
1338 	}
1339 	i915_request_put(rq);
1340 
1341 	result[0] = create_user_vma(A->vm, SZ_64K);
1342 	if (IS_ERR(result[0])) {
1343 		err = PTR_ERR(result[0]);
1344 		goto err_ref1;
1345 	}
1346 
1347 	result[1] = create_user_vma(A->vm, SZ_64K);
1348 	if (IS_ERR(result[1])) {
1349 		err = PTR_ERR(result[1]);
1350 		goto err_result0;
1351 	}
1352 
1353 	rq = record_registers(A, result[0], result[1], sema);
1354 	if (IS_ERR(rq)) {
1355 		err = PTR_ERR(rq);
1356 		goto err_result1;
1357 	}
1358 
1359 	err = poison_registers(B, poison, sema);
1360 	if (err) {
1361 		WRITE_ONCE(*sema, -1);
1362 		i915_request_put(rq);
1363 		goto err_result1;
1364 	}
1365 
1366 	if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1367 		i915_request_put(rq);
1368 		err = -ETIME;
1369 		goto err_result1;
1370 	}
1371 	i915_request_put(rq);
1372 
1373 	err = compare_isolation(engine, ref, result, A, poison);
1374 
1375 err_result1:
1376 	i915_vma_put(result[1]);
1377 err_result0:
1378 	i915_vma_put(result[0]);
1379 err_ref1:
1380 	i915_vma_put(ref[1]);
1381 err_ref0:
1382 	i915_vma_put(ref[0]);
1383 err_B:
1384 	intel_context_put(B);
1385 err_A:
1386 	intel_context_put(A);
1387 	return err;
1388 }
1389 
1390 static bool skip_isolation(const struct intel_engine_cs *engine)
1391 {
1392 	if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1393 		return true;
1394 
1395 	if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1396 		return true;
1397 
1398 	return false;
1399 }
1400 
1401 static int live_lrc_isolation(void *arg)
1402 {
1403 	struct intel_gt *gt = arg;
1404 	struct intel_engine_cs *engine;
1405 	enum intel_engine_id id;
1406 	const u32 poison[] = {
1407 		STACK_MAGIC,
1408 		0x3a3a3a3a,
1409 		0x5c5c5c5c,
1410 		0xffffffff,
1411 		0xffff0000,
1412 	};
1413 	int err = 0;
1414 
1415 	/*
1416 	 * Our goal is try and verify that per-context state cannot be
1417 	 * tampered with by another non-privileged client.
1418 	 *
1419 	 * We take the list of context registers from the LRI in the default
1420 	 * context image and attempt to modify that list from a remote context.
1421 	 */
1422 
1423 	for_each_engine(engine, gt, id) {
1424 		int i;
1425 
1426 		/* Just don't even ask */
1427 		if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1428 		    skip_isolation(engine))
1429 			continue;
1430 
1431 		intel_engine_pm_get(engine);
1432 		for (i = 0; i < ARRAY_SIZE(poison); i++) {
1433 			int result;
1434 
1435 			result = __lrc_isolation(engine, poison[i]);
1436 			if (result && !err)
1437 				err = result;
1438 
1439 			result = __lrc_isolation(engine, ~poison[i]);
1440 			if (result && !err)
1441 				err = result;
1442 		}
1443 		intel_engine_pm_put(engine);
1444 		if (igt_flush_test(gt->i915)) {
1445 			err = -EIO;
1446 			break;
1447 		}
1448 	}
1449 
1450 	return err;
1451 }
1452 
1453 static int indirect_ctx_submit_req(struct intel_context *ce)
1454 {
1455 	struct i915_request *rq;
1456 	int err = 0;
1457 
1458 	rq = intel_context_create_request(ce);
1459 	if (IS_ERR(rq))
1460 		return PTR_ERR(rq);
1461 
1462 	i915_request_get(rq);
1463 	i915_request_add(rq);
1464 
1465 	if (i915_request_wait(rq, 0, HZ / 5) < 0)
1466 		err = -ETIME;
1467 
1468 	i915_request_put(rq);
1469 
1470 	return err;
1471 }
1472 
1473 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1474 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1475 
1476 static u32 *
1477 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1478 {
1479 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1480 		MI_SRM_LRM_GLOBAL_GTT |
1481 		MI_LRI_LRM_CS_MMIO;
1482 	*cs++ = i915_mmio_reg_offset(RING_START(0));
1483 	*cs++ = i915_ggtt_offset(ce->state) +
1484 		context_wa_bb_offset(ce) +
1485 		CTX_BB_CANARY_OFFSET;
1486 	*cs++ = 0;
1487 
1488 	return cs;
1489 }
1490 
1491 static void
1492 indirect_ctx_bb_setup(struct intel_context *ce)
1493 {
1494 	u32 *cs = context_indirect_bb(ce);
1495 
1496 	cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1497 
1498 	setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1499 }
1500 
1501 static bool check_ring_start(struct intel_context *ce)
1502 {
1503 	const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1504 		LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1505 
1506 	if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1507 		return true;
1508 
1509 	pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1510 	       ctx_bb[CTX_BB_CANARY_INDEX],
1511 	       ce->lrc_reg_state[CTX_RING_START]);
1512 
1513 	return false;
1514 }
1515 
1516 static int indirect_ctx_bb_check(struct intel_context *ce)
1517 {
1518 	int err;
1519 
1520 	err = indirect_ctx_submit_req(ce);
1521 	if (err)
1522 		return err;
1523 
1524 	if (!check_ring_start(ce))
1525 		return -EINVAL;
1526 
1527 	return 0;
1528 }
1529 
1530 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1531 {
1532 	struct intel_context *a, *b;
1533 	int err;
1534 
1535 	a = intel_context_create(engine);
1536 	if (IS_ERR(a))
1537 		return PTR_ERR(a);
1538 	err = intel_context_pin(a);
1539 	if (err)
1540 		goto put_a;
1541 
1542 	b = intel_context_create(engine);
1543 	if (IS_ERR(b)) {
1544 		err = PTR_ERR(b);
1545 		goto unpin_a;
1546 	}
1547 	err = intel_context_pin(b);
1548 	if (err)
1549 		goto put_b;
1550 
1551 	/* We use the already reserved extra page in context state */
1552 	if (!a->wa_bb_page) {
1553 		GEM_BUG_ON(b->wa_bb_page);
1554 		GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1555 		goto unpin_b;
1556 	}
1557 
1558 	/*
1559 	 * In order to test that our per context bb is truly per context,
1560 	 * and executes at the intended spot on context restoring process,
1561 	 * make the batch store the ring start value to memory.
1562 	 * As ring start is restored apriori of starting the indirect ctx bb and
1563 	 * as it will be different for each context, it fits to this purpose.
1564 	 */
1565 	indirect_ctx_bb_setup(a);
1566 	indirect_ctx_bb_setup(b);
1567 
1568 	err = indirect_ctx_bb_check(a);
1569 	if (err)
1570 		goto unpin_b;
1571 
1572 	err = indirect_ctx_bb_check(b);
1573 
1574 unpin_b:
1575 	intel_context_unpin(b);
1576 put_b:
1577 	intel_context_put(b);
1578 unpin_a:
1579 	intel_context_unpin(a);
1580 put_a:
1581 	intel_context_put(a);
1582 
1583 	return err;
1584 }
1585 
1586 static int live_lrc_indirect_ctx_bb(void *arg)
1587 {
1588 	struct intel_gt *gt = arg;
1589 	struct intel_engine_cs *engine;
1590 	enum intel_engine_id id;
1591 	int err = 0;
1592 
1593 	for_each_engine(engine, gt, id) {
1594 		intel_engine_pm_get(engine);
1595 		err = __live_lrc_indirect_ctx_bb(engine);
1596 		intel_engine_pm_put(engine);
1597 
1598 		if (igt_flush_test(gt->i915))
1599 			err = -EIO;
1600 
1601 		if (err)
1602 			break;
1603 	}
1604 
1605 	return err;
1606 }
1607 
1608 static void garbage_reset(struct intel_engine_cs *engine,
1609 			  struct i915_request *rq)
1610 {
1611 	const unsigned int bit = I915_RESET_ENGINE + engine->id;
1612 	unsigned long *lock = &engine->gt->reset.flags;
1613 
1614 	local_bh_disable();
1615 	if (!test_and_set_bit(bit, lock)) {
1616 		tasklet_disable(&engine->sched_engine->tasklet);
1617 
1618 		if (!rq->fence.error)
1619 			__intel_engine_reset_bh(engine, NULL);
1620 
1621 		tasklet_enable(&engine->sched_engine->tasklet);
1622 		clear_and_wake_up_bit(bit, lock);
1623 	}
1624 	local_bh_enable();
1625 }
1626 
1627 static struct i915_request *garbage(struct intel_context *ce,
1628 				    struct rnd_state *prng)
1629 {
1630 	struct i915_request *rq;
1631 	int err;
1632 
1633 	err = intel_context_pin(ce);
1634 	if (err)
1635 		return ERR_PTR(err);
1636 
1637 	prandom_bytes_state(prng,
1638 			    ce->lrc_reg_state,
1639 			    ce->engine->context_size -
1640 			    LRC_STATE_OFFSET);
1641 
1642 	rq = intel_context_create_request(ce);
1643 	if (IS_ERR(rq)) {
1644 		err = PTR_ERR(rq);
1645 		goto err_unpin;
1646 	}
1647 
1648 	i915_request_get(rq);
1649 	i915_request_add(rq);
1650 	return rq;
1651 
1652 err_unpin:
1653 	intel_context_unpin(ce);
1654 	return ERR_PTR(err);
1655 }
1656 
1657 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1658 {
1659 	struct intel_context *ce;
1660 	struct i915_request *hang;
1661 	int err = 0;
1662 
1663 	ce = intel_context_create(engine);
1664 	if (IS_ERR(ce))
1665 		return PTR_ERR(ce);
1666 
1667 	hang = garbage(ce, prng);
1668 	if (IS_ERR(hang)) {
1669 		err = PTR_ERR(hang);
1670 		goto err_ce;
1671 	}
1672 
1673 	if (wait_for_submit(engine, hang, HZ / 2)) {
1674 		i915_request_put(hang);
1675 		err = -ETIME;
1676 		goto err_ce;
1677 	}
1678 
1679 	intel_context_set_banned(ce);
1680 	garbage_reset(engine, hang);
1681 
1682 	intel_engine_flush_submission(engine);
1683 	if (!hang->fence.error) {
1684 		i915_request_put(hang);
1685 		pr_err("%s: corrupted context was not reset\n",
1686 		       engine->name);
1687 		err = -EINVAL;
1688 		goto err_ce;
1689 	}
1690 
1691 	if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1692 		pr_err("%s: corrupted context did not recover\n",
1693 		       engine->name);
1694 		i915_request_put(hang);
1695 		err = -EIO;
1696 		goto err_ce;
1697 	}
1698 	i915_request_put(hang);
1699 
1700 err_ce:
1701 	intel_context_put(ce);
1702 	return err;
1703 }
1704 
1705 static int live_lrc_garbage(void *arg)
1706 {
1707 	struct intel_gt *gt = arg;
1708 	struct intel_engine_cs *engine;
1709 	enum intel_engine_id id;
1710 
1711 	/*
1712 	 * Verify that we can recover if one context state is completely
1713 	 * corrupted.
1714 	 */
1715 
1716 	if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1717 		return 0;
1718 
1719 	for_each_engine(engine, gt, id) {
1720 		I915_RND_STATE(prng);
1721 		int err = 0, i;
1722 
1723 		if (!intel_has_reset_engine(engine->gt))
1724 			continue;
1725 
1726 		intel_engine_pm_get(engine);
1727 		for (i = 0; i < 3; i++) {
1728 			err = __lrc_garbage(engine, &prng);
1729 			if (err)
1730 				break;
1731 		}
1732 		intel_engine_pm_put(engine);
1733 
1734 		if (igt_flush_test(gt->i915))
1735 			err = -EIO;
1736 		if (err)
1737 			return err;
1738 	}
1739 
1740 	return 0;
1741 }
1742 
1743 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1744 {
1745 	struct intel_context *ce;
1746 	struct i915_request *rq;
1747 	IGT_TIMEOUT(end_time);
1748 	int err;
1749 
1750 	ce = intel_context_create(engine);
1751 	if (IS_ERR(ce))
1752 		return PTR_ERR(ce);
1753 
1754 	ce->runtime.num_underflow = 0;
1755 	ce->runtime.max_underflow = 0;
1756 
1757 	do {
1758 		unsigned int loop = 1024;
1759 
1760 		while (loop) {
1761 			rq = intel_context_create_request(ce);
1762 			if (IS_ERR(rq)) {
1763 				err = PTR_ERR(rq);
1764 				goto err_rq;
1765 			}
1766 
1767 			if (--loop == 0)
1768 				i915_request_get(rq);
1769 
1770 			i915_request_add(rq);
1771 		}
1772 
1773 		if (__igt_timeout(end_time, NULL))
1774 			break;
1775 
1776 		i915_request_put(rq);
1777 	} while (1);
1778 
1779 	err = i915_request_wait(rq, 0, HZ / 5);
1780 	if (err < 0) {
1781 		pr_err("%s: request not completed!\n", engine->name);
1782 		goto err_wait;
1783 	}
1784 
1785 	igt_flush_test(engine->i915);
1786 
1787 	pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1788 		engine->name,
1789 		intel_context_get_total_runtime_ns(ce),
1790 		intel_context_get_avg_runtime_ns(ce));
1791 
1792 	err = 0;
1793 	if (ce->runtime.num_underflow) {
1794 		pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1795 		       engine->name,
1796 		       ce->runtime.num_underflow,
1797 		       ce->runtime.max_underflow);
1798 		GEM_TRACE_DUMP();
1799 		err = -EOVERFLOW;
1800 	}
1801 
1802 err_wait:
1803 	i915_request_put(rq);
1804 err_rq:
1805 	intel_context_put(ce);
1806 	return err;
1807 }
1808 
1809 static int live_pphwsp_runtime(void *arg)
1810 {
1811 	struct intel_gt *gt = arg;
1812 	struct intel_engine_cs *engine;
1813 	enum intel_engine_id id;
1814 	int err = 0;
1815 
1816 	/*
1817 	 * Check that cumulative context runtime as stored in the pphwsp[16]
1818 	 * is monotonic.
1819 	 */
1820 
1821 	for_each_engine(engine, gt, id) {
1822 		err = __live_pphwsp_runtime(engine);
1823 		if (err)
1824 			break;
1825 	}
1826 
1827 	if (igt_flush_test(gt->i915))
1828 		err = -EIO;
1829 
1830 	return err;
1831 }
1832 
1833 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1834 {
1835 	static const struct i915_subtest tests[] = {
1836 		SUBTEST(live_lrc_layout),
1837 		SUBTEST(live_lrc_fixed),
1838 		SUBTEST(live_lrc_state),
1839 		SUBTEST(live_lrc_gpr),
1840 		SUBTEST(live_lrc_isolation),
1841 		SUBTEST(live_lrc_timestamp),
1842 		SUBTEST(live_lrc_garbage),
1843 		SUBTEST(live_pphwsp_runtime),
1844 		SUBTEST(live_lrc_indirect_ctx_bb),
1845 	};
1846 
1847 	if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1848 		return 0;
1849 
1850 	return intel_gt_live_subtests(tests, &i915->gt);
1851 }
1852