1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include <linux/prime_numbers.h>
7 
8 #include "gem/i915_gem_internal.h"
9 
10 #include "i915_selftest.h"
11 #include "intel_engine_heartbeat.h"
12 #include "intel_engine_pm.h"
13 #include "intel_reset.h"
14 #include "intel_ring.h"
15 #include "selftest_engine_heartbeat.h"
16 #include "selftests/i915_random.h"
17 #include "selftests/igt_flush_test.h"
18 #include "selftests/igt_live_test.h"
19 #include "selftests/igt_spinner.h"
20 #include "selftests/lib_sw_fence.h"
21 #include "shmem_utils.h"
22 
23 #include "gem/selftests/igt_gem_utils.h"
24 #include "gem/selftests/mock_context.h"
25 
26 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
27 #define NUM_GPR 16
28 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
29 
30 static struct i915_vma *create_scratch(struct intel_gt *gt)
31 {
32 	return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
33 }
34 
35 static bool is_active(struct i915_request *rq)
36 {
37 	if (i915_request_is_active(rq))
38 		return true;
39 
40 	if (i915_request_on_hold(rq))
41 		return true;
42 
43 	if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
44 		return true;
45 
46 	return false;
47 }
48 
49 static int wait_for_submit(struct intel_engine_cs *engine,
50 			   struct i915_request *rq,
51 			   unsigned long timeout)
52 {
53 	/* Ignore our own attempts to suppress excess tasklets */
54 	tasklet_hi_schedule(&engine->sched_engine->tasklet);
55 
56 	timeout += jiffies;
57 	do {
58 		bool done = time_after(jiffies, timeout);
59 
60 		if (i915_request_completed(rq)) /* that was quick! */
61 			return 0;
62 
63 		/* Wait until the HW has acknowleged the submission (or err) */
64 		intel_engine_flush_submission(engine);
65 		if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
66 			return 0;
67 
68 		if (done)
69 			return -ETIME;
70 
71 		cond_resched();
72 	} while (1);
73 }
74 
75 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
76 {
77 	const u32 offset =
78 		i915_ggtt_offset(ce->engine->status_page.vma) +
79 		offset_in_page(slot);
80 	struct i915_request *rq;
81 	u32 *cs;
82 
83 	rq = intel_context_create_request(ce);
84 	if (IS_ERR(rq))
85 		return PTR_ERR(rq);
86 
87 	cs = intel_ring_begin(rq, 4);
88 	if (IS_ERR(cs)) {
89 		i915_request_add(rq);
90 		return PTR_ERR(cs);
91 	}
92 
93 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
94 	*cs++ = offset;
95 	*cs++ = 0;
96 	*cs++ = 1;
97 
98 	intel_ring_advance(rq, cs);
99 
100 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
101 	i915_request_add(rq);
102 	return 0;
103 }
104 
105 static int context_flush(struct intel_context *ce, long timeout)
106 {
107 	struct i915_request *rq;
108 	struct dma_fence *fence;
109 	int err = 0;
110 
111 	rq = intel_engine_create_kernel_request(ce->engine);
112 	if (IS_ERR(rq))
113 		return PTR_ERR(rq);
114 
115 	fence = i915_active_fence_get(&ce->timeline->last_request);
116 	if (fence) {
117 		i915_request_await_dma_fence(rq, fence);
118 		dma_fence_put(fence);
119 	}
120 
121 	rq = i915_request_get(rq);
122 	i915_request_add(rq);
123 	if (i915_request_wait(rq, 0, timeout) < 0)
124 		err = -ETIME;
125 	i915_request_put(rq);
126 
127 	rmb(); /* We know the request is written, make sure all state is too! */
128 	return err;
129 }
130 
131 static int get_lri_mask(struct intel_engine_cs *engine, u32 lri)
132 {
133 	if ((lri & MI_LRI_LRM_CS_MMIO) == 0)
134 		return ~0u;
135 
136 	if (GRAPHICS_VER(engine->i915) < 12)
137 		return 0xfff;
138 
139 	switch (engine->class) {
140 	default:
141 	case RENDER_CLASS:
142 	case COMPUTE_CLASS:
143 		return 0x07ff;
144 	case COPY_ENGINE_CLASS:
145 		return 0x0fff;
146 	case VIDEO_DECODE_CLASS:
147 	case VIDEO_ENHANCEMENT_CLASS:
148 		return 0x3fff;
149 	}
150 }
151 
152 static int live_lrc_layout(void *arg)
153 {
154 	struct intel_gt *gt = arg;
155 	struct intel_engine_cs *engine;
156 	enum intel_engine_id id;
157 	u32 *lrc;
158 	int err;
159 
160 	/*
161 	 * Check the registers offsets we use to create the initial reg state
162 	 * match the layout saved by HW.
163 	 */
164 
165 	lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
166 	if (!lrc)
167 		return -ENOMEM;
168 	GEM_BUG_ON(offset_in_page(lrc));
169 
170 	err = 0;
171 	for_each_engine(engine, gt, id) {
172 		u32 *hw;
173 		int dw;
174 
175 		if (!engine->default_state)
176 			continue;
177 
178 		hw = shmem_pin_map(engine->default_state);
179 		if (!hw) {
180 			err = -ENOMEM;
181 			break;
182 		}
183 		hw += LRC_STATE_OFFSET / sizeof(*hw);
184 
185 		__lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
186 				engine->kernel_context, engine, true);
187 
188 		dw = 0;
189 		do {
190 			u32 lri = READ_ONCE(hw[dw]);
191 			u32 lri_mask;
192 
193 			if (lri == 0) {
194 				dw++;
195 				continue;
196 			}
197 
198 			if (lrc[dw] == 0) {
199 				pr_debug("%s: skipped instruction %x at dword %d\n",
200 					 engine->name, lri, dw);
201 				dw++;
202 				continue;
203 			}
204 
205 			if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
206 				pr_err("%s: Expected LRI command at dword %d, found %08x\n",
207 				       engine->name, dw, lri);
208 				err = -EINVAL;
209 				break;
210 			}
211 
212 			if (lrc[dw] != lri) {
213 				pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
214 				       engine->name, dw, lri, lrc[dw]);
215 				err = -EINVAL;
216 				break;
217 			}
218 
219 			/*
220 			 * When bit 19 of MI_LOAD_REGISTER_IMM instruction
221 			 * opcode is set on Gen12+ devices, HW does not
222 			 * care about certain register address offsets, and
223 			 * instead check the following for valid address
224 			 * ranges on specific engines:
225 			 * RCS && CCS: BITS(0 - 10)
226 			 * BCS: BITS(0 - 11)
227 			 * VECS && VCS: BITS(0 - 13)
228 			 */
229 			lri_mask = get_lri_mask(engine, lri);
230 
231 			lri &= 0x7f;
232 			lri++;
233 			dw++;
234 
235 			while (lri) {
236 				u32 offset = READ_ONCE(hw[dw]);
237 
238 				if ((offset ^ lrc[dw]) & lri_mask) {
239 					pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
240 					       engine->name, dw, offset, lrc[dw]);
241 					err = -EINVAL;
242 					break;
243 				}
244 
245 				/*
246 				 * Skip over the actual register value as we
247 				 * expect that to differ.
248 				 */
249 				dw += 2;
250 				lri -= 2;
251 			}
252 		} while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
253 
254 		if (err) {
255 			pr_info("%s: HW register image:\n", engine->name);
256 			igt_hexdump(hw, PAGE_SIZE);
257 
258 			pr_info("%s: SW register image:\n", engine->name);
259 			igt_hexdump(lrc, PAGE_SIZE);
260 		}
261 
262 		shmem_unpin_map(engine->default_state, hw);
263 		if (err)
264 			break;
265 	}
266 
267 	free_page((unsigned long)lrc);
268 	return err;
269 }
270 
271 static int find_offset(const u32 *lri, u32 offset)
272 {
273 	int i;
274 
275 	for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
276 		if (lri[i] == offset)
277 			return i;
278 
279 	return -1;
280 }
281 
282 static int live_lrc_fixed(void *arg)
283 {
284 	struct intel_gt *gt = arg;
285 	struct intel_engine_cs *engine;
286 	enum intel_engine_id id;
287 	int err = 0;
288 
289 	/*
290 	 * Check the assumed register offsets match the actual locations in
291 	 * the context image.
292 	 */
293 
294 	for_each_engine(engine, gt, id) {
295 		const struct {
296 			u32 reg;
297 			u32 offset;
298 			const char *name;
299 		} tbl[] = {
300 			{
301 				i915_mmio_reg_offset(RING_START(engine->mmio_base)),
302 				CTX_RING_START - 1,
303 				"RING_START"
304 			},
305 			{
306 				i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
307 				CTX_RING_CTL - 1,
308 				"RING_CTL"
309 			},
310 			{
311 				i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
312 				CTX_RING_HEAD - 1,
313 				"RING_HEAD"
314 			},
315 			{
316 				i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
317 				CTX_RING_TAIL - 1,
318 				"RING_TAIL"
319 			},
320 			{
321 				i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
322 				lrc_ring_mi_mode(engine),
323 				"RING_MI_MODE"
324 			},
325 			{
326 				i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
327 				CTX_BB_STATE - 1,
328 				"BB_STATE"
329 			},
330 			{
331 				i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
332 				lrc_ring_wa_bb_per_ctx(engine),
333 				"RING_BB_PER_CTX_PTR"
334 			},
335 			{
336 				i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
337 				lrc_ring_indirect_ptr(engine),
338 				"RING_INDIRECT_CTX_PTR"
339 			},
340 			{
341 				i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
342 				lrc_ring_indirect_offset(engine),
343 				"RING_INDIRECT_CTX_OFFSET"
344 			},
345 			{
346 				i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
347 				CTX_TIMESTAMP - 1,
348 				"RING_CTX_TIMESTAMP"
349 			},
350 			{
351 				i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
352 				lrc_ring_gpr0(engine),
353 				"RING_CS_GPR0"
354 			},
355 			{
356 				i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
357 				lrc_ring_cmd_buf_cctl(engine),
358 				"RING_CMD_BUF_CCTL"
359 			},
360 			{ },
361 		}, *t;
362 		u32 *hw;
363 
364 		if (!engine->default_state)
365 			continue;
366 
367 		hw = shmem_pin_map(engine->default_state);
368 		if (!hw) {
369 			err = -ENOMEM;
370 			break;
371 		}
372 		hw += LRC_STATE_OFFSET / sizeof(*hw);
373 
374 		for (t = tbl; t->name; t++) {
375 			int dw = find_offset(hw, t->reg);
376 
377 			if (dw != t->offset) {
378 				pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
379 				       engine->name,
380 				       t->name,
381 				       t->reg,
382 				       dw,
383 				       t->offset);
384 				err = -EINVAL;
385 			}
386 		}
387 
388 		shmem_unpin_map(engine->default_state, hw);
389 	}
390 
391 	return err;
392 }
393 
394 static int __live_lrc_state(struct intel_engine_cs *engine,
395 			    struct i915_vma *scratch)
396 {
397 	struct intel_context *ce;
398 	struct i915_request *rq;
399 	struct i915_gem_ww_ctx ww;
400 	enum {
401 		RING_START_IDX = 0,
402 		RING_TAIL_IDX,
403 		MAX_IDX
404 	};
405 	u32 expected[MAX_IDX];
406 	u32 *cs;
407 	int err;
408 	int n;
409 
410 	ce = intel_context_create(engine);
411 	if (IS_ERR(ce))
412 		return PTR_ERR(ce);
413 
414 	i915_gem_ww_ctx_init(&ww, false);
415 retry:
416 	err = i915_gem_object_lock(scratch->obj, &ww);
417 	if (!err)
418 		err = intel_context_pin_ww(ce, &ww);
419 	if (err)
420 		goto err_put;
421 
422 	rq = i915_request_create(ce);
423 	if (IS_ERR(rq)) {
424 		err = PTR_ERR(rq);
425 		goto err_unpin;
426 	}
427 
428 	cs = intel_ring_begin(rq, 4 * MAX_IDX);
429 	if (IS_ERR(cs)) {
430 		err = PTR_ERR(cs);
431 		i915_request_add(rq);
432 		goto err_unpin;
433 	}
434 
435 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
436 	*cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
437 	*cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
438 	*cs++ = 0;
439 
440 	expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
441 
442 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
443 	*cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
444 	*cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
445 	*cs++ = 0;
446 
447 	err = i915_request_await_object(rq, scratch->obj, true);
448 	if (!err)
449 		err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
450 
451 	i915_request_get(rq);
452 	i915_request_add(rq);
453 	if (err)
454 		goto err_rq;
455 
456 	intel_engine_flush_submission(engine);
457 	expected[RING_TAIL_IDX] = ce->ring->tail;
458 
459 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
460 		err = -ETIME;
461 		goto err_rq;
462 	}
463 
464 	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
465 	if (IS_ERR(cs)) {
466 		err = PTR_ERR(cs);
467 		goto err_rq;
468 	}
469 
470 	for (n = 0; n < MAX_IDX; n++) {
471 		if (cs[n] != expected[n]) {
472 			pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
473 			       engine->name, n, cs[n], expected[n]);
474 			err = -EINVAL;
475 			break;
476 		}
477 	}
478 
479 	i915_gem_object_unpin_map(scratch->obj);
480 
481 err_rq:
482 	i915_request_put(rq);
483 err_unpin:
484 	intel_context_unpin(ce);
485 err_put:
486 	if (err == -EDEADLK) {
487 		err = i915_gem_ww_ctx_backoff(&ww);
488 		if (!err)
489 			goto retry;
490 	}
491 	i915_gem_ww_ctx_fini(&ww);
492 	intel_context_put(ce);
493 	return err;
494 }
495 
496 static int live_lrc_state(void *arg)
497 {
498 	struct intel_gt *gt = arg;
499 	struct intel_engine_cs *engine;
500 	struct i915_vma *scratch;
501 	enum intel_engine_id id;
502 	int err = 0;
503 
504 	/*
505 	 * Check the live register state matches what we expect for this
506 	 * intel_context.
507 	 */
508 
509 	scratch = create_scratch(gt);
510 	if (IS_ERR(scratch))
511 		return PTR_ERR(scratch);
512 
513 	for_each_engine(engine, gt, id) {
514 		err = __live_lrc_state(engine, scratch);
515 		if (err)
516 			break;
517 	}
518 
519 	if (igt_flush_test(gt->i915))
520 		err = -EIO;
521 
522 	i915_vma_unpin_and_release(&scratch, 0);
523 	return err;
524 }
525 
526 static int gpr_make_dirty(struct intel_context *ce)
527 {
528 	struct i915_request *rq;
529 	u32 *cs;
530 	int n;
531 
532 	rq = intel_context_create_request(ce);
533 	if (IS_ERR(rq))
534 		return PTR_ERR(rq);
535 
536 	cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
537 	if (IS_ERR(cs)) {
538 		i915_request_add(rq);
539 		return PTR_ERR(cs);
540 	}
541 
542 	*cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
543 	for (n = 0; n < NUM_GPR_DW; n++) {
544 		*cs++ = CS_GPR(ce->engine, n);
545 		*cs++ = STACK_MAGIC;
546 	}
547 	*cs++ = MI_NOOP;
548 
549 	intel_ring_advance(rq, cs);
550 
551 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
552 	i915_request_add(rq);
553 
554 	return 0;
555 }
556 
557 static struct i915_request *
558 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
559 {
560 	const u32 offset =
561 		i915_ggtt_offset(ce->engine->status_page.vma) +
562 		offset_in_page(slot);
563 	struct i915_request *rq;
564 	u32 *cs;
565 	int err;
566 	int n;
567 
568 	rq = intel_context_create_request(ce);
569 	if (IS_ERR(rq))
570 		return rq;
571 
572 	cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
573 	if (IS_ERR(cs)) {
574 		i915_request_add(rq);
575 		return ERR_CAST(cs);
576 	}
577 
578 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
579 	*cs++ = MI_NOOP;
580 
581 	*cs++ = MI_SEMAPHORE_WAIT |
582 		MI_SEMAPHORE_GLOBAL_GTT |
583 		MI_SEMAPHORE_POLL |
584 		MI_SEMAPHORE_SAD_NEQ_SDD;
585 	*cs++ = 0;
586 	*cs++ = offset;
587 	*cs++ = 0;
588 
589 	for (n = 0; n < NUM_GPR_DW; n++) {
590 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
591 		*cs++ = CS_GPR(ce->engine, n);
592 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
593 		*cs++ = 0;
594 	}
595 
596 	i915_vma_lock(scratch);
597 	err = i915_request_await_object(rq, scratch->obj, true);
598 	if (!err)
599 		err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
600 	i915_vma_unlock(scratch);
601 
602 	i915_request_get(rq);
603 	i915_request_add(rq);
604 	if (err) {
605 		i915_request_put(rq);
606 		rq = ERR_PTR(err);
607 	}
608 
609 	return rq;
610 }
611 
612 static int __live_lrc_gpr(struct intel_engine_cs *engine,
613 			  struct i915_vma *scratch,
614 			  bool preempt)
615 {
616 	u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
617 	struct intel_context *ce;
618 	struct i915_request *rq;
619 	u32 *cs;
620 	int err;
621 	int n;
622 
623 	if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
624 		return 0; /* GPR only on rcs0 for gen8 */
625 
626 	err = gpr_make_dirty(engine->kernel_context);
627 	if (err)
628 		return err;
629 
630 	ce = intel_context_create(engine);
631 	if (IS_ERR(ce))
632 		return PTR_ERR(ce);
633 
634 	rq = __gpr_read(ce, scratch, slot);
635 	if (IS_ERR(rq)) {
636 		err = PTR_ERR(rq);
637 		goto err_put;
638 	}
639 
640 	err = wait_for_submit(engine, rq, HZ / 2);
641 	if (err)
642 		goto err_rq;
643 
644 	if (preempt) {
645 		err = gpr_make_dirty(engine->kernel_context);
646 		if (err)
647 			goto err_rq;
648 
649 		err = emit_semaphore_signal(engine->kernel_context, slot);
650 		if (err)
651 			goto err_rq;
652 
653 		err = wait_for_submit(engine, rq, HZ / 2);
654 		if (err)
655 			goto err_rq;
656 	} else {
657 		slot[0] = 1;
658 		wmb();
659 	}
660 
661 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
662 		err = -ETIME;
663 		goto err_rq;
664 	}
665 
666 	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
667 	if (IS_ERR(cs)) {
668 		err = PTR_ERR(cs);
669 		goto err_rq;
670 	}
671 
672 	for (n = 0; n < NUM_GPR_DW; n++) {
673 		if (cs[n]) {
674 			pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
675 			       engine->name,
676 			       n / 2, n & 1 ? "udw" : "ldw",
677 			       cs[n]);
678 			err = -EINVAL;
679 			break;
680 		}
681 	}
682 
683 	i915_gem_object_unpin_map(scratch->obj);
684 
685 err_rq:
686 	memset32(&slot[0], -1, 4);
687 	wmb();
688 	i915_request_put(rq);
689 err_put:
690 	intel_context_put(ce);
691 	return err;
692 }
693 
694 static int live_lrc_gpr(void *arg)
695 {
696 	struct intel_gt *gt = arg;
697 	struct intel_engine_cs *engine;
698 	struct i915_vma *scratch;
699 	enum intel_engine_id id;
700 	int err = 0;
701 
702 	/*
703 	 * Check that GPR registers are cleared in new contexts as we need
704 	 * to avoid leaking any information from previous contexts.
705 	 */
706 
707 	scratch = create_scratch(gt);
708 	if (IS_ERR(scratch))
709 		return PTR_ERR(scratch);
710 
711 	for_each_engine(engine, gt, id) {
712 		st_engine_heartbeat_disable(engine);
713 
714 		err = __live_lrc_gpr(engine, scratch, false);
715 		if (err)
716 			goto err;
717 
718 		err = __live_lrc_gpr(engine, scratch, true);
719 		if (err)
720 			goto err;
721 
722 err:
723 		st_engine_heartbeat_enable(engine);
724 		if (igt_flush_test(gt->i915))
725 			err = -EIO;
726 		if (err)
727 			break;
728 	}
729 
730 	i915_vma_unpin_and_release(&scratch, 0);
731 	return err;
732 }
733 
734 static struct i915_request *
735 create_timestamp(struct intel_context *ce, void *slot, int idx)
736 {
737 	const u32 offset =
738 		i915_ggtt_offset(ce->engine->status_page.vma) +
739 		offset_in_page(slot);
740 	struct i915_request *rq;
741 	u32 *cs;
742 	int err;
743 
744 	rq = intel_context_create_request(ce);
745 	if (IS_ERR(rq))
746 		return rq;
747 
748 	cs = intel_ring_begin(rq, 10);
749 	if (IS_ERR(cs)) {
750 		err = PTR_ERR(cs);
751 		goto err;
752 	}
753 
754 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
755 	*cs++ = MI_NOOP;
756 
757 	*cs++ = MI_SEMAPHORE_WAIT |
758 		MI_SEMAPHORE_GLOBAL_GTT |
759 		MI_SEMAPHORE_POLL |
760 		MI_SEMAPHORE_SAD_NEQ_SDD;
761 	*cs++ = 0;
762 	*cs++ = offset;
763 	*cs++ = 0;
764 
765 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
766 	*cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
767 	*cs++ = offset + idx * sizeof(u32);
768 	*cs++ = 0;
769 
770 	intel_ring_advance(rq, cs);
771 
772 	err = 0;
773 err:
774 	i915_request_get(rq);
775 	i915_request_add(rq);
776 	if (err) {
777 		i915_request_put(rq);
778 		return ERR_PTR(err);
779 	}
780 
781 	return rq;
782 }
783 
784 struct lrc_timestamp {
785 	struct intel_engine_cs *engine;
786 	struct intel_context *ce[2];
787 	u32 poison;
788 };
789 
790 static bool timestamp_advanced(u32 start, u32 end)
791 {
792 	return (s32)(end - start) > 0;
793 }
794 
795 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
796 {
797 	u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
798 	struct i915_request *rq;
799 	u32 timestamp;
800 	int err = 0;
801 
802 	arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
803 	rq = create_timestamp(arg->ce[0], slot, 1);
804 	if (IS_ERR(rq))
805 		return PTR_ERR(rq);
806 
807 	err = wait_for_submit(rq->engine, rq, HZ / 2);
808 	if (err)
809 		goto err;
810 
811 	if (preempt) {
812 		arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
813 		err = emit_semaphore_signal(arg->ce[1], slot);
814 		if (err)
815 			goto err;
816 	} else {
817 		slot[0] = 1;
818 		wmb();
819 	}
820 
821 	/* And wait for switch to kernel (to save our context to memory) */
822 	err = context_flush(arg->ce[0], HZ / 2);
823 	if (err)
824 		goto err;
825 
826 	if (!timestamp_advanced(arg->poison, slot[1])) {
827 		pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
828 		       arg->engine->name, preempt ? "preempt" : "simple",
829 		       arg->poison, slot[1]);
830 		err = -EINVAL;
831 	}
832 
833 	timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
834 	if (!timestamp_advanced(slot[1], timestamp)) {
835 		pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
836 		       arg->engine->name, preempt ? "preempt" : "simple",
837 		       slot[1], timestamp);
838 		err = -EINVAL;
839 	}
840 
841 err:
842 	memset32(slot, -1, 4);
843 	i915_request_put(rq);
844 	return err;
845 }
846 
847 static int live_lrc_timestamp(void *arg)
848 {
849 	struct lrc_timestamp data = {};
850 	struct intel_gt *gt = arg;
851 	enum intel_engine_id id;
852 	const u32 poison[] = {
853 		0,
854 		S32_MAX,
855 		(u32)S32_MAX + 1,
856 		U32_MAX,
857 	};
858 
859 	/*
860 	 * We want to verify that the timestamp is saved and restore across
861 	 * context switches and is monotonic.
862 	 *
863 	 * So we do this with a little bit of LRC poisoning to check various
864 	 * boundary conditions, and see what happens if we preempt the context
865 	 * with a second request (carrying more poison into the timestamp).
866 	 */
867 
868 	for_each_engine(data.engine, gt, id) {
869 		int i, err = 0;
870 
871 		st_engine_heartbeat_disable(data.engine);
872 
873 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
874 			struct intel_context *tmp;
875 
876 			tmp = intel_context_create(data.engine);
877 			if (IS_ERR(tmp)) {
878 				err = PTR_ERR(tmp);
879 				goto err;
880 			}
881 
882 			err = intel_context_pin(tmp);
883 			if (err) {
884 				intel_context_put(tmp);
885 				goto err;
886 			}
887 
888 			data.ce[i] = tmp;
889 		}
890 
891 		for (i = 0; i < ARRAY_SIZE(poison); i++) {
892 			data.poison = poison[i];
893 
894 			err = __lrc_timestamp(&data, false);
895 			if (err)
896 				break;
897 
898 			err = __lrc_timestamp(&data, true);
899 			if (err)
900 				break;
901 		}
902 
903 err:
904 		st_engine_heartbeat_enable(data.engine);
905 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
906 			if (!data.ce[i])
907 				break;
908 
909 			intel_context_unpin(data.ce[i]);
910 			intel_context_put(data.ce[i]);
911 		}
912 
913 		if (igt_flush_test(gt->i915))
914 			err = -EIO;
915 		if (err)
916 			return err;
917 	}
918 
919 	return 0;
920 }
921 
922 static struct i915_vma *
923 create_user_vma(struct i915_address_space *vm, unsigned long size)
924 {
925 	struct drm_i915_gem_object *obj;
926 	struct i915_vma *vma;
927 	int err;
928 
929 	obj = i915_gem_object_create_internal(vm->i915, size);
930 	if (IS_ERR(obj))
931 		return ERR_CAST(obj);
932 
933 	vma = i915_vma_instance(obj, vm, NULL);
934 	if (IS_ERR(vma)) {
935 		i915_gem_object_put(obj);
936 		return vma;
937 	}
938 
939 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
940 	if (err) {
941 		i915_gem_object_put(obj);
942 		return ERR_PTR(err);
943 	}
944 
945 	return vma;
946 }
947 
948 static u32 safe_poison(u32 offset, u32 poison)
949 {
950 	/*
951 	 * Do not enable predication as it will nop all subsequent commands,
952 	 * not only disabling the tests (by preventing all the other SRM) but
953 	 * also preventing the arbitration events at the end of the request.
954 	 */
955 	if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)))
956 		poison &= ~REG_BIT(0);
957 
958 	return poison;
959 }
960 
961 static struct i915_vma *
962 store_context(struct intel_context *ce, struct i915_vma *scratch)
963 {
964 	struct i915_vma *batch;
965 	u32 dw, x, *cs, *hw;
966 	u32 *defaults;
967 
968 	batch = create_user_vma(ce->vm, SZ_64K);
969 	if (IS_ERR(batch))
970 		return batch;
971 
972 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
973 	if (IS_ERR(cs)) {
974 		i915_vma_put(batch);
975 		return ERR_CAST(cs);
976 	}
977 
978 	defaults = shmem_pin_map(ce->engine->default_state);
979 	if (!defaults) {
980 		i915_gem_object_unpin_map(batch->obj);
981 		i915_vma_put(batch);
982 		return ERR_PTR(-ENOMEM);
983 	}
984 
985 	x = 0;
986 	dw = 0;
987 	hw = defaults;
988 	hw += LRC_STATE_OFFSET / sizeof(*hw);
989 	do {
990 		u32 len = hw[dw] & 0x7f;
991 
992 		if (hw[dw] == 0) {
993 			dw++;
994 			continue;
995 		}
996 
997 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
998 			dw += len + 2;
999 			continue;
1000 		}
1001 
1002 		dw++;
1003 		len = (len + 1) / 2;
1004 		while (len--) {
1005 			*cs++ = MI_STORE_REGISTER_MEM_GEN8;
1006 			*cs++ = hw[dw];
1007 			*cs++ = lower_32_bits(scratch->node.start + x);
1008 			*cs++ = upper_32_bits(scratch->node.start + x);
1009 
1010 			dw += 2;
1011 			x += 4;
1012 		}
1013 	} while (dw < PAGE_SIZE / sizeof(u32) &&
1014 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1015 
1016 	*cs++ = MI_BATCH_BUFFER_END;
1017 
1018 	shmem_unpin_map(ce->engine->default_state, defaults);
1019 
1020 	i915_gem_object_flush_map(batch->obj);
1021 	i915_gem_object_unpin_map(batch->obj);
1022 
1023 	return batch;
1024 }
1025 
1026 static int move_to_active(struct i915_request *rq,
1027 			  struct i915_vma *vma,
1028 			  unsigned int flags)
1029 {
1030 	int err;
1031 
1032 	i915_vma_lock(vma);
1033 	err = i915_request_await_object(rq, vma->obj, flags);
1034 	if (!err)
1035 		err = i915_vma_move_to_active(vma, rq, flags);
1036 	i915_vma_unlock(vma);
1037 
1038 	return err;
1039 }
1040 
1041 static struct i915_request *
1042 record_registers(struct intel_context *ce,
1043 		 struct i915_vma *before,
1044 		 struct i915_vma *after,
1045 		 u32 *sema)
1046 {
1047 	struct i915_vma *b_before, *b_after;
1048 	struct i915_request *rq;
1049 	u32 *cs;
1050 	int err;
1051 
1052 	b_before = store_context(ce, before);
1053 	if (IS_ERR(b_before))
1054 		return ERR_CAST(b_before);
1055 
1056 	b_after = store_context(ce, after);
1057 	if (IS_ERR(b_after)) {
1058 		rq = ERR_CAST(b_after);
1059 		goto err_before;
1060 	}
1061 
1062 	rq = intel_context_create_request(ce);
1063 	if (IS_ERR(rq))
1064 		goto err_after;
1065 
1066 	err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1067 	if (err)
1068 		goto err_rq;
1069 
1070 	err = move_to_active(rq, b_before, 0);
1071 	if (err)
1072 		goto err_rq;
1073 
1074 	err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1075 	if (err)
1076 		goto err_rq;
1077 
1078 	err = move_to_active(rq, b_after, 0);
1079 	if (err)
1080 		goto err_rq;
1081 
1082 	cs = intel_ring_begin(rq, 14);
1083 	if (IS_ERR(cs)) {
1084 		err = PTR_ERR(cs);
1085 		goto err_rq;
1086 	}
1087 
1088 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1089 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1090 	*cs++ = lower_32_bits(b_before->node.start);
1091 	*cs++ = upper_32_bits(b_before->node.start);
1092 
1093 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1094 	*cs++ = MI_SEMAPHORE_WAIT |
1095 		MI_SEMAPHORE_GLOBAL_GTT |
1096 		MI_SEMAPHORE_POLL |
1097 		MI_SEMAPHORE_SAD_NEQ_SDD;
1098 	*cs++ = 0;
1099 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1100 		offset_in_page(sema);
1101 	*cs++ = 0;
1102 	*cs++ = MI_NOOP;
1103 
1104 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1105 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1106 	*cs++ = lower_32_bits(b_after->node.start);
1107 	*cs++ = upper_32_bits(b_after->node.start);
1108 
1109 	intel_ring_advance(rq, cs);
1110 
1111 	WRITE_ONCE(*sema, 0);
1112 	i915_request_get(rq);
1113 	i915_request_add(rq);
1114 err_after:
1115 	i915_vma_put(b_after);
1116 err_before:
1117 	i915_vma_put(b_before);
1118 	return rq;
1119 
1120 err_rq:
1121 	i915_request_add(rq);
1122 	rq = ERR_PTR(err);
1123 	goto err_after;
1124 }
1125 
1126 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1127 {
1128 	struct i915_vma *batch;
1129 	u32 dw, *cs, *hw;
1130 	u32 *defaults;
1131 
1132 	batch = create_user_vma(ce->vm, SZ_64K);
1133 	if (IS_ERR(batch))
1134 		return batch;
1135 
1136 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1137 	if (IS_ERR(cs)) {
1138 		i915_vma_put(batch);
1139 		return ERR_CAST(cs);
1140 	}
1141 
1142 	defaults = shmem_pin_map(ce->engine->default_state);
1143 	if (!defaults) {
1144 		i915_gem_object_unpin_map(batch->obj);
1145 		i915_vma_put(batch);
1146 		return ERR_PTR(-ENOMEM);
1147 	}
1148 
1149 	dw = 0;
1150 	hw = defaults;
1151 	hw += LRC_STATE_OFFSET / sizeof(*hw);
1152 	do {
1153 		u32 len = hw[dw] & 0x7f;
1154 
1155 		if (hw[dw] == 0) {
1156 			dw++;
1157 			continue;
1158 		}
1159 
1160 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1161 			dw += len + 2;
1162 			continue;
1163 		}
1164 
1165 		dw++;
1166 		len = (len + 1) / 2;
1167 		*cs++ = MI_LOAD_REGISTER_IMM(len);
1168 		while (len--) {
1169 			*cs++ = hw[dw];
1170 			*cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,
1171 								  MI_LRI_LRM_CS_MMIO),
1172 					    poison);
1173 			dw += 2;
1174 		}
1175 	} while (dw < PAGE_SIZE / sizeof(u32) &&
1176 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1177 
1178 	*cs++ = MI_BATCH_BUFFER_END;
1179 
1180 	shmem_unpin_map(ce->engine->default_state, defaults);
1181 
1182 	i915_gem_object_flush_map(batch->obj);
1183 	i915_gem_object_unpin_map(batch->obj);
1184 
1185 	return batch;
1186 }
1187 
1188 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1189 {
1190 	struct i915_request *rq;
1191 	struct i915_vma *batch;
1192 	u32 *cs;
1193 	int err;
1194 
1195 	batch = load_context(ce, poison);
1196 	if (IS_ERR(batch))
1197 		return PTR_ERR(batch);
1198 
1199 	rq = intel_context_create_request(ce);
1200 	if (IS_ERR(rq)) {
1201 		err = PTR_ERR(rq);
1202 		goto err_batch;
1203 	}
1204 
1205 	err = move_to_active(rq, batch, 0);
1206 	if (err)
1207 		goto err_rq;
1208 
1209 	cs = intel_ring_begin(rq, 8);
1210 	if (IS_ERR(cs)) {
1211 		err = PTR_ERR(cs);
1212 		goto err_rq;
1213 	}
1214 
1215 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1216 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1217 	*cs++ = lower_32_bits(batch->node.start);
1218 	*cs++ = upper_32_bits(batch->node.start);
1219 
1220 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1221 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1222 		offset_in_page(sema);
1223 	*cs++ = 0;
1224 	*cs++ = 1;
1225 
1226 	intel_ring_advance(rq, cs);
1227 
1228 	rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1229 err_rq:
1230 	i915_request_add(rq);
1231 err_batch:
1232 	i915_vma_put(batch);
1233 	return err;
1234 }
1235 
1236 static bool is_moving(u32 a, u32 b)
1237 {
1238 	return a != b;
1239 }
1240 
1241 static int compare_isolation(struct intel_engine_cs *engine,
1242 			     struct i915_vma *ref[2],
1243 			     struct i915_vma *result[2],
1244 			     struct intel_context *ce,
1245 			     u32 poison)
1246 {
1247 	u32 x, dw, *hw, *lrc;
1248 	u32 *A[2], *B[2];
1249 	u32 *defaults;
1250 	int err = 0;
1251 
1252 	A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1253 	if (IS_ERR(A[0]))
1254 		return PTR_ERR(A[0]);
1255 
1256 	A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1257 	if (IS_ERR(A[1])) {
1258 		err = PTR_ERR(A[1]);
1259 		goto err_A0;
1260 	}
1261 
1262 	B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1263 	if (IS_ERR(B[0])) {
1264 		err = PTR_ERR(B[0]);
1265 		goto err_A1;
1266 	}
1267 
1268 	B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1269 	if (IS_ERR(B[1])) {
1270 		err = PTR_ERR(B[1]);
1271 		goto err_B0;
1272 	}
1273 
1274 	lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1275 					       i915_coherent_map_type(engine->i915,
1276 								      ce->state->obj,
1277 								      false));
1278 	if (IS_ERR(lrc)) {
1279 		err = PTR_ERR(lrc);
1280 		goto err_B1;
1281 	}
1282 	lrc += LRC_STATE_OFFSET / sizeof(*hw);
1283 
1284 	defaults = shmem_pin_map(ce->engine->default_state);
1285 	if (!defaults) {
1286 		err = -ENOMEM;
1287 		goto err_lrc;
1288 	}
1289 
1290 	x = 0;
1291 	dw = 0;
1292 	hw = defaults;
1293 	hw += LRC_STATE_OFFSET / sizeof(*hw);
1294 	do {
1295 		u32 len = hw[dw] & 0x7f;
1296 
1297 		if (hw[dw] == 0) {
1298 			dw++;
1299 			continue;
1300 		}
1301 
1302 		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1303 			dw += len + 2;
1304 			continue;
1305 		}
1306 
1307 		dw++;
1308 		len = (len + 1) / 2;
1309 		while (len--) {
1310 			if (!is_moving(A[0][x], A[1][x]) &&
1311 			    (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1312 				switch (hw[dw] & 4095) {
1313 				case 0x30: /* RING_HEAD */
1314 				case 0x34: /* RING_TAIL */
1315 					break;
1316 
1317 				default:
1318 					pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1319 					       engine->name, dw,
1320 					       hw[dw], hw[dw + 1],
1321 					       A[0][x], B[0][x], B[1][x],
1322 					       poison, lrc[dw + 1]);
1323 					err = -EINVAL;
1324 				}
1325 			}
1326 			dw += 2;
1327 			x++;
1328 		}
1329 	} while (dw < PAGE_SIZE / sizeof(u32) &&
1330 		 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1331 
1332 	shmem_unpin_map(ce->engine->default_state, defaults);
1333 err_lrc:
1334 	i915_gem_object_unpin_map(ce->state->obj);
1335 err_B1:
1336 	i915_gem_object_unpin_map(result[1]->obj);
1337 err_B0:
1338 	i915_gem_object_unpin_map(result[0]->obj);
1339 err_A1:
1340 	i915_gem_object_unpin_map(ref[1]->obj);
1341 err_A0:
1342 	i915_gem_object_unpin_map(ref[0]->obj);
1343 	return err;
1344 }
1345 
1346 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1347 {
1348 	u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1349 	struct i915_vma *ref[2], *result[2];
1350 	struct intel_context *A, *B;
1351 	struct i915_request *rq;
1352 	int err;
1353 
1354 	A = intel_context_create(engine);
1355 	if (IS_ERR(A))
1356 		return PTR_ERR(A);
1357 
1358 	B = intel_context_create(engine);
1359 	if (IS_ERR(B)) {
1360 		err = PTR_ERR(B);
1361 		goto err_A;
1362 	}
1363 
1364 	ref[0] = create_user_vma(A->vm, SZ_64K);
1365 	if (IS_ERR(ref[0])) {
1366 		err = PTR_ERR(ref[0]);
1367 		goto err_B;
1368 	}
1369 
1370 	ref[1] = create_user_vma(A->vm, SZ_64K);
1371 	if (IS_ERR(ref[1])) {
1372 		err = PTR_ERR(ref[1]);
1373 		goto err_ref0;
1374 	}
1375 
1376 	rq = record_registers(A, ref[0], ref[1], sema);
1377 	if (IS_ERR(rq)) {
1378 		err = PTR_ERR(rq);
1379 		goto err_ref1;
1380 	}
1381 
1382 	WRITE_ONCE(*sema, 1);
1383 	wmb();
1384 
1385 	if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1386 		i915_request_put(rq);
1387 		err = -ETIME;
1388 		goto err_ref1;
1389 	}
1390 	i915_request_put(rq);
1391 
1392 	result[0] = create_user_vma(A->vm, SZ_64K);
1393 	if (IS_ERR(result[0])) {
1394 		err = PTR_ERR(result[0]);
1395 		goto err_ref1;
1396 	}
1397 
1398 	result[1] = create_user_vma(A->vm, SZ_64K);
1399 	if (IS_ERR(result[1])) {
1400 		err = PTR_ERR(result[1]);
1401 		goto err_result0;
1402 	}
1403 
1404 	rq = record_registers(A, result[0], result[1], sema);
1405 	if (IS_ERR(rq)) {
1406 		err = PTR_ERR(rq);
1407 		goto err_result1;
1408 	}
1409 
1410 	err = poison_registers(B, poison, sema);
1411 	if (err) {
1412 		WRITE_ONCE(*sema, -1);
1413 		i915_request_put(rq);
1414 		goto err_result1;
1415 	}
1416 
1417 	if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1418 		i915_request_put(rq);
1419 		err = -ETIME;
1420 		goto err_result1;
1421 	}
1422 	i915_request_put(rq);
1423 
1424 	err = compare_isolation(engine, ref, result, A, poison);
1425 
1426 err_result1:
1427 	i915_vma_put(result[1]);
1428 err_result0:
1429 	i915_vma_put(result[0]);
1430 err_ref1:
1431 	i915_vma_put(ref[1]);
1432 err_ref0:
1433 	i915_vma_put(ref[0]);
1434 err_B:
1435 	intel_context_put(B);
1436 err_A:
1437 	intel_context_put(A);
1438 	return err;
1439 }
1440 
1441 static bool skip_isolation(const struct intel_engine_cs *engine)
1442 {
1443 	if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1444 		return true;
1445 
1446 	if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1447 		return true;
1448 
1449 	return false;
1450 }
1451 
1452 static int live_lrc_isolation(void *arg)
1453 {
1454 	struct intel_gt *gt = arg;
1455 	struct intel_engine_cs *engine;
1456 	enum intel_engine_id id;
1457 	const u32 poison[] = {
1458 		STACK_MAGIC,
1459 		0x3a3a3a3a,
1460 		0x5c5c5c5c,
1461 		0xffffffff,
1462 		0xffff0000,
1463 	};
1464 	int err = 0;
1465 
1466 	/*
1467 	 * Our goal is try and verify that per-context state cannot be
1468 	 * tampered with by another non-privileged client.
1469 	 *
1470 	 * We take the list of context registers from the LRI in the default
1471 	 * context image and attempt to modify that list from a remote context.
1472 	 */
1473 
1474 	for_each_engine(engine, gt, id) {
1475 		int i;
1476 
1477 		/* Just don't even ask */
1478 		if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1479 		    skip_isolation(engine))
1480 			continue;
1481 
1482 		intel_engine_pm_get(engine);
1483 		for (i = 0; i < ARRAY_SIZE(poison); i++) {
1484 			int result;
1485 
1486 			result = __lrc_isolation(engine, poison[i]);
1487 			if (result && !err)
1488 				err = result;
1489 
1490 			result = __lrc_isolation(engine, ~poison[i]);
1491 			if (result && !err)
1492 				err = result;
1493 		}
1494 		intel_engine_pm_put(engine);
1495 		if (igt_flush_test(gt->i915)) {
1496 			err = -EIO;
1497 			break;
1498 		}
1499 	}
1500 
1501 	return err;
1502 }
1503 
1504 static int indirect_ctx_submit_req(struct intel_context *ce)
1505 {
1506 	struct i915_request *rq;
1507 	int err = 0;
1508 
1509 	rq = intel_context_create_request(ce);
1510 	if (IS_ERR(rq))
1511 		return PTR_ERR(rq);
1512 
1513 	i915_request_get(rq);
1514 	i915_request_add(rq);
1515 
1516 	if (i915_request_wait(rq, 0, HZ / 5) < 0)
1517 		err = -ETIME;
1518 
1519 	i915_request_put(rq);
1520 
1521 	return err;
1522 }
1523 
1524 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1525 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1526 
1527 static u32 *
1528 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1529 {
1530 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1531 		MI_SRM_LRM_GLOBAL_GTT |
1532 		MI_LRI_LRM_CS_MMIO;
1533 	*cs++ = i915_mmio_reg_offset(RING_START(0));
1534 	*cs++ = i915_ggtt_offset(ce->state) +
1535 		context_wa_bb_offset(ce) +
1536 		CTX_BB_CANARY_OFFSET;
1537 	*cs++ = 0;
1538 
1539 	return cs;
1540 }
1541 
1542 static void
1543 indirect_ctx_bb_setup(struct intel_context *ce)
1544 {
1545 	u32 *cs = context_indirect_bb(ce);
1546 
1547 	cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1548 
1549 	setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1550 }
1551 
1552 static bool check_ring_start(struct intel_context *ce)
1553 {
1554 	const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1555 		LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1556 
1557 	if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1558 		return true;
1559 
1560 	pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1561 	       ctx_bb[CTX_BB_CANARY_INDEX],
1562 	       ce->lrc_reg_state[CTX_RING_START]);
1563 
1564 	return false;
1565 }
1566 
1567 static int indirect_ctx_bb_check(struct intel_context *ce)
1568 {
1569 	int err;
1570 
1571 	err = indirect_ctx_submit_req(ce);
1572 	if (err)
1573 		return err;
1574 
1575 	if (!check_ring_start(ce))
1576 		return -EINVAL;
1577 
1578 	return 0;
1579 }
1580 
1581 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1582 {
1583 	struct intel_context *a, *b;
1584 	int err;
1585 
1586 	a = intel_context_create(engine);
1587 	if (IS_ERR(a))
1588 		return PTR_ERR(a);
1589 	err = intel_context_pin(a);
1590 	if (err)
1591 		goto put_a;
1592 
1593 	b = intel_context_create(engine);
1594 	if (IS_ERR(b)) {
1595 		err = PTR_ERR(b);
1596 		goto unpin_a;
1597 	}
1598 	err = intel_context_pin(b);
1599 	if (err)
1600 		goto put_b;
1601 
1602 	/* We use the already reserved extra page in context state */
1603 	if (!a->wa_bb_page) {
1604 		GEM_BUG_ON(b->wa_bb_page);
1605 		GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1606 		goto unpin_b;
1607 	}
1608 
1609 	/*
1610 	 * In order to test that our per context bb is truly per context,
1611 	 * and executes at the intended spot on context restoring process,
1612 	 * make the batch store the ring start value to memory.
1613 	 * As ring start is restored apriori of starting the indirect ctx bb and
1614 	 * as it will be different for each context, it fits to this purpose.
1615 	 */
1616 	indirect_ctx_bb_setup(a);
1617 	indirect_ctx_bb_setup(b);
1618 
1619 	err = indirect_ctx_bb_check(a);
1620 	if (err)
1621 		goto unpin_b;
1622 
1623 	err = indirect_ctx_bb_check(b);
1624 
1625 unpin_b:
1626 	intel_context_unpin(b);
1627 put_b:
1628 	intel_context_put(b);
1629 unpin_a:
1630 	intel_context_unpin(a);
1631 put_a:
1632 	intel_context_put(a);
1633 
1634 	return err;
1635 }
1636 
1637 static int live_lrc_indirect_ctx_bb(void *arg)
1638 {
1639 	struct intel_gt *gt = arg;
1640 	struct intel_engine_cs *engine;
1641 	enum intel_engine_id id;
1642 	int err = 0;
1643 
1644 	for_each_engine(engine, gt, id) {
1645 		intel_engine_pm_get(engine);
1646 		err = __live_lrc_indirect_ctx_bb(engine);
1647 		intel_engine_pm_put(engine);
1648 
1649 		if (igt_flush_test(gt->i915))
1650 			err = -EIO;
1651 
1652 		if (err)
1653 			break;
1654 	}
1655 
1656 	return err;
1657 }
1658 
1659 static void garbage_reset(struct intel_engine_cs *engine,
1660 			  struct i915_request *rq)
1661 {
1662 	const unsigned int bit = I915_RESET_ENGINE + engine->id;
1663 	unsigned long *lock = &engine->gt->reset.flags;
1664 
1665 	local_bh_disable();
1666 	if (!test_and_set_bit(bit, lock)) {
1667 		tasklet_disable(&engine->sched_engine->tasklet);
1668 
1669 		if (!rq->fence.error)
1670 			__intel_engine_reset_bh(engine, NULL);
1671 
1672 		tasklet_enable(&engine->sched_engine->tasklet);
1673 		clear_and_wake_up_bit(bit, lock);
1674 	}
1675 	local_bh_enable();
1676 }
1677 
1678 static struct i915_request *garbage(struct intel_context *ce,
1679 				    struct rnd_state *prng)
1680 {
1681 	struct i915_request *rq;
1682 	int err;
1683 
1684 	err = intel_context_pin(ce);
1685 	if (err)
1686 		return ERR_PTR(err);
1687 
1688 	prandom_bytes_state(prng,
1689 			    ce->lrc_reg_state,
1690 			    ce->engine->context_size -
1691 			    LRC_STATE_OFFSET);
1692 
1693 	rq = intel_context_create_request(ce);
1694 	if (IS_ERR(rq)) {
1695 		err = PTR_ERR(rq);
1696 		goto err_unpin;
1697 	}
1698 
1699 	i915_request_get(rq);
1700 	i915_request_add(rq);
1701 	return rq;
1702 
1703 err_unpin:
1704 	intel_context_unpin(ce);
1705 	return ERR_PTR(err);
1706 }
1707 
1708 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1709 {
1710 	struct intel_context *ce;
1711 	struct i915_request *hang;
1712 	int err = 0;
1713 
1714 	ce = intel_context_create(engine);
1715 	if (IS_ERR(ce))
1716 		return PTR_ERR(ce);
1717 
1718 	hang = garbage(ce, prng);
1719 	if (IS_ERR(hang)) {
1720 		err = PTR_ERR(hang);
1721 		goto err_ce;
1722 	}
1723 
1724 	if (wait_for_submit(engine, hang, HZ / 2)) {
1725 		i915_request_put(hang);
1726 		err = -ETIME;
1727 		goto err_ce;
1728 	}
1729 
1730 	intel_context_set_banned(ce);
1731 	garbage_reset(engine, hang);
1732 
1733 	intel_engine_flush_submission(engine);
1734 	if (!hang->fence.error) {
1735 		i915_request_put(hang);
1736 		pr_err("%s: corrupted context was not reset\n",
1737 		       engine->name);
1738 		err = -EINVAL;
1739 		goto err_ce;
1740 	}
1741 
1742 	if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1743 		pr_err("%s: corrupted context did not recover\n",
1744 		       engine->name);
1745 		i915_request_put(hang);
1746 		err = -EIO;
1747 		goto err_ce;
1748 	}
1749 	i915_request_put(hang);
1750 
1751 err_ce:
1752 	intel_context_put(ce);
1753 	return err;
1754 }
1755 
1756 static int live_lrc_garbage(void *arg)
1757 {
1758 	struct intel_gt *gt = arg;
1759 	struct intel_engine_cs *engine;
1760 	enum intel_engine_id id;
1761 
1762 	/*
1763 	 * Verify that we can recover if one context state is completely
1764 	 * corrupted.
1765 	 */
1766 
1767 	if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1768 		return 0;
1769 
1770 	for_each_engine(engine, gt, id) {
1771 		I915_RND_STATE(prng);
1772 		int err = 0, i;
1773 
1774 		if (!intel_has_reset_engine(engine->gt))
1775 			continue;
1776 
1777 		intel_engine_pm_get(engine);
1778 		for (i = 0; i < 3; i++) {
1779 			err = __lrc_garbage(engine, &prng);
1780 			if (err)
1781 				break;
1782 		}
1783 		intel_engine_pm_put(engine);
1784 
1785 		if (igt_flush_test(gt->i915))
1786 			err = -EIO;
1787 		if (err)
1788 			return err;
1789 	}
1790 
1791 	return 0;
1792 }
1793 
1794 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1795 {
1796 	struct intel_context *ce;
1797 	struct i915_request *rq;
1798 	IGT_TIMEOUT(end_time);
1799 	int err;
1800 
1801 	ce = intel_context_create(engine);
1802 	if (IS_ERR(ce))
1803 		return PTR_ERR(ce);
1804 
1805 	ce->stats.runtime.num_underflow = 0;
1806 	ce->stats.runtime.max_underflow = 0;
1807 
1808 	do {
1809 		unsigned int loop = 1024;
1810 
1811 		while (loop) {
1812 			rq = intel_context_create_request(ce);
1813 			if (IS_ERR(rq)) {
1814 				err = PTR_ERR(rq);
1815 				goto err_rq;
1816 			}
1817 
1818 			if (--loop == 0)
1819 				i915_request_get(rq);
1820 
1821 			i915_request_add(rq);
1822 		}
1823 
1824 		if (__igt_timeout(end_time, NULL))
1825 			break;
1826 
1827 		i915_request_put(rq);
1828 	} while (1);
1829 
1830 	err = i915_request_wait(rq, 0, HZ / 5);
1831 	if (err < 0) {
1832 		pr_err("%s: request not completed!\n", engine->name);
1833 		goto err_wait;
1834 	}
1835 
1836 	igt_flush_test(engine->i915);
1837 
1838 	pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1839 		engine->name,
1840 		intel_context_get_total_runtime_ns(ce),
1841 		intel_context_get_avg_runtime_ns(ce));
1842 
1843 	err = 0;
1844 	if (ce->stats.runtime.num_underflow) {
1845 		pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1846 		       engine->name,
1847 		       ce->stats.runtime.num_underflow,
1848 		       ce->stats.runtime.max_underflow);
1849 		GEM_TRACE_DUMP();
1850 		err = -EOVERFLOW;
1851 	}
1852 
1853 err_wait:
1854 	i915_request_put(rq);
1855 err_rq:
1856 	intel_context_put(ce);
1857 	return err;
1858 }
1859 
1860 static int live_pphwsp_runtime(void *arg)
1861 {
1862 	struct intel_gt *gt = arg;
1863 	struct intel_engine_cs *engine;
1864 	enum intel_engine_id id;
1865 	int err = 0;
1866 
1867 	/*
1868 	 * Check that cumulative context runtime as stored in the pphwsp[16]
1869 	 * is monotonic.
1870 	 */
1871 
1872 	for_each_engine(engine, gt, id) {
1873 		err = __live_pphwsp_runtime(engine);
1874 		if (err)
1875 			break;
1876 	}
1877 
1878 	if (igt_flush_test(gt->i915))
1879 		err = -EIO;
1880 
1881 	return err;
1882 }
1883 
1884 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1885 {
1886 	static const struct i915_subtest tests[] = {
1887 		SUBTEST(live_lrc_layout),
1888 		SUBTEST(live_lrc_fixed),
1889 		SUBTEST(live_lrc_state),
1890 		SUBTEST(live_lrc_gpr),
1891 		SUBTEST(live_lrc_isolation),
1892 		SUBTEST(live_lrc_timestamp),
1893 		SUBTEST(live_lrc_garbage),
1894 		SUBTEST(live_pphwsp_runtime),
1895 		SUBTEST(live_lrc_indirect_ctx_bb),
1896 	};
1897 
1898 	if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1899 		return 0;
1900 
1901 	return intel_gt_live_subtests(tests, to_gt(i915));
1902 }
1903