1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include "gem/i915_gem_internal.h"
7 #include "gem/i915_gem_pm.h"
8 #include "gt/intel_engine_user.h"
9 #include "gt/intel_gt.h"
10 #include "i915_selftest.h"
11 #include "intel_reset.h"
12 
13 #include "selftests/igt_flush_test.h"
14 #include "selftests/igt_reset.h"
15 #include "selftests/igt_spinner.h"
16 #include "selftests/intel_scheduler_helpers.h"
17 #include "selftests/mock_drm.h"
18 
19 #include "gem/selftests/igt_gem_utils.h"
20 #include "gem/selftests/mock_context.h"
21 
22 static const struct wo_register {
23 	enum intel_platform platform;
24 	u32 reg;
25 } wo_registers[] = {
26 	{ INTEL_GEMINILAKE, 0x731c }
27 };
28 
29 struct wa_lists {
30 	struct i915_wa_list gt_wa_list;
31 	struct {
32 		struct i915_wa_list wa_list;
33 		struct i915_wa_list ctx_wa_list;
34 	} engine[I915_NUM_ENGINES];
35 };
36 
37 static int request_add_sync(struct i915_request *rq, int err)
38 {
39 	i915_request_get(rq);
40 	i915_request_add(rq);
41 	if (i915_request_wait(rq, 0, HZ / 5) < 0)
42 		err = -EIO;
43 	i915_request_put(rq);
44 
45 	return err;
46 }
47 
48 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
49 {
50 	int err = 0;
51 
52 	i915_request_get(rq);
53 	i915_request_add(rq);
54 	if (spin && !igt_wait_for_spinner(spin, rq))
55 		err = -ETIMEDOUT;
56 	i915_request_put(rq);
57 
58 	return err;
59 }
60 
61 static void
62 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
63 {
64 	struct intel_engine_cs *engine;
65 	enum intel_engine_id id;
66 
67 	memset(lists, 0, sizeof(*lists));
68 
69 	wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
70 	gt_init_workarounds(gt, &lists->gt_wa_list);
71 	wa_init_finish(&lists->gt_wa_list);
72 
73 	for_each_engine(engine, gt, id) {
74 		struct i915_wa_list *wal = &lists->engine[id].wa_list;
75 
76 		wa_init_start(wal, "REF", engine->name);
77 		engine_init_workarounds(engine, wal);
78 		wa_init_finish(wal);
79 
80 		__intel_engine_init_ctx_wa(engine,
81 					   &lists->engine[id].ctx_wa_list,
82 					   "CTX_REF");
83 	}
84 }
85 
86 static void
87 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
88 {
89 	struct intel_engine_cs *engine;
90 	enum intel_engine_id id;
91 
92 	for_each_engine(engine, gt, id)
93 		intel_wa_list_free(&lists->engine[id].wa_list);
94 
95 	intel_wa_list_free(&lists->gt_wa_list);
96 }
97 
98 static struct drm_i915_gem_object *
99 read_nonprivs(struct intel_context *ce)
100 {
101 	struct intel_engine_cs *engine = ce->engine;
102 	const u32 base = engine->mmio_base;
103 	struct drm_i915_gem_object *result;
104 	struct i915_request *rq;
105 	struct i915_vma *vma;
106 	u32 srm, *cs;
107 	int err;
108 	int i;
109 
110 	result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
111 	if (IS_ERR(result))
112 		return result;
113 
114 	i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
115 
116 	cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
117 	if (IS_ERR(cs)) {
118 		err = PTR_ERR(cs);
119 		goto err_obj;
120 	}
121 	memset(cs, 0xc5, PAGE_SIZE);
122 	i915_gem_object_flush_map(result);
123 	i915_gem_object_unpin_map(result);
124 
125 	vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
126 	if (IS_ERR(vma)) {
127 		err = PTR_ERR(vma);
128 		goto err_obj;
129 	}
130 
131 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
132 	if (err)
133 		goto err_obj;
134 
135 	rq = intel_context_create_request(ce);
136 	if (IS_ERR(rq)) {
137 		err = PTR_ERR(rq);
138 		goto err_pin;
139 	}
140 
141 	i915_vma_lock(vma);
142 	err = i915_request_await_object(rq, vma->obj, true);
143 	if (err == 0)
144 		err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
145 	i915_vma_unlock(vma);
146 	if (err)
147 		goto err_req;
148 
149 	srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
150 	if (GRAPHICS_VER(engine->i915) >= 8)
151 		srm++;
152 
153 	cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
154 	if (IS_ERR(cs)) {
155 		err = PTR_ERR(cs);
156 		goto err_req;
157 	}
158 
159 	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
160 		*cs++ = srm;
161 		*cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
162 		*cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
163 		*cs++ = 0;
164 	}
165 	intel_ring_advance(rq, cs);
166 
167 	i915_request_add(rq);
168 	i915_vma_unpin(vma);
169 
170 	return result;
171 
172 err_req:
173 	i915_request_add(rq);
174 err_pin:
175 	i915_vma_unpin(vma);
176 err_obj:
177 	i915_gem_object_put(result);
178 	return ERR_PTR(err);
179 }
180 
181 static u32
182 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
183 {
184 	i915_reg_t reg = i < engine->whitelist.count ?
185 			 engine->whitelist.list[i].reg :
186 			 RING_NOPID(engine->mmio_base);
187 
188 	return i915_mmio_reg_offset(reg);
189 }
190 
191 static void
192 print_results(const struct intel_engine_cs *engine, const u32 *results)
193 {
194 	unsigned int i;
195 
196 	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
197 		u32 expected = get_whitelist_reg(engine, i);
198 		u32 actual = results[i];
199 
200 		pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
201 			i, expected, actual);
202 	}
203 }
204 
205 static int check_whitelist(struct intel_context *ce)
206 {
207 	struct intel_engine_cs *engine = ce->engine;
208 	struct drm_i915_gem_object *results;
209 	struct intel_wedge_me wedge;
210 	u32 *vaddr;
211 	int err;
212 	int i;
213 
214 	results = read_nonprivs(ce);
215 	if (IS_ERR(results))
216 		return PTR_ERR(results);
217 
218 	err = 0;
219 	i915_gem_object_lock(results, NULL);
220 	intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
221 		err = i915_gem_object_set_to_cpu_domain(results, false);
222 
223 	if (intel_gt_is_wedged(engine->gt))
224 		err = -EIO;
225 	if (err)
226 		goto out_put;
227 
228 	vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
229 	if (IS_ERR(vaddr)) {
230 		err = PTR_ERR(vaddr);
231 		goto out_put;
232 	}
233 
234 	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
235 		u32 expected = get_whitelist_reg(engine, i);
236 		u32 actual = vaddr[i];
237 
238 		if (expected != actual) {
239 			print_results(engine, vaddr);
240 			pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
241 			       i, expected, actual);
242 
243 			err = -EINVAL;
244 			break;
245 		}
246 	}
247 
248 	i915_gem_object_unpin_map(results);
249 out_put:
250 	i915_gem_object_unlock(results);
251 	i915_gem_object_put(results);
252 	return err;
253 }
254 
255 static int do_device_reset(struct intel_engine_cs *engine)
256 {
257 	intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
258 	return 0;
259 }
260 
261 static int do_engine_reset(struct intel_engine_cs *engine)
262 {
263 	return intel_engine_reset(engine, "live_workarounds");
264 }
265 
266 static int do_guc_reset(struct intel_engine_cs *engine)
267 {
268 	/* Currently a no-op as the reset is handled by GuC */
269 	return 0;
270 }
271 
272 static int
273 switch_to_scratch_context(struct intel_engine_cs *engine,
274 			  struct igt_spinner *spin,
275 			  struct i915_request **rq)
276 {
277 	struct intel_context *ce;
278 	int err = 0;
279 
280 	ce = intel_context_create(engine);
281 	if (IS_ERR(ce))
282 		return PTR_ERR(ce);
283 
284 	*rq = igt_spinner_create_request(spin, ce, MI_NOOP);
285 	intel_context_put(ce);
286 
287 	if (IS_ERR(*rq)) {
288 		spin = NULL;
289 		err = PTR_ERR(*rq);
290 		goto err;
291 	}
292 
293 	err = request_add_spin(*rq, spin);
294 err:
295 	if (err && spin)
296 		igt_spinner_end(spin);
297 
298 	return err;
299 }
300 
301 static int check_whitelist_across_reset(struct intel_engine_cs *engine,
302 					int (*reset)(struct intel_engine_cs *),
303 					const char *name)
304 {
305 	struct intel_context *ce, *tmp;
306 	struct igt_spinner spin;
307 	struct i915_request *rq;
308 	intel_wakeref_t wakeref;
309 	int err;
310 
311 	pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
312 		engine->whitelist.count, engine->name, name);
313 
314 	ce = intel_context_create(engine);
315 	if (IS_ERR(ce))
316 		return PTR_ERR(ce);
317 
318 	err = igt_spinner_init(&spin, engine->gt);
319 	if (err)
320 		goto out_ctx;
321 
322 	err = check_whitelist(ce);
323 	if (err) {
324 		pr_err("Invalid whitelist *before* %s reset!\n", name);
325 		goto out_spin;
326 	}
327 
328 	err = switch_to_scratch_context(engine, &spin, &rq);
329 	if (err)
330 		goto out_spin;
331 
332 	/* Ensure the spinner hasn't aborted */
333 	if (i915_request_completed(rq)) {
334 		pr_err("%s spinner failed to start\n", name);
335 		err = -ETIMEDOUT;
336 		goto out_spin;
337 	}
338 
339 	with_intel_runtime_pm(engine->uncore->rpm, wakeref)
340 		err = reset(engine);
341 
342 	/* Ensure the reset happens and kills the engine */
343 	if (err == 0)
344 		err = intel_selftest_wait_for_rq(rq);
345 
346 	igt_spinner_end(&spin);
347 
348 	if (err) {
349 		pr_err("%s reset failed\n", name);
350 		goto out_spin;
351 	}
352 
353 	err = check_whitelist(ce);
354 	if (err) {
355 		pr_err("Whitelist not preserved in context across %s reset!\n",
356 		       name);
357 		goto out_spin;
358 	}
359 
360 	tmp = intel_context_create(engine);
361 	if (IS_ERR(tmp)) {
362 		err = PTR_ERR(tmp);
363 		goto out_spin;
364 	}
365 	intel_context_put(ce);
366 	ce = tmp;
367 
368 	err = check_whitelist(ce);
369 	if (err) {
370 		pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
371 		       name);
372 		goto out_spin;
373 	}
374 
375 out_spin:
376 	igt_spinner_fini(&spin);
377 out_ctx:
378 	intel_context_put(ce);
379 	return err;
380 }
381 
382 static struct i915_vma *create_batch(struct i915_address_space *vm)
383 {
384 	struct drm_i915_gem_object *obj;
385 	struct i915_vma *vma;
386 	int err;
387 
388 	obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
389 	if (IS_ERR(obj))
390 		return ERR_CAST(obj);
391 
392 	vma = i915_vma_instance(obj, vm, NULL);
393 	if (IS_ERR(vma)) {
394 		err = PTR_ERR(vma);
395 		goto err_obj;
396 	}
397 
398 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
399 	if (err)
400 		goto err_obj;
401 
402 	return vma;
403 
404 err_obj:
405 	i915_gem_object_put(obj);
406 	return ERR_PTR(err);
407 }
408 
409 static u32 reg_write(u32 old, u32 new, u32 rsvd)
410 {
411 	if (rsvd == 0x0000ffff) {
412 		old &= ~(new >> 16);
413 		old |= new & (new >> 16);
414 	} else {
415 		old &= ~rsvd;
416 		old |= new & rsvd;
417 	}
418 
419 	return old;
420 }
421 
422 static bool wo_register(struct intel_engine_cs *engine, u32 reg)
423 {
424 	enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
425 	int i;
426 
427 	if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
428 	     RING_FORCE_TO_NONPRIV_ACCESS_WR)
429 		return true;
430 
431 	for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
432 		if (wo_registers[i].platform == platform &&
433 		    wo_registers[i].reg == reg)
434 			return true;
435 	}
436 
437 	return false;
438 }
439 
440 static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
441 {
442 	reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
443 	switch (reg) {
444 	case 0x358:
445 	case 0x35c:
446 	case 0x3a8:
447 		return true;
448 
449 	default:
450 		return false;
451 	}
452 }
453 
454 static bool ro_register(u32 reg)
455 {
456 	if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
457 	     RING_FORCE_TO_NONPRIV_ACCESS_RD)
458 		return true;
459 
460 	return false;
461 }
462 
463 static int whitelist_writable_count(struct intel_engine_cs *engine)
464 {
465 	int count = engine->whitelist.count;
466 	int i;
467 
468 	for (i = 0; i < engine->whitelist.count; i++) {
469 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
470 
471 		if (ro_register(reg))
472 			count--;
473 	}
474 
475 	return count;
476 }
477 
478 static int check_dirty_whitelist(struct intel_context *ce)
479 {
480 	const u32 values[] = {
481 		0x00000000,
482 		0x01010101,
483 		0x10100101,
484 		0x03030303,
485 		0x30300303,
486 		0x05050505,
487 		0x50500505,
488 		0x0f0f0f0f,
489 		0xf00ff00f,
490 		0x10101010,
491 		0xf0f01010,
492 		0x30303030,
493 		0xa0a03030,
494 		0x50505050,
495 		0xc0c05050,
496 		0xf0f0f0f0,
497 		0x11111111,
498 		0x33333333,
499 		0x55555555,
500 		0x0000ffff,
501 		0x00ff00ff,
502 		0xff0000ff,
503 		0xffff00ff,
504 		0xffffffff,
505 	};
506 	struct intel_engine_cs *engine = ce->engine;
507 	struct i915_vma *scratch;
508 	struct i915_vma *batch;
509 	int err = 0, i, v, sz;
510 	u32 *cs, *results;
511 
512 	sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
513 	scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
514 	if (IS_ERR(scratch))
515 		return PTR_ERR(scratch);
516 
517 	batch = create_batch(ce->vm);
518 	if (IS_ERR(batch)) {
519 		err = PTR_ERR(batch);
520 		goto out_scratch;
521 	}
522 
523 	for (i = 0; i < engine->whitelist.count; i++) {
524 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
525 		struct i915_gem_ww_ctx ww;
526 		u64 addr = scratch->node.start;
527 		struct i915_request *rq;
528 		u32 srm, lrm, rsvd;
529 		u32 expect;
530 		int idx;
531 		bool ro_reg;
532 
533 		if (wo_register(engine, reg))
534 			continue;
535 
536 		if (timestamp(engine, reg))
537 			continue; /* timestamps are expected to autoincrement */
538 
539 		ro_reg = ro_register(reg);
540 
541 		i915_gem_ww_ctx_init(&ww, false);
542 retry:
543 		cs = NULL;
544 		err = i915_gem_object_lock(scratch->obj, &ww);
545 		if (!err)
546 			err = i915_gem_object_lock(batch->obj, &ww);
547 		if (!err)
548 			err = intel_context_pin_ww(ce, &ww);
549 		if (err)
550 			goto out;
551 
552 		cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
553 		if (IS_ERR(cs)) {
554 			err = PTR_ERR(cs);
555 			goto out_ctx;
556 		}
557 
558 		results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
559 		if (IS_ERR(results)) {
560 			err = PTR_ERR(results);
561 			goto out_unmap_batch;
562 		}
563 
564 		/* Clear non priv flags */
565 		reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
566 
567 		srm = MI_STORE_REGISTER_MEM;
568 		lrm = MI_LOAD_REGISTER_MEM;
569 		if (GRAPHICS_VER(engine->i915) >= 8)
570 			lrm++, srm++;
571 
572 		pr_debug("%s: Writing garbage to %x\n",
573 			 engine->name, reg);
574 
575 		/* SRM original */
576 		*cs++ = srm;
577 		*cs++ = reg;
578 		*cs++ = lower_32_bits(addr);
579 		*cs++ = upper_32_bits(addr);
580 
581 		idx = 1;
582 		for (v = 0; v < ARRAY_SIZE(values); v++) {
583 			/* LRI garbage */
584 			*cs++ = MI_LOAD_REGISTER_IMM(1);
585 			*cs++ = reg;
586 			*cs++ = values[v];
587 
588 			/* SRM result */
589 			*cs++ = srm;
590 			*cs++ = reg;
591 			*cs++ = lower_32_bits(addr + sizeof(u32) * idx);
592 			*cs++ = upper_32_bits(addr + sizeof(u32) * idx);
593 			idx++;
594 		}
595 		for (v = 0; v < ARRAY_SIZE(values); v++) {
596 			/* LRI garbage */
597 			*cs++ = MI_LOAD_REGISTER_IMM(1);
598 			*cs++ = reg;
599 			*cs++ = ~values[v];
600 
601 			/* SRM result */
602 			*cs++ = srm;
603 			*cs++ = reg;
604 			*cs++ = lower_32_bits(addr + sizeof(u32) * idx);
605 			*cs++ = upper_32_bits(addr + sizeof(u32) * idx);
606 			idx++;
607 		}
608 		GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
609 
610 		/* LRM original -- don't leave garbage in the context! */
611 		*cs++ = lrm;
612 		*cs++ = reg;
613 		*cs++ = lower_32_bits(addr);
614 		*cs++ = upper_32_bits(addr);
615 
616 		*cs++ = MI_BATCH_BUFFER_END;
617 
618 		i915_gem_object_flush_map(batch->obj);
619 		i915_gem_object_unpin_map(batch->obj);
620 		intel_gt_chipset_flush(engine->gt);
621 		cs = NULL;
622 
623 		rq = i915_request_create(ce);
624 		if (IS_ERR(rq)) {
625 			err = PTR_ERR(rq);
626 			goto out_unmap_scratch;
627 		}
628 
629 		if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
630 			err = engine->emit_init_breadcrumb(rq);
631 			if (err)
632 				goto err_request;
633 		}
634 
635 		err = i915_request_await_object(rq, batch->obj, false);
636 		if (err == 0)
637 			err = i915_vma_move_to_active(batch, rq, 0);
638 		if (err)
639 			goto err_request;
640 
641 		err = i915_request_await_object(rq, scratch->obj, true);
642 		if (err == 0)
643 			err = i915_vma_move_to_active(scratch, rq,
644 						      EXEC_OBJECT_WRITE);
645 		if (err)
646 			goto err_request;
647 
648 		err = engine->emit_bb_start(rq,
649 					    batch->node.start, PAGE_SIZE,
650 					    0);
651 		if (err)
652 			goto err_request;
653 
654 err_request:
655 		err = request_add_sync(rq, err);
656 		if (err) {
657 			pr_err("%s: Futzing %x timedout; cancelling test\n",
658 			       engine->name, reg);
659 			intel_gt_set_wedged(engine->gt);
660 			goto out_unmap_scratch;
661 		}
662 
663 		GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
664 		if (!ro_reg) {
665 			/* detect write masking */
666 			rsvd = results[ARRAY_SIZE(values)];
667 			if (!rsvd) {
668 				pr_err("%s: Unable to write to whitelisted register %x\n",
669 				       engine->name, reg);
670 				err = -EINVAL;
671 				goto out_unmap_scratch;
672 			}
673 		} else {
674 			rsvd = 0;
675 		}
676 
677 		expect = results[0];
678 		idx = 1;
679 		for (v = 0; v < ARRAY_SIZE(values); v++) {
680 			if (ro_reg)
681 				expect = results[0];
682 			else
683 				expect = reg_write(expect, values[v], rsvd);
684 
685 			if (results[idx] != expect)
686 				err++;
687 			idx++;
688 		}
689 		for (v = 0; v < ARRAY_SIZE(values); v++) {
690 			if (ro_reg)
691 				expect = results[0];
692 			else
693 				expect = reg_write(expect, ~values[v], rsvd);
694 
695 			if (results[idx] != expect)
696 				err++;
697 			idx++;
698 		}
699 		if (err) {
700 			pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
701 			       engine->name, err, reg);
702 
703 			if (ro_reg)
704 				pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
705 					engine->name, reg, results[0]);
706 			else
707 				pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
708 					engine->name, reg, results[0], rsvd);
709 
710 			expect = results[0];
711 			idx = 1;
712 			for (v = 0; v < ARRAY_SIZE(values); v++) {
713 				u32 w = values[v];
714 
715 				if (ro_reg)
716 					expect = results[0];
717 				else
718 					expect = reg_write(expect, w, rsvd);
719 				pr_info("Wrote %08x, read %08x, expect %08x\n",
720 					w, results[idx], expect);
721 				idx++;
722 			}
723 			for (v = 0; v < ARRAY_SIZE(values); v++) {
724 				u32 w = ~values[v];
725 
726 				if (ro_reg)
727 					expect = results[0];
728 				else
729 					expect = reg_write(expect, w, rsvd);
730 				pr_info("Wrote %08x, read %08x, expect %08x\n",
731 					w, results[idx], expect);
732 				idx++;
733 			}
734 
735 			err = -EINVAL;
736 		}
737 out_unmap_scratch:
738 		i915_gem_object_unpin_map(scratch->obj);
739 out_unmap_batch:
740 		if (cs)
741 			i915_gem_object_unpin_map(batch->obj);
742 out_ctx:
743 		intel_context_unpin(ce);
744 out:
745 		if (err == -EDEADLK) {
746 			err = i915_gem_ww_ctx_backoff(&ww);
747 			if (!err)
748 				goto retry;
749 		}
750 		i915_gem_ww_ctx_fini(&ww);
751 		if (err)
752 			break;
753 	}
754 
755 	if (igt_flush_test(engine->i915))
756 		err = -EIO;
757 
758 	i915_vma_unpin_and_release(&batch, 0);
759 out_scratch:
760 	i915_vma_unpin_and_release(&scratch, 0);
761 	return err;
762 }
763 
764 static int live_dirty_whitelist(void *arg)
765 {
766 	struct intel_gt *gt = arg;
767 	struct intel_engine_cs *engine;
768 	enum intel_engine_id id;
769 
770 	/* Can the user write to the whitelisted registers? */
771 
772 	if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
773 		return 0;
774 
775 	for_each_engine(engine, gt, id) {
776 		struct intel_context *ce;
777 		int err;
778 
779 		if (engine->whitelist.count == 0)
780 			continue;
781 
782 		ce = intel_context_create(engine);
783 		if (IS_ERR(ce))
784 			return PTR_ERR(ce);
785 
786 		err = check_dirty_whitelist(ce);
787 		intel_context_put(ce);
788 		if (err)
789 			return err;
790 	}
791 
792 	return 0;
793 }
794 
795 static int live_reset_whitelist(void *arg)
796 {
797 	struct intel_gt *gt = arg;
798 	struct intel_engine_cs *engine;
799 	enum intel_engine_id id;
800 	int err = 0;
801 
802 	/* If we reset the gpu, we should not lose the RING_NONPRIV */
803 	igt_global_reset_lock(gt);
804 
805 	for_each_engine(engine, gt, id) {
806 		if (engine->whitelist.count == 0)
807 			continue;
808 
809 		if (intel_has_reset_engine(gt)) {
810 			if (intel_engine_uses_guc(engine)) {
811 				struct intel_selftest_saved_policy saved;
812 				int err2;
813 
814 				err = intel_selftest_modify_policy(engine, &saved,
815 								   SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
816 				if (err)
817 					goto out;
818 
819 				err = check_whitelist_across_reset(engine,
820 								   do_guc_reset,
821 								   "guc");
822 
823 				err2 = intel_selftest_restore_policy(engine, &saved);
824 				if (err == 0)
825 					err = err2;
826 			} else {
827 				err = check_whitelist_across_reset(engine,
828 								   do_engine_reset,
829 								   "engine");
830 			}
831 
832 			if (err)
833 				goto out;
834 		}
835 
836 		if (intel_has_gpu_reset(gt)) {
837 			err = check_whitelist_across_reset(engine,
838 							   do_device_reset,
839 							   "device");
840 			if (err)
841 				goto out;
842 		}
843 	}
844 
845 out:
846 	igt_global_reset_unlock(gt);
847 	return err;
848 }
849 
850 static int read_whitelisted_registers(struct intel_context *ce,
851 				      struct i915_vma *results)
852 {
853 	struct intel_engine_cs *engine = ce->engine;
854 	struct i915_request *rq;
855 	int i, err = 0;
856 	u32 srm, *cs;
857 
858 	rq = intel_context_create_request(ce);
859 	if (IS_ERR(rq))
860 		return PTR_ERR(rq);
861 
862 	i915_vma_lock(results);
863 	err = i915_request_await_object(rq, results->obj, true);
864 	if (err == 0)
865 		err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
866 	i915_vma_unlock(results);
867 	if (err)
868 		goto err_req;
869 
870 	srm = MI_STORE_REGISTER_MEM;
871 	if (GRAPHICS_VER(engine->i915) >= 8)
872 		srm++;
873 
874 	cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
875 	if (IS_ERR(cs)) {
876 		err = PTR_ERR(cs);
877 		goto err_req;
878 	}
879 
880 	for (i = 0; i < engine->whitelist.count; i++) {
881 		u64 offset = results->node.start + sizeof(u32) * i;
882 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
883 
884 		/* Clear non priv flags */
885 		reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
886 
887 		*cs++ = srm;
888 		*cs++ = reg;
889 		*cs++ = lower_32_bits(offset);
890 		*cs++ = upper_32_bits(offset);
891 	}
892 	intel_ring_advance(rq, cs);
893 
894 err_req:
895 	return request_add_sync(rq, err);
896 }
897 
898 static int scrub_whitelisted_registers(struct intel_context *ce)
899 {
900 	struct intel_engine_cs *engine = ce->engine;
901 	struct i915_request *rq;
902 	struct i915_vma *batch;
903 	int i, err = 0;
904 	u32 *cs;
905 
906 	batch = create_batch(ce->vm);
907 	if (IS_ERR(batch))
908 		return PTR_ERR(batch);
909 
910 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
911 	if (IS_ERR(cs)) {
912 		err = PTR_ERR(cs);
913 		goto err_batch;
914 	}
915 
916 	*cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
917 	for (i = 0; i < engine->whitelist.count; i++) {
918 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
919 
920 		if (ro_register(reg))
921 			continue;
922 
923 		/* Clear non priv flags */
924 		reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
925 
926 		*cs++ = reg;
927 		*cs++ = 0xffffffff;
928 	}
929 	*cs++ = MI_BATCH_BUFFER_END;
930 
931 	i915_gem_object_flush_map(batch->obj);
932 	intel_gt_chipset_flush(engine->gt);
933 
934 	rq = intel_context_create_request(ce);
935 	if (IS_ERR(rq)) {
936 		err = PTR_ERR(rq);
937 		goto err_unpin;
938 	}
939 
940 	if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
941 		err = engine->emit_init_breadcrumb(rq);
942 		if (err)
943 			goto err_request;
944 	}
945 
946 	i915_vma_lock(batch);
947 	err = i915_request_await_object(rq, batch->obj, false);
948 	if (err == 0)
949 		err = i915_vma_move_to_active(batch, rq, 0);
950 	i915_vma_unlock(batch);
951 	if (err)
952 		goto err_request;
953 
954 	/* Perform the writes from an unprivileged "user" batch */
955 	err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
956 
957 err_request:
958 	err = request_add_sync(rq, err);
959 
960 err_unpin:
961 	i915_gem_object_unpin_map(batch->obj);
962 err_batch:
963 	i915_vma_unpin_and_release(&batch, 0);
964 	return err;
965 }
966 
967 struct regmask {
968 	i915_reg_t reg;
969 	u8 graphics_ver;
970 };
971 
972 static bool find_reg(struct drm_i915_private *i915,
973 		     i915_reg_t reg,
974 		     const struct regmask *tbl,
975 		     unsigned long count)
976 {
977 	u32 offset = i915_mmio_reg_offset(reg);
978 
979 	while (count--) {
980 		if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
981 		    i915_mmio_reg_offset(tbl->reg) == offset)
982 			return true;
983 		tbl++;
984 	}
985 
986 	return false;
987 }
988 
989 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
990 {
991 	/* Alas, we must pardon some whitelists. Mistakes already made */
992 	static const struct regmask pardon[] = {
993 		{ GEN9_CTX_PREEMPT_REG, 9 },
994 		{ _MMIO(0xb118), 9 }, /* GEN8_L3SQCREG4 */
995 	};
996 
997 	return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
998 }
999 
1000 static bool result_eq(struct intel_engine_cs *engine,
1001 		      u32 a, u32 b, i915_reg_t reg)
1002 {
1003 	if (a != b && !pardon_reg(engine->i915, reg)) {
1004 		pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
1005 		       i915_mmio_reg_offset(reg), a, b);
1006 		return false;
1007 	}
1008 
1009 	return true;
1010 }
1011 
1012 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
1013 {
1014 	/* Some registers do not seem to behave and our writes unreadable */
1015 	static const struct regmask wo[] = {
1016 		{ GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
1017 	};
1018 
1019 	return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
1020 }
1021 
1022 static bool result_neq(struct intel_engine_cs *engine,
1023 		       u32 a, u32 b, i915_reg_t reg)
1024 {
1025 	if (a == b && !writeonly_reg(engine->i915, reg)) {
1026 		pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
1027 		       i915_mmio_reg_offset(reg), a);
1028 		return false;
1029 	}
1030 
1031 	return true;
1032 }
1033 
1034 static int
1035 check_whitelisted_registers(struct intel_engine_cs *engine,
1036 			    struct i915_vma *A,
1037 			    struct i915_vma *B,
1038 			    bool (*fn)(struct intel_engine_cs *engine,
1039 				       u32 a, u32 b,
1040 				       i915_reg_t reg))
1041 {
1042 	u32 *a, *b;
1043 	int i, err;
1044 
1045 	a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1046 	if (IS_ERR(a))
1047 		return PTR_ERR(a);
1048 
1049 	b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1050 	if (IS_ERR(b)) {
1051 		err = PTR_ERR(b);
1052 		goto err_a;
1053 	}
1054 
1055 	err = 0;
1056 	for (i = 0; i < engine->whitelist.count; i++) {
1057 		const struct i915_wa *wa = &engine->whitelist.list[i];
1058 
1059 		if (i915_mmio_reg_offset(wa->reg) &
1060 		    RING_FORCE_TO_NONPRIV_ACCESS_RD)
1061 			continue;
1062 
1063 		if (!fn(engine, a[i], b[i], wa->reg))
1064 			err = -EINVAL;
1065 	}
1066 
1067 	i915_gem_object_unpin_map(B->obj);
1068 err_a:
1069 	i915_gem_object_unpin_map(A->obj);
1070 	return err;
1071 }
1072 
1073 static int live_isolated_whitelist(void *arg)
1074 {
1075 	struct intel_gt *gt = arg;
1076 	struct {
1077 		struct i915_vma *scratch[2];
1078 	} client[2] = {};
1079 	struct intel_engine_cs *engine;
1080 	enum intel_engine_id id;
1081 	int i, err = 0;
1082 
1083 	/*
1084 	 * Check that a write into a whitelist register works, but
1085 	 * invisible to a second context.
1086 	 */
1087 
1088 	if (!intel_engines_has_context_isolation(gt->i915))
1089 		return 0;
1090 
1091 	for (i = 0; i < ARRAY_SIZE(client); i++) {
1092 		client[i].scratch[0] =
1093 			__vm_create_scratch_for_read_pinned(gt->vm, 4096);
1094 		if (IS_ERR(client[i].scratch[0])) {
1095 			err = PTR_ERR(client[i].scratch[0]);
1096 			goto err;
1097 		}
1098 
1099 		client[i].scratch[1] =
1100 			__vm_create_scratch_for_read_pinned(gt->vm, 4096);
1101 		if (IS_ERR(client[i].scratch[1])) {
1102 			err = PTR_ERR(client[i].scratch[1]);
1103 			i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1104 			goto err;
1105 		}
1106 	}
1107 
1108 	for_each_engine(engine, gt, id) {
1109 		struct intel_context *ce[2];
1110 
1111 		if (!engine->kernel_context->vm)
1112 			continue;
1113 
1114 		if (!whitelist_writable_count(engine))
1115 			continue;
1116 
1117 		ce[0] = intel_context_create(engine);
1118 		if (IS_ERR(ce[0])) {
1119 			err = PTR_ERR(ce[0]);
1120 			break;
1121 		}
1122 		ce[1] = intel_context_create(engine);
1123 		if (IS_ERR(ce[1])) {
1124 			err = PTR_ERR(ce[1]);
1125 			intel_context_put(ce[0]);
1126 			break;
1127 		}
1128 
1129 		/* Read default values */
1130 		err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1131 		if (err)
1132 			goto err_ce;
1133 
1134 		/* Try to overwrite registers (should only affect ctx0) */
1135 		err = scrub_whitelisted_registers(ce[0]);
1136 		if (err)
1137 			goto err_ce;
1138 
1139 		/* Read values from ctx1, we expect these to be defaults */
1140 		err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1141 		if (err)
1142 			goto err_ce;
1143 
1144 		/* Verify that both reads return the same default values */
1145 		err = check_whitelisted_registers(engine,
1146 						  client[0].scratch[0],
1147 						  client[1].scratch[0],
1148 						  result_eq);
1149 		if (err)
1150 			goto err_ce;
1151 
1152 		/* Read back the updated values in ctx0 */
1153 		err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1154 		if (err)
1155 			goto err_ce;
1156 
1157 		/* User should be granted privilege to overwhite regs */
1158 		err = check_whitelisted_registers(engine,
1159 						  client[0].scratch[0],
1160 						  client[0].scratch[1],
1161 						  result_neq);
1162 err_ce:
1163 		intel_context_put(ce[1]);
1164 		intel_context_put(ce[0]);
1165 		if (err)
1166 			break;
1167 	}
1168 
1169 err:
1170 	for (i = 0; i < ARRAY_SIZE(client); i++) {
1171 		i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1172 		i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1173 	}
1174 
1175 	if (igt_flush_test(gt->i915))
1176 		err = -EIO;
1177 
1178 	return err;
1179 }
1180 
1181 static bool
1182 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1183 		const char *str)
1184 {
1185 	struct intel_engine_cs *engine;
1186 	enum intel_engine_id id;
1187 	bool ok = true;
1188 
1189 	ok &= wa_list_verify(gt, &lists->gt_wa_list, str);
1190 
1191 	for_each_engine(engine, gt, id) {
1192 		struct intel_context *ce;
1193 
1194 		ce = intel_context_create(engine);
1195 		if (IS_ERR(ce))
1196 			return false;
1197 
1198 		ok &= engine_wa_list_verify(ce,
1199 					    &lists->engine[id].wa_list,
1200 					    str) == 0;
1201 
1202 		ok &= engine_wa_list_verify(ce,
1203 					    &lists->engine[id].ctx_wa_list,
1204 					    str) == 0;
1205 
1206 		intel_context_put(ce);
1207 	}
1208 
1209 	return ok;
1210 }
1211 
1212 static int
1213 live_gpu_reset_workarounds(void *arg)
1214 {
1215 	struct intel_gt *gt = arg;
1216 	intel_wakeref_t wakeref;
1217 	struct wa_lists *lists;
1218 	bool ok;
1219 
1220 	if (!intel_has_gpu_reset(gt))
1221 		return 0;
1222 
1223 	lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1224 	if (!lists)
1225 		return -ENOMEM;
1226 
1227 	pr_info("Verifying after GPU reset...\n");
1228 
1229 	igt_global_reset_lock(gt);
1230 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1231 
1232 	reference_lists_init(gt, lists);
1233 
1234 	ok = verify_wa_lists(gt, lists, "before reset");
1235 	if (!ok)
1236 		goto out;
1237 
1238 	intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1239 
1240 	ok = verify_wa_lists(gt, lists, "after reset");
1241 
1242 out:
1243 	reference_lists_fini(gt, lists);
1244 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1245 	igt_global_reset_unlock(gt);
1246 	kfree(lists);
1247 
1248 	return ok ? 0 : -ESRCH;
1249 }
1250 
1251 static int
1252 live_engine_reset_workarounds(void *arg)
1253 {
1254 	struct intel_gt *gt = arg;
1255 	struct intel_engine_cs *engine;
1256 	enum intel_engine_id id;
1257 	struct intel_context *ce;
1258 	struct igt_spinner spin;
1259 	struct i915_request *rq;
1260 	intel_wakeref_t wakeref;
1261 	struct wa_lists *lists;
1262 	int ret = 0;
1263 
1264 	if (!intel_has_reset_engine(gt))
1265 		return 0;
1266 
1267 	lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1268 	if (!lists)
1269 		return -ENOMEM;
1270 
1271 	igt_global_reset_lock(gt);
1272 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1273 
1274 	reference_lists_init(gt, lists);
1275 
1276 	for_each_engine(engine, gt, id) {
1277 		struct intel_selftest_saved_policy saved;
1278 		bool using_guc = intel_engine_uses_guc(engine);
1279 		bool ok;
1280 		int ret2;
1281 
1282 		pr_info("Verifying after %s reset...\n", engine->name);
1283 		ret = intel_selftest_modify_policy(engine, &saved,
1284 						   SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1285 		if (ret)
1286 			break;
1287 
1288 		ce = intel_context_create(engine);
1289 		if (IS_ERR(ce)) {
1290 			ret = PTR_ERR(ce);
1291 			goto restore;
1292 		}
1293 
1294 		if (!using_guc) {
1295 			ok = verify_wa_lists(gt, lists, "before reset");
1296 			if (!ok) {
1297 				ret = -ESRCH;
1298 				goto err;
1299 			}
1300 
1301 			ret = intel_engine_reset(engine, "live_workarounds:idle");
1302 			if (ret) {
1303 				pr_err("%s: Reset failed while idle\n", engine->name);
1304 				goto err;
1305 			}
1306 
1307 			ok = verify_wa_lists(gt, lists, "after idle reset");
1308 			if (!ok) {
1309 				ret = -ESRCH;
1310 				goto err;
1311 			}
1312 		}
1313 
1314 		ret = igt_spinner_init(&spin, engine->gt);
1315 		if (ret)
1316 			goto err;
1317 
1318 		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1319 		if (IS_ERR(rq)) {
1320 			ret = PTR_ERR(rq);
1321 			igt_spinner_fini(&spin);
1322 			goto err;
1323 		}
1324 
1325 		ret = request_add_spin(rq, &spin);
1326 		if (ret) {
1327 			pr_err("%s: Spinner failed to start\n", engine->name);
1328 			igt_spinner_fini(&spin);
1329 			goto err;
1330 		}
1331 
1332 		/* Ensure the spinner hasn't aborted */
1333 		if (i915_request_completed(rq)) {
1334 			ret = -ETIMEDOUT;
1335 			goto skip;
1336 		}
1337 
1338 		if (!using_guc) {
1339 			ret = intel_engine_reset(engine, "live_workarounds:active");
1340 			if (ret) {
1341 				pr_err("%s: Reset failed on an active spinner\n",
1342 				       engine->name);
1343 				igt_spinner_fini(&spin);
1344 				goto err;
1345 			}
1346 		}
1347 
1348 		/* Ensure the reset happens and kills the engine */
1349 		if (ret == 0)
1350 			ret = intel_selftest_wait_for_rq(rq);
1351 
1352 skip:
1353 		igt_spinner_end(&spin);
1354 		igt_spinner_fini(&spin);
1355 
1356 		ok = verify_wa_lists(gt, lists, "after busy reset");
1357 		if (!ok)
1358 			ret = -ESRCH;
1359 
1360 err:
1361 		intel_context_put(ce);
1362 
1363 restore:
1364 		ret2 = intel_selftest_restore_policy(engine, &saved);
1365 		if (ret == 0)
1366 			ret = ret2;
1367 		if (ret)
1368 			break;
1369 	}
1370 
1371 	reference_lists_fini(gt, lists);
1372 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1373 	igt_global_reset_unlock(gt);
1374 	kfree(lists);
1375 
1376 	igt_flush_test(gt->i915);
1377 
1378 	return ret;
1379 }
1380 
1381 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1382 {
1383 	static const struct i915_subtest tests[] = {
1384 		SUBTEST(live_dirty_whitelist),
1385 		SUBTEST(live_reset_whitelist),
1386 		SUBTEST(live_isolated_whitelist),
1387 		SUBTEST(live_gpu_reset_workarounds),
1388 		SUBTEST(live_engine_reset_workarounds),
1389 	};
1390 
1391 	if (intel_gt_is_wedged(to_gt(i915)))
1392 		return 0;
1393 
1394 	return intel_gt_live_subtests(tests, to_gt(i915));
1395 }
1396