1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include "gem/i915_gem_pm.h"
7 #include "gt/intel_engine_user.h"
8 #include "gt/intel_gt.h"
9 #include "i915_selftest.h"
10 #include "intel_reset.h"
11 
12 #include "selftests/igt_flush_test.h"
13 #include "selftests/igt_reset.h"
14 #include "selftests/igt_spinner.h"
15 #include "selftests/mock_drm.h"
16 
17 #include "gem/selftests/igt_gem_utils.h"
18 #include "gem/selftests/mock_context.h"
19 
20 static const struct wo_register {
21 	enum intel_platform platform;
22 	u32 reg;
23 } wo_registers[] = {
24 	{ INTEL_GEMINILAKE, 0x731c }
25 };
26 
27 struct wa_lists {
28 	struct i915_wa_list gt_wa_list;
29 	struct {
30 		struct i915_wa_list wa_list;
31 		struct i915_wa_list ctx_wa_list;
32 	} engine[I915_NUM_ENGINES];
33 };
34 
35 static int request_add_sync(struct i915_request *rq, int err)
36 {
37 	i915_request_get(rq);
38 	i915_request_add(rq);
39 	if (i915_request_wait(rq, 0, HZ / 5) < 0)
40 		err = -EIO;
41 	i915_request_put(rq);
42 
43 	return err;
44 }
45 
46 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
47 {
48 	int err = 0;
49 
50 	i915_request_get(rq);
51 	i915_request_add(rq);
52 	if (spin && !igt_wait_for_spinner(spin, rq))
53 		err = -ETIMEDOUT;
54 	i915_request_put(rq);
55 
56 	return err;
57 }
58 
59 static void
60 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
61 {
62 	struct intel_engine_cs *engine;
63 	enum intel_engine_id id;
64 
65 	memset(lists, 0, sizeof(*lists));
66 
67 	wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
68 	gt_init_workarounds(gt->i915, &lists->gt_wa_list);
69 	wa_init_finish(&lists->gt_wa_list);
70 
71 	for_each_engine(engine, gt, id) {
72 		struct i915_wa_list *wal = &lists->engine[id].wa_list;
73 
74 		wa_init_start(wal, "REF", engine->name);
75 		engine_init_workarounds(engine, wal);
76 		wa_init_finish(wal);
77 
78 		__intel_engine_init_ctx_wa(engine,
79 					   &lists->engine[id].ctx_wa_list,
80 					   "CTX_REF");
81 	}
82 }
83 
84 static void
85 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
86 {
87 	struct intel_engine_cs *engine;
88 	enum intel_engine_id id;
89 
90 	for_each_engine(engine, gt, id)
91 		intel_wa_list_free(&lists->engine[id].wa_list);
92 
93 	intel_wa_list_free(&lists->gt_wa_list);
94 }
95 
96 static struct drm_i915_gem_object *
97 read_nonprivs(struct intel_context *ce)
98 {
99 	struct intel_engine_cs *engine = ce->engine;
100 	const u32 base = engine->mmio_base;
101 	struct drm_i915_gem_object *result;
102 	struct i915_request *rq;
103 	struct i915_vma *vma;
104 	u32 srm, *cs;
105 	int err;
106 	int i;
107 
108 	result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
109 	if (IS_ERR(result))
110 		return result;
111 
112 	i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
113 
114 	cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
115 	if (IS_ERR(cs)) {
116 		err = PTR_ERR(cs);
117 		goto err_obj;
118 	}
119 	memset(cs, 0xc5, PAGE_SIZE);
120 	i915_gem_object_flush_map(result);
121 	i915_gem_object_unpin_map(result);
122 
123 	vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
124 	if (IS_ERR(vma)) {
125 		err = PTR_ERR(vma);
126 		goto err_obj;
127 	}
128 
129 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
130 	if (err)
131 		goto err_obj;
132 
133 	rq = intel_context_create_request(ce);
134 	if (IS_ERR(rq)) {
135 		err = PTR_ERR(rq);
136 		goto err_pin;
137 	}
138 
139 	i915_vma_lock(vma);
140 	err = i915_request_await_object(rq, vma->obj, true);
141 	if (err == 0)
142 		err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
143 	i915_vma_unlock(vma);
144 	if (err)
145 		goto err_req;
146 
147 	srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
148 	if (INTEL_GEN(engine->i915) >= 8)
149 		srm++;
150 
151 	cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
152 	if (IS_ERR(cs)) {
153 		err = PTR_ERR(cs);
154 		goto err_req;
155 	}
156 
157 	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
158 		*cs++ = srm;
159 		*cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
160 		*cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
161 		*cs++ = 0;
162 	}
163 	intel_ring_advance(rq, cs);
164 
165 	i915_request_add(rq);
166 	i915_vma_unpin(vma);
167 
168 	return result;
169 
170 err_req:
171 	i915_request_add(rq);
172 err_pin:
173 	i915_vma_unpin(vma);
174 err_obj:
175 	i915_gem_object_put(result);
176 	return ERR_PTR(err);
177 }
178 
179 static u32
180 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
181 {
182 	i915_reg_t reg = i < engine->whitelist.count ?
183 			 engine->whitelist.list[i].reg :
184 			 RING_NOPID(engine->mmio_base);
185 
186 	return i915_mmio_reg_offset(reg);
187 }
188 
189 static void
190 print_results(const struct intel_engine_cs *engine, const u32 *results)
191 {
192 	unsigned int i;
193 
194 	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
195 		u32 expected = get_whitelist_reg(engine, i);
196 		u32 actual = results[i];
197 
198 		pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
199 			i, expected, actual);
200 	}
201 }
202 
203 static int check_whitelist(struct intel_context *ce)
204 {
205 	struct intel_engine_cs *engine = ce->engine;
206 	struct drm_i915_gem_object *results;
207 	struct intel_wedge_me wedge;
208 	u32 *vaddr;
209 	int err;
210 	int i;
211 
212 	results = read_nonprivs(ce);
213 	if (IS_ERR(results))
214 		return PTR_ERR(results);
215 
216 	err = 0;
217 	i915_gem_object_lock(results, NULL);
218 	intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
219 		err = i915_gem_object_set_to_cpu_domain(results, false);
220 
221 	if (intel_gt_is_wedged(engine->gt))
222 		err = -EIO;
223 	if (err)
224 		goto out_put;
225 
226 	vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
227 	if (IS_ERR(vaddr)) {
228 		err = PTR_ERR(vaddr);
229 		goto out_put;
230 	}
231 
232 	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
233 		u32 expected = get_whitelist_reg(engine, i);
234 		u32 actual = vaddr[i];
235 
236 		if (expected != actual) {
237 			print_results(engine, vaddr);
238 			pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
239 			       i, expected, actual);
240 
241 			err = -EINVAL;
242 			break;
243 		}
244 	}
245 
246 	i915_gem_object_unpin_map(results);
247 out_put:
248 	i915_gem_object_unlock(results);
249 	i915_gem_object_put(results);
250 	return err;
251 }
252 
253 static int do_device_reset(struct intel_engine_cs *engine)
254 {
255 	intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
256 	return 0;
257 }
258 
259 static int do_engine_reset(struct intel_engine_cs *engine)
260 {
261 	return intel_engine_reset(engine, "live_workarounds");
262 }
263 
264 static int
265 switch_to_scratch_context(struct intel_engine_cs *engine,
266 			  struct igt_spinner *spin)
267 {
268 	struct intel_context *ce;
269 	struct i915_request *rq;
270 	int err = 0;
271 
272 	ce = intel_context_create(engine);
273 	if (IS_ERR(ce))
274 		return PTR_ERR(ce);
275 
276 	rq = igt_spinner_create_request(spin, ce, MI_NOOP);
277 	intel_context_put(ce);
278 
279 	if (IS_ERR(rq)) {
280 		spin = NULL;
281 		err = PTR_ERR(rq);
282 		goto err;
283 	}
284 
285 	err = request_add_spin(rq, spin);
286 err:
287 	if (err && spin)
288 		igt_spinner_end(spin);
289 
290 	return err;
291 }
292 
293 static int check_whitelist_across_reset(struct intel_engine_cs *engine,
294 					int (*reset)(struct intel_engine_cs *),
295 					const char *name)
296 {
297 	struct intel_context *ce, *tmp;
298 	struct igt_spinner spin;
299 	intel_wakeref_t wakeref;
300 	int err;
301 
302 	pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
303 		engine->whitelist.count, engine->name, name);
304 
305 	ce = intel_context_create(engine);
306 	if (IS_ERR(ce))
307 		return PTR_ERR(ce);
308 
309 	err = igt_spinner_init(&spin, engine->gt);
310 	if (err)
311 		goto out_ctx;
312 
313 	err = check_whitelist(ce);
314 	if (err) {
315 		pr_err("Invalid whitelist *before* %s reset!\n", name);
316 		goto out_spin;
317 	}
318 
319 	err = switch_to_scratch_context(engine, &spin);
320 	if (err)
321 		goto out_spin;
322 
323 	with_intel_runtime_pm(engine->uncore->rpm, wakeref)
324 		err = reset(engine);
325 
326 	igt_spinner_end(&spin);
327 
328 	if (err) {
329 		pr_err("%s reset failed\n", name);
330 		goto out_spin;
331 	}
332 
333 	err = check_whitelist(ce);
334 	if (err) {
335 		pr_err("Whitelist not preserved in context across %s reset!\n",
336 		       name);
337 		goto out_spin;
338 	}
339 
340 	tmp = intel_context_create(engine);
341 	if (IS_ERR(tmp)) {
342 		err = PTR_ERR(tmp);
343 		goto out_spin;
344 	}
345 	intel_context_put(ce);
346 	ce = tmp;
347 
348 	err = check_whitelist(ce);
349 	if (err) {
350 		pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
351 		       name);
352 		goto out_spin;
353 	}
354 
355 out_spin:
356 	igt_spinner_fini(&spin);
357 out_ctx:
358 	intel_context_put(ce);
359 	return err;
360 }
361 
362 static struct i915_vma *create_batch(struct i915_address_space *vm)
363 {
364 	struct drm_i915_gem_object *obj;
365 	struct i915_vma *vma;
366 	int err;
367 
368 	obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
369 	if (IS_ERR(obj))
370 		return ERR_CAST(obj);
371 
372 	vma = i915_vma_instance(obj, vm, NULL);
373 	if (IS_ERR(vma)) {
374 		err = PTR_ERR(vma);
375 		goto err_obj;
376 	}
377 
378 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
379 	if (err)
380 		goto err_obj;
381 
382 	return vma;
383 
384 err_obj:
385 	i915_gem_object_put(obj);
386 	return ERR_PTR(err);
387 }
388 
389 static u32 reg_write(u32 old, u32 new, u32 rsvd)
390 {
391 	if (rsvd == 0x0000ffff) {
392 		old &= ~(new >> 16);
393 		old |= new & (new >> 16);
394 	} else {
395 		old &= ~rsvd;
396 		old |= new & rsvd;
397 	}
398 
399 	return old;
400 }
401 
402 static bool wo_register(struct intel_engine_cs *engine, u32 reg)
403 {
404 	enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
405 	int i;
406 
407 	if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
408 	     RING_FORCE_TO_NONPRIV_ACCESS_WR)
409 		return true;
410 
411 	for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
412 		if (wo_registers[i].platform == platform &&
413 		    wo_registers[i].reg == reg)
414 			return true;
415 	}
416 
417 	return false;
418 }
419 
420 static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
421 {
422 	reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
423 	switch (reg) {
424 	case 0x358:
425 	case 0x35c:
426 	case 0x3a8:
427 		return true;
428 
429 	default:
430 		return false;
431 	}
432 }
433 
434 static bool ro_register(u32 reg)
435 {
436 	if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
437 	     RING_FORCE_TO_NONPRIV_ACCESS_RD)
438 		return true;
439 
440 	return false;
441 }
442 
443 static int whitelist_writable_count(struct intel_engine_cs *engine)
444 {
445 	int count = engine->whitelist.count;
446 	int i;
447 
448 	for (i = 0; i < engine->whitelist.count; i++) {
449 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
450 
451 		if (ro_register(reg))
452 			count--;
453 	}
454 
455 	return count;
456 }
457 
458 static int check_dirty_whitelist(struct intel_context *ce)
459 {
460 	const u32 values[] = {
461 		0x00000000,
462 		0x01010101,
463 		0x10100101,
464 		0x03030303,
465 		0x30300303,
466 		0x05050505,
467 		0x50500505,
468 		0x0f0f0f0f,
469 		0xf00ff00f,
470 		0x10101010,
471 		0xf0f01010,
472 		0x30303030,
473 		0xa0a03030,
474 		0x50505050,
475 		0xc0c05050,
476 		0xf0f0f0f0,
477 		0x11111111,
478 		0x33333333,
479 		0x55555555,
480 		0x0000ffff,
481 		0x00ff00ff,
482 		0xff0000ff,
483 		0xffff00ff,
484 		0xffffffff,
485 	};
486 	struct intel_engine_cs *engine = ce->engine;
487 	struct i915_vma *scratch;
488 	struct i915_vma *batch;
489 	int err = 0, i, v, sz;
490 	u32 *cs, *results;
491 
492 	sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
493 	scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
494 	if (IS_ERR(scratch))
495 		return PTR_ERR(scratch);
496 
497 	batch = create_batch(ce->vm);
498 	if (IS_ERR(batch)) {
499 		err = PTR_ERR(batch);
500 		goto out_scratch;
501 	}
502 
503 	for (i = 0; i < engine->whitelist.count; i++) {
504 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
505 		struct i915_gem_ww_ctx ww;
506 		u64 addr = scratch->node.start;
507 		struct i915_request *rq;
508 		u32 srm, lrm, rsvd;
509 		u32 expect;
510 		int idx;
511 		bool ro_reg;
512 
513 		if (wo_register(engine, reg))
514 			continue;
515 
516 		if (timestamp(engine, reg))
517 			continue; /* timestamps are expected to autoincrement */
518 
519 		ro_reg = ro_register(reg);
520 
521 		i915_gem_ww_ctx_init(&ww, false);
522 retry:
523 		cs = NULL;
524 		err = i915_gem_object_lock(scratch->obj, &ww);
525 		if (!err)
526 			err = i915_gem_object_lock(batch->obj, &ww);
527 		if (!err)
528 			err = intel_context_pin_ww(ce, &ww);
529 		if (err)
530 			goto out;
531 
532 		cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
533 		if (IS_ERR(cs)) {
534 			err = PTR_ERR(cs);
535 			goto out_ctx;
536 		}
537 
538 		results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
539 		if (IS_ERR(results)) {
540 			err = PTR_ERR(results);
541 			goto out_unmap_batch;
542 		}
543 
544 		/* Clear non priv flags */
545 		reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
546 
547 		srm = MI_STORE_REGISTER_MEM;
548 		lrm = MI_LOAD_REGISTER_MEM;
549 		if (INTEL_GEN(engine->i915) >= 8)
550 			lrm++, srm++;
551 
552 		pr_debug("%s: Writing garbage to %x\n",
553 			 engine->name, reg);
554 
555 		/* SRM original */
556 		*cs++ = srm;
557 		*cs++ = reg;
558 		*cs++ = lower_32_bits(addr);
559 		*cs++ = upper_32_bits(addr);
560 
561 		idx = 1;
562 		for (v = 0; v < ARRAY_SIZE(values); v++) {
563 			/* LRI garbage */
564 			*cs++ = MI_LOAD_REGISTER_IMM(1);
565 			*cs++ = reg;
566 			*cs++ = values[v];
567 
568 			/* SRM result */
569 			*cs++ = srm;
570 			*cs++ = reg;
571 			*cs++ = lower_32_bits(addr + sizeof(u32) * idx);
572 			*cs++ = upper_32_bits(addr + sizeof(u32) * idx);
573 			idx++;
574 		}
575 		for (v = 0; v < ARRAY_SIZE(values); v++) {
576 			/* LRI garbage */
577 			*cs++ = MI_LOAD_REGISTER_IMM(1);
578 			*cs++ = reg;
579 			*cs++ = ~values[v];
580 
581 			/* SRM result */
582 			*cs++ = srm;
583 			*cs++ = reg;
584 			*cs++ = lower_32_bits(addr + sizeof(u32) * idx);
585 			*cs++ = upper_32_bits(addr + sizeof(u32) * idx);
586 			idx++;
587 		}
588 		GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
589 
590 		/* LRM original -- don't leave garbage in the context! */
591 		*cs++ = lrm;
592 		*cs++ = reg;
593 		*cs++ = lower_32_bits(addr);
594 		*cs++ = upper_32_bits(addr);
595 
596 		*cs++ = MI_BATCH_BUFFER_END;
597 
598 		i915_gem_object_flush_map(batch->obj);
599 		i915_gem_object_unpin_map(batch->obj);
600 		intel_gt_chipset_flush(engine->gt);
601 		cs = NULL;
602 
603 		rq = i915_request_create(ce);
604 		if (IS_ERR(rq)) {
605 			err = PTR_ERR(rq);
606 			goto out_unmap_scratch;
607 		}
608 
609 		if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
610 			err = engine->emit_init_breadcrumb(rq);
611 			if (err)
612 				goto err_request;
613 		}
614 
615 		err = i915_request_await_object(rq, batch->obj, false);
616 		if (err == 0)
617 			err = i915_vma_move_to_active(batch, rq, 0);
618 		if (err)
619 			goto err_request;
620 
621 		err = i915_request_await_object(rq, scratch->obj, true);
622 		if (err == 0)
623 			err = i915_vma_move_to_active(scratch, rq,
624 						      EXEC_OBJECT_WRITE);
625 		if (err)
626 			goto err_request;
627 
628 		err = engine->emit_bb_start(rq,
629 					    batch->node.start, PAGE_SIZE,
630 					    0);
631 		if (err)
632 			goto err_request;
633 
634 err_request:
635 		err = request_add_sync(rq, err);
636 		if (err) {
637 			pr_err("%s: Futzing %x timedout; cancelling test\n",
638 			       engine->name, reg);
639 			intel_gt_set_wedged(engine->gt);
640 			goto out_unmap_scratch;
641 		}
642 
643 		GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
644 		if (!ro_reg) {
645 			/* detect write masking */
646 			rsvd = results[ARRAY_SIZE(values)];
647 			if (!rsvd) {
648 				pr_err("%s: Unable to write to whitelisted register %x\n",
649 				       engine->name, reg);
650 				err = -EINVAL;
651 				goto out_unmap_scratch;
652 			}
653 		} else {
654 			rsvd = 0;
655 		}
656 
657 		expect = results[0];
658 		idx = 1;
659 		for (v = 0; v < ARRAY_SIZE(values); v++) {
660 			if (ro_reg)
661 				expect = results[0];
662 			else
663 				expect = reg_write(expect, values[v], rsvd);
664 
665 			if (results[idx] != expect)
666 				err++;
667 			idx++;
668 		}
669 		for (v = 0; v < ARRAY_SIZE(values); v++) {
670 			if (ro_reg)
671 				expect = results[0];
672 			else
673 				expect = reg_write(expect, ~values[v], rsvd);
674 
675 			if (results[idx] != expect)
676 				err++;
677 			idx++;
678 		}
679 		if (err) {
680 			pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
681 			       engine->name, err, reg);
682 
683 			if (ro_reg)
684 				pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
685 					engine->name, reg, results[0]);
686 			else
687 				pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
688 					engine->name, reg, results[0], rsvd);
689 
690 			expect = results[0];
691 			idx = 1;
692 			for (v = 0; v < ARRAY_SIZE(values); v++) {
693 				u32 w = values[v];
694 
695 				if (ro_reg)
696 					expect = results[0];
697 				else
698 					expect = reg_write(expect, w, rsvd);
699 				pr_info("Wrote %08x, read %08x, expect %08x\n",
700 					w, results[idx], expect);
701 				idx++;
702 			}
703 			for (v = 0; v < ARRAY_SIZE(values); v++) {
704 				u32 w = ~values[v];
705 
706 				if (ro_reg)
707 					expect = results[0];
708 				else
709 					expect = reg_write(expect, w, rsvd);
710 				pr_info("Wrote %08x, read %08x, expect %08x\n",
711 					w, results[idx], expect);
712 				idx++;
713 			}
714 
715 			err = -EINVAL;
716 		}
717 out_unmap_scratch:
718 		i915_gem_object_unpin_map(scratch->obj);
719 out_unmap_batch:
720 		if (cs)
721 			i915_gem_object_unpin_map(batch->obj);
722 out_ctx:
723 		intel_context_unpin(ce);
724 out:
725 		if (err == -EDEADLK) {
726 			err = i915_gem_ww_ctx_backoff(&ww);
727 			if (!err)
728 				goto retry;
729 		}
730 		i915_gem_ww_ctx_fini(&ww);
731 		if (err)
732 			break;
733 	}
734 
735 	if (igt_flush_test(engine->i915))
736 		err = -EIO;
737 
738 	i915_vma_unpin_and_release(&batch, 0);
739 out_scratch:
740 	i915_vma_unpin_and_release(&scratch, 0);
741 	return err;
742 }
743 
744 static int live_dirty_whitelist(void *arg)
745 {
746 	struct intel_gt *gt = arg;
747 	struct intel_engine_cs *engine;
748 	enum intel_engine_id id;
749 
750 	/* Can the user write to the whitelisted registers? */
751 
752 	if (INTEL_GEN(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
753 		return 0;
754 
755 	for_each_engine(engine, gt, id) {
756 		struct intel_context *ce;
757 		int err;
758 
759 		if (engine->whitelist.count == 0)
760 			continue;
761 
762 		ce = intel_context_create(engine);
763 		if (IS_ERR(ce))
764 			return PTR_ERR(ce);
765 
766 		err = check_dirty_whitelist(ce);
767 		intel_context_put(ce);
768 		if (err)
769 			return err;
770 	}
771 
772 	return 0;
773 }
774 
775 static int live_reset_whitelist(void *arg)
776 {
777 	struct intel_gt *gt = arg;
778 	struct intel_engine_cs *engine;
779 	enum intel_engine_id id;
780 	int err = 0;
781 
782 	/* If we reset the gpu, we should not lose the RING_NONPRIV */
783 	igt_global_reset_lock(gt);
784 
785 	for_each_engine(engine, gt, id) {
786 		if (engine->whitelist.count == 0)
787 			continue;
788 
789 		if (intel_has_reset_engine(gt)) {
790 			err = check_whitelist_across_reset(engine,
791 							   do_engine_reset,
792 							   "engine");
793 			if (err)
794 				goto out;
795 		}
796 
797 		if (intel_has_gpu_reset(gt)) {
798 			err = check_whitelist_across_reset(engine,
799 							   do_device_reset,
800 							   "device");
801 			if (err)
802 				goto out;
803 		}
804 	}
805 
806 out:
807 	igt_global_reset_unlock(gt);
808 	return err;
809 }
810 
811 static int read_whitelisted_registers(struct intel_context *ce,
812 				      struct i915_vma *results)
813 {
814 	struct intel_engine_cs *engine = ce->engine;
815 	struct i915_request *rq;
816 	int i, err = 0;
817 	u32 srm, *cs;
818 
819 	rq = intel_context_create_request(ce);
820 	if (IS_ERR(rq))
821 		return PTR_ERR(rq);
822 
823 	i915_vma_lock(results);
824 	err = i915_request_await_object(rq, results->obj, true);
825 	if (err == 0)
826 		err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
827 	i915_vma_unlock(results);
828 	if (err)
829 		goto err_req;
830 
831 	srm = MI_STORE_REGISTER_MEM;
832 	if (INTEL_GEN(engine->i915) >= 8)
833 		srm++;
834 
835 	cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
836 	if (IS_ERR(cs)) {
837 		err = PTR_ERR(cs);
838 		goto err_req;
839 	}
840 
841 	for (i = 0; i < engine->whitelist.count; i++) {
842 		u64 offset = results->node.start + sizeof(u32) * i;
843 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
844 
845 		/* Clear non priv flags */
846 		reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
847 
848 		*cs++ = srm;
849 		*cs++ = reg;
850 		*cs++ = lower_32_bits(offset);
851 		*cs++ = upper_32_bits(offset);
852 	}
853 	intel_ring_advance(rq, cs);
854 
855 err_req:
856 	return request_add_sync(rq, err);
857 }
858 
859 static int scrub_whitelisted_registers(struct intel_context *ce)
860 {
861 	struct intel_engine_cs *engine = ce->engine;
862 	struct i915_request *rq;
863 	struct i915_vma *batch;
864 	int i, err = 0;
865 	u32 *cs;
866 
867 	batch = create_batch(ce->vm);
868 	if (IS_ERR(batch))
869 		return PTR_ERR(batch);
870 
871 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
872 	if (IS_ERR(cs)) {
873 		err = PTR_ERR(cs);
874 		goto err_batch;
875 	}
876 
877 	*cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
878 	for (i = 0; i < engine->whitelist.count; i++) {
879 		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
880 
881 		if (ro_register(reg))
882 			continue;
883 
884 		/* Clear non priv flags */
885 		reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
886 
887 		*cs++ = reg;
888 		*cs++ = 0xffffffff;
889 	}
890 	*cs++ = MI_BATCH_BUFFER_END;
891 
892 	i915_gem_object_flush_map(batch->obj);
893 	intel_gt_chipset_flush(engine->gt);
894 
895 	rq = intel_context_create_request(ce);
896 	if (IS_ERR(rq)) {
897 		err = PTR_ERR(rq);
898 		goto err_unpin;
899 	}
900 
901 	if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
902 		err = engine->emit_init_breadcrumb(rq);
903 		if (err)
904 			goto err_request;
905 	}
906 
907 	i915_vma_lock(batch);
908 	err = i915_request_await_object(rq, batch->obj, false);
909 	if (err == 0)
910 		err = i915_vma_move_to_active(batch, rq, 0);
911 	i915_vma_unlock(batch);
912 	if (err)
913 		goto err_request;
914 
915 	/* Perform the writes from an unprivileged "user" batch */
916 	err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
917 
918 err_request:
919 	err = request_add_sync(rq, err);
920 
921 err_unpin:
922 	i915_gem_object_unpin_map(batch->obj);
923 err_batch:
924 	i915_vma_unpin_and_release(&batch, 0);
925 	return err;
926 }
927 
928 struct regmask {
929 	i915_reg_t reg;
930 	unsigned long gen_mask;
931 };
932 
933 static bool find_reg(struct drm_i915_private *i915,
934 		     i915_reg_t reg,
935 		     const struct regmask *tbl,
936 		     unsigned long count)
937 {
938 	u32 offset = i915_mmio_reg_offset(reg);
939 
940 	while (count--) {
941 		if (INTEL_INFO(i915)->gen_mask & tbl->gen_mask &&
942 		    i915_mmio_reg_offset(tbl->reg) == offset)
943 			return true;
944 		tbl++;
945 	}
946 
947 	return false;
948 }
949 
950 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
951 {
952 	/* Alas, we must pardon some whitelists. Mistakes already made */
953 	static const struct regmask pardon[] = {
954 		{ GEN9_CTX_PREEMPT_REG, INTEL_GEN_MASK(9, 9) },
955 		{ GEN8_L3SQCREG4, INTEL_GEN_MASK(9, 9) },
956 	};
957 
958 	return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
959 }
960 
961 static bool result_eq(struct intel_engine_cs *engine,
962 		      u32 a, u32 b, i915_reg_t reg)
963 {
964 	if (a != b && !pardon_reg(engine->i915, reg)) {
965 		pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
966 		       i915_mmio_reg_offset(reg), a, b);
967 		return false;
968 	}
969 
970 	return true;
971 }
972 
973 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
974 {
975 	/* Some registers do not seem to behave and our writes unreadable */
976 	static const struct regmask wo[] = {
977 		{ GEN9_SLICE_COMMON_ECO_CHICKEN1, INTEL_GEN_MASK(9, 9) },
978 	};
979 
980 	return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
981 }
982 
983 static bool result_neq(struct intel_engine_cs *engine,
984 		       u32 a, u32 b, i915_reg_t reg)
985 {
986 	if (a == b && !writeonly_reg(engine->i915, reg)) {
987 		pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
988 		       i915_mmio_reg_offset(reg), a);
989 		return false;
990 	}
991 
992 	return true;
993 }
994 
995 static int
996 check_whitelisted_registers(struct intel_engine_cs *engine,
997 			    struct i915_vma *A,
998 			    struct i915_vma *B,
999 			    bool (*fn)(struct intel_engine_cs *engine,
1000 				       u32 a, u32 b,
1001 				       i915_reg_t reg))
1002 {
1003 	u32 *a, *b;
1004 	int i, err;
1005 
1006 	a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1007 	if (IS_ERR(a))
1008 		return PTR_ERR(a);
1009 
1010 	b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1011 	if (IS_ERR(b)) {
1012 		err = PTR_ERR(b);
1013 		goto err_a;
1014 	}
1015 
1016 	err = 0;
1017 	for (i = 0; i < engine->whitelist.count; i++) {
1018 		const struct i915_wa *wa = &engine->whitelist.list[i];
1019 
1020 		if (i915_mmio_reg_offset(wa->reg) &
1021 		    RING_FORCE_TO_NONPRIV_ACCESS_RD)
1022 			continue;
1023 
1024 		if (!fn(engine, a[i], b[i], wa->reg))
1025 			err = -EINVAL;
1026 	}
1027 
1028 	i915_gem_object_unpin_map(B->obj);
1029 err_a:
1030 	i915_gem_object_unpin_map(A->obj);
1031 	return err;
1032 }
1033 
1034 static int live_isolated_whitelist(void *arg)
1035 {
1036 	struct intel_gt *gt = arg;
1037 	struct {
1038 		struct i915_vma *scratch[2];
1039 	} client[2] = {};
1040 	struct intel_engine_cs *engine;
1041 	enum intel_engine_id id;
1042 	int i, err = 0;
1043 
1044 	/*
1045 	 * Check that a write into a whitelist register works, but
1046 	 * invisible to a second context.
1047 	 */
1048 
1049 	if (!intel_engines_has_context_isolation(gt->i915))
1050 		return 0;
1051 
1052 	for (i = 0; i < ARRAY_SIZE(client); i++) {
1053 		client[i].scratch[0] =
1054 			__vm_create_scratch_for_read_pinned(gt->vm, 4096);
1055 		if (IS_ERR(client[i].scratch[0])) {
1056 			err = PTR_ERR(client[i].scratch[0]);
1057 			goto err;
1058 		}
1059 
1060 		client[i].scratch[1] =
1061 			__vm_create_scratch_for_read_pinned(gt->vm, 4096);
1062 		if (IS_ERR(client[i].scratch[1])) {
1063 			err = PTR_ERR(client[i].scratch[1]);
1064 			i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1065 			goto err;
1066 		}
1067 	}
1068 
1069 	for_each_engine(engine, gt, id) {
1070 		struct intel_context *ce[2];
1071 
1072 		if (!engine->kernel_context->vm)
1073 			continue;
1074 
1075 		if (!whitelist_writable_count(engine))
1076 			continue;
1077 
1078 		ce[0] = intel_context_create(engine);
1079 		if (IS_ERR(ce[0])) {
1080 			err = PTR_ERR(ce[0]);
1081 			break;
1082 		}
1083 		ce[1] = intel_context_create(engine);
1084 		if (IS_ERR(ce[1])) {
1085 			err = PTR_ERR(ce[1]);
1086 			intel_context_put(ce[0]);
1087 			break;
1088 		}
1089 
1090 		/* Read default values */
1091 		err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1092 		if (err)
1093 			goto err_ce;
1094 
1095 		/* Try to overwrite registers (should only affect ctx0) */
1096 		err = scrub_whitelisted_registers(ce[0]);
1097 		if (err)
1098 			goto err_ce;
1099 
1100 		/* Read values from ctx1, we expect these to be defaults */
1101 		err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1102 		if (err)
1103 			goto err_ce;
1104 
1105 		/* Verify that both reads return the same default values */
1106 		err = check_whitelisted_registers(engine,
1107 						  client[0].scratch[0],
1108 						  client[1].scratch[0],
1109 						  result_eq);
1110 		if (err)
1111 			goto err_ce;
1112 
1113 		/* Read back the updated values in ctx0 */
1114 		err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1115 		if (err)
1116 			goto err_ce;
1117 
1118 		/* User should be granted privilege to overwhite regs */
1119 		err = check_whitelisted_registers(engine,
1120 						  client[0].scratch[0],
1121 						  client[0].scratch[1],
1122 						  result_neq);
1123 err_ce:
1124 		intel_context_put(ce[1]);
1125 		intel_context_put(ce[0]);
1126 		if (err)
1127 			break;
1128 	}
1129 
1130 err:
1131 	for (i = 0; i < ARRAY_SIZE(client); i++) {
1132 		i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1133 		i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1134 	}
1135 
1136 	if (igt_flush_test(gt->i915))
1137 		err = -EIO;
1138 
1139 	return err;
1140 }
1141 
1142 static bool
1143 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1144 		const char *str)
1145 {
1146 	struct intel_engine_cs *engine;
1147 	enum intel_engine_id id;
1148 	bool ok = true;
1149 
1150 	ok &= wa_list_verify(gt->uncore, &lists->gt_wa_list, str);
1151 
1152 	for_each_engine(engine, gt, id) {
1153 		struct intel_context *ce;
1154 
1155 		ce = intel_context_create(engine);
1156 		if (IS_ERR(ce))
1157 			return false;
1158 
1159 		ok &= engine_wa_list_verify(ce,
1160 					    &lists->engine[id].wa_list,
1161 					    str) == 0;
1162 
1163 		ok &= engine_wa_list_verify(ce,
1164 					    &lists->engine[id].ctx_wa_list,
1165 					    str) == 0;
1166 
1167 		intel_context_put(ce);
1168 	}
1169 
1170 	return ok;
1171 }
1172 
1173 static int
1174 live_gpu_reset_workarounds(void *arg)
1175 {
1176 	struct intel_gt *gt = arg;
1177 	intel_wakeref_t wakeref;
1178 	struct wa_lists lists;
1179 	bool ok;
1180 
1181 	if (!intel_has_gpu_reset(gt))
1182 		return 0;
1183 
1184 	pr_info("Verifying after GPU reset...\n");
1185 
1186 	igt_global_reset_lock(gt);
1187 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1188 
1189 	reference_lists_init(gt, &lists);
1190 
1191 	ok = verify_wa_lists(gt, &lists, "before reset");
1192 	if (!ok)
1193 		goto out;
1194 
1195 	intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1196 
1197 	ok = verify_wa_lists(gt, &lists, "after reset");
1198 
1199 out:
1200 	reference_lists_fini(gt, &lists);
1201 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1202 	igt_global_reset_unlock(gt);
1203 
1204 	return ok ? 0 : -ESRCH;
1205 }
1206 
1207 static int
1208 live_engine_reset_workarounds(void *arg)
1209 {
1210 	struct intel_gt *gt = arg;
1211 	struct intel_engine_cs *engine;
1212 	enum intel_engine_id id;
1213 	struct intel_context *ce;
1214 	struct igt_spinner spin;
1215 	struct i915_request *rq;
1216 	intel_wakeref_t wakeref;
1217 	struct wa_lists lists;
1218 	int ret = 0;
1219 
1220 	if (!intel_has_reset_engine(gt))
1221 		return 0;
1222 
1223 	igt_global_reset_lock(gt);
1224 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1225 
1226 	reference_lists_init(gt, &lists);
1227 
1228 	for_each_engine(engine, gt, id) {
1229 		bool ok;
1230 
1231 		pr_info("Verifying after %s reset...\n", engine->name);
1232 		ce = intel_context_create(engine);
1233 		if (IS_ERR(ce)) {
1234 			ret = PTR_ERR(ce);
1235 			break;
1236 		}
1237 
1238 		ok = verify_wa_lists(gt, &lists, "before reset");
1239 		if (!ok) {
1240 			ret = -ESRCH;
1241 			goto err;
1242 		}
1243 
1244 		ret = intel_engine_reset(engine, "live_workarounds:idle");
1245 		if (ret) {
1246 			pr_err("%s: Reset failed while idle\n", engine->name);
1247 			goto err;
1248 		}
1249 
1250 		ok = verify_wa_lists(gt, &lists, "after idle reset");
1251 		if (!ok) {
1252 			ret = -ESRCH;
1253 			goto err;
1254 		}
1255 
1256 		ret = igt_spinner_init(&spin, engine->gt);
1257 		if (ret)
1258 			goto err;
1259 
1260 		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1261 		if (IS_ERR(rq)) {
1262 			ret = PTR_ERR(rq);
1263 			igt_spinner_fini(&spin);
1264 			goto err;
1265 		}
1266 
1267 		ret = request_add_spin(rq, &spin);
1268 		if (ret) {
1269 			pr_err("%s: Spinner failed to start\n", engine->name);
1270 			igt_spinner_fini(&spin);
1271 			goto err;
1272 		}
1273 
1274 		ret = intel_engine_reset(engine, "live_workarounds:active");
1275 		if (ret) {
1276 			pr_err("%s: Reset failed on an active spinner\n",
1277 			       engine->name);
1278 			igt_spinner_fini(&spin);
1279 			goto err;
1280 		}
1281 
1282 		igt_spinner_end(&spin);
1283 		igt_spinner_fini(&spin);
1284 
1285 		ok = verify_wa_lists(gt, &lists, "after busy reset");
1286 		if (!ok) {
1287 			ret = -ESRCH;
1288 			goto err;
1289 		}
1290 
1291 err:
1292 		intel_context_put(ce);
1293 		if (ret)
1294 			break;
1295 	}
1296 
1297 	reference_lists_fini(gt, &lists);
1298 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1299 	igt_global_reset_unlock(gt);
1300 
1301 	igt_flush_test(gt->i915);
1302 
1303 	return ret;
1304 }
1305 
1306 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1307 {
1308 	static const struct i915_subtest tests[] = {
1309 		SUBTEST(live_dirty_whitelist),
1310 		SUBTEST(live_reset_whitelist),
1311 		SUBTEST(live_isolated_whitelist),
1312 		SUBTEST(live_gpu_reset_workarounds),
1313 		SUBTEST(live_engine_reset_workarounds),
1314 	};
1315 
1316 	if (intel_gt_is_wedged(&i915->gt))
1317 		return 0;
1318 
1319 	return intel_gt_live_subtests(tests, &i915->gt);
1320 }
1321