1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "i915_selftest.h"
7 
8 #include "gem/i915_gem_internal.h"
9 #include "gem/i915_gem_region.h"
10 
11 #include "gen8_engine_cs.h"
12 #include "i915_gem_ww.h"
13 #include "intel_engine_regs.h"
14 #include "intel_gpu_commands.h"
15 #include "intel_context.h"
16 #include "intel_gt.h"
17 #include "intel_ring.h"
18 
19 #include "selftests/igt_flush_test.h"
20 #include "selftests/i915_random.h"
21 
22 static void vma_set_qw(struct i915_vma *vma, u64 addr, u64 val)
23 {
24 	GEM_BUG_ON(addr < i915_vma_offset(vma));
25 	GEM_BUG_ON(addr >= i915_vma_offset(vma) + i915_vma_size(vma) + sizeof(val));
26 	memset64(page_mask_bits(vma->obj->mm.mapping) +
27 		 (addr - i915_vma_offset(vma)), val, 1);
28 }
29 
30 static int
31 pte_tlbinv(struct intel_context *ce,
32 	   struct i915_vma *va,
33 	   struct i915_vma *vb,
34 	   u64 align,
35 	   void (*tlbinv)(struct i915_address_space *vm, u64 addr, u64 length),
36 	   u64 length,
37 	   struct rnd_state *prng)
38 {
39 	const unsigned int pat_index =
40 		i915_gem_get_pat_index(ce->vm->i915, I915_CACHE_NONE);
41 	struct drm_i915_gem_object *batch;
42 	struct drm_mm_node vb_node;
43 	struct i915_request *rq;
44 	struct i915_vma *vma;
45 	u64 addr;
46 	int err;
47 	u32 *cs;
48 
49 	batch = i915_gem_object_create_internal(ce->vm->i915, 4096);
50 	if (IS_ERR(batch))
51 		return PTR_ERR(batch);
52 
53 	vma = i915_vma_instance(batch, ce->vm, NULL);
54 	if (IS_ERR(vma)) {
55 		err = PTR_ERR(vma);
56 		goto out;
57 	}
58 
59 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
60 	if (err)
61 		goto out;
62 
63 	/* Pin va at random but aligned offset after vma */
64 	addr = round_up(vma->node.start + vma->node.size, align);
65 	/* MI_CONDITIONAL_BATCH_BUFFER_END limits address to 48b */
66 	addr = igt_random_offset(prng, addr, min(ce->vm->total, BIT_ULL(48)),
67 				 va->size, align);
68 	err = i915_vma_pin(va,  0, 0, addr | PIN_OFFSET_FIXED | PIN_USER);
69 	if (err) {
70 		pr_err("Cannot pin at %llx+%llx\n", addr, va->size);
71 		goto out;
72 	}
73 	GEM_BUG_ON(i915_vma_offset(va) != addr);
74 	if (vb != va) {
75 		vb_node = vb->node;
76 		vb->node = va->node; /* overwrites the _same_ PTE  */
77 	}
78 
79 	/*
80 	 * Now choose random dword at the 1st pinned page.
81 	 *
82 	 * SZ_64K pages on dg1 require that the whole PT be marked
83 	 * containing 64KiB entries. So we make sure that vma
84 	 * covers the whole PT, despite being randomly aligned to 64KiB
85 	 * and restrict our sampling to the 2MiB PT within where
86 	 * we know that we will be using 64KiB pages.
87 	 */
88 	if (align == SZ_64K)
89 		addr = round_up(addr, SZ_2M);
90 	addr = igt_random_offset(prng, addr, addr + align, 8, 8);
91 
92 	if (va != vb)
93 		pr_info("%s(%s): Sampling %llx, with alignment %llx, using PTE size %x (phys %x, sg %x), invalidate:%llx+%llx\n",
94 			ce->engine->name, va->obj->mm.region->name ?: "smem",
95 			addr, align, va->resource->page_sizes_gtt,
96 			va->page_sizes.phys, va->page_sizes.sg,
97 			addr & -length, length);
98 
99 	cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
100 	*cs++ = MI_NOOP; /* for later termination */
101 	/*
102 	 * Sample the target to see if we spot the updated backing store.
103 	 * Gen8 VCS compares immediate value with bitwise-and of two
104 	 * consecutive DWORDS pointed by addr, other gen/engines compare value
105 	 * with DWORD pointed by addr. Moreover we want to exercise DWORD size
106 	 * invalidations. To fulfill all these requirements below values
107 	 * have been chosen.
108 	 */
109 	*cs++ = MI_CONDITIONAL_BATCH_BUFFER_END | MI_DO_COMPARE | 2;
110 	*cs++ = 0; /* break if *addr == 0 */
111 	*cs++ = lower_32_bits(addr);
112 	*cs++ = upper_32_bits(addr);
113 	vma_set_qw(va, addr, -1);
114 	vma_set_qw(vb, addr, 0);
115 
116 	/* Keep sampling until we get bored */
117 	*cs++ = MI_BATCH_BUFFER_START | BIT(8) | 1;
118 	*cs++ = lower_32_bits(i915_vma_offset(vma));
119 	*cs++ = upper_32_bits(i915_vma_offset(vma));
120 
121 	i915_gem_object_flush_map(batch);
122 
123 	rq = i915_request_create(ce);
124 	if (IS_ERR(rq)) {
125 		err = PTR_ERR(rq);
126 		goto out_va;
127 	}
128 
129 	err = rq->engine->emit_bb_start(rq, i915_vma_offset(vma), 0, 0);
130 	if (err) {
131 		i915_request_add(rq);
132 		goto out_va;
133 	}
134 
135 	i915_request_get(rq);
136 	i915_request_add(rq);
137 
138 	/* Short sleep to sanitycheck the batch is spinning before we begin */
139 	msleep(10);
140 	if (va == vb) {
141 		if (!i915_request_completed(rq)) {
142 			pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n",
143 			       ce->engine->name, va->obj->mm.region->name ?: "smem",
144 			       addr, align, va->resource->page_sizes_gtt,
145 			       va->page_sizes.phys, va->page_sizes.sg);
146 			err = -EIO;
147 		}
148 	} else if (!i915_request_completed(rq)) {
149 		struct i915_vma_resource vb_res = {
150 			.bi.pages = vb->obj->mm.pages,
151 			.bi.page_sizes = vb->obj->mm.page_sizes,
152 			.start = i915_vma_offset(vb),
153 			.vma_size = i915_vma_size(vb)
154 		};
155 		unsigned int pte_flags = 0;
156 
157 		/* Flip the PTE between A and B */
158 		if (i915_gem_object_is_lmem(vb->obj))
159 			pte_flags |= PTE_LM;
160 		ce->vm->insert_entries(ce->vm, &vb_res, pat_index, pte_flags);
161 
162 		/* Flush the PTE update to concurrent HW */
163 		tlbinv(ce->vm, addr & -length, length);
164 
165 		if (wait_for(i915_request_completed(rq), HZ / 2)) {
166 			pr_err("%s: Request did not complete; the COND_BBE did not read the updated PTE\n",
167 			       ce->engine->name);
168 			err = -EINVAL;
169 		}
170 	} else {
171 		pr_err("Spinner ended unexpectedly\n");
172 		err = -EIO;
173 	}
174 	i915_request_put(rq);
175 
176 	cs = page_mask_bits(batch->mm.mapping);
177 	*cs = MI_BATCH_BUFFER_END;
178 	wmb();
179 
180 out_va:
181 	if (vb != va)
182 		vb->node = vb_node;
183 	i915_vma_unpin(va);
184 	if (i915_vma_unbind_unlocked(va))
185 		err = -EIO;
186 out:
187 	i915_gem_object_put(batch);
188 	return err;
189 }
190 
191 static struct drm_i915_gem_object *create_lmem(struct intel_gt *gt)
192 {
193 	/*
194 	 * Allocation of largest possible page size allows to test all types
195 	 * of pages.
196 	 */
197 	return i915_gem_object_create_lmem(gt->i915, SZ_1G, I915_BO_ALLOC_CONTIGUOUS);
198 }
199 
200 static struct drm_i915_gem_object *create_smem(struct intel_gt *gt)
201 {
202 	/*
203 	 * SZ_64K pages require covering the whole 2M PT (gen8 to tgl/dg1).
204 	 * While that does not require the whole 2M block to be contiguous
205 	 * it is easier to make it so, since we need that for SZ_2M pagees.
206 	 * Since we randomly offset the start of the vma, we need a 4M object
207 	 * so that there is a 2M range within it is suitable for SZ_64K PTE.
208 	 */
209 	return i915_gem_object_create_internal(gt->i915, SZ_4M);
210 }
211 
212 static int
213 mem_tlbinv(struct intel_gt *gt,
214 	   struct drm_i915_gem_object *(*create_fn)(struct intel_gt *),
215 	   void (*tlbinv)(struct i915_address_space *vm, u64 addr, u64 length))
216 {
217 	unsigned int ppgtt_size = RUNTIME_INFO(gt->i915)->ppgtt_size;
218 	struct intel_engine_cs *engine;
219 	struct drm_i915_gem_object *A, *B;
220 	struct i915_ppgtt *ppgtt;
221 	struct i915_vma *va, *vb;
222 	enum intel_engine_id id;
223 	I915_RND_STATE(prng);
224 	void *vaddr;
225 	int err;
226 
227 	/*
228 	 * Check that the TLB invalidate is able to revoke an active
229 	 * page. We load a page into a spinning COND_BBE loop and then
230 	 * remap that page to a new physical address. The old address, and
231 	 * so the loop keeps spinning, is retained in the TLB cache until
232 	 * we issue an invalidate.
233 	 */
234 
235 	A = create_fn(gt);
236 	if (IS_ERR(A))
237 		return PTR_ERR(A);
238 
239 	vaddr = i915_gem_object_pin_map_unlocked(A, I915_MAP_WC);
240 	if (IS_ERR(vaddr)) {
241 		err = PTR_ERR(vaddr);
242 		goto out_a;
243 	}
244 
245 	B = create_fn(gt);
246 	if (IS_ERR(B)) {
247 		err = PTR_ERR(B);
248 		goto out_a;
249 	}
250 
251 	vaddr = i915_gem_object_pin_map_unlocked(B, I915_MAP_WC);
252 	if (IS_ERR(vaddr)) {
253 		err = PTR_ERR(vaddr);
254 		goto out_b;
255 	}
256 
257 	GEM_BUG_ON(A->base.size != B->base.size);
258 	if ((A->mm.page_sizes.phys | B->mm.page_sizes.phys) & (A->base.size - 1))
259 		pr_warn("Failed to allocate contiguous pages for size %zx\n",
260 			A->base.size);
261 
262 	ppgtt = i915_ppgtt_create(gt, 0);
263 	if (IS_ERR(ppgtt)) {
264 		err = PTR_ERR(ppgtt);
265 		goto out_b;
266 	}
267 
268 	va = i915_vma_instance(A, &ppgtt->vm, NULL);
269 	if (IS_ERR(va)) {
270 		err = PTR_ERR(va);
271 		goto out_vm;
272 	}
273 
274 	vb = i915_vma_instance(B, &ppgtt->vm, NULL);
275 	if (IS_ERR(vb)) {
276 		err = PTR_ERR(vb);
277 		goto out_vm;
278 	}
279 
280 	err = 0;
281 	for_each_engine(engine, gt, id) {
282 		struct i915_gem_ww_ctx ww;
283 		struct intel_context *ce;
284 		int bit;
285 
286 		ce = intel_context_create(engine);
287 		if (IS_ERR(ce)) {
288 			err = PTR_ERR(ce);
289 			break;
290 		}
291 
292 		i915_vm_put(ce->vm);
293 		ce->vm = i915_vm_get(&ppgtt->vm);
294 
295 		for_i915_gem_ww(&ww, err, true)
296 			err = intel_context_pin_ww(ce, &ww);
297 		if (err)
298 			goto err_put;
299 
300 		for_each_set_bit(bit,
301 				 (unsigned long *)&RUNTIME_INFO(gt->i915)->page_sizes,
302 				 BITS_PER_TYPE(RUNTIME_INFO(gt->i915)->page_sizes)) {
303 			unsigned int len;
304 
305 			if (BIT_ULL(bit) < i915_vm_obj_min_alignment(va->vm, va->obj))
306 				continue;
307 
308 			/* sanitycheck the semaphore wake up */
309 			err = pte_tlbinv(ce, va, va,
310 					 BIT_ULL(bit),
311 					 NULL, SZ_4K,
312 					 &prng);
313 			if (err)
314 				goto err_unpin;
315 
316 			for (len = 2; len <= ppgtt_size; len = min(2 * len, ppgtt_size)) {
317 				err = pte_tlbinv(ce, va, vb,
318 						 BIT_ULL(bit),
319 						 tlbinv,
320 						 BIT_ULL(len),
321 						 &prng);
322 				if (err)
323 					goto err_unpin;
324 				if (len == ppgtt_size)
325 					break;
326 			}
327 		}
328 err_unpin:
329 		intel_context_unpin(ce);
330 err_put:
331 		intel_context_put(ce);
332 		if (err)
333 			break;
334 	}
335 
336 	if (igt_flush_test(gt->i915))
337 		err = -EIO;
338 
339 out_vm:
340 	i915_vm_put(&ppgtt->vm);
341 out_b:
342 	i915_gem_object_put(B);
343 out_a:
344 	i915_gem_object_put(A);
345 	return err;
346 }
347 
348 static void tlbinv_full(struct i915_address_space *vm, u64 addr, u64 length)
349 {
350 	intel_gt_invalidate_tlb(vm->gt, intel_gt_tlb_seqno(vm->gt) | 1);
351 }
352 
353 static int invalidate_full(void *arg)
354 {
355 	struct intel_gt *gt = arg;
356 	int err;
357 
358 	if (GRAPHICS_VER(gt->i915) < 8)
359 		return 0; /* TLB invalidate not implemented */
360 
361 	err = mem_tlbinv(gt, create_smem, tlbinv_full);
362 	if (err == 0)
363 		err = mem_tlbinv(gt, create_lmem, tlbinv_full);
364 	if (err == -ENODEV || err == -ENXIO)
365 		err = 0;
366 
367 	return err;
368 }
369 
370 int intel_tlb_live_selftests(struct drm_i915_private *i915)
371 {
372 	static const struct i915_subtest tests[] = {
373 		SUBTEST(invalidate_full),
374 	};
375 	struct intel_gt *gt;
376 	unsigned int i;
377 
378 	for_each_gt(gt, i915, i) {
379 		int err;
380 
381 		if (intel_gt_is_wedged(gt))
382 			continue;
383 
384 		err = intel_gt_live_subtests(tests, gt);
385 		if (err)
386 			return err;
387 	}
388 
389 	return 0;
390 }
391