1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2017 Intel Corporation
5  */
6 
7 #include <linux/prime_numbers.h>
8 
9 #include "gt/intel_engine_pm.h"
10 #include "gt/intel_gpu_commands.h"
11 #include "gt/intel_gt.h"
12 #include "gt/intel_gt_pm.h"
13 #include "gt/intel_ring.h"
14 
15 #include "i915_selftest.h"
16 #include "selftests/i915_random.h"
17 
18 struct context {
19 	struct drm_i915_gem_object *obj;
20 	struct intel_engine_cs *engine;
21 };
22 
23 static int cpu_set(struct context *ctx, unsigned long offset, u32 v)
24 {
25 	unsigned int needs_clflush;
26 	struct page *page;
27 	void *map;
28 	u32 *cpu;
29 	int err;
30 
31 	i915_gem_object_lock(ctx->obj, NULL);
32 	err = i915_gem_object_prepare_write(ctx->obj, &needs_clflush);
33 	if (err)
34 		goto out;
35 
36 	page = i915_gem_object_get_page(ctx->obj, offset >> PAGE_SHIFT);
37 	map = kmap_atomic(page);
38 	cpu = map + offset_in_page(offset);
39 
40 	if (needs_clflush & CLFLUSH_BEFORE)
41 		drm_clflush_virt_range(cpu, sizeof(*cpu));
42 
43 	*cpu = v;
44 
45 	if (needs_clflush & CLFLUSH_AFTER)
46 		drm_clflush_virt_range(cpu, sizeof(*cpu));
47 
48 	kunmap_atomic(map);
49 	i915_gem_object_finish_access(ctx->obj);
50 
51 out:
52 	i915_gem_object_unlock(ctx->obj);
53 	return err;
54 }
55 
56 static int cpu_get(struct context *ctx, unsigned long offset, u32 *v)
57 {
58 	unsigned int needs_clflush;
59 	struct page *page;
60 	void *map;
61 	u32 *cpu;
62 	int err;
63 
64 	i915_gem_object_lock(ctx->obj, NULL);
65 	err = i915_gem_object_prepare_read(ctx->obj, &needs_clflush);
66 	if (err)
67 		goto out;
68 
69 	page = i915_gem_object_get_page(ctx->obj, offset >> PAGE_SHIFT);
70 	map = kmap_atomic(page);
71 	cpu = map + offset_in_page(offset);
72 
73 	if (needs_clflush & CLFLUSH_BEFORE)
74 		drm_clflush_virt_range(cpu, sizeof(*cpu));
75 
76 	*v = *cpu;
77 
78 	kunmap_atomic(map);
79 	i915_gem_object_finish_access(ctx->obj);
80 
81 out:
82 	i915_gem_object_unlock(ctx->obj);
83 	return err;
84 }
85 
86 static int gtt_set(struct context *ctx, unsigned long offset, u32 v)
87 {
88 	struct i915_vma *vma;
89 	u32 __iomem *map;
90 	int err = 0;
91 
92 	i915_gem_object_lock(ctx->obj, NULL);
93 	err = i915_gem_object_set_to_gtt_domain(ctx->obj, true);
94 	i915_gem_object_unlock(ctx->obj);
95 	if (err)
96 		return err;
97 
98 	vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, PIN_MAPPABLE);
99 	if (IS_ERR(vma))
100 		return PTR_ERR(vma);
101 
102 	intel_gt_pm_get(vma->vm->gt);
103 
104 	map = i915_vma_pin_iomap(vma);
105 	i915_vma_unpin(vma);
106 	if (IS_ERR(map)) {
107 		err = PTR_ERR(map);
108 		goto out_rpm;
109 	}
110 
111 	iowrite32(v, &map[offset / sizeof(*map)]);
112 	i915_vma_unpin_iomap(vma);
113 
114 out_rpm:
115 	intel_gt_pm_put(vma->vm->gt);
116 	return err;
117 }
118 
119 static int gtt_get(struct context *ctx, unsigned long offset, u32 *v)
120 {
121 	struct i915_vma *vma;
122 	u32 __iomem *map;
123 	int err = 0;
124 
125 	i915_gem_object_lock(ctx->obj, NULL);
126 	err = i915_gem_object_set_to_gtt_domain(ctx->obj, false);
127 	i915_gem_object_unlock(ctx->obj);
128 	if (err)
129 		return err;
130 
131 	vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, PIN_MAPPABLE);
132 	if (IS_ERR(vma))
133 		return PTR_ERR(vma);
134 
135 	intel_gt_pm_get(vma->vm->gt);
136 
137 	map = i915_vma_pin_iomap(vma);
138 	i915_vma_unpin(vma);
139 	if (IS_ERR(map)) {
140 		err = PTR_ERR(map);
141 		goto out_rpm;
142 	}
143 
144 	*v = ioread32(&map[offset / sizeof(*map)]);
145 	i915_vma_unpin_iomap(vma);
146 
147 out_rpm:
148 	intel_gt_pm_put(vma->vm->gt);
149 	return err;
150 }
151 
152 static int wc_set(struct context *ctx, unsigned long offset, u32 v)
153 {
154 	u32 *map;
155 	int err;
156 
157 	i915_gem_object_lock(ctx->obj, NULL);
158 	err = i915_gem_object_set_to_wc_domain(ctx->obj, true);
159 	i915_gem_object_unlock(ctx->obj);
160 	if (err)
161 		return err;
162 
163 	map = i915_gem_object_pin_map(ctx->obj, I915_MAP_WC);
164 	if (IS_ERR(map))
165 		return PTR_ERR(map);
166 
167 	map[offset / sizeof(*map)] = v;
168 
169 	__i915_gem_object_flush_map(ctx->obj, offset, sizeof(*map));
170 	i915_gem_object_unpin_map(ctx->obj);
171 
172 	return 0;
173 }
174 
175 static int wc_get(struct context *ctx, unsigned long offset, u32 *v)
176 {
177 	u32 *map;
178 	int err;
179 
180 	i915_gem_object_lock(ctx->obj, NULL);
181 	err = i915_gem_object_set_to_wc_domain(ctx->obj, false);
182 	i915_gem_object_unlock(ctx->obj);
183 	if (err)
184 		return err;
185 
186 	map = i915_gem_object_pin_map(ctx->obj, I915_MAP_WC);
187 	if (IS_ERR(map))
188 		return PTR_ERR(map);
189 
190 	*v = map[offset / sizeof(*map)];
191 	i915_gem_object_unpin_map(ctx->obj);
192 
193 	return 0;
194 }
195 
196 static int gpu_set(struct context *ctx, unsigned long offset, u32 v)
197 {
198 	struct i915_request *rq;
199 	struct i915_vma *vma;
200 	u32 *cs;
201 	int err;
202 
203 	i915_gem_object_lock(ctx->obj, NULL);
204 	err = i915_gem_object_set_to_gtt_domain(ctx->obj, true);
205 	if (err)
206 		goto out_unlock;
207 
208 	vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, 0);
209 	if (IS_ERR(vma)) {
210 		err = PTR_ERR(vma);
211 		goto out_unlock;
212 	}
213 
214 	rq = intel_engine_create_kernel_request(ctx->engine);
215 	if (IS_ERR(rq)) {
216 		err = PTR_ERR(rq);
217 		goto out_unpin;
218 	}
219 
220 	cs = intel_ring_begin(rq, 4);
221 	if (IS_ERR(cs)) {
222 		err = PTR_ERR(cs);
223 		goto out_rq;
224 	}
225 
226 	if (INTEL_GEN(ctx->engine->i915) >= 8) {
227 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
228 		*cs++ = lower_32_bits(i915_ggtt_offset(vma) + offset);
229 		*cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset);
230 		*cs++ = v;
231 	} else if (INTEL_GEN(ctx->engine->i915) >= 4) {
232 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
233 		*cs++ = 0;
234 		*cs++ = i915_ggtt_offset(vma) + offset;
235 		*cs++ = v;
236 	} else {
237 		*cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
238 		*cs++ = i915_ggtt_offset(vma) + offset;
239 		*cs++ = v;
240 		*cs++ = MI_NOOP;
241 	}
242 	intel_ring_advance(rq, cs);
243 
244 	err = i915_request_await_object(rq, vma->obj, true);
245 	if (err == 0)
246 		err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
247 
248 out_rq:
249 	i915_request_add(rq);
250 out_unpin:
251 	i915_vma_unpin(vma);
252 out_unlock:
253 	i915_gem_object_unlock(ctx->obj);
254 
255 	return err;
256 }
257 
258 static bool always_valid(struct context *ctx)
259 {
260 	return true;
261 }
262 
263 static bool needs_fence_registers(struct context *ctx)
264 {
265 	struct intel_gt *gt = ctx->engine->gt;
266 
267 	if (intel_gt_is_wedged(gt))
268 		return false;
269 
270 	return gt->ggtt->num_fences;
271 }
272 
273 static bool needs_mi_store_dword(struct context *ctx)
274 {
275 	if (intel_gt_is_wedged(ctx->engine->gt))
276 		return false;
277 
278 	return intel_engine_can_store_dword(ctx->engine);
279 }
280 
281 static const struct igt_coherency_mode {
282 	const char *name;
283 	int (*set)(struct context *ctx, unsigned long offset, u32 v);
284 	int (*get)(struct context *ctx, unsigned long offset, u32 *v);
285 	bool (*valid)(struct context *ctx);
286 } igt_coherency_mode[] = {
287 	{ "cpu", cpu_set, cpu_get, always_valid },
288 	{ "gtt", gtt_set, gtt_get, needs_fence_registers },
289 	{ "wc", wc_set, wc_get, always_valid },
290 	{ "gpu", gpu_set, NULL, needs_mi_store_dword },
291 	{ },
292 };
293 
294 static struct intel_engine_cs *
295 random_engine(struct drm_i915_private *i915, struct rnd_state *prng)
296 {
297 	struct intel_engine_cs *engine;
298 	unsigned int count;
299 
300 	count = 0;
301 	for_each_uabi_engine(engine, i915)
302 		count++;
303 
304 	count = i915_prandom_u32_max_state(count, prng);
305 	for_each_uabi_engine(engine, i915)
306 		if (count-- == 0)
307 			return engine;
308 
309 	return NULL;
310 }
311 
312 static int igt_gem_coherency(void *arg)
313 {
314 	const unsigned int ncachelines = PAGE_SIZE/64;
315 	struct drm_i915_private *i915 = arg;
316 	const struct igt_coherency_mode *read, *write, *over;
317 	unsigned long count, n;
318 	u32 *offsets, *values;
319 	I915_RND_STATE(prng);
320 	struct context ctx;
321 	int err = 0;
322 
323 	/*
324 	 * We repeatedly write, overwrite and read from a sequence of
325 	 * cachelines in order to try and detect incoherency (unflushed writes
326 	 * from either the CPU or GPU). Each setter/getter uses our cache
327 	 * domain API which should prevent incoherency.
328 	 */
329 
330 	offsets = kmalloc_array(ncachelines, 2*sizeof(u32), GFP_KERNEL);
331 	if (!offsets)
332 		return -ENOMEM;
333 	for (count = 0; count < ncachelines; count++)
334 		offsets[count] = count * 64 + 4 * (count % 16);
335 
336 	values = offsets + ncachelines;
337 
338 	ctx.engine = random_engine(i915, &prng);
339 	if (!ctx.engine) {
340 		err = -ENODEV;
341 		goto out_free;
342 	}
343 	pr_info("%s: using %s\n", __func__, ctx.engine->name);
344 	intel_engine_pm_get(ctx.engine);
345 
346 	for (over = igt_coherency_mode; over->name; over++) {
347 		if (!over->set)
348 			continue;
349 
350 		if (!over->valid(&ctx))
351 			continue;
352 
353 		for (write = igt_coherency_mode; write->name; write++) {
354 			if (!write->set)
355 				continue;
356 
357 			if (!write->valid(&ctx))
358 				continue;
359 
360 			for (read = igt_coherency_mode; read->name; read++) {
361 				if (!read->get)
362 					continue;
363 
364 				if (!read->valid(&ctx))
365 					continue;
366 
367 				for_each_prime_number_from(count, 1, ncachelines) {
368 					ctx.obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
369 					if (IS_ERR(ctx.obj)) {
370 						err = PTR_ERR(ctx.obj);
371 						goto out_pm;
372 					}
373 
374 					i915_random_reorder(offsets, ncachelines, &prng);
375 					for (n = 0; n < count; n++)
376 						values[n] = prandom_u32_state(&prng);
377 
378 					for (n = 0; n < count; n++) {
379 						err = over->set(&ctx, offsets[n], ~values[n]);
380 						if (err) {
381 							pr_err("Failed to set stale value[%ld/%ld] in object using %s, err=%d\n",
382 							       n, count, over->name, err);
383 							goto put_object;
384 						}
385 					}
386 
387 					for (n = 0; n < count; n++) {
388 						err = write->set(&ctx, offsets[n], values[n]);
389 						if (err) {
390 							pr_err("Failed to set value[%ld/%ld] in object using %s, err=%d\n",
391 							       n, count, write->name, err);
392 							goto put_object;
393 						}
394 					}
395 
396 					for (n = 0; n < count; n++) {
397 						u32 found;
398 
399 						err = read->get(&ctx, offsets[n], &found);
400 						if (err) {
401 							pr_err("Failed to get value[%ld/%ld] in object using %s, err=%d\n",
402 							       n, count, read->name, err);
403 							goto put_object;
404 						}
405 
406 						if (found != values[n]) {
407 							pr_err("Value[%ld/%ld] mismatch, (overwrite with %s) wrote [%s] %x read [%s] %x (inverse %x), at offset %x\n",
408 							       n, count, over->name,
409 							       write->name, values[n],
410 							       read->name, found,
411 							       ~values[n], offsets[n]);
412 							err = -EINVAL;
413 							goto put_object;
414 						}
415 					}
416 
417 					i915_gem_object_put(ctx.obj);
418 				}
419 			}
420 		}
421 	}
422 out_pm:
423 	intel_engine_pm_put(ctx.engine);
424 out_free:
425 	kfree(offsets);
426 	return err;
427 
428 put_object:
429 	i915_gem_object_put(ctx.obj);
430 	goto out_pm;
431 }
432 
433 int i915_gem_coherency_live_selftests(struct drm_i915_private *i915)
434 {
435 	static const struct i915_subtest tests[] = {
436 		SUBTEST(igt_gem_coherency),
437 	};
438 
439 	return i915_subtests(tests, i915);
440 }
441