1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2016 Intel Corporation
5  */
6 
7 #include <linux/highmem.h>
8 #include <linux/prime_numbers.h>
9 
10 #include "gem/i915_gem_internal.h"
11 #include "gem/i915_gem_region.h"
12 #include "gem/i915_gem_ttm.h"
13 #include "gem/i915_gem_ttm_move.h"
14 #include "gt/intel_engine_pm.h"
15 #include "gt/intel_gpu_commands.h"
16 #include "gt/intel_gt.h"
17 #include "gt/intel_gt_pm.h"
18 #include "gt/intel_migrate.h"
19 #include "i915_ttm_buddy_manager.h"
20 
21 #include "huge_gem_object.h"
22 #include "i915_selftest.h"
23 #include "selftests/i915_random.h"
24 #include "selftests/igt_flush_test.h"
25 #include "selftests/igt_reset.h"
26 #include "selftests/igt_mmap.h"
27 
28 struct tile {
29 	unsigned int width;
30 	unsigned int height;
31 	unsigned int stride;
32 	unsigned int size;
33 	unsigned int tiling;
34 	unsigned int swizzle;
35 };
36 
37 static u64 swizzle_bit(unsigned int bit, u64 offset)
38 {
39 	return (offset & BIT_ULL(bit)) >> (bit - 6);
40 }
41 
42 static u64 tiled_offset(const struct tile *tile, u64 v)
43 {
44 	u64 x, y;
45 
46 	if (tile->tiling == I915_TILING_NONE)
47 		return v;
48 
49 	y = div64_u64_rem(v, tile->stride, &x);
50 	v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height;
51 
52 	if (tile->tiling == I915_TILING_X) {
53 		v += y * tile->width;
54 		v += div64_u64_rem(x, tile->width, &x) << tile->size;
55 		v += x;
56 	} else if (tile->width == 128) {
57 		const unsigned int ytile_span = 16;
58 		const unsigned int ytile_height = 512;
59 
60 		v += y * ytile_span;
61 		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
62 		v += x;
63 	} else {
64 		const unsigned int ytile_span = 32;
65 		const unsigned int ytile_height = 256;
66 
67 		v += y * ytile_span;
68 		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
69 		v += x;
70 	}
71 
72 	switch (tile->swizzle) {
73 	case I915_BIT_6_SWIZZLE_9:
74 		v ^= swizzle_bit(9, v);
75 		break;
76 	case I915_BIT_6_SWIZZLE_9_10:
77 		v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
78 		break;
79 	case I915_BIT_6_SWIZZLE_9_11:
80 		v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
81 		break;
82 	case I915_BIT_6_SWIZZLE_9_10_11:
83 		v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
84 		break;
85 	}
86 
87 	return v;
88 }
89 
90 static int check_partial_mapping(struct drm_i915_gem_object *obj,
91 				 const struct tile *tile,
92 				 struct rnd_state *prng)
93 {
94 	const unsigned long npages = obj->base.size / PAGE_SIZE;
95 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
96 	struct i915_ggtt_view view;
97 	struct i915_vma *vma;
98 	unsigned long page;
99 	u32 __iomem *io;
100 	struct page *p;
101 	unsigned int n;
102 	u64 offset;
103 	u32 *cpu;
104 	int err;
105 
106 	err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
107 	if (err) {
108 		pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
109 		       tile->tiling, tile->stride, err);
110 		return err;
111 	}
112 
113 	GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
114 	GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
115 
116 	i915_gem_object_lock(obj, NULL);
117 	err = i915_gem_object_set_to_gtt_domain(obj, true);
118 	i915_gem_object_unlock(obj);
119 	if (err) {
120 		pr_err("Failed to flush to GTT write domain; err=%d\n", err);
121 		return err;
122 	}
123 
124 	page = i915_prandom_u32_max_state(npages, prng);
125 	view = compute_partial_view(obj, page, MIN_CHUNK_PAGES);
126 
127 	vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
128 	if (IS_ERR(vma)) {
129 		pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
130 		       page, (int)PTR_ERR(vma));
131 		return PTR_ERR(vma);
132 	}
133 
134 	n = page - view.partial.offset;
135 	GEM_BUG_ON(n >= view.partial.size);
136 
137 	io = i915_vma_pin_iomap(vma);
138 	i915_vma_unpin(vma);
139 	if (IS_ERR(io)) {
140 		pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
141 		       page, (int)PTR_ERR(io));
142 		err = PTR_ERR(io);
143 		goto out;
144 	}
145 
146 	iowrite32(page, io + n * PAGE_SIZE / sizeof(*io));
147 	i915_vma_unpin_iomap(vma);
148 
149 	offset = tiled_offset(tile, page << PAGE_SHIFT);
150 	if (offset >= obj->base.size)
151 		goto out;
152 
153 	intel_gt_flush_ggtt_writes(to_gt(i915));
154 
155 	p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
156 	cpu = kmap(p) + offset_in_page(offset);
157 	drm_clflush_virt_range(cpu, sizeof(*cpu));
158 	if (*cpu != (u32)page) {
159 		pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
160 		       page, n,
161 		       view.partial.offset,
162 		       view.partial.size,
163 		       vma->size >> PAGE_SHIFT,
164 		       tile->tiling ? tile_row_pages(obj) : 0,
165 		       vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
166 		       offset >> PAGE_SHIFT,
167 		       (unsigned int)offset_in_page(offset),
168 		       offset,
169 		       (u32)page, *cpu);
170 		err = -EINVAL;
171 	}
172 	*cpu = 0;
173 	drm_clflush_virt_range(cpu, sizeof(*cpu));
174 	kunmap(p);
175 
176 out:
177 	i915_gem_object_lock(obj, NULL);
178 	i915_vma_destroy(vma);
179 	i915_gem_object_unlock(obj);
180 	return err;
181 }
182 
183 static int check_partial_mappings(struct drm_i915_gem_object *obj,
184 				  const struct tile *tile,
185 				  unsigned long end_time)
186 {
187 	const unsigned int nreal = obj->scratch / PAGE_SIZE;
188 	const unsigned long npages = obj->base.size / PAGE_SIZE;
189 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
190 	struct i915_vma *vma;
191 	unsigned long page;
192 	int err;
193 
194 	err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
195 	if (err) {
196 		pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
197 		       tile->tiling, tile->stride, err);
198 		return err;
199 	}
200 
201 	GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
202 	GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
203 
204 	i915_gem_object_lock(obj, NULL);
205 	err = i915_gem_object_set_to_gtt_domain(obj, true);
206 	i915_gem_object_unlock(obj);
207 	if (err) {
208 		pr_err("Failed to flush to GTT write domain; err=%d\n", err);
209 		return err;
210 	}
211 
212 	for_each_prime_number_from(page, 1, npages) {
213 		struct i915_ggtt_view view =
214 			compute_partial_view(obj, page, MIN_CHUNK_PAGES);
215 		u32 __iomem *io;
216 		struct page *p;
217 		unsigned int n;
218 		u64 offset;
219 		u32 *cpu;
220 
221 		GEM_BUG_ON(view.partial.size > nreal);
222 		cond_resched();
223 
224 		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
225 		if (IS_ERR(vma)) {
226 			pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
227 			       page, (int)PTR_ERR(vma));
228 			return PTR_ERR(vma);
229 		}
230 
231 		n = page - view.partial.offset;
232 		GEM_BUG_ON(n >= view.partial.size);
233 
234 		io = i915_vma_pin_iomap(vma);
235 		i915_vma_unpin(vma);
236 		if (IS_ERR(io)) {
237 			pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
238 			       page, (int)PTR_ERR(io));
239 			return PTR_ERR(io);
240 		}
241 
242 		iowrite32(page, io + n * PAGE_SIZE / sizeof(*io));
243 		i915_vma_unpin_iomap(vma);
244 
245 		offset = tiled_offset(tile, page << PAGE_SHIFT);
246 		if (offset >= obj->base.size)
247 			continue;
248 
249 		intel_gt_flush_ggtt_writes(to_gt(i915));
250 
251 		p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
252 		cpu = kmap(p) + offset_in_page(offset);
253 		drm_clflush_virt_range(cpu, sizeof(*cpu));
254 		if (*cpu != (u32)page) {
255 			pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
256 			       page, n,
257 			       view.partial.offset,
258 			       view.partial.size,
259 			       vma->size >> PAGE_SHIFT,
260 			       tile->tiling ? tile_row_pages(obj) : 0,
261 			       vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
262 			       offset >> PAGE_SHIFT,
263 			       (unsigned int)offset_in_page(offset),
264 			       offset,
265 			       (u32)page, *cpu);
266 			err = -EINVAL;
267 		}
268 		*cpu = 0;
269 		drm_clflush_virt_range(cpu, sizeof(*cpu));
270 		kunmap(p);
271 		if (err)
272 			return err;
273 
274 		i915_gem_object_lock(obj, NULL);
275 		i915_vma_destroy(vma);
276 		i915_gem_object_unlock(obj);
277 
278 		if (igt_timeout(end_time,
279 				"%s: timed out after tiling=%d stride=%d\n",
280 				__func__, tile->tiling, tile->stride))
281 			return -EINTR;
282 	}
283 
284 	return 0;
285 }
286 
287 static unsigned int
288 setup_tile_size(struct tile *tile, struct drm_i915_private *i915)
289 {
290 	if (GRAPHICS_VER(i915) <= 2) {
291 		tile->height = 16;
292 		tile->width = 128;
293 		tile->size = 11;
294 	} else if (tile->tiling == I915_TILING_Y &&
295 		   HAS_128_BYTE_Y_TILING(i915)) {
296 		tile->height = 32;
297 		tile->width = 128;
298 		tile->size = 12;
299 	} else {
300 		tile->height = 8;
301 		tile->width = 512;
302 		tile->size = 12;
303 	}
304 
305 	if (GRAPHICS_VER(i915) < 4)
306 		return 8192 / tile->width;
307 	else if (GRAPHICS_VER(i915) < 7)
308 		return 128 * I965_FENCE_MAX_PITCH_VAL / tile->width;
309 	else
310 		return 128 * GEN7_FENCE_MAX_PITCH_VAL / tile->width;
311 }
312 
313 static int igt_partial_tiling(void *arg)
314 {
315 	const unsigned int nreal = 1 << 12; /* largest tile row x2 */
316 	struct drm_i915_private *i915 = arg;
317 	struct drm_i915_gem_object *obj;
318 	intel_wakeref_t wakeref;
319 	int tiling;
320 	int err;
321 
322 	if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
323 		return 0;
324 
325 	/* We want to check the page mapping and fencing of a large object
326 	 * mmapped through the GTT. The object we create is larger than can
327 	 * possibly be mmaped as a whole, and so we must use partial GGTT vma.
328 	 * We then check that a write through each partial GGTT vma ends up
329 	 * in the right set of pages within the object, and with the expected
330 	 * tiling, which we verify by manual swizzling.
331 	 */
332 
333 	obj = huge_gem_object(i915,
334 			      nreal << PAGE_SHIFT,
335 			      (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
336 	if (IS_ERR(obj))
337 		return PTR_ERR(obj);
338 
339 	err = i915_gem_object_pin_pages_unlocked(obj);
340 	if (err) {
341 		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
342 		       nreal, obj->base.size / PAGE_SIZE, err);
343 		goto out;
344 	}
345 
346 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
347 
348 	if (1) {
349 		IGT_TIMEOUT(end);
350 		struct tile tile;
351 
352 		tile.height = 1;
353 		tile.width = 1;
354 		tile.size = 0;
355 		tile.stride = 0;
356 		tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
357 		tile.tiling = I915_TILING_NONE;
358 
359 		err = check_partial_mappings(obj, &tile, end);
360 		if (err && err != -EINTR)
361 			goto out_unlock;
362 	}
363 
364 	for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) {
365 		IGT_TIMEOUT(end);
366 		unsigned int max_pitch;
367 		unsigned int pitch;
368 		struct tile tile;
369 
370 		if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
371 			/*
372 			 * The swizzling pattern is actually unknown as it
373 			 * varies based on physical address of each page.
374 			 * See i915_gem_detect_bit_6_swizzle().
375 			 */
376 			break;
377 
378 		tile.tiling = tiling;
379 		switch (tiling) {
380 		case I915_TILING_X:
381 			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x;
382 			break;
383 		case I915_TILING_Y:
384 			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y;
385 			break;
386 		}
387 
388 		GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN);
389 		if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
390 		    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
391 			continue;
392 
393 		max_pitch = setup_tile_size(&tile, i915);
394 
395 		for (pitch = max_pitch; pitch; pitch >>= 1) {
396 			tile.stride = tile.width * pitch;
397 			err = check_partial_mappings(obj, &tile, end);
398 			if (err == -EINTR)
399 				goto next_tiling;
400 			if (err)
401 				goto out_unlock;
402 
403 			if (pitch > 2 && GRAPHICS_VER(i915) >= 4) {
404 				tile.stride = tile.width * (pitch - 1);
405 				err = check_partial_mappings(obj, &tile, end);
406 				if (err == -EINTR)
407 					goto next_tiling;
408 				if (err)
409 					goto out_unlock;
410 			}
411 
412 			if (pitch < max_pitch && GRAPHICS_VER(i915) >= 4) {
413 				tile.stride = tile.width * (pitch + 1);
414 				err = check_partial_mappings(obj, &tile, end);
415 				if (err == -EINTR)
416 					goto next_tiling;
417 				if (err)
418 					goto out_unlock;
419 			}
420 		}
421 
422 		if (GRAPHICS_VER(i915) >= 4) {
423 			for_each_prime_number(pitch, max_pitch) {
424 				tile.stride = tile.width * pitch;
425 				err = check_partial_mappings(obj, &tile, end);
426 				if (err == -EINTR)
427 					goto next_tiling;
428 				if (err)
429 					goto out_unlock;
430 			}
431 		}
432 
433 next_tiling: ;
434 	}
435 
436 out_unlock:
437 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
438 	i915_gem_object_unpin_pages(obj);
439 out:
440 	i915_gem_object_put(obj);
441 	return err;
442 }
443 
444 static int igt_smoke_tiling(void *arg)
445 {
446 	const unsigned int nreal = 1 << 12; /* largest tile row x2 */
447 	struct drm_i915_private *i915 = arg;
448 	struct drm_i915_gem_object *obj;
449 	intel_wakeref_t wakeref;
450 	I915_RND_STATE(prng);
451 	unsigned long count;
452 	IGT_TIMEOUT(end);
453 	int err;
454 
455 	if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
456 		return 0;
457 
458 	/*
459 	 * igt_partial_tiling() does an exhastive check of partial tiling
460 	 * chunking, but will undoubtably run out of time. Here, we do a
461 	 * randomised search and hope over many runs of 1s with different
462 	 * seeds we will do a thorough check.
463 	 *
464 	 * Remember to look at the st_seed if we see a flip-flop in BAT!
465 	 */
466 
467 	if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
468 		return 0;
469 
470 	obj = huge_gem_object(i915,
471 			      nreal << PAGE_SHIFT,
472 			      (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
473 	if (IS_ERR(obj))
474 		return PTR_ERR(obj);
475 
476 	err = i915_gem_object_pin_pages_unlocked(obj);
477 	if (err) {
478 		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
479 		       nreal, obj->base.size / PAGE_SIZE, err);
480 		goto out;
481 	}
482 
483 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
484 
485 	count = 0;
486 	do {
487 		struct tile tile;
488 
489 		tile.tiling =
490 			i915_prandom_u32_max_state(I915_TILING_Y + 1, &prng);
491 		switch (tile.tiling) {
492 		case I915_TILING_NONE:
493 			tile.height = 1;
494 			tile.width = 1;
495 			tile.size = 0;
496 			tile.stride = 0;
497 			tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
498 			break;
499 
500 		case I915_TILING_X:
501 			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x;
502 			break;
503 		case I915_TILING_Y:
504 			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y;
505 			break;
506 		}
507 
508 		if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
509 		    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
510 			continue;
511 
512 		if (tile.tiling != I915_TILING_NONE) {
513 			unsigned int max_pitch = setup_tile_size(&tile, i915);
514 
515 			tile.stride =
516 				i915_prandom_u32_max_state(max_pitch, &prng);
517 			tile.stride = (1 + tile.stride) * tile.width;
518 			if (GRAPHICS_VER(i915) < 4)
519 				tile.stride = rounddown_pow_of_two(tile.stride);
520 		}
521 
522 		err = check_partial_mapping(obj, &tile, &prng);
523 		if (err)
524 			break;
525 
526 		count++;
527 	} while (!__igt_timeout(end, NULL));
528 
529 	pr_info("%s: Completed %lu trials\n", __func__, count);
530 
531 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
532 	i915_gem_object_unpin_pages(obj);
533 out:
534 	i915_gem_object_put(obj);
535 	return err;
536 }
537 
538 static int make_obj_busy(struct drm_i915_gem_object *obj)
539 {
540 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
541 	struct intel_engine_cs *engine;
542 
543 	for_each_uabi_engine(engine, i915) {
544 		struct i915_request *rq;
545 		struct i915_vma *vma;
546 		struct i915_gem_ww_ctx ww;
547 		int err;
548 
549 		vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
550 		if (IS_ERR(vma))
551 			return PTR_ERR(vma);
552 
553 		i915_gem_ww_ctx_init(&ww, false);
554 retry:
555 		err = i915_gem_object_lock(obj, &ww);
556 		if (!err)
557 			err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
558 		if (err)
559 			goto err;
560 
561 		rq = intel_engine_create_kernel_request(engine);
562 		if (IS_ERR(rq)) {
563 			err = PTR_ERR(rq);
564 			goto err_unpin;
565 		}
566 
567 		err = i915_request_await_object(rq, vma->obj, true);
568 		if (err == 0)
569 			err = i915_vma_move_to_active(vma, rq,
570 						      EXEC_OBJECT_WRITE);
571 
572 		i915_request_add(rq);
573 err_unpin:
574 		i915_vma_unpin(vma);
575 err:
576 		if (err == -EDEADLK) {
577 			err = i915_gem_ww_ctx_backoff(&ww);
578 			if (!err)
579 				goto retry;
580 		}
581 		i915_gem_ww_ctx_fini(&ww);
582 		if (err)
583 			return err;
584 	}
585 
586 	i915_gem_object_put(obj); /* leave it only alive via its active ref */
587 	return 0;
588 }
589 
590 static enum i915_mmap_type default_mapping(struct drm_i915_private *i915)
591 {
592 	if (HAS_LMEM(i915))
593 		return I915_MMAP_TYPE_FIXED;
594 
595 	return I915_MMAP_TYPE_GTT;
596 }
597 
598 static struct drm_i915_gem_object *
599 create_sys_or_internal(struct drm_i915_private *i915,
600 		       unsigned long size)
601 {
602 	if (HAS_LMEM(i915)) {
603 		struct intel_memory_region *sys_region =
604 			i915->mm.regions[INTEL_REGION_SMEM];
605 
606 		return __i915_gem_object_create_user(i915, size, &sys_region, 1);
607 	}
608 
609 	return i915_gem_object_create_internal(i915, size);
610 }
611 
612 static bool assert_mmap_offset(struct drm_i915_private *i915,
613 			       unsigned long size,
614 			       int expected)
615 {
616 	struct drm_i915_gem_object *obj;
617 	u64 offset;
618 	int ret;
619 
620 	obj = create_sys_or_internal(i915, size);
621 	if (IS_ERR(obj))
622 		return expected && expected == PTR_ERR(obj);
623 
624 	ret = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL);
625 	i915_gem_object_put(obj);
626 
627 	return ret == expected;
628 }
629 
630 static void disable_retire_worker(struct drm_i915_private *i915)
631 {
632 	i915_gem_driver_unregister__shrinker(i915);
633 	intel_gt_pm_get(to_gt(i915));
634 	cancel_delayed_work_sync(&to_gt(i915)->requests.retire_work);
635 }
636 
637 static void restore_retire_worker(struct drm_i915_private *i915)
638 {
639 	igt_flush_test(i915);
640 	intel_gt_pm_put(to_gt(i915));
641 	i915_gem_driver_register__shrinker(i915);
642 }
643 
644 static void mmap_offset_lock(struct drm_i915_private *i915)
645 	__acquires(&i915->drm.vma_offset_manager->vm_lock)
646 {
647 	write_lock(&i915->drm.vma_offset_manager->vm_lock);
648 }
649 
650 static void mmap_offset_unlock(struct drm_i915_private *i915)
651 	__releases(&i915->drm.vma_offset_manager->vm_lock)
652 {
653 	write_unlock(&i915->drm.vma_offset_manager->vm_lock);
654 }
655 
656 static int igt_mmap_offset_exhaustion(void *arg)
657 {
658 	struct drm_i915_private *i915 = arg;
659 	struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
660 	struct drm_i915_gem_object *obj;
661 	struct drm_mm_node *hole, *next;
662 	int loop, err = 0;
663 	u64 offset;
664 	int enospc = HAS_LMEM(i915) ? -ENXIO : -ENOSPC;
665 
666 	/* Disable background reaper */
667 	disable_retire_worker(i915);
668 	GEM_BUG_ON(!to_gt(i915)->awake);
669 	intel_gt_retire_requests(to_gt(i915));
670 	i915_gem_drain_freed_objects(i915);
671 
672 	/* Trim the device mmap space to only a page */
673 	mmap_offset_lock(i915);
674 	loop = 1; /* PAGE_SIZE units */
675 	list_for_each_entry_safe(hole, next, &mm->hole_stack, hole_stack) {
676 		struct drm_mm_node *resv;
677 
678 		resv = kzalloc(sizeof(*resv), GFP_NOWAIT);
679 		if (!resv) {
680 			err = -ENOMEM;
681 			goto out_park;
682 		}
683 
684 		resv->start = drm_mm_hole_node_start(hole) + loop;
685 		resv->size = hole->hole_size - loop;
686 		resv->color = -1ul;
687 		loop = 0;
688 
689 		if (!resv->size) {
690 			kfree(resv);
691 			continue;
692 		}
693 
694 		pr_debug("Reserving hole [%llx + %llx]\n",
695 			 resv->start, resv->size);
696 
697 		err = drm_mm_reserve_node(mm, resv);
698 		if (err) {
699 			pr_err("Failed to trim VMA manager, err=%d\n", err);
700 			kfree(resv);
701 			goto out_park;
702 		}
703 	}
704 	GEM_BUG_ON(!list_is_singular(&mm->hole_stack));
705 	mmap_offset_unlock(i915);
706 
707 	/* Just fits! */
708 	if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) {
709 		pr_err("Unable to insert object into single page hole\n");
710 		err = -EINVAL;
711 		goto out;
712 	}
713 
714 	/* Too large */
715 	if (!assert_mmap_offset(i915, 2 * PAGE_SIZE, enospc)) {
716 		pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n");
717 		err = -EINVAL;
718 		goto out;
719 	}
720 
721 	/* Fill the hole, further allocation attempts should then fail */
722 	obj = create_sys_or_internal(i915, PAGE_SIZE);
723 	if (IS_ERR(obj)) {
724 		err = PTR_ERR(obj);
725 		pr_err("Unable to create object for reclaimed hole\n");
726 		goto out;
727 	}
728 
729 	err = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL);
730 	if (err) {
731 		pr_err("Unable to insert object into reclaimed hole\n");
732 		goto err_obj;
733 	}
734 
735 	if (!assert_mmap_offset(i915, PAGE_SIZE, enospc)) {
736 		pr_err("Unexpectedly succeeded in inserting object into no holes!\n");
737 		err = -EINVAL;
738 		goto err_obj;
739 	}
740 
741 	i915_gem_object_put(obj);
742 
743 	/* Now fill with busy dead objects that we expect to reap */
744 	for (loop = 0; loop < 3; loop++) {
745 		if (intel_gt_is_wedged(to_gt(i915)))
746 			break;
747 
748 		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
749 		if (IS_ERR(obj)) {
750 			err = PTR_ERR(obj);
751 			goto out;
752 		}
753 
754 		err = make_obj_busy(obj);
755 		if (err) {
756 			pr_err("[loop %d] Failed to busy the object\n", loop);
757 			goto err_obj;
758 		}
759 	}
760 
761 out:
762 	mmap_offset_lock(i915);
763 out_park:
764 	drm_mm_for_each_node_safe(hole, next, mm) {
765 		if (hole->color != -1ul)
766 			continue;
767 
768 		drm_mm_remove_node(hole);
769 		kfree(hole);
770 	}
771 	mmap_offset_unlock(i915);
772 	restore_retire_worker(i915);
773 	return err;
774 err_obj:
775 	i915_gem_object_put(obj);
776 	goto out;
777 }
778 
779 static int gtt_set(struct drm_i915_gem_object *obj)
780 {
781 	struct i915_vma *vma;
782 	void __iomem *map;
783 	int err = 0;
784 
785 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
786 	if (IS_ERR(vma))
787 		return PTR_ERR(vma);
788 
789 	intel_gt_pm_get(vma->vm->gt);
790 	map = i915_vma_pin_iomap(vma);
791 	i915_vma_unpin(vma);
792 	if (IS_ERR(map)) {
793 		err = PTR_ERR(map);
794 		goto out;
795 	}
796 
797 	memset_io(map, POISON_INUSE, obj->base.size);
798 	i915_vma_unpin_iomap(vma);
799 
800 out:
801 	intel_gt_pm_put(vma->vm->gt);
802 	return err;
803 }
804 
805 static int gtt_check(struct drm_i915_gem_object *obj)
806 {
807 	struct i915_vma *vma;
808 	void __iomem *map;
809 	int err = 0;
810 
811 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
812 	if (IS_ERR(vma))
813 		return PTR_ERR(vma);
814 
815 	intel_gt_pm_get(vma->vm->gt);
816 	map = i915_vma_pin_iomap(vma);
817 	i915_vma_unpin(vma);
818 	if (IS_ERR(map)) {
819 		err = PTR_ERR(map);
820 		goto out;
821 	}
822 
823 	if (memchr_inv((void __force *)map, POISON_FREE, obj->base.size)) {
824 		pr_err("%s: Write via mmap did not land in backing store (GTT)\n",
825 		       obj->mm.region->name);
826 		err = -EINVAL;
827 	}
828 	i915_vma_unpin_iomap(vma);
829 
830 out:
831 	intel_gt_pm_put(vma->vm->gt);
832 	return err;
833 }
834 
835 static int wc_set(struct drm_i915_gem_object *obj)
836 {
837 	void *vaddr;
838 
839 	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
840 	if (IS_ERR(vaddr))
841 		return PTR_ERR(vaddr);
842 
843 	memset(vaddr, POISON_INUSE, obj->base.size);
844 	i915_gem_object_flush_map(obj);
845 	i915_gem_object_unpin_map(obj);
846 
847 	return 0;
848 }
849 
850 static int wc_check(struct drm_i915_gem_object *obj)
851 {
852 	void *vaddr;
853 	int err = 0;
854 
855 	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
856 	if (IS_ERR(vaddr))
857 		return PTR_ERR(vaddr);
858 
859 	if (memchr_inv(vaddr, POISON_FREE, obj->base.size)) {
860 		pr_err("%s: Write via mmap did not land in backing store (WC)\n",
861 		       obj->mm.region->name);
862 		err = -EINVAL;
863 	}
864 	i915_gem_object_unpin_map(obj);
865 
866 	return err;
867 }
868 
869 static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type)
870 {
871 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
872 	bool no_map;
873 
874 	if (obj->ops->mmap_offset)
875 		return type == I915_MMAP_TYPE_FIXED;
876 	else if (type == I915_MMAP_TYPE_FIXED)
877 		return false;
878 
879 	if (type == I915_MMAP_TYPE_GTT &&
880 	    !i915_ggtt_has_aperture(to_gt(i915)->ggtt))
881 		return false;
882 
883 	i915_gem_object_lock(obj, NULL);
884 	no_map = (type != I915_MMAP_TYPE_GTT &&
885 		  !i915_gem_object_has_struct_page(obj) &&
886 		  !i915_gem_object_has_iomem(obj));
887 	i915_gem_object_unlock(obj);
888 
889 	return !no_map;
890 }
891 
892 #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24))
893 static int __igt_mmap(struct drm_i915_private *i915,
894 		      struct drm_i915_gem_object *obj,
895 		      enum i915_mmap_type type)
896 {
897 	struct vm_area_struct *area;
898 	unsigned long addr;
899 	int err, i;
900 	u64 offset;
901 
902 	if (!can_mmap(obj, type))
903 		return 0;
904 
905 	err = wc_set(obj);
906 	if (err == -ENXIO)
907 		err = gtt_set(obj);
908 	if (err)
909 		return err;
910 
911 	err = __assign_mmap_offset(obj, type, &offset, NULL);
912 	if (err)
913 		return err;
914 
915 	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
916 	if (IS_ERR_VALUE(addr))
917 		return addr;
918 
919 	pr_debug("igt_mmap(%s, %d) @ %lx\n", obj->mm.region->name, type, addr);
920 
921 	mmap_read_lock(current->mm);
922 	area = vma_lookup(current->mm, addr);
923 	mmap_read_unlock(current->mm);
924 	if (!area) {
925 		pr_err("%s: Did not create a vm_area_struct for the mmap\n",
926 		       obj->mm.region->name);
927 		err = -EINVAL;
928 		goto out_unmap;
929 	}
930 
931 	for (i = 0; i < obj->base.size / sizeof(u32); i++) {
932 		u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux)));
933 		u32 x;
934 
935 		if (get_user(x, ux)) {
936 			pr_err("%s: Unable to read from mmap, offset:%zd\n",
937 			       obj->mm.region->name, i * sizeof(x));
938 			err = -EFAULT;
939 			goto out_unmap;
940 		}
941 
942 		if (x != expand32(POISON_INUSE)) {
943 			pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n",
944 			       obj->mm.region->name,
945 			       i * sizeof(x), x, expand32(POISON_INUSE));
946 			err = -EINVAL;
947 			goto out_unmap;
948 		}
949 
950 		x = expand32(POISON_FREE);
951 		if (put_user(x, ux)) {
952 			pr_err("%s: Unable to write to mmap, offset:%zd\n",
953 			       obj->mm.region->name, i * sizeof(x));
954 			err = -EFAULT;
955 			goto out_unmap;
956 		}
957 	}
958 
959 	if (type == I915_MMAP_TYPE_GTT)
960 		intel_gt_flush_ggtt_writes(to_gt(i915));
961 
962 	err = wc_check(obj);
963 	if (err == -ENXIO)
964 		err = gtt_check(obj);
965 out_unmap:
966 	vm_munmap(addr, obj->base.size);
967 	return err;
968 }
969 
970 static int igt_mmap(void *arg)
971 {
972 	struct drm_i915_private *i915 = arg;
973 	struct intel_memory_region *mr;
974 	enum intel_region_id id;
975 
976 	for_each_memory_region(mr, i915, id) {
977 		unsigned long sizes[] = {
978 			PAGE_SIZE,
979 			mr->min_page_size,
980 			SZ_4M,
981 		};
982 		int i;
983 
984 		if (mr->private)
985 			continue;
986 
987 		for (i = 0; i < ARRAY_SIZE(sizes); i++) {
988 			struct drm_i915_gem_object *obj;
989 			int err;
990 
991 			obj = __i915_gem_object_create_user(i915, sizes[i], &mr, 1);
992 			if (obj == ERR_PTR(-ENODEV))
993 				continue;
994 
995 			if (IS_ERR(obj))
996 				return PTR_ERR(obj);
997 
998 			err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT);
999 			if (err == 0)
1000 				err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC);
1001 			if (err == 0)
1002 				err = __igt_mmap(i915, obj, I915_MMAP_TYPE_FIXED);
1003 
1004 			i915_gem_object_put(obj);
1005 			if (err)
1006 				return err;
1007 		}
1008 	}
1009 
1010 	return 0;
1011 }
1012 
1013 static void igt_close_objects(struct drm_i915_private *i915,
1014 			      struct list_head *objects)
1015 {
1016 	struct drm_i915_gem_object *obj, *on;
1017 
1018 	list_for_each_entry_safe(obj, on, objects, st_link) {
1019 		i915_gem_object_lock(obj, NULL);
1020 		if (i915_gem_object_has_pinned_pages(obj))
1021 			i915_gem_object_unpin_pages(obj);
1022 		/* No polluting the memory region between tests */
1023 		__i915_gem_object_put_pages(obj);
1024 		i915_gem_object_unlock(obj);
1025 		list_del(&obj->st_link);
1026 		i915_gem_object_put(obj);
1027 	}
1028 
1029 	cond_resched();
1030 
1031 	i915_gem_drain_freed_objects(i915);
1032 }
1033 
1034 static void igt_make_evictable(struct list_head *objects)
1035 {
1036 	struct drm_i915_gem_object *obj;
1037 
1038 	list_for_each_entry(obj, objects, st_link) {
1039 		i915_gem_object_lock(obj, NULL);
1040 		if (i915_gem_object_has_pinned_pages(obj))
1041 			i915_gem_object_unpin_pages(obj);
1042 		i915_gem_object_unlock(obj);
1043 	}
1044 
1045 	cond_resched();
1046 }
1047 
1048 static int igt_fill_mappable(struct intel_memory_region *mr,
1049 			     struct list_head *objects)
1050 {
1051 	u64 size, total;
1052 	int err;
1053 
1054 	total = 0;
1055 	size = mr->io_size;
1056 	do {
1057 		struct drm_i915_gem_object *obj;
1058 
1059 		obj = i915_gem_object_create_region(mr, size, 0, 0);
1060 		if (IS_ERR(obj)) {
1061 			err = PTR_ERR(obj);
1062 			goto err_close;
1063 		}
1064 
1065 		list_add(&obj->st_link, objects);
1066 
1067 		err = i915_gem_object_pin_pages_unlocked(obj);
1068 		if (err) {
1069 			if (err != -ENXIO && err != -ENOMEM)
1070 				goto err_close;
1071 
1072 			if (size == mr->min_page_size) {
1073 				err = 0;
1074 				break;
1075 			}
1076 
1077 			size >>= 1;
1078 			continue;
1079 		}
1080 
1081 		total += obj->base.size;
1082 	} while (1);
1083 
1084 	pr_info("%s filled=%lluMiB\n", __func__, total >> 20);
1085 	return 0;
1086 
1087 err_close:
1088 	igt_close_objects(mr->i915, objects);
1089 	return err;
1090 }
1091 
1092 static int ___igt_mmap_migrate(struct drm_i915_private *i915,
1093 			       struct drm_i915_gem_object *obj,
1094 			       unsigned long addr,
1095 			       bool unfaultable)
1096 {
1097 	struct vm_area_struct *area;
1098 	int err = 0, i;
1099 
1100 	pr_info("igt_mmap(%s, %d) @ %lx\n",
1101 		obj->mm.region->name, I915_MMAP_TYPE_FIXED, addr);
1102 
1103 	mmap_read_lock(current->mm);
1104 	area = vma_lookup(current->mm, addr);
1105 	mmap_read_unlock(current->mm);
1106 	if (!area) {
1107 		pr_err("%s: Did not create a vm_area_struct for the mmap\n",
1108 		       obj->mm.region->name);
1109 		err = -EINVAL;
1110 		goto out_unmap;
1111 	}
1112 
1113 	for (i = 0; i < obj->base.size / sizeof(u32); i++) {
1114 		u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux)));
1115 		u32 x;
1116 
1117 		if (get_user(x, ux)) {
1118 			err = -EFAULT;
1119 			if (!unfaultable) {
1120 				pr_err("%s: Unable to read from mmap, offset:%zd\n",
1121 				       obj->mm.region->name, i * sizeof(x));
1122 				goto out_unmap;
1123 			}
1124 
1125 			continue;
1126 		}
1127 
1128 		if (unfaultable) {
1129 			pr_err("%s: Faulted unmappable memory\n",
1130 			       obj->mm.region->name);
1131 			err = -EINVAL;
1132 			goto out_unmap;
1133 		}
1134 
1135 		if (x != expand32(POISON_INUSE)) {
1136 			pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n",
1137 			       obj->mm.region->name,
1138 			       i * sizeof(x), x, expand32(POISON_INUSE));
1139 			err = -EINVAL;
1140 			goto out_unmap;
1141 		}
1142 
1143 		x = expand32(POISON_FREE);
1144 		if (put_user(x, ux)) {
1145 			pr_err("%s: Unable to write to mmap, offset:%zd\n",
1146 			       obj->mm.region->name, i * sizeof(x));
1147 			err = -EFAULT;
1148 			goto out_unmap;
1149 		}
1150 	}
1151 
1152 	if (unfaultable) {
1153 		if (err == -EFAULT)
1154 			err = 0;
1155 	} else {
1156 		obj->flags &= ~I915_BO_ALLOC_GPU_ONLY;
1157 		err = wc_check(obj);
1158 	}
1159 out_unmap:
1160 	vm_munmap(addr, obj->base.size);
1161 	return err;
1162 }
1163 
1164 #define IGT_MMAP_MIGRATE_TOPDOWN     (1 << 0)
1165 #define IGT_MMAP_MIGRATE_FILL        (1 << 1)
1166 #define IGT_MMAP_MIGRATE_EVICTABLE   (1 << 2)
1167 #define IGT_MMAP_MIGRATE_UNFAULTABLE (1 << 3)
1168 #define IGT_MMAP_MIGRATE_FAIL_GPU    (1 << 4)
1169 static int __igt_mmap_migrate(struct intel_memory_region **placements,
1170 			      int n_placements,
1171 			      struct intel_memory_region *expected_mr,
1172 			      unsigned int flags)
1173 {
1174 	struct drm_i915_private *i915 = placements[0]->i915;
1175 	struct drm_i915_gem_object *obj;
1176 	struct i915_request *rq = NULL;
1177 	unsigned long addr;
1178 	LIST_HEAD(objects);
1179 	u64 offset;
1180 	int err;
1181 
1182 	obj = __i915_gem_object_create_user(i915, PAGE_SIZE,
1183 					    placements,
1184 					    n_placements);
1185 	if (IS_ERR(obj))
1186 		return PTR_ERR(obj);
1187 
1188 	if (flags & IGT_MMAP_MIGRATE_TOPDOWN)
1189 		obj->flags |= I915_BO_ALLOC_GPU_ONLY;
1190 
1191 	err = __assign_mmap_offset(obj, I915_MMAP_TYPE_FIXED, &offset, NULL);
1192 	if (err)
1193 		goto out_put;
1194 
1195 	/*
1196 	 * This will eventually create a GEM context, due to opening dummy drm
1197 	 * file, which needs a tiny amount of mappable device memory for the top
1198 	 * level paging structures(and perhaps scratch), so make sure we
1199 	 * allocate early, to avoid tears.
1200 	 */
1201 	addr = igt_mmap_offset(i915, offset, obj->base.size,
1202 			       PROT_WRITE, MAP_SHARED);
1203 	if (IS_ERR_VALUE(addr)) {
1204 		err = addr;
1205 		goto out_put;
1206 	}
1207 
1208 	if (flags & IGT_MMAP_MIGRATE_FILL) {
1209 		err = igt_fill_mappable(placements[0], &objects);
1210 		if (err)
1211 			goto out_put;
1212 	}
1213 
1214 	err = i915_gem_object_lock(obj, NULL);
1215 	if (err)
1216 		goto out_put;
1217 
1218 	err = i915_gem_object_pin_pages(obj);
1219 	if (err) {
1220 		i915_gem_object_unlock(obj);
1221 		goto out_put;
1222 	}
1223 
1224 	err = intel_context_migrate_clear(to_gt(i915)->migrate.context, NULL,
1225 					  obj->mm.pages->sgl, obj->cache_level,
1226 					  i915_gem_object_is_lmem(obj),
1227 					  expand32(POISON_INUSE), &rq);
1228 	i915_gem_object_unpin_pages(obj);
1229 	if (rq) {
1230 		err = dma_resv_reserve_fences(obj->base.resv, 1);
1231 		if (!err)
1232 			dma_resv_add_fence(obj->base.resv, &rq->fence,
1233 					   DMA_RESV_USAGE_KERNEL);
1234 		i915_request_put(rq);
1235 	}
1236 	i915_gem_object_unlock(obj);
1237 	if (err)
1238 		goto out_put;
1239 
1240 	if (flags & IGT_MMAP_MIGRATE_EVICTABLE)
1241 		igt_make_evictable(&objects);
1242 
1243 	if (flags & IGT_MMAP_MIGRATE_FAIL_GPU) {
1244 		err = i915_gem_object_lock(obj, NULL);
1245 		if (err)
1246 			goto out_put;
1247 
1248 		/*
1249 		 * Ensure we only simulate the gpu failuire when faulting the
1250 		 * pages.
1251 		 */
1252 		err = i915_gem_object_wait_moving_fence(obj, true);
1253 		i915_gem_object_unlock(obj);
1254 		if (err)
1255 			goto out_put;
1256 		i915_ttm_migrate_set_failure_modes(true, false);
1257 	}
1258 
1259 	err = ___igt_mmap_migrate(i915, obj, addr,
1260 				  flags & IGT_MMAP_MIGRATE_UNFAULTABLE);
1261 
1262 	if (!err && obj->mm.region != expected_mr) {
1263 		pr_err("%s region mismatch %s\n", __func__, expected_mr->name);
1264 		err = -EINVAL;
1265 	}
1266 
1267 	if (flags & IGT_MMAP_MIGRATE_FAIL_GPU) {
1268 		struct intel_gt *gt;
1269 		unsigned int id;
1270 
1271 		i915_ttm_migrate_set_failure_modes(false, false);
1272 
1273 		for_each_gt(gt, i915, id) {
1274 			intel_wakeref_t wakeref;
1275 			bool wedged;
1276 
1277 			mutex_lock(&gt->reset.mutex);
1278 			wedged = test_bit(I915_WEDGED, &gt->reset.flags);
1279 			mutex_unlock(&gt->reset.mutex);
1280 			if (!wedged) {
1281 				pr_err("gt(%u) not wedged\n", id);
1282 				err = -EINVAL;
1283 				continue;
1284 			}
1285 
1286 			wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1287 			igt_global_reset_lock(gt);
1288 			intel_gt_reset(gt, ALL_ENGINES, NULL);
1289 			igt_global_reset_unlock(gt);
1290 			intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1291 		}
1292 
1293 		if (!i915_gem_object_has_unknown_state(obj)) {
1294 			pr_err("object missing unknown_state\n");
1295 			err = -EINVAL;
1296 		}
1297 	}
1298 
1299 out_put:
1300 	i915_gem_object_put(obj);
1301 	igt_close_objects(i915, &objects);
1302 	return err;
1303 }
1304 
1305 static int igt_mmap_migrate(void *arg)
1306 {
1307 	struct drm_i915_private *i915 = arg;
1308 	struct intel_memory_region *system = i915->mm.regions[INTEL_REGION_SMEM];
1309 	struct intel_memory_region *mr;
1310 	enum intel_region_id id;
1311 
1312 	for_each_memory_region(mr, i915, id) {
1313 		struct intel_memory_region *mixed[] = { mr, system };
1314 		struct intel_memory_region *single[] = { mr };
1315 		struct ttm_resource_manager *man = mr->region_private;
1316 		resource_size_t saved_io_size;
1317 		int err;
1318 
1319 		if (mr->private)
1320 			continue;
1321 
1322 		if (!mr->io_size)
1323 			continue;
1324 
1325 		/*
1326 		 * For testing purposes let's force small BAR, if not already
1327 		 * present.
1328 		 */
1329 		saved_io_size = mr->io_size;
1330 		if (mr->io_size == mr->total) {
1331 			resource_size_t io_size = mr->io_size;
1332 
1333 			io_size = rounddown_pow_of_two(io_size >> 1);
1334 			if (io_size < PAGE_SIZE)
1335 				continue;
1336 
1337 			mr->io_size = io_size;
1338 			i915_ttm_buddy_man_force_visible_size(man,
1339 							      io_size >> PAGE_SHIFT);
1340 		}
1341 
1342 		/*
1343 		 * Allocate in the mappable portion, should be no suprises here.
1344 		 */
1345 		err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), mr, 0);
1346 		if (err)
1347 			goto out_io_size;
1348 
1349 		/*
1350 		 * Allocate in the non-mappable portion, but force migrating to
1351 		 * the mappable portion on fault (LMEM -> LMEM)
1352 		 */
1353 		err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr,
1354 					 IGT_MMAP_MIGRATE_TOPDOWN |
1355 					 IGT_MMAP_MIGRATE_FILL |
1356 					 IGT_MMAP_MIGRATE_EVICTABLE);
1357 		if (err)
1358 			goto out_io_size;
1359 
1360 		/*
1361 		 * Allocate in the non-mappable portion, but force spilling into
1362 		 * system memory on fault (LMEM -> SMEM)
1363 		 */
1364 		err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), system,
1365 					 IGT_MMAP_MIGRATE_TOPDOWN |
1366 					 IGT_MMAP_MIGRATE_FILL);
1367 		if (err)
1368 			goto out_io_size;
1369 
1370 		/*
1371 		 * Allocate in the non-mappable portion, but since the mappable
1372 		 * portion is already full, and we can't spill to system memory,
1373 		 * then we should expect the fault to fail.
1374 		 */
1375 		err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr,
1376 					 IGT_MMAP_MIGRATE_TOPDOWN |
1377 					 IGT_MMAP_MIGRATE_FILL |
1378 					 IGT_MMAP_MIGRATE_UNFAULTABLE);
1379 		if (err)
1380 			goto out_io_size;
1381 
1382 		/*
1383 		 * Allocate in the non-mappable portion, but force migrating to
1384 		 * the mappable portion on fault (LMEM -> LMEM). We then also
1385 		 * simulate a gpu error when moving the pages when faulting the
1386 		 * pages, which should result in wedging the gpu and returning
1387 		 * SIGBUS in the fault handler, since we can't fallback to
1388 		 * memcpy.
1389 		 */
1390 		err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr,
1391 					 IGT_MMAP_MIGRATE_TOPDOWN |
1392 					 IGT_MMAP_MIGRATE_FILL |
1393 					 IGT_MMAP_MIGRATE_EVICTABLE |
1394 					 IGT_MMAP_MIGRATE_FAIL_GPU |
1395 					 IGT_MMAP_MIGRATE_UNFAULTABLE);
1396 out_io_size:
1397 		mr->io_size = saved_io_size;
1398 		i915_ttm_buddy_man_force_visible_size(man,
1399 						      mr->io_size >> PAGE_SHIFT);
1400 		if (err)
1401 			return err;
1402 	}
1403 
1404 	return 0;
1405 }
1406 
1407 static const char *repr_mmap_type(enum i915_mmap_type type)
1408 {
1409 	switch (type) {
1410 	case I915_MMAP_TYPE_GTT: return "gtt";
1411 	case I915_MMAP_TYPE_WB: return "wb";
1412 	case I915_MMAP_TYPE_WC: return "wc";
1413 	case I915_MMAP_TYPE_UC: return "uc";
1414 	case I915_MMAP_TYPE_FIXED: return "fixed";
1415 	default: return "unknown";
1416 	}
1417 }
1418 
1419 static bool can_access(struct drm_i915_gem_object *obj)
1420 {
1421 	bool access;
1422 
1423 	i915_gem_object_lock(obj, NULL);
1424 	access = i915_gem_object_has_struct_page(obj) ||
1425 		i915_gem_object_has_iomem(obj);
1426 	i915_gem_object_unlock(obj);
1427 
1428 	return access;
1429 }
1430 
1431 static int __igt_mmap_access(struct drm_i915_private *i915,
1432 			     struct drm_i915_gem_object *obj,
1433 			     enum i915_mmap_type type)
1434 {
1435 	unsigned long __user *ptr;
1436 	unsigned long A, B;
1437 	unsigned long x, y;
1438 	unsigned long addr;
1439 	int err;
1440 	u64 offset;
1441 
1442 	memset(&A, 0xAA, sizeof(A));
1443 	memset(&B, 0xBB, sizeof(B));
1444 
1445 	if (!can_mmap(obj, type) || !can_access(obj))
1446 		return 0;
1447 
1448 	err = __assign_mmap_offset(obj, type, &offset, NULL);
1449 	if (err)
1450 		return err;
1451 
1452 	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
1453 	if (IS_ERR_VALUE(addr))
1454 		return addr;
1455 	ptr = (unsigned long __user *)addr;
1456 
1457 	err = __put_user(A, ptr);
1458 	if (err) {
1459 		pr_err("%s(%s): failed to write into user mmap\n",
1460 		       obj->mm.region->name, repr_mmap_type(type));
1461 		goto out_unmap;
1462 	}
1463 
1464 	intel_gt_flush_ggtt_writes(to_gt(i915));
1465 
1466 	err = access_process_vm(current, addr, &x, sizeof(x), 0);
1467 	if (err != sizeof(x)) {
1468 		pr_err("%s(%s): access_process_vm() read failed\n",
1469 		       obj->mm.region->name, repr_mmap_type(type));
1470 		goto out_unmap;
1471 	}
1472 
1473 	err = access_process_vm(current, addr, &B, sizeof(B), FOLL_WRITE);
1474 	if (err != sizeof(B)) {
1475 		pr_err("%s(%s): access_process_vm() write failed\n",
1476 		       obj->mm.region->name, repr_mmap_type(type));
1477 		goto out_unmap;
1478 	}
1479 
1480 	intel_gt_flush_ggtt_writes(to_gt(i915));
1481 
1482 	err = __get_user(y, ptr);
1483 	if (err) {
1484 		pr_err("%s(%s): failed to read from user mmap\n",
1485 		       obj->mm.region->name, repr_mmap_type(type));
1486 		goto out_unmap;
1487 	}
1488 
1489 	if (x != A || y != B) {
1490 		pr_err("%s(%s): failed to read/write values, found (%lx, %lx)\n",
1491 		       obj->mm.region->name, repr_mmap_type(type),
1492 		       x, y);
1493 		err = -EINVAL;
1494 		goto out_unmap;
1495 	}
1496 
1497 out_unmap:
1498 	vm_munmap(addr, obj->base.size);
1499 	return err;
1500 }
1501 
1502 static int igt_mmap_access(void *arg)
1503 {
1504 	struct drm_i915_private *i915 = arg;
1505 	struct intel_memory_region *mr;
1506 	enum intel_region_id id;
1507 
1508 	for_each_memory_region(mr, i915, id) {
1509 		struct drm_i915_gem_object *obj;
1510 		int err;
1511 
1512 		if (mr->private)
1513 			continue;
1514 
1515 		obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1);
1516 		if (obj == ERR_PTR(-ENODEV))
1517 			continue;
1518 
1519 		if (IS_ERR(obj))
1520 			return PTR_ERR(obj);
1521 
1522 		err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT);
1523 		if (err == 0)
1524 			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB);
1525 		if (err == 0)
1526 			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WC);
1527 		if (err == 0)
1528 			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_UC);
1529 		if (err == 0)
1530 			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_FIXED);
1531 
1532 		i915_gem_object_put(obj);
1533 		if (err)
1534 			return err;
1535 	}
1536 
1537 	return 0;
1538 }
1539 
1540 static int __igt_mmap_gpu(struct drm_i915_private *i915,
1541 			  struct drm_i915_gem_object *obj,
1542 			  enum i915_mmap_type type)
1543 {
1544 	struct intel_engine_cs *engine;
1545 	unsigned long addr;
1546 	u32 __user *ux;
1547 	u32 bbe;
1548 	int err;
1549 	u64 offset;
1550 
1551 	/*
1552 	 * Verify that the mmap access into the backing store aligns with
1553 	 * that of the GPU, i.e. that mmap is indeed writing into the same
1554 	 * page as being read by the GPU.
1555 	 */
1556 
1557 	if (!can_mmap(obj, type))
1558 		return 0;
1559 
1560 	err = wc_set(obj);
1561 	if (err == -ENXIO)
1562 		err = gtt_set(obj);
1563 	if (err)
1564 		return err;
1565 
1566 	err = __assign_mmap_offset(obj, type, &offset, NULL);
1567 	if (err)
1568 		return err;
1569 
1570 	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
1571 	if (IS_ERR_VALUE(addr))
1572 		return addr;
1573 
1574 	ux = u64_to_user_ptr((u64)addr);
1575 	bbe = MI_BATCH_BUFFER_END;
1576 	if (put_user(bbe, ux)) {
1577 		pr_err("%s: Unable to write to mmap\n", obj->mm.region->name);
1578 		err = -EFAULT;
1579 		goto out_unmap;
1580 	}
1581 
1582 	if (type == I915_MMAP_TYPE_GTT)
1583 		intel_gt_flush_ggtt_writes(to_gt(i915));
1584 
1585 	for_each_uabi_engine(engine, i915) {
1586 		struct i915_request *rq;
1587 		struct i915_vma *vma;
1588 		struct i915_gem_ww_ctx ww;
1589 
1590 		vma = i915_vma_instance(obj, engine->kernel_context->vm, NULL);
1591 		if (IS_ERR(vma)) {
1592 			err = PTR_ERR(vma);
1593 			goto out_unmap;
1594 		}
1595 
1596 		i915_gem_ww_ctx_init(&ww, false);
1597 retry:
1598 		err = i915_gem_object_lock(obj, &ww);
1599 		if (!err)
1600 			err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
1601 		if (err)
1602 			goto out_ww;
1603 
1604 		rq = i915_request_create(engine->kernel_context);
1605 		if (IS_ERR(rq)) {
1606 			err = PTR_ERR(rq);
1607 			goto out_unpin;
1608 		}
1609 
1610 		err = i915_request_await_object(rq, vma->obj, false);
1611 		if (err == 0)
1612 			err = i915_vma_move_to_active(vma, rq, 0);
1613 
1614 		err = engine->emit_bb_start(rq, vma->node.start, 0, 0);
1615 		i915_request_get(rq);
1616 		i915_request_add(rq);
1617 
1618 		if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1619 			struct drm_printer p =
1620 				drm_info_printer(engine->i915->drm.dev);
1621 
1622 			pr_err("%s(%s, %s): Failed to execute batch\n",
1623 			       __func__, engine->name, obj->mm.region->name);
1624 			intel_engine_dump(engine, &p,
1625 					  "%s\n", engine->name);
1626 
1627 			intel_gt_set_wedged(engine->gt);
1628 			err = -EIO;
1629 		}
1630 		i915_request_put(rq);
1631 
1632 out_unpin:
1633 		i915_vma_unpin(vma);
1634 out_ww:
1635 		if (err == -EDEADLK) {
1636 			err = i915_gem_ww_ctx_backoff(&ww);
1637 			if (!err)
1638 				goto retry;
1639 		}
1640 		i915_gem_ww_ctx_fini(&ww);
1641 		if (err)
1642 			goto out_unmap;
1643 	}
1644 
1645 out_unmap:
1646 	vm_munmap(addr, obj->base.size);
1647 	return err;
1648 }
1649 
1650 static int igt_mmap_gpu(void *arg)
1651 {
1652 	struct drm_i915_private *i915 = arg;
1653 	struct intel_memory_region *mr;
1654 	enum intel_region_id id;
1655 
1656 	for_each_memory_region(mr, i915, id) {
1657 		struct drm_i915_gem_object *obj;
1658 		int err;
1659 
1660 		if (mr->private)
1661 			continue;
1662 
1663 		obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1);
1664 		if (obj == ERR_PTR(-ENODEV))
1665 			continue;
1666 
1667 		if (IS_ERR(obj))
1668 			return PTR_ERR(obj);
1669 
1670 		err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT);
1671 		if (err == 0)
1672 			err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC);
1673 		if (err == 0)
1674 			err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_FIXED);
1675 
1676 		i915_gem_object_put(obj);
1677 		if (err)
1678 			return err;
1679 	}
1680 
1681 	return 0;
1682 }
1683 
1684 static int check_present_pte(pte_t *pte, unsigned long addr, void *data)
1685 {
1686 	if (!pte_present(*pte) || pte_none(*pte)) {
1687 		pr_err("missing PTE:%lx\n",
1688 		       (addr - (unsigned long)data) >> PAGE_SHIFT);
1689 		return -EINVAL;
1690 	}
1691 
1692 	return 0;
1693 }
1694 
1695 static int check_absent_pte(pte_t *pte, unsigned long addr, void *data)
1696 {
1697 	if (pte_present(*pte) && !pte_none(*pte)) {
1698 		pr_err("present PTE:%lx; expected to be revoked\n",
1699 		       (addr - (unsigned long)data) >> PAGE_SHIFT);
1700 		return -EINVAL;
1701 	}
1702 
1703 	return 0;
1704 }
1705 
1706 static int check_present(unsigned long addr, unsigned long len)
1707 {
1708 	return apply_to_page_range(current->mm, addr, len,
1709 				   check_present_pte, (void *)addr);
1710 }
1711 
1712 static int check_absent(unsigned long addr, unsigned long len)
1713 {
1714 	return apply_to_page_range(current->mm, addr, len,
1715 				   check_absent_pte, (void *)addr);
1716 }
1717 
1718 static int prefault_range(u64 start, u64 len)
1719 {
1720 	const char __user *addr, *end;
1721 	char __maybe_unused c;
1722 	int err;
1723 
1724 	addr = u64_to_user_ptr(start);
1725 	end = addr + len;
1726 
1727 	for (; addr < end; addr += PAGE_SIZE) {
1728 		err = __get_user(c, addr);
1729 		if (err)
1730 			return err;
1731 	}
1732 
1733 	return __get_user(c, end - 1);
1734 }
1735 
1736 static int __igt_mmap_revoke(struct drm_i915_private *i915,
1737 			     struct drm_i915_gem_object *obj,
1738 			     enum i915_mmap_type type)
1739 {
1740 	unsigned long addr;
1741 	int err;
1742 	u64 offset;
1743 
1744 	if (!can_mmap(obj, type))
1745 		return 0;
1746 
1747 	err = __assign_mmap_offset(obj, type, &offset, NULL);
1748 	if (err)
1749 		return err;
1750 
1751 	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
1752 	if (IS_ERR_VALUE(addr))
1753 		return addr;
1754 
1755 	err = prefault_range(addr, obj->base.size);
1756 	if (err)
1757 		goto out_unmap;
1758 
1759 	err = check_present(addr, obj->base.size);
1760 	if (err) {
1761 		pr_err("%s: was not present\n", obj->mm.region->name);
1762 		goto out_unmap;
1763 	}
1764 
1765 	/*
1766 	 * After unbinding the object from the GGTT, its address may be reused
1767 	 * for other objects. Ergo we have to revoke the previous mmap PTE
1768 	 * access as it no longer points to the same object.
1769 	 */
1770 	i915_gem_object_lock(obj, NULL);
1771 	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
1772 	i915_gem_object_unlock(obj);
1773 	if (err) {
1774 		pr_err("Failed to unbind object!\n");
1775 		goto out_unmap;
1776 	}
1777 
1778 	if (type != I915_MMAP_TYPE_GTT) {
1779 		i915_gem_object_lock(obj, NULL);
1780 		__i915_gem_object_put_pages(obj);
1781 		i915_gem_object_unlock(obj);
1782 		if (i915_gem_object_has_pages(obj)) {
1783 			pr_err("Failed to put-pages object!\n");
1784 			err = -EINVAL;
1785 			goto out_unmap;
1786 		}
1787 	}
1788 
1789 	err = check_absent(addr, obj->base.size);
1790 	if (err) {
1791 		pr_err("%s: was not absent\n", obj->mm.region->name);
1792 		goto out_unmap;
1793 	}
1794 
1795 out_unmap:
1796 	vm_munmap(addr, obj->base.size);
1797 	return err;
1798 }
1799 
1800 static int igt_mmap_revoke(void *arg)
1801 {
1802 	struct drm_i915_private *i915 = arg;
1803 	struct intel_memory_region *mr;
1804 	enum intel_region_id id;
1805 
1806 	for_each_memory_region(mr, i915, id) {
1807 		struct drm_i915_gem_object *obj;
1808 		int err;
1809 
1810 		if (mr->private)
1811 			continue;
1812 
1813 		obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1);
1814 		if (obj == ERR_PTR(-ENODEV))
1815 			continue;
1816 
1817 		if (IS_ERR(obj))
1818 			return PTR_ERR(obj);
1819 
1820 		err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT);
1821 		if (err == 0)
1822 			err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC);
1823 		if (err == 0)
1824 			err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_FIXED);
1825 
1826 		i915_gem_object_put(obj);
1827 		if (err)
1828 			return err;
1829 	}
1830 
1831 	return 0;
1832 }
1833 
1834 int i915_gem_mman_live_selftests(struct drm_i915_private *i915)
1835 {
1836 	static const struct i915_subtest tests[] = {
1837 		SUBTEST(igt_partial_tiling),
1838 		SUBTEST(igt_smoke_tiling),
1839 		SUBTEST(igt_mmap_offset_exhaustion),
1840 		SUBTEST(igt_mmap),
1841 		SUBTEST(igt_mmap_migrate),
1842 		SUBTEST(igt_mmap_access),
1843 		SUBTEST(igt_mmap_revoke),
1844 		SUBTEST(igt_mmap_gpu),
1845 	};
1846 
1847 	return i915_subtests(tests, i915);
1848 }
1849