1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2017 Intel Corporation
5  */
6 
7 #include <linux/prime_numbers.h>
8 #include <linux/string_helpers.h>
9 #include <linux/swap.h>
10 
11 #include "i915_selftest.h"
12 
13 #include "gem/i915_gem_internal.h"
14 #include "gem/i915_gem_lmem.h"
15 #include "gem/i915_gem_pm.h"
16 #include "gem/i915_gem_region.h"
17 
18 #include "gt/intel_gt.h"
19 
20 #include "igt_gem_utils.h"
21 #include "mock_context.h"
22 
23 #include "selftests/mock_drm.h"
24 #include "selftests/mock_gem_device.h"
25 #include "selftests/mock_region.h"
26 #include "selftests/i915_random.h"
27 
28 static struct i915_gem_context *hugepage_ctx(struct drm_i915_private *i915,
29 					     struct file *file)
30 {
31 	struct i915_gem_context *ctx = live_context(i915, file);
32 	struct i915_address_space *vm;
33 
34 	if (IS_ERR(ctx))
35 		return ctx;
36 
37 	vm = ctx->vm;
38 	if (vm)
39 		WRITE_ONCE(vm->scrub_64K, true);
40 
41 	return ctx;
42 }
43 
44 static const unsigned int page_sizes[] = {
45 	I915_GTT_PAGE_SIZE_2M,
46 	I915_GTT_PAGE_SIZE_64K,
47 	I915_GTT_PAGE_SIZE_4K,
48 };
49 
50 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
51 					  u64 rem)
52 {
53 	int i;
54 
55 	for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
56 		unsigned int page_size = page_sizes[i];
57 
58 		if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
59 			return page_size;
60 	}
61 
62 	return 0;
63 }
64 
65 static void huge_pages_free_pages(struct sg_table *st)
66 {
67 	struct scatterlist *sg;
68 
69 	for (sg = st->sgl; sg; sg = __sg_next(sg)) {
70 		if (sg_page(sg))
71 			__free_pages(sg_page(sg), get_order(sg->length));
72 	}
73 
74 	sg_free_table(st);
75 	kfree(st);
76 }
77 
78 static int get_huge_pages(struct drm_i915_gem_object *obj)
79 {
80 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
81 	unsigned int page_mask = obj->mm.page_mask;
82 	struct sg_table *st;
83 	struct scatterlist *sg;
84 	unsigned int sg_page_sizes;
85 	u64 rem;
86 
87 	/* restricted by sg_alloc_table */
88 	if (overflows_type(obj->base.size >> PAGE_SHIFT, unsigned int))
89 		return -E2BIG;
90 
91 	st = kmalloc(sizeof(*st), GFP);
92 	if (!st)
93 		return -ENOMEM;
94 
95 	if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
96 		kfree(st);
97 		return -ENOMEM;
98 	}
99 
100 	rem = obj->base.size;
101 	sg = st->sgl;
102 	st->nents = 0;
103 	sg_page_sizes = 0;
104 
105 	/*
106 	 * Our goal here is simple, we want to greedily fill the object from
107 	 * largest to smallest page-size, while ensuring that we use *every*
108 	 * page-size as per the given page-mask.
109 	 */
110 	do {
111 		unsigned int bit = ilog2(page_mask);
112 		unsigned int page_size = BIT(bit);
113 		int order = get_order(page_size);
114 
115 		do {
116 			struct page *page;
117 
118 			GEM_BUG_ON(order > MAX_ORDER);
119 			page = alloc_pages(GFP | __GFP_ZERO, order);
120 			if (!page)
121 				goto err;
122 
123 			sg_set_page(sg, page, page_size, 0);
124 			sg_page_sizes |= page_size;
125 			st->nents++;
126 
127 			rem -= page_size;
128 			if (!rem) {
129 				sg_mark_end(sg);
130 				break;
131 			}
132 
133 			sg = __sg_next(sg);
134 		} while ((rem - ((page_size-1) & page_mask)) >= page_size);
135 
136 		page_mask &= (page_size-1);
137 	} while (page_mask);
138 
139 	if (i915_gem_gtt_prepare_pages(obj, st))
140 		goto err;
141 
142 	GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
143 	__i915_gem_object_set_pages(obj, st);
144 
145 	return 0;
146 
147 err:
148 	sg_set_page(sg, NULL, 0, 0);
149 	sg_mark_end(sg);
150 	huge_pages_free_pages(st);
151 
152 	return -ENOMEM;
153 }
154 
155 static void put_huge_pages(struct drm_i915_gem_object *obj,
156 			   struct sg_table *pages)
157 {
158 	i915_gem_gtt_finish_pages(obj, pages);
159 	huge_pages_free_pages(pages);
160 
161 	obj->mm.dirty = false;
162 
163 	__start_cpu_write(obj);
164 }
165 
166 static const struct drm_i915_gem_object_ops huge_page_ops = {
167 	.name = "huge-gem",
168 	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
169 	.get_pages = get_huge_pages,
170 	.put_pages = put_huge_pages,
171 };
172 
173 static struct drm_i915_gem_object *
174 huge_pages_object(struct drm_i915_private *i915,
175 		  u64 size,
176 		  unsigned int page_mask)
177 {
178 	static struct lock_class_key lock_class;
179 	struct drm_i915_gem_object *obj;
180 	unsigned int cache_level;
181 
182 	GEM_BUG_ON(!size);
183 	GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
184 
185 	if (size >> PAGE_SHIFT > INT_MAX)
186 		return ERR_PTR(-E2BIG);
187 
188 	if (overflows_type(size, obj->base.size))
189 		return ERR_PTR(-E2BIG);
190 
191 	obj = i915_gem_object_alloc();
192 	if (!obj)
193 		return ERR_PTR(-ENOMEM);
194 
195 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
196 	i915_gem_object_init(obj, &huge_page_ops, &lock_class, 0);
197 	obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
198 	i915_gem_object_set_volatile(obj);
199 
200 	obj->write_domain = I915_GEM_DOMAIN_CPU;
201 	obj->read_domains = I915_GEM_DOMAIN_CPU;
202 
203 	cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
204 	i915_gem_object_set_cache_coherency(obj, cache_level);
205 
206 	obj->mm.page_mask = page_mask;
207 
208 	return obj;
209 }
210 
211 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
212 {
213 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
214 	const u64 max_len = rounddown_pow_of_two(UINT_MAX);
215 	struct sg_table *st;
216 	struct scatterlist *sg;
217 	u64 rem;
218 
219 	/* restricted by sg_alloc_table */
220 	if (overflows_type(obj->base.size >> PAGE_SHIFT, unsigned int))
221 		return -E2BIG;
222 
223 	st = kmalloc(sizeof(*st), GFP);
224 	if (!st)
225 		return -ENOMEM;
226 
227 	if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
228 		kfree(st);
229 		return -ENOMEM;
230 	}
231 
232 	/* Use optimal page sized chunks to fill in the sg table */
233 	rem = obj->base.size;
234 	sg = st->sgl;
235 	st->nents = 0;
236 	do {
237 		unsigned int page_size = get_largest_page_size(i915, rem);
238 		unsigned int len = min(page_size * div_u64(rem, page_size),
239 				       max_len);
240 
241 		GEM_BUG_ON(!page_size);
242 
243 		sg->offset = 0;
244 		sg->length = len;
245 		sg_dma_len(sg) = len;
246 		sg_dma_address(sg) = page_size;
247 
248 		st->nents++;
249 
250 		rem -= len;
251 		if (!rem) {
252 			sg_mark_end(sg);
253 			break;
254 		}
255 
256 		sg = sg_next(sg);
257 	} while (1);
258 
259 	i915_sg_trim(st);
260 
261 	__i915_gem_object_set_pages(obj, st);
262 
263 	return 0;
264 }
265 
266 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
267 {
268 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
269 	struct sg_table *st;
270 	struct scatterlist *sg;
271 	unsigned int page_size;
272 
273 	st = kmalloc(sizeof(*st), GFP);
274 	if (!st)
275 		return -ENOMEM;
276 
277 	if (sg_alloc_table(st, 1, GFP)) {
278 		kfree(st);
279 		return -ENOMEM;
280 	}
281 
282 	sg = st->sgl;
283 	st->nents = 1;
284 
285 	page_size = get_largest_page_size(i915, obj->base.size);
286 	GEM_BUG_ON(!page_size);
287 
288 	sg->offset = 0;
289 	sg->length = obj->base.size;
290 	sg_dma_len(sg) = obj->base.size;
291 	sg_dma_address(sg) = page_size;
292 
293 	__i915_gem_object_set_pages(obj, st);
294 
295 	return 0;
296 #undef GFP
297 }
298 
299 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
300 				 struct sg_table *pages)
301 {
302 	sg_free_table(pages);
303 	kfree(pages);
304 }
305 
306 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
307 				struct sg_table *pages)
308 {
309 	fake_free_huge_pages(obj, pages);
310 	obj->mm.dirty = false;
311 }
312 
313 static const struct drm_i915_gem_object_ops fake_ops = {
314 	.name = "fake-gem",
315 	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
316 	.get_pages = fake_get_huge_pages,
317 	.put_pages = fake_put_huge_pages,
318 };
319 
320 static const struct drm_i915_gem_object_ops fake_ops_single = {
321 	.name = "fake-gem",
322 	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
323 	.get_pages = fake_get_huge_pages_single,
324 	.put_pages = fake_put_huge_pages,
325 };
326 
327 static struct drm_i915_gem_object *
328 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
329 {
330 	static struct lock_class_key lock_class;
331 	struct drm_i915_gem_object *obj;
332 
333 	GEM_BUG_ON(!size);
334 	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
335 
336 	if (size >> PAGE_SHIFT > UINT_MAX)
337 		return ERR_PTR(-E2BIG);
338 
339 	if (overflows_type(size, obj->base.size))
340 		return ERR_PTR(-E2BIG);
341 
342 	obj = i915_gem_object_alloc();
343 	if (!obj)
344 		return ERR_PTR(-ENOMEM);
345 
346 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
347 
348 	if (single)
349 		i915_gem_object_init(obj, &fake_ops_single, &lock_class, 0);
350 	else
351 		i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
352 
353 	i915_gem_object_set_volatile(obj);
354 
355 	obj->write_domain = I915_GEM_DOMAIN_CPU;
356 	obj->read_domains = I915_GEM_DOMAIN_CPU;
357 	obj->cache_level = I915_CACHE_NONE;
358 
359 	return obj;
360 }
361 
362 static int igt_check_page_sizes(struct i915_vma *vma)
363 {
364 	struct drm_i915_private *i915 = vma->vm->i915;
365 	unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
366 	struct drm_i915_gem_object *obj = vma->obj;
367 	int err;
368 
369 	/* We have to wait for the async bind to complete before our asserts */
370 	err = i915_vma_sync(vma);
371 	if (err)
372 		return err;
373 
374 	if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
375 		pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
376 		       vma->page_sizes.sg & ~supported, supported);
377 		err = -EINVAL;
378 	}
379 
380 	if (!HAS_PAGE_SIZES(i915, vma->resource->page_sizes_gtt)) {
381 		pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
382 		       vma->resource->page_sizes_gtt & ~supported, supported);
383 		err = -EINVAL;
384 	}
385 
386 	if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
387 		pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
388 		       vma->page_sizes.phys, obj->mm.page_sizes.phys);
389 		err = -EINVAL;
390 	}
391 
392 	if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
393 		pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
394 		       vma->page_sizes.sg, obj->mm.page_sizes.sg);
395 		err = -EINVAL;
396 	}
397 
398 	/*
399 	 * The dma-api is like a box of chocolates when it comes to the
400 	 * alignment of dma addresses, however for LMEM we have total control
401 	 * and so can guarantee alignment, likewise when we allocate our blocks
402 	 * they should appear in descending order, and if we know that we align
403 	 * to the largest page size for the GTT address, we should be able to
404 	 * assert that if we see 2M physical pages then we should also get 2M
405 	 * GTT pages. If we don't then something might be wrong in our
406 	 * construction of the backing pages.
407 	 *
408 	 * Maintaining alignment is required to utilise huge pages in the ppGGT.
409 	 */
410 	if (i915_gem_object_is_lmem(obj) &&
411 	    IS_ALIGNED(i915_vma_offset(vma), SZ_2M) &&
412 	    vma->page_sizes.sg & SZ_2M &&
413 	    vma->resource->page_sizes_gtt < SZ_2M) {
414 		pr_err("gtt pages mismatch for LMEM, expected 2M GTT pages, sg(%u), gtt(%u)\n",
415 		       vma->page_sizes.sg, vma->resource->page_sizes_gtt);
416 		err = -EINVAL;
417 	}
418 
419 	return err;
420 }
421 
422 static int igt_mock_exhaust_device_supported_pages(void *arg)
423 {
424 	struct i915_ppgtt *ppgtt = arg;
425 	struct drm_i915_private *i915 = ppgtt->vm.i915;
426 	unsigned int saved_mask = RUNTIME_INFO(i915)->page_sizes;
427 	struct drm_i915_gem_object *obj;
428 	struct i915_vma *vma;
429 	int i, j, single;
430 	int err;
431 
432 	/*
433 	 * Sanity check creating objects with every valid page support
434 	 * combination for our mock device.
435 	 */
436 
437 	for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
438 		unsigned int combination = SZ_4K; /* Required for ppGTT */
439 
440 		for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
441 			if (i & BIT(j))
442 				combination |= page_sizes[j];
443 		}
444 
445 		RUNTIME_INFO(i915)->page_sizes = combination;
446 
447 		for (single = 0; single <= 1; ++single) {
448 			obj = fake_huge_pages_object(i915, combination, !!single);
449 			if (IS_ERR(obj)) {
450 				err = PTR_ERR(obj);
451 				goto out_device;
452 			}
453 
454 			if (obj->base.size != combination) {
455 				pr_err("obj->base.size=%zu, expected=%u\n",
456 				       obj->base.size, combination);
457 				err = -EINVAL;
458 				goto out_put;
459 			}
460 
461 			vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
462 			if (IS_ERR(vma)) {
463 				err = PTR_ERR(vma);
464 				goto out_put;
465 			}
466 
467 			err = i915_vma_pin(vma, 0, 0, PIN_USER);
468 			if (err)
469 				goto out_put;
470 
471 			err = igt_check_page_sizes(vma);
472 
473 			if (vma->page_sizes.sg != combination) {
474 				pr_err("page_sizes.sg=%u, expected=%u\n",
475 				       vma->page_sizes.sg, combination);
476 				err = -EINVAL;
477 			}
478 
479 			i915_vma_unpin(vma);
480 			i915_gem_object_put(obj);
481 
482 			if (err)
483 				goto out_device;
484 		}
485 	}
486 
487 	goto out_device;
488 
489 out_put:
490 	i915_gem_object_put(obj);
491 out_device:
492 	RUNTIME_INFO(i915)->page_sizes = saved_mask;
493 
494 	return err;
495 }
496 
497 static int igt_mock_memory_region_huge_pages(void *arg)
498 {
499 	const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS };
500 	struct i915_ppgtt *ppgtt = arg;
501 	struct drm_i915_private *i915 = ppgtt->vm.i915;
502 	unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
503 	struct intel_memory_region *mem;
504 	struct drm_i915_gem_object *obj;
505 	struct i915_vma *vma;
506 	int bit;
507 	int err = 0;
508 
509 	mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0);
510 	if (IS_ERR(mem)) {
511 		pr_err("%s failed to create memory region\n", __func__);
512 		return PTR_ERR(mem);
513 	}
514 
515 	for_each_set_bit(bit, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
516 		unsigned int page_size = BIT(bit);
517 		resource_size_t phys;
518 		int i;
519 
520 		for (i = 0; i < ARRAY_SIZE(flags); ++i) {
521 			obj = i915_gem_object_create_region(mem,
522 							    page_size, page_size,
523 							    flags[i]);
524 			if (IS_ERR(obj)) {
525 				err = PTR_ERR(obj);
526 				goto out_region;
527 			}
528 
529 			vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
530 			if (IS_ERR(vma)) {
531 				err = PTR_ERR(vma);
532 				goto out_put;
533 			}
534 
535 			err = i915_vma_pin(vma, 0, 0, PIN_USER);
536 			if (err)
537 				goto out_put;
538 
539 			err = igt_check_page_sizes(vma);
540 			if (err)
541 				goto out_unpin;
542 
543 			phys = i915_gem_object_get_dma_address(obj, 0);
544 			if (!IS_ALIGNED(phys, page_size)) {
545 				pr_err("%s addr misaligned(%pa) page_size=%u\n",
546 				       __func__, &phys, page_size);
547 				err = -EINVAL;
548 				goto out_unpin;
549 			}
550 
551 			if (vma->resource->page_sizes_gtt != page_size) {
552 				pr_err("%s page_sizes.gtt=%u, expected=%u\n",
553 				       __func__, vma->resource->page_sizes_gtt,
554 				       page_size);
555 				err = -EINVAL;
556 				goto out_unpin;
557 			}
558 
559 			i915_vma_unpin(vma);
560 			__i915_gem_object_put_pages(obj);
561 			i915_gem_object_put(obj);
562 		}
563 	}
564 
565 	goto out_region;
566 
567 out_unpin:
568 	i915_vma_unpin(vma);
569 out_put:
570 	i915_gem_object_put(obj);
571 out_region:
572 	intel_memory_region_destroy(mem);
573 	return err;
574 }
575 
576 static int igt_mock_ppgtt_misaligned_dma(void *arg)
577 {
578 	struct i915_ppgtt *ppgtt = arg;
579 	struct drm_i915_private *i915 = ppgtt->vm.i915;
580 	unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
581 	struct drm_i915_gem_object *obj;
582 	int bit;
583 	int err;
584 
585 	/*
586 	 * Sanity check dma misalignment for huge pages -- the dma addresses we
587 	 * insert into the paging structures need to always respect the page
588 	 * size alignment.
589 	 */
590 
591 	bit = ilog2(I915_GTT_PAGE_SIZE_64K);
592 
593 	for_each_set_bit_from(bit, &supported,
594 			      ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
595 		IGT_TIMEOUT(end_time);
596 		unsigned int page_size = BIT(bit);
597 		unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
598 		unsigned int offset;
599 		unsigned int size =
600 			round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
601 		struct i915_vma *vma;
602 
603 		obj = fake_huge_pages_object(i915, size, true);
604 		if (IS_ERR(obj))
605 			return PTR_ERR(obj);
606 
607 		if (obj->base.size != size) {
608 			pr_err("obj->base.size=%zu, expected=%u\n",
609 			       obj->base.size, size);
610 			err = -EINVAL;
611 			goto out_put;
612 		}
613 
614 		err = i915_gem_object_pin_pages_unlocked(obj);
615 		if (err)
616 			goto out_put;
617 
618 		/* Force the page size for this object */
619 		obj->mm.page_sizes.sg = page_size;
620 
621 		vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
622 		if (IS_ERR(vma)) {
623 			err = PTR_ERR(vma);
624 			goto out_unpin;
625 		}
626 
627 		err = i915_vma_pin(vma, 0, 0, flags);
628 		if (err)
629 			goto out_unpin;
630 
631 
632 		err = igt_check_page_sizes(vma);
633 
634 		if (vma->resource->page_sizes_gtt != page_size) {
635 			pr_err("page_sizes.gtt=%u, expected %u\n",
636 			       vma->resource->page_sizes_gtt, page_size);
637 			err = -EINVAL;
638 		}
639 
640 		i915_vma_unpin(vma);
641 
642 		if (err)
643 			goto out_unpin;
644 
645 		/*
646 		 * Try all the other valid offsets until the next
647 		 * boundary -- should always fall back to using 4K
648 		 * pages.
649 		 */
650 		for (offset = 4096; offset < page_size; offset += 4096) {
651 			err = i915_vma_unbind_unlocked(vma);
652 			if (err)
653 				goto out_unpin;
654 
655 			err = i915_vma_pin(vma, 0, 0, flags | offset);
656 			if (err)
657 				goto out_unpin;
658 
659 			err = igt_check_page_sizes(vma);
660 
661 			if (vma->resource->page_sizes_gtt != I915_GTT_PAGE_SIZE_4K) {
662 				pr_err("page_sizes.gtt=%u, expected %llu\n",
663 				       vma->resource->page_sizes_gtt,
664 				       I915_GTT_PAGE_SIZE_4K);
665 				err = -EINVAL;
666 			}
667 
668 			i915_vma_unpin(vma);
669 
670 			if (err)
671 				goto out_unpin;
672 
673 			if (igt_timeout(end_time,
674 					"%s timed out at offset %x with page-size %x\n",
675 					__func__, offset, page_size))
676 				break;
677 		}
678 
679 		i915_gem_object_lock(obj, NULL);
680 		i915_gem_object_unpin_pages(obj);
681 		__i915_gem_object_put_pages(obj);
682 		i915_gem_object_unlock(obj);
683 		i915_gem_object_put(obj);
684 	}
685 
686 	return 0;
687 
688 out_unpin:
689 	i915_gem_object_lock(obj, NULL);
690 	i915_gem_object_unpin_pages(obj);
691 	i915_gem_object_unlock(obj);
692 out_put:
693 	i915_gem_object_put(obj);
694 
695 	return err;
696 }
697 
698 static void close_object_list(struct list_head *objects,
699 			      struct i915_ppgtt *ppgtt)
700 {
701 	struct drm_i915_gem_object *obj, *on;
702 
703 	list_for_each_entry_safe(obj, on, objects, st_link) {
704 		list_del(&obj->st_link);
705 		i915_gem_object_lock(obj, NULL);
706 		i915_gem_object_unpin_pages(obj);
707 		__i915_gem_object_put_pages(obj);
708 		i915_gem_object_unlock(obj);
709 		i915_gem_object_put(obj);
710 	}
711 }
712 
713 static int igt_mock_ppgtt_huge_fill(void *arg)
714 {
715 	struct i915_ppgtt *ppgtt = arg;
716 	struct drm_i915_private *i915 = ppgtt->vm.i915;
717 	unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT;
718 	unsigned long page_num;
719 	bool single = false;
720 	LIST_HEAD(objects);
721 	IGT_TIMEOUT(end_time);
722 	int err = -ENODEV;
723 
724 	for_each_prime_number_from(page_num, 1, max_pages) {
725 		struct drm_i915_gem_object *obj;
726 		u64 size = page_num << PAGE_SHIFT;
727 		struct i915_vma *vma;
728 		unsigned int expected_gtt = 0;
729 		int i;
730 
731 		obj = fake_huge_pages_object(i915, size, single);
732 		if (IS_ERR(obj)) {
733 			err = PTR_ERR(obj);
734 			break;
735 		}
736 
737 		if (obj->base.size != size) {
738 			pr_err("obj->base.size=%zd, expected=%llu\n",
739 			       obj->base.size, size);
740 			i915_gem_object_put(obj);
741 			err = -EINVAL;
742 			break;
743 		}
744 
745 		err = i915_gem_object_pin_pages_unlocked(obj);
746 		if (err) {
747 			i915_gem_object_put(obj);
748 			break;
749 		}
750 
751 		list_add(&obj->st_link, &objects);
752 
753 		vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
754 		if (IS_ERR(vma)) {
755 			err = PTR_ERR(vma);
756 			break;
757 		}
758 
759 		err = i915_vma_pin(vma, 0, 0, PIN_USER);
760 		if (err)
761 			break;
762 
763 		err = igt_check_page_sizes(vma);
764 		if (err) {
765 			i915_vma_unpin(vma);
766 			break;
767 		}
768 
769 		/*
770 		 * Figure out the expected gtt page size knowing that we go from
771 		 * largest to smallest page size sg chunks, and that we align to
772 		 * the largest page size.
773 		 */
774 		for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
775 			unsigned int page_size = page_sizes[i];
776 
777 			if (HAS_PAGE_SIZES(i915, page_size) &&
778 			    size >= page_size) {
779 				expected_gtt |= page_size;
780 				size &= page_size-1;
781 			}
782 		}
783 
784 		GEM_BUG_ON(!expected_gtt);
785 		GEM_BUG_ON(size);
786 
787 		if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
788 			expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
789 
790 		i915_vma_unpin(vma);
791 
792 		if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
793 			if (!IS_ALIGNED(vma->node.start,
794 					I915_GTT_PAGE_SIZE_2M)) {
795 				pr_err("node.start(%llx) not aligned to 2M\n",
796 				       vma->node.start);
797 				err = -EINVAL;
798 				break;
799 			}
800 
801 			if (!IS_ALIGNED(vma->node.size,
802 					I915_GTT_PAGE_SIZE_2M)) {
803 				pr_err("node.size(%llx) not aligned to 2M\n",
804 				       vma->node.size);
805 				err = -EINVAL;
806 				break;
807 			}
808 		}
809 
810 		if (vma->resource->page_sizes_gtt != expected_gtt) {
811 			pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
812 			       vma->resource->page_sizes_gtt, expected_gtt,
813 			       obj->base.size, str_yes_no(!!single));
814 			err = -EINVAL;
815 			break;
816 		}
817 
818 		if (igt_timeout(end_time,
819 				"%s timed out at size %zd\n",
820 				__func__, obj->base.size))
821 			break;
822 
823 		single = !single;
824 	}
825 
826 	close_object_list(&objects, ppgtt);
827 
828 	if (err == -ENOMEM || err == -ENOSPC)
829 		err = 0;
830 
831 	return err;
832 }
833 
834 static int igt_mock_ppgtt_64K(void *arg)
835 {
836 	struct i915_ppgtt *ppgtt = arg;
837 	struct drm_i915_private *i915 = ppgtt->vm.i915;
838 	struct drm_i915_gem_object *obj;
839 	const struct object_info {
840 		unsigned int size;
841 		unsigned int gtt;
842 		unsigned int offset;
843 	} objects[] = {
844 		/* Cases with forced padding/alignment */
845 		{
846 			.size = SZ_64K,
847 			.gtt = I915_GTT_PAGE_SIZE_64K,
848 			.offset = 0,
849 		},
850 		{
851 			.size = SZ_64K + SZ_4K,
852 			.gtt = I915_GTT_PAGE_SIZE_4K,
853 			.offset = 0,
854 		},
855 		{
856 			.size = SZ_64K - SZ_4K,
857 			.gtt = I915_GTT_PAGE_SIZE_4K,
858 			.offset = 0,
859 		},
860 		{
861 			.size = SZ_2M,
862 			.gtt = I915_GTT_PAGE_SIZE_64K,
863 			.offset = 0,
864 		},
865 		{
866 			.size = SZ_2M - SZ_4K,
867 			.gtt = I915_GTT_PAGE_SIZE_4K,
868 			.offset = 0,
869 		},
870 		{
871 			.size = SZ_2M + SZ_4K,
872 			.gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
873 			.offset = 0,
874 		},
875 		{
876 			.size = SZ_2M + SZ_64K,
877 			.gtt = I915_GTT_PAGE_SIZE_64K,
878 			.offset = 0,
879 		},
880 		{
881 			.size = SZ_2M - SZ_64K,
882 			.gtt = I915_GTT_PAGE_SIZE_64K,
883 			.offset = 0,
884 		},
885 		/* Try without any forced padding/alignment */
886 		{
887 			.size = SZ_64K,
888 			.offset = SZ_2M,
889 			.gtt = I915_GTT_PAGE_SIZE_4K,
890 		},
891 		{
892 			.size = SZ_128K,
893 			.offset = SZ_2M - SZ_64K,
894 			.gtt = I915_GTT_PAGE_SIZE_4K,
895 		},
896 	};
897 	struct i915_vma *vma;
898 	int i, single;
899 	int err;
900 
901 	/*
902 	 * Sanity check some of the trickiness with 64K pages -- either we can
903 	 * safely mark the whole page-table(2M block) as 64K, or we have to
904 	 * always fallback to 4K.
905 	 */
906 
907 	if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
908 		return 0;
909 
910 	for (i = 0; i < ARRAY_SIZE(objects); ++i) {
911 		unsigned int size = objects[i].size;
912 		unsigned int expected_gtt = objects[i].gtt;
913 		unsigned int offset = objects[i].offset;
914 		unsigned int flags = PIN_USER;
915 
916 		for (single = 0; single <= 1; single++) {
917 			obj = fake_huge_pages_object(i915, size, !!single);
918 			if (IS_ERR(obj))
919 				return PTR_ERR(obj);
920 
921 			err = i915_gem_object_pin_pages_unlocked(obj);
922 			if (err)
923 				goto out_object_put;
924 
925 			/*
926 			 * Disable 2M pages -- We only want to use 64K/4K pages
927 			 * for this test.
928 			 */
929 			obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
930 
931 			vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
932 			if (IS_ERR(vma)) {
933 				err = PTR_ERR(vma);
934 				goto out_object_unpin;
935 			}
936 
937 			if (offset)
938 				flags |= PIN_OFFSET_FIXED | offset;
939 
940 			err = i915_vma_pin(vma, 0, 0, flags);
941 			if (err)
942 				goto out_object_unpin;
943 
944 			err = igt_check_page_sizes(vma);
945 			if (err)
946 				goto out_vma_unpin;
947 
948 			if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
949 				if (!IS_ALIGNED(vma->node.start,
950 						I915_GTT_PAGE_SIZE_2M)) {
951 					pr_err("node.start(%llx) not aligned to 2M\n",
952 					       vma->node.start);
953 					err = -EINVAL;
954 					goto out_vma_unpin;
955 				}
956 
957 				if (!IS_ALIGNED(vma->node.size,
958 						I915_GTT_PAGE_SIZE_2M)) {
959 					pr_err("node.size(%llx) not aligned to 2M\n",
960 					       vma->node.size);
961 					err = -EINVAL;
962 					goto out_vma_unpin;
963 				}
964 			}
965 
966 			if (vma->resource->page_sizes_gtt != expected_gtt) {
967 				pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
968 				       vma->resource->page_sizes_gtt,
969 				       expected_gtt, i, str_yes_no(!!single));
970 				err = -EINVAL;
971 				goto out_vma_unpin;
972 			}
973 
974 			i915_vma_unpin(vma);
975 			i915_gem_object_lock(obj, NULL);
976 			i915_gem_object_unpin_pages(obj);
977 			__i915_gem_object_put_pages(obj);
978 			i915_gem_object_unlock(obj);
979 			i915_gem_object_put(obj);
980 
981 			i915_gem_drain_freed_objects(i915);
982 		}
983 	}
984 
985 	return 0;
986 
987 out_vma_unpin:
988 	i915_vma_unpin(vma);
989 out_object_unpin:
990 	i915_gem_object_lock(obj, NULL);
991 	i915_gem_object_unpin_pages(obj);
992 	i915_gem_object_unlock(obj);
993 out_object_put:
994 	i915_gem_object_put(obj);
995 
996 	return err;
997 }
998 
999 static int gpu_write(struct intel_context *ce,
1000 		     struct i915_vma *vma,
1001 		     u32 dw,
1002 		     u32 val)
1003 {
1004 	int err;
1005 
1006 	i915_gem_object_lock(vma->obj, NULL);
1007 	err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
1008 	i915_gem_object_unlock(vma->obj);
1009 	if (err)
1010 		return err;
1011 
1012 	return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32),
1013 			       vma->size >> PAGE_SHIFT, val);
1014 }
1015 
1016 static int
1017 __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1018 {
1019 	unsigned int needs_flush;
1020 	unsigned long n;
1021 	int err;
1022 
1023 	i915_gem_object_lock(obj, NULL);
1024 	err = i915_gem_object_prepare_read(obj, &needs_flush);
1025 	if (err)
1026 		goto err_unlock;
1027 
1028 	for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
1029 		u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1030 
1031 		if (needs_flush & CLFLUSH_BEFORE)
1032 			drm_clflush_virt_range(ptr, PAGE_SIZE);
1033 
1034 		if (ptr[dword] != val) {
1035 			pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1036 			       n, dword, ptr[dword], val);
1037 			kunmap_atomic(ptr);
1038 			err = -EINVAL;
1039 			break;
1040 		}
1041 
1042 		kunmap_atomic(ptr);
1043 	}
1044 
1045 	i915_gem_object_finish_access(obj);
1046 err_unlock:
1047 	i915_gem_object_unlock(obj);
1048 
1049 	return err;
1050 }
1051 
1052 static int __cpu_check_vmap(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1053 {
1054 	unsigned long n = obj->base.size >> PAGE_SHIFT;
1055 	u32 *ptr;
1056 	int err;
1057 
1058 	err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1059 	if (err)
1060 		return err;
1061 
1062 	ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1063 	if (IS_ERR(ptr))
1064 		return PTR_ERR(ptr);
1065 
1066 	ptr += dword;
1067 	while (n--) {
1068 		if (*ptr != val) {
1069 			pr_err("base[%u]=%08x, val=%08x\n",
1070 			       dword, *ptr, val);
1071 			err = -EINVAL;
1072 			break;
1073 		}
1074 
1075 		ptr += PAGE_SIZE / sizeof(*ptr);
1076 	}
1077 
1078 	i915_gem_object_unpin_map(obj);
1079 	return err;
1080 }
1081 
1082 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1083 {
1084 	if (i915_gem_object_has_struct_page(obj))
1085 		return __cpu_check_shmem(obj, dword, val);
1086 	else
1087 		return __cpu_check_vmap(obj, dword, val);
1088 }
1089 
1090 static int __igt_write_huge(struct intel_context *ce,
1091 			    struct drm_i915_gem_object *obj,
1092 			    u64 size, u64 offset,
1093 			    u32 dword, u32 val)
1094 {
1095 	unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1096 	struct i915_vma *vma;
1097 	int err;
1098 
1099 	vma = i915_vma_instance(obj, ce->vm, NULL);
1100 	if (IS_ERR(vma))
1101 		return PTR_ERR(vma);
1102 
1103 	err = i915_vma_pin(vma, size, 0, flags | offset);
1104 	if (err) {
1105 		/*
1106 		 * The ggtt may have some pages reserved so
1107 		 * refrain from erroring out.
1108 		 */
1109 		if (err == -ENOSPC && i915_is_ggtt(ce->vm))
1110 			err = 0;
1111 
1112 		return err;
1113 	}
1114 
1115 	err = igt_check_page_sizes(vma);
1116 	if (err)
1117 		goto out_vma_unpin;
1118 
1119 	err = gpu_write(ce, vma, dword, val);
1120 	if (err) {
1121 		pr_err("gpu-write failed at offset=%llx\n", offset);
1122 		goto out_vma_unpin;
1123 	}
1124 
1125 	err = cpu_check(obj, dword, val);
1126 	if (err) {
1127 		pr_err("cpu-check failed at offset=%llx\n", offset);
1128 		goto out_vma_unpin;
1129 	}
1130 
1131 out_vma_unpin:
1132 	i915_vma_unpin(vma);
1133 	return err;
1134 }
1135 
1136 static int igt_write_huge(struct drm_i915_private *i915,
1137 			  struct drm_i915_gem_object *obj)
1138 {
1139 	struct i915_gem_engines *engines;
1140 	struct i915_gem_engines_iter it;
1141 	struct intel_context *ce;
1142 	I915_RND_STATE(prng);
1143 	IGT_TIMEOUT(end_time);
1144 	unsigned int max_page_size;
1145 	unsigned int count;
1146 	struct i915_gem_context *ctx;
1147 	struct file *file;
1148 	u64 max;
1149 	u64 num;
1150 	u64 size;
1151 	int *order;
1152 	int i, n;
1153 	int err = 0;
1154 
1155 	file = mock_file(i915);
1156 	if (IS_ERR(file))
1157 		return PTR_ERR(file);
1158 
1159 	ctx = hugepage_ctx(i915, file);
1160 	if (IS_ERR(ctx)) {
1161 		err = PTR_ERR(ctx);
1162 		goto out;
1163 	}
1164 
1165 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1166 
1167 	size = obj->base.size;
1168 	if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
1169 	    !HAS_64K_PAGES(i915))
1170 		size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1171 
1172 	n = 0;
1173 	count = 0;
1174 	max = U64_MAX;
1175 	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1176 		count++;
1177 		if (!intel_engine_can_store_dword(ce->engine))
1178 			continue;
1179 
1180 		max = min(max, ce->vm->total);
1181 		n++;
1182 	}
1183 	i915_gem_context_unlock_engines(ctx);
1184 	if (!n)
1185 		goto out;
1186 
1187 	/*
1188 	 * To keep things interesting when alternating between engines in our
1189 	 * randomized order, lets also make feeding to the same engine a few
1190 	 * times in succession a possibility by enlarging the permutation array.
1191 	 */
1192 	order = i915_random_order(count * count, &prng);
1193 	if (!order)
1194 		return -ENOMEM;
1195 
1196 	max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1197 	max = div_u64(max - size, max_page_size);
1198 
1199 	/*
1200 	 * Try various offsets in an ascending/descending fashion until we
1201 	 * timeout -- we want to avoid issues hidden by effectively always using
1202 	 * offset = 0.
1203 	 */
1204 	i = 0;
1205 	engines = i915_gem_context_lock_engines(ctx);
1206 	for_each_prime_number_from(num, 0, max) {
1207 		u64 offset_low = num * max_page_size;
1208 		u64 offset_high = (max - num) * max_page_size;
1209 		u32 dword = offset_in_page(num) / 4;
1210 		struct intel_context *ce;
1211 
1212 		ce = engines->engines[order[i] % engines->num_engines];
1213 		i = (i + 1) % (count * count);
1214 		if (!ce || !intel_engine_can_store_dword(ce->engine))
1215 			continue;
1216 
1217 		/*
1218 		 * In order to utilize 64K pages we need to both pad the vma
1219 		 * size and ensure the vma offset is at the start of the pt
1220 		 * boundary, however to improve coverage we opt for testing both
1221 		 * aligned and unaligned offsets.
1222 		 *
1223 		 * With PS64 this is no longer the case, but to ensure we
1224 		 * sometimes get the compact layout for smaller objects, apply
1225 		 * the round_up anyway.
1226 		 */
1227 		if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1228 			offset_low = round_down(offset_low,
1229 						I915_GTT_PAGE_SIZE_2M);
1230 
1231 		err = __igt_write_huge(ce, obj, size, offset_low,
1232 				       dword, num + 1);
1233 		if (err)
1234 			break;
1235 
1236 		err = __igt_write_huge(ce, obj, size, offset_high,
1237 				       dword, num + 1);
1238 		if (err)
1239 			break;
1240 
1241 		if (igt_timeout(end_time,
1242 				"%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1243 				__func__, ce->engine->name, offset_low, offset_high,
1244 				max_page_size))
1245 			break;
1246 	}
1247 	i915_gem_context_unlock_engines(ctx);
1248 
1249 	kfree(order);
1250 
1251 out:
1252 	fput(file);
1253 	return err;
1254 }
1255 
1256 typedef struct drm_i915_gem_object *
1257 (*igt_create_fn)(struct drm_i915_private *i915, u32 size, u32 flags);
1258 
1259 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1260 {
1261 	return i915->mm.gemfs && has_transparent_hugepage();
1262 }
1263 
1264 static struct drm_i915_gem_object *
1265 igt_create_shmem(struct drm_i915_private *i915, u32 size, u32 flags)
1266 {
1267 	if (!igt_can_allocate_thp(i915)) {
1268 		pr_info("%s missing THP support, skipping\n", __func__);
1269 		return ERR_PTR(-ENODEV);
1270 	}
1271 
1272 	return i915_gem_object_create_shmem(i915, size);
1273 }
1274 
1275 static struct drm_i915_gem_object *
1276 igt_create_internal(struct drm_i915_private *i915, u32 size, u32 flags)
1277 {
1278 	return i915_gem_object_create_internal(i915, size);
1279 }
1280 
1281 static struct drm_i915_gem_object *
1282 igt_create_system(struct drm_i915_private *i915, u32 size, u32 flags)
1283 {
1284 	return huge_pages_object(i915, size, size);
1285 }
1286 
1287 static struct drm_i915_gem_object *
1288 igt_create_local(struct drm_i915_private *i915, u32 size, u32 flags)
1289 {
1290 	return i915_gem_object_create_lmem(i915, size, flags);
1291 }
1292 
1293 static u32 igt_random_size(struct rnd_state *prng,
1294 			   u32 min_page_size,
1295 			   u32 max_page_size)
1296 {
1297 	u64 mask;
1298 	u32 size;
1299 
1300 	GEM_BUG_ON(!is_power_of_2(min_page_size));
1301 	GEM_BUG_ON(!is_power_of_2(max_page_size));
1302 	GEM_BUG_ON(min_page_size < PAGE_SIZE);
1303 	GEM_BUG_ON(min_page_size > max_page_size);
1304 
1305 	mask = ((max_page_size << 1ULL) - 1) & PAGE_MASK;
1306 	size = prandom_u32_state(prng) & mask;
1307 	if (size < min_page_size)
1308 		size |= min_page_size;
1309 
1310 	return size;
1311 }
1312 
1313 static int igt_ppgtt_smoke_huge(void *arg)
1314 {
1315 	struct drm_i915_private *i915 = arg;
1316 	struct drm_i915_gem_object *obj;
1317 	I915_RND_STATE(prng);
1318 	struct {
1319 		igt_create_fn fn;
1320 		u32 min;
1321 		u32 max;
1322 	} backends[] = {
1323 		{ igt_create_internal, SZ_64K, SZ_2M,  },
1324 		{ igt_create_shmem,    SZ_64K, SZ_32M, },
1325 		{ igt_create_local,    SZ_64K, SZ_1G,  },
1326 	};
1327 	int err;
1328 	int i;
1329 
1330 	/*
1331 	 * Sanity check that the HW uses huge pages correctly through our
1332 	 * various backends -- ensure that our writes land in the right place.
1333 	 */
1334 
1335 	for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1336 		u32 min = backends[i].min;
1337 		u32 max = backends[i].max;
1338 		u32 size = max;
1339 
1340 try_again:
1341 		size = igt_random_size(&prng, min, rounddown_pow_of_two(size));
1342 
1343 		obj = backends[i].fn(i915, size, 0);
1344 		if (IS_ERR(obj)) {
1345 			err = PTR_ERR(obj);
1346 			if (err == -E2BIG) {
1347 				size >>= 1;
1348 				goto try_again;
1349 			} else if (err == -ENODEV) {
1350 				err = 0;
1351 				continue;
1352 			}
1353 
1354 			return err;
1355 		}
1356 
1357 		err = i915_gem_object_pin_pages_unlocked(obj);
1358 		if (err) {
1359 			if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) {
1360 				i915_gem_object_put(obj);
1361 				size >>= 1;
1362 				goto try_again;
1363 			}
1364 			goto out_put;
1365 		}
1366 
1367 		if (obj->mm.page_sizes.phys < min) {
1368 			pr_info("%s unable to allocate huge-page(s) with size=%u, i=%d\n",
1369 				__func__, size, i);
1370 			err = -ENOMEM;
1371 			goto out_unpin;
1372 		}
1373 
1374 		err = igt_write_huge(i915, obj);
1375 		if (err) {
1376 			pr_err("%s write-huge failed with size=%u, i=%d\n",
1377 			       __func__, size, i);
1378 		}
1379 out_unpin:
1380 		i915_gem_object_lock(obj, NULL);
1381 		i915_gem_object_unpin_pages(obj);
1382 		__i915_gem_object_put_pages(obj);
1383 		i915_gem_object_unlock(obj);
1384 out_put:
1385 		i915_gem_object_put(obj);
1386 
1387 		if (err == -ENOMEM || err == -ENXIO)
1388 			err = 0;
1389 
1390 		if (err)
1391 			break;
1392 
1393 		cond_resched();
1394 	}
1395 
1396 	return err;
1397 }
1398 
1399 static int igt_ppgtt_sanity_check(void *arg)
1400 {
1401 	struct drm_i915_private *i915 = arg;
1402 	unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
1403 	struct {
1404 		igt_create_fn fn;
1405 		unsigned int flags;
1406 	} backends[] = {
1407 		{ igt_create_system, 0,                        },
1408 		{ igt_create_local,  0,                        },
1409 		{ igt_create_local,  I915_BO_ALLOC_CONTIGUOUS, },
1410 	};
1411 	struct {
1412 		u32 size;
1413 		u32 pages;
1414 	} combos[] = {
1415 		{ SZ_64K,		SZ_64K		},
1416 		{ SZ_2M,		SZ_2M		},
1417 		{ SZ_2M,		SZ_64K		},
1418 		{ SZ_2M - SZ_64K,	SZ_64K		},
1419 		{ SZ_2M - SZ_4K,	SZ_64K | SZ_4K	},
1420 		{ SZ_2M + SZ_4K,	SZ_64K | SZ_4K	},
1421 		{ SZ_2M + SZ_4K,	SZ_2M  | SZ_4K	},
1422 		{ SZ_2M + SZ_64K,	SZ_2M  | SZ_64K },
1423 		{ SZ_2M + SZ_64K,	SZ_64K		},
1424 	};
1425 	int i, j;
1426 	int err;
1427 
1428 	if (supported == I915_GTT_PAGE_SIZE_4K)
1429 		return 0;
1430 
1431 	/*
1432 	 * Sanity check that the HW behaves with a limited set of combinations.
1433 	 * We already have a bunch of randomised testing, which should give us
1434 	 * a decent amount of variation between runs, however we should keep
1435 	 * this to limit the chances of introducing a temporary regression, by
1436 	 * testing the most obvious cases that might make something blow up.
1437 	 */
1438 
1439 	for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1440 		for (j = 0; j < ARRAY_SIZE(combos); ++j) {
1441 			struct drm_i915_gem_object *obj;
1442 			u32 size = combos[j].size;
1443 			u32 pages = combos[j].pages;
1444 
1445 			obj = backends[i].fn(i915, size, backends[i].flags);
1446 			if (IS_ERR(obj)) {
1447 				err = PTR_ERR(obj);
1448 				if (err == -ENODEV) {
1449 					pr_info("Device lacks local memory, skipping\n");
1450 					err = 0;
1451 					break;
1452 				}
1453 
1454 				return err;
1455 			}
1456 
1457 			err = i915_gem_object_pin_pages_unlocked(obj);
1458 			if (err) {
1459 				i915_gem_object_put(obj);
1460 				goto out;
1461 			}
1462 
1463 			GEM_BUG_ON(pages > obj->base.size);
1464 			pages = pages & supported;
1465 
1466 			if (pages)
1467 				obj->mm.page_sizes.sg = pages;
1468 
1469 			err = igt_write_huge(i915, obj);
1470 
1471 			i915_gem_object_lock(obj, NULL);
1472 			i915_gem_object_unpin_pages(obj);
1473 			__i915_gem_object_put_pages(obj);
1474 			i915_gem_object_unlock(obj);
1475 			i915_gem_object_put(obj);
1476 
1477 			if (err) {
1478 				pr_err("%s write-huge failed with size=%u pages=%u i=%d, j=%d\n",
1479 				       __func__, size, pages, i, j);
1480 				goto out;
1481 			}
1482 		}
1483 
1484 		cond_resched();
1485 	}
1486 
1487 out:
1488 	if (err == -ENOMEM)
1489 		err = 0;
1490 
1491 	return err;
1492 }
1493 
1494 static int igt_ppgtt_compact(void *arg)
1495 {
1496 	struct drm_i915_private *i915 = arg;
1497 	struct drm_i915_gem_object *obj;
1498 	int err;
1499 
1500 	/*
1501 	 * Simple test to catch issues with compact 64K pages -- since the pt is
1502 	 * compacted to 256B that gives us 32 entries per pt, however since the
1503 	 * backing page for the pt is 4K, any extra entries we might incorrectly
1504 	 * write out should be ignored by the HW. If ever hit such a case this
1505 	 * test should catch it since some of our writes would land in scratch.
1506 	 */
1507 
1508 	if (!HAS_64K_PAGES(i915)) {
1509 		pr_info("device lacks compact 64K page support, skipping\n");
1510 		return 0;
1511 	}
1512 
1513 	if (!HAS_LMEM(i915)) {
1514 		pr_info("device lacks LMEM support, skipping\n");
1515 		return 0;
1516 	}
1517 
1518 	/* We want the range to cover multiple page-table boundaries. */
1519 	obj = i915_gem_object_create_lmem(i915, SZ_4M, 0);
1520 	if (IS_ERR(obj))
1521 		return PTR_ERR(obj);
1522 
1523 	err = i915_gem_object_pin_pages_unlocked(obj);
1524 	if (err)
1525 		goto out_put;
1526 
1527 	if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1528 		pr_info("LMEM compact unable to allocate huge-page(s)\n");
1529 		goto out_unpin;
1530 	}
1531 
1532 	/*
1533 	 * Disable 2M GTT pages by forcing the page-size to 64K for the GTT
1534 	 * insertion.
1535 	 */
1536 	obj->mm.page_sizes.sg = I915_GTT_PAGE_SIZE_64K;
1537 
1538 	err = igt_write_huge(i915, obj);
1539 	if (err)
1540 		pr_err("LMEM compact write-huge failed\n");
1541 
1542 out_unpin:
1543 	i915_gem_object_unpin_pages(obj);
1544 out_put:
1545 	i915_gem_object_put(obj);
1546 
1547 	if (err == -ENOMEM)
1548 		err = 0;
1549 
1550 	return err;
1551 }
1552 
1553 static int igt_ppgtt_mixed(void *arg)
1554 {
1555 	struct drm_i915_private *i915 = arg;
1556 	const unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
1557 	struct drm_i915_gem_object *obj, *on;
1558 	struct i915_gem_engines *engines;
1559 	struct i915_gem_engines_iter it;
1560 	struct i915_address_space *vm;
1561 	struct i915_gem_context *ctx;
1562 	struct intel_context *ce;
1563 	struct file *file;
1564 	I915_RND_STATE(prng);
1565 	LIST_HEAD(objects);
1566 	struct intel_memory_region *mr;
1567 	struct i915_vma *vma;
1568 	unsigned int count;
1569 	u32 i, addr;
1570 	int *order;
1571 	int n, err;
1572 
1573 	/*
1574 	 * Sanity check mixing 4K and 64K pages within the same page-table via
1575 	 * the new PS64 TLB hint.
1576 	 */
1577 
1578 	if (!HAS_64K_PAGES(i915)) {
1579 		pr_info("device lacks PS64, skipping\n");
1580 		return 0;
1581 	}
1582 
1583 	file = mock_file(i915);
1584 	if (IS_ERR(file))
1585 		return PTR_ERR(file);
1586 
1587 	ctx = hugepage_ctx(i915, file);
1588 	if (IS_ERR(ctx)) {
1589 		err = PTR_ERR(ctx);
1590 		goto out;
1591 	}
1592 	vm = i915_gem_context_get_eb_vm(ctx);
1593 
1594 	i = 0;
1595 	addr = 0;
1596 	do {
1597 		u32 sz;
1598 
1599 		sz = i915_prandom_u32_max_state(SZ_4M, &prng);
1600 		sz = max_t(u32, sz, SZ_4K);
1601 
1602 		mr = i915->mm.regions[INTEL_REGION_LMEM_0];
1603 		if (i & 1)
1604 			mr = i915->mm.regions[INTEL_REGION_SMEM];
1605 
1606 		obj = i915_gem_object_create_region(mr, sz, 0, 0);
1607 		if (IS_ERR(obj)) {
1608 			err = PTR_ERR(obj);
1609 			goto out_vm;
1610 		}
1611 
1612 		list_add_tail(&obj->st_link, &objects);
1613 
1614 		vma = i915_vma_instance(obj, vm, NULL);
1615 		if (IS_ERR(vma)) {
1616 			err = PTR_ERR(vma);
1617 			goto err_put;
1618 		}
1619 
1620 		addr = round_up(addr, mr->min_page_size);
1621 		err = i915_vma_pin(vma, 0, 0, addr | flags);
1622 		if (err)
1623 			goto err_put;
1624 
1625 		if (mr->type == INTEL_MEMORY_LOCAL &&
1626 		    (vma->resource->page_sizes_gtt & I915_GTT_PAGE_SIZE_4K)) {
1627 			err = -EINVAL;
1628 			goto err_put;
1629 		}
1630 
1631 		addr += obj->base.size;
1632 		i++;
1633 	} while (addr <= SZ_16M);
1634 
1635 	n = 0;
1636 	count = 0;
1637 	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1638 		count++;
1639 		if (!intel_engine_can_store_dword(ce->engine))
1640 			continue;
1641 
1642 		n++;
1643 	}
1644 	i915_gem_context_unlock_engines(ctx);
1645 	if (!n)
1646 		goto err_put;
1647 
1648 	order = i915_random_order(count * count, &prng);
1649 	if (!order) {
1650 		err = -ENOMEM;
1651 		goto err_put;
1652 	}
1653 
1654 	i = 0;
1655 	addr = 0;
1656 	engines = i915_gem_context_lock_engines(ctx);
1657 	list_for_each_entry(obj, &objects, st_link) {
1658 		u32 rnd = i915_prandom_u32_max_state(UINT_MAX, &prng);
1659 
1660 		addr = round_up(addr, obj->mm.region->min_page_size);
1661 
1662 		ce = engines->engines[order[i] % engines->num_engines];
1663 		i = (i + 1) % (count * count);
1664 		if (!ce || !intel_engine_can_store_dword(ce->engine))
1665 			continue;
1666 
1667 		err = __igt_write_huge(ce, obj, obj->base.size, addr, 0, rnd);
1668 		if (err)
1669 			break;
1670 
1671 		err = __igt_write_huge(ce, obj, obj->base.size, addr,
1672 				       offset_in_page(rnd) / sizeof(u32), rnd + 1);
1673 		if (err)
1674 			break;
1675 
1676 		err = __igt_write_huge(ce, obj, obj->base.size, addr,
1677 				       (PAGE_SIZE / sizeof(u32)) - 1,
1678 				       rnd + 2);
1679 		if (err)
1680 			break;
1681 
1682 		addr += obj->base.size;
1683 
1684 		cond_resched();
1685 	}
1686 
1687 	i915_gem_context_unlock_engines(ctx);
1688 	kfree(order);
1689 err_put:
1690 	list_for_each_entry_safe(obj, on, &objects, st_link) {
1691 		list_del(&obj->st_link);
1692 		i915_gem_object_put(obj);
1693 	}
1694 out_vm:
1695 	i915_vm_put(vm);
1696 out:
1697 	fput(file);
1698 	return err;
1699 }
1700 
1701 static int igt_tmpfs_fallback(void *arg)
1702 {
1703 	struct drm_i915_private *i915 = arg;
1704 	struct i915_address_space *vm;
1705 	struct i915_gem_context *ctx;
1706 	struct vfsmount *gemfs = i915->mm.gemfs;
1707 	struct drm_i915_gem_object *obj;
1708 	struct i915_vma *vma;
1709 	struct file *file;
1710 	u32 *vaddr;
1711 	int err = 0;
1712 
1713 	file = mock_file(i915);
1714 	if (IS_ERR(file))
1715 		return PTR_ERR(file);
1716 
1717 	ctx = hugepage_ctx(i915, file);
1718 	if (IS_ERR(ctx)) {
1719 		err = PTR_ERR(ctx);
1720 		goto out;
1721 	}
1722 	vm = i915_gem_context_get_eb_vm(ctx);
1723 
1724 	/*
1725 	 * Make sure that we don't burst into a ball of flames upon falling back
1726 	 * to tmpfs, which we rely on if on the off-chance we encouter a failure
1727 	 * when setting up gemfs.
1728 	 */
1729 
1730 	i915->mm.gemfs = NULL;
1731 
1732 	obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
1733 	if (IS_ERR(obj)) {
1734 		err = PTR_ERR(obj);
1735 		goto out_restore;
1736 	}
1737 
1738 	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1739 	if (IS_ERR(vaddr)) {
1740 		err = PTR_ERR(vaddr);
1741 		goto out_put;
1742 	}
1743 	*vaddr = 0xdeadbeaf;
1744 
1745 	__i915_gem_object_flush_map(obj, 0, 64);
1746 	i915_gem_object_unpin_map(obj);
1747 
1748 	vma = i915_vma_instance(obj, vm, NULL);
1749 	if (IS_ERR(vma)) {
1750 		err = PTR_ERR(vma);
1751 		goto out_put;
1752 	}
1753 
1754 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
1755 	if (err)
1756 		goto out_put;
1757 
1758 	err = igt_check_page_sizes(vma);
1759 
1760 	i915_vma_unpin(vma);
1761 out_put:
1762 	i915_gem_object_put(obj);
1763 out_restore:
1764 	i915->mm.gemfs = gemfs;
1765 
1766 	i915_vm_put(vm);
1767 out:
1768 	fput(file);
1769 	return err;
1770 }
1771 
1772 static int igt_shrink_thp(void *arg)
1773 {
1774 	struct drm_i915_private *i915 = arg;
1775 	struct i915_address_space *vm;
1776 	struct i915_gem_context *ctx;
1777 	struct drm_i915_gem_object *obj;
1778 	struct i915_gem_engines_iter it;
1779 	struct intel_context *ce;
1780 	struct i915_vma *vma;
1781 	struct file *file;
1782 	unsigned int flags = PIN_USER;
1783 	unsigned int n;
1784 	intel_wakeref_t wf;
1785 	bool should_swap;
1786 	int err;
1787 
1788 	if (!igt_can_allocate_thp(i915)) {
1789 		pr_info("missing THP support, skipping\n");
1790 		return 0;
1791 	}
1792 
1793 	file = mock_file(i915);
1794 	if (IS_ERR(file))
1795 		return PTR_ERR(file);
1796 
1797 	ctx = hugepage_ctx(i915, file);
1798 	if (IS_ERR(ctx)) {
1799 		err = PTR_ERR(ctx);
1800 		goto out;
1801 	}
1802 	vm = i915_gem_context_get_eb_vm(ctx);
1803 
1804 	/*
1805 	 * Sanity check shrinking huge-paged object -- make sure nothing blows
1806 	 * up.
1807 	 */
1808 
1809 	obj = i915_gem_object_create_shmem(i915, SZ_2M);
1810 	if (IS_ERR(obj)) {
1811 		err = PTR_ERR(obj);
1812 		goto out_vm;
1813 	}
1814 
1815 	vma = i915_vma_instance(obj, vm, NULL);
1816 	if (IS_ERR(vma)) {
1817 		err = PTR_ERR(vma);
1818 		goto out_put;
1819 	}
1820 
1821 	wf = intel_runtime_pm_get(&i915->runtime_pm); /* active shrink */
1822 
1823 	err = i915_vma_pin(vma, 0, 0, flags);
1824 	if (err)
1825 		goto out_wf;
1826 
1827 	if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1828 		pr_info("failed to allocate THP, finishing test early\n");
1829 		goto out_unpin;
1830 	}
1831 
1832 	err = igt_check_page_sizes(vma);
1833 	if (err)
1834 		goto out_unpin;
1835 
1836 	n = 0;
1837 
1838 	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1839 		if (!intel_engine_can_store_dword(ce->engine))
1840 			continue;
1841 
1842 		err = gpu_write(ce, vma, n++, 0xdeadbeaf);
1843 		if (err)
1844 			break;
1845 	}
1846 	i915_gem_context_unlock_engines(ctx);
1847 	/*
1848 	 * Nuke everything *before* we unpin the pages so we can be reasonably
1849 	 * sure that when later checking get_nr_swap_pages() that some random
1850 	 * leftover object doesn't steal the remaining swap space.
1851 	 */
1852 	i915_gem_shrink(NULL, i915, -1UL, NULL,
1853 			I915_SHRINK_BOUND |
1854 			I915_SHRINK_UNBOUND |
1855 			I915_SHRINK_ACTIVE);
1856 	i915_vma_unpin(vma);
1857 	if (err)
1858 		goto out_wf;
1859 
1860 	/*
1861 	 * Now that the pages are *unpinned* shrinking should invoke
1862 	 * shmem to truncate our pages, if we have available swap.
1863 	 */
1864 	should_swap = get_nr_swap_pages() > 0;
1865 	i915_gem_shrink(NULL, i915, -1UL, NULL,
1866 			I915_SHRINK_BOUND |
1867 			I915_SHRINK_UNBOUND |
1868 			I915_SHRINK_ACTIVE |
1869 			I915_SHRINK_WRITEBACK);
1870 	if (should_swap == i915_gem_object_has_pages(obj)) {
1871 		pr_err("unexpected pages mismatch, should_swap=%s\n",
1872 		       str_yes_no(should_swap));
1873 		err = -EINVAL;
1874 		goto out_wf;
1875 	}
1876 
1877 	if (should_swap == (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys)) {
1878 		pr_err("unexpected residual page-size bits, should_swap=%s\n",
1879 		       str_yes_no(should_swap));
1880 		err = -EINVAL;
1881 		goto out_wf;
1882 	}
1883 
1884 	err = i915_vma_pin(vma, 0, 0, flags);
1885 	if (err)
1886 		goto out_wf;
1887 
1888 	while (n--) {
1889 		err = cpu_check(obj, n, 0xdeadbeaf);
1890 		if (err)
1891 			break;
1892 	}
1893 
1894 out_unpin:
1895 	i915_vma_unpin(vma);
1896 out_wf:
1897 	intel_runtime_pm_put(&i915->runtime_pm, wf);
1898 out_put:
1899 	i915_gem_object_put(obj);
1900 out_vm:
1901 	i915_vm_put(vm);
1902 out:
1903 	fput(file);
1904 	return err;
1905 }
1906 
1907 int i915_gem_huge_page_mock_selftests(void)
1908 {
1909 	static const struct i915_subtest tests[] = {
1910 		SUBTEST(igt_mock_exhaust_device_supported_pages),
1911 		SUBTEST(igt_mock_memory_region_huge_pages),
1912 		SUBTEST(igt_mock_ppgtt_misaligned_dma),
1913 		SUBTEST(igt_mock_ppgtt_huge_fill),
1914 		SUBTEST(igt_mock_ppgtt_64K),
1915 	};
1916 	struct drm_i915_private *dev_priv;
1917 	struct i915_ppgtt *ppgtt;
1918 	int err;
1919 
1920 	dev_priv = mock_gem_device();
1921 	if (!dev_priv)
1922 		return -ENOMEM;
1923 
1924 	/* Pretend to be a device which supports the 48b PPGTT */
1925 	RUNTIME_INFO(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
1926 	RUNTIME_INFO(dev_priv)->ppgtt_size = 48;
1927 
1928 	ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
1929 	if (IS_ERR(ppgtt)) {
1930 		err = PTR_ERR(ppgtt);
1931 		goto out_unlock;
1932 	}
1933 
1934 	if (!i915_vm_is_4lvl(&ppgtt->vm)) {
1935 		pr_err("failed to create 48b PPGTT\n");
1936 		err = -EINVAL;
1937 		goto out_put;
1938 	}
1939 
1940 	/* If we were ever hit this then it's time to mock the 64K scratch */
1941 	if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1942 		pr_err("PPGTT missing 64K scratch page\n");
1943 		err = -EINVAL;
1944 		goto out_put;
1945 	}
1946 
1947 	err = i915_subtests(tests, ppgtt);
1948 
1949 out_put:
1950 	i915_vm_put(&ppgtt->vm);
1951 out_unlock:
1952 	mock_destroy_device(dev_priv);
1953 	return err;
1954 }
1955 
1956 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915)
1957 {
1958 	static const struct i915_subtest tests[] = {
1959 		SUBTEST(igt_shrink_thp),
1960 		SUBTEST(igt_tmpfs_fallback),
1961 		SUBTEST(igt_ppgtt_smoke_huge),
1962 		SUBTEST(igt_ppgtt_sanity_check),
1963 		SUBTEST(igt_ppgtt_compact),
1964 		SUBTEST(igt_ppgtt_mixed),
1965 	};
1966 
1967 	if (!HAS_PPGTT(i915)) {
1968 		pr_info("PPGTT not supported, skipping live-selftests\n");
1969 		return 0;
1970 	}
1971 
1972 	if (intel_gt_is_wedged(to_gt(i915)))
1973 		return 0;
1974 
1975 	return i915_live_subtests(tests, i915);
1976 }
1977