1b414fcd5SChris Wilson /*
2b414fcd5SChris Wilson  * SPDX-License-Identifier: MIT
3b414fcd5SChris Wilson  *
4b414fcd5SChris Wilson  * Copyright © 2016 Intel Corporation
5b414fcd5SChris Wilson  */
6b414fcd5SChris Wilson 
7b414fcd5SChris Wilson #include <linux/prime_numbers.h>
8b414fcd5SChris Wilson 
9a1c8a09eSTvrtko Ursulin #include "gt/intel_gt.h"
10b414fcd5SChris Wilson #include "gt/intel_gt_pm.h"
1110be98a7SChris Wilson #include "huge_gem_object.h"
12b414fcd5SChris Wilson #include "i915_selftest.h"
1307e98eb0SChris Wilson #include "selftests/i915_random.h"
14b414fcd5SChris Wilson #include "selftests/igt_flush_test.h"
15b414fcd5SChris Wilson 
16b414fcd5SChris Wilson struct tile {
17b414fcd5SChris Wilson 	unsigned int width;
18b414fcd5SChris Wilson 	unsigned int height;
19b414fcd5SChris Wilson 	unsigned int stride;
20b414fcd5SChris Wilson 	unsigned int size;
21b414fcd5SChris Wilson 	unsigned int tiling;
22b414fcd5SChris Wilson 	unsigned int swizzle;
23b414fcd5SChris Wilson };
24b414fcd5SChris Wilson 
25b414fcd5SChris Wilson static u64 swizzle_bit(unsigned int bit, u64 offset)
26b414fcd5SChris Wilson {
27b414fcd5SChris Wilson 	return (offset & BIT_ULL(bit)) >> (bit - 6);
28b414fcd5SChris Wilson }
29b414fcd5SChris Wilson 
30b414fcd5SChris Wilson static u64 tiled_offset(const struct tile *tile, u64 v)
31b414fcd5SChris Wilson {
32b414fcd5SChris Wilson 	u64 x, y;
33b414fcd5SChris Wilson 
34b414fcd5SChris Wilson 	if (tile->tiling == I915_TILING_NONE)
35b414fcd5SChris Wilson 		return v;
36b414fcd5SChris Wilson 
37b414fcd5SChris Wilson 	y = div64_u64_rem(v, tile->stride, &x);
38b414fcd5SChris Wilson 	v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height;
39b414fcd5SChris Wilson 
40b414fcd5SChris Wilson 	if (tile->tiling == I915_TILING_X) {
41b414fcd5SChris Wilson 		v += y * tile->width;
42b414fcd5SChris Wilson 		v += div64_u64_rem(x, tile->width, &x) << tile->size;
43b414fcd5SChris Wilson 		v += x;
44b414fcd5SChris Wilson 	} else if (tile->width == 128) {
45b414fcd5SChris Wilson 		const unsigned int ytile_span = 16;
46b414fcd5SChris Wilson 		const unsigned int ytile_height = 512;
47b414fcd5SChris Wilson 
48b414fcd5SChris Wilson 		v += y * ytile_span;
49b414fcd5SChris Wilson 		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
50b414fcd5SChris Wilson 		v += x;
51b414fcd5SChris Wilson 	} else {
52b414fcd5SChris Wilson 		const unsigned int ytile_span = 32;
53b414fcd5SChris Wilson 		const unsigned int ytile_height = 256;
54b414fcd5SChris Wilson 
55b414fcd5SChris Wilson 		v += y * ytile_span;
56b414fcd5SChris Wilson 		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
57b414fcd5SChris Wilson 		v += x;
58b414fcd5SChris Wilson 	}
59b414fcd5SChris Wilson 
60b414fcd5SChris Wilson 	switch (tile->swizzle) {
61b414fcd5SChris Wilson 	case I915_BIT_6_SWIZZLE_9:
62b414fcd5SChris Wilson 		v ^= swizzle_bit(9, v);
63b414fcd5SChris Wilson 		break;
64b414fcd5SChris Wilson 	case I915_BIT_6_SWIZZLE_9_10:
65b414fcd5SChris Wilson 		v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
66b414fcd5SChris Wilson 		break;
67b414fcd5SChris Wilson 	case I915_BIT_6_SWIZZLE_9_11:
68b414fcd5SChris Wilson 		v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
69b414fcd5SChris Wilson 		break;
70b414fcd5SChris Wilson 	case I915_BIT_6_SWIZZLE_9_10_11:
71b414fcd5SChris Wilson 		v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
72b414fcd5SChris Wilson 		break;
73b414fcd5SChris Wilson 	}
74b414fcd5SChris Wilson 
75b414fcd5SChris Wilson 	return v;
76b414fcd5SChris Wilson }
77b414fcd5SChris Wilson 
78b414fcd5SChris Wilson static int check_partial_mapping(struct drm_i915_gem_object *obj,
79b414fcd5SChris Wilson 				 const struct tile *tile,
8007e98eb0SChris Wilson 				 struct rnd_state *prng)
8107e98eb0SChris Wilson {
8207e98eb0SChris Wilson 	const unsigned long npages = obj->base.size / PAGE_SIZE;
8307e98eb0SChris Wilson 	struct i915_ggtt_view view;
8407e98eb0SChris Wilson 	struct i915_vma *vma;
8507e98eb0SChris Wilson 	unsigned long page;
8607e98eb0SChris Wilson 	u32 __iomem *io;
8707e98eb0SChris Wilson 	struct page *p;
8807e98eb0SChris Wilson 	unsigned int n;
8907e98eb0SChris Wilson 	u64 offset;
9007e98eb0SChris Wilson 	u32 *cpu;
9107e98eb0SChris Wilson 	int err;
9207e98eb0SChris Wilson 
9307e98eb0SChris Wilson 	err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
9407e98eb0SChris Wilson 	if (err) {
9507e98eb0SChris Wilson 		pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
9607e98eb0SChris Wilson 		       tile->tiling, tile->stride, err);
9707e98eb0SChris Wilson 		return err;
9807e98eb0SChris Wilson 	}
9907e98eb0SChris Wilson 
10007e98eb0SChris Wilson 	GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
10107e98eb0SChris Wilson 	GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
10207e98eb0SChris Wilson 
10307e98eb0SChris Wilson 	i915_gem_object_lock(obj);
10407e98eb0SChris Wilson 	err = i915_gem_object_set_to_gtt_domain(obj, true);
10507e98eb0SChris Wilson 	i915_gem_object_unlock(obj);
10607e98eb0SChris Wilson 	if (err) {
10707e98eb0SChris Wilson 		pr_err("Failed to flush to GTT write domain; err=%d\n", err);
10807e98eb0SChris Wilson 		return err;
10907e98eb0SChris Wilson 	}
11007e98eb0SChris Wilson 
11107e98eb0SChris Wilson 	page = i915_prandom_u32_max_state(npages, prng);
11207e98eb0SChris Wilson 	view = compute_partial_view(obj, page, MIN_CHUNK_PAGES);
11307e98eb0SChris Wilson 
11407e98eb0SChris Wilson 	vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
11507e98eb0SChris Wilson 	if (IS_ERR(vma)) {
11607e98eb0SChris Wilson 		pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
11707e98eb0SChris Wilson 		       page, (int)PTR_ERR(vma));
11807e98eb0SChris Wilson 		return PTR_ERR(vma);
11907e98eb0SChris Wilson 	}
12007e98eb0SChris Wilson 
12107e98eb0SChris Wilson 	n = page - view.partial.offset;
12207e98eb0SChris Wilson 	GEM_BUG_ON(n >= view.partial.size);
12307e98eb0SChris Wilson 
12407e98eb0SChris Wilson 	io = i915_vma_pin_iomap(vma);
12507e98eb0SChris Wilson 	i915_vma_unpin(vma);
12607e98eb0SChris Wilson 	if (IS_ERR(io)) {
12707e98eb0SChris Wilson 		pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
12807e98eb0SChris Wilson 		       page, (int)PTR_ERR(io));
12907e98eb0SChris Wilson 		err = PTR_ERR(io);
13007e98eb0SChris Wilson 		goto out;
13107e98eb0SChris Wilson 	}
13207e98eb0SChris Wilson 
13307e98eb0SChris Wilson 	iowrite32(page, io + n * PAGE_SIZE / sizeof(*io));
13407e98eb0SChris Wilson 	i915_vma_unpin_iomap(vma);
13507e98eb0SChris Wilson 
13607e98eb0SChris Wilson 	offset = tiled_offset(tile, page << PAGE_SHIFT);
13707e98eb0SChris Wilson 	if (offset >= obj->base.size)
13807e98eb0SChris Wilson 		goto out;
13907e98eb0SChris Wilson 
14007e98eb0SChris Wilson 	intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt);
14107e98eb0SChris Wilson 
14207e98eb0SChris Wilson 	p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
14307e98eb0SChris Wilson 	cpu = kmap(p) + offset_in_page(offset);
14407e98eb0SChris Wilson 	drm_clflush_virt_range(cpu, sizeof(*cpu));
14507e98eb0SChris Wilson 	if (*cpu != (u32)page) {
14607e98eb0SChris Wilson 		pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
14707e98eb0SChris Wilson 		       page, n,
14807e98eb0SChris Wilson 		       view.partial.offset,
14907e98eb0SChris Wilson 		       view.partial.size,
15007e98eb0SChris Wilson 		       vma->size >> PAGE_SHIFT,
15107e98eb0SChris Wilson 		       tile->tiling ? tile_row_pages(obj) : 0,
15207e98eb0SChris Wilson 		       vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
15307e98eb0SChris Wilson 		       offset >> PAGE_SHIFT,
15407e98eb0SChris Wilson 		       (unsigned int)offset_in_page(offset),
15507e98eb0SChris Wilson 		       offset,
15607e98eb0SChris Wilson 		       (u32)page, *cpu);
15707e98eb0SChris Wilson 		err = -EINVAL;
15807e98eb0SChris Wilson 	}
15907e98eb0SChris Wilson 	*cpu = 0;
16007e98eb0SChris Wilson 	drm_clflush_virt_range(cpu, sizeof(*cpu));
16107e98eb0SChris Wilson 	kunmap(p);
16207e98eb0SChris Wilson 
16307e98eb0SChris Wilson out:
16407e98eb0SChris Wilson 	i915_vma_destroy(vma);
16507e98eb0SChris Wilson 	return err;
16607e98eb0SChris Wilson }
16707e98eb0SChris Wilson 
16807e98eb0SChris Wilson static int check_partial_mappings(struct drm_i915_gem_object *obj,
16907e98eb0SChris Wilson 				  const struct tile *tile,
170b414fcd5SChris Wilson 				  unsigned long end_time)
171b414fcd5SChris Wilson {
172b414fcd5SChris Wilson 	const unsigned int nreal = obj->scratch / PAGE_SIZE;
173b414fcd5SChris Wilson 	const unsigned long npages = obj->base.size / PAGE_SIZE;
174b414fcd5SChris Wilson 	struct i915_vma *vma;
175b414fcd5SChris Wilson 	unsigned long page;
176b414fcd5SChris Wilson 	int err;
177b414fcd5SChris Wilson 
178b414fcd5SChris Wilson 	err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
179b414fcd5SChris Wilson 	if (err) {
180b414fcd5SChris Wilson 		pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
181b414fcd5SChris Wilson 		       tile->tiling, tile->stride, err);
182b414fcd5SChris Wilson 		return err;
183b414fcd5SChris Wilson 	}
184b414fcd5SChris Wilson 
185b414fcd5SChris Wilson 	GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
186b414fcd5SChris Wilson 	GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
187b414fcd5SChris Wilson 
18887d1372dSChris Wilson 	i915_gem_object_lock(obj);
18987d1372dSChris Wilson 	err = i915_gem_object_set_to_gtt_domain(obj, true);
19087d1372dSChris Wilson 	i915_gem_object_unlock(obj);
19187d1372dSChris Wilson 	if (err) {
19287d1372dSChris Wilson 		pr_err("Failed to flush to GTT write domain; err=%d\n", err);
19387d1372dSChris Wilson 		return err;
19487d1372dSChris Wilson 	}
19587d1372dSChris Wilson 
196b414fcd5SChris Wilson 	for_each_prime_number_from(page, 1, npages) {
197b414fcd5SChris Wilson 		struct i915_ggtt_view view =
198b414fcd5SChris Wilson 			compute_partial_view(obj, page, MIN_CHUNK_PAGES);
199b414fcd5SChris Wilson 		u32 __iomem *io;
200b414fcd5SChris Wilson 		struct page *p;
201b414fcd5SChris Wilson 		unsigned int n;
202b414fcd5SChris Wilson 		u64 offset;
203b414fcd5SChris Wilson 		u32 *cpu;
204b414fcd5SChris Wilson 
205b414fcd5SChris Wilson 		GEM_BUG_ON(view.partial.size > nreal);
206b414fcd5SChris Wilson 		cond_resched();
207b414fcd5SChris Wilson 
208b414fcd5SChris Wilson 		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
209b414fcd5SChris Wilson 		if (IS_ERR(vma)) {
210b414fcd5SChris Wilson 			pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
211b414fcd5SChris Wilson 			       page, (int)PTR_ERR(vma));
212b414fcd5SChris Wilson 			return PTR_ERR(vma);
213b414fcd5SChris Wilson 		}
214b414fcd5SChris Wilson 
215b414fcd5SChris Wilson 		n = page - view.partial.offset;
216b414fcd5SChris Wilson 		GEM_BUG_ON(n >= view.partial.size);
217b414fcd5SChris Wilson 
218b414fcd5SChris Wilson 		io = i915_vma_pin_iomap(vma);
219b414fcd5SChris Wilson 		i915_vma_unpin(vma);
220b414fcd5SChris Wilson 		if (IS_ERR(io)) {
221b414fcd5SChris Wilson 			pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
222b414fcd5SChris Wilson 			       page, (int)PTR_ERR(io));
223b414fcd5SChris Wilson 			return PTR_ERR(io);
224b414fcd5SChris Wilson 		}
225b414fcd5SChris Wilson 
226b414fcd5SChris Wilson 		iowrite32(page, io + n * PAGE_SIZE / sizeof(*io));
227b414fcd5SChris Wilson 		i915_vma_unpin_iomap(vma);
228b414fcd5SChris Wilson 
229b414fcd5SChris Wilson 		offset = tiled_offset(tile, page << PAGE_SHIFT);
230b414fcd5SChris Wilson 		if (offset >= obj->base.size)
231b414fcd5SChris Wilson 			continue;
232b414fcd5SChris Wilson 
233a1c8a09eSTvrtko Ursulin 		intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt);
234b414fcd5SChris Wilson 
235b414fcd5SChris Wilson 		p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
236b414fcd5SChris Wilson 		cpu = kmap(p) + offset_in_page(offset);
237b414fcd5SChris Wilson 		drm_clflush_virt_range(cpu, sizeof(*cpu));
238b414fcd5SChris Wilson 		if (*cpu != (u32)page) {
239b414fcd5SChris Wilson 			pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
240b414fcd5SChris Wilson 			       page, n,
241b414fcd5SChris Wilson 			       view.partial.offset,
242b414fcd5SChris Wilson 			       view.partial.size,
243b414fcd5SChris Wilson 			       vma->size >> PAGE_SHIFT,
244b414fcd5SChris Wilson 			       tile->tiling ? tile_row_pages(obj) : 0,
245b414fcd5SChris Wilson 			       vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
246b414fcd5SChris Wilson 			       offset >> PAGE_SHIFT,
247b414fcd5SChris Wilson 			       (unsigned int)offset_in_page(offset),
248b414fcd5SChris Wilson 			       offset,
249b414fcd5SChris Wilson 			       (u32)page, *cpu);
250b414fcd5SChris Wilson 			err = -EINVAL;
251b414fcd5SChris Wilson 		}
252b414fcd5SChris Wilson 		*cpu = 0;
253b414fcd5SChris Wilson 		drm_clflush_virt_range(cpu, sizeof(*cpu));
254b414fcd5SChris Wilson 		kunmap(p);
255b414fcd5SChris Wilson 		if (err)
256b414fcd5SChris Wilson 			return err;
257b414fcd5SChris Wilson 
258b414fcd5SChris Wilson 		i915_vma_destroy(vma);
25907e98eb0SChris Wilson 
26007e98eb0SChris Wilson 		if (igt_timeout(end_time,
26107e98eb0SChris Wilson 				"%s: timed out after tiling=%d stride=%d\n",
26207e98eb0SChris Wilson 				__func__, tile->tiling, tile->stride))
26307e98eb0SChris Wilson 			return -EINTR;
264b414fcd5SChris Wilson 	}
265b414fcd5SChris Wilson 
266b414fcd5SChris Wilson 	return 0;
267b414fcd5SChris Wilson }
268b414fcd5SChris Wilson 
26907e98eb0SChris Wilson static unsigned int
27007e98eb0SChris Wilson setup_tile_size(struct tile *tile, struct drm_i915_private *i915)
27107e98eb0SChris Wilson {
27207e98eb0SChris Wilson 	if (INTEL_GEN(i915) <= 2) {
27307e98eb0SChris Wilson 		tile->height = 16;
27407e98eb0SChris Wilson 		tile->width = 128;
27507e98eb0SChris Wilson 		tile->size = 11;
27607e98eb0SChris Wilson 	} else if (tile->tiling == I915_TILING_Y &&
27707e98eb0SChris Wilson 		   HAS_128_BYTE_Y_TILING(i915)) {
27807e98eb0SChris Wilson 		tile->height = 32;
27907e98eb0SChris Wilson 		tile->width = 128;
28007e98eb0SChris Wilson 		tile->size = 12;
28107e98eb0SChris Wilson 	} else {
28207e98eb0SChris Wilson 		tile->height = 8;
28307e98eb0SChris Wilson 		tile->width = 512;
28407e98eb0SChris Wilson 		tile->size = 12;
28507e98eb0SChris Wilson 	}
28607e98eb0SChris Wilson 
28707e98eb0SChris Wilson 	if (INTEL_GEN(i915) < 4)
28807e98eb0SChris Wilson 		return 8192 / tile->width;
28907e98eb0SChris Wilson 	else if (INTEL_GEN(i915) < 7)
29007e98eb0SChris Wilson 		return 128 * I965_FENCE_MAX_PITCH_VAL / tile->width;
29107e98eb0SChris Wilson 	else
29207e98eb0SChris Wilson 		return 128 * GEN7_FENCE_MAX_PITCH_VAL / tile->width;
29307e98eb0SChris Wilson }
29407e98eb0SChris Wilson 
295b414fcd5SChris Wilson static int igt_partial_tiling(void *arg)
296b414fcd5SChris Wilson {
297b414fcd5SChris Wilson 	const unsigned int nreal = 1 << 12; /* largest tile row x2 */
298b414fcd5SChris Wilson 	struct drm_i915_private *i915 = arg;
299b414fcd5SChris Wilson 	struct drm_i915_gem_object *obj;
300b414fcd5SChris Wilson 	intel_wakeref_t wakeref;
301b414fcd5SChris Wilson 	int tiling;
302b414fcd5SChris Wilson 	int err;
303b414fcd5SChris Wilson 
304b414fcd5SChris Wilson 	/* We want to check the page mapping and fencing of a large object
305b414fcd5SChris Wilson 	 * mmapped through the GTT. The object we create is larger than can
306b414fcd5SChris Wilson 	 * possibly be mmaped as a whole, and so we must use partial GGTT vma.
307b414fcd5SChris Wilson 	 * We then check that a write through each partial GGTT vma ends up
308b414fcd5SChris Wilson 	 * in the right set of pages within the object, and with the expected
309b414fcd5SChris Wilson 	 * tiling, which we verify by manual swizzling.
310b414fcd5SChris Wilson 	 */
311b414fcd5SChris Wilson 
312b414fcd5SChris Wilson 	obj = huge_gem_object(i915,
313b414fcd5SChris Wilson 			      nreal << PAGE_SHIFT,
314b414fcd5SChris Wilson 			      (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
315b414fcd5SChris Wilson 	if (IS_ERR(obj))
316b414fcd5SChris Wilson 		return PTR_ERR(obj);
317b414fcd5SChris Wilson 
318b414fcd5SChris Wilson 	err = i915_gem_object_pin_pages(obj);
319b414fcd5SChris Wilson 	if (err) {
320b414fcd5SChris Wilson 		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
321b414fcd5SChris Wilson 		       nreal, obj->base.size / PAGE_SIZE, err);
322b414fcd5SChris Wilson 		goto out;
323b414fcd5SChris Wilson 	}
324b414fcd5SChris Wilson 
325d858d569SDaniele Ceraolo Spurio 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
326b414fcd5SChris Wilson 
327b414fcd5SChris Wilson 	if (1) {
328b414fcd5SChris Wilson 		IGT_TIMEOUT(end);
329b414fcd5SChris Wilson 		struct tile tile;
330b414fcd5SChris Wilson 
331b414fcd5SChris Wilson 		tile.height = 1;
332b414fcd5SChris Wilson 		tile.width = 1;
333b414fcd5SChris Wilson 		tile.size = 0;
334b414fcd5SChris Wilson 		tile.stride = 0;
335b414fcd5SChris Wilson 		tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
336b414fcd5SChris Wilson 		tile.tiling = I915_TILING_NONE;
337b414fcd5SChris Wilson 
33807e98eb0SChris Wilson 		err = check_partial_mappings(obj, &tile, end);
339b414fcd5SChris Wilson 		if (err && err != -EINTR)
340b414fcd5SChris Wilson 			goto out_unlock;
341b414fcd5SChris Wilson 	}
342b414fcd5SChris Wilson 
343b414fcd5SChris Wilson 	for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) {
344b414fcd5SChris Wilson 		IGT_TIMEOUT(end);
345b414fcd5SChris Wilson 		unsigned int max_pitch;
346b414fcd5SChris Wilson 		unsigned int pitch;
347b414fcd5SChris Wilson 		struct tile tile;
348b414fcd5SChris Wilson 
349b414fcd5SChris Wilson 		if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
350b414fcd5SChris Wilson 			/*
351b414fcd5SChris Wilson 			 * The swizzling pattern is actually unknown as it
352b414fcd5SChris Wilson 			 * varies based on physical address of each page.
353b414fcd5SChris Wilson 			 * See i915_gem_detect_bit_6_swizzle().
354b414fcd5SChris Wilson 			 */
355b414fcd5SChris Wilson 			break;
356b414fcd5SChris Wilson 
357b414fcd5SChris Wilson 		tile.tiling = tiling;
358b414fcd5SChris Wilson 		switch (tiling) {
359b414fcd5SChris Wilson 		case I915_TILING_X:
360972c646fSChris Wilson 			tile.swizzle = i915->ggtt.bit_6_swizzle_x;
361b414fcd5SChris Wilson 			break;
362b414fcd5SChris Wilson 		case I915_TILING_Y:
363972c646fSChris Wilson 			tile.swizzle = i915->ggtt.bit_6_swizzle_y;
364b414fcd5SChris Wilson 			break;
365b414fcd5SChris Wilson 		}
366b414fcd5SChris Wilson 
367b414fcd5SChris Wilson 		GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN);
368b414fcd5SChris Wilson 		if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
369b414fcd5SChris Wilson 		    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
370b414fcd5SChris Wilson 			continue;
371b414fcd5SChris Wilson 
37207e98eb0SChris Wilson 		max_pitch = setup_tile_size(&tile, i915);
373b414fcd5SChris Wilson 
374b414fcd5SChris Wilson 		for (pitch = max_pitch; pitch; pitch >>= 1) {
375b414fcd5SChris Wilson 			tile.stride = tile.width * pitch;
37607e98eb0SChris Wilson 			err = check_partial_mappings(obj, &tile, end);
377b414fcd5SChris Wilson 			if (err == -EINTR)
378b414fcd5SChris Wilson 				goto next_tiling;
379b414fcd5SChris Wilson 			if (err)
380b414fcd5SChris Wilson 				goto out_unlock;
381b414fcd5SChris Wilson 
382b414fcd5SChris Wilson 			if (pitch > 2 && INTEL_GEN(i915) >= 4) {
383b414fcd5SChris Wilson 				tile.stride = tile.width * (pitch - 1);
38407e98eb0SChris Wilson 				err = check_partial_mappings(obj, &tile, end);
385b414fcd5SChris Wilson 				if (err == -EINTR)
386b414fcd5SChris Wilson 					goto next_tiling;
387b414fcd5SChris Wilson 				if (err)
388b414fcd5SChris Wilson 					goto out_unlock;
389b414fcd5SChris Wilson 			}
390b414fcd5SChris Wilson 
391b414fcd5SChris Wilson 			if (pitch < max_pitch && INTEL_GEN(i915) >= 4) {
392b414fcd5SChris Wilson 				tile.stride = tile.width * (pitch + 1);
39307e98eb0SChris Wilson 				err = check_partial_mappings(obj, &tile, end);
394b414fcd5SChris Wilson 				if (err == -EINTR)
395b414fcd5SChris Wilson 					goto next_tiling;
396b414fcd5SChris Wilson 				if (err)
397b414fcd5SChris Wilson 					goto out_unlock;
398b414fcd5SChris Wilson 			}
399b414fcd5SChris Wilson 		}
400b414fcd5SChris Wilson 
401b414fcd5SChris Wilson 		if (INTEL_GEN(i915) >= 4) {
402b414fcd5SChris Wilson 			for_each_prime_number(pitch, max_pitch) {
403b414fcd5SChris Wilson 				tile.stride = tile.width * pitch;
40407e98eb0SChris Wilson 				err = check_partial_mappings(obj, &tile, end);
405b414fcd5SChris Wilson 				if (err == -EINTR)
406b414fcd5SChris Wilson 					goto next_tiling;
407b414fcd5SChris Wilson 				if (err)
408b414fcd5SChris Wilson 					goto out_unlock;
409b414fcd5SChris Wilson 			}
410b414fcd5SChris Wilson 		}
411b414fcd5SChris Wilson 
412b414fcd5SChris Wilson next_tiling: ;
413b414fcd5SChris Wilson 	}
414b414fcd5SChris Wilson 
415b414fcd5SChris Wilson out_unlock:
416d858d569SDaniele Ceraolo Spurio 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
417b414fcd5SChris Wilson 	i915_gem_object_unpin_pages(obj);
418b414fcd5SChris Wilson out:
419b414fcd5SChris Wilson 	i915_gem_object_put(obj);
420b414fcd5SChris Wilson 	return err;
421b414fcd5SChris Wilson }
422b414fcd5SChris Wilson 
42307e98eb0SChris Wilson static int igt_smoke_tiling(void *arg)
42407e98eb0SChris Wilson {
42507e98eb0SChris Wilson 	const unsigned int nreal = 1 << 12; /* largest tile row x2 */
42607e98eb0SChris Wilson 	struct drm_i915_private *i915 = arg;
42707e98eb0SChris Wilson 	struct drm_i915_gem_object *obj;
42807e98eb0SChris Wilson 	intel_wakeref_t wakeref;
42907e98eb0SChris Wilson 	I915_RND_STATE(prng);
43007e98eb0SChris Wilson 	unsigned long count;
43107e98eb0SChris Wilson 	IGT_TIMEOUT(end);
43207e98eb0SChris Wilson 	int err;
43307e98eb0SChris Wilson 
43407e98eb0SChris Wilson 	/*
43507e98eb0SChris Wilson 	 * igt_partial_tiling() does an exhastive check of partial tiling
43607e98eb0SChris Wilson 	 * chunking, but will undoubtably run out of time. Here, we do a
43707e98eb0SChris Wilson 	 * randomised search and hope over many runs of 1s with different
43807e98eb0SChris Wilson 	 * seeds we will do a thorough check.
43907e98eb0SChris Wilson 	 *
44007e98eb0SChris Wilson 	 * Remember to look at the st_seed if we see a flip-flop in BAT!
44107e98eb0SChris Wilson 	 */
44207e98eb0SChris Wilson 
44307e98eb0SChris Wilson 	if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
44407e98eb0SChris Wilson 		return 0;
44507e98eb0SChris Wilson 
44607e98eb0SChris Wilson 	obj = huge_gem_object(i915,
44707e98eb0SChris Wilson 			      nreal << PAGE_SHIFT,
44807e98eb0SChris Wilson 			      (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
44907e98eb0SChris Wilson 	if (IS_ERR(obj))
45007e98eb0SChris Wilson 		return PTR_ERR(obj);
45107e98eb0SChris Wilson 
45207e98eb0SChris Wilson 	err = i915_gem_object_pin_pages(obj);
45307e98eb0SChris Wilson 	if (err) {
45407e98eb0SChris Wilson 		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
45507e98eb0SChris Wilson 		       nreal, obj->base.size / PAGE_SIZE, err);
45607e98eb0SChris Wilson 		goto out;
45707e98eb0SChris Wilson 	}
45807e98eb0SChris Wilson 
45907e98eb0SChris Wilson 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
46007e98eb0SChris Wilson 
46107e98eb0SChris Wilson 	count = 0;
46207e98eb0SChris Wilson 	do {
46307e98eb0SChris Wilson 		struct tile tile;
46407e98eb0SChris Wilson 
46507e98eb0SChris Wilson 		tile.tiling =
46607e98eb0SChris Wilson 			i915_prandom_u32_max_state(I915_TILING_Y + 1, &prng);
46707e98eb0SChris Wilson 		switch (tile.tiling) {
46807e98eb0SChris Wilson 		case I915_TILING_NONE:
46907e98eb0SChris Wilson 			tile.height = 1;
47007e98eb0SChris Wilson 			tile.width = 1;
47107e98eb0SChris Wilson 			tile.size = 0;
47207e98eb0SChris Wilson 			tile.stride = 0;
47307e98eb0SChris Wilson 			tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
47407e98eb0SChris Wilson 			break;
47507e98eb0SChris Wilson 
47607e98eb0SChris Wilson 		case I915_TILING_X:
477972c646fSChris Wilson 			tile.swizzle = i915->ggtt.bit_6_swizzle_x;
47807e98eb0SChris Wilson 			break;
47907e98eb0SChris Wilson 		case I915_TILING_Y:
480972c646fSChris Wilson 			tile.swizzle = i915->ggtt.bit_6_swizzle_y;
48107e98eb0SChris Wilson 			break;
48207e98eb0SChris Wilson 		}
48307e98eb0SChris Wilson 
48407e98eb0SChris Wilson 		if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
48507e98eb0SChris Wilson 		    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
48607e98eb0SChris Wilson 			continue;
48707e98eb0SChris Wilson 
48807e98eb0SChris Wilson 		if (tile.tiling != I915_TILING_NONE) {
48907e98eb0SChris Wilson 			unsigned int max_pitch = setup_tile_size(&tile, i915);
49007e98eb0SChris Wilson 
49107e98eb0SChris Wilson 			tile.stride =
49207e98eb0SChris Wilson 				i915_prandom_u32_max_state(max_pitch, &prng);
49307e98eb0SChris Wilson 			tile.stride = (1 + tile.stride) * tile.width;
49407e98eb0SChris Wilson 			if (INTEL_GEN(i915) < 4)
49507e98eb0SChris Wilson 				tile.stride = rounddown_pow_of_two(tile.stride);
49607e98eb0SChris Wilson 		}
49707e98eb0SChris Wilson 
49807e98eb0SChris Wilson 		err = check_partial_mapping(obj, &tile, &prng);
49907e98eb0SChris Wilson 		if (err)
50007e98eb0SChris Wilson 			break;
50107e98eb0SChris Wilson 
50207e98eb0SChris Wilson 		count++;
50307e98eb0SChris Wilson 	} while (!__igt_timeout(end, NULL));
50407e98eb0SChris Wilson 
50507e98eb0SChris Wilson 	pr_info("%s: Completed %lu trials\n", __func__, count);
50607e98eb0SChris Wilson 
50707e98eb0SChris Wilson 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
50807e98eb0SChris Wilson 	i915_gem_object_unpin_pages(obj);
50907e98eb0SChris Wilson out:
51007e98eb0SChris Wilson 	i915_gem_object_put(obj);
51107e98eb0SChris Wilson 	return err;
51207e98eb0SChris Wilson }
51307e98eb0SChris Wilson 
514b414fcd5SChris Wilson static int make_obj_busy(struct drm_i915_gem_object *obj)
515b414fcd5SChris Wilson {
516b414fcd5SChris Wilson 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
5178f856c74SChris Wilson 	struct intel_engine_cs *engine;
5188f856c74SChris Wilson 	enum intel_engine_id id;
519b414fcd5SChris Wilson 	struct i915_vma *vma;
520b414fcd5SChris Wilson 	int err;
521b414fcd5SChris Wilson 
522b414fcd5SChris Wilson 	vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
523b414fcd5SChris Wilson 	if (IS_ERR(vma))
524b414fcd5SChris Wilson 		return PTR_ERR(vma);
525b414fcd5SChris Wilson 
526b414fcd5SChris Wilson 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
527b414fcd5SChris Wilson 	if (err)
528b414fcd5SChris Wilson 		return err;
529b414fcd5SChris Wilson 
5308f856c74SChris Wilson 	for_each_engine(engine, i915, id) {
5318f856c74SChris Wilson 		struct i915_request *rq;
5328f856c74SChris Wilson 
5338f856c74SChris Wilson 		rq = i915_request_create(engine->kernel_context);
534b414fcd5SChris Wilson 		if (IS_ERR(rq)) {
535b414fcd5SChris Wilson 			i915_vma_unpin(vma);
536b414fcd5SChris Wilson 			return PTR_ERR(rq);
537b414fcd5SChris Wilson 		}
538b414fcd5SChris Wilson 
5396951e589SChris Wilson 		i915_vma_lock(vma);
54070d6894dSChris Wilson 		err = i915_request_await_object(rq, vma->obj, true);
54170d6894dSChris Wilson 		if (err == 0)
54270d6894dSChris Wilson 			err = i915_vma_move_to_active(vma, rq,
54370d6894dSChris Wilson 						      EXEC_OBJECT_WRITE);
5446951e589SChris Wilson 		i915_vma_unlock(vma);
545b414fcd5SChris Wilson 
546b414fcd5SChris Wilson 		i915_request_add(rq);
5478f856c74SChris Wilson 	}
548b414fcd5SChris Wilson 
549b414fcd5SChris Wilson 	i915_vma_unpin(vma);
550c017cf6bSChris Wilson 	i915_gem_object_put(obj); /* leave it only alive via its active ref */
551b414fcd5SChris Wilson 
552b414fcd5SChris Wilson 	return err;
553b414fcd5SChris Wilson }
554b414fcd5SChris Wilson 
555b414fcd5SChris Wilson static bool assert_mmap_offset(struct drm_i915_private *i915,
556b414fcd5SChris Wilson 			       unsigned long size,
557b414fcd5SChris Wilson 			       int expected)
558b414fcd5SChris Wilson {
559b414fcd5SChris Wilson 	struct drm_i915_gem_object *obj;
560b414fcd5SChris Wilson 	int err;
561b414fcd5SChris Wilson 
562b414fcd5SChris Wilson 	obj = i915_gem_object_create_internal(i915, size);
563b414fcd5SChris Wilson 	if (IS_ERR(obj))
564b414fcd5SChris Wilson 		return PTR_ERR(obj);
565b414fcd5SChris Wilson 
566b414fcd5SChris Wilson 	err = create_mmap_offset(obj);
567b414fcd5SChris Wilson 	i915_gem_object_put(obj);
568b414fcd5SChris Wilson 
569b414fcd5SChris Wilson 	return err == expected;
570b414fcd5SChris Wilson }
571b414fcd5SChris Wilson 
572b414fcd5SChris Wilson static void disable_retire_worker(struct drm_i915_private *i915)
573b414fcd5SChris Wilson {
574c29579d2SChris Wilson 	i915_gem_driver_unregister__shrinker(i915);
5750c91621cSChris Wilson 	intel_gt_pm_get(&i915->gt);
57666101975SChris Wilson 	cancel_delayed_work_sync(&i915->gt.requests.retire_work);
577b414fcd5SChris Wilson }
578b414fcd5SChris Wilson 
579b414fcd5SChris Wilson static void restore_retire_worker(struct drm_i915_private *i915)
580b414fcd5SChris Wilson {
5817e805762SChris Wilson 	igt_flush_test(i915);
5820c91621cSChris Wilson 	intel_gt_pm_put(&i915->gt);
583c29579d2SChris Wilson 	i915_gem_driver_register__shrinker(i915);
584b414fcd5SChris Wilson }
585b414fcd5SChris Wilson 
586f63dfc14SChris Wilson static void mmap_offset_lock(struct drm_i915_private *i915)
587f63dfc14SChris Wilson 	__acquires(&i915->drm.vma_offset_manager->vm_lock)
588f63dfc14SChris Wilson {
589f63dfc14SChris Wilson 	write_lock(&i915->drm.vma_offset_manager->vm_lock);
590f63dfc14SChris Wilson }
591f63dfc14SChris Wilson 
592f63dfc14SChris Wilson static void mmap_offset_unlock(struct drm_i915_private *i915)
593f63dfc14SChris Wilson 	__releases(&i915->drm.vma_offset_manager->vm_lock)
594f63dfc14SChris Wilson {
595f63dfc14SChris Wilson 	write_unlock(&i915->drm.vma_offset_manager->vm_lock);
596f63dfc14SChris Wilson }
597f63dfc14SChris Wilson 
598b414fcd5SChris Wilson static int igt_mmap_offset_exhaustion(void *arg)
599b414fcd5SChris Wilson {
600b414fcd5SChris Wilson 	struct drm_i915_private *i915 = arg;
601b414fcd5SChris Wilson 	struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
602b414fcd5SChris Wilson 	struct drm_i915_gem_object *obj;
603b414fcd5SChris Wilson 	struct drm_mm_node resv, *hole;
604b414fcd5SChris Wilson 	u64 hole_start, hole_end;
605b414fcd5SChris Wilson 	int loop, err;
606b414fcd5SChris Wilson 
607b414fcd5SChris Wilson 	/* Disable background reaper */
608b414fcd5SChris Wilson 	disable_retire_worker(i915);
609b414fcd5SChris Wilson 	GEM_BUG_ON(!i915->gt.awake);
610b414fcd5SChris Wilson 
611b414fcd5SChris Wilson 	/* Trim the device mmap space to only a page */
612b414fcd5SChris Wilson 	memset(&resv, 0, sizeof(resv));
613b414fcd5SChris Wilson 	drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
614b414fcd5SChris Wilson 		resv.start = hole_start;
615b414fcd5SChris Wilson 		resv.size = hole_end - hole_start - 1; /* PAGE_SIZE units */
616f63dfc14SChris Wilson 		mmap_offset_lock(i915);
617b414fcd5SChris Wilson 		err = drm_mm_reserve_node(mm, &resv);
618f63dfc14SChris Wilson 		mmap_offset_unlock(i915);
619b414fcd5SChris Wilson 		if (err) {
620b414fcd5SChris Wilson 			pr_err("Failed to trim VMA manager, err=%d\n", err);
621b414fcd5SChris Wilson 			goto out_park;
622b414fcd5SChris Wilson 		}
623b414fcd5SChris Wilson 		break;
624b414fcd5SChris Wilson 	}
625b414fcd5SChris Wilson 
626b414fcd5SChris Wilson 	/* Just fits! */
627b414fcd5SChris Wilson 	if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) {
628b414fcd5SChris Wilson 		pr_err("Unable to insert object into single page hole\n");
629b414fcd5SChris Wilson 		err = -EINVAL;
630b414fcd5SChris Wilson 		goto out;
631b414fcd5SChris Wilson 	}
632b414fcd5SChris Wilson 
633b414fcd5SChris Wilson 	/* Too large */
634b414fcd5SChris Wilson 	if (!assert_mmap_offset(i915, 2 * PAGE_SIZE, -ENOSPC)) {
635b414fcd5SChris Wilson 		pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n");
636b414fcd5SChris Wilson 		err = -EINVAL;
637b414fcd5SChris Wilson 		goto out;
638b414fcd5SChris Wilson 	}
639b414fcd5SChris Wilson 
640b414fcd5SChris Wilson 	/* Fill the hole, further allocation attempts should then fail */
641b414fcd5SChris Wilson 	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
642b414fcd5SChris Wilson 	if (IS_ERR(obj)) {
643b414fcd5SChris Wilson 		err = PTR_ERR(obj);
644b414fcd5SChris Wilson 		goto out;
645b414fcd5SChris Wilson 	}
646b414fcd5SChris Wilson 
647b414fcd5SChris Wilson 	err = create_mmap_offset(obj);
648b414fcd5SChris Wilson 	if (err) {
649b414fcd5SChris Wilson 		pr_err("Unable to insert object into reclaimed hole\n");
650b414fcd5SChris Wilson 		goto err_obj;
651b414fcd5SChris Wilson 	}
652b414fcd5SChris Wilson 
653b414fcd5SChris Wilson 	if (!assert_mmap_offset(i915, PAGE_SIZE, -ENOSPC)) {
654b414fcd5SChris Wilson 		pr_err("Unexpectedly succeeded in inserting object into no holes!\n");
655b414fcd5SChris Wilson 		err = -EINVAL;
656b414fcd5SChris Wilson 		goto err_obj;
657b414fcd5SChris Wilson 	}
658b414fcd5SChris Wilson 
659b414fcd5SChris Wilson 	i915_gem_object_put(obj);
660b414fcd5SChris Wilson 
661b414fcd5SChris Wilson 	/* Now fill with busy dead objects that we expect to reap */
662b414fcd5SChris Wilson 	for (loop = 0; loop < 3; loop++) {
663cb823ed9SChris Wilson 		if (intel_gt_is_wedged(&i915->gt))
664b414fcd5SChris Wilson 			break;
665b414fcd5SChris Wilson 
666b414fcd5SChris Wilson 		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
667b414fcd5SChris Wilson 		if (IS_ERR(obj)) {
668b414fcd5SChris Wilson 			err = PTR_ERR(obj);
669b414fcd5SChris Wilson 			goto out;
670b414fcd5SChris Wilson 		}
671b414fcd5SChris Wilson 
672b414fcd5SChris Wilson 		err = make_obj_busy(obj);
673b414fcd5SChris Wilson 		if (err) {
674b414fcd5SChris Wilson 			pr_err("[loop %d] Failed to busy the object\n", loop);
675b414fcd5SChris Wilson 			goto err_obj;
676b414fcd5SChris Wilson 		}
677b414fcd5SChris Wilson 	}
678b414fcd5SChris Wilson 
679b414fcd5SChris Wilson out:
680f63dfc14SChris Wilson 	mmap_offset_lock(i915);
681b414fcd5SChris Wilson 	drm_mm_remove_node(&resv);
682f63dfc14SChris Wilson 	mmap_offset_unlock(i915);
683b414fcd5SChris Wilson out_park:
684b414fcd5SChris Wilson 	restore_retire_worker(i915);
685b414fcd5SChris Wilson 	return err;
686b414fcd5SChris Wilson err_obj:
687b414fcd5SChris Wilson 	i915_gem_object_put(obj);
688b414fcd5SChris Wilson 	goto out;
689b414fcd5SChris Wilson }
690b414fcd5SChris Wilson 
691b414fcd5SChris Wilson int i915_gem_mman_live_selftests(struct drm_i915_private *i915)
692b414fcd5SChris Wilson {
693b414fcd5SChris Wilson 	static const struct i915_subtest tests[] = {
694b414fcd5SChris Wilson 		SUBTEST(igt_partial_tiling),
69507e98eb0SChris Wilson 		SUBTEST(igt_smoke_tiling),
696b414fcd5SChris Wilson 		SUBTEST(igt_mmap_offset_exhaustion),
697b414fcd5SChris Wilson 	};
698b414fcd5SChris Wilson 
699b414fcd5SChris Wilson 	return i915_subtests(tests, i915);
700b414fcd5SChris Wilson }
701