1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2008 Intel Corporation
5  */
6 
7 #include <linux/string.h>
8 #include <linux/bitops.h>
9 
10 #include "i915_drv.h"
11 #include "i915_gem.h"
12 #include "i915_gem_ioctls.h"
13 #include "i915_gem_mman.h"
14 #include "i915_gem_object.h"
15 #include "i915_gem_tiling.h"
16 #include "i915_reg.h"
17 
18 /**
19  * DOC: buffer object tiling
20  *
21  * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
22  * interface to declare fence register requirements.
23  *
24  * In principle GEM doesn't care at all about the internal data layout of an
25  * object, and hence it also doesn't care about tiling or swizzling. There's two
26  * exceptions:
27  *
28  * - For X and Y tiling the hardware provides detilers for CPU access, so called
29  *   fences. Since there's only a limited amount of them the kernel must manage
30  *   these, and therefore userspace must tell the kernel the object tiling if it
31  *   wants to use fences for detiling.
32  * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
33  *   depends upon the physical page frame number. When swapping such objects the
34  *   page frame number might change and the kernel must be able to fix this up
35  *   and hence now the tiling. Note that on a subset of platforms with
36  *   asymmetric memory channel population the swizzling pattern changes in an
37  *   unknown way, and for those the kernel simply forbids swapping completely.
38  *
39  * Since neither of this applies for new tiling layouts on modern platforms like
40  * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
41  * Anything else can be handled in userspace entirely without the kernel's
42  * invovlement.
43  */
44 
45 /**
46  * i915_gem_fence_size - required global GTT size for a fence
47  * @i915: i915 device
48  * @size: object size
49  * @tiling: tiling mode
50  * @stride: tiling stride
51  *
52  * Return the required global GTT size for a fence (view of a tiled object),
53  * taking into account potential fence register mapping.
54  */
55 u32 i915_gem_fence_size(struct drm_i915_private *i915,
56 			u32 size, unsigned int tiling, unsigned int stride)
57 {
58 	u32 ggtt_size;
59 
60 	GEM_BUG_ON(!size);
61 
62 	if (tiling == I915_TILING_NONE)
63 		return size;
64 
65 	GEM_BUG_ON(!stride);
66 
67 	if (GRAPHICS_VER(i915) >= 4) {
68 		stride *= i915_gem_tile_height(tiling);
69 		GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
70 		return roundup(size, stride);
71 	}
72 
73 	/* Previous chips need a power-of-two fence region when tiling */
74 	if (GRAPHICS_VER(i915) == 3)
75 		ggtt_size = 1024*1024;
76 	else
77 		ggtt_size = 512*1024;
78 
79 	while (ggtt_size < size)
80 		ggtt_size <<= 1;
81 
82 	return ggtt_size;
83 }
84 
85 /**
86  * i915_gem_fence_alignment - required global GTT alignment for a fence
87  * @i915: i915 device
88  * @size: object size
89  * @tiling: tiling mode
90  * @stride: tiling stride
91  *
92  * Return the required global GTT alignment for a fence (a view of a tiled
93  * object), taking into account potential fence register mapping.
94  */
95 u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
96 			     unsigned int tiling, unsigned int stride)
97 {
98 	GEM_BUG_ON(!size);
99 
100 	/*
101 	 * Minimum alignment is 4k (GTT page size), but might be greater
102 	 * if a fence register is needed for the object.
103 	 */
104 	if (tiling == I915_TILING_NONE)
105 		return I915_GTT_MIN_ALIGNMENT;
106 
107 	if (GRAPHICS_VER(i915) >= 4)
108 		return I965_FENCE_PAGE;
109 
110 	/*
111 	 * Previous chips need to be aligned to the size of the smallest
112 	 * fence register that can contain the object.
113 	 */
114 	return i915_gem_fence_size(i915, size, tiling, stride);
115 }
116 
117 /* Check pitch constraints for all chips & tiling formats */
118 static bool
119 i915_tiling_ok(struct drm_i915_gem_object *obj,
120 	       unsigned int tiling, unsigned int stride)
121 {
122 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
123 	unsigned int tile_width;
124 
125 	/* Linear is always fine */
126 	if (tiling == I915_TILING_NONE)
127 		return true;
128 
129 	if (tiling > I915_TILING_LAST)
130 		return false;
131 
132 	/* check maximum stride & object size */
133 	/* i965+ stores the end address of the gtt mapping in the fence
134 	 * reg, so dont bother to check the size */
135 	if (GRAPHICS_VER(i915) >= 7) {
136 		if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
137 			return false;
138 	} else if (GRAPHICS_VER(i915) >= 4) {
139 		if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
140 			return false;
141 	} else {
142 		if (stride > 8192)
143 			return false;
144 
145 		if (!is_power_of_2(stride))
146 			return false;
147 	}
148 
149 	if (GRAPHICS_VER(i915) == 2 ||
150 	    (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
151 		tile_width = 128;
152 	else
153 		tile_width = 512;
154 
155 	if (!stride || !IS_ALIGNED(stride, tile_width))
156 		return false;
157 
158 	return true;
159 }
160 
161 static bool i915_vma_fence_prepare(struct i915_vma *vma,
162 				   int tiling_mode, unsigned int stride)
163 {
164 	struct drm_i915_private *i915 = vma->vm->i915;
165 	u32 size, alignment;
166 
167 	if (!i915_vma_is_map_and_fenceable(vma))
168 		return true;
169 
170 	size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
171 	if (i915_vma_size(vma) < size)
172 		return false;
173 
174 	alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
175 	if (!IS_ALIGNED(i915_ggtt_offset(vma), alignment))
176 		return false;
177 
178 	return true;
179 }
180 
181 /* Make the current GTT allocation valid for the change in tiling. */
182 static int
183 i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
184 			      int tiling_mode, unsigned int stride)
185 {
186 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
187 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
188 	struct i915_vma *vma, *vn;
189 	LIST_HEAD(unbind);
190 	int ret = 0;
191 
192 	if (tiling_mode == I915_TILING_NONE)
193 		return 0;
194 
195 	mutex_lock(&ggtt->vm.mutex);
196 
197 	spin_lock(&obj->vma.lock);
198 	for_each_ggtt_vma(vma, obj) {
199 		GEM_BUG_ON(vma->vm != &ggtt->vm);
200 
201 		if (i915_vma_fence_prepare(vma, tiling_mode, stride))
202 			continue;
203 
204 		list_move(&vma->vm_link, &unbind);
205 	}
206 	spin_unlock(&obj->vma.lock);
207 
208 	list_for_each_entry_safe(vma, vn, &unbind, vm_link) {
209 		ret = __i915_vma_unbind(vma);
210 		if (ret) {
211 			/* Restore the remaining vma on an error */
212 			list_splice(&unbind, &ggtt->vm.bound_list);
213 			break;
214 		}
215 	}
216 
217 	mutex_unlock(&ggtt->vm.mutex);
218 
219 	return ret;
220 }
221 
222 bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
223 {
224 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
225 
226 	return to_gt(i915)->ggtt->bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
227 		i915_gem_object_is_tiled(obj);
228 }
229 
230 int
231 i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
232 			   unsigned int tiling, unsigned int stride)
233 {
234 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
235 	struct i915_vma *vma;
236 	int err;
237 
238 	/* Make sure we don't cross-contaminate obj->tiling_and_stride */
239 	BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);
240 
241 	GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
242 	GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
243 
244 	if ((tiling | stride) == obj->tiling_and_stride)
245 		return 0;
246 
247 	if (i915_gem_object_is_framebuffer(obj))
248 		return -EBUSY;
249 
250 	/* We need to rebind the object if its current allocation
251 	 * no longer meets the alignment restrictions for its new
252 	 * tiling mode. Otherwise we can just leave it alone, but
253 	 * need to ensure that any fence register is updated before
254 	 * the next fenced (either through the GTT or by the BLT unit
255 	 * on older GPUs) access.
256 	 *
257 	 * After updating the tiling parameters, we then flag whether
258 	 * we need to update an associated fence register. Note this
259 	 * has to also include the unfenced register the GPU uses
260 	 * whilst executing a fenced command for an untiled object.
261 	 */
262 
263 	i915_gem_object_lock(obj, NULL);
264 	if (i915_gem_object_is_framebuffer(obj)) {
265 		i915_gem_object_unlock(obj);
266 		return -EBUSY;
267 	}
268 
269 	err = i915_gem_object_fence_prepare(obj, tiling, stride);
270 	if (err) {
271 		i915_gem_object_unlock(obj);
272 		return err;
273 	}
274 
275 	/* If the memory has unknown (i.e. varying) swizzling, we pin the
276 	 * pages to prevent them being swapped out and causing corruption
277 	 * due to the change in swizzling.
278 	 */
279 	if (i915_gem_object_has_pages(obj) &&
280 	    obj->mm.madv == I915_MADV_WILLNEED &&
281 	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
282 		if (tiling == I915_TILING_NONE) {
283 			GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
284 			i915_gem_object_clear_tiling_quirk(obj);
285 			i915_gem_object_make_shrinkable(obj);
286 		}
287 		if (!i915_gem_object_is_tiled(obj)) {
288 			GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
289 			i915_gem_object_make_unshrinkable(obj);
290 			i915_gem_object_set_tiling_quirk(obj);
291 		}
292 	}
293 
294 	spin_lock(&obj->vma.lock);
295 	for_each_ggtt_vma(vma, obj) {
296 		vma->fence_size =
297 			i915_gem_fence_size(i915, vma->size, tiling, stride);
298 		vma->fence_alignment =
299 			i915_gem_fence_alignment(i915,
300 						 vma->size, tiling, stride);
301 
302 		if (vma->fence)
303 			vma->fence->dirty = true;
304 	}
305 	spin_unlock(&obj->vma.lock);
306 
307 	obj->tiling_and_stride = tiling | stride;
308 
309 	/* Try to preallocate memory required to save swizzling on put-pages */
310 	if (i915_gem_object_needs_bit17_swizzle(obj)) {
311 		if (!obj->bit_17) {
312 			obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT,
313 						    GFP_KERNEL);
314 		}
315 	} else {
316 		bitmap_free(obj->bit_17);
317 		obj->bit_17 = NULL;
318 	}
319 
320 	i915_gem_object_unlock(obj);
321 
322 	/* Force the fence to be reacquired for GTT access */
323 	i915_gem_object_release_mmap_gtt(obj);
324 
325 	return 0;
326 }
327 
328 /**
329  * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
330  * @dev: DRM device
331  * @data: data pointer for the ioctl
332  * @file: DRM file for the ioctl call
333  *
334  * Sets the tiling mode of an object, returning the required swizzling of
335  * bit 6 of addresses in the object.
336  *
337  * Called by the user via ioctl.
338  *
339  * Returns:
340  * Zero on success, negative errno on failure.
341  */
342 int
343 i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
344 			  struct drm_file *file)
345 {
346 	struct drm_i915_private *dev_priv = to_i915(dev);
347 	struct drm_i915_gem_set_tiling *args = data;
348 	struct drm_i915_gem_object *obj;
349 	int err;
350 
351 	if (!to_gt(dev_priv)->ggtt->num_fences)
352 		return -EOPNOTSUPP;
353 
354 	obj = i915_gem_object_lookup(file, args->handle);
355 	if (!obj)
356 		return -ENOENT;
357 
358 	/*
359 	 * The tiling mode of proxy objects is handled by its generator, and
360 	 * not allowed to be changed by userspace.
361 	 */
362 	if (i915_gem_object_is_proxy(obj)) {
363 		err = -ENXIO;
364 		goto err;
365 	}
366 
367 	if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
368 		err = -EINVAL;
369 		goto err;
370 	}
371 
372 	if (args->tiling_mode == I915_TILING_NONE) {
373 		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
374 		args->stride = 0;
375 	} else {
376 		if (args->tiling_mode == I915_TILING_X)
377 			args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x;
378 		else
379 			args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y;
380 
381 		/* Hide bit 17 swizzling from the user.  This prevents old Mesa
382 		 * from aborting the application on sw fallbacks to bit 17,
383 		 * and we use the pread/pwrite bit17 paths to swizzle for it.
384 		 * If there was a user that was relying on the swizzle
385 		 * information for drm_intel_bo_map()ed reads/writes this would
386 		 * break it, but we don't have any of those.
387 		 */
388 		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
389 			args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
390 		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
391 			args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
392 
393 		/* If we can't handle the swizzling, make it untiled. */
394 		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
395 			args->tiling_mode = I915_TILING_NONE;
396 			args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
397 			args->stride = 0;
398 		}
399 	}
400 
401 	err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
402 
403 	/* We have to maintain this existing ABI... */
404 	args->stride = i915_gem_object_get_stride(obj);
405 	args->tiling_mode = i915_gem_object_get_tiling(obj);
406 
407 err:
408 	i915_gem_object_put(obj);
409 	return err;
410 }
411 
412 /**
413  * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
414  * @dev: DRM device
415  * @data: data pointer for the ioctl
416  * @file: DRM file for the ioctl call
417  *
418  * Returns the current tiling mode and required bit 6 swizzling for the object.
419  *
420  * Called by the user via ioctl.
421  *
422  * Returns:
423  * Zero on success, negative errno on failure.
424  */
425 int
426 i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
427 			  struct drm_file *file)
428 {
429 	struct drm_i915_gem_get_tiling *args = data;
430 	struct drm_i915_private *dev_priv = to_i915(dev);
431 	struct drm_i915_gem_object *obj;
432 	int err = -ENOENT;
433 
434 	if (!to_gt(dev_priv)->ggtt->num_fences)
435 		return -EOPNOTSUPP;
436 
437 	rcu_read_lock();
438 	obj = i915_gem_object_lookup_rcu(file, args->handle);
439 	if (obj) {
440 		args->tiling_mode =
441 			READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
442 		err = 0;
443 	}
444 	rcu_read_unlock();
445 	if (unlikely(err))
446 		return err;
447 
448 	switch (args->tiling_mode) {
449 	case I915_TILING_X:
450 		args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x;
451 		break;
452 	case I915_TILING_Y:
453 		args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y;
454 		break;
455 	default:
456 	case I915_TILING_NONE:
457 		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
458 		break;
459 	}
460 
461 	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
462 	if (dev_priv->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
463 		args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
464 	else
465 		args->phys_swizzle_mode = args->swizzle_mode;
466 	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
467 		args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
468 	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
469 		args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
470 
471 	return 0;
472 }
473