1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2008 Intel Corporation 5 */ 6 7 #include <linux/string.h> 8 #include <linux/bitops.h> 9 #include <drm/i915_drm.h> 10 11 #include "i915_drv.h" 12 #include "i915_gem.h" 13 #include "i915_gem_ioctls.h" 14 #include "i915_gem_mman.h" 15 #include "i915_gem_object.h" 16 17 /** 18 * DOC: buffer object tiling 19 * 20 * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace 21 * interface to declare fence register requirements. 22 * 23 * In principle GEM doesn't care at all about the internal data layout of an 24 * object, and hence it also doesn't care about tiling or swizzling. There's two 25 * exceptions: 26 * 27 * - For X and Y tiling the hardware provides detilers for CPU access, so called 28 * fences. Since there's only a limited amount of them the kernel must manage 29 * these, and therefore userspace must tell the kernel the object tiling if it 30 * wants to use fences for detiling. 31 * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which 32 * depends upon the physical page frame number. When swapping such objects the 33 * page frame number might change and the kernel must be able to fix this up 34 * and hence now the tiling. Note that on a subset of platforms with 35 * asymmetric memory channel population the swizzling pattern changes in an 36 * unknown way, and for those the kernel simply forbids swapping completely. 37 * 38 * Since neither of this applies for new tiling layouts on modern platforms like 39 * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled. 40 * Anything else can be handled in userspace entirely without the kernel's 41 * invovlement. 42 */ 43 44 /** 45 * i915_gem_fence_size - required global GTT size for a fence 46 * @i915: i915 device 47 * @size: object size 48 * @tiling: tiling mode 49 * @stride: tiling stride 50 * 51 * Return the required global GTT size for a fence (view of a tiled object), 52 * taking into account potential fence register mapping. 53 */ 54 u32 i915_gem_fence_size(struct drm_i915_private *i915, 55 u32 size, unsigned int tiling, unsigned int stride) 56 { 57 u32 ggtt_size; 58 59 GEM_BUG_ON(!size); 60 61 if (tiling == I915_TILING_NONE) 62 return size; 63 64 GEM_BUG_ON(!stride); 65 66 if (INTEL_GEN(i915) >= 4) { 67 stride *= i915_gem_tile_height(tiling); 68 GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE)); 69 return roundup(size, stride); 70 } 71 72 /* Previous chips need a power-of-two fence region when tiling */ 73 if (IS_GEN(i915, 3)) 74 ggtt_size = 1024*1024; 75 else 76 ggtt_size = 512*1024; 77 78 while (ggtt_size < size) 79 ggtt_size <<= 1; 80 81 return ggtt_size; 82 } 83 84 /** 85 * i915_gem_fence_alignment - required global GTT alignment for a fence 86 * @i915: i915 device 87 * @size: object size 88 * @tiling: tiling mode 89 * @stride: tiling stride 90 * 91 * Return the required global GTT alignment for a fence (a view of a tiled 92 * object), taking into account potential fence register mapping. 93 */ 94 u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size, 95 unsigned int tiling, unsigned int stride) 96 { 97 GEM_BUG_ON(!size); 98 99 /* 100 * Minimum alignment is 4k (GTT page size), but might be greater 101 * if a fence register is needed for the object. 102 */ 103 if (tiling == I915_TILING_NONE) 104 return I915_GTT_MIN_ALIGNMENT; 105 106 if (INTEL_GEN(i915) >= 4) 107 return I965_FENCE_PAGE; 108 109 /* 110 * Previous chips need to be aligned to the size of the smallest 111 * fence register that can contain the object. 112 */ 113 return i915_gem_fence_size(i915, size, tiling, stride); 114 } 115 116 /* Check pitch constriants for all chips & tiling formats */ 117 static bool 118 i915_tiling_ok(struct drm_i915_gem_object *obj, 119 unsigned int tiling, unsigned int stride) 120 { 121 struct drm_i915_private *i915 = to_i915(obj->base.dev); 122 unsigned int tile_width; 123 124 /* Linear is always fine */ 125 if (tiling == I915_TILING_NONE) 126 return true; 127 128 if (tiling > I915_TILING_LAST) 129 return false; 130 131 /* check maximum stride & object size */ 132 /* i965+ stores the end address of the gtt mapping in the fence 133 * reg, so dont bother to check the size */ 134 if (INTEL_GEN(i915) >= 7) { 135 if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL) 136 return false; 137 } else if (INTEL_GEN(i915) >= 4) { 138 if (stride / 128 > I965_FENCE_MAX_PITCH_VAL) 139 return false; 140 } else { 141 if (stride > 8192) 142 return false; 143 144 if (!is_power_of_2(stride)) 145 return false; 146 } 147 148 if (IS_GEN(i915, 2) || 149 (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915))) 150 tile_width = 128; 151 else 152 tile_width = 512; 153 154 if (!stride || !IS_ALIGNED(stride, tile_width)) 155 return false; 156 157 return true; 158 } 159 160 static bool i915_vma_fence_prepare(struct i915_vma *vma, 161 int tiling_mode, unsigned int stride) 162 { 163 struct drm_i915_private *i915 = vma->vm->i915; 164 u32 size, alignment; 165 166 if (!i915_vma_is_map_and_fenceable(vma)) 167 return true; 168 169 size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride); 170 if (vma->node.size < size) 171 return false; 172 173 alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride); 174 if (!IS_ALIGNED(vma->node.start, alignment)) 175 return false; 176 177 return true; 178 } 179 180 /* Make the current GTT allocation valid for the change in tiling. */ 181 static int 182 i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj, 183 int tiling_mode, unsigned int stride) 184 { 185 struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt; 186 struct i915_vma *vma; 187 int ret = 0; 188 189 if (tiling_mode == I915_TILING_NONE) 190 return 0; 191 192 mutex_lock(&ggtt->vm.mutex); 193 for_each_ggtt_vma(vma, obj) { 194 if (i915_vma_fence_prepare(vma, tiling_mode, stride)) 195 continue; 196 197 ret = __i915_vma_unbind(vma); 198 if (ret) 199 break; 200 } 201 mutex_unlock(&ggtt->vm.mutex); 202 203 return ret; 204 } 205 206 int 207 i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 208 unsigned int tiling, unsigned int stride) 209 { 210 struct drm_i915_private *i915 = to_i915(obj->base.dev); 211 struct i915_vma *vma; 212 int err; 213 214 /* Make sure we don't cross-contaminate obj->tiling_and_stride */ 215 BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK); 216 217 GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride)); 218 GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE)); 219 220 if ((tiling | stride) == obj->tiling_and_stride) 221 return 0; 222 223 if (i915_gem_object_is_framebuffer(obj)) 224 return -EBUSY; 225 226 /* We need to rebind the object if its current allocation 227 * no longer meets the alignment restrictions for its new 228 * tiling mode. Otherwise we can just leave it alone, but 229 * need to ensure that any fence register is updated before 230 * the next fenced (either through the GTT or by the BLT unit 231 * on older GPUs) access. 232 * 233 * After updating the tiling parameters, we then flag whether 234 * we need to update an associated fence register. Note this 235 * has to also include the unfenced register the GPU uses 236 * whilst executing a fenced command for an untiled object. 237 */ 238 239 i915_gem_object_lock(obj); 240 if (i915_gem_object_is_framebuffer(obj)) { 241 i915_gem_object_unlock(obj); 242 return -EBUSY; 243 } 244 245 err = i915_gem_object_fence_prepare(obj, tiling, stride); 246 if (err) { 247 i915_gem_object_unlock(obj); 248 return err; 249 } 250 251 /* If the memory has unknown (i.e. varying) swizzling, we pin the 252 * pages to prevent them being swapped out and causing corruption 253 * due to the change in swizzling. 254 */ 255 mutex_lock(&obj->mm.lock); 256 if (i915_gem_object_has_pages(obj) && 257 obj->mm.madv == I915_MADV_WILLNEED && 258 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 259 if (tiling == I915_TILING_NONE) { 260 GEM_BUG_ON(!obj->mm.quirked); 261 __i915_gem_object_unpin_pages(obj); 262 obj->mm.quirked = false; 263 } 264 if (!i915_gem_object_is_tiled(obj)) { 265 GEM_BUG_ON(obj->mm.quirked); 266 __i915_gem_object_pin_pages(obj); 267 obj->mm.quirked = true; 268 } 269 } 270 mutex_unlock(&obj->mm.lock); 271 272 for_each_ggtt_vma(vma, obj) { 273 vma->fence_size = 274 i915_gem_fence_size(i915, vma->size, tiling, stride); 275 vma->fence_alignment = 276 i915_gem_fence_alignment(i915, 277 vma->size, tiling, stride); 278 279 if (vma->fence) 280 vma->fence->dirty = true; 281 } 282 283 obj->tiling_and_stride = tiling | stride; 284 i915_gem_object_unlock(obj); 285 286 /* Force the fence to be reacquired for GTT access */ 287 i915_gem_object_release_mmap(obj); 288 289 /* Try to preallocate memory required to save swizzling on put-pages */ 290 if (i915_gem_object_needs_bit17_swizzle(obj)) { 291 if (!obj->bit_17) { 292 obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT, 293 GFP_KERNEL); 294 } 295 } else { 296 bitmap_free(obj->bit_17); 297 obj->bit_17 = NULL; 298 } 299 300 return 0; 301 } 302 303 /** 304 * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode 305 * @dev: DRM device 306 * @data: data pointer for the ioctl 307 * @file: DRM file for the ioctl call 308 * 309 * Sets the tiling mode of an object, returning the required swizzling of 310 * bit 6 of addresses in the object. 311 * 312 * Called by the user via ioctl. 313 * 314 * Returns: 315 * Zero on success, negative errno on failure. 316 */ 317 int 318 i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, 319 struct drm_file *file) 320 { 321 struct drm_i915_private *dev_priv = to_i915(dev); 322 struct drm_i915_gem_set_tiling *args = data; 323 struct drm_i915_gem_object *obj; 324 int err; 325 326 if (!dev_priv->ggtt.num_fences) 327 return -EOPNOTSUPP; 328 329 obj = i915_gem_object_lookup(file, args->handle); 330 if (!obj) 331 return -ENOENT; 332 333 /* 334 * The tiling mode of proxy objects is handled by its generator, and 335 * not allowed to be changed by userspace. 336 */ 337 if (i915_gem_object_is_proxy(obj)) { 338 err = -ENXIO; 339 goto err; 340 } 341 342 if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) { 343 err = -EINVAL; 344 goto err; 345 } 346 347 if (args->tiling_mode == I915_TILING_NONE) { 348 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 349 args->stride = 0; 350 } else { 351 if (args->tiling_mode == I915_TILING_X) 352 args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_x; 353 else 354 args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_y; 355 356 /* Hide bit 17 swizzling from the user. This prevents old Mesa 357 * from aborting the application on sw fallbacks to bit 17, 358 * and we use the pread/pwrite bit17 paths to swizzle for it. 359 * If there was a user that was relying on the swizzle 360 * information for drm_intel_bo_map()ed reads/writes this would 361 * break it, but we don't have any of those. 362 */ 363 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) 364 args->swizzle_mode = I915_BIT_6_SWIZZLE_9; 365 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 366 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 367 368 /* If we can't handle the swizzling, make it untiled. */ 369 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) { 370 args->tiling_mode = I915_TILING_NONE; 371 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 372 args->stride = 0; 373 } 374 } 375 376 err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride); 377 378 /* We have to maintain this existing ABI... */ 379 args->stride = i915_gem_object_get_stride(obj); 380 args->tiling_mode = i915_gem_object_get_tiling(obj); 381 382 err: 383 i915_gem_object_put(obj); 384 return err; 385 } 386 387 /** 388 * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode 389 * @dev: DRM device 390 * @data: data pointer for the ioctl 391 * @file: DRM file for the ioctl call 392 * 393 * Returns the current tiling mode and required bit 6 swizzling for the object. 394 * 395 * Called by the user via ioctl. 396 * 397 * Returns: 398 * Zero on success, negative errno on failure. 399 */ 400 int 401 i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, 402 struct drm_file *file) 403 { 404 struct drm_i915_gem_get_tiling *args = data; 405 struct drm_i915_private *dev_priv = to_i915(dev); 406 struct drm_i915_gem_object *obj; 407 int err = -ENOENT; 408 409 if (!dev_priv->ggtt.num_fences) 410 return -EOPNOTSUPP; 411 412 rcu_read_lock(); 413 obj = i915_gem_object_lookup_rcu(file, args->handle); 414 if (obj) { 415 args->tiling_mode = 416 READ_ONCE(obj->tiling_and_stride) & TILING_MASK; 417 err = 0; 418 } 419 rcu_read_unlock(); 420 if (unlikely(err)) 421 return err; 422 423 switch (args->tiling_mode) { 424 case I915_TILING_X: 425 args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_x; 426 break; 427 case I915_TILING_Y: 428 args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_y; 429 break; 430 default: 431 case I915_TILING_NONE: 432 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 433 break; 434 } 435 436 /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ 437 if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) 438 args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN; 439 else 440 args->phys_swizzle_mode = args->swizzle_mode; 441 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) 442 args->swizzle_mode = I915_BIT_6_SWIZZLE_9; 443 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 444 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 445 446 return 0; 447 } 448