1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/mman.h> 8 #include <linux/sizes.h> 9 10 #include "i915_drv.h" 11 #include "i915_gem_gtt.h" 12 #include "i915_gem_ioctls.h" 13 #include "i915_gem_object.h" 14 #include "i915_vma.h" 15 #include "intel_drv.h" 16 17 static inline bool 18 __vma_matches(struct vm_area_struct *vma, struct file *filp, 19 unsigned long addr, unsigned long size) 20 { 21 if (vma->vm_file != filp) 22 return false; 23 24 return vma->vm_start == addr && 25 (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); 26 } 27 28 /** 29 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address 30 * it is mapped to. 31 * @dev: drm device 32 * @data: ioctl data blob 33 * @file: drm file 34 * 35 * While the mapping holds a reference on the contents of the object, it doesn't 36 * imply a ref on the object itself. 37 * 38 * IMPORTANT: 39 * 40 * DRM driver writers who look a this function as an example for how to do GEM 41 * mmap support, please don't implement mmap support like here. The modern way 42 * to implement DRM mmap support is with an mmap offset ioctl (like 43 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. 44 * That way debug tooling like valgrind will understand what's going on, hiding 45 * the mmap call in a driver private ioctl will break that. The i915 driver only 46 * does cpu mmaps this way because we didn't know better. 47 */ 48 int 49 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 50 struct drm_file *file) 51 { 52 struct drm_i915_gem_mmap *args = data; 53 struct drm_i915_gem_object *obj; 54 unsigned long addr; 55 56 if (args->flags & ~(I915_MMAP_WC)) 57 return -EINVAL; 58 59 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) 60 return -ENODEV; 61 62 obj = i915_gem_object_lookup(file, args->handle); 63 if (!obj) 64 return -ENOENT; 65 66 /* prime objects have no backing filp to GEM mmap 67 * pages from. 68 */ 69 if (!obj->base.filp) { 70 addr = -ENXIO; 71 goto err; 72 } 73 74 if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { 75 addr = -EINVAL; 76 goto err; 77 } 78 79 addr = vm_mmap(obj->base.filp, 0, args->size, 80 PROT_READ | PROT_WRITE, MAP_SHARED, 81 args->offset); 82 if (IS_ERR_VALUE(addr)) 83 goto err; 84 85 if (args->flags & I915_MMAP_WC) { 86 struct mm_struct *mm = current->mm; 87 struct vm_area_struct *vma; 88 89 if (down_write_killable(&mm->mmap_sem)) { 90 addr = -EINTR; 91 goto err; 92 } 93 vma = find_vma(mm, addr); 94 if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) 95 vma->vm_page_prot = 96 pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 97 else 98 addr = -ENOMEM; 99 up_write(&mm->mmap_sem); 100 if (IS_ERR_VALUE(addr)) 101 goto err; 102 103 /* This may race, but that's ok, it only gets set */ 104 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU); 105 } 106 i915_gem_object_put(obj); 107 108 args->addr_ptr = (u64)addr; 109 return 0; 110 111 err: 112 i915_gem_object_put(obj); 113 return addr; 114 } 115 116 static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) 117 { 118 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; 119 } 120 121 /** 122 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps 123 * 124 * A history of the GTT mmap interface: 125 * 126 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to 127 * aligned and suitable for fencing, and still fit into the available 128 * mappable space left by the pinned display objects. A classic problem 129 * we called the page-fault-of-doom where we would ping-pong between 130 * two objects that could not fit inside the GTT and so the memcpy 131 * would page one object in at the expense of the other between every 132 * single byte. 133 * 134 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none 135 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the 136 * object is too large for the available space (or simply too large 137 * for the mappable aperture!), a view is created instead and faulted 138 * into userspace. (This view is aligned and sized appropriately for 139 * fenced access.) 140 * 141 * 2 - Recognise WC as a separate cache domain so that we can flush the 142 * delayed writes via GTT before performing direct access via WC. 143 * 144 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial 145 * pagefault; swapin remains transparent. 146 * 147 * Restrictions: 148 * 149 * * snoopable objects cannot be accessed via the GTT. It can cause machine 150 * hangs on some architectures, corruption on others. An attempt to service 151 * a GTT page fault from a snoopable object will generate a SIGBUS. 152 * 153 * * the object must be able to fit into RAM (physical memory, though no 154 * limited to the mappable aperture). 155 * 156 * 157 * Caveats: 158 * 159 * * a new GTT page fault will synchronize rendering from the GPU and flush 160 * all data to system memory. Subsequent access will not be synchronized. 161 * 162 * * all mappings are revoked on runtime device suspend. 163 * 164 * * there are only 8, 16 or 32 fence registers to share between all users 165 * (older machines require fence register for display and blitter access 166 * as well). Contention of the fence registers will cause the previous users 167 * to be unmapped and any new access will generate new page faults. 168 * 169 * * running out of memory while servicing a fault may generate a SIGBUS, 170 * rather than the expected SIGSEGV. 171 */ 172 int i915_gem_mmap_gtt_version(void) 173 { 174 return 3; 175 } 176 177 static inline struct i915_ggtt_view 178 compute_partial_view(const struct drm_i915_gem_object *obj, 179 pgoff_t page_offset, 180 unsigned int chunk) 181 { 182 struct i915_ggtt_view view; 183 184 if (i915_gem_object_is_tiled(obj)) 185 chunk = roundup(chunk, tile_row_pages(obj)); 186 187 view.type = I915_GGTT_VIEW_PARTIAL; 188 view.partial.offset = rounddown(page_offset, chunk); 189 view.partial.size = 190 min_t(unsigned int, chunk, 191 (obj->base.size >> PAGE_SHIFT) - view.partial.offset); 192 193 /* If the partial covers the entire object, just create a normal VMA. */ 194 if (chunk >= obj->base.size >> PAGE_SHIFT) 195 view.type = I915_GGTT_VIEW_NORMAL; 196 197 return view; 198 } 199 200 /** 201 * i915_gem_fault - fault a page into the GTT 202 * @vmf: fault info 203 * 204 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 205 * from userspace. The fault handler takes care of binding the object to 206 * the GTT (if needed), allocating and programming a fence register (again, 207 * only if needed based on whether the old reg is still valid or the object 208 * is tiled) and inserting a new PTE into the faulting process. 209 * 210 * Note that the faulting process may involve evicting existing objects 211 * from the GTT and/or fence registers to make room. So performance may 212 * suffer if the GTT working set is large or there are few fence registers 213 * left. 214 * 215 * The current feature set supported by i915_gem_fault() and thus GTT mmaps 216 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version). 217 */ 218 vm_fault_t i915_gem_fault(struct vm_fault *vmf) 219 { 220 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) 221 struct vm_area_struct *area = vmf->vma; 222 struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data); 223 struct drm_device *dev = obj->base.dev; 224 struct drm_i915_private *i915 = to_i915(dev); 225 struct intel_runtime_pm *rpm = &i915->runtime_pm; 226 struct i915_ggtt *ggtt = &i915->ggtt; 227 bool write = area->vm_flags & VM_WRITE; 228 intel_wakeref_t wakeref; 229 struct i915_vma *vma; 230 pgoff_t page_offset; 231 int srcu; 232 int ret; 233 234 /* Sanity check that we allow writing into this object */ 235 if (i915_gem_object_is_readonly(obj) && write) 236 return VM_FAULT_SIGBUS; 237 238 /* We don't use vmf->pgoff since that has the fake offset */ 239 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; 240 241 trace_i915_gem_object_fault(obj, page_offset, true, write); 242 243 ret = i915_gem_object_pin_pages(obj); 244 if (ret) 245 goto err; 246 247 wakeref = intel_runtime_pm_get(rpm); 248 249 srcu = i915_reset_trylock(i915); 250 if (srcu < 0) { 251 ret = srcu; 252 goto err_rpm; 253 } 254 255 ret = i915_mutex_lock_interruptible(dev); 256 if (ret) 257 goto err_reset; 258 259 /* Access to snoopable pages through the GTT is incoherent. */ 260 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) { 261 ret = -EFAULT; 262 goto err_unlock; 263 } 264 265 /* Now pin it into the GTT as needed */ 266 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 267 PIN_MAPPABLE | 268 PIN_NONBLOCK | 269 PIN_NONFAULT); 270 if (IS_ERR(vma)) { 271 /* Use a partial view if it is bigger than available space */ 272 struct i915_ggtt_view view = 273 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); 274 unsigned int flags; 275 276 flags = PIN_MAPPABLE; 277 if (view.type == I915_GGTT_VIEW_NORMAL) 278 flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ 279 280 /* 281 * Userspace is now writing through an untracked VMA, abandon 282 * all hope that the hardware is able to track future writes. 283 */ 284 obj->frontbuffer_ggtt_origin = ORIGIN_CPU; 285 286 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); 287 if (IS_ERR(vma) && !view.type) { 288 flags = PIN_MAPPABLE; 289 view.type = I915_GGTT_VIEW_PARTIAL; 290 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); 291 } 292 } 293 if (IS_ERR(vma)) { 294 ret = PTR_ERR(vma); 295 goto err_unlock; 296 } 297 298 ret = i915_vma_pin_fence(vma); 299 if (ret) 300 goto err_unpin; 301 302 /* Finally, remap it using the new GTT offset */ 303 ret = remap_io_mapping(area, 304 area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT), 305 (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT, 306 min_t(u64, vma->size, area->vm_end - area->vm_start), 307 &ggtt->iomap); 308 if (ret) 309 goto err_fence; 310 311 /* Mark as being mmapped into userspace for later revocation */ 312 assert_rpm_wakelock_held(rpm); 313 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) 314 list_add(&obj->userfault_link, &i915->ggtt.userfault_list); 315 if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) 316 intel_wakeref_auto(&i915->ggtt.userfault_wakeref, 317 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); 318 GEM_BUG_ON(!obj->userfault_count); 319 320 i915_vma_set_ggtt_write(vma); 321 322 err_fence: 323 i915_vma_unpin_fence(vma); 324 err_unpin: 325 __i915_vma_unpin(vma); 326 err_unlock: 327 mutex_unlock(&dev->struct_mutex); 328 err_reset: 329 i915_reset_unlock(i915, srcu); 330 err_rpm: 331 intel_runtime_pm_put(rpm, wakeref); 332 i915_gem_object_unpin_pages(obj); 333 err: 334 switch (ret) { 335 case -EIO: 336 /* 337 * We eat errors when the gpu is terminally wedged to avoid 338 * userspace unduly crashing (gl has no provisions for mmaps to 339 * fail). But any other -EIO isn't ours (e.g. swap in failure) 340 * and so needs to be reported. 341 */ 342 if (!i915_terminally_wedged(i915)) 343 return VM_FAULT_SIGBUS; 344 /* else: fall through */ 345 case -EAGAIN: 346 /* 347 * EAGAIN means the gpu is hung and we'll wait for the error 348 * handler to reset everything when re-faulting in 349 * i915_mutex_lock_interruptible. 350 */ 351 case 0: 352 case -ERESTARTSYS: 353 case -EINTR: 354 case -EBUSY: 355 /* 356 * EBUSY is ok: this just means that another thread 357 * already did the job. 358 */ 359 return VM_FAULT_NOPAGE; 360 case -ENOMEM: 361 return VM_FAULT_OOM; 362 case -ENOSPC: 363 case -EFAULT: 364 return VM_FAULT_SIGBUS; 365 default: 366 WARN_ONCE(ret, "unhandled error in %s: %i\n", __func__, ret); 367 return VM_FAULT_SIGBUS; 368 } 369 } 370 371 void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) 372 { 373 struct i915_vma *vma; 374 375 GEM_BUG_ON(!obj->userfault_count); 376 377 obj->userfault_count = 0; 378 list_del(&obj->userfault_link); 379 drm_vma_node_unmap(&obj->base.vma_node, 380 obj->base.dev->anon_inode->i_mapping); 381 382 for_each_ggtt_vma(vma, obj) 383 i915_vma_unset_userfault(vma); 384 } 385 386 /** 387 * i915_gem_object_release_mmap - remove physical page mappings 388 * @obj: obj in question 389 * 390 * Preserve the reservation of the mmapping with the DRM core code, but 391 * relinquish ownership of the pages back to the system. 392 * 393 * It is vital that we remove the page mapping if we have mapped a tiled 394 * object through the GTT and then lose the fence register due to 395 * resource pressure. Similarly if the object has been moved out of the 396 * aperture, than pages mapped into userspace must be revoked. Removing the 397 * mapping will then trigger a page fault on the next user access, allowing 398 * fixup by i915_gem_fault(). 399 */ 400 void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) 401 { 402 struct drm_i915_private *i915 = to_i915(obj->base.dev); 403 intel_wakeref_t wakeref; 404 405 /* Serialisation between user GTT access and our code depends upon 406 * revoking the CPU's PTE whilst the mutex is held. The next user 407 * pagefault then has to wait until we release the mutex. 408 * 409 * Note that RPM complicates somewhat by adding an additional 410 * requirement that operations to the GGTT be made holding the RPM 411 * wakeref. 412 */ 413 lockdep_assert_held(&i915->drm.struct_mutex); 414 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 415 416 if (!obj->userfault_count) 417 goto out; 418 419 __i915_gem_object_release_mmap(obj); 420 421 /* Ensure that the CPU's PTE are revoked and there are not outstanding 422 * memory transactions from userspace before we return. The TLB 423 * flushing implied above by changing the PTE above *should* be 424 * sufficient, an extra barrier here just provides us with a bit 425 * of paranoid documentation about our requirement to serialise 426 * memory writes before touching registers / GSM. 427 */ 428 wmb(); 429 430 out: 431 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 432 } 433 434 static int create_mmap_offset(struct drm_i915_gem_object *obj) 435 { 436 struct drm_i915_private *i915 = to_i915(obj->base.dev); 437 int err; 438 439 err = drm_gem_create_mmap_offset(&obj->base); 440 if (likely(!err)) 441 return 0; 442 443 /* Attempt to reap some mmap space from dead objects */ 444 do { 445 err = i915_gem_wait_for_idle(i915, 446 I915_WAIT_INTERRUPTIBLE, 447 MAX_SCHEDULE_TIMEOUT); 448 if (err) 449 break; 450 451 i915_gem_drain_freed_objects(i915); 452 err = drm_gem_create_mmap_offset(&obj->base); 453 if (!err) 454 break; 455 456 } while (flush_delayed_work(&i915->gem.retire_work)); 457 458 return err; 459 } 460 461 int 462 i915_gem_mmap_gtt(struct drm_file *file, 463 struct drm_device *dev, 464 u32 handle, 465 u64 *offset) 466 { 467 struct drm_i915_gem_object *obj; 468 int ret; 469 470 obj = i915_gem_object_lookup(file, handle); 471 if (!obj) 472 return -ENOENT; 473 474 ret = create_mmap_offset(obj); 475 if (ret == 0) 476 *offset = drm_vma_node_offset_addr(&obj->base.vma_node); 477 478 i915_gem_object_put(obj); 479 return ret; 480 } 481 482 /** 483 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 484 * @dev: DRM device 485 * @data: GTT mapping ioctl data 486 * @file: GEM object info 487 * 488 * Simply returns the fake offset to userspace so it can mmap it. 489 * The mmap call will end up in drm_gem_mmap(), which will set things 490 * up so we can get faults in the handler above. 491 * 492 * The fault handler will take care of binding the object into the GTT 493 * (since it may have been evicted to make room for something), allocating 494 * a fence register, and mapping the appropriate aperture address into 495 * userspace. 496 */ 497 int 498 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 499 struct drm_file *file) 500 { 501 struct drm_i915_gem_mmap_gtt *args = data; 502 503 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); 504 } 505 506 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 507 #include "selftests/i915_gem_mman.c" 508 #endif 509