1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2012-2014 Intel Corporation 5 */ 6 7 #include <linux/mmu_context.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/mempolicy.h> 10 #include <linux/swap.h> 11 #include <linux/sched/mm.h> 12 13 #include <drm/i915_drm.h> 14 15 #include "i915_drv.h" 16 #include "i915_gem_ioctls.h" 17 #include "i915_gem_object.h" 18 #include "i915_scatterlist.h" 19 20 struct i915_mm_struct { 21 struct mm_struct *mm; 22 struct drm_i915_private *i915; 23 struct i915_mmu_notifier *mn; 24 struct hlist_node node; 25 struct kref kref; 26 struct work_struct work; 27 }; 28 29 #if defined(CONFIG_MMU_NOTIFIER) 30 #include <linux/interval_tree.h> 31 32 struct i915_mmu_notifier { 33 spinlock_t lock; 34 struct hlist_node node; 35 struct mmu_notifier mn; 36 struct rb_root_cached objects; 37 struct i915_mm_struct *mm; 38 }; 39 40 struct i915_mmu_object { 41 struct i915_mmu_notifier *mn; 42 struct drm_i915_gem_object *obj; 43 struct interval_tree_node it; 44 }; 45 46 static void add_object(struct i915_mmu_object *mo) 47 { 48 GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb)); 49 interval_tree_insert(&mo->it, &mo->mn->objects); 50 } 51 52 static void del_object(struct i915_mmu_object *mo) 53 { 54 if (RB_EMPTY_NODE(&mo->it.rb)) 55 return; 56 57 interval_tree_remove(&mo->it, &mo->mn->objects); 58 RB_CLEAR_NODE(&mo->it.rb); 59 } 60 61 static void 62 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value) 63 { 64 struct i915_mmu_object *mo = obj->userptr.mmu_object; 65 66 /* 67 * During mm_invalidate_range we need to cancel any userptr that 68 * overlaps the range being invalidated. Doing so requires the 69 * struct_mutex, and that risks recursion. In order to cause 70 * recursion, the user must alias the userptr address space with 71 * a GTT mmapping (possible with a MAP_FIXED) - then when we have 72 * to invalidate that mmaping, mm_invalidate_range is called with 73 * the userptr address *and* the struct_mutex held. To prevent that 74 * we set a flag under the i915_mmu_notifier spinlock to indicate 75 * whether this object is valid. 76 */ 77 if (!mo) 78 return; 79 80 spin_lock(&mo->mn->lock); 81 if (value) 82 add_object(mo); 83 else 84 del_object(mo); 85 spin_unlock(&mo->mn->lock); 86 } 87 88 static int 89 userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, 90 const struct mmu_notifier_range *range) 91 { 92 struct i915_mmu_notifier *mn = 93 container_of(_mn, struct i915_mmu_notifier, mn); 94 struct interval_tree_node *it; 95 struct mutex *unlock = NULL; 96 unsigned long end; 97 int ret = 0; 98 99 if (RB_EMPTY_ROOT(&mn->objects.rb_root)) 100 return 0; 101 102 /* interval ranges are inclusive, but invalidate range is exclusive */ 103 end = range->end - 1; 104 105 spin_lock(&mn->lock); 106 it = interval_tree_iter_first(&mn->objects, range->start, end); 107 while (it) { 108 struct drm_i915_gem_object *obj; 109 110 if (!mmu_notifier_range_blockable(range)) { 111 ret = -EAGAIN; 112 break; 113 } 114 115 /* 116 * The mmu_object is released late when destroying the 117 * GEM object so it is entirely possible to gain a 118 * reference on an object in the process of being freed 119 * since our serialisation is via the spinlock and not 120 * the struct_mutex - and consequently use it after it 121 * is freed and then double free it. To prevent that 122 * use-after-free we only acquire a reference on the 123 * object if it is not in the process of being destroyed. 124 */ 125 obj = container_of(it, struct i915_mmu_object, it)->obj; 126 if (!kref_get_unless_zero(&obj->base.refcount)) { 127 it = interval_tree_iter_next(it, range->start, end); 128 continue; 129 } 130 spin_unlock(&mn->lock); 131 132 if (!unlock) { 133 unlock = &mn->mm->i915->drm.struct_mutex; 134 135 switch (mutex_trylock_recursive(unlock)) { 136 default: 137 case MUTEX_TRYLOCK_FAILED: 138 if (mutex_lock_killable_nested(unlock, I915_MM_SHRINKER)) { 139 i915_gem_object_put(obj); 140 return -EINTR; 141 } 142 /* fall through */ 143 case MUTEX_TRYLOCK_SUCCESS: 144 break; 145 146 case MUTEX_TRYLOCK_RECURSIVE: 147 unlock = ERR_PTR(-EEXIST); 148 break; 149 } 150 } 151 152 ret = i915_gem_object_unbind(obj, 153 I915_GEM_OBJECT_UNBIND_ACTIVE); 154 if (ret == 0) 155 ret = __i915_gem_object_put_pages(obj, I915_MM_SHRINKER); 156 i915_gem_object_put(obj); 157 if (ret) 158 goto unlock; 159 160 spin_lock(&mn->lock); 161 162 /* 163 * As we do not (yet) protect the mmu from concurrent insertion 164 * over this range, there is no guarantee that this search will 165 * terminate given a pathologic workload. 166 */ 167 it = interval_tree_iter_first(&mn->objects, range->start, end); 168 } 169 spin_unlock(&mn->lock); 170 171 unlock: 172 if (!IS_ERR_OR_NULL(unlock)) 173 mutex_unlock(unlock); 174 175 return ret; 176 177 } 178 179 static const struct mmu_notifier_ops i915_gem_userptr_notifier = { 180 .invalidate_range_start = userptr_mn_invalidate_range_start, 181 }; 182 183 static struct i915_mmu_notifier * 184 i915_mmu_notifier_create(struct i915_mm_struct *mm) 185 { 186 struct i915_mmu_notifier *mn; 187 188 mn = kmalloc(sizeof(*mn), GFP_KERNEL); 189 if (mn == NULL) 190 return ERR_PTR(-ENOMEM); 191 192 spin_lock_init(&mn->lock); 193 mn->mn.ops = &i915_gem_userptr_notifier; 194 mn->objects = RB_ROOT_CACHED; 195 mn->mm = mm; 196 197 return mn; 198 } 199 200 static void 201 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 202 { 203 struct i915_mmu_object *mo; 204 205 mo = fetch_and_zero(&obj->userptr.mmu_object); 206 if (!mo) 207 return; 208 209 spin_lock(&mo->mn->lock); 210 del_object(mo); 211 spin_unlock(&mo->mn->lock); 212 kfree(mo); 213 } 214 215 static struct i915_mmu_notifier * 216 i915_mmu_notifier_find(struct i915_mm_struct *mm) 217 { 218 struct i915_mmu_notifier *mn; 219 int err = 0; 220 221 mn = mm->mn; 222 if (mn) 223 return mn; 224 225 mn = i915_mmu_notifier_create(mm); 226 if (IS_ERR(mn)) 227 err = PTR_ERR(mn); 228 229 down_write(&mm->mm->mmap_sem); 230 mutex_lock(&mm->i915->mm_lock); 231 if (mm->mn == NULL && !err) { 232 /* Protected by mmap_sem (write-lock) */ 233 err = __mmu_notifier_register(&mn->mn, mm->mm); 234 if (!err) { 235 /* Protected by mm_lock */ 236 mm->mn = fetch_and_zero(&mn); 237 } 238 } else if (mm->mn) { 239 /* 240 * Someone else raced and successfully installed the mmu 241 * notifier, we can cancel our own errors. 242 */ 243 err = 0; 244 } 245 mutex_unlock(&mm->i915->mm_lock); 246 up_write(&mm->mm->mmap_sem); 247 248 if (mn && !IS_ERR(mn)) 249 kfree(mn); 250 251 return err ? ERR_PTR(err) : mm->mn; 252 } 253 254 static int 255 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 256 unsigned flags) 257 { 258 struct i915_mmu_notifier *mn; 259 struct i915_mmu_object *mo; 260 261 if (flags & I915_USERPTR_UNSYNCHRONIZED) 262 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; 263 264 if (WARN_ON(obj->userptr.mm == NULL)) 265 return -EINVAL; 266 267 mn = i915_mmu_notifier_find(obj->userptr.mm); 268 if (IS_ERR(mn)) 269 return PTR_ERR(mn); 270 271 mo = kzalloc(sizeof(*mo), GFP_KERNEL); 272 if (!mo) 273 return -ENOMEM; 274 275 mo->mn = mn; 276 mo->obj = obj; 277 mo->it.start = obj->userptr.ptr; 278 mo->it.last = obj->userptr.ptr + obj->base.size - 1; 279 RB_CLEAR_NODE(&mo->it.rb); 280 281 obj->userptr.mmu_object = mo; 282 return 0; 283 } 284 285 static void 286 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 287 struct mm_struct *mm) 288 { 289 if (mn == NULL) 290 return; 291 292 mmu_notifier_unregister(&mn->mn, mm); 293 kfree(mn); 294 } 295 296 #else 297 298 static void 299 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value) 300 { 301 } 302 303 static void 304 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) 305 { 306 } 307 308 static int 309 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, 310 unsigned flags) 311 { 312 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) 313 return -ENODEV; 314 315 if (!capable(CAP_SYS_ADMIN)) 316 return -EPERM; 317 318 return 0; 319 } 320 321 static void 322 i915_mmu_notifier_free(struct i915_mmu_notifier *mn, 323 struct mm_struct *mm) 324 { 325 } 326 327 #endif 328 329 static struct i915_mm_struct * 330 __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) 331 { 332 struct i915_mm_struct *mm; 333 334 /* Protected by dev_priv->mm_lock */ 335 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) 336 if (mm->mm == real) 337 return mm; 338 339 return NULL; 340 } 341 342 static int 343 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) 344 { 345 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 346 struct i915_mm_struct *mm; 347 int ret = 0; 348 349 /* During release of the GEM object we hold the struct_mutex. This 350 * precludes us from calling mmput() at that time as that may be 351 * the last reference and so call exit_mmap(). exit_mmap() will 352 * attempt to reap the vma, and if we were holding a GTT mmap 353 * would then call drm_gem_vm_close() and attempt to reacquire 354 * the struct mutex. So in order to avoid that recursion, we have 355 * to defer releasing the mm reference until after we drop the 356 * struct_mutex, i.e. we need to schedule a worker to do the clean 357 * up. 358 */ 359 mutex_lock(&dev_priv->mm_lock); 360 mm = __i915_mm_struct_find(dev_priv, current->mm); 361 if (mm == NULL) { 362 mm = kmalloc(sizeof(*mm), GFP_KERNEL); 363 if (mm == NULL) { 364 ret = -ENOMEM; 365 goto out; 366 } 367 368 kref_init(&mm->kref); 369 mm->i915 = to_i915(obj->base.dev); 370 371 mm->mm = current->mm; 372 mmgrab(current->mm); 373 374 mm->mn = NULL; 375 376 /* Protected by dev_priv->mm_lock */ 377 hash_add(dev_priv->mm_structs, 378 &mm->node, (unsigned long)mm->mm); 379 } else 380 kref_get(&mm->kref); 381 382 obj->userptr.mm = mm; 383 out: 384 mutex_unlock(&dev_priv->mm_lock); 385 return ret; 386 } 387 388 static void 389 __i915_mm_struct_free__worker(struct work_struct *work) 390 { 391 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); 392 i915_mmu_notifier_free(mm->mn, mm->mm); 393 mmdrop(mm->mm); 394 kfree(mm); 395 } 396 397 static void 398 __i915_mm_struct_free(struct kref *kref) 399 { 400 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); 401 402 /* Protected by dev_priv->mm_lock */ 403 hash_del(&mm->node); 404 mutex_unlock(&mm->i915->mm_lock); 405 406 INIT_WORK(&mm->work, __i915_mm_struct_free__worker); 407 queue_work(mm->i915->mm.userptr_wq, &mm->work); 408 } 409 410 static void 411 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) 412 { 413 if (obj->userptr.mm == NULL) 414 return; 415 416 kref_put_mutex(&obj->userptr.mm->kref, 417 __i915_mm_struct_free, 418 &to_i915(obj->base.dev)->mm_lock); 419 obj->userptr.mm = NULL; 420 } 421 422 struct get_pages_work { 423 struct work_struct work; 424 struct drm_i915_gem_object *obj; 425 struct task_struct *task; 426 }; 427 428 static struct sg_table * 429 __i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj, 430 struct page **pvec, int num_pages) 431 { 432 unsigned int max_segment = i915_sg_segment_size(); 433 struct sg_table *st; 434 unsigned int sg_page_sizes; 435 int ret; 436 437 st = kmalloc(sizeof(*st), GFP_KERNEL); 438 if (!st) 439 return ERR_PTR(-ENOMEM); 440 441 alloc_table: 442 ret = __sg_alloc_table_from_pages(st, pvec, num_pages, 443 0, num_pages << PAGE_SHIFT, 444 max_segment, 445 GFP_KERNEL); 446 if (ret) { 447 kfree(st); 448 return ERR_PTR(ret); 449 } 450 451 ret = i915_gem_gtt_prepare_pages(obj, st); 452 if (ret) { 453 sg_free_table(st); 454 455 if (max_segment > PAGE_SIZE) { 456 max_segment = PAGE_SIZE; 457 goto alloc_table; 458 } 459 460 kfree(st); 461 return ERR_PTR(ret); 462 } 463 464 sg_page_sizes = i915_sg_page_sizes(st->sgl); 465 466 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 467 468 return st; 469 } 470 471 static void 472 __i915_gem_userptr_get_pages_worker(struct work_struct *_work) 473 { 474 struct get_pages_work *work = container_of(_work, typeof(*work), work); 475 struct drm_i915_gem_object *obj = work->obj; 476 const int npages = obj->base.size >> PAGE_SHIFT; 477 struct page **pvec; 478 int pinned, ret; 479 480 ret = -ENOMEM; 481 pinned = 0; 482 483 pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 484 if (pvec != NULL) { 485 struct mm_struct *mm = obj->userptr.mm->mm; 486 unsigned int flags = 0; 487 488 if (!i915_gem_object_is_readonly(obj)) 489 flags |= FOLL_WRITE; 490 491 ret = -EFAULT; 492 if (mmget_not_zero(mm)) { 493 down_read(&mm->mmap_sem); 494 while (pinned < npages) { 495 ret = get_user_pages_remote 496 (work->task, mm, 497 obj->userptr.ptr + pinned * PAGE_SIZE, 498 npages - pinned, 499 flags, 500 pvec + pinned, NULL, NULL); 501 if (ret < 0) 502 break; 503 504 pinned += ret; 505 } 506 up_read(&mm->mmap_sem); 507 mmput(mm); 508 } 509 } 510 511 mutex_lock(&obj->mm.lock); 512 if (obj->userptr.work == &work->work) { 513 struct sg_table *pages = ERR_PTR(ret); 514 515 if (pinned == npages) { 516 pages = __i915_gem_userptr_alloc_pages(obj, pvec, 517 npages); 518 if (!IS_ERR(pages)) { 519 pinned = 0; 520 pages = NULL; 521 } 522 } 523 524 obj->userptr.work = ERR_CAST(pages); 525 if (IS_ERR(pages)) 526 __i915_gem_userptr_set_active(obj, false); 527 } 528 mutex_unlock(&obj->mm.lock); 529 530 release_pages(pvec, pinned); 531 kvfree(pvec); 532 533 i915_gem_object_put(obj); 534 put_task_struct(work->task); 535 kfree(work); 536 } 537 538 static struct sg_table * 539 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj) 540 { 541 struct get_pages_work *work; 542 543 /* Spawn a worker so that we can acquire the 544 * user pages without holding our mutex. Access 545 * to the user pages requires mmap_sem, and we have 546 * a strict lock ordering of mmap_sem, struct_mutex - 547 * we already hold struct_mutex here and so cannot 548 * call gup without encountering a lock inversion. 549 * 550 * Userspace will keep on repeating the operation 551 * (thanks to EAGAIN) until either we hit the fast 552 * path or the worker completes. If the worker is 553 * cancelled or superseded, the task is still run 554 * but the results ignored. (This leads to 555 * complications that we may have a stray object 556 * refcount that we need to be wary of when 557 * checking for existing objects during creation.) 558 * If the worker encounters an error, it reports 559 * that error back to this function through 560 * obj->userptr.work = ERR_PTR. 561 */ 562 work = kmalloc(sizeof(*work), GFP_KERNEL); 563 if (work == NULL) 564 return ERR_PTR(-ENOMEM); 565 566 obj->userptr.work = &work->work; 567 568 work->obj = i915_gem_object_get(obj); 569 570 work->task = current; 571 get_task_struct(work->task); 572 573 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); 574 queue_work(to_i915(obj->base.dev)->mm.userptr_wq, &work->work); 575 576 return ERR_PTR(-EAGAIN); 577 } 578 579 static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) 580 { 581 const int num_pages = obj->base.size >> PAGE_SHIFT; 582 struct mm_struct *mm = obj->userptr.mm->mm; 583 struct page **pvec; 584 struct sg_table *pages; 585 bool active; 586 int pinned; 587 588 /* If userspace should engineer that these pages are replaced in 589 * the vma between us binding this page into the GTT and completion 590 * of rendering... Their loss. If they change the mapping of their 591 * pages they need to create a new bo to point to the new vma. 592 * 593 * However, that still leaves open the possibility of the vma 594 * being copied upon fork. Which falls under the same userspace 595 * synchronisation issue as a regular bo, except that this time 596 * the process may not be expecting that a particular piece of 597 * memory is tied to the GPU. 598 * 599 * Fortunately, we can hook into the mmu_notifier in order to 600 * discard the page references prior to anything nasty happening 601 * to the vma (discard or cloning) which should prevent the more 602 * egregious cases from causing harm. 603 */ 604 605 if (obj->userptr.work) { 606 /* active flag should still be held for the pending work */ 607 if (IS_ERR(obj->userptr.work)) 608 return PTR_ERR(obj->userptr.work); 609 else 610 return -EAGAIN; 611 } 612 613 pvec = NULL; 614 pinned = 0; 615 616 if (mm == current->mm) { 617 pvec = kvmalloc_array(num_pages, sizeof(struct page *), 618 GFP_KERNEL | 619 __GFP_NORETRY | 620 __GFP_NOWARN); 621 if (pvec) /* defer to worker if malloc fails */ 622 pinned = __get_user_pages_fast(obj->userptr.ptr, 623 num_pages, 624 !i915_gem_object_is_readonly(obj), 625 pvec); 626 } 627 628 active = false; 629 if (pinned < 0) { 630 pages = ERR_PTR(pinned); 631 pinned = 0; 632 } else if (pinned < num_pages) { 633 pages = __i915_gem_userptr_get_pages_schedule(obj); 634 active = pages == ERR_PTR(-EAGAIN); 635 } else { 636 pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages); 637 active = !IS_ERR(pages); 638 } 639 if (active) 640 __i915_gem_userptr_set_active(obj, true); 641 642 if (IS_ERR(pages)) 643 release_pages(pvec, pinned); 644 kvfree(pvec); 645 646 return PTR_ERR_OR_ZERO(pages); 647 } 648 649 static void 650 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, 651 struct sg_table *pages) 652 { 653 struct sgt_iter sgt_iter; 654 struct page *page; 655 656 /* Cancel any inflight work and force them to restart their gup */ 657 obj->userptr.work = NULL; 658 __i915_gem_userptr_set_active(obj, false); 659 if (!pages) 660 return; 661 662 __i915_gem_object_release_shmem(obj, pages, true); 663 i915_gem_gtt_finish_pages(obj, pages); 664 665 /* 666 * We always mark objects as dirty when they are used by the GPU, 667 * just in case. However, if we set the vma as being read-only we know 668 * that the object will never have been written to. 669 */ 670 if (i915_gem_object_is_readonly(obj)) 671 obj->mm.dirty = false; 672 673 for_each_sgt_page(page, sgt_iter, pages) { 674 if (obj->mm.dirty) 675 /* 676 * As this may not be anonymous memory (e.g. shmem) 677 * but exist on a real mapping, we have to lock 678 * the page in order to dirty it -- holding 679 * the page reference is not sufficient to 680 * prevent the inode from being truncated. 681 * Play safe and take the lock. 682 */ 683 set_page_dirty_lock(page); 684 685 mark_page_accessed(page); 686 put_page(page); 687 } 688 obj->mm.dirty = false; 689 690 sg_free_table(pages); 691 kfree(pages); 692 } 693 694 static void 695 i915_gem_userptr_release(struct drm_i915_gem_object *obj) 696 { 697 i915_gem_userptr_release__mmu_notifier(obj); 698 i915_gem_userptr_release__mm_struct(obj); 699 } 700 701 static int 702 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) 703 { 704 if (obj->userptr.mmu_object) 705 return 0; 706 707 return i915_gem_userptr_init__mmu_notifier(obj, 0); 708 } 709 710 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { 711 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 712 I915_GEM_OBJECT_IS_SHRINKABLE | 713 I915_GEM_OBJECT_ASYNC_CANCEL, 714 .get_pages = i915_gem_userptr_get_pages, 715 .put_pages = i915_gem_userptr_put_pages, 716 .dmabuf_export = i915_gem_userptr_dmabuf_export, 717 .release = i915_gem_userptr_release, 718 }; 719 720 /* 721 * Creates a new mm object that wraps some normal memory from the process 722 * context - user memory. 723 * 724 * We impose several restrictions upon the memory being mapped 725 * into the GPU. 726 * 1. It must be page aligned (both start/end addresses, i.e ptr and size). 727 * 2. It must be normal system memory, not a pointer into another map of IO 728 * space (e.g. it must not be a GTT mmapping of another object). 729 * 3. We only allow a bo as large as we could in theory map into the GTT, 730 * that is we limit the size to the total size of the GTT. 731 * 4. The bo is marked as being snoopable. The backing pages are left 732 * accessible directly by the CPU, but reads and writes by the GPU may 733 * incur the cost of a snoop (unless you have an LLC architecture). 734 * 735 * Synchronisation between multiple users and the GPU is left to userspace 736 * through the normal set-domain-ioctl. The kernel will enforce that the 737 * GPU relinquishes the VMA before it is returned back to the system 738 * i.e. upon free(), munmap() or process termination. However, the userspace 739 * malloc() library may not immediately relinquish the VMA after free() and 740 * instead reuse it whilst the GPU is still reading and writing to the VMA. 741 * Caveat emptor. 742 * 743 * Also note, that the object created here is not currently a "first class" 744 * object, in that several ioctls are banned. These are the CPU access 745 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use 746 * direct access via your pointer rather than use those ioctls. Another 747 * restriction is that we do not allow userptr surfaces to be pinned to the 748 * hardware and so we reject any attempt to create a framebuffer out of a 749 * userptr. 750 * 751 * If you think this is a good interface to use to pass GPU memory between 752 * drivers, please use dma-buf instead. In fact, wherever possible use 753 * dma-buf instead. 754 */ 755 int 756 i915_gem_userptr_ioctl(struct drm_device *dev, 757 void *data, 758 struct drm_file *file) 759 { 760 struct drm_i915_private *dev_priv = to_i915(dev); 761 struct drm_i915_gem_userptr *args = data; 762 struct drm_i915_gem_object *obj; 763 int ret; 764 u32 handle; 765 766 if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) { 767 /* We cannot support coherent userptr objects on hw without 768 * LLC and broken snooping. 769 */ 770 return -ENODEV; 771 } 772 773 if (args->flags & ~(I915_USERPTR_READ_ONLY | 774 I915_USERPTR_UNSYNCHRONIZED)) 775 return -EINVAL; 776 777 if (!args->user_size) 778 return -EINVAL; 779 780 if (offset_in_page(args->user_ptr | args->user_size)) 781 return -EINVAL; 782 783 if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size)) 784 return -EFAULT; 785 786 if (args->flags & I915_USERPTR_READ_ONLY) { 787 struct i915_address_space *vm; 788 789 /* 790 * On almost all of the older hw, we cannot tell the GPU that 791 * a page is readonly. 792 */ 793 vm = dev_priv->kernel_context->vm; 794 if (!vm || !vm->has_read_only) 795 return -ENODEV; 796 } 797 798 obj = i915_gem_object_alloc(); 799 if (obj == NULL) 800 return -ENOMEM; 801 802 drm_gem_private_object_init(dev, &obj->base, args->user_size); 803 i915_gem_object_init(obj, &i915_gem_userptr_ops); 804 obj->read_domains = I915_GEM_DOMAIN_CPU; 805 obj->write_domain = I915_GEM_DOMAIN_CPU; 806 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); 807 808 obj->userptr.ptr = args->user_ptr; 809 if (args->flags & I915_USERPTR_READ_ONLY) 810 i915_gem_object_set_readonly(obj); 811 812 /* And keep a pointer to the current->mm for resolving the user pages 813 * at binding. This means that we need to hook into the mmu_notifier 814 * in order to detect if the mmu is destroyed. 815 */ 816 ret = i915_gem_userptr_init__mm_struct(obj); 817 if (ret == 0) 818 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); 819 if (ret == 0) 820 ret = drm_gem_handle_create(file, &obj->base, &handle); 821 822 /* drop reference from allocate - handle holds it now */ 823 i915_gem_object_put(obj); 824 if (ret) 825 return ret; 826 827 args->handle = handle; 828 return 0; 829 } 830 831 int i915_gem_init_userptr(struct drm_i915_private *dev_priv) 832 { 833 mutex_init(&dev_priv->mm_lock); 834 hash_init(dev_priv->mm_structs); 835 836 dev_priv->mm.userptr_wq = 837 alloc_workqueue("i915-userptr-acquire", 838 WQ_HIGHPRI | WQ_UNBOUND, 839 0); 840 if (!dev_priv->mm.userptr_wq) 841 return -ENOMEM; 842 843 return 0; 844 } 845 846 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv) 847 { 848 destroy_workqueue(dev_priv->mm.userptr_wq); 849 } 850