1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2012-2014 Intel Corporation
5 *
6 * Based on amdgpu_mn, which bears the following notice:
7 *
8 * Copyright 2014 Advanced Micro Devices, Inc.
9 * All Rights Reserved.
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the
13 * "Software"), to deal in the Software without restriction, including
14 * without limitation the rights to use, copy, modify, merge, publish,
15 * distribute, sub license, and/or sell copies of the Software, and to
16 * permit persons to whom the Software is furnished to do so, subject to
17 * the following conditions:
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 * The above copyright notice and this permission notice (including the
28 * next paragraph) shall be included in all copies or substantial portions
29 * of the Software.
30 *
31 */
32 /*
33 * Authors:
34 * Christian König <christian.koenig@amd.com>
35 */
36
37 #include <linux/mmu_context.h>
38 #include <linux/mempolicy.h>
39 #include <linux/swap.h>
40 #include <linux/sched/mm.h>
41
42 #include "i915_drv.h"
43 #include "i915_gem_ioctls.h"
44 #include "i915_gem_object.h"
45 #include "i915_gem_userptr.h"
46 #include "i915_scatterlist.h"
47
48 #ifdef CONFIG_MMU_NOTIFIER
49
50 /**
51 * i915_gem_userptr_invalidate - callback to notify about mm change
52 *
53 * @mni: the range (mm) is about to update
54 * @range: details on the invalidation
55 * @cur_seq: Value to pass to mmu_interval_set_seq()
56 *
57 * Block for operations on BOs to finish and mark pages as accessed and
58 * potentially dirty.
59 */
i915_gem_userptr_invalidate(struct mmu_interval_notifier * mni,const struct mmu_notifier_range * range,unsigned long cur_seq)60 static bool i915_gem_userptr_invalidate(struct mmu_interval_notifier *mni,
61 const struct mmu_notifier_range *range,
62 unsigned long cur_seq)
63 {
64 struct drm_i915_gem_object *obj = container_of(mni, struct drm_i915_gem_object, userptr.notifier);
65 struct drm_i915_private *i915 = to_i915(obj->base.dev);
66 long r;
67
68 if (!mmu_notifier_range_blockable(range))
69 return false;
70
71 write_lock(&i915->mm.notifier_lock);
72
73 mmu_interval_set_seq(mni, cur_seq);
74
75 write_unlock(&i915->mm.notifier_lock);
76
77 /*
78 * We don't wait when the process is exiting. This is valid
79 * because the object will be cleaned up anyway.
80 *
81 * This is also temporarily required as a hack, because we
82 * cannot currently force non-consistent batch buffers to preempt
83 * and reschedule by waiting on it, hanging processes on exit.
84 */
85 if (current->flags & PF_EXITING)
86 return true;
87
88 /* we will unbind on next submission, still have userptr pins */
89 r = dma_resv_wait_timeout(obj->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
90 MAX_SCHEDULE_TIMEOUT);
91 if (r <= 0)
92 drm_err(&i915->drm, "(%ld) failed to wait for idle\n", r);
93
94 return true;
95 }
96
97 static const struct mmu_interval_notifier_ops i915_gem_userptr_notifier_ops = {
98 .invalidate = i915_gem_userptr_invalidate,
99 };
100
101 static int
i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object * obj)102 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj)
103 {
104 return mmu_interval_notifier_insert(&obj->userptr.notifier, current->mm,
105 obj->userptr.ptr, obj->base.size,
106 &i915_gem_userptr_notifier_ops);
107 }
108
i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object * obj)109 static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj)
110 {
111 struct page **pvec = NULL;
112
113 assert_object_held_shared(obj);
114
115 if (!--obj->userptr.page_ref) {
116 pvec = obj->userptr.pvec;
117 obj->userptr.pvec = NULL;
118 }
119 GEM_BUG_ON(obj->userptr.page_ref < 0);
120
121 if (pvec) {
122 const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
123
124 unpin_user_pages(pvec, num_pages);
125 kvfree(pvec);
126 }
127 }
128
i915_gem_userptr_get_pages(struct drm_i915_gem_object * obj)129 static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
130 {
131 unsigned int max_segment = i915_sg_segment_size(obj->base.dev->dev);
132 struct sg_table *st;
133 struct page **pvec;
134 unsigned int num_pages; /* limited by sg_alloc_table_from_pages_segment */
135 int ret;
136
137 if (overflows_type(obj->base.size >> PAGE_SHIFT, num_pages))
138 return -E2BIG;
139
140 num_pages = obj->base.size >> PAGE_SHIFT;
141 st = kmalloc(sizeof(*st), GFP_KERNEL);
142 if (!st)
143 return -ENOMEM;
144
145 if (!obj->userptr.page_ref) {
146 ret = -EAGAIN;
147 goto err_free;
148 }
149
150 obj->userptr.page_ref++;
151 pvec = obj->userptr.pvec;
152
153 alloc_table:
154 ret = sg_alloc_table_from_pages_segment(st, pvec, num_pages, 0,
155 num_pages << PAGE_SHIFT,
156 max_segment, GFP_KERNEL);
157 if (ret)
158 goto err;
159
160 ret = i915_gem_gtt_prepare_pages(obj, st);
161 if (ret) {
162 sg_free_table(st);
163
164 if (max_segment > PAGE_SIZE) {
165 max_segment = PAGE_SIZE;
166 goto alloc_table;
167 }
168
169 goto err;
170 }
171
172 WARN_ON_ONCE(!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE));
173 if (i915_gem_object_can_bypass_llc(obj))
174 obj->cache_dirty = true;
175
176 __i915_gem_object_set_pages(obj, st);
177
178 return 0;
179
180 err:
181 i915_gem_object_userptr_drop_ref(obj);
182 err_free:
183 kfree(st);
184 return ret;
185 }
186
187 static void
i915_gem_userptr_put_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)188 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
189 struct sg_table *pages)
190 {
191 struct sgt_iter sgt_iter;
192 struct page *page;
193
194 if (!pages)
195 return;
196
197 __i915_gem_object_release_shmem(obj, pages, true);
198 i915_gem_gtt_finish_pages(obj, pages);
199
200 /*
201 * We always mark objects as dirty when they are used by the GPU,
202 * just in case. However, if we set the vma as being read-only we know
203 * that the object will never have been written to.
204 */
205 if (i915_gem_object_is_readonly(obj))
206 obj->mm.dirty = false;
207
208 for_each_sgt_page(page, sgt_iter, pages) {
209 if (obj->mm.dirty && trylock_page(page)) {
210 /*
211 * As this may not be anonymous memory (e.g. shmem)
212 * but exist on a real mapping, we have to lock
213 * the page in order to dirty it -- holding
214 * the page reference is not sufficient to
215 * prevent the inode from being truncated.
216 * Play safe and take the lock.
217 *
218 * However...!
219 *
220 * The mmu-notifier can be invalidated for a
221 * migrate_folio, that is alreadying holding the lock
222 * on the folio. Such a try_to_unmap() will result
223 * in us calling put_pages() and so recursively try
224 * to lock the page. We avoid that deadlock with
225 * a trylock_page() and in exchange we risk missing
226 * some page dirtying.
227 */
228 set_page_dirty(page);
229 unlock_page(page);
230 }
231
232 mark_page_accessed(page);
233 }
234 obj->mm.dirty = false;
235
236 sg_free_table(pages);
237 kfree(pages);
238
239 i915_gem_object_userptr_drop_ref(obj);
240 }
241
i915_gem_object_userptr_unbind(struct drm_i915_gem_object * obj)242 static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj)
243 {
244 struct sg_table *pages;
245 int err;
246
247 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
248 if (err)
249 return err;
250
251 if (GEM_WARN_ON(i915_gem_object_has_pinned_pages(obj)))
252 return -EBUSY;
253
254 assert_object_held(obj);
255
256 pages = __i915_gem_object_unset_pages(obj);
257 if (!IS_ERR_OR_NULL(pages))
258 i915_gem_userptr_put_pages(obj, pages);
259
260 return err;
261 }
262
i915_gem_object_userptr_submit_init(struct drm_i915_gem_object * obj)263 int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj)
264 {
265 const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
266 struct page **pvec;
267 unsigned int gup_flags = 0;
268 unsigned long notifier_seq;
269 int pinned, ret;
270
271 if (obj->userptr.notifier.mm != current->mm)
272 return -EFAULT;
273
274 notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier);
275
276 ret = i915_gem_object_lock_interruptible(obj, NULL);
277 if (ret)
278 return ret;
279
280 if (notifier_seq == obj->userptr.notifier_seq && obj->userptr.pvec) {
281 i915_gem_object_unlock(obj);
282 return 0;
283 }
284
285 ret = i915_gem_object_userptr_unbind(obj);
286 i915_gem_object_unlock(obj);
287 if (ret)
288 return ret;
289
290 pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL);
291 if (!pvec)
292 return -ENOMEM;
293
294 if (!i915_gem_object_is_readonly(obj))
295 gup_flags |= FOLL_WRITE;
296
297 pinned = 0;
298 while (pinned < num_pages) {
299 ret = pin_user_pages_fast(obj->userptr.ptr + pinned * PAGE_SIZE,
300 num_pages - pinned, gup_flags,
301 &pvec[pinned]);
302 if (ret < 0)
303 goto out;
304
305 pinned += ret;
306 }
307
308 ret = i915_gem_object_lock_interruptible(obj, NULL);
309 if (ret)
310 goto out;
311
312 if (mmu_interval_read_retry(&obj->userptr.notifier,
313 !obj->userptr.page_ref ? notifier_seq :
314 obj->userptr.notifier_seq)) {
315 ret = -EAGAIN;
316 goto out_unlock;
317 }
318
319 if (!obj->userptr.page_ref++) {
320 obj->userptr.pvec = pvec;
321 obj->userptr.notifier_seq = notifier_seq;
322 pvec = NULL;
323 ret = ____i915_gem_object_get_pages(obj);
324 }
325
326 obj->userptr.page_ref--;
327
328 out_unlock:
329 i915_gem_object_unlock(obj);
330
331 out:
332 if (pvec) {
333 unpin_user_pages(pvec, pinned);
334 kvfree(pvec);
335 }
336
337 return ret;
338 }
339
i915_gem_object_userptr_submit_done(struct drm_i915_gem_object * obj)340 int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj)
341 {
342 if (mmu_interval_read_retry(&obj->userptr.notifier,
343 obj->userptr.notifier_seq)) {
344 /* We collided with the mmu notifier, need to retry */
345
346 return -EAGAIN;
347 }
348
349 return 0;
350 }
351
i915_gem_object_userptr_validate(struct drm_i915_gem_object * obj)352 int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj)
353 {
354 int err;
355
356 err = i915_gem_object_userptr_submit_init(obj);
357 if (err)
358 return err;
359
360 err = i915_gem_object_lock_interruptible(obj, NULL);
361 if (!err) {
362 /*
363 * Since we only check validity, not use the pages,
364 * it doesn't matter if we collide with the mmu notifier,
365 * and -EAGAIN handling is not required.
366 */
367 err = i915_gem_object_pin_pages(obj);
368 if (!err)
369 i915_gem_object_unpin_pages(obj);
370
371 i915_gem_object_unlock(obj);
372 }
373
374 return err;
375 }
376
377 static void
i915_gem_userptr_release(struct drm_i915_gem_object * obj)378 i915_gem_userptr_release(struct drm_i915_gem_object *obj)
379 {
380 GEM_WARN_ON(obj->userptr.page_ref);
381
382 if (!obj->userptr.notifier.mm)
383 return;
384
385 mmu_interval_notifier_remove(&obj->userptr.notifier);
386 obj->userptr.notifier.mm = NULL;
387 }
388
389 static int
i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object * obj)390 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
391 {
392 drm_dbg(obj->base.dev, "Exporting userptr no longer allowed\n");
393
394 return -EINVAL;
395 }
396
397 static int
i915_gem_userptr_pwrite(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pwrite * args)398 i915_gem_userptr_pwrite(struct drm_i915_gem_object *obj,
399 const struct drm_i915_gem_pwrite *args)
400 {
401 drm_dbg(obj->base.dev, "pwrite to userptr no longer allowed\n");
402
403 return -EINVAL;
404 }
405
406 static int
i915_gem_userptr_pread(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pread * args)407 i915_gem_userptr_pread(struct drm_i915_gem_object *obj,
408 const struct drm_i915_gem_pread *args)
409 {
410 drm_dbg(obj->base.dev, "pread from userptr no longer allowed\n");
411
412 return -EINVAL;
413 }
414
415 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
416 .name = "i915_gem_object_userptr",
417 .flags = I915_GEM_OBJECT_IS_SHRINKABLE |
418 I915_GEM_OBJECT_NO_MMAP |
419 I915_GEM_OBJECT_IS_PROXY,
420 .get_pages = i915_gem_userptr_get_pages,
421 .put_pages = i915_gem_userptr_put_pages,
422 .dmabuf_export = i915_gem_userptr_dmabuf_export,
423 .pwrite = i915_gem_userptr_pwrite,
424 .pread = i915_gem_userptr_pread,
425 .release = i915_gem_userptr_release,
426 };
427
428 #endif
429
430 static int
probe_range(struct mm_struct * mm,unsigned long addr,unsigned long len)431 probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len)
432 {
433 VMA_ITERATOR(vmi, mm, addr);
434 struct vm_area_struct *vma;
435 unsigned long end = addr + len;
436
437 mmap_read_lock(mm);
438 for_each_vma_range(vmi, vma, end) {
439 /* Check for holes, note that we also update the addr below */
440 if (vma->vm_start > addr)
441 break;
442
443 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
444 break;
445
446 addr = vma->vm_end;
447 }
448 mmap_read_unlock(mm);
449
450 if (vma || addr < end)
451 return -EFAULT;
452 return 0;
453 }
454
455 /*
456 * Creates a new mm object that wraps some normal memory from the process
457 * context - user memory.
458 *
459 * We impose several restrictions upon the memory being mapped
460 * into the GPU.
461 * 1. It must be page aligned (both start/end addresses, i.e ptr and size).
462 * 2. It must be normal system memory, not a pointer into another map of IO
463 * space (e.g. it must not be a GTT mmapping of another object).
464 * 3. We only allow a bo as large as we could in theory map into the GTT,
465 * that is we limit the size to the total size of the GTT.
466 * 4. The bo is marked as being snoopable. The backing pages are left
467 * accessible directly by the CPU, but reads and writes by the GPU may
468 * incur the cost of a snoop (unless you have an LLC architecture).
469 *
470 * Synchronisation between multiple users and the GPU is left to userspace
471 * through the normal set-domain-ioctl. The kernel will enforce that the
472 * GPU relinquishes the VMA before it is returned back to the system
473 * i.e. upon free(), munmap() or process termination. However, the userspace
474 * malloc() library may not immediately relinquish the VMA after free() and
475 * instead reuse it whilst the GPU is still reading and writing to the VMA.
476 * Caveat emptor.
477 *
478 * Also note, that the object created here is not currently a "first class"
479 * object, in that several ioctls are banned. These are the CPU access
480 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use
481 * direct access via your pointer rather than use those ioctls. Another
482 * restriction is that we do not allow userptr surfaces to be pinned to the
483 * hardware and so we reject any attempt to create a framebuffer out of a
484 * userptr.
485 *
486 * If you think this is a good interface to use to pass GPU memory between
487 * drivers, please use dma-buf instead. In fact, wherever possible use
488 * dma-buf instead.
489 */
490 int
i915_gem_userptr_ioctl(struct drm_device * dev,void * data,struct drm_file * file)491 i915_gem_userptr_ioctl(struct drm_device *dev,
492 void *data,
493 struct drm_file *file)
494 {
495 static struct lock_class_key __maybe_unused lock_class;
496 struct drm_i915_private *dev_priv = to_i915(dev);
497 struct drm_i915_gem_userptr *args = data;
498 struct drm_i915_gem_object __maybe_unused *obj;
499 int __maybe_unused ret;
500 u32 __maybe_unused handle;
501
502 if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
503 /* We cannot support coherent userptr objects on hw without
504 * LLC and broken snooping.
505 */
506 return -ENODEV;
507 }
508
509 if (args->flags & ~(I915_USERPTR_READ_ONLY |
510 I915_USERPTR_UNSYNCHRONIZED |
511 I915_USERPTR_PROBE))
512 return -EINVAL;
513
514 if (i915_gem_object_size_2big(args->user_size))
515 return -E2BIG;
516
517 if (!args->user_size)
518 return -EINVAL;
519
520 if (offset_in_page(args->user_ptr | args->user_size))
521 return -EINVAL;
522
523 if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
524 return -EFAULT;
525
526 if (args->flags & I915_USERPTR_UNSYNCHRONIZED)
527 return -ENODEV;
528
529 if (args->flags & I915_USERPTR_READ_ONLY) {
530 /*
531 * On almost all of the older hw, we cannot tell the GPU that
532 * a page is readonly.
533 */
534 if (!to_gt(dev_priv)->vm->has_read_only)
535 return -ENODEV;
536 }
537
538 if (args->flags & I915_USERPTR_PROBE) {
539 /*
540 * Check that the range pointed to represents real struct
541 * pages and not iomappings (at this moment in time!)
542 */
543 ret = probe_range(current->mm, args->user_ptr, args->user_size);
544 if (ret)
545 return ret;
546 }
547
548 #ifdef CONFIG_MMU_NOTIFIER
549 obj = i915_gem_object_alloc();
550 if (obj == NULL)
551 return -ENOMEM;
552
553 drm_gem_private_object_init(dev, &obj->base, args->user_size);
554 i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class,
555 I915_BO_ALLOC_USER);
556 obj->mem_flags = I915_BO_FLAG_STRUCT_PAGE;
557 obj->read_domains = I915_GEM_DOMAIN_CPU;
558 obj->write_domain = I915_GEM_DOMAIN_CPU;
559 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
560
561 obj->userptr.ptr = args->user_ptr;
562 obj->userptr.notifier_seq = ULONG_MAX;
563 if (args->flags & I915_USERPTR_READ_ONLY)
564 i915_gem_object_set_readonly(obj);
565
566 /* And keep a pointer to the current->mm for resolving the user pages
567 * at binding. This means that we need to hook into the mmu_notifier
568 * in order to detect if the mmu is destroyed.
569 */
570 ret = i915_gem_userptr_init__mmu_notifier(obj);
571 if (ret == 0)
572 ret = drm_gem_handle_create(file, &obj->base, &handle);
573
574 /* drop reference from allocate - handle holds it now */
575 i915_gem_object_put(obj);
576 if (ret)
577 return ret;
578
579 args->handle = handle;
580 return 0;
581 #else
582 return -ENODEV;
583 #endif
584 }
585
i915_gem_init_userptr(struct drm_i915_private * dev_priv)586 int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
587 {
588 #ifdef CONFIG_MMU_NOTIFIER
589 rwlock_init(&dev_priv->mm.notifier_lock);
590 #endif
591
592 return 0;
593 }
594
i915_gem_cleanup_userptr(struct drm_i915_private * dev_priv)595 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
596 {
597 }
598