1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2012-2014 Intel Corporation
5  *
6   * Based on amdgpu_mn, which bears the following notice:
7  *
8  * Copyright 2014 Advanced Micro Devices, Inc.
9  * All Rights Reserved.
10  *
11  * Permission is hereby granted, free of charge, to any person obtaining a
12  * copy of this software and associated documentation files (the
13  * "Software"), to deal in the Software without restriction, including
14  * without limitation the rights to use, copy, modify, merge, publish,
15  * distribute, sub license, and/or sell copies of the Software, and to
16  * permit persons to whom the Software is furnished to do so, subject to
17  * the following conditions:
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  * The above copyright notice and this permission notice (including the
28  * next paragraph) shall be included in all copies or substantial portions
29  * of the Software.
30  *
31  */
32 /*
33  * Authors:
34  *    Christian König <christian.koenig@amd.com>
35  */
36 
37 #include <linux/mmu_context.h>
38 #include <linux/mempolicy.h>
39 #include <linux/swap.h>
40 #include <linux/sched/mm.h>
41 
42 #include "i915_drv.h"
43 #include "i915_gem_ioctls.h"
44 #include "i915_gem_object.h"
45 #include "i915_gem_userptr.h"
46 #include "i915_scatterlist.h"
47 
48 #ifdef CONFIG_MMU_NOTIFIER
49 
50 /**
51  * i915_gem_userptr_invalidate - callback to notify about mm change
52  *
53  * @mni: the range (mm) is about to update
54  * @range: details on the invalidation
55  * @cur_seq: Value to pass to mmu_interval_set_seq()
56  *
57  * Block for operations on BOs to finish and mark pages as accessed and
58  * potentially dirty.
59  */
i915_gem_userptr_invalidate(struct mmu_interval_notifier * mni,const struct mmu_notifier_range * range,unsigned long cur_seq)60 static bool i915_gem_userptr_invalidate(struct mmu_interval_notifier *mni,
61 					const struct mmu_notifier_range *range,
62 					unsigned long cur_seq)
63 {
64 	struct drm_i915_gem_object *obj = container_of(mni, struct drm_i915_gem_object, userptr.notifier);
65 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
66 	long r;
67 
68 	if (!mmu_notifier_range_blockable(range))
69 		return false;
70 
71 	write_lock(&i915->mm.notifier_lock);
72 
73 	mmu_interval_set_seq(mni, cur_seq);
74 
75 	write_unlock(&i915->mm.notifier_lock);
76 
77 	/*
78 	 * We don't wait when the process is exiting. This is valid
79 	 * because the object will be cleaned up anyway.
80 	 *
81 	 * This is also temporarily required as a hack, because we
82 	 * cannot currently force non-consistent batch buffers to preempt
83 	 * and reschedule by waiting on it, hanging processes on exit.
84 	 */
85 	if (current->flags & PF_EXITING)
86 		return true;
87 
88 	/* we will unbind on next submission, still have userptr pins */
89 	r = dma_resv_wait_timeout(obj->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
90 				  MAX_SCHEDULE_TIMEOUT);
91 	if (r <= 0)
92 		drm_err(&i915->drm, "(%ld) failed to wait for idle\n", r);
93 
94 	return true;
95 }
96 
97 static const struct mmu_interval_notifier_ops i915_gem_userptr_notifier_ops = {
98 	.invalidate = i915_gem_userptr_invalidate,
99 };
100 
101 static int
i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object * obj)102 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj)
103 {
104 	return mmu_interval_notifier_insert(&obj->userptr.notifier, current->mm,
105 					    obj->userptr.ptr, obj->base.size,
106 					    &i915_gem_userptr_notifier_ops);
107 }
108 
i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object * obj)109 static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj)
110 {
111 	struct page **pvec = NULL;
112 
113 	assert_object_held_shared(obj);
114 
115 	if (!--obj->userptr.page_ref) {
116 		pvec = obj->userptr.pvec;
117 		obj->userptr.pvec = NULL;
118 	}
119 	GEM_BUG_ON(obj->userptr.page_ref < 0);
120 
121 	if (pvec) {
122 		const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
123 
124 		unpin_user_pages(pvec, num_pages);
125 		kvfree(pvec);
126 	}
127 }
128 
i915_gem_userptr_get_pages(struct drm_i915_gem_object * obj)129 static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
130 {
131 	unsigned int max_segment = i915_sg_segment_size(obj->base.dev->dev);
132 	struct sg_table *st;
133 	struct page **pvec;
134 	unsigned int num_pages; /* limited by sg_alloc_table_from_pages_segment */
135 	int ret;
136 
137 	if (overflows_type(obj->base.size >> PAGE_SHIFT, num_pages))
138 		return -E2BIG;
139 
140 	num_pages = obj->base.size >> PAGE_SHIFT;
141 	st = kmalloc(sizeof(*st), GFP_KERNEL);
142 	if (!st)
143 		return -ENOMEM;
144 
145 	if (!obj->userptr.page_ref) {
146 		ret = -EAGAIN;
147 		goto err_free;
148 	}
149 
150 	obj->userptr.page_ref++;
151 	pvec = obj->userptr.pvec;
152 
153 alloc_table:
154 	ret = sg_alloc_table_from_pages_segment(st, pvec, num_pages, 0,
155 						num_pages << PAGE_SHIFT,
156 						max_segment, GFP_KERNEL);
157 	if (ret)
158 		goto err;
159 
160 	ret = i915_gem_gtt_prepare_pages(obj, st);
161 	if (ret) {
162 		sg_free_table(st);
163 
164 		if (max_segment > PAGE_SIZE) {
165 			max_segment = PAGE_SIZE;
166 			goto alloc_table;
167 		}
168 
169 		goto err;
170 	}
171 
172 	WARN_ON_ONCE(!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE));
173 	if (i915_gem_object_can_bypass_llc(obj))
174 		obj->cache_dirty = true;
175 
176 	__i915_gem_object_set_pages(obj, st);
177 
178 	return 0;
179 
180 err:
181 	i915_gem_object_userptr_drop_ref(obj);
182 err_free:
183 	kfree(st);
184 	return ret;
185 }
186 
187 static void
i915_gem_userptr_put_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)188 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
189 			   struct sg_table *pages)
190 {
191 	struct sgt_iter sgt_iter;
192 	struct page *page;
193 
194 	if (!pages)
195 		return;
196 
197 	__i915_gem_object_release_shmem(obj, pages, true);
198 	i915_gem_gtt_finish_pages(obj, pages);
199 
200 	/*
201 	 * We always mark objects as dirty when they are used by the GPU,
202 	 * just in case. However, if we set the vma as being read-only we know
203 	 * that the object will never have been written to.
204 	 */
205 	if (i915_gem_object_is_readonly(obj))
206 		obj->mm.dirty = false;
207 
208 	for_each_sgt_page(page, sgt_iter, pages) {
209 		if (obj->mm.dirty && trylock_page(page)) {
210 			/*
211 			 * As this may not be anonymous memory (e.g. shmem)
212 			 * but exist on a real mapping, we have to lock
213 			 * the page in order to dirty it -- holding
214 			 * the page reference is not sufficient to
215 			 * prevent the inode from being truncated.
216 			 * Play safe and take the lock.
217 			 *
218 			 * However...!
219 			 *
220 			 * The mmu-notifier can be invalidated for a
221 			 * migrate_folio, that is alreadying holding the lock
222 			 * on the folio. Such a try_to_unmap() will result
223 			 * in us calling put_pages() and so recursively try
224 			 * to lock the page. We avoid that deadlock with
225 			 * a trylock_page() and in exchange we risk missing
226 			 * some page dirtying.
227 			 */
228 			set_page_dirty(page);
229 			unlock_page(page);
230 		}
231 
232 		mark_page_accessed(page);
233 	}
234 	obj->mm.dirty = false;
235 
236 	sg_free_table(pages);
237 	kfree(pages);
238 
239 	i915_gem_object_userptr_drop_ref(obj);
240 }
241 
i915_gem_object_userptr_unbind(struct drm_i915_gem_object * obj)242 static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj)
243 {
244 	struct sg_table *pages;
245 	int err;
246 
247 	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
248 	if (err)
249 		return err;
250 
251 	if (GEM_WARN_ON(i915_gem_object_has_pinned_pages(obj)))
252 		return -EBUSY;
253 
254 	assert_object_held(obj);
255 
256 	pages = __i915_gem_object_unset_pages(obj);
257 	if (!IS_ERR_OR_NULL(pages))
258 		i915_gem_userptr_put_pages(obj, pages);
259 
260 	return err;
261 }
262 
i915_gem_object_userptr_submit_init(struct drm_i915_gem_object * obj)263 int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj)
264 {
265 	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
266 	struct page **pvec;
267 	unsigned int gup_flags = 0;
268 	unsigned long notifier_seq;
269 	int pinned, ret;
270 
271 	if (obj->userptr.notifier.mm != current->mm)
272 		return -EFAULT;
273 
274 	notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier);
275 
276 	ret = i915_gem_object_lock_interruptible(obj, NULL);
277 	if (ret)
278 		return ret;
279 
280 	if (notifier_seq == obj->userptr.notifier_seq && obj->userptr.pvec) {
281 		i915_gem_object_unlock(obj);
282 		return 0;
283 	}
284 
285 	ret = i915_gem_object_userptr_unbind(obj);
286 	i915_gem_object_unlock(obj);
287 	if (ret)
288 		return ret;
289 
290 	pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL);
291 	if (!pvec)
292 		return -ENOMEM;
293 
294 	if (!i915_gem_object_is_readonly(obj))
295 		gup_flags |= FOLL_WRITE;
296 
297 	pinned = 0;
298 	while (pinned < num_pages) {
299 		ret = pin_user_pages_fast(obj->userptr.ptr + pinned * PAGE_SIZE,
300 					  num_pages - pinned, gup_flags,
301 					  &pvec[pinned]);
302 		if (ret < 0)
303 			goto out;
304 
305 		pinned += ret;
306 	}
307 
308 	ret = i915_gem_object_lock_interruptible(obj, NULL);
309 	if (ret)
310 		goto out;
311 
312 	if (mmu_interval_read_retry(&obj->userptr.notifier,
313 		!obj->userptr.page_ref ? notifier_seq :
314 		obj->userptr.notifier_seq)) {
315 		ret = -EAGAIN;
316 		goto out_unlock;
317 	}
318 
319 	if (!obj->userptr.page_ref++) {
320 		obj->userptr.pvec = pvec;
321 		obj->userptr.notifier_seq = notifier_seq;
322 		pvec = NULL;
323 		ret = ____i915_gem_object_get_pages(obj);
324 	}
325 
326 	obj->userptr.page_ref--;
327 
328 out_unlock:
329 	i915_gem_object_unlock(obj);
330 
331 out:
332 	if (pvec) {
333 		unpin_user_pages(pvec, pinned);
334 		kvfree(pvec);
335 	}
336 
337 	return ret;
338 }
339 
i915_gem_object_userptr_submit_done(struct drm_i915_gem_object * obj)340 int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj)
341 {
342 	if (mmu_interval_read_retry(&obj->userptr.notifier,
343 				    obj->userptr.notifier_seq)) {
344 		/* We collided with the mmu notifier, need to retry */
345 
346 		return -EAGAIN;
347 	}
348 
349 	return 0;
350 }
351 
i915_gem_object_userptr_validate(struct drm_i915_gem_object * obj)352 int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj)
353 {
354 	int err;
355 
356 	err = i915_gem_object_userptr_submit_init(obj);
357 	if (err)
358 		return err;
359 
360 	err = i915_gem_object_lock_interruptible(obj, NULL);
361 	if (!err) {
362 		/*
363 		 * Since we only check validity, not use the pages,
364 		 * it doesn't matter if we collide with the mmu notifier,
365 		 * and -EAGAIN handling is not required.
366 		 */
367 		err = i915_gem_object_pin_pages(obj);
368 		if (!err)
369 			i915_gem_object_unpin_pages(obj);
370 
371 		i915_gem_object_unlock(obj);
372 	}
373 
374 	return err;
375 }
376 
377 static void
i915_gem_userptr_release(struct drm_i915_gem_object * obj)378 i915_gem_userptr_release(struct drm_i915_gem_object *obj)
379 {
380 	GEM_WARN_ON(obj->userptr.page_ref);
381 
382 	if (!obj->userptr.notifier.mm)
383 		return;
384 
385 	mmu_interval_notifier_remove(&obj->userptr.notifier);
386 	obj->userptr.notifier.mm = NULL;
387 }
388 
389 static int
i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object * obj)390 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
391 {
392 	drm_dbg(obj->base.dev, "Exporting userptr no longer allowed\n");
393 
394 	return -EINVAL;
395 }
396 
397 static int
i915_gem_userptr_pwrite(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pwrite * args)398 i915_gem_userptr_pwrite(struct drm_i915_gem_object *obj,
399 			const struct drm_i915_gem_pwrite *args)
400 {
401 	drm_dbg(obj->base.dev, "pwrite to userptr no longer allowed\n");
402 
403 	return -EINVAL;
404 }
405 
406 static int
i915_gem_userptr_pread(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pread * args)407 i915_gem_userptr_pread(struct drm_i915_gem_object *obj,
408 		       const struct drm_i915_gem_pread *args)
409 {
410 	drm_dbg(obj->base.dev, "pread from userptr no longer allowed\n");
411 
412 	return -EINVAL;
413 }
414 
415 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
416 	.name = "i915_gem_object_userptr",
417 	.flags = I915_GEM_OBJECT_IS_SHRINKABLE |
418 		 I915_GEM_OBJECT_NO_MMAP |
419 		 I915_GEM_OBJECT_IS_PROXY,
420 	.get_pages = i915_gem_userptr_get_pages,
421 	.put_pages = i915_gem_userptr_put_pages,
422 	.dmabuf_export = i915_gem_userptr_dmabuf_export,
423 	.pwrite = i915_gem_userptr_pwrite,
424 	.pread = i915_gem_userptr_pread,
425 	.release = i915_gem_userptr_release,
426 };
427 
428 #endif
429 
430 static int
probe_range(struct mm_struct * mm,unsigned long addr,unsigned long len)431 probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len)
432 {
433 	VMA_ITERATOR(vmi, mm, addr);
434 	struct vm_area_struct *vma;
435 	unsigned long end = addr + len;
436 
437 	mmap_read_lock(mm);
438 	for_each_vma_range(vmi, vma, end) {
439 		/* Check for holes, note that we also update the addr below */
440 		if (vma->vm_start > addr)
441 			break;
442 
443 		if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
444 			break;
445 
446 		addr = vma->vm_end;
447 	}
448 	mmap_read_unlock(mm);
449 
450 	if (vma || addr < end)
451 		return -EFAULT;
452 	return 0;
453 }
454 
455 /*
456  * Creates a new mm object that wraps some normal memory from the process
457  * context - user memory.
458  *
459  * We impose several restrictions upon the memory being mapped
460  * into the GPU.
461  * 1. It must be page aligned (both start/end addresses, i.e ptr and size).
462  * 2. It must be normal system memory, not a pointer into another map of IO
463  *    space (e.g. it must not be a GTT mmapping of another object).
464  * 3. We only allow a bo as large as we could in theory map into the GTT,
465  *    that is we limit the size to the total size of the GTT.
466  * 4. The bo is marked as being snoopable. The backing pages are left
467  *    accessible directly by the CPU, but reads and writes by the GPU may
468  *    incur the cost of a snoop (unless you have an LLC architecture).
469  *
470  * Synchronisation between multiple users and the GPU is left to userspace
471  * through the normal set-domain-ioctl. The kernel will enforce that the
472  * GPU relinquishes the VMA before it is returned back to the system
473  * i.e. upon free(), munmap() or process termination. However, the userspace
474  * malloc() library may not immediately relinquish the VMA after free() and
475  * instead reuse it whilst the GPU is still reading and writing to the VMA.
476  * Caveat emptor.
477  *
478  * Also note, that the object created here is not currently a "first class"
479  * object, in that several ioctls are banned. These are the CPU access
480  * ioctls: mmap(), pwrite and pread. In practice, you are expected to use
481  * direct access via your pointer rather than use those ioctls. Another
482  * restriction is that we do not allow userptr surfaces to be pinned to the
483  * hardware and so we reject any attempt to create a framebuffer out of a
484  * userptr.
485  *
486  * If you think this is a good interface to use to pass GPU memory between
487  * drivers, please use dma-buf instead. In fact, wherever possible use
488  * dma-buf instead.
489  */
490 int
i915_gem_userptr_ioctl(struct drm_device * dev,void * data,struct drm_file * file)491 i915_gem_userptr_ioctl(struct drm_device *dev,
492 		       void *data,
493 		       struct drm_file *file)
494 {
495 	static struct lock_class_key __maybe_unused lock_class;
496 	struct drm_i915_private *dev_priv = to_i915(dev);
497 	struct drm_i915_gem_userptr *args = data;
498 	struct drm_i915_gem_object __maybe_unused *obj;
499 	int __maybe_unused ret;
500 	u32 __maybe_unused handle;
501 
502 	if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
503 		/* We cannot support coherent userptr objects on hw without
504 		 * LLC and broken snooping.
505 		 */
506 		return -ENODEV;
507 	}
508 
509 	if (args->flags & ~(I915_USERPTR_READ_ONLY |
510 			    I915_USERPTR_UNSYNCHRONIZED |
511 			    I915_USERPTR_PROBE))
512 		return -EINVAL;
513 
514 	if (i915_gem_object_size_2big(args->user_size))
515 		return -E2BIG;
516 
517 	if (!args->user_size)
518 		return -EINVAL;
519 
520 	if (offset_in_page(args->user_ptr | args->user_size))
521 		return -EINVAL;
522 
523 	if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
524 		return -EFAULT;
525 
526 	if (args->flags & I915_USERPTR_UNSYNCHRONIZED)
527 		return -ENODEV;
528 
529 	if (args->flags & I915_USERPTR_READ_ONLY) {
530 		/*
531 		 * On almost all of the older hw, we cannot tell the GPU that
532 		 * a page is readonly.
533 		 */
534 		if (!to_gt(dev_priv)->vm->has_read_only)
535 			return -ENODEV;
536 	}
537 
538 	if (args->flags & I915_USERPTR_PROBE) {
539 		/*
540 		 * Check that the range pointed to represents real struct
541 		 * pages and not iomappings (at this moment in time!)
542 		 */
543 		ret = probe_range(current->mm, args->user_ptr, args->user_size);
544 		if (ret)
545 			return ret;
546 	}
547 
548 #ifdef CONFIG_MMU_NOTIFIER
549 	obj = i915_gem_object_alloc();
550 	if (obj == NULL)
551 		return -ENOMEM;
552 
553 	drm_gem_private_object_init(dev, &obj->base, args->user_size);
554 	i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class,
555 			     I915_BO_ALLOC_USER);
556 	obj->mem_flags = I915_BO_FLAG_STRUCT_PAGE;
557 	obj->read_domains = I915_GEM_DOMAIN_CPU;
558 	obj->write_domain = I915_GEM_DOMAIN_CPU;
559 	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
560 
561 	obj->userptr.ptr = args->user_ptr;
562 	obj->userptr.notifier_seq = ULONG_MAX;
563 	if (args->flags & I915_USERPTR_READ_ONLY)
564 		i915_gem_object_set_readonly(obj);
565 
566 	/* And keep a pointer to the current->mm for resolving the user pages
567 	 * at binding. This means that we need to hook into the mmu_notifier
568 	 * in order to detect if the mmu is destroyed.
569 	 */
570 	ret = i915_gem_userptr_init__mmu_notifier(obj);
571 	if (ret == 0)
572 		ret = drm_gem_handle_create(file, &obj->base, &handle);
573 
574 	/* drop reference from allocate - handle holds it now */
575 	i915_gem_object_put(obj);
576 	if (ret)
577 		return ret;
578 
579 	args->handle = handle;
580 	return 0;
581 #else
582 	return -ENODEV;
583 #endif
584 }
585 
i915_gem_init_userptr(struct drm_i915_private * dev_priv)586 int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
587 {
588 #ifdef CONFIG_MMU_NOTIFIER
589 	rwlock_init(&dev_priv->mm.notifier_lock);
590 #endif
591 
592 	return 0;
593 }
594 
i915_gem_cleanup_userptr(struct drm_i915_private * dev_priv)595 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
596 {
597 }
598