1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include <linux/anon_inodes.h>
8 #include <linux/mman.h>
9 #include <linux/pfn_t.h>
10 #include <linux/sizes.h>
11 
12 #include <drm/drm_cache.h>
13 
14 #include "gt/intel_gt.h"
15 #include "gt/intel_gt_requests.h"
16 
17 #include "i915_drv.h"
18 #include "i915_gem_evict.h"
19 #include "i915_gem_gtt.h"
20 #include "i915_gem_ioctls.h"
21 #include "i915_gem_object.h"
22 #include "i915_gem_mman.h"
23 #include "i915_mm.h"
24 #include "i915_trace.h"
25 #include "i915_user_extensions.h"
26 #include "i915_gem_ttm.h"
27 #include "i915_vma.h"
28 
29 static inline bool
30 __vma_matches(struct vm_area_struct *vma, struct file *filp,
31 	      unsigned long addr, unsigned long size)
32 {
33 	if (vma->vm_file != filp)
34 		return false;
35 
36 	return vma->vm_start == addr &&
37 	       (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
38 }
39 
40 /**
41  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
42  *			 it is mapped to.
43  * @dev: drm device
44  * @data: ioctl data blob
45  * @file: drm file
46  *
47  * While the mapping holds a reference on the contents of the object, it doesn't
48  * imply a ref on the object itself.
49  *
50  * IMPORTANT:
51  *
52  * DRM driver writers who look a this function as an example for how to do GEM
53  * mmap support, please don't implement mmap support like here. The modern way
54  * to implement DRM mmap support is with an mmap offset ioctl (like
55  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
56  * That way debug tooling like valgrind will understand what's going on, hiding
57  * the mmap call in a driver private ioctl will break that. The i915 driver only
58  * does cpu mmaps this way because we didn't know better.
59  */
60 int
61 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
62 		    struct drm_file *file)
63 {
64 	struct drm_i915_private *i915 = to_i915(dev);
65 	struct drm_i915_gem_mmap *args = data;
66 	struct drm_i915_gem_object *obj;
67 	unsigned long addr;
68 
69 	/*
70 	 * mmap ioctl is disallowed for all discrete platforms,
71 	 * and for all platforms with GRAPHICS_VER > 12.
72 	 */
73 	if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0))
74 		return -EOPNOTSUPP;
75 
76 	if (args->flags & ~(I915_MMAP_WC))
77 		return -EINVAL;
78 
79 	if (args->flags & I915_MMAP_WC && !pat_enabled())
80 		return -ENODEV;
81 
82 	obj = i915_gem_object_lookup(file, args->handle);
83 	if (!obj)
84 		return -ENOENT;
85 
86 	/* prime objects have no backing filp to GEM mmap
87 	 * pages from.
88 	 */
89 	if (!obj->base.filp) {
90 		addr = -ENXIO;
91 		goto err;
92 	}
93 
94 	if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
95 		addr = -EINVAL;
96 		goto err;
97 	}
98 
99 	addr = vm_mmap(obj->base.filp, 0, args->size,
100 		       PROT_READ | PROT_WRITE, MAP_SHARED,
101 		       args->offset);
102 	if (IS_ERR_VALUE(addr))
103 		goto err;
104 
105 	if (args->flags & I915_MMAP_WC) {
106 		struct mm_struct *mm = current->mm;
107 		struct vm_area_struct *vma;
108 
109 		if (mmap_write_lock_killable(mm)) {
110 			addr = -EINTR;
111 			goto err;
112 		}
113 		vma = find_vma(mm, addr);
114 		if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
115 			vma->vm_page_prot =
116 				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
117 		else
118 			addr = -ENOMEM;
119 		mmap_write_unlock(mm);
120 		if (IS_ERR_VALUE(addr))
121 			goto err;
122 	}
123 	i915_gem_object_put(obj);
124 
125 	args->addr_ptr = (u64)addr;
126 	return 0;
127 
128 err:
129 	i915_gem_object_put(obj);
130 	return addr;
131 }
132 
133 static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
134 {
135 	return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
136 }
137 
138 /**
139  * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
140  *
141  * A history of the GTT mmap interface:
142  *
143  * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
144  *     aligned and suitable for fencing, and still fit into the available
145  *     mappable space left by the pinned display objects. A classic problem
146  *     we called the page-fault-of-doom where we would ping-pong between
147  *     two objects that could not fit inside the GTT and so the memcpy
148  *     would page one object in at the expense of the other between every
149  *     single byte.
150  *
151  * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
152  *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
153  *     object is too large for the available space (or simply too large
154  *     for the mappable aperture!), a view is created instead and faulted
155  *     into userspace. (This view is aligned and sized appropriately for
156  *     fenced access.)
157  *
158  * 2 - Recognise WC as a separate cache domain so that we can flush the
159  *     delayed writes via GTT before performing direct access via WC.
160  *
161  * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
162  *     pagefault; swapin remains transparent.
163  *
164  * 4 - Support multiple fault handlers per object depending on object's
165  *     backing storage (a.k.a. MMAP_OFFSET).
166  *
167  * Restrictions:
168  *
169  *  * snoopable objects cannot be accessed via the GTT. It can cause machine
170  *    hangs on some architectures, corruption on others. An attempt to service
171  *    a GTT page fault from a snoopable object will generate a SIGBUS.
172  *
173  *  * the object must be able to fit into RAM (physical memory, though no
174  *    limited to the mappable aperture).
175  *
176  *
177  * Caveats:
178  *
179  *  * a new GTT page fault will synchronize rendering from the GPU and flush
180  *    all data to system memory. Subsequent access will not be synchronized.
181  *
182  *  * all mappings are revoked on runtime device suspend.
183  *
184  *  * there are only 8, 16 or 32 fence registers to share between all users
185  *    (older machines require fence register for display and blitter access
186  *    as well). Contention of the fence registers will cause the previous users
187  *    to be unmapped and any new access will generate new page faults.
188  *
189  *  * running out of memory while servicing a fault may generate a SIGBUS,
190  *    rather than the expected SIGSEGV.
191  */
192 int i915_gem_mmap_gtt_version(void)
193 {
194 	return 4;
195 }
196 
197 static inline struct i915_gtt_view
198 compute_partial_view(const struct drm_i915_gem_object *obj,
199 		     pgoff_t page_offset,
200 		     unsigned int chunk)
201 {
202 	struct i915_gtt_view view;
203 
204 	if (i915_gem_object_is_tiled(obj))
205 		chunk = roundup(chunk, tile_row_pages(obj) ?: 1);
206 
207 	view.type = I915_GTT_VIEW_PARTIAL;
208 	view.partial.offset = rounddown(page_offset, chunk);
209 	view.partial.size =
210 		min_t(unsigned int, chunk,
211 		      (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
212 
213 	/* If the partial covers the entire object, just create a normal VMA. */
214 	if (chunk >= obj->base.size >> PAGE_SHIFT)
215 		view.type = I915_GTT_VIEW_NORMAL;
216 
217 	return view;
218 }
219 
220 static vm_fault_t i915_error_to_vmf_fault(int err)
221 {
222 	switch (err) {
223 	default:
224 		WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err);
225 		fallthrough;
226 	case -EIO: /* shmemfs failure from swap device */
227 	case -EFAULT: /* purged object */
228 	case -ENODEV: /* bad object, how did you get here! */
229 	case -ENXIO: /* unable to access backing store (on device) */
230 		return VM_FAULT_SIGBUS;
231 
232 	case -ENOMEM: /* our allocation failure */
233 		return VM_FAULT_OOM;
234 
235 	case 0:
236 	case -EAGAIN:
237 	case -ENOSPC: /* transient failure to evict? */
238 	case -ENOBUFS: /* temporarily out of fences? */
239 	case -ERESTARTSYS:
240 	case -EINTR:
241 	case -EBUSY:
242 		/*
243 		 * EBUSY is ok: this just means that another thread
244 		 * already did the job.
245 		 */
246 		return VM_FAULT_NOPAGE;
247 	}
248 }
249 
250 static vm_fault_t vm_fault_cpu(struct vm_fault *vmf)
251 {
252 	struct vm_area_struct *area = vmf->vma;
253 	struct i915_mmap_offset *mmo = area->vm_private_data;
254 	struct drm_i915_gem_object *obj = mmo->obj;
255 	resource_size_t iomap;
256 	int err;
257 
258 	/* Sanity check that we allow writing into this object */
259 	if (unlikely(i915_gem_object_is_readonly(obj) &&
260 		     area->vm_flags & VM_WRITE))
261 		return VM_FAULT_SIGBUS;
262 
263 	if (i915_gem_object_lock_interruptible(obj, NULL))
264 		return VM_FAULT_NOPAGE;
265 
266 	err = i915_gem_object_pin_pages(obj);
267 	if (err)
268 		goto out;
269 
270 	iomap = -1;
271 	if (!i915_gem_object_has_struct_page(obj)) {
272 		iomap = obj->mm.region->iomap.base;
273 		iomap -= obj->mm.region->region.start;
274 	}
275 
276 	/* PTEs are revoked in obj->ops->put_pages() */
277 	err = remap_io_sg(area,
278 			  area->vm_start, area->vm_end - area->vm_start,
279 			  obj->mm.pages->sgl, iomap);
280 
281 	if (area->vm_flags & VM_WRITE) {
282 		GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
283 		obj->mm.dirty = true;
284 	}
285 
286 	i915_gem_object_unpin_pages(obj);
287 
288 out:
289 	i915_gem_object_unlock(obj);
290 	return i915_error_to_vmf_fault(err);
291 }
292 
293 static vm_fault_t vm_fault_gtt(struct vm_fault *vmf)
294 {
295 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
296 	struct vm_area_struct *area = vmf->vma;
297 	struct i915_mmap_offset *mmo = area->vm_private_data;
298 	struct drm_i915_gem_object *obj = mmo->obj;
299 	struct drm_device *dev = obj->base.dev;
300 	struct drm_i915_private *i915 = to_i915(dev);
301 	struct intel_runtime_pm *rpm = &i915->runtime_pm;
302 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
303 	bool write = area->vm_flags & VM_WRITE;
304 	struct i915_gem_ww_ctx ww;
305 	intel_wakeref_t wakeref;
306 	struct i915_vma *vma;
307 	pgoff_t page_offset;
308 	int srcu;
309 	int ret;
310 
311 	/* We don't use vmf->pgoff since that has the fake offset */
312 	page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
313 
314 	trace_i915_gem_object_fault(obj, page_offset, true, write);
315 
316 	wakeref = intel_runtime_pm_get(rpm);
317 
318 	i915_gem_ww_ctx_init(&ww, true);
319 retry:
320 	ret = i915_gem_object_lock(obj, &ww);
321 	if (ret)
322 		goto err_rpm;
323 
324 	/* Sanity check that we allow writing into this object */
325 	if (i915_gem_object_is_readonly(obj) && write) {
326 		ret = -EFAULT;
327 		goto err_rpm;
328 	}
329 
330 	ret = i915_gem_object_pin_pages(obj);
331 	if (ret)
332 		goto err_rpm;
333 
334 	ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu);
335 	if (ret)
336 		goto err_pages;
337 
338 	/* Now pin it into the GTT as needed */
339 	vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
340 					  PIN_MAPPABLE |
341 					  PIN_NONBLOCK /* NOWARN */ |
342 					  PIN_NOEVICT);
343 	if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
344 		/* Use a partial view if it is bigger than available space */
345 		struct i915_gtt_view view =
346 			compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
347 		unsigned int flags;
348 
349 		flags = PIN_MAPPABLE | PIN_NOSEARCH;
350 		if (view.type == I915_GTT_VIEW_NORMAL)
351 			flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
352 
353 		/*
354 		 * Userspace is now writing through an untracked VMA, abandon
355 		 * all hope that the hardware is able to track future writes.
356 		 */
357 
358 		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
359 		if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
360 			flags = PIN_MAPPABLE;
361 			view.type = I915_GTT_VIEW_PARTIAL;
362 			vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
363 		}
364 
365 		/*
366 		 * The entire mappable GGTT is pinned? Unexpected!
367 		 * Try to evict the object we locked too, as normally we skip it
368 		 * due to lack of short term pinning inside execbuf.
369 		 */
370 		if (vma == ERR_PTR(-ENOSPC)) {
371 			ret = mutex_lock_interruptible(&ggtt->vm.mutex);
372 			if (!ret) {
373 				ret = i915_gem_evict_vm(&ggtt->vm, &ww, NULL);
374 				mutex_unlock(&ggtt->vm.mutex);
375 			}
376 			if (ret)
377 				goto err_reset;
378 			vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
379 		}
380 	}
381 	if (IS_ERR(vma)) {
382 		ret = PTR_ERR(vma);
383 		goto err_reset;
384 	}
385 
386 	/* Access to snoopable pages through the GTT is incoherent. */
387 	/*
388 	 * For objects created by userspace through GEM_CREATE with pat_index
389 	 * set by set_pat extension, coherency is managed by userspace, make
390 	 * sure we don't fail handling the vm fault by calling
391 	 * i915_gem_object_has_cache_level() which always return true for such
392 	 * objects. Otherwise this helper function would fall back to checking
393 	 * whether the object is un-cached.
394 	 */
395 	if (!(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||
396 	      HAS_LLC(i915))) {
397 		ret = -EFAULT;
398 		goto err_unpin;
399 	}
400 
401 	ret = i915_vma_pin_fence(vma);
402 	if (ret)
403 		goto err_unpin;
404 
405 	/* Finally, remap it using the new GTT offset */
406 	ret = remap_io_mapping(area,
407 			       area->vm_start + (vma->gtt_view.partial.offset << PAGE_SHIFT),
408 			       (ggtt->gmadr.start + i915_ggtt_offset(vma)) >> PAGE_SHIFT,
409 			       min_t(u64, vma->size, area->vm_end - area->vm_start),
410 			       &ggtt->iomap);
411 	if (ret)
412 		goto err_fence;
413 
414 	assert_rpm_wakelock_held(rpm);
415 
416 	/* Mark as being mmapped into userspace for later revocation */
417 	mutex_lock(&to_gt(i915)->ggtt->vm.mutex);
418 	if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
419 		list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list);
420 	mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);
421 
422 	/* Track the mmo associated with the fenced vma */
423 	vma->mmo = mmo;
424 
425 	if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
426 		intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref,
427 				   msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
428 
429 	if (write) {
430 		GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
431 		i915_vma_set_ggtt_write(vma);
432 		obj->mm.dirty = true;
433 	}
434 
435 err_fence:
436 	i915_vma_unpin_fence(vma);
437 err_unpin:
438 	__i915_vma_unpin(vma);
439 err_reset:
440 	intel_gt_reset_unlock(ggtt->vm.gt, srcu);
441 err_pages:
442 	i915_gem_object_unpin_pages(obj);
443 err_rpm:
444 	if (ret == -EDEADLK) {
445 		ret = i915_gem_ww_ctx_backoff(&ww);
446 		if (!ret)
447 			goto retry;
448 	}
449 	i915_gem_ww_ctx_fini(&ww);
450 	intel_runtime_pm_put(rpm, wakeref);
451 	return i915_error_to_vmf_fault(ret);
452 }
453 
454 static int
455 vm_access(struct vm_area_struct *area, unsigned long addr,
456 	  void *buf, int len, int write)
457 {
458 	struct i915_mmap_offset *mmo = area->vm_private_data;
459 	struct drm_i915_gem_object *obj = mmo->obj;
460 	struct i915_gem_ww_ctx ww;
461 	void *vaddr;
462 	int err = 0;
463 
464 	if (i915_gem_object_is_readonly(obj) && write)
465 		return -EACCES;
466 
467 	addr -= area->vm_start;
468 	if (range_overflows_t(u64, addr, len, obj->base.size))
469 		return -EINVAL;
470 
471 	i915_gem_ww_ctx_init(&ww, true);
472 retry:
473 	err = i915_gem_object_lock(obj, &ww);
474 	if (err)
475 		goto out;
476 
477 	/* As this is primarily for debugging, let's focus on simplicity */
478 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC);
479 	if (IS_ERR(vaddr)) {
480 		err = PTR_ERR(vaddr);
481 		goto out;
482 	}
483 
484 	if (write) {
485 		memcpy(vaddr + addr, buf, len);
486 		__i915_gem_object_flush_map(obj, addr, len);
487 	} else {
488 		memcpy(buf, vaddr + addr, len);
489 	}
490 
491 	i915_gem_object_unpin_map(obj);
492 out:
493 	if (err == -EDEADLK) {
494 		err = i915_gem_ww_ctx_backoff(&ww);
495 		if (!err)
496 			goto retry;
497 	}
498 	i915_gem_ww_ctx_fini(&ww);
499 
500 	if (err)
501 		return err;
502 
503 	return len;
504 }
505 
506 void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
507 {
508 	struct i915_vma *vma;
509 
510 	GEM_BUG_ON(!obj->userfault_count);
511 
512 	for_each_ggtt_vma(vma, obj)
513 		i915_vma_revoke_mmap(vma);
514 
515 	GEM_BUG_ON(obj->userfault_count);
516 }
517 
518 /*
519  * It is vital that we remove the page mapping if we have mapped a tiled
520  * object through the GTT and then lose the fence register due to
521  * resource pressure. Similarly if the object has been moved out of the
522  * aperture, than pages mapped into userspace must be revoked. Removing the
523  * mapping will then trigger a page fault on the next user access, allowing
524  * fixup by vm_fault_gtt().
525  */
526 void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
527 {
528 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
529 	intel_wakeref_t wakeref;
530 
531 	/*
532 	 * Serialisation between user GTT access and our code depends upon
533 	 * revoking the CPU's PTE whilst the mutex is held. The next user
534 	 * pagefault then has to wait until we release the mutex.
535 	 *
536 	 * Note that RPM complicates somewhat by adding an additional
537 	 * requirement that operations to the GGTT be made holding the RPM
538 	 * wakeref.
539 	 */
540 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
541 	mutex_lock(&to_gt(i915)->ggtt->vm.mutex);
542 
543 	if (!obj->userfault_count)
544 		goto out;
545 
546 	__i915_gem_object_release_mmap_gtt(obj);
547 
548 	/*
549 	 * Ensure that the CPU's PTE are revoked and there are not outstanding
550 	 * memory transactions from userspace before we return. The TLB
551 	 * flushing implied above by changing the PTE above *should* be
552 	 * sufficient, an extra barrier here just provides us with a bit
553 	 * of paranoid documentation about our requirement to serialise
554 	 * memory writes before touching registers / GSM.
555 	 */
556 	wmb();
557 
558 out:
559 	mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);
560 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
561 }
562 
563 void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj)
564 {
565 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
566 	struct ttm_device *bdev = bo->bdev;
567 
568 	drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
569 
570 	/*
571 	 * We have exclusive access here via runtime suspend. All other callers
572 	 * must first grab the rpm wakeref.
573 	 */
574 	GEM_BUG_ON(!obj->userfault_count);
575 	list_del(&obj->userfault_link);
576 	obj->userfault_count = 0;
577 }
578 
579 void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj)
580 {
581 	struct i915_mmap_offset *mmo, *mn;
582 
583 	if (obj->ops->unmap_virtual)
584 		obj->ops->unmap_virtual(obj);
585 
586 	spin_lock(&obj->mmo.lock);
587 	rbtree_postorder_for_each_entry_safe(mmo, mn,
588 					     &obj->mmo.offsets, offset) {
589 		/*
590 		 * vma_node_unmap for GTT mmaps handled already in
591 		 * __i915_gem_object_release_mmap_gtt
592 		 */
593 		if (mmo->mmap_type == I915_MMAP_TYPE_GTT)
594 			continue;
595 
596 		spin_unlock(&obj->mmo.lock);
597 		drm_vma_node_unmap(&mmo->vma_node,
598 				   obj->base.dev->anon_inode->i_mapping);
599 		spin_lock(&obj->mmo.lock);
600 	}
601 	spin_unlock(&obj->mmo.lock);
602 }
603 
604 static struct i915_mmap_offset *
605 lookup_mmo(struct drm_i915_gem_object *obj,
606 	   enum i915_mmap_type mmap_type)
607 {
608 	struct rb_node *rb;
609 
610 	spin_lock(&obj->mmo.lock);
611 	rb = obj->mmo.offsets.rb_node;
612 	while (rb) {
613 		struct i915_mmap_offset *mmo =
614 			rb_entry(rb, typeof(*mmo), offset);
615 
616 		if (mmo->mmap_type == mmap_type) {
617 			spin_unlock(&obj->mmo.lock);
618 			return mmo;
619 		}
620 
621 		if (mmo->mmap_type < mmap_type)
622 			rb = rb->rb_right;
623 		else
624 			rb = rb->rb_left;
625 	}
626 	spin_unlock(&obj->mmo.lock);
627 
628 	return NULL;
629 }
630 
631 static struct i915_mmap_offset *
632 insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo)
633 {
634 	struct rb_node *rb, **p;
635 
636 	spin_lock(&obj->mmo.lock);
637 	rb = NULL;
638 	p = &obj->mmo.offsets.rb_node;
639 	while (*p) {
640 		struct i915_mmap_offset *pos;
641 
642 		rb = *p;
643 		pos = rb_entry(rb, typeof(*pos), offset);
644 
645 		if (pos->mmap_type == mmo->mmap_type) {
646 			spin_unlock(&obj->mmo.lock);
647 			drm_vma_offset_remove(obj->base.dev->vma_offset_manager,
648 					      &mmo->vma_node);
649 			kfree(mmo);
650 			return pos;
651 		}
652 
653 		if (pos->mmap_type < mmo->mmap_type)
654 			p = &rb->rb_right;
655 		else
656 			p = &rb->rb_left;
657 	}
658 	rb_link_node(&mmo->offset, rb, p);
659 	rb_insert_color(&mmo->offset, &obj->mmo.offsets);
660 	spin_unlock(&obj->mmo.lock);
661 
662 	return mmo;
663 }
664 
665 static struct i915_mmap_offset *
666 mmap_offset_attach(struct drm_i915_gem_object *obj,
667 		   enum i915_mmap_type mmap_type,
668 		   struct drm_file *file)
669 {
670 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
671 	struct i915_mmap_offset *mmo;
672 	int err;
673 
674 	GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops);
675 
676 	mmo = lookup_mmo(obj, mmap_type);
677 	if (mmo)
678 		goto out;
679 
680 	mmo = kmalloc(sizeof(*mmo), GFP_KERNEL);
681 	if (!mmo)
682 		return ERR_PTR(-ENOMEM);
683 
684 	mmo->obj = obj;
685 	mmo->mmap_type = mmap_type;
686 	drm_vma_node_reset(&mmo->vma_node);
687 
688 	err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
689 				 &mmo->vma_node, obj->base.size / PAGE_SIZE);
690 	if (likely(!err))
691 		goto insert;
692 
693 	/* Attempt to reap some mmap space from dead objects */
694 	err = intel_gt_retire_requests_timeout(to_gt(i915), MAX_SCHEDULE_TIMEOUT,
695 					       NULL);
696 	if (err)
697 		goto err;
698 
699 	i915_gem_drain_freed_objects(i915);
700 	err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
701 				 &mmo->vma_node, obj->base.size / PAGE_SIZE);
702 	if (err)
703 		goto err;
704 
705 insert:
706 	mmo = insert_mmo(obj, mmo);
707 	GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo);
708 out:
709 	if (file)
710 		drm_vma_node_allow_once(&mmo->vma_node, file);
711 	return mmo;
712 
713 err:
714 	kfree(mmo);
715 	return ERR_PTR(err);
716 }
717 
718 static int
719 __assign_mmap_offset(struct drm_i915_gem_object *obj,
720 		     enum i915_mmap_type mmap_type,
721 		     u64 *offset, struct drm_file *file)
722 {
723 	struct i915_mmap_offset *mmo;
724 
725 	if (i915_gem_object_never_mmap(obj))
726 		return -ENODEV;
727 
728 	if (obj->ops->mmap_offset)  {
729 		if (mmap_type != I915_MMAP_TYPE_FIXED)
730 			return -ENODEV;
731 
732 		*offset = obj->ops->mmap_offset(obj);
733 		return 0;
734 	}
735 
736 	if (mmap_type == I915_MMAP_TYPE_FIXED)
737 		return -ENODEV;
738 
739 	if (mmap_type != I915_MMAP_TYPE_GTT &&
740 	    !i915_gem_object_has_struct_page(obj) &&
741 	    !i915_gem_object_has_iomem(obj))
742 		return -ENODEV;
743 
744 	mmo = mmap_offset_attach(obj, mmap_type, file);
745 	if (IS_ERR(mmo))
746 		return PTR_ERR(mmo);
747 
748 	*offset = drm_vma_node_offset_addr(&mmo->vma_node);
749 	return 0;
750 }
751 
752 static int
753 __assign_mmap_offset_handle(struct drm_file *file,
754 			    u32 handle,
755 			    enum i915_mmap_type mmap_type,
756 			    u64 *offset)
757 {
758 	struct drm_i915_gem_object *obj;
759 	int err;
760 
761 	obj = i915_gem_object_lookup(file, handle);
762 	if (!obj)
763 		return -ENOENT;
764 
765 	err = i915_gem_object_lock_interruptible(obj, NULL);
766 	if (err)
767 		goto out_put;
768 	err = __assign_mmap_offset(obj, mmap_type, offset, file);
769 	i915_gem_object_unlock(obj);
770 out_put:
771 	i915_gem_object_put(obj);
772 	return err;
773 }
774 
775 int
776 i915_gem_dumb_mmap_offset(struct drm_file *file,
777 			  struct drm_device *dev,
778 			  u32 handle,
779 			  u64 *offset)
780 {
781 	struct drm_i915_private *i915 = to_i915(dev);
782 	enum i915_mmap_type mmap_type;
783 
784 	if (HAS_LMEM(to_i915(dev)))
785 		mmap_type = I915_MMAP_TYPE_FIXED;
786 	else if (pat_enabled())
787 		mmap_type = I915_MMAP_TYPE_WC;
788 	else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
789 		return -ENODEV;
790 	else
791 		mmap_type = I915_MMAP_TYPE_GTT;
792 
793 	return __assign_mmap_offset_handle(file, handle, mmap_type, offset);
794 }
795 
796 /**
797  * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing
798  * @dev: DRM device
799  * @data: GTT mapping ioctl data
800  * @file: GEM object info
801  *
802  * Simply returns the fake offset to userspace so it can mmap it.
803  * The mmap call will end up in drm_gem_mmap(), which will set things
804  * up so we can get faults in the handler above.
805  *
806  * The fault handler will take care of binding the object into the GTT
807  * (since it may have been evicted to make room for something), allocating
808  * a fence register, and mapping the appropriate aperture address into
809  * userspace.
810  */
811 int
812 i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
813 			   struct drm_file *file)
814 {
815 	struct drm_i915_private *i915 = to_i915(dev);
816 	struct drm_i915_gem_mmap_offset *args = data;
817 	enum i915_mmap_type type;
818 	int err;
819 
820 	/*
821 	 * Historically we failed to check args.pad and args.offset
822 	 * and so we cannot use those fields for user input and we cannot
823 	 * add -EINVAL for them as the ABI is fixed, i.e. old userspace
824 	 * may be feeding in garbage in those fields.
825 	 *
826 	 * if (args->pad) return -EINVAL; is verbotten!
827 	 */
828 
829 	err = i915_user_extensions(u64_to_user_ptr(args->extensions),
830 				   NULL, 0, NULL);
831 	if (err)
832 		return err;
833 
834 	switch (args->flags) {
835 	case I915_MMAP_OFFSET_GTT:
836 		if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
837 			return -ENODEV;
838 		type = I915_MMAP_TYPE_GTT;
839 		break;
840 
841 	case I915_MMAP_OFFSET_WC:
842 		if (!pat_enabled())
843 			return -ENODEV;
844 		type = I915_MMAP_TYPE_WC;
845 		break;
846 
847 	case I915_MMAP_OFFSET_WB:
848 		type = I915_MMAP_TYPE_WB;
849 		break;
850 
851 	case I915_MMAP_OFFSET_UC:
852 		if (!pat_enabled())
853 			return -ENODEV;
854 		type = I915_MMAP_TYPE_UC;
855 		break;
856 
857 	case I915_MMAP_OFFSET_FIXED:
858 		type = I915_MMAP_TYPE_FIXED;
859 		break;
860 
861 	default:
862 		return -EINVAL;
863 	}
864 
865 	return __assign_mmap_offset_handle(file, args->handle, type, &args->offset);
866 }
867 
868 static void vm_open(struct vm_area_struct *vma)
869 {
870 	struct i915_mmap_offset *mmo = vma->vm_private_data;
871 	struct drm_i915_gem_object *obj = mmo->obj;
872 
873 	GEM_BUG_ON(!obj);
874 	i915_gem_object_get(obj);
875 }
876 
877 static void vm_close(struct vm_area_struct *vma)
878 {
879 	struct i915_mmap_offset *mmo = vma->vm_private_data;
880 	struct drm_i915_gem_object *obj = mmo->obj;
881 
882 	GEM_BUG_ON(!obj);
883 	i915_gem_object_put(obj);
884 }
885 
886 static const struct vm_operations_struct vm_ops_gtt = {
887 	.fault = vm_fault_gtt,
888 	.access = vm_access,
889 	.open = vm_open,
890 	.close = vm_close,
891 };
892 
893 static const struct vm_operations_struct vm_ops_cpu = {
894 	.fault = vm_fault_cpu,
895 	.access = vm_access,
896 	.open = vm_open,
897 	.close = vm_close,
898 };
899 
900 static int singleton_release(struct inode *inode, struct file *file)
901 {
902 	struct drm_i915_private *i915 = file->private_data;
903 
904 	cmpxchg(&i915->gem.mmap_singleton, file, NULL);
905 	drm_dev_put(&i915->drm);
906 
907 	return 0;
908 }
909 
910 static const struct file_operations singleton_fops = {
911 	.owner = THIS_MODULE,
912 	.release = singleton_release,
913 };
914 
915 static struct file *mmap_singleton(struct drm_i915_private *i915)
916 {
917 	struct file *file;
918 
919 	rcu_read_lock();
920 	file = READ_ONCE(i915->gem.mmap_singleton);
921 	if (file && !get_file_rcu(file))
922 		file = NULL;
923 	rcu_read_unlock();
924 	if (file)
925 		return file;
926 
927 	file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR);
928 	if (IS_ERR(file))
929 		return file;
930 
931 	/* Everyone shares a single global address space */
932 	file->f_mapping = i915->drm.anon_inode->i_mapping;
933 
934 	smp_store_mb(i915->gem.mmap_singleton, file);
935 	drm_dev_get(&i915->drm);
936 
937 	return file;
938 }
939 
940 static int
941 i915_gem_object_mmap(struct drm_i915_gem_object *obj,
942 		     struct i915_mmap_offset *mmo,
943 		     struct vm_area_struct *vma)
944 {
945 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
946 	struct drm_device *dev = &i915->drm;
947 	struct file *anon;
948 
949 	if (i915_gem_object_is_readonly(obj)) {
950 		if (vma->vm_flags & VM_WRITE) {
951 			i915_gem_object_put(obj);
952 			return -EINVAL;
953 		}
954 		vm_flags_clear(vma, VM_MAYWRITE);
955 	}
956 
957 	anon = mmap_singleton(to_i915(dev));
958 	if (IS_ERR(anon)) {
959 		i915_gem_object_put(obj);
960 		return PTR_ERR(anon);
961 	}
962 
963 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
964 
965 	/*
966 	 * We keep the ref on mmo->obj, not vm_file, but we require
967 	 * vma->vm_file->f_mapping, see vma_link(), for later revocation.
968 	 * Our userspace is accustomed to having per-file resource cleanup
969 	 * (i.e. contexts, objects and requests) on their close(fd), which
970 	 * requires avoiding extraneous references to their filp, hence why
971 	 * we prefer to use an anonymous file for their mmaps.
972 	 */
973 	vma_set_file(vma, anon);
974 	/* Drop the initial creation reference, the vma is now holding one. */
975 	fput(anon);
976 
977 	if (obj->ops->mmap_ops) {
978 		vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags));
979 		vma->vm_ops = obj->ops->mmap_ops;
980 		vma->vm_private_data = obj->base.vma_node.driver_private;
981 		return 0;
982 	}
983 
984 	vma->vm_private_data = mmo;
985 
986 	switch (mmo->mmap_type) {
987 	case I915_MMAP_TYPE_WC:
988 		vma->vm_page_prot =
989 			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
990 		vma->vm_ops = &vm_ops_cpu;
991 		break;
992 
993 	case I915_MMAP_TYPE_FIXED:
994 		GEM_WARN_ON(1);
995 		fallthrough;
996 	case I915_MMAP_TYPE_WB:
997 		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
998 		vma->vm_ops = &vm_ops_cpu;
999 		break;
1000 
1001 	case I915_MMAP_TYPE_UC:
1002 		vma->vm_page_prot =
1003 			pgprot_noncached(vm_get_page_prot(vma->vm_flags));
1004 		vma->vm_ops = &vm_ops_cpu;
1005 		break;
1006 
1007 	case I915_MMAP_TYPE_GTT:
1008 		vma->vm_page_prot =
1009 			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1010 		vma->vm_ops = &vm_ops_gtt;
1011 		break;
1012 	}
1013 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1014 
1015 	return 0;
1016 }
1017 
1018 /*
1019  * This overcomes the limitation in drm_gem_mmap's assignment of a
1020  * drm_gem_object as the vma->vm_private_data. Since we need to
1021  * be able to resolve multiple mmap offsets which could be tied
1022  * to a single gem object.
1023  */
1024 int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1025 {
1026 	struct drm_vma_offset_node *node;
1027 	struct drm_file *priv = filp->private_data;
1028 	struct drm_device *dev = priv->minor->dev;
1029 	struct drm_i915_gem_object *obj = NULL;
1030 	struct i915_mmap_offset *mmo = NULL;
1031 
1032 	if (drm_dev_is_unplugged(dev))
1033 		return -ENODEV;
1034 
1035 	rcu_read_lock();
1036 	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
1037 	node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
1038 						  vma->vm_pgoff,
1039 						  vma_pages(vma));
1040 	if (node && drm_vma_node_is_allowed(node, priv)) {
1041 		/*
1042 		 * Skip 0-refcnted objects as it is in the process of being
1043 		 * destroyed and will be invalid when the vma manager lock
1044 		 * is released.
1045 		 */
1046 		if (!node->driver_private) {
1047 			mmo = container_of(node, struct i915_mmap_offset, vma_node);
1048 			obj = i915_gem_object_get_rcu(mmo->obj);
1049 
1050 			GEM_BUG_ON(obj && obj->ops->mmap_ops);
1051 		} else {
1052 			obj = i915_gem_object_get_rcu
1053 				(container_of(node, struct drm_i915_gem_object,
1054 					      base.vma_node));
1055 
1056 			GEM_BUG_ON(obj && !obj->ops->mmap_ops);
1057 		}
1058 	}
1059 	drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
1060 	rcu_read_unlock();
1061 	if (!obj)
1062 		return node ? -EACCES : -EINVAL;
1063 
1064 	return i915_gem_object_mmap(obj, mmo, vma);
1065 }
1066 
1067 int i915_gem_fb_mmap(struct drm_i915_gem_object *obj, struct vm_area_struct *vma)
1068 {
1069 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1070 	struct drm_device *dev = &i915->drm;
1071 	struct i915_mmap_offset *mmo = NULL;
1072 	enum i915_mmap_type mmap_type;
1073 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1074 
1075 	if (drm_dev_is_unplugged(dev))
1076 		return -ENODEV;
1077 
1078 	/* handle ttm object */
1079 	if (obj->ops->mmap_ops) {
1080 		/*
1081 		 * ttm fault handler, ttm_bo_vm_fault_reserved() uses fake offset
1082 		 * to calculate page offset so set that up.
1083 		 */
1084 		vma->vm_pgoff += drm_vma_node_start(&obj->base.vma_node);
1085 	} else {
1086 		/* handle stolen and smem objects */
1087 		mmap_type = i915_ggtt_has_aperture(ggtt) ? I915_MMAP_TYPE_GTT : I915_MMAP_TYPE_WC;
1088 		mmo = mmap_offset_attach(obj, mmap_type, NULL);
1089 		if (IS_ERR(mmo))
1090 			return PTR_ERR(mmo);
1091 	}
1092 
1093 	/*
1094 	 * When we install vm_ops for mmap we are too late for
1095 	 * the vm_ops->open() which increases the ref_count of
1096 	 * this obj and then it gets decreased by the vm_ops->close().
1097 	 * To balance this increase the obj ref_count here.
1098 	 */
1099 	obj = i915_gem_object_get(obj);
1100 	return i915_gem_object_mmap(obj, mmo, vma);
1101 }
1102 
1103 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1104 #include "selftests/i915_gem_mman.c"
1105 #endif
1106