1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include <linux/anon_inodes.h>
8 #include <linux/mman.h>
9 #include <linux/pfn_t.h>
10 #include <linux/sizes.h>
11 
12 #include "gt/intel_gt.h"
13 #include "gt/intel_gt_requests.h"
14 
15 #include "i915_drv.h"
16 #include "i915_gem_gtt.h"
17 #include "i915_gem_ioctls.h"
18 #include "i915_gem_object.h"
19 #include "i915_gem_mman.h"
20 #include "i915_trace.h"
21 #include "i915_user_extensions.h"
22 #include "i915_gem_ttm.h"
23 #include "i915_vma.h"
24 
25 static inline bool
26 __vma_matches(struct vm_area_struct *vma, struct file *filp,
27 	      unsigned long addr, unsigned long size)
28 {
29 	if (vma->vm_file != filp)
30 		return false;
31 
32 	return vma->vm_start == addr &&
33 	       (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
34 }
35 
36 /**
37  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
38  *			 it is mapped to.
39  * @dev: drm device
40  * @data: ioctl data blob
41  * @file: drm file
42  *
43  * While the mapping holds a reference on the contents of the object, it doesn't
44  * imply a ref on the object itself.
45  *
46  * IMPORTANT:
47  *
48  * DRM driver writers who look a this function as an example for how to do GEM
49  * mmap support, please don't implement mmap support like here. The modern way
50  * to implement DRM mmap support is with an mmap offset ioctl (like
51  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
52  * That way debug tooling like valgrind will understand what's going on, hiding
53  * the mmap call in a driver private ioctl will break that. The i915 driver only
54  * does cpu mmaps this way because we didn't know better.
55  */
56 int
57 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
58 		    struct drm_file *file)
59 {
60 	struct drm_i915_private *i915 = to_i915(dev);
61 	struct drm_i915_gem_mmap *args = data;
62 	struct drm_i915_gem_object *obj;
63 	unsigned long addr;
64 
65 	/*
66 	 * mmap ioctl is disallowed for all discrete platforms,
67 	 * and for all platforms with GRAPHICS_VER > 12.
68 	 */
69 	if (IS_DGFX(i915) || GRAPHICS_VER(i915) > 12)
70 		return -EOPNOTSUPP;
71 
72 	if (args->flags & ~(I915_MMAP_WC))
73 		return -EINVAL;
74 
75 	if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
76 		return -ENODEV;
77 
78 	obj = i915_gem_object_lookup(file, args->handle);
79 	if (!obj)
80 		return -ENOENT;
81 
82 	/* prime objects have no backing filp to GEM mmap
83 	 * pages from.
84 	 */
85 	if (!obj->base.filp) {
86 		addr = -ENXIO;
87 		goto err;
88 	}
89 
90 	if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
91 		addr = -EINVAL;
92 		goto err;
93 	}
94 
95 	addr = vm_mmap(obj->base.filp, 0, args->size,
96 		       PROT_READ | PROT_WRITE, MAP_SHARED,
97 		       args->offset);
98 	if (IS_ERR_VALUE(addr))
99 		goto err;
100 
101 	if (args->flags & I915_MMAP_WC) {
102 		struct mm_struct *mm = current->mm;
103 		struct vm_area_struct *vma;
104 
105 		if (mmap_write_lock_killable(mm)) {
106 			addr = -EINTR;
107 			goto err;
108 		}
109 		vma = find_vma(mm, addr);
110 		if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
111 			vma->vm_page_prot =
112 				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
113 		else
114 			addr = -ENOMEM;
115 		mmap_write_unlock(mm);
116 		if (IS_ERR_VALUE(addr))
117 			goto err;
118 	}
119 	i915_gem_object_put(obj);
120 
121 	args->addr_ptr = (u64)addr;
122 	return 0;
123 
124 err:
125 	i915_gem_object_put(obj);
126 	return addr;
127 }
128 
129 static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
130 {
131 	return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
132 }
133 
134 /**
135  * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
136  *
137  * A history of the GTT mmap interface:
138  *
139  * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
140  *     aligned and suitable for fencing, and still fit into the available
141  *     mappable space left by the pinned display objects. A classic problem
142  *     we called the page-fault-of-doom where we would ping-pong between
143  *     two objects that could not fit inside the GTT and so the memcpy
144  *     would page one object in at the expense of the other between every
145  *     single byte.
146  *
147  * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
148  *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
149  *     object is too large for the available space (or simply too large
150  *     for the mappable aperture!), a view is created instead and faulted
151  *     into userspace. (This view is aligned and sized appropriately for
152  *     fenced access.)
153  *
154  * 2 - Recognise WC as a separate cache domain so that we can flush the
155  *     delayed writes via GTT before performing direct access via WC.
156  *
157  * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
158  *     pagefault; swapin remains transparent.
159  *
160  * 4 - Support multiple fault handlers per object depending on object's
161  *     backing storage (a.k.a. MMAP_OFFSET).
162  *
163  * Restrictions:
164  *
165  *  * snoopable objects cannot be accessed via the GTT. It can cause machine
166  *    hangs on some architectures, corruption on others. An attempt to service
167  *    a GTT page fault from a snoopable object will generate a SIGBUS.
168  *
169  *  * the object must be able to fit into RAM (physical memory, though no
170  *    limited to the mappable aperture).
171  *
172  *
173  * Caveats:
174  *
175  *  * a new GTT page fault will synchronize rendering from the GPU and flush
176  *    all data to system memory. Subsequent access will not be synchronized.
177  *
178  *  * all mappings are revoked on runtime device suspend.
179  *
180  *  * there are only 8, 16 or 32 fence registers to share between all users
181  *    (older machines require fence register for display and blitter access
182  *    as well). Contention of the fence registers will cause the previous users
183  *    to be unmapped and any new access will generate new page faults.
184  *
185  *  * running out of memory while servicing a fault may generate a SIGBUS,
186  *    rather than the expected SIGSEGV.
187  */
188 int i915_gem_mmap_gtt_version(void)
189 {
190 	return 4;
191 }
192 
193 static inline struct i915_ggtt_view
194 compute_partial_view(const struct drm_i915_gem_object *obj,
195 		     pgoff_t page_offset,
196 		     unsigned int chunk)
197 {
198 	struct i915_ggtt_view view;
199 
200 	if (i915_gem_object_is_tiled(obj))
201 		chunk = roundup(chunk, tile_row_pages(obj) ?: 1);
202 
203 	view.type = I915_GGTT_VIEW_PARTIAL;
204 	view.partial.offset = rounddown(page_offset, chunk);
205 	view.partial.size =
206 		min_t(unsigned int, chunk,
207 		      (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
208 
209 	/* If the partial covers the entire object, just create a normal VMA. */
210 	if (chunk >= obj->base.size >> PAGE_SHIFT)
211 		view.type = I915_GGTT_VIEW_NORMAL;
212 
213 	return view;
214 }
215 
216 static vm_fault_t i915_error_to_vmf_fault(int err)
217 {
218 	switch (err) {
219 	default:
220 		WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err);
221 		fallthrough;
222 	case -EIO: /* shmemfs failure from swap device */
223 	case -EFAULT: /* purged object */
224 	case -ENODEV: /* bad object, how did you get here! */
225 	case -ENXIO: /* unable to access backing store (on device) */
226 		return VM_FAULT_SIGBUS;
227 
228 	case -ENOMEM: /* our allocation failure */
229 		return VM_FAULT_OOM;
230 
231 	case 0:
232 	case -EAGAIN:
233 	case -ENOSPC: /* transient failure to evict? */
234 	case -ERESTARTSYS:
235 	case -EINTR:
236 	case -EBUSY:
237 		/*
238 		 * EBUSY is ok: this just means that another thread
239 		 * already did the job.
240 		 */
241 		return VM_FAULT_NOPAGE;
242 	}
243 }
244 
245 static vm_fault_t vm_fault_cpu(struct vm_fault *vmf)
246 {
247 	struct vm_area_struct *area = vmf->vma;
248 	struct i915_mmap_offset *mmo = area->vm_private_data;
249 	struct drm_i915_gem_object *obj = mmo->obj;
250 	resource_size_t iomap;
251 	int err;
252 
253 	/* Sanity check that we allow writing into this object */
254 	if (unlikely(i915_gem_object_is_readonly(obj) &&
255 		     area->vm_flags & VM_WRITE))
256 		return VM_FAULT_SIGBUS;
257 
258 	if (i915_gem_object_lock_interruptible(obj, NULL))
259 		return VM_FAULT_NOPAGE;
260 
261 	err = i915_gem_object_pin_pages(obj);
262 	if (err)
263 		goto out;
264 
265 	iomap = -1;
266 	if (!i915_gem_object_has_struct_page(obj)) {
267 		iomap = obj->mm.region->iomap.base;
268 		iomap -= obj->mm.region->region.start;
269 	}
270 
271 	/* PTEs are revoked in obj->ops->put_pages() */
272 	err = remap_io_sg(area,
273 			  area->vm_start, area->vm_end - area->vm_start,
274 			  obj->mm.pages->sgl, iomap);
275 
276 	if (area->vm_flags & VM_WRITE) {
277 		GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
278 		obj->mm.dirty = true;
279 	}
280 
281 	i915_gem_object_unpin_pages(obj);
282 
283 out:
284 	i915_gem_object_unlock(obj);
285 	return i915_error_to_vmf_fault(err);
286 }
287 
288 static vm_fault_t vm_fault_gtt(struct vm_fault *vmf)
289 {
290 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
291 	struct vm_area_struct *area = vmf->vma;
292 	struct i915_mmap_offset *mmo = area->vm_private_data;
293 	struct drm_i915_gem_object *obj = mmo->obj;
294 	struct drm_device *dev = obj->base.dev;
295 	struct drm_i915_private *i915 = to_i915(dev);
296 	struct intel_runtime_pm *rpm = &i915->runtime_pm;
297 	struct i915_ggtt *ggtt = &i915->ggtt;
298 	bool write = area->vm_flags & VM_WRITE;
299 	struct i915_gem_ww_ctx ww;
300 	intel_wakeref_t wakeref;
301 	struct i915_vma *vma;
302 	pgoff_t page_offset;
303 	int srcu;
304 	int ret;
305 
306 	/* We don't use vmf->pgoff since that has the fake offset */
307 	page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
308 
309 	trace_i915_gem_object_fault(obj, page_offset, true, write);
310 
311 	wakeref = intel_runtime_pm_get(rpm);
312 
313 	i915_gem_ww_ctx_init(&ww, true);
314 retry:
315 	ret = i915_gem_object_lock(obj, &ww);
316 	if (ret)
317 		goto err_rpm;
318 
319 	/* Sanity check that we allow writing into this object */
320 	if (i915_gem_object_is_readonly(obj) && write) {
321 		ret = -EFAULT;
322 		goto err_rpm;
323 	}
324 
325 	ret = i915_gem_object_pin_pages(obj);
326 	if (ret)
327 		goto err_rpm;
328 
329 	ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu);
330 	if (ret)
331 		goto err_pages;
332 
333 	/* Now pin it into the GTT as needed */
334 	vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
335 					  PIN_MAPPABLE |
336 					  PIN_NONBLOCK /* NOWARN */ |
337 					  PIN_NOEVICT);
338 	if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
339 		/* Use a partial view if it is bigger than available space */
340 		struct i915_ggtt_view view =
341 			compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
342 		unsigned int flags;
343 
344 		flags = PIN_MAPPABLE | PIN_NOSEARCH;
345 		if (view.type == I915_GGTT_VIEW_NORMAL)
346 			flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
347 
348 		/*
349 		 * Userspace is now writing through an untracked VMA, abandon
350 		 * all hope that the hardware is able to track future writes.
351 		 */
352 
353 		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
354 		if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
355 			flags = PIN_MAPPABLE;
356 			view.type = I915_GGTT_VIEW_PARTIAL;
357 			vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
358 		}
359 
360 		/* The entire mappable GGTT is pinned? Unexpected! */
361 		GEM_BUG_ON(vma == ERR_PTR(-ENOSPC));
362 	}
363 	if (IS_ERR(vma)) {
364 		ret = PTR_ERR(vma);
365 		goto err_reset;
366 	}
367 
368 	/* Access to snoopable pages through the GTT is incoherent. */
369 	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
370 		ret = -EFAULT;
371 		goto err_unpin;
372 	}
373 
374 	ret = i915_vma_pin_fence(vma);
375 	if (ret)
376 		goto err_unpin;
377 
378 	/* Finally, remap it using the new GTT offset */
379 	ret = remap_io_mapping(area,
380 			       area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
381 			       (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
382 			       min_t(u64, vma->size, area->vm_end - area->vm_start),
383 			       &ggtt->iomap);
384 	if (ret)
385 		goto err_fence;
386 
387 	assert_rpm_wakelock_held(rpm);
388 
389 	/* Mark as being mmapped into userspace for later revocation */
390 	mutex_lock(&i915->ggtt.vm.mutex);
391 	if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
392 		list_add(&obj->userfault_link, &i915->ggtt.userfault_list);
393 	mutex_unlock(&i915->ggtt.vm.mutex);
394 
395 	/* Track the mmo associated with the fenced vma */
396 	vma->mmo = mmo;
397 
398 	if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
399 		intel_wakeref_auto(&i915->ggtt.userfault_wakeref,
400 				   msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
401 
402 	if (write) {
403 		GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
404 		i915_vma_set_ggtt_write(vma);
405 		obj->mm.dirty = true;
406 	}
407 
408 err_fence:
409 	i915_vma_unpin_fence(vma);
410 err_unpin:
411 	__i915_vma_unpin(vma);
412 err_reset:
413 	intel_gt_reset_unlock(ggtt->vm.gt, srcu);
414 err_pages:
415 	i915_gem_object_unpin_pages(obj);
416 err_rpm:
417 	if (ret == -EDEADLK) {
418 		ret = i915_gem_ww_ctx_backoff(&ww);
419 		if (!ret)
420 			goto retry;
421 	}
422 	i915_gem_ww_ctx_fini(&ww);
423 	intel_runtime_pm_put(rpm, wakeref);
424 	return i915_error_to_vmf_fault(ret);
425 }
426 
427 static int
428 vm_access(struct vm_area_struct *area, unsigned long addr,
429 	  void *buf, int len, int write)
430 {
431 	struct i915_mmap_offset *mmo = area->vm_private_data;
432 	struct drm_i915_gem_object *obj = mmo->obj;
433 	struct i915_gem_ww_ctx ww;
434 	void *vaddr;
435 	int err = 0;
436 
437 	if (i915_gem_object_is_readonly(obj) && write)
438 		return -EACCES;
439 
440 	addr -= area->vm_start;
441 	if (addr >= obj->base.size)
442 		return -EINVAL;
443 
444 	i915_gem_ww_ctx_init(&ww, true);
445 retry:
446 	err = i915_gem_object_lock(obj, &ww);
447 	if (err)
448 		goto out;
449 
450 	/* As this is primarily for debugging, let's focus on simplicity */
451 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC);
452 	if (IS_ERR(vaddr)) {
453 		err = PTR_ERR(vaddr);
454 		goto out;
455 	}
456 
457 	if (write) {
458 		memcpy(vaddr + addr, buf, len);
459 		__i915_gem_object_flush_map(obj, addr, len);
460 	} else {
461 		memcpy(buf, vaddr + addr, len);
462 	}
463 
464 	i915_gem_object_unpin_map(obj);
465 out:
466 	if (err == -EDEADLK) {
467 		err = i915_gem_ww_ctx_backoff(&ww);
468 		if (!err)
469 			goto retry;
470 	}
471 	i915_gem_ww_ctx_fini(&ww);
472 
473 	if (err)
474 		return err;
475 
476 	return len;
477 }
478 
479 void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
480 {
481 	struct i915_vma *vma;
482 
483 	GEM_BUG_ON(!obj->userfault_count);
484 
485 	for_each_ggtt_vma(vma, obj)
486 		i915_vma_revoke_mmap(vma);
487 
488 	GEM_BUG_ON(obj->userfault_count);
489 }
490 
491 /*
492  * It is vital that we remove the page mapping if we have mapped a tiled
493  * object through the GTT and then lose the fence register due to
494  * resource pressure. Similarly if the object has been moved out of the
495  * aperture, than pages mapped into userspace must be revoked. Removing the
496  * mapping will then trigger a page fault on the next user access, allowing
497  * fixup by vm_fault_gtt().
498  */
499 void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
500 {
501 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
502 	intel_wakeref_t wakeref;
503 
504 	/*
505 	 * Serialisation between user GTT access and our code depends upon
506 	 * revoking the CPU's PTE whilst the mutex is held. The next user
507 	 * pagefault then has to wait until we release the mutex.
508 	 *
509 	 * Note that RPM complicates somewhat by adding an additional
510 	 * requirement that operations to the GGTT be made holding the RPM
511 	 * wakeref.
512 	 */
513 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
514 	mutex_lock(&i915->ggtt.vm.mutex);
515 
516 	if (!obj->userfault_count)
517 		goto out;
518 
519 	__i915_gem_object_release_mmap_gtt(obj);
520 
521 	/*
522 	 * Ensure that the CPU's PTE are revoked and there are not outstanding
523 	 * memory transactions from userspace before we return. The TLB
524 	 * flushing implied above by changing the PTE above *should* be
525 	 * sufficient, an extra barrier here just provides us with a bit
526 	 * of paranoid documentation about our requirement to serialise
527 	 * memory writes before touching registers / GSM.
528 	 */
529 	wmb();
530 
531 out:
532 	mutex_unlock(&i915->ggtt.vm.mutex);
533 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
534 }
535 
536 void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj)
537 {
538 	struct i915_mmap_offset *mmo, *mn;
539 
540 	spin_lock(&obj->mmo.lock);
541 	rbtree_postorder_for_each_entry_safe(mmo, mn,
542 					     &obj->mmo.offsets, offset) {
543 		/*
544 		 * vma_node_unmap for GTT mmaps handled already in
545 		 * __i915_gem_object_release_mmap_gtt
546 		 */
547 		if (mmo->mmap_type == I915_MMAP_TYPE_GTT)
548 			continue;
549 
550 		spin_unlock(&obj->mmo.lock);
551 		drm_vma_node_unmap(&mmo->vma_node,
552 				   obj->base.dev->anon_inode->i_mapping);
553 		spin_lock(&obj->mmo.lock);
554 	}
555 	spin_unlock(&obj->mmo.lock);
556 }
557 
558 static struct i915_mmap_offset *
559 lookup_mmo(struct drm_i915_gem_object *obj,
560 	   enum i915_mmap_type mmap_type)
561 {
562 	struct rb_node *rb;
563 
564 	spin_lock(&obj->mmo.lock);
565 	rb = obj->mmo.offsets.rb_node;
566 	while (rb) {
567 		struct i915_mmap_offset *mmo =
568 			rb_entry(rb, typeof(*mmo), offset);
569 
570 		if (mmo->mmap_type == mmap_type) {
571 			spin_unlock(&obj->mmo.lock);
572 			return mmo;
573 		}
574 
575 		if (mmo->mmap_type < mmap_type)
576 			rb = rb->rb_right;
577 		else
578 			rb = rb->rb_left;
579 	}
580 	spin_unlock(&obj->mmo.lock);
581 
582 	return NULL;
583 }
584 
585 static struct i915_mmap_offset *
586 insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo)
587 {
588 	struct rb_node *rb, **p;
589 
590 	spin_lock(&obj->mmo.lock);
591 	rb = NULL;
592 	p = &obj->mmo.offsets.rb_node;
593 	while (*p) {
594 		struct i915_mmap_offset *pos;
595 
596 		rb = *p;
597 		pos = rb_entry(rb, typeof(*pos), offset);
598 
599 		if (pos->mmap_type == mmo->mmap_type) {
600 			spin_unlock(&obj->mmo.lock);
601 			drm_vma_offset_remove(obj->base.dev->vma_offset_manager,
602 					      &mmo->vma_node);
603 			kfree(mmo);
604 			return pos;
605 		}
606 
607 		if (pos->mmap_type < mmo->mmap_type)
608 			p = &rb->rb_right;
609 		else
610 			p = &rb->rb_left;
611 	}
612 	rb_link_node(&mmo->offset, rb, p);
613 	rb_insert_color(&mmo->offset, &obj->mmo.offsets);
614 	spin_unlock(&obj->mmo.lock);
615 
616 	return mmo;
617 }
618 
619 static struct i915_mmap_offset *
620 mmap_offset_attach(struct drm_i915_gem_object *obj,
621 		   enum i915_mmap_type mmap_type,
622 		   struct drm_file *file)
623 {
624 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
625 	struct i915_mmap_offset *mmo;
626 	int err;
627 
628 	GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops);
629 
630 	mmo = lookup_mmo(obj, mmap_type);
631 	if (mmo)
632 		goto out;
633 
634 	mmo = kmalloc(sizeof(*mmo), GFP_KERNEL);
635 	if (!mmo)
636 		return ERR_PTR(-ENOMEM);
637 
638 	mmo->obj = obj;
639 	mmo->mmap_type = mmap_type;
640 	drm_vma_node_reset(&mmo->vma_node);
641 
642 	err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
643 				 &mmo->vma_node, obj->base.size / PAGE_SIZE);
644 	if (likely(!err))
645 		goto insert;
646 
647 	/* Attempt to reap some mmap space from dead objects */
648 	err = intel_gt_retire_requests_timeout(&i915->gt, MAX_SCHEDULE_TIMEOUT,
649 					       NULL);
650 	if (err)
651 		goto err;
652 
653 	i915_gem_drain_freed_objects(i915);
654 	err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
655 				 &mmo->vma_node, obj->base.size / PAGE_SIZE);
656 	if (err)
657 		goto err;
658 
659 insert:
660 	mmo = insert_mmo(obj, mmo);
661 	GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo);
662 out:
663 	if (file)
664 		drm_vma_node_allow(&mmo->vma_node, file);
665 	return mmo;
666 
667 err:
668 	kfree(mmo);
669 	return ERR_PTR(err);
670 }
671 
672 static int
673 __assign_mmap_offset(struct drm_i915_gem_object *obj,
674 		     enum i915_mmap_type mmap_type,
675 		     u64 *offset, struct drm_file *file)
676 {
677 	struct i915_mmap_offset *mmo;
678 
679 	if (i915_gem_object_never_mmap(obj))
680 		return -ENODEV;
681 
682 	if (obj->ops->mmap_offset)  {
683 		if (mmap_type != I915_MMAP_TYPE_FIXED)
684 			return -ENODEV;
685 
686 		*offset = obj->ops->mmap_offset(obj);
687 		return 0;
688 	}
689 
690 	if (mmap_type == I915_MMAP_TYPE_FIXED)
691 		return -ENODEV;
692 
693 	if (mmap_type != I915_MMAP_TYPE_GTT &&
694 	    !i915_gem_object_has_struct_page(obj) &&
695 	    !i915_gem_object_has_iomem(obj))
696 		return -ENODEV;
697 
698 	mmo = mmap_offset_attach(obj, mmap_type, file);
699 	if (IS_ERR(mmo))
700 		return PTR_ERR(mmo);
701 
702 	*offset = drm_vma_node_offset_addr(&mmo->vma_node);
703 	return 0;
704 }
705 
706 static int
707 __assign_mmap_offset_handle(struct drm_file *file,
708 			    u32 handle,
709 			    enum i915_mmap_type mmap_type,
710 			    u64 *offset)
711 {
712 	struct drm_i915_gem_object *obj;
713 	int err;
714 
715 	obj = i915_gem_object_lookup(file, handle);
716 	if (!obj)
717 		return -ENOENT;
718 
719 	err = i915_gem_object_lock_interruptible(obj, NULL);
720 	if (err)
721 		goto out_put;
722 	err = __assign_mmap_offset(obj, mmap_type, offset, file);
723 	i915_gem_object_unlock(obj);
724 out_put:
725 	i915_gem_object_put(obj);
726 	return err;
727 }
728 
729 int
730 i915_gem_dumb_mmap_offset(struct drm_file *file,
731 			  struct drm_device *dev,
732 			  u32 handle,
733 			  u64 *offset)
734 {
735 	enum i915_mmap_type mmap_type;
736 
737 	if (HAS_LMEM(to_i915(dev)))
738 		mmap_type = I915_MMAP_TYPE_FIXED;
739 	else if (boot_cpu_has(X86_FEATURE_PAT))
740 		mmap_type = I915_MMAP_TYPE_WC;
741 	else if (!i915_ggtt_has_aperture(&to_i915(dev)->ggtt))
742 		return -ENODEV;
743 	else
744 		mmap_type = I915_MMAP_TYPE_GTT;
745 
746 	return __assign_mmap_offset_handle(file, handle, mmap_type, offset);
747 }
748 
749 /**
750  * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing
751  * @dev: DRM device
752  * @data: GTT mapping ioctl data
753  * @file: GEM object info
754  *
755  * Simply returns the fake offset to userspace so it can mmap it.
756  * The mmap call will end up in drm_gem_mmap(), which will set things
757  * up so we can get faults in the handler above.
758  *
759  * The fault handler will take care of binding the object into the GTT
760  * (since it may have been evicted to make room for something), allocating
761  * a fence register, and mapping the appropriate aperture address into
762  * userspace.
763  */
764 int
765 i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
766 			   struct drm_file *file)
767 {
768 	struct drm_i915_private *i915 = to_i915(dev);
769 	struct drm_i915_gem_mmap_offset *args = data;
770 	enum i915_mmap_type type;
771 	int err;
772 
773 	/*
774 	 * Historically we failed to check args.pad and args.offset
775 	 * and so we cannot use those fields for user input and we cannot
776 	 * add -EINVAL for them as the ABI is fixed, i.e. old userspace
777 	 * may be feeding in garbage in those fields.
778 	 *
779 	 * if (args->pad) return -EINVAL; is verbotten!
780 	 */
781 
782 	err = i915_user_extensions(u64_to_user_ptr(args->extensions),
783 				   NULL, 0, NULL);
784 	if (err)
785 		return err;
786 
787 	switch (args->flags) {
788 	case I915_MMAP_OFFSET_GTT:
789 		if (!i915_ggtt_has_aperture(&i915->ggtt))
790 			return -ENODEV;
791 		type = I915_MMAP_TYPE_GTT;
792 		break;
793 
794 	case I915_MMAP_OFFSET_WC:
795 		if (!boot_cpu_has(X86_FEATURE_PAT))
796 			return -ENODEV;
797 		type = I915_MMAP_TYPE_WC;
798 		break;
799 
800 	case I915_MMAP_OFFSET_WB:
801 		type = I915_MMAP_TYPE_WB;
802 		break;
803 
804 	case I915_MMAP_OFFSET_UC:
805 		if (!boot_cpu_has(X86_FEATURE_PAT))
806 			return -ENODEV;
807 		type = I915_MMAP_TYPE_UC;
808 		break;
809 
810 	case I915_MMAP_OFFSET_FIXED:
811 		type = I915_MMAP_TYPE_FIXED;
812 		break;
813 
814 	default:
815 		return -EINVAL;
816 	}
817 
818 	return __assign_mmap_offset_handle(file, args->handle, type, &args->offset);
819 }
820 
821 static void vm_open(struct vm_area_struct *vma)
822 {
823 	struct i915_mmap_offset *mmo = vma->vm_private_data;
824 	struct drm_i915_gem_object *obj = mmo->obj;
825 
826 	GEM_BUG_ON(!obj);
827 	i915_gem_object_get(obj);
828 }
829 
830 static void vm_close(struct vm_area_struct *vma)
831 {
832 	struct i915_mmap_offset *mmo = vma->vm_private_data;
833 	struct drm_i915_gem_object *obj = mmo->obj;
834 
835 	GEM_BUG_ON(!obj);
836 	i915_gem_object_put(obj);
837 }
838 
839 static const struct vm_operations_struct vm_ops_gtt = {
840 	.fault = vm_fault_gtt,
841 	.access = vm_access,
842 	.open = vm_open,
843 	.close = vm_close,
844 };
845 
846 static const struct vm_operations_struct vm_ops_cpu = {
847 	.fault = vm_fault_cpu,
848 	.access = vm_access,
849 	.open = vm_open,
850 	.close = vm_close,
851 };
852 
853 static int singleton_release(struct inode *inode, struct file *file)
854 {
855 	struct drm_i915_private *i915 = file->private_data;
856 
857 	cmpxchg(&i915->gem.mmap_singleton, file, NULL);
858 	drm_dev_put(&i915->drm);
859 
860 	return 0;
861 }
862 
863 static const struct file_operations singleton_fops = {
864 	.owner = THIS_MODULE,
865 	.release = singleton_release,
866 };
867 
868 static struct file *mmap_singleton(struct drm_i915_private *i915)
869 {
870 	struct file *file;
871 
872 	rcu_read_lock();
873 	file = READ_ONCE(i915->gem.mmap_singleton);
874 	if (file && !get_file_rcu(file))
875 		file = NULL;
876 	rcu_read_unlock();
877 	if (file)
878 		return file;
879 
880 	file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR);
881 	if (IS_ERR(file))
882 		return file;
883 
884 	/* Everyone shares a single global address space */
885 	file->f_mapping = i915->drm.anon_inode->i_mapping;
886 
887 	smp_store_mb(i915->gem.mmap_singleton, file);
888 	drm_dev_get(&i915->drm);
889 
890 	return file;
891 }
892 
893 /*
894  * This overcomes the limitation in drm_gem_mmap's assignment of a
895  * drm_gem_object as the vma->vm_private_data. Since we need to
896  * be able to resolve multiple mmap offsets which could be tied
897  * to a single gem object.
898  */
899 int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)
900 {
901 	struct drm_vma_offset_node *node;
902 	struct drm_file *priv = filp->private_data;
903 	struct drm_device *dev = priv->minor->dev;
904 	struct drm_i915_gem_object *obj = NULL;
905 	struct i915_mmap_offset *mmo = NULL;
906 	struct file *anon;
907 
908 	if (drm_dev_is_unplugged(dev))
909 		return -ENODEV;
910 
911 	rcu_read_lock();
912 	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
913 	node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
914 						  vma->vm_pgoff,
915 						  vma_pages(vma));
916 	if (node && drm_vma_node_is_allowed(node, priv)) {
917 		/*
918 		 * Skip 0-refcnted objects as it is in the process of being
919 		 * destroyed and will be invalid when the vma manager lock
920 		 * is released.
921 		 */
922 		if (!node->driver_private) {
923 			mmo = container_of(node, struct i915_mmap_offset, vma_node);
924 			obj = i915_gem_object_get_rcu(mmo->obj);
925 
926 			GEM_BUG_ON(obj && obj->ops->mmap_ops);
927 		} else {
928 			obj = i915_gem_object_get_rcu
929 				(container_of(node, struct drm_i915_gem_object,
930 					      base.vma_node));
931 
932 			GEM_BUG_ON(obj && !obj->ops->mmap_ops);
933 		}
934 	}
935 	drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
936 	rcu_read_unlock();
937 	if (!obj)
938 		return node ? -EACCES : -EINVAL;
939 
940 	if (i915_gem_object_is_readonly(obj)) {
941 		if (vma->vm_flags & VM_WRITE) {
942 			i915_gem_object_put(obj);
943 			return -EINVAL;
944 		}
945 		vma->vm_flags &= ~VM_MAYWRITE;
946 	}
947 
948 	anon = mmap_singleton(to_i915(dev));
949 	if (IS_ERR(anon)) {
950 		i915_gem_object_put(obj);
951 		return PTR_ERR(anon);
952 	}
953 
954 	vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
955 
956 	/*
957 	 * We keep the ref on mmo->obj, not vm_file, but we require
958 	 * vma->vm_file->f_mapping, see vma_link(), for later revocation.
959 	 * Our userspace is accustomed to having per-file resource cleanup
960 	 * (i.e. contexts, objects and requests) on their close(fd), which
961 	 * requires avoiding extraneous references to their filp, hence why
962 	 * we prefer to use an anonymous file for their mmaps.
963 	 */
964 	vma_set_file(vma, anon);
965 	/* Drop the initial creation reference, the vma is now holding one. */
966 	fput(anon);
967 
968 	if (obj->ops->mmap_ops) {
969 		vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags));
970 		vma->vm_ops = obj->ops->mmap_ops;
971 		vma->vm_private_data = node->driver_private;
972 		return 0;
973 	}
974 
975 	vma->vm_private_data = mmo;
976 
977 	switch (mmo->mmap_type) {
978 	case I915_MMAP_TYPE_WC:
979 		vma->vm_page_prot =
980 			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
981 		vma->vm_ops = &vm_ops_cpu;
982 		break;
983 
984 	case I915_MMAP_TYPE_FIXED:
985 		GEM_WARN_ON(1);
986 		fallthrough;
987 	case I915_MMAP_TYPE_WB:
988 		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
989 		vma->vm_ops = &vm_ops_cpu;
990 		break;
991 
992 	case I915_MMAP_TYPE_UC:
993 		vma->vm_page_prot =
994 			pgprot_noncached(vm_get_page_prot(vma->vm_flags));
995 		vma->vm_ops = &vm_ops_cpu;
996 		break;
997 
998 	case I915_MMAP_TYPE_GTT:
999 		vma->vm_page_prot =
1000 			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1001 		vma->vm_ops = &vm_ops_gtt;
1002 		break;
1003 	}
1004 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1005 
1006 	return 0;
1007 }
1008 
1009 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1010 #include "selftests/i915_gem_mman.c"
1011 #endif
1012