1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/ktime.h>
29 #include <linux/module.h>
30 #include <linux/pagemap.h>
31 #include <linux/pci.h>
32 #include <linux/dma-buf.h>
33 
34 #include <drm/amdgpu_drm.h>
35 #include <drm/drm_drv.h>
36 #include <drm/drm_gem_ttm_helper.h>
37 #include <drm/ttm/ttm_tt.h>
38 
39 #include "amdgpu.h"
40 #include "amdgpu_display.h"
41 #include "amdgpu_dma_buf.h"
42 #include "amdgpu_hmm.h"
43 #include "amdgpu_xgmi.h"
44 
45 static const struct drm_gem_object_funcs amdgpu_gem_object_funcs;
46 
47 static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
48 {
49 	struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
50 	struct drm_device *ddev = bo->base.dev;
51 	vm_fault_t ret;
52 	int idx;
53 
54 	ret = ttm_bo_vm_reserve(bo, vmf);
55 	if (ret)
56 		return ret;
57 
58 	if (drm_dev_enter(ddev, &idx)) {
59 		ret = amdgpu_bo_fault_reserve_notify(bo);
60 		if (ret) {
61 			drm_dev_exit(idx);
62 			goto unlock;
63 		}
64 
65 		ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
66 					       TTM_BO_VM_NUM_PREFAULT);
67 
68 		drm_dev_exit(idx);
69 	} else {
70 		ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
71 	}
72 	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
73 		return ret;
74 
75 unlock:
76 	dma_resv_unlock(bo->base.resv);
77 	return ret;
78 }
79 
80 static const struct vm_operations_struct amdgpu_gem_vm_ops = {
81 	.fault = amdgpu_gem_fault,
82 	.open = ttm_bo_vm_open,
83 	.close = ttm_bo_vm_close,
84 	.access = ttm_bo_vm_access
85 };
86 
87 static void amdgpu_gem_object_free(struct drm_gem_object *gobj)
88 {
89 	struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj);
90 
91 	if (robj) {
92 		amdgpu_hmm_unregister(robj);
93 		amdgpu_bo_unref(&robj);
94 	}
95 }
96 
97 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
98 			     int alignment, u32 initial_domain,
99 			     u64 flags, enum ttm_bo_type type,
100 			     struct dma_resv *resv,
101 			     struct drm_gem_object **obj)
102 {
103 	struct amdgpu_bo *bo;
104 	struct amdgpu_bo_user *ubo;
105 	struct amdgpu_bo_param bp;
106 	int r;
107 
108 	memset(&bp, 0, sizeof(bp));
109 	*obj = NULL;
110 
111 	bp.size = size;
112 	bp.byte_align = alignment;
113 	bp.type = type;
114 	bp.resv = resv;
115 	bp.preferred_domain = initial_domain;
116 	bp.flags = flags;
117 	bp.domain = initial_domain;
118 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
119 
120 	r = amdgpu_bo_create_user(adev, &bp, &ubo);
121 	if (r)
122 		return r;
123 
124 	bo = &ubo->bo;
125 	*obj = &bo->tbo.base;
126 	(*obj)->funcs = &amdgpu_gem_object_funcs;
127 
128 	return 0;
129 }
130 
131 void amdgpu_gem_force_release(struct amdgpu_device *adev)
132 {
133 	struct drm_device *ddev = adev_to_drm(adev);
134 	struct drm_file *file;
135 
136 	mutex_lock(&ddev->filelist_mutex);
137 
138 	list_for_each_entry(file, &ddev->filelist, lhead) {
139 		struct drm_gem_object *gobj;
140 		int handle;
141 
142 		WARN_ONCE(1, "Still active user space clients!\n");
143 		spin_lock(&file->table_lock);
144 		idr_for_each_entry(&file->object_idr, gobj, handle) {
145 			WARN_ONCE(1, "And also active allocations!\n");
146 			drm_gem_object_put(gobj);
147 		}
148 		idr_destroy(&file->object_idr);
149 		spin_unlock(&file->table_lock);
150 	}
151 
152 	mutex_unlock(&ddev->filelist_mutex);
153 }
154 
155 /*
156  * Call from drm_gem_handle_create which appear in both new and open ioctl
157  * case.
158  */
159 static int amdgpu_gem_object_open(struct drm_gem_object *obj,
160 				  struct drm_file *file_priv)
161 {
162 	struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
163 	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
164 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
165 	struct amdgpu_vm *vm = &fpriv->vm;
166 	struct amdgpu_bo_va *bo_va;
167 	struct mm_struct *mm;
168 	int r;
169 
170 	mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
171 	if (mm && mm != current->mm)
172 		return -EPERM;
173 
174 	if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
175 	    abo->tbo.base.resv != vm->root.bo->tbo.base.resv)
176 		return -EPERM;
177 
178 	r = amdgpu_bo_reserve(abo, false);
179 	if (r)
180 		return r;
181 
182 	bo_va = amdgpu_vm_bo_find(vm, abo);
183 	if (!bo_va) {
184 		bo_va = amdgpu_vm_bo_add(adev, vm, abo);
185 	} else {
186 		++bo_va->ref_count;
187 	}
188 	amdgpu_bo_unreserve(abo);
189 	return 0;
190 }
191 
192 static void amdgpu_gem_object_close(struct drm_gem_object *obj,
193 				    struct drm_file *file_priv)
194 {
195 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
196 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
197 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
198 	struct amdgpu_vm *vm = &fpriv->vm;
199 
200 	struct amdgpu_bo_list_entry vm_pd;
201 	struct list_head list, duplicates;
202 	struct dma_fence *fence = NULL;
203 	struct ttm_validate_buffer tv;
204 	struct ww_acquire_ctx ticket;
205 	struct amdgpu_bo_va *bo_va;
206 	long r;
207 
208 	INIT_LIST_HEAD(&list);
209 	INIT_LIST_HEAD(&duplicates);
210 
211 	tv.bo = &bo->tbo;
212 	tv.num_shared = 2;
213 	list_add(&tv.head, &list);
214 
215 	amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
216 
217 	r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates);
218 	if (r) {
219 		dev_err(adev->dev, "leaking bo va because "
220 			"we fail to reserve bo (%ld)\n", r);
221 		return;
222 	}
223 	bo_va = amdgpu_vm_bo_find(vm, bo);
224 	if (!bo_va || --bo_va->ref_count)
225 		goto out_unlock;
226 
227 	amdgpu_vm_bo_del(adev, bo_va);
228 	if (!amdgpu_vm_ready(vm))
229 		goto out_unlock;
230 
231 	r = amdgpu_vm_clear_freed(adev, vm, &fence);
232 	if (r || !fence)
233 		goto out_unlock;
234 
235 	amdgpu_bo_fence(bo, fence, true);
236 	dma_fence_put(fence);
237 
238 out_unlock:
239 	if (unlikely(r < 0))
240 		dev_err(adev->dev, "failed to clear page "
241 			"tables on GEM object close (%ld)\n", r);
242 	ttm_eu_backoff_reservation(&ticket, &list);
243 }
244 
245 static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
246 {
247 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
248 
249 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
250 		return -EPERM;
251 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
252 		return -EPERM;
253 
254 	/* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
255 	 * for debugger access to invisible VRAM. Should have used MAP_SHARED
256 	 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
257 	 * becoming writable and makes is_cow_mapping(vm_flags) false.
258 	 */
259 	if (is_cow_mapping(vma->vm_flags) &&
260 	    !(vma->vm_flags & VM_ACCESS_FLAGS))
261 		vm_flags_clear(vma, VM_MAYWRITE);
262 
263 	return drm_gem_ttm_mmap(obj, vma);
264 }
265 
266 static const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
267 	.free = amdgpu_gem_object_free,
268 	.open = amdgpu_gem_object_open,
269 	.close = amdgpu_gem_object_close,
270 	.export = amdgpu_gem_prime_export,
271 	.vmap = drm_gem_ttm_vmap,
272 	.vunmap = drm_gem_ttm_vunmap,
273 	.mmap = amdgpu_gem_object_mmap,
274 	.vm_ops = &amdgpu_gem_vm_ops,
275 };
276 
277 /*
278  * GEM ioctls.
279  */
280 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
281 			    struct drm_file *filp)
282 {
283 	struct amdgpu_device *adev = drm_to_adev(dev);
284 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
285 	struct amdgpu_vm *vm = &fpriv->vm;
286 	union drm_amdgpu_gem_create *args = data;
287 	uint64_t flags = args->in.domain_flags;
288 	uint64_t size = args->in.bo_size;
289 	struct dma_resv *resv = NULL;
290 	struct drm_gem_object *gobj;
291 	uint32_t handle, initial_domain;
292 	int r;
293 
294 	/* reject invalid gem flags */
295 	if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
296 		      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
297 		      AMDGPU_GEM_CREATE_CPU_GTT_USWC |
298 		      AMDGPU_GEM_CREATE_VRAM_CLEARED |
299 		      AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
300 		      AMDGPU_GEM_CREATE_EXPLICIT_SYNC |
301 		      AMDGPU_GEM_CREATE_ENCRYPTED |
302 		      AMDGPU_GEM_CREATE_DISCARDABLE))
303 		return -EINVAL;
304 
305 	/* reject invalid gem domains */
306 	if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
307 		return -EINVAL;
308 
309 	if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
310 		DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
311 		return -EINVAL;
312 	}
313 
314 	/* create a gem object to contain this object in */
315 	if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
316 	    AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
317 		if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
318 			/* if gds bo is created from user space, it must be
319 			 * passed to bo list
320 			 */
321 			DRM_ERROR("GDS bo cannot be per-vm-bo\n");
322 			return -EINVAL;
323 		}
324 		flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
325 	}
326 
327 	if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
328 		r = amdgpu_bo_reserve(vm->root.bo, false);
329 		if (r)
330 			return r;
331 
332 		resv = vm->root.bo->tbo.base.resv;
333 	}
334 
335 	initial_domain = (u32)(0xffffffff & args->in.domains);
336 retry:
337 	r = amdgpu_gem_object_create(adev, size, args->in.alignment,
338 				     initial_domain,
339 				     flags, ttm_bo_type_device, resv, &gobj);
340 	if (r && r != -ERESTARTSYS) {
341 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
342 			flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
343 			goto retry;
344 		}
345 
346 		if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
347 			initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
348 			goto retry;
349 		}
350 		DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
351 				size, initial_domain, args->in.alignment, r);
352 	}
353 
354 	if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
355 		if (!r) {
356 			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
357 
358 			abo->parent = amdgpu_bo_ref(vm->root.bo);
359 		}
360 		amdgpu_bo_unreserve(vm->root.bo);
361 	}
362 	if (r)
363 		return r;
364 
365 	r = drm_gem_handle_create(filp, gobj, &handle);
366 	/* drop reference from allocate - handle holds it now */
367 	drm_gem_object_put(gobj);
368 	if (r)
369 		return r;
370 
371 	memset(args, 0, sizeof(*args));
372 	args->out.handle = handle;
373 	return 0;
374 }
375 
376 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
377 			     struct drm_file *filp)
378 {
379 	struct ttm_operation_ctx ctx = { true, false };
380 	struct amdgpu_device *adev = drm_to_adev(dev);
381 	struct drm_amdgpu_gem_userptr *args = data;
382 	struct drm_gem_object *gobj;
383 	struct hmm_range *range;
384 	struct amdgpu_bo *bo;
385 	uint32_t handle;
386 	int r;
387 
388 	args->addr = untagged_addr(args->addr);
389 
390 	if (offset_in_page(args->addr | args->size))
391 		return -EINVAL;
392 
393 	/* reject unknown flag values */
394 	if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
395 	    AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
396 	    AMDGPU_GEM_USERPTR_REGISTER))
397 		return -EINVAL;
398 
399 	if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
400 	     !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
401 
402 		/* if we want to write to it we must install a MMU notifier */
403 		return -EACCES;
404 	}
405 
406 	/* create a gem object to contain this object in */
407 	r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
408 				     0, ttm_bo_type_device, NULL, &gobj);
409 	if (r)
410 		return r;
411 
412 	bo = gem_to_amdgpu_bo(gobj);
413 	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
414 	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
415 	r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
416 	if (r)
417 		goto release_object;
418 
419 	r = amdgpu_hmm_register(bo, args->addr);
420 	if (r)
421 		goto release_object;
422 
423 	if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
424 		r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
425 						 &range);
426 		if (r)
427 			goto release_object;
428 
429 		r = amdgpu_bo_reserve(bo, true);
430 		if (r)
431 			goto user_pages_done;
432 
433 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
434 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
435 		amdgpu_bo_unreserve(bo);
436 		if (r)
437 			goto user_pages_done;
438 	}
439 
440 	r = drm_gem_handle_create(filp, gobj, &handle);
441 	if (r)
442 		goto user_pages_done;
443 
444 	args->handle = handle;
445 
446 user_pages_done:
447 	if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
448 		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);
449 
450 release_object:
451 	drm_gem_object_put(gobj);
452 
453 	return r;
454 }
455 
456 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
457 			  struct drm_device *dev,
458 			  uint32_t handle, uint64_t *offset_p)
459 {
460 	struct drm_gem_object *gobj;
461 	struct amdgpu_bo *robj;
462 
463 	gobj = drm_gem_object_lookup(filp, handle);
464 	if (gobj == NULL) {
465 		return -ENOENT;
466 	}
467 	robj = gem_to_amdgpu_bo(gobj);
468 	if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
469 	    (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
470 		drm_gem_object_put(gobj);
471 		return -EPERM;
472 	}
473 	*offset_p = amdgpu_bo_mmap_offset(robj);
474 	drm_gem_object_put(gobj);
475 	return 0;
476 }
477 
478 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
479 			  struct drm_file *filp)
480 {
481 	union drm_amdgpu_gem_mmap *args = data;
482 	uint32_t handle = args->in.handle;
483 	memset(args, 0, sizeof(*args));
484 	return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
485 }
486 
487 /**
488  * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
489  *
490  * @timeout_ns: timeout in ns
491  *
492  * Calculate the timeout in jiffies from an absolute timeout in ns.
493  */
494 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
495 {
496 	unsigned long timeout_jiffies;
497 	ktime_t timeout;
498 
499 	/* clamp timeout if it's to large */
500 	if (((int64_t)timeout_ns) < 0)
501 		return MAX_SCHEDULE_TIMEOUT;
502 
503 	timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
504 	if (ktime_to_ns(timeout) < 0)
505 		return 0;
506 
507 	timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
508 	/*  clamp timeout to avoid unsigned-> signed overflow */
509 	if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT )
510 		return MAX_SCHEDULE_TIMEOUT - 1;
511 
512 	return timeout_jiffies;
513 }
514 
515 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
516 			      struct drm_file *filp)
517 {
518 	union drm_amdgpu_gem_wait_idle *args = data;
519 	struct drm_gem_object *gobj;
520 	struct amdgpu_bo *robj;
521 	uint32_t handle = args->in.handle;
522 	unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
523 	int r = 0;
524 	long ret;
525 
526 	gobj = drm_gem_object_lookup(filp, handle);
527 	if (gobj == NULL) {
528 		return -ENOENT;
529 	}
530 	robj = gem_to_amdgpu_bo(gobj);
531 	ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,
532 				    true, timeout);
533 
534 	/* ret == 0 means not signaled,
535 	 * ret > 0 means signaled
536 	 * ret < 0 means interrupted before timeout
537 	 */
538 	if (ret >= 0) {
539 		memset(args, 0, sizeof(*args));
540 		args->out.status = (ret == 0);
541 	} else
542 		r = ret;
543 
544 	drm_gem_object_put(gobj);
545 	return r;
546 }
547 
548 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
549 				struct drm_file *filp)
550 {
551 	struct drm_amdgpu_gem_metadata *args = data;
552 	struct drm_gem_object *gobj;
553 	struct amdgpu_bo *robj;
554 	int r = -1;
555 
556 	DRM_DEBUG("%d \n", args->handle);
557 	gobj = drm_gem_object_lookup(filp, args->handle);
558 	if (gobj == NULL)
559 		return -ENOENT;
560 	robj = gem_to_amdgpu_bo(gobj);
561 
562 	r = amdgpu_bo_reserve(robj, false);
563 	if (unlikely(r != 0))
564 		goto out;
565 
566 	if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
567 		amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
568 		r = amdgpu_bo_get_metadata(robj, args->data.data,
569 					   sizeof(args->data.data),
570 					   &args->data.data_size_bytes,
571 					   &args->data.flags);
572 	} else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
573 		if (args->data.data_size_bytes > sizeof(args->data.data)) {
574 			r = -EINVAL;
575 			goto unreserve;
576 		}
577 		r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
578 		if (!r)
579 			r = amdgpu_bo_set_metadata(robj, args->data.data,
580 						   args->data.data_size_bytes,
581 						   args->data.flags);
582 	}
583 
584 unreserve:
585 	amdgpu_bo_unreserve(robj);
586 out:
587 	drm_gem_object_put(gobj);
588 	return r;
589 }
590 
591 /**
592  * amdgpu_gem_va_update_vm -update the bo_va in its VM
593  *
594  * @adev: amdgpu_device pointer
595  * @vm: vm to update
596  * @bo_va: bo_va to update
597  * @operation: map, unmap or clear
598  *
599  * Update the bo_va directly after setting its address. Errors are not
600  * vital here, so they are not reported back to userspace.
601  */
602 static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
603 				    struct amdgpu_vm *vm,
604 				    struct amdgpu_bo_va *bo_va,
605 				    uint32_t operation)
606 {
607 	int r;
608 
609 	if (!amdgpu_vm_ready(vm))
610 		return;
611 
612 	r = amdgpu_vm_clear_freed(adev, vm, NULL);
613 	if (r)
614 		goto error;
615 
616 	if (operation == AMDGPU_VA_OP_MAP ||
617 	    operation == AMDGPU_VA_OP_REPLACE) {
618 		r = amdgpu_vm_bo_update(adev, bo_va, false);
619 		if (r)
620 			goto error;
621 	}
622 
623 	r = amdgpu_vm_update_pdes(adev, vm, false);
624 
625 error:
626 	if (r && r != -ERESTARTSYS)
627 		DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
628 }
629 
630 /**
631  * amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags
632  *
633  * @adev: amdgpu_device pointer
634  * @flags: GEM UAPI flags
635  *
636  * Returns the GEM UAPI flags mapped into hardware for the ASIC.
637  */
638 uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags)
639 {
640 	uint64_t pte_flag = 0;
641 
642 	if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
643 		pte_flag |= AMDGPU_PTE_EXECUTABLE;
644 	if (flags & AMDGPU_VM_PAGE_READABLE)
645 		pte_flag |= AMDGPU_PTE_READABLE;
646 	if (flags & AMDGPU_VM_PAGE_WRITEABLE)
647 		pte_flag |= AMDGPU_PTE_WRITEABLE;
648 	if (flags & AMDGPU_VM_PAGE_PRT)
649 		pte_flag |= AMDGPU_PTE_PRT;
650 	if (flags & AMDGPU_VM_PAGE_NOALLOC)
651 		pte_flag |= AMDGPU_PTE_NOALLOC;
652 
653 	if (adev->gmc.gmc_funcs->map_mtype)
654 		pte_flag |= amdgpu_gmc_map_mtype(adev,
655 						 flags & AMDGPU_VM_MTYPE_MASK);
656 
657 	return pte_flag;
658 }
659 
660 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
661 			  struct drm_file *filp)
662 {
663 	const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
664 		AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
665 		AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
666 		AMDGPU_VM_PAGE_NOALLOC;
667 	const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
668 		AMDGPU_VM_PAGE_PRT;
669 
670 	struct drm_amdgpu_gem_va *args = data;
671 	struct drm_gem_object *gobj;
672 	struct amdgpu_device *adev = drm_to_adev(dev);
673 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
674 	struct amdgpu_bo *abo;
675 	struct amdgpu_bo_va *bo_va;
676 	struct amdgpu_bo_list_entry vm_pd;
677 	struct ttm_validate_buffer tv;
678 	struct ww_acquire_ctx ticket;
679 	struct list_head list, duplicates;
680 	uint64_t va_flags;
681 	uint64_t vm_size;
682 	int r = 0;
683 
684 	if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
685 		dev_dbg(dev->dev,
686 			"va_address 0x%LX is in reserved area 0x%LX\n",
687 			args->va_address, AMDGPU_VA_RESERVED_SIZE);
688 		return -EINVAL;
689 	}
690 
691 	if (args->va_address >= AMDGPU_GMC_HOLE_START &&
692 	    args->va_address < AMDGPU_GMC_HOLE_END) {
693 		dev_dbg(dev->dev,
694 			"va_address 0x%LX is in VA hole 0x%LX-0x%LX\n",
695 			args->va_address, AMDGPU_GMC_HOLE_START,
696 			AMDGPU_GMC_HOLE_END);
697 		return -EINVAL;
698 	}
699 
700 	args->va_address &= AMDGPU_GMC_HOLE_MASK;
701 
702 	vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
703 	vm_size -= AMDGPU_VA_RESERVED_SIZE;
704 	if (args->va_address + args->map_size > vm_size) {
705 		dev_dbg(dev->dev,
706 			"va_address 0x%llx is in top reserved area 0x%llx\n",
707 			args->va_address + args->map_size, vm_size);
708 		return -EINVAL;
709 	}
710 
711 	if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
712 		dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
713 			args->flags);
714 		return -EINVAL;
715 	}
716 
717 	switch (args->operation) {
718 	case AMDGPU_VA_OP_MAP:
719 	case AMDGPU_VA_OP_UNMAP:
720 	case AMDGPU_VA_OP_CLEAR:
721 	case AMDGPU_VA_OP_REPLACE:
722 		break;
723 	default:
724 		dev_dbg(dev->dev, "unsupported operation %d\n",
725 			args->operation);
726 		return -EINVAL;
727 	}
728 
729 	INIT_LIST_HEAD(&list);
730 	INIT_LIST_HEAD(&duplicates);
731 	if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
732 	    !(args->flags & AMDGPU_VM_PAGE_PRT)) {
733 		gobj = drm_gem_object_lookup(filp, args->handle);
734 		if (gobj == NULL)
735 			return -ENOENT;
736 		abo = gem_to_amdgpu_bo(gobj);
737 		tv.bo = &abo->tbo;
738 		if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
739 			tv.num_shared = 1;
740 		else
741 			tv.num_shared = 0;
742 		list_add(&tv.head, &list);
743 	} else {
744 		gobj = NULL;
745 		abo = NULL;
746 	}
747 
748 	amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd);
749 
750 	r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
751 	if (r)
752 		goto error_unref;
753 
754 	if (abo) {
755 		bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
756 		if (!bo_va) {
757 			r = -ENOENT;
758 			goto error_backoff;
759 		}
760 	} else if (args->operation != AMDGPU_VA_OP_CLEAR) {
761 		bo_va = fpriv->prt_va;
762 	} else {
763 		bo_va = NULL;
764 	}
765 
766 	switch (args->operation) {
767 	case AMDGPU_VA_OP_MAP:
768 		va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
769 		r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
770 				     args->offset_in_bo, args->map_size,
771 				     va_flags);
772 		break;
773 	case AMDGPU_VA_OP_UNMAP:
774 		r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
775 		break;
776 
777 	case AMDGPU_VA_OP_CLEAR:
778 		r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
779 						args->va_address,
780 						args->map_size);
781 		break;
782 	case AMDGPU_VA_OP_REPLACE:
783 		va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
784 		r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
785 					     args->offset_in_bo, args->map_size,
786 					     va_flags);
787 		break;
788 	default:
789 		break;
790 	}
791 	if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !amdgpu_vm_debug)
792 		amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
793 					args->operation);
794 
795 error_backoff:
796 	ttm_eu_backoff_reservation(&ticket, &list);
797 
798 error_unref:
799 	drm_gem_object_put(gobj);
800 	return r;
801 }
802 
803 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
804 			struct drm_file *filp)
805 {
806 	struct amdgpu_device *adev = drm_to_adev(dev);
807 	struct drm_amdgpu_gem_op *args = data;
808 	struct drm_gem_object *gobj;
809 	struct amdgpu_vm_bo_base *base;
810 	struct amdgpu_bo *robj;
811 	int r;
812 
813 	gobj = drm_gem_object_lookup(filp, args->handle);
814 	if (gobj == NULL) {
815 		return -ENOENT;
816 	}
817 	robj = gem_to_amdgpu_bo(gobj);
818 
819 	r = amdgpu_bo_reserve(robj, false);
820 	if (unlikely(r))
821 		goto out;
822 
823 	switch (args->op) {
824 	case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
825 		struct drm_amdgpu_gem_create_in info;
826 		void __user *out = u64_to_user_ptr(args->value);
827 
828 		info.bo_size = robj->tbo.base.size;
829 		info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;
830 		info.domains = robj->preferred_domains;
831 		info.domain_flags = robj->flags;
832 		amdgpu_bo_unreserve(robj);
833 		if (copy_to_user(out, &info, sizeof(info)))
834 			r = -EFAULT;
835 		break;
836 	}
837 	case AMDGPU_GEM_OP_SET_PLACEMENT:
838 		if (robj->tbo.base.import_attach &&
839 		    args->value & AMDGPU_GEM_DOMAIN_VRAM) {
840 			r = -EINVAL;
841 			amdgpu_bo_unreserve(robj);
842 			break;
843 		}
844 		if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
845 			r = -EPERM;
846 			amdgpu_bo_unreserve(robj);
847 			break;
848 		}
849 		for (base = robj->vm_bo; base; base = base->next)
850 			if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
851 				amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {
852 				r = -EINVAL;
853 				amdgpu_bo_unreserve(robj);
854 				goto out;
855 			}
856 
857 
858 		robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
859 							AMDGPU_GEM_DOMAIN_GTT |
860 							AMDGPU_GEM_DOMAIN_CPU);
861 		robj->allowed_domains = robj->preferred_domains;
862 		if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
863 			robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
864 
865 		if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
866 			amdgpu_vm_bo_invalidate(adev, robj, true);
867 
868 		amdgpu_bo_unreserve(robj);
869 		break;
870 	default:
871 		amdgpu_bo_unreserve(robj);
872 		r = -EINVAL;
873 	}
874 
875 out:
876 	drm_gem_object_put(gobj);
877 	return r;
878 }
879 
880 static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
881 				  int width,
882 				  int cpp,
883 				  bool tiled)
884 {
885 	int aligned = width;
886 	int pitch_mask = 0;
887 
888 	switch (cpp) {
889 	case 1:
890 		pitch_mask = 255;
891 		break;
892 	case 2:
893 		pitch_mask = 127;
894 		break;
895 	case 3:
896 	case 4:
897 		pitch_mask = 63;
898 		break;
899 	}
900 
901 	aligned += pitch_mask;
902 	aligned &= ~pitch_mask;
903 	return aligned * cpp;
904 }
905 
906 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
907 			    struct drm_device *dev,
908 			    struct drm_mode_create_dumb *args)
909 {
910 	struct amdgpu_device *adev = drm_to_adev(dev);
911 	struct drm_gem_object *gobj;
912 	uint32_t handle;
913 	u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
914 		    AMDGPU_GEM_CREATE_CPU_GTT_USWC |
915 		    AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
916 	u32 domain;
917 	int r;
918 
919 	/*
920 	 * The buffer returned from this function should be cleared, but
921 	 * it can only be done if the ring is enabled or we'll fail to
922 	 * create the buffer.
923 	 */
924 	if (adev->mman.buffer_funcs_enabled)
925 		flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
926 
927 	args->pitch = amdgpu_gem_align_pitch(adev, args->width,
928 					     DIV_ROUND_UP(args->bpp, 8), 0);
929 	args->size = (u64)args->pitch * args->height;
930 	args->size = ALIGN(args->size, PAGE_SIZE);
931 	domain = amdgpu_bo_get_preferred_domain(adev,
932 				amdgpu_display_supported_domains(adev, flags));
933 	r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
934 				     ttm_bo_type_device, NULL, &gobj);
935 	if (r)
936 		return -ENOMEM;
937 
938 	r = drm_gem_handle_create(file_priv, gobj, &handle);
939 	/* drop reference from allocate - handle holds it now */
940 	drm_gem_object_put(gobj);
941 	if (r) {
942 		return r;
943 	}
944 	args->handle = handle;
945 	return 0;
946 }
947 
948 #if defined(CONFIG_DEBUG_FS)
949 static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
950 {
951 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
952 	struct drm_device *dev = adev_to_drm(adev);
953 	struct drm_file *file;
954 	int r;
955 
956 	r = mutex_lock_interruptible(&dev->filelist_mutex);
957 	if (r)
958 		return r;
959 
960 	list_for_each_entry(file, &dev->filelist, lhead) {
961 		struct task_struct *task;
962 		struct drm_gem_object *gobj;
963 		int id;
964 
965 		/*
966 		 * Although we have a valid reference on file->pid, that does
967 		 * not guarantee that the task_struct who called get_pid() is
968 		 * still alive (e.g. get_pid(current) => fork() => exit()).
969 		 * Therefore, we need to protect this ->comm access using RCU.
970 		 */
971 		rcu_read_lock();
972 		task = pid_task(file->pid, PIDTYPE_PID);
973 		seq_printf(m, "pid %8d command %s:\n", pid_nr(file->pid),
974 			   task ? task->comm : "<unknown>");
975 		rcu_read_unlock();
976 
977 		spin_lock(&file->table_lock);
978 		idr_for_each_entry(&file->object_idr, gobj, id) {
979 			struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
980 
981 			amdgpu_bo_print_info(id, bo, m);
982 		}
983 		spin_unlock(&file->table_lock);
984 	}
985 
986 	mutex_unlock(&dev->filelist_mutex);
987 	return 0;
988 }
989 
990 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);
991 
992 #endif
993 
994 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
995 {
996 #if defined(CONFIG_DEBUG_FS)
997 	struct drm_minor *minor = adev_to_drm(adev)->primary;
998 	struct dentry *root = minor->debugfs_root;
999 
1000 	debugfs_create_file("amdgpu_gem_info", 0444, root, adev,
1001 			    &amdgpu_debugfs_gem_info_fops);
1002 #endif
1003 }
1004