1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * based on nouveau_prime.c
23  *
24  * Authors: Alex Deucher
25  */
26 
27 /**
28  * DOC: PRIME Buffer Sharing
29  *
30  * The following callback implementations are used for :ref:`sharing GEM buffer
31  * objects between different devices via PRIME <prime_buffer_sharing>`.
32  */
33 
34 #include "amdgpu.h"
35 #include "amdgpu_display.h"
36 #include "amdgpu_gem.h"
37 #include "amdgpu_dma_buf.h"
38 #include <drm/amdgpu_drm.h>
39 #include <linux/dma-buf.h>
40 #include <linux/dma-fence-array.h>
41 
42 /**
43  * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
44  * @obj: GEM BO
45  *
46  * Sets up an in-kernel virtual mapping of the BO's memory.
47  *
48  * Returns:
49  * The virtual address of the mapping or an error pointer.
50  */
51 void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj)
52 {
53 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
54 	int ret;
55 
56 	ret = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages,
57 			  &bo->dma_buf_vmap);
58 	if (ret)
59 		return ERR_PTR(ret);
60 
61 	return bo->dma_buf_vmap.virtual;
62 }
63 
64 /**
65  * amdgpu_gem_prime_vunmap - &dma_buf_ops.vunmap implementation
66  * @obj: GEM BO
67  * @vaddr: Virtual address (unused)
68  *
69  * Tears down the in-kernel virtual mapping of the BO's memory.
70  */
71 void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
72 {
73 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
74 
75 	ttm_bo_kunmap(&bo->dma_buf_vmap);
76 }
77 
78 /**
79  * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation
80  * @obj: GEM BO
81  * @vma: Virtual memory area
82  *
83  * Sets up a userspace mapping of the BO's memory in the given
84  * virtual memory area.
85  *
86  * Returns:
87  * 0 on success or a negative error code on failure.
88  */
89 int amdgpu_gem_prime_mmap(struct drm_gem_object *obj,
90 			  struct vm_area_struct *vma)
91 {
92 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
93 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
94 	unsigned asize = amdgpu_bo_size(bo);
95 	int ret;
96 
97 	if (!vma->vm_file)
98 		return -ENODEV;
99 
100 	if (adev == NULL)
101 		return -ENODEV;
102 
103 	/* Check for valid size. */
104 	if (asize < vma->vm_end - vma->vm_start)
105 		return -EINVAL;
106 
107 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) ||
108 	    (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
109 		return -EPERM;
110 	}
111 	vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT;
112 
113 	/* prime mmap does not need to check access, so allow here */
114 	ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data);
115 	if (ret)
116 		return ret;
117 
118 	ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev);
119 	drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data);
120 
121 	return ret;
122 }
123 
124 static int
125 __dma_resv_make_exclusive(struct dma_resv *obj)
126 {
127 	struct dma_fence **fences;
128 	unsigned int count;
129 	int r;
130 
131 	if (!dma_resv_get_list(obj)) /* no shared fences to convert */
132 		return 0;
133 
134 	r = dma_resv_get_fences_rcu(obj, NULL, &count, &fences);
135 	if (r)
136 		return r;
137 
138 	if (count == 0) {
139 		/* Now that was unexpected. */
140 	} else if (count == 1) {
141 		dma_resv_add_excl_fence(obj, fences[0]);
142 		dma_fence_put(fences[0]);
143 		kfree(fences);
144 	} else {
145 		struct dma_fence_array *array;
146 
147 		array = dma_fence_array_create(count, fences,
148 					       dma_fence_context_alloc(1), 0,
149 					       false);
150 		if (!array)
151 			goto err_fences_put;
152 
153 		dma_resv_add_excl_fence(obj, &array->base);
154 		dma_fence_put(&array->base);
155 	}
156 
157 	return 0;
158 
159 err_fences_put:
160 	while (count--)
161 		dma_fence_put(fences[count]);
162 	kfree(fences);
163 	return -ENOMEM;
164 }
165 
166 /**
167  * amdgpu_dma_buf_attach - &dma_buf_ops.attach implementation
168  *
169  * @dmabuf: DMA-buf where we attach to
170  * @attach: attachment to add
171  *
172  * Add the attachment as user to the exported DMA-buf.
173  */
174 static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
175 				 struct dma_buf_attachment *attach)
176 {
177 	struct drm_gem_object *obj = dmabuf->priv;
178 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
179 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
180 	int r;
181 
182 	if (attach->dev->driver == adev->dev->driver)
183 		return 0;
184 
185 	r = amdgpu_bo_reserve(bo, false);
186 	if (unlikely(r != 0))
187 		return r;
188 
189 	/*
190 	 * We only create shared fences for internal use, but importers
191 	 * of the dmabuf rely on exclusive fences for implicitly
192 	 * tracking write hazards. As any of the current fences may
193 	 * correspond to a write, we need to convert all existing
194 	 * fences on the reservation object into a single exclusive
195 	 * fence.
196 	 */
197 	r = __dma_resv_make_exclusive(bo->tbo.base.resv);
198 	if (r)
199 		return r;
200 
201 	bo->prime_shared_count++;
202 	amdgpu_bo_unreserve(bo);
203 	return 0;
204 }
205 
206 /**
207  * amdgpu_dma_buf_detach - &dma_buf_ops.detach implementation
208  *
209  * @dmabuf: DMA-buf where we remove the attachment from
210  * @attach: the attachment to remove
211  *
212  * Called when an attachment is removed from the DMA-buf.
213  */
214 static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf,
215 				  struct dma_buf_attachment *attach)
216 {
217 	struct drm_gem_object *obj = dmabuf->priv;
218 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
219 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
220 
221 	if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count)
222 		bo->prime_shared_count--;
223 }
224 
225 /**
226  * amdgpu_dma_buf_pin - &dma_buf_ops.pin implementation
227  *
228  * @attach: attachment to pin down
229  *
230  * Pin the BO which is backing the DMA-buf so that it can't move any more.
231  */
232 static int amdgpu_dma_buf_pin(struct dma_buf_attachment *attach)
233 {
234 	struct drm_gem_object *obj = attach->dmabuf->priv;
235 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
236 
237 	/* pin buffer into GTT */
238 	return amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
239 }
240 
241 /**
242  * amdgpu_dma_buf_unpin - &dma_buf_ops.unpin implementation
243  *
244  * @attach: attachment to unpin
245  *
246  * Unpin a previously pinned BO to make it movable again.
247  */
248 static void amdgpu_dma_buf_unpin(struct dma_buf_attachment *attach)
249 {
250 	struct drm_gem_object *obj = attach->dmabuf->priv;
251 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
252 
253 	amdgpu_bo_unpin(bo);
254 }
255 
256 /**
257  * amdgpu_dma_buf_map - &dma_buf_ops.map_dma_buf implementation
258  * @attach: DMA-buf attachment
259  * @dir: DMA direction
260  *
261  * Makes sure that the shared DMA buffer can be accessed by the target device.
262  * For now, simply pins it to the GTT domain, where it should be accessible by
263  * all DMA devices.
264  *
265  * Returns:
266  * sg_table filled with the DMA addresses to use or ERR_PRT with negative error
267  * code.
268  */
269 static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach,
270 					   enum dma_data_direction dir)
271 {
272 	struct dma_buf *dma_buf = attach->dmabuf;
273 	struct drm_gem_object *obj = dma_buf->priv;
274 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
275 	struct sg_table *sgt;
276 	long r;
277 
278 	if (!bo->pin_count) {
279 		/* move buffer into GTT */
280 		struct ttm_operation_ctx ctx = { false, false };
281 
282 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
283 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
284 		if (r)
285 			return ERR_PTR(r);
286 
287 	} else if (!(amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type) &
288 		     AMDGPU_GEM_DOMAIN_GTT)) {
289 		return ERR_PTR(-EBUSY);
290 	}
291 
292 	sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages, bo->tbo.num_pages);
293 	if (IS_ERR(sgt))
294 		return sgt;
295 
296 	if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir,
297 			      DMA_ATTR_SKIP_CPU_SYNC))
298 		goto error_free;
299 
300 	return sgt;
301 
302 error_free:
303 	sg_free_table(sgt);
304 	kfree(sgt);
305 	return ERR_PTR(-ENOMEM);
306 }
307 
308 /**
309  * amdgpu_dma_buf_unmap - &dma_buf_ops.unmap_dma_buf implementation
310  * @attach: DMA-buf attachment
311  * @sgt: sg_table to unmap
312  * @dir: DMA direction
313  *
314  * This is called when a shared DMA buffer no longer needs to be accessible by
315  * another device. For now, simply unpins the buffer from GTT.
316  */
317 static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach,
318 				 struct sg_table *sgt,
319 				 enum dma_data_direction dir)
320 {
321 	dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
322 	sg_free_table(sgt);
323 	kfree(sgt);
324 }
325 
326 /**
327  * amdgpu_dma_buf_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation
328  * @dma_buf: Shared DMA buffer
329  * @direction: Direction of DMA transfer
330  *
331  * This is called before CPU access to the shared DMA buffer's memory. If it's
332  * a read access, the buffer is moved to the GTT domain if possible, for optimal
333  * CPU read performance.
334  *
335  * Returns:
336  * 0 on success or a negative error code on failure.
337  */
338 static int amdgpu_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
339 					   enum dma_data_direction direction)
340 {
341 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv);
342 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
343 	struct ttm_operation_ctx ctx = { true, false };
344 	u32 domain = amdgpu_display_supported_domains(adev, bo->flags);
345 	int ret;
346 	bool reads = (direction == DMA_BIDIRECTIONAL ||
347 		      direction == DMA_FROM_DEVICE);
348 
349 	if (!reads || !(domain & AMDGPU_GEM_DOMAIN_GTT))
350 		return 0;
351 
352 	/* move to gtt */
353 	ret = amdgpu_bo_reserve(bo, false);
354 	if (unlikely(ret != 0))
355 		return ret;
356 
357 	if (!bo->pin_count && (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) {
358 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
359 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
360 	}
361 
362 	amdgpu_bo_unreserve(bo);
363 	return ret;
364 }
365 
366 const struct dma_buf_ops amdgpu_dmabuf_ops = {
367 	.attach = amdgpu_dma_buf_attach,
368 	.detach = amdgpu_dma_buf_detach,
369 	.pin = amdgpu_dma_buf_pin,
370 	.unpin = amdgpu_dma_buf_unpin,
371 	.map_dma_buf = amdgpu_dma_buf_map,
372 	.unmap_dma_buf = amdgpu_dma_buf_unmap,
373 	.release = drm_gem_dmabuf_release,
374 	.begin_cpu_access = amdgpu_dma_buf_begin_cpu_access,
375 	.mmap = drm_gem_dmabuf_mmap,
376 	.vmap = drm_gem_dmabuf_vmap,
377 	.vunmap = drm_gem_dmabuf_vunmap,
378 };
379 
380 /**
381  * amdgpu_gem_prime_export - &drm_driver.gem_prime_export implementation
382  * @gobj: GEM BO
383  * @flags: Flags such as DRM_CLOEXEC and DRM_RDWR.
384  *
385  * The main work is done by the &drm_gem_prime_export helper.
386  *
387  * Returns:
388  * Shared DMA buffer representing the GEM BO from the given device.
389  */
390 struct dma_buf *amdgpu_gem_prime_export(struct drm_gem_object *gobj,
391 					int flags)
392 {
393 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
394 	struct dma_buf *buf;
395 
396 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) ||
397 	    bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
398 		return ERR_PTR(-EPERM);
399 
400 	buf = drm_gem_prime_export(gobj, flags);
401 	if (!IS_ERR(buf))
402 		buf->ops = &amdgpu_dmabuf_ops;
403 
404 	return buf;
405 }
406 
407 /**
408  * amdgpu_dma_buf_create_obj - create BO for DMA-buf import
409  *
410  * @dev: DRM device
411  * @dma_buf: DMA-buf
412  *
413  * Creates an empty SG BO for DMA-buf import.
414  *
415  * Returns:
416  * A new GEM BO of the given DRM device, representing the memory
417  * described by the given DMA-buf attachment and scatter/gather table.
418  */
419 static struct drm_gem_object *
420 amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf)
421 {
422 	struct dma_resv *resv = dma_buf->resv;
423 	struct amdgpu_device *adev = dev->dev_private;
424 	struct amdgpu_bo *bo;
425 	struct amdgpu_bo_param bp;
426 	int ret;
427 
428 	memset(&bp, 0, sizeof(bp));
429 	bp.size = dma_buf->size;
430 	bp.byte_align = PAGE_SIZE;
431 	bp.domain = AMDGPU_GEM_DOMAIN_CPU;
432 	bp.flags = 0;
433 	bp.type = ttm_bo_type_sg;
434 	bp.resv = resv;
435 	dma_resv_lock(resv, NULL);
436 	ret = amdgpu_bo_create(adev, &bp, &bo);
437 	if (ret)
438 		goto error;
439 
440 	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
441 	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
442 	if (dma_buf->ops != &amdgpu_dmabuf_ops)
443 		bo->prime_shared_count = 1;
444 
445 	dma_resv_unlock(resv);
446 	return &bo->tbo.base;
447 
448 error:
449 	dma_resv_unlock(resv);
450 	return ERR_PTR(ret);
451 }
452 
453 /**
454  * amdgpu_dma_buf_move_notify - &attach.move_notify implementation
455  *
456  * @attach: the DMA-buf attachment
457  *
458  * Invalidate the DMA-buf attachment, making sure that the we re-create the
459  * mapping before the next use.
460  */
461 static void
462 amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach)
463 {
464 	struct drm_gem_object *obj = attach->importer_priv;
465 	struct ww_acquire_ctx *ticket = dma_resv_locking_ctx(obj->resv);
466 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
467 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
468 	struct ttm_operation_ctx ctx = { false, false };
469 	struct ttm_placement placement = {};
470 	struct amdgpu_vm_bo_base *bo_base;
471 	int r;
472 
473 	if (bo->tbo.mem.mem_type == TTM_PL_SYSTEM)
474 		return;
475 
476 	r = ttm_bo_validate(&bo->tbo, &placement, &ctx);
477 	if (r) {
478 		DRM_ERROR("Failed to invalidate DMA-buf import (%d))\n", r);
479 		return;
480 	}
481 
482 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
483 		struct amdgpu_vm *vm = bo_base->vm;
484 		struct dma_resv *resv = vm->root.base.bo->tbo.base.resv;
485 
486 		if (ticket) {
487 			/* When we get an error here it means that somebody
488 			 * else is holding the VM lock and updating page tables
489 			 * So we can just continue here.
490 			 */
491 			r = dma_resv_lock(resv, ticket);
492 			if (r)
493 				continue;
494 
495 		} else {
496 			/* TODO: This is more problematic and we actually need
497 			 * to allow page tables updates without holding the
498 			 * lock.
499 			 */
500 			if (!dma_resv_trylock(resv))
501 				continue;
502 		}
503 
504 		r = amdgpu_vm_clear_freed(adev, vm, NULL);
505 		if (!r)
506 			r = amdgpu_vm_handle_moved(adev, vm);
507 
508 		if (r && r != -EBUSY)
509 			DRM_ERROR("Failed to invalidate VM page tables (%d))\n",
510 				  r);
511 
512 		dma_resv_unlock(resv);
513 	}
514 }
515 
516 static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops = {
517 	.move_notify = amdgpu_dma_buf_move_notify
518 };
519 
520 /**
521  * amdgpu_gem_prime_import - &drm_driver.gem_prime_import implementation
522  * @dev: DRM device
523  * @dma_buf: Shared DMA buffer
524  *
525  * Import a dma_buf into a the driver and potentially create a new GEM object.
526  *
527  * Returns:
528  * GEM BO representing the shared DMA buffer for the given device.
529  */
530 struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev,
531 					       struct dma_buf *dma_buf)
532 {
533 	struct dma_buf_attachment *attach;
534 	struct drm_gem_object *obj;
535 
536 	if (dma_buf->ops == &amdgpu_dmabuf_ops) {
537 		obj = dma_buf->priv;
538 		if (obj->dev == dev) {
539 			/*
540 			 * Importing dmabuf exported from out own gem increases
541 			 * refcount on gem itself instead of f_count of dmabuf.
542 			 */
543 			drm_gem_object_get(obj);
544 			return obj;
545 		}
546 	}
547 
548 	obj = amdgpu_dma_buf_create_obj(dev, dma_buf);
549 	if (IS_ERR(obj))
550 		return obj;
551 
552 	attach = dma_buf_dynamic_attach(dma_buf, dev->dev,
553 					&amdgpu_dma_buf_attach_ops, obj);
554 	if (IS_ERR(attach)) {
555 		drm_gem_object_put(obj);
556 		return ERR_CAST(attach);
557 	}
558 
559 	get_dma_buf(dma_buf);
560 	obj->import_attach = attach;
561 	return obj;
562 }
563