1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * based on nouveau_prime.c 23 * 24 * Authors: Alex Deucher 25 */ 26 27 /** 28 * DOC: PRIME Buffer Sharing 29 * 30 * The following callback implementations are used for :ref:`sharing GEM buffer 31 * objects between different devices via PRIME <prime_buffer_sharing>`. 32 */ 33 34 #include "amdgpu.h" 35 #include "amdgpu_display.h" 36 #include "amdgpu_gem.h" 37 #include "amdgpu_dma_buf.h" 38 #include <drm/amdgpu_drm.h> 39 #include <linux/dma-buf.h> 40 #include <linux/dma-fence-array.h> 41 42 /** 43 * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation 44 * @obj: GEM BO 45 * 46 * Sets up an in-kernel virtual mapping of the BO's memory. 47 * 48 * Returns: 49 * The virtual address of the mapping or an error pointer. 50 */ 51 void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj) 52 { 53 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 54 int ret; 55 56 ret = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, 57 &bo->dma_buf_vmap); 58 if (ret) 59 return ERR_PTR(ret); 60 61 return bo->dma_buf_vmap.virtual; 62 } 63 64 /** 65 * amdgpu_gem_prime_vunmap - &dma_buf_ops.vunmap implementation 66 * @obj: GEM BO 67 * @vaddr: Virtual address (unused) 68 * 69 * Tears down the in-kernel virtual mapping of the BO's memory. 70 */ 71 void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) 72 { 73 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 74 75 ttm_bo_kunmap(&bo->dma_buf_vmap); 76 } 77 78 /** 79 * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation 80 * @obj: GEM BO 81 * @vma: Virtual memory area 82 * 83 * Sets up a userspace mapping of the BO's memory in the given 84 * virtual memory area. 85 * 86 * Returns: 87 * 0 on success or a negative error code on failure. 88 */ 89 int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, 90 struct vm_area_struct *vma) 91 { 92 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 93 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 94 unsigned asize = amdgpu_bo_size(bo); 95 int ret; 96 97 if (!vma->vm_file) 98 return -ENODEV; 99 100 if (adev == NULL) 101 return -ENODEV; 102 103 /* Check for valid size. */ 104 if (asize < vma->vm_end - vma->vm_start) 105 return -EINVAL; 106 107 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || 108 (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { 109 return -EPERM; 110 } 111 vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT; 112 113 /* prime mmap does not need to check access, so allow here */ 114 ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data); 115 if (ret) 116 return ret; 117 118 ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev); 119 drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data); 120 121 return ret; 122 } 123 124 static int 125 __dma_resv_make_exclusive(struct dma_resv *obj) 126 { 127 struct dma_fence **fences; 128 unsigned int count; 129 int r; 130 131 if (!dma_resv_get_list(obj)) /* no shared fences to convert */ 132 return 0; 133 134 r = dma_resv_get_fences_rcu(obj, NULL, &count, &fences); 135 if (r) 136 return r; 137 138 if (count == 0) { 139 /* Now that was unexpected. */ 140 } else if (count == 1) { 141 dma_resv_add_excl_fence(obj, fences[0]); 142 dma_fence_put(fences[0]); 143 kfree(fences); 144 } else { 145 struct dma_fence_array *array; 146 147 array = dma_fence_array_create(count, fences, 148 dma_fence_context_alloc(1), 0, 149 false); 150 if (!array) 151 goto err_fences_put; 152 153 dma_resv_add_excl_fence(obj, &array->base); 154 dma_fence_put(&array->base); 155 } 156 157 return 0; 158 159 err_fences_put: 160 while (count--) 161 dma_fence_put(fences[count]); 162 kfree(fences); 163 return -ENOMEM; 164 } 165 166 /** 167 * amdgpu_dma_buf_attach - &dma_buf_ops.attach implementation 168 * 169 * @dmabuf: DMA-buf where we attach to 170 * @attach: attachment to add 171 * 172 * Add the attachment as user to the exported DMA-buf. 173 */ 174 static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, 175 struct dma_buf_attachment *attach) 176 { 177 struct drm_gem_object *obj = dmabuf->priv; 178 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 179 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 180 int r; 181 182 if (attach->dev->driver == adev->dev->driver) 183 return 0; 184 185 r = amdgpu_bo_reserve(bo, false); 186 if (unlikely(r != 0)) 187 return r; 188 189 /* 190 * We only create shared fences for internal use, but importers 191 * of the dmabuf rely on exclusive fences for implicitly 192 * tracking write hazards. As any of the current fences may 193 * correspond to a write, we need to convert all existing 194 * fences on the reservation object into a single exclusive 195 * fence. 196 */ 197 r = __dma_resv_make_exclusive(bo->tbo.base.resv); 198 if (r) 199 return r; 200 201 bo->prime_shared_count++; 202 amdgpu_bo_unreserve(bo); 203 return 0; 204 } 205 206 /** 207 * amdgpu_dma_buf_detach - &dma_buf_ops.detach implementation 208 * 209 * @dmabuf: DMA-buf where we remove the attachment from 210 * @attach: the attachment to remove 211 * 212 * Called when an attachment is removed from the DMA-buf. 213 */ 214 static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf, 215 struct dma_buf_attachment *attach) 216 { 217 struct drm_gem_object *obj = dmabuf->priv; 218 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 219 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 220 221 if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count) 222 bo->prime_shared_count--; 223 } 224 225 /** 226 * amdgpu_dma_buf_pin - &dma_buf_ops.pin implementation 227 * 228 * @attach: attachment to pin down 229 * 230 * Pin the BO which is backing the DMA-buf so that it can't move any more. 231 */ 232 static int amdgpu_dma_buf_pin(struct dma_buf_attachment *attach) 233 { 234 struct drm_gem_object *obj = attach->dmabuf->priv; 235 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 236 237 /* pin buffer into GTT */ 238 return amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 239 } 240 241 /** 242 * amdgpu_dma_buf_unpin - &dma_buf_ops.unpin implementation 243 * 244 * @attach: attachment to unpin 245 * 246 * Unpin a previously pinned BO to make it movable again. 247 */ 248 static void amdgpu_dma_buf_unpin(struct dma_buf_attachment *attach) 249 { 250 struct drm_gem_object *obj = attach->dmabuf->priv; 251 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 252 253 amdgpu_bo_unpin(bo); 254 } 255 256 /** 257 * amdgpu_dma_buf_map - &dma_buf_ops.map_dma_buf implementation 258 * @attach: DMA-buf attachment 259 * @dir: DMA direction 260 * 261 * Makes sure that the shared DMA buffer can be accessed by the target device. 262 * For now, simply pins it to the GTT domain, where it should be accessible by 263 * all DMA devices. 264 * 265 * Returns: 266 * sg_table filled with the DMA addresses to use or ERR_PRT with negative error 267 * code. 268 */ 269 static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach, 270 enum dma_data_direction dir) 271 { 272 struct dma_buf *dma_buf = attach->dmabuf; 273 struct drm_gem_object *obj = dma_buf->priv; 274 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 275 struct sg_table *sgt; 276 long r; 277 278 if (!bo->pin_count) { 279 /* move buffer into GTT */ 280 struct ttm_operation_ctx ctx = { false, false }; 281 282 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 283 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 284 if (r) 285 return ERR_PTR(r); 286 287 } else if (!(amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type) & 288 AMDGPU_GEM_DOMAIN_GTT)) { 289 return ERR_PTR(-EBUSY); 290 } 291 292 sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages, bo->tbo.num_pages); 293 if (IS_ERR(sgt)) 294 return sgt; 295 296 if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir, 297 DMA_ATTR_SKIP_CPU_SYNC)) 298 goto error_free; 299 300 return sgt; 301 302 error_free: 303 sg_free_table(sgt); 304 kfree(sgt); 305 return ERR_PTR(-ENOMEM); 306 } 307 308 /** 309 * amdgpu_dma_buf_unmap - &dma_buf_ops.unmap_dma_buf implementation 310 * @attach: DMA-buf attachment 311 * @sgt: sg_table to unmap 312 * @dir: DMA direction 313 * 314 * This is called when a shared DMA buffer no longer needs to be accessible by 315 * another device. For now, simply unpins the buffer from GTT. 316 */ 317 static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach, 318 struct sg_table *sgt, 319 enum dma_data_direction dir) 320 { 321 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); 322 sg_free_table(sgt); 323 kfree(sgt); 324 } 325 326 /** 327 * amdgpu_dma_buf_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation 328 * @dma_buf: Shared DMA buffer 329 * @direction: Direction of DMA transfer 330 * 331 * This is called before CPU access to the shared DMA buffer's memory. If it's 332 * a read access, the buffer is moved to the GTT domain if possible, for optimal 333 * CPU read performance. 334 * 335 * Returns: 336 * 0 on success or a negative error code on failure. 337 */ 338 static int amdgpu_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, 339 enum dma_data_direction direction) 340 { 341 struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv); 342 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 343 struct ttm_operation_ctx ctx = { true, false }; 344 u32 domain = amdgpu_display_supported_domains(adev, bo->flags); 345 int ret; 346 bool reads = (direction == DMA_BIDIRECTIONAL || 347 direction == DMA_FROM_DEVICE); 348 349 if (!reads || !(domain & AMDGPU_GEM_DOMAIN_GTT)) 350 return 0; 351 352 /* move to gtt */ 353 ret = amdgpu_bo_reserve(bo, false); 354 if (unlikely(ret != 0)) 355 return ret; 356 357 if (!bo->pin_count && (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) { 358 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 359 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 360 } 361 362 amdgpu_bo_unreserve(bo); 363 return ret; 364 } 365 366 const struct dma_buf_ops amdgpu_dmabuf_ops = { 367 .attach = amdgpu_dma_buf_attach, 368 .detach = amdgpu_dma_buf_detach, 369 .pin = amdgpu_dma_buf_pin, 370 .unpin = amdgpu_dma_buf_unpin, 371 .map_dma_buf = amdgpu_dma_buf_map, 372 .unmap_dma_buf = amdgpu_dma_buf_unmap, 373 .release = drm_gem_dmabuf_release, 374 .begin_cpu_access = amdgpu_dma_buf_begin_cpu_access, 375 .mmap = drm_gem_dmabuf_mmap, 376 .vmap = drm_gem_dmabuf_vmap, 377 .vunmap = drm_gem_dmabuf_vunmap, 378 }; 379 380 /** 381 * amdgpu_gem_prime_export - &drm_driver.gem_prime_export implementation 382 * @gobj: GEM BO 383 * @flags: Flags such as DRM_CLOEXEC and DRM_RDWR. 384 * 385 * The main work is done by the &drm_gem_prime_export helper. 386 * 387 * Returns: 388 * Shared DMA buffer representing the GEM BO from the given device. 389 */ 390 struct dma_buf *amdgpu_gem_prime_export(struct drm_gem_object *gobj, 391 int flags) 392 { 393 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); 394 struct dma_buf *buf; 395 396 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || 397 bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) 398 return ERR_PTR(-EPERM); 399 400 buf = drm_gem_prime_export(gobj, flags); 401 if (!IS_ERR(buf)) 402 buf->ops = &amdgpu_dmabuf_ops; 403 404 return buf; 405 } 406 407 /** 408 * amdgpu_dma_buf_create_obj - create BO for DMA-buf import 409 * 410 * @dev: DRM device 411 * @dma_buf: DMA-buf 412 * 413 * Creates an empty SG BO for DMA-buf import. 414 * 415 * Returns: 416 * A new GEM BO of the given DRM device, representing the memory 417 * described by the given DMA-buf attachment and scatter/gather table. 418 */ 419 static struct drm_gem_object * 420 amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) 421 { 422 struct dma_resv *resv = dma_buf->resv; 423 struct amdgpu_device *adev = dev->dev_private; 424 struct amdgpu_bo *bo; 425 struct amdgpu_bo_param bp; 426 int ret; 427 428 memset(&bp, 0, sizeof(bp)); 429 bp.size = dma_buf->size; 430 bp.byte_align = PAGE_SIZE; 431 bp.domain = AMDGPU_GEM_DOMAIN_CPU; 432 bp.flags = 0; 433 bp.type = ttm_bo_type_sg; 434 bp.resv = resv; 435 dma_resv_lock(resv, NULL); 436 ret = amdgpu_bo_create(adev, &bp, &bo); 437 if (ret) 438 goto error; 439 440 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 441 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 442 if (dma_buf->ops != &amdgpu_dmabuf_ops) 443 bo->prime_shared_count = 1; 444 445 dma_resv_unlock(resv); 446 return &bo->tbo.base; 447 448 error: 449 dma_resv_unlock(resv); 450 return ERR_PTR(ret); 451 } 452 453 /** 454 * amdgpu_dma_buf_move_notify - &attach.move_notify implementation 455 * 456 * @attach: the DMA-buf attachment 457 * 458 * Invalidate the DMA-buf attachment, making sure that the we re-create the 459 * mapping before the next use. 460 */ 461 static void 462 amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach) 463 { 464 struct drm_gem_object *obj = attach->importer_priv; 465 struct ww_acquire_ctx *ticket = dma_resv_locking_ctx(obj->resv); 466 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 467 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 468 struct ttm_operation_ctx ctx = { false, false }; 469 struct ttm_placement placement = {}; 470 struct amdgpu_vm_bo_base *bo_base; 471 int r; 472 473 if (bo->tbo.mem.mem_type == TTM_PL_SYSTEM) 474 return; 475 476 r = ttm_bo_validate(&bo->tbo, &placement, &ctx); 477 if (r) { 478 DRM_ERROR("Failed to invalidate DMA-buf import (%d))\n", r); 479 return; 480 } 481 482 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 483 struct amdgpu_vm *vm = bo_base->vm; 484 struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; 485 486 if (ticket) { 487 /* When we get an error here it means that somebody 488 * else is holding the VM lock and updating page tables 489 * So we can just continue here. 490 */ 491 r = dma_resv_lock(resv, ticket); 492 if (r) 493 continue; 494 495 } else { 496 /* TODO: This is more problematic and we actually need 497 * to allow page tables updates without holding the 498 * lock. 499 */ 500 if (!dma_resv_trylock(resv)) 501 continue; 502 } 503 504 r = amdgpu_vm_clear_freed(adev, vm, NULL); 505 if (!r) 506 r = amdgpu_vm_handle_moved(adev, vm); 507 508 if (r && r != -EBUSY) 509 DRM_ERROR("Failed to invalidate VM page tables (%d))\n", 510 r); 511 512 dma_resv_unlock(resv); 513 } 514 } 515 516 static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops = { 517 .move_notify = amdgpu_dma_buf_move_notify 518 }; 519 520 /** 521 * amdgpu_gem_prime_import - &drm_driver.gem_prime_import implementation 522 * @dev: DRM device 523 * @dma_buf: Shared DMA buffer 524 * 525 * Import a dma_buf into a the driver and potentially create a new GEM object. 526 * 527 * Returns: 528 * GEM BO representing the shared DMA buffer for the given device. 529 */ 530 struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, 531 struct dma_buf *dma_buf) 532 { 533 struct dma_buf_attachment *attach; 534 struct drm_gem_object *obj; 535 536 if (dma_buf->ops == &amdgpu_dmabuf_ops) { 537 obj = dma_buf->priv; 538 if (obj->dev == dev) { 539 /* 540 * Importing dmabuf exported from out own gem increases 541 * refcount on gem itself instead of f_count of dmabuf. 542 */ 543 drm_gem_object_get(obj); 544 return obj; 545 } 546 } 547 548 obj = amdgpu_dma_buf_create_obj(dev, dma_buf); 549 if (IS_ERR(obj)) 550 return obj; 551 552 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, 553 &amdgpu_dma_buf_attach_ops, obj); 554 if (IS_ERR(attach)) { 555 drm_gem_object_put(obj); 556 return ERR_CAST(attach); 557 } 558 559 get_dma_buf(dma_buf); 560 obj->import_attach = attach; 561 return obj; 562 } 563