1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * based on nouveau_prime.c 23 * 24 * Authors: Alex Deucher 25 */ 26 27 /** 28 * DOC: PRIME Buffer Sharing 29 * 30 * The following callback implementations are used for :ref:`sharing GEM buffer 31 * objects between different devices via PRIME <prime_buffer_sharing>`. 32 */ 33 34 #include "amdgpu.h" 35 #include "amdgpu_display.h" 36 #include "amdgpu_gem.h" 37 #include "amdgpu_dma_buf.h" 38 #include <drm/amdgpu_drm.h> 39 #include <linux/dma-buf.h> 40 #include <linux/dma-fence-array.h> 41 42 /** 43 * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation 44 * @obj: GEM BO 45 * 46 * Sets up an in-kernel virtual mapping of the BO's memory. 47 * 48 * Returns: 49 * The virtual address of the mapping or an error pointer. 50 */ 51 void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj) 52 { 53 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 54 int ret; 55 56 ret = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, 57 &bo->dma_buf_vmap); 58 if (ret) 59 return ERR_PTR(ret); 60 61 return bo->dma_buf_vmap.virtual; 62 } 63 64 /** 65 * amdgpu_gem_prime_vunmap - &dma_buf_ops.vunmap implementation 66 * @obj: GEM BO 67 * @vaddr: Virtual address (unused) 68 * 69 * Tears down the in-kernel virtual mapping of the BO's memory. 70 */ 71 void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) 72 { 73 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 74 75 ttm_bo_kunmap(&bo->dma_buf_vmap); 76 } 77 78 /** 79 * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation 80 * @obj: GEM BO 81 * @vma: Virtual memory area 82 * 83 * Sets up a userspace mapping of the BO's memory in the given 84 * virtual memory area. 85 * 86 * Returns: 87 * 0 on success or a negative error code on failure. 88 */ 89 int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, 90 struct vm_area_struct *vma) 91 { 92 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 93 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 94 unsigned asize = amdgpu_bo_size(bo); 95 int ret; 96 97 if (!vma->vm_file) 98 return -ENODEV; 99 100 if (adev == NULL) 101 return -ENODEV; 102 103 /* Check for valid size. */ 104 if (asize < vma->vm_end - vma->vm_start) 105 return -EINVAL; 106 107 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || 108 (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { 109 return -EPERM; 110 } 111 vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT; 112 113 /* prime mmap does not need to check access, so allow here */ 114 ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data); 115 if (ret) 116 return ret; 117 118 ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev); 119 drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data); 120 121 return ret; 122 } 123 124 static int 125 __dma_resv_make_exclusive(struct dma_resv *obj) 126 { 127 struct dma_fence **fences; 128 unsigned int count; 129 int r; 130 131 if (!dma_resv_get_list(obj)) /* no shared fences to convert */ 132 return 0; 133 134 r = dma_resv_get_fences_rcu(obj, NULL, &count, &fences); 135 if (r) 136 return r; 137 138 if (count == 0) { 139 /* Now that was unexpected. */ 140 } else if (count == 1) { 141 dma_resv_add_excl_fence(obj, fences[0]); 142 dma_fence_put(fences[0]); 143 kfree(fences); 144 } else { 145 struct dma_fence_array *array; 146 147 array = dma_fence_array_create(count, fences, 148 dma_fence_context_alloc(1), 0, 149 false); 150 if (!array) 151 goto err_fences_put; 152 153 dma_resv_add_excl_fence(obj, &array->base); 154 dma_fence_put(&array->base); 155 } 156 157 return 0; 158 159 err_fences_put: 160 while (count--) 161 dma_fence_put(fences[count]); 162 kfree(fences); 163 return -ENOMEM; 164 } 165 166 /** 167 * amdgpu_dma_buf_attach - &dma_buf_ops.attach implementation 168 * 169 * @dmabuf: DMA-buf where we attach to 170 * @attach: attachment to add 171 * 172 * Add the attachment as user to the exported DMA-buf. 173 */ 174 static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, 175 struct dma_buf_attachment *attach) 176 { 177 struct drm_gem_object *obj = dmabuf->priv; 178 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 179 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 180 int r; 181 182 if (attach->dev->driver == adev->dev->driver) 183 return 0; 184 185 r = amdgpu_bo_reserve(bo, false); 186 if (unlikely(r != 0)) 187 return r; 188 189 /* 190 * We only create shared fences for internal use, but importers 191 * of the dmabuf rely on exclusive fences for implicitly 192 * tracking write hazards. As any of the current fences may 193 * correspond to a write, we need to convert all existing 194 * fences on the reservation object into a single exclusive 195 * fence. 196 */ 197 r = __dma_resv_make_exclusive(bo->tbo.base.resv); 198 if (r) 199 return r; 200 201 bo->prime_shared_count++; 202 amdgpu_bo_unreserve(bo); 203 return 0; 204 } 205 206 /** 207 * amdgpu_dma_buf_detach - &dma_buf_ops.detach implementation 208 * 209 * @dmabuf: DMA-buf where we remove the attachment from 210 * @attach: the attachment to remove 211 * 212 * Called when an attachment is removed from the DMA-buf. 213 */ 214 static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf, 215 struct dma_buf_attachment *attach) 216 { 217 struct drm_gem_object *obj = dmabuf->priv; 218 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 219 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 220 221 if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count) 222 bo->prime_shared_count--; 223 } 224 225 /** 226 * amdgpu_dma_buf_map - &dma_buf_ops.map_dma_buf implementation 227 * @attach: DMA-buf attachment 228 * @dir: DMA direction 229 * 230 * Makes sure that the shared DMA buffer can be accessed by the target device. 231 * For now, simply pins it to the GTT domain, where it should be accessible by 232 * all DMA devices. 233 * 234 * Returns: 235 * sg_table filled with the DMA addresses to use or ERR_PRT with negative error 236 * code. 237 */ 238 static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach, 239 enum dma_data_direction dir) 240 { 241 struct dma_buf *dma_buf = attach->dmabuf; 242 struct drm_gem_object *obj = dma_buf->priv; 243 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 244 struct sg_table *sgt; 245 long r; 246 247 r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 248 if (r) 249 return ERR_PTR(r); 250 251 sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages, bo->tbo.num_pages); 252 if (IS_ERR(sgt)) 253 return sgt; 254 255 if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir, 256 DMA_ATTR_SKIP_CPU_SYNC)) 257 goto error_free; 258 259 return sgt; 260 261 error_free: 262 sg_free_table(sgt); 263 kfree(sgt); 264 return ERR_PTR(-ENOMEM); 265 } 266 267 /** 268 * amdgpu_dma_buf_unmap - &dma_buf_ops.unmap_dma_buf implementation 269 * @attach: DMA-buf attachment 270 * @sgt: sg_table to unmap 271 * @dir: DMA direction 272 * 273 * This is called when a shared DMA buffer no longer needs to be accessible by 274 * another device. For now, simply unpins the buffer from GTT. 275 */ 276 static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach, 277 struct sg_table *sgt, 278 enum dma_data_direction dir) 279 { 280 struct drm_gem_object *obj = attach->dmabuf->priv; 281 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 282 283 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); 284 sg_free_table(sgt); 285 kfree(sgt); 286 amdgpu_bo_unpin(bo); 287 } 288 289 /** 290 * amdgpu_dma_buf_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation 291 * @dma_buf: Shared DMA buffer 292 * @direction: Direction of DMA transfer 293 * 294 * This is called before CPU access to the shared DMA buffer's memory. If it's 295 * a read access, the buffer is moved to the GTT domain if possible, for optimal 296 * CPU read performance. 297 * 298 * Returns: 299 * 0 on success or a negative error code on failure. 300 */ 301 static int amdgpu_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, 302 enum dma_data_direction direction) 303 { 304 struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv); 305 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 306 struct ttm_operation_ctx ctx = { true, false }; 307 u32 domain = amdgpu_display_supported_domains(adev, bo->flags); 308 int ret; 309 bool reads = (direction == DMA_BIDIRECTIONAL || 310 direction == DMA_FROM_DEVICE); 311 312 if (!reads || !(domain & AMDGPU_GEM_DOMAIN_GTT)) 313 return 0; 314 315 /* move to gtt */ 316 ret = amdgpu_bo_reserve(bo, false); 317 if (unlikely(ret != 0)) 318 return ret; 319 320 if (!bo->pin_count && (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) { 321 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 322 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 323 } 324 325 amdgpu_bo_unreserve(bo); 326 return ret; 327 } 328 329 const struct dma_buf_ops amdgpu_dmabuf_ops = { 330 .dynamic_mapping = true, 331 .attach = amdgpu_dma_buf_attach, 332 .detach = amdgpu_dma_buf_detach, 333 .map_dma_buf = amdgpu_dma_buf_map, 334 .unmap_dma_buf = amdgpu_dma_buf_unmap, 335 .release = drm_gem_dmabuf_release, 336 .begin_cpu_access = amdgpu_dma_buf_begin_cpu_access, 337 .mmap = drm_gem_dmabuf_mmap, 338 .vmap = drm_gem_dmabuf_vmap, 339 .vunmap = drm_gem_dmabuf_vunmap, 340 }; 341 342 /** 343 * amdgpu_gem_prime_export - &drm_driver.gem_prime_export implementation 344 * @gobj: GEM BO 345 * @flags: Flags such as DRM_CLOEXEC and DRM_RDWR. 346 * 347 * The main work is done by the &drm_gem_prime_export helper. 348 * 349 * Returns: 350 * Shared DMA buffer representing the GEM BO from the given device. 351 */ 352 struct dma_buf *amdgpu_gem_prime_export(struct drm_gem_object *gobj, 353 int flags) 354 { 355 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); 356 struct dma_buf *buf; 357 358 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || 359 bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) 360 return ERR_PTR(-EPERM); 361 362 buf = drm_gem_prime_export(gobj, flags); 363 if (!IS_ERR(buf)) 364 buf->ops = &amdgpu_dmabuf_ops; 365 366 return buf; 367 } 368 369 /** 370 * amdgpu_dma_buf_create_obj - create BO for DMA-buf import 371 * 372 * @dev: DRM device 373 * @dma_buf: DMA-buf 374 * 375 * Creates an empty SG BO for DMA-buf import. 376 * 377 * Returns: 378 * A new GEM BO of the given DRM device, representing the memory 379 * described by the given DMA-buf attachment and scatter/gather table. 380 */ 381 static struct drm_gem_object * 382 amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) 383 { 384 struct dma_resv *resv = dma_buf->resv; 385 struct amdgpu_device *adev = dev->dev_private; 386 struct amdgpu_bo *bo; 387 struct amdgpu_bo_param bp; 388 int ret; 389 390 memset(&bp, 0, sizeof(bp)); 391 bp.size = dma_buf->size; 392 bp.byte_align = PAGE_SIZE; 393 bp.domain = AMDGPU_GEM_DOMAIN_CPU; 394 bp.flags = 0; 395 bp.type = ttm_bo_type_sg; 396 bp.resv = resv; 397 dma_resv_lock(resv, NULL); 398 ret = amdgpu_bo_create(adev, &bp, &bo); 399 if (ret) 400 goto error; 401 402 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 403 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 404 if (dma_buf->ops != &amdgpu_dmabuf_ops) 405 bo->prime_shared_count = 1; 406 407 dma_resv_unlock(resv); 408 return &bo->tbo.base; 409 410 error: 411 dma_resv_unlock(resv); 412 return ERR_PTR(ret); 413 } 414 415 /** 416 * amdgpu_gem_prime_import - &drm_driver.gem_prime_import implementation 417 * @dev: DRM device 418 * @dma_buf: Shared DMA buffer 419 * 420 * Import a dma_buf into a the driver and potentially create a new GEM object. 421 * 422 * Returns: 423 * GEM BO representing the shared DMA buffer for the given device. 424 */ 425 struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, 426 struct dma_buf *dma_buf) 427 { 428 struct dma_buf_attachment *attach; 429 struct drm_gem_object *obj; 430 431 if (dma_buf->ops == &amdgpu_dmabuf_ops) { 432 obj = dma_buf->priv; 433 if (obj->dev == dev) { 434 /* 435 * Importing dmabuf exported from out own gem increases 436 * refcount on gem itself instead of f_count of dmabuf. 437 */ 438 drm_gem_object_get(obj); 439 return obj; 440 } 441 } 442 443 obj = amdgpu_dma_buf_create_obj(dev, dma_buf); 444 if (IS_ERR(obj)) 445 return obj; 446 447 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, true); 448 if (IS_ERR(attach)) { 449 drm_gem_object_put(obj); 450 return ERR_CAST(attach); 451 } 452 453 get_dma_buf(dma_buf); 454 obj->import_attach = attach; 455 return obj; 456 } 457