1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/set_memory.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_cache.h> 13 #include <drm/drm_debugfs.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_utils.h> 16 17 #include "ivpu_drv.h" 18 #include "ivpu_gem.h" 19 #include "ivpu_hw.h" 20 #include "ivpu_mmu.h" 21 #include "ivpu_mmu_context.h" 22 23 MODULE_IMPORT_NS(DMA_BUF); 24 25 static const struct drm_gem_object_funcs ivpu_gem_funcs; 26 27 static struct lock_class_key prime_bo_lock_class_key; 28 29 static int __must_check prime_alloc_pages_locked(struct ivpu_bo *bo) 30 { 31 /* Pages are managed by the underlying dma-buf */ 32 return 0; 33 } 34 35 static void prime_free_pages_locked(struct ivpu_bo *bo) 36 { 37 /* Pages are managed by the underlying dma-buf */ 38 } 39 40 static int prime_map_pages_locked(struct ivpu_bo *bo) 41 { 42 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 43 struct sg_table *sgt; 44 45 sgt = dma_buf_map_attachment_unlocked(bo->base.import_attach, DMA_BIDIRECTIONAL); 46 if (IS_ERR(sgt)) { 47 ivpu_err(vdev, "Failed to map attachment: %ld\n", PTR_ERR(sgt)); 48 return PTR_ERR(sgt); 49 } 50 51 bo->sgt = sgt; 52 return 0; 53 } 54 55 static void prime_unmap_pages_locked(struct ivpu_bo *bo) 56 { 57 dma_buf_unmap_attachment_unlocked(bo->base.import_attach, bo->sgt, DMA_BIDIRECTIONAL); 58 bo->sgt = NULL; 59 } 60 61 static const struct ivpu_bo_ops prime_ops = { 62 .type = IVPU_BO_TYPE_PRIME, 63 .name = "prime", 64 .alloc_pages = prime_alloc_pages_locked, 65 .free_pages = prime_free_pages_locked, 66 .map_pages = prime_map_pages_locked, 67 .unmap_pages = prime_unmap_pages_locked, 68 }; 69 70 static int __must_check shmem_alloc_pages_locked(struct ivpu_bo *bo) 71 { 72 int npages = bo->base.size >> PAGE_SHIFT; 73 struct page **pages; 74 75 pages = drm_gem_get_pages(&bo->base); 76 if (IS_ERR(pages)) 77 return PTR_ERR(pages); 78 79 if (bo->flags & DRM_IVPU_BO_WC) 80 set_pages_array_wc(pages, npages); 81 else if (bo->flags & DRM_IVPU_BO_UNCACHED) 82 set_pages_array_uc(pages, npages); 83 84 bo->pages = pages; 85 return 0; 86 } 87 88 static void shmem_free_pages_locked(struct ivpu_bo *bo) 89 { 90 if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED) 91 set_pages_array_wb(bo->pages, bo->base.size >> PAGE_SHIFT); 92 93 drm_gem_put_pages(&bo->base, bo->pages, true, false); 94 bo->pages = NULL; 95 } 96 97 static int ivpu_bo_map_pages_locked(struct ivpu_bo *bo) 98 { 99 int npages = bo->base.size >> PAGE_SHIFT; 100 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 101 struct sg_table *sgt; 102 int ret; 103 104 sgt = drm_prime_pages_to_sg(&vdev->drm, bo->pages, npages); 105 if (IS_ERR(sgt)) { 106 ivpu_err(vdev, "Failed to allocate sgtable\n"); 107 return PTR_ERR(sgt); 108 } 109 110 ret = dma_map_sgtable(vdev->drm.dev, sgt, DMA_BIDIRECTIONAL, 0); 111 if (ret) { 112 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); 113 goto err_free_sgt; 114 } 115 116 bo->sgt = sgt; 117 return 0; 118 119 err_free_sgt: 120 kfree(sgt); 121 return ret; 122 } 123 124 static void ivpu_bo_unmap_pages_locked(struct ivpu_bo *bo) 125 { 126 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 127 128 dma_unmap_sgtable(vdev->drm.dev, bo->sgt, DMA_BIDIRECTIONAL, 0); 129 sg_free_table(bo->sgt); 130 kfree(bo->sgt); 131 bo->sgt = NULL; 132 } 133 134 static const struct ivpu_bo_ops shmem_ops = { 135 .type = IVPU_BO_TYPE_SHMEM, 136 .name = "shmem", 137 .alloc_pages = shmem_alloc_pages_locked, 138 .free_pages = shmem_free_pages_locked, 139 .map_pages = ivpu_bo_map_pages_locked, 140 .unmap_pages = ivpu_bo_unmap_pages_locked, 141 }; 142 143 static int __must_check internal_alloc_pages_locked(struct ivpu_bo *bo) 144 { 145 unsigned int i, npages = bo->base.size >> PAGE_SHIFT; 146 struct page **pages; 147 int ret; 148 149 pages = kvmalloc_array(npages, sizeof(*bo->pages), GFP_KERNEL); 150 if (!pages) 151 return -ENOMEM; 152 153 for (i = 0; i < npages; i++) { 154 pages[i] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); 155 if (!pages[i]) { 156 ret = -ENOMEM; 157 goto err_free_pages; 158 } 159 cond_resched(); 160 } 161 162 bo->pages = pages; 163 return 0; 164 165 err_free_pages: 166 while (i--) 167 put_page(pages[i]); 168 kvfree(pages); 169 return ret; 170 } 171 172 static void internal_free_pages_locked(struct ivpu_bo *bo) 173 { 174 unsigned int i, npages = bo->base.size >> PAGE_SHIFT; 175 176 if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED) 177 set_pages_array_wb(bo->pages, bo->base.size >> PAGE_SHIFT); 178 179 for (i = 0; i < npages; i++) 180 put_page(bo->pages[i]); 181 182 kvfree(bo->pages); 183 bo->pages = NULL; 184 } 185 186 static const struct ivpu_bo_ops internal_ops = { 187 .type = IVPU_BO_TYPE_INTERNAL, 188 .name = "internal", 189 .alloc_pages = internal_alloc_pages_locked, 190 .free_pages = internal_free_pages_locked, 191 .map_pages = ivpu_bo_map_pages_locked, 192 .unmap_pages = ivpu_bo_unmap_pages_locked, 193 }; 194 195 static int __must_check ivpu_bo_alloc_and_map_pages_locked(struct ivpu_bo *bo) 196 { 197 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 198 int ret; 199 200 lockdep_assert_held(&bo->lock); 201 drm_WARN_ON(&vdev->drm, bo->sgt); 202 203 ret = bo->ops->alloc_pages(bo); 204 if (ret) { 205 ivpu_err(vdev, "Failed to allocate pages for BO: %d", ret); 206 return ret; 207 } 208 209 ret = bo->ops->map_pages(bo); 210 if (ret) { 211 ivpu_err(vdev, "Failed to map pages for BO: %d", ret); 212 goto err_free_pages; 213 } 214 return ret; 215 216 err_free_pages: 217 bo->ops->free_pages(bo); 218 return ret; 219 } 220 221 static void ivpu_bo_unmap_and_free_pages(struct ivpu_bo *bo) 222 { 223 mutex_lock(&bo->lock); 224 225 WARN_ON(!bo->sgt); 226 bo->ops->unmap_pages(bo); 227 WARN_ON(bo->sgt); 228 bo->ops->free_pages(bo); 229 WARN_ON(bo->pages); 230 231 mutex_unlock(&bo->lock); 232 } 233 234 /* 235 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. 236 * 237 * This function pins physical memory pages, then maps the physical pages 238 * to IOMMU address space and finally updates the VPU MMU page tables 239 * to allow the VPU to translate VPU address to IOMMU address. 240 */ 241 int __must_check ivpu_bo_pin(struct ivpu_bo *bo) 242 { 243 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 244 int ret = 0; 245 246 mutex_lock(&bo->lock); 247 248 if (!bo->vpu_addr) { 249 ivpu_err(vdev, "vpu_addr not set for BO ctx_id: %d handle: %d\n", 250 bo->ctx->id, bo->handle); 251 ret = -EINVAL; 252 goto unlock; 253 } 254 255 if (!bo->sgt) { 256 ret = ivpu_bo_alloc_and_map_pages_locked(bo); 257 if (ret) 258 goto unlock; 259 } 260 261 if (!bo->mmu_mapped) { 262 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, bo->sgt, 263 ivpu_bo_is_snooped(bo)); 264 if (ret) { 265 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); 266 goto unlock; 267 } 268 bo->mmu_mapped = true; 269 } 270 271 unlock: 272 mutex_unlock(&bo->lock); 273 274 return ret; 275 } 276 277 static int 278 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, 279 const struct ivpu_addr_range *range) 280 { 281 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 282 int ret; 283 284 if (!range) { 285 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) 286 range = &vdev->hw->ranges.shave; 287 else if (bo->flags & DRM_IVPU_BO_DMA_MEM) 288 range = &vdev->hw->ranges.dma; 289 else 290 range = &vdev->hw->ranges.user; 291 } 292 293 mutex_lock(&ctx->lock); 294 ret = ivpu_mmu_context_insert_node_locked(ctx, range, bo->base.size, &bo->mm_node); 295 if (!ret) { 296 bo->ctx = ctx; 297 bo->vpu_addr = bo->mm_node.start; 298 list_add_tail(&bo->ctx_node, &ctx->bo_list); 299 } 300 mutex_unlock(&ctx->lock); 301 302 return ret; 303 } 304 305 static void ivpu_bo_free_vpu_addr(struct ivpu_bo *bo) 306 { 307 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 308 struct ivpu_mmu_context *ctx = bo->ctx; 309 310 ivpu_dbg(vdev, BO, "remove from ctx: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n", 311 ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped); 312 313 mutex_lock(&bo->lock); 314 315 if (bo->mmu_mapped) { 316 drm_WARN_ON(&vdev->drm, !bo->sgt); 317 ivpu_mmu_context_unmap_sgt(vdev, ctx, bo->vpu_addr, bo->sgt); 318 bo->mmu_mapped = false; 319 } 320 321 mutex_lock(&ctx->lock); 322 list_del(&bo->ctx_node); 323 bo->vpu_addr = 0; 324 bo->ctx = NULL; 325 ivpu_mmu_context_remove_node_locked(ctx, &bo->mm_node); 326 mutex_unlock(&ctx->lock); 327 328 mutex_unlock(&bo->lock); 329 } 330 331 void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx) 332 { 333 struct ivpu_bo *bo, *tmp; 334 335 list_for_each_entry_safe(bo, tmp, &ctx->bo_list, ctx_node) 336 ivpu_bo_free_vpu_addr(bo); 337 } 338 339 static struct ivpu_bo * 340 ivpu_bo_alloc(struct ivpu_device *vdev, struct ivpu_mmu_context *mmu_context, 341 u64 size, u32 flags, const struct ivpu_bo_ops *ops, 342 const struct ivpu_addr_range *range, u64 user_ptr) 343 { 344 struct ivpu_bo *bo; 345 int ret = 0; 346 347 if (drm_WARN_ON(&vdev->drm, size == 0 || !PAGE_ALIGNED(size))) 348 return ERR_PTR(-EINVAL); 349 350 switch (flags & DRM_IVPU_BO_CACHE_MASK) { 351 case DRM_IVPU_BO_CACHED: 352 case DRM_IVPU_BO_UNCACHED: 353 case DRM_IVPU_BO_WC: 354 break; 355 default: 356 return ERR_PTR(-EINVAL); 357 } 358 359 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 360 if (!bo) 361 return ERR_PTR(-ENOMEM); 362 363 mutex_init(&bo->lock); 364 bo->base.funcs = &ivpu_gem_funcs; 365 bo->flags = flags; 366 bo->ops = ops; 367 bo->user_ptr = user_ptr; 368 369 if (ops->type == IVPU_BO_TYPE_SHMEM) 370 ret = drm_gem_object_init(&vdev->drm, &bo->base, size); 371 else 372 drm_gem_private_object_init(&vdev->drm, &bo->base, size); 373 374 if (ret) { 375 ivpu_err(vdev, "Failed to initialize drm object\n"); 376 goto err_free; 377 } 378 379 if (flags & DRM_IVPU_BO_MAPPABLE) { 380 ret = drm_gem_create_mmap_offset(&bo->base); 381 if (ret) { 382 ivpu_err(vdev, "Failed to allocate mmap offset\n"); 383 goto err_release; 384 } 385 } 386 387 if (mmu_context) { 388 ret = ivpu_bo_alloc_vpu_addr(bo, mmu_context, range); 389 if (ret) { 390 ivpu_err(vdev, "Failed to add BO to context: %d\n", ret); 391 goto err_release; 392 } 393 } 394 395 return bo; 396 397 err_release: 398 drm_gem_object_release(&bo->base); 399 err_free: 400 kfree(bo); 401 return ERR_PTR(ret); 402 } 403 404 static void ivpu_bo_free(struct drm_gem_object *obj) 405 { 406 struct ivpu_bo *bo = to_ivpu_bo(obj); 407 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 408 409 if (bo->ctx) 410 ivpu_dbg(vdev, BO, "free: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n", 411 bo->ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped); 412 else 413 ivpu_dbg(vdev, BO, "free: ctx (released) allocated %d mmu_mapped %d\n", 414 (bool)bo->sgt, bo->mmu_mapped); 415 416 drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); 417 418 vunmap(bo->kvaddr); 419 420 if (bo->ctx) 421 ivpu_bo_free_vpu_addr(bo); 422 423 if (bo->sgt) 424 ivpu_bo_unmap_and_free_pages(bo); 425 426 if (bo->base.import_attach) 427 drm_prime_gem_destroy(&bo->base, bo->sgt); 428 429 drm_gem_object_release(&bo->base); 430 431 mutex_destroy(&bo->lock); 432 kfree(bo); 433 } 434 435 static int ivpu_bo_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 436 { 437 struct ivpu_bo *bo = to_ivpu_bo(obj); 438 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 439 440 ivpu_dbg(vdev, BO, "mmap: ctx %u handle %u vpu_addr 0x%llx size %zu type %s", 441 bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size, bo->ops->name); 442 443 if (obj->import_attach) { 444 /* Drop the reference drm_gem_mmap_obj() acquired.*/ 445 drm_gem_object_put(obj); 446 vma->vm_private_data = NULL; 447 return dma_buf_mmap(obj->dma_buf, vma, 0); 448 } 449 450 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND); 451 vma->vm_page_prot = ivpu_bo_pgprot(bo, vm_get_page_prot(vma->vm_flags)); 452 453 return 0; 454 } 455 456 static struct sg_table *ivpu_bo_get_sg_table(struct drm_gem_object *obj) 457 { 458 struct ivpu_bo *bo = to_ivpu_bo(obj); 459 loff_t npages = obj->size >> PAGE_SHIFT; 460 int ret = 0; 461 462 mutex_lock(&bo->lock); 463 464 if (!bo->sgt) 465 ret = ivpu_bo_alloc_and_map_pages_locked(bo); 466 467 mutex_unlock(&bo->lock); 468 469 if (ret) 470 return ERR_PTR(ret); 471 472 return drm_prime_pages_to_sg(obj->dev, bo->pages, npages); 473 } 474 475 static vm_fault_t ivpu_vm_fault(struct vm_fault *vmf) 476 { 477 struct vm_area_struct *vma = vmf->vma; 478 struct drm_gem_object *obj = vma->vm_private_data; 479 struct ivpu_bo *bo = to_ivpu_bo(obj); 480 loff_t npages = obj->size >> PAGE_SHIFT; 481 pgoff_t page_offset; 482 struct page *page; 483 vm_fault_t ret; 484 int err; 485 486 mutex_lock(&bo->lock); 487 488 if (!bo->sgt) { 489 err = ivpu_bo_alloc_and_map_pages_locked(bo); 490 if (err) { 491 ret = vmf_error(err); 492 goto unlock; 493 } 494 } 495 496 /* We don't use vmf->pgoff since that has the fake offset */ 497 page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 498 if (page_offset >= npages) { 499 ret = VM_FAULT_SIGBUS; 500 } else { 501 page = bo->pages[page_offset]; 502 ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page)); 503 } 504 505 unlock: 506 mutex_unlock(&bo->lock); 507 508 return ret; 509 } 510 511 static const struct vm_operations_struct ivpu_vm_ops = { 512 .fault = ivpu_vm_fault, 513 .open = drm_gem_vm_open, 514 .close = drm_gem_vm_close, 515 }; 516 517 static const struct drm_gem_object_funcs ivpu_gem_funcs = { 518 .free = ivpu_bo_free, 519 .mmap = ivpu_bo_mmap, 520 .vm_ops = &ivpu_vm_ops, 521 .get_sg_table = ivpu_bo_get_sg_table, 522 }; 523 524 int 525 ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 526 { 527 struct ivpu_file_priv *file_priv = file->driver_priv; 528 struct ivpu_device *vdev = file_priv->vdev; 529 struct drm_ivpu_bo_create *args = data; 530 u64 size = PAGE_ALIGN(args->size); 531 struct ivpu_bo *bo; 532 int ret; 533 534 if (args->flags & ~DRM_IVPU_BO_FLAGS) 535 return -EINVAL; 536 537 if (size == 0) 538 return -EINVAL; 539 540 bo = ivpu_bo_alloc(vdev, &file_priv->ctx, size, args->flags, &shmem_ops, NULL, 0); 541 if (IS_ERR(bo)) { 542 ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)", 543 bo, file_priv->ctx.id, args->size, args->flags); 544 return PTR_ERR(bo); 545 } 546 547 ret = drm_gem_handle_create(file, &bo->base, &bo->handle); 548 if (!ret) { 549 args->vpu_addr = bo->vpu_addr; 550 args->handle = bo->handle; 551 } 552 553 drm_gem_object_put(&bo->base); 554 555 ivpu_dbg(vdev, BO, "alloc shmem: ctx %u vpu_addr 0x%llx size %zu flags 0x%x\n", 556 file_priv->ctx.id, bo->vpu_addr, bo->base.size, bo->flags); 557 558 return ret; 559 } 560 561 struct ivpu_bo * 562 ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags) 563 { 564 const struct ivpu_addr_range *range; 565 struct ivpu_addr_range fixed_range; 566 struct ivpu_bo *bo; 567 pgprot_t prot; 568 int ret; 569 570 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr)); 571 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); 572 573 if (vpu_addr) { 574 fixed_range.start = vpu_addr; 575 fixed_range.end = vpu_addr + size; 576 range = &fixed_range; 577 } else { 578 range = &vdev->hw->ranges.global; 579 } 580 581 bo = ivpu_bo_alloc(vdev, &vdev->gctx, size, flags, &internal_ops, range, 0); 582 if (IS_ERR(bo)) { 583 ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)", 584 bo, vpu_addr, size, flags); 585 return NULL; 586 } 587 588 ret = ivpu_bo_pin(bo); 589 if (ret) 590 goto err_put; 591 592 if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED) 593 drm_clflush_pages(bo->pages, bo->base.size >> PAGE_SHIFT); 594 595 if (bo->flags & DRM_IVPU_BO_WC) 596 set_pages_array_wc(bo->pages, bo->base.size >> PAGE_SHIFT); 597 else if (bo->flags & DRM_IVPU_BO_UNCACHED) 598 set_pages_array_uc(bo->pages, bo->base.size >> PAGE_SHIFT); 599 600 prot = ivpu_bo_pgprot(bo, PAGE_KERNEL); 601 bo->kvaddr = vmap(bo->pages, bo->base.size >> PAGE_SHIFT, VM_MAP, prot); 602 if (!bo->kvaddr) { 603 ivpu_err(vdev, "Failed to map BO into kernel virtual memory\n"); 604 goto err_put; 605 } 606 607 ivpu_dbg(vdev, BO, "alloc internal: ctx 0 vpu_addr 0x%llx size %zu flags 0x%x\n", 608 bo->vpu_addr, bo->base.size, flags); 609 610 return bo; 611 612 err_put: 613 drm_gem_object_put(&bo->base); 614 return NULL; 615 } 616 617 void ivpu_bo_free_internal(struct ivpu_bo *bo) 618 { 619 drm_gem_object_put(&bo->base); 620 } 621 622 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *buf) 623 { 624 struct ivpu_device *vdev = to_ivpu_device(dev); 625 struct dma_buf_attachment *attach; 626 struct ivpu_bo *bo; 627 628 attach = dma_buf_attach(buf, dev->dev); 629 if (IS_ERR(attach)) 630 return ERR_CAST(attach); 631 632 get_dma_buf(buf); 633 634 bo = ivpu_bo_alloc(vdev, NULL, buf->size, DRM_IVPU_BO_MAPPABLE, &prime_ops, NULL, 0); 635 if (IS_ERR(bo)) { 636 ivpu_err(vdev, "Failed to import BO: %pe (size %lu)", bo, buf->size); 637 goto err_detach; 638 } 639 640 lockdep_set_class(&bo->lock, &prime_bo_lock_class_key); 641 642 bo->base.import_attach = attach; 643 644 return &bo->base; 645 646 err_detach: 647 dma_buf_detach(buf, attach); 648 dma_buf_put(buf); 649 return ERR_CAST(bo); 650 } 651 652 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 653 { 654 struct ivpu_file_priv *file_priv = file->driver_priv; 655 struct ivpu_device *vdev = to_ivpu_device(dev); 656 struct drm_ivpu_bo_info *args = data; 657 struct drm_gem_object *obj; 658 struct ivpu_bo *bo; 659 int ret = 0; 660 661 obj = drm_gem_object_lookup(file, args->handle); 662 if (!obj) 663 return -ENOENT; 664 665 bo = to_ivpu_bo(obj); 666 667 mutex_lock(&bo->lock); 668 669 if (!bo->ctx) { 670 ret = ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, NULL); 671 if (ret) { 672 ivpu_err(vdev, "Failed to allocate vpu_addr: %d\n", ret); 673 goto unlock; 674 } 675 } 676 677 args->flags = bo->flags; 678 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); 679 args->vpu_addr = bo->vpu_addr; 680 args->size = obj->size; 681 unlock: 682 mutex_unlock(&bo->lock); 683 drm_gem_object_put(obj); 684 return ret; 685 } 686 687 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 688 { 689 struct drm_ivpu_bo_wait *args = data; 690 struct drm_gem_object *obj; 691 unsigned long timeout; 692 long ret; 693 694 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 695 696 obj = drm_gem_object_lookup(file, args->handle); 697 if (!obj) 698 return -EINVAL; 699 700 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); 701 if (ret == 0) { 702 ret = -ETIMEDOUT; 703 } else if (ret > 0) { 704 ret = 0; 705 args->job_status = to_ivpu_bo(obj)->job_status; 706 } 707 708 drm_gem_object_put(obj); 709 710 return ret; 711 } 712 713 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) 714 { 715 unsigned long dma_refcount = 0; 716 717 if (bo->base.dma_buf && bo->base.dma_buf->file) 718 dma_refcount = atomic_long_read(&bo->base.dma_buf->file->f_count); 719 720 drm_printf(p, "%5u %6d %16llx %10lu %10u %12lu %14s\n", 721 bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size, 722 kref_read(&bo->base.refcount), dma_refcount, bo->ops->name); 723 } 724 725 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) 726 { 727 struct ivpu_device *vdev = to_ivpu_device(dev); 728 struct ivpu_file_priv *file_priv; 729 unsigned long ctx_id; 730 struct ivpu_bo *bo; 731 732 drm_printf(p, "%5s %6s %16s %10s %10s %12s %14s\n", 733 "ctx", "handle", "vpu_addr", "size", "refcount", "dma_refcount", "type"); 734 735 mutex_lock(&vdev->gctx.lock); 736 list_for_each_entry(bo, &vdev->gctx.bo_list, ctx_node) 737 ivpu_bo_print_info(bo, p); 738 mutex_unlock(&vdev->gctx.lock); 739 740 xa_for_each(&vdev->context_xa, ctx_id, file_priv) { 741 file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id); 742 if (!file_priv) 743 continue; 744 745 mutex_lock(&file_priv->ctx.lock); 746 list_for_each_entry(bo, &file_priv->ctx.bo_list, ctx_node) 747 ivpu_bo_print_info(bo, p); 748 mutex_unlock(&file_priv->ctx.lock); 749 750 ivpu_file_priv_put(&file_priv); 751 } 752 } 753 754 void ivpu_bo_list_print(struct drm_device *dev) 755 { 756 struct drm_printer p = drm_info_printer(dev->dev); 757 758 ivpu_bo_list(dev, &p); 759 } 760