1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include <linux/dma-map-ops.h> 8 #include <linux/vmalloc.h> 9 #include <linux/spinlock.h> 10 #include <linux/shmem_fs.h> 11 #include <linux/dma-buf.h> 12 #include <linux/pfn_t.h> 13 14 #include <drm/drm_prime.h> 15 16 #include "msm_drv.h" 17 #include "msm_fence.h" 18 #include "msm_gem.h" 19 #include "msm_gpu.h" 20 #include "msm_mmu.h" 21 22 static dma_addr_t physaddr(struct drm_gem_object *obj) 23 { 24 struct msm_gem_object *msm_obj = to_msm_bo(obj); 25 struct msm_drm_private *priv = obj->dev->dev_private; 26 return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) + 27 priv->vram.paddr; 28 } 29 30 static bool use_pages(struct drm_gem_object *obj) 31 { 32 struct msm_gem_object *msm_obj = to_msm_bo(obj); 33 return !msm_obj->vram_node; 34 } 35 36 /* 37 * Cache sync.. this is a bit over-complicated, to fit dma-mapping 38 * API. Really GPU cache is out of scope here (handled on cmdstream) 39 * and all we need to do is invalidate newly allocated pages before 40 * mapping to CPU as uncached/writecombine. 41 * 42 * On top of this, we have the added headache, that depending on 43 * display generation, the display's iommu may be wired up to either 44 * the toplevel drm device (mdss), or to the mdp sub-node, meaning 45 * that here we either have dma-direct or iommu ops. 46 * 47 * Let this be a cautionary tail of abstraction gone wrong. 48 */ 49 50 static void sync_for_device(struct msm_gem_object *msm_obj) 51 { 52 struct device *dev = msm_obj->base.dev->dev; 53 54 dma_map_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0); 55 } 56 57 static void sync_for_cpu(struct msm_gem_object *msm_obj) 58 { 59 struct device *dev = msm_obj->base.dev->dev; 60 61 dma_unmap_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0); 62 } 63 64 static void update_lru_active(struct drm_gem_object *obj) 65 { 66 struct msm_drm_private *priv = obj->dev->dev_private; 67 struct msm_gem_object *msm_obj = to_msm_bo(obj); 68 69 GEM_WARN_ON(!msm_obj->pages); 70 71 if (msm_obj->pin_count) { 72 drm_gem_lru_move_tail_locked(&priv->lru.pinned, obj); 73 } else if (msm_obj->madv == MSM_MADV_WILLNEED) { 74 drm_gem_lru_move_tail_locked(&priv->lru.willneed, obj); 75 } else { 76 GEM_WARN_ON(msm_obj->madv != MSM_MADV_DONTNEED); 77 78 drm_gem_lru_move_tail_locked(&priv->lru.dontneed, obj); 79 } 80 } 81 82 static void update_lru_locked(struct drm_gem_object *obj) 83 { 84 struct msm_drm_private *priv = obj->dev->dev_private; 85 struct msm_gem_object *msm_obj = to_msm_bo(obj); 86 87 msm_gem_assert_locked(&msm_obj->base); 88 89 if (!msm_obj->pages) { 90 GEM_WARN_ON(msm_obj->pin_count); 91 92 drm_gem_lru_move_tail_locked(&priv->lru.unbacked, obj); 93 } else { 94 update_lru_active(obj); 95 } 96 } 97 98 static void update_lru(struct drm_gem_object *obj) 99 { 100 struct msm_drm_private *priv = obj->dev->dev_private; 101 102 mutex_lock(&priv->lru.lock); 103 update_lru_locked(obj); 104 mutex_unlock(&priv->lru.lock); 105 } 106 107 /* allocate pages from VRAM carveout, used when no IOMMU: */ 108 static struct page **get_pages_vram(struct drm_gem_object *obj, int npages) 109 { 110 struct msm_gem_object *msm_obj = to_msm_bo(obj); 111 struct msm_drm_private *priv = obj->dev->dev_private; 112 dma_addr_t paddr; 113 struct page **p; 114 int ret, i; 115 116 p = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 117 if (!p) 118 return ERR_PTR(-ENOMEM); 119 120 spin_lock(&priv->vram.lock); 121 ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node, npages); 122 spin_unlock(&priv->vram.lock); 123 if (ret) { 124 kvfree(p); 125 return ERR_PTR(ret); 126 } 127 128 paddr = physaddr(obj); 129 for (i = 0; i < npages; i++) { 130 p[i] = pfn_to_page(__phys_to_pfn(paddr)); 131 paddr += PAGE_SIZE; 132 } 133 134 return p; 135 } 136 137 static struct page **get_pages(struct drm_gem_object *obj) 138 { 139 struct msm_gem_object *msm_obj = to_msm_bo(obj); 140 141 msm_gem_assert_locked(obj); 142 143 if (!msm_obj->pages) { 144 struct drm_device *dev = obj->dev; 145 struct page **p; 146 int npages = obj->size >> PAGE_SHIFT; 147 148 if (use_pages(obj)) 149 p = drm_gem_get_pages(obj); 150 else 151 p = get_pages_vram(obj, npages); 152 153 if (IS_ERR(p)) { 154 DRM_DEV_ERROR(dev->dev, "could not get pages: %ld\n", 155 PTR_ERR(p)); 156 return p; 157 } 158 159 msm_obj->pages = p; 160 161 msm_obj->sgt = drm_prime_pages_to_sg(obj->dev, p, npages); 162 if (IS_ERR(msm_obj->sgt)) { 163 void *ptr = ERR_CAST(msm_obj->sgt); 164 165 DRM_DEV_ERROR(dev->dev, "failed to allocate sgt\n"); 166 msm_obj->sgt = NULL; 167 return ptr; 168 } 169 170 /* For non-cached buffers, ensure the new pages are clean 171 * because display controller, GPU, etc. are not coherent: 172 */ 173 if (msm_obj->flags & MSM_BO_WC) 174 sync_for_device(msm_obj); 175 176 update_lru(obj); 177 } 178 179 return msm_obj->pages; 180 } 181 182 static void put_pages_vram(struct drm_gem_object *obj) 183 { 184 struct msm_gem_object *msm_obj = to_msm_bo(obj); 185 struct msm_drm_private *priv = obj->dev->dev_private; 186 187 spin_lock(&priv->vram.lock); 188 drm_mm_remove_node(msm_obj->vram_node); 189 spin_unlock(&priv->vram.lock); 190 191 kvfree(msm_obj->pages); 192 } 193 194 static void put_pages(struct drm_gem_object *obj) 195 { 196 struct msm_gem_object *msm_obj = to_msm_bo(obj); 197 198 if (msm_obj->pages) { 199 if (msm_obj->sgt) { 200 /* For non-cached buffers, ensure the new 201 * pages are clean because display controller, 202 * GPU, etc. are not coherent: 203 */ 204 if (msm_obj->flags & MSM_BO_WC) 205 sync_for_cpu(msm_obj); 206 207 sg_free_table(msm_obj->sgt); 208 kfree(msm_obj->sgt); 209 msm_obj->sgt = NULL; 210 } 211 212 if (use_pages(obj)) 213 drm_gem_put_pages(obj, msm_obj->pages, true, false); 214 else 215 put_pages_vram(obj); 216 217 msm_obj->pages = NULL; 218 update_lru(obj); 219 } 220 } 221 222 static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj, 223 unsigned madv) 224 { 225 struct msm_drm_private *priv = obj->dev->dev_private; 226 struct msm_gem_object *msm_obj = to_msm_bo(obj); 227 struct page **p; 228 229 msm_gem_assert_locked(obj); 230 231 if (GEM_WARN_ON(msm_obj->madv > madv)) { 232 DRM_DEV_ERROR(obj->dev->dev, "Invalid madv state: %u vs %u\n", 233 msm_obj->madv, madv); 234 return ERR_PTR(-EBUSY); 235 } 236 237 p = get_pages(obj); 238 if (IS_ERR(p)) 239 return p; 240 241 mutex_lock(&priv->lru.lock); 242 msm_obj->pin_count++; 243 update_lru_locked(obj); 244 mutex_unlock(&priv->lru.lock); 245 246 return p; 247 } 248 249 struct page **msm_gem_pin_pages(struct drm_gem_object *obj) 250 { 251 struct page **p; 252 253 msm_gem_lock(obj); 254 p = msm_gem_pin_pages_locked(obj, MSM_MADV_WILLNEED); 255 msm_gem_unlock(obj); 256 257 return p; 258 } 259 260 void msm_gem_unpin_pages(struct drm_gem_object *obj) 261 { 262 msm_gem_lock(obj); 263 msm_gem_unpin_locked(obj); 264 msm_gem_unlock(obj); 265 } 266 267 static pgprot_t msm_gem_pgprot(struct msm_gem_object *msm_obj, pgprot_t prot) 268 { 269 if (msm_obj->flags & MSM_BO_WC) 270 return pgprot_writecombine(prot); 271 return prot; 272 } 273 274 static vm_fault_t msm_gem_fault(struct vm_fault *vmf) 275 { 276 struct vm_area_struct *vma = vmf->vma; 277 struct drm_gem_object *obj = vma->vm_private_data; 278 struct msm_gem_object *msm_obj = to_msm_bo(obj); 279 struct page **pages; 280 unsigned long pfn; 281 pgoff_t pgoff; 282 int err; 283 vm_fault_t ret; 284 285 /* 286 * vm_ops.open/drm_gem_mmap_obj and close get and put 287 * a reference on obj. So, we dont need to hold one here. 288 */ 289 err = msm_gem_lock_interruptible(obj); 290 if (err) { 291 ret = VM_FAULT_NOPAGE; 292 goto out; 293 } 294 295 if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) { 296 msm_gem_unlock(obj); 297 return VM_FAULT_SIGBUS; 298 } 299 300 /* make sure we have pages attached now */ 301 pages = get_pages(obj); 302 if (IS_ERR(pages)) { 303 ret = vmf_error(PTR_ERR(pages)); 304 goto out_unlock; 305 } 306 307 /* We don't use vmf->pgoff since that has the fake offset: */ 308 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 309 310 pfn = page_to_pfn(pages[pgoff]); 311 312 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address, 313 pfn, pfn << PAGE_SHIFT); 314 315 ret = vmf_insert_pfn(vma, vmf->address, pfn); 316 317 out_unlock: 318 msm_gem_unlock(obj); 319 out: 320 return ret; 321 } 322 323 /** get mmap offset */ 324 static uint64_t mmap_offset(struct drm_gem_object *obj) 325 { 326 struct drm_device *dev = obj->dev; 327 int ret; 328 329 msm_gem_assert_locked(obj); 330 331 /* Make it mmapable */ 332 ret = drm_gem_create_mmap_offset(obj); 333 334 if (ret) { 335 DRM_DEV_ERROR(dev->dev, "could not allocate mmap offset\n"); 336 return 0; 337 } 338 339 return drm_vma_node_offset_addr(&obj->vma_node); 340 } 341 342 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj) 343 { 344 uint64_t offset; 345 346 msm_gem_lock(obj); 347 offset = mmap_offset(obj); 348 msm_gem_unlock(obj); 349 return offset; 350 } 351 352 static struct msm_gem_vma *add_vma(struct drm_gem_object *obj, 353 struct msm_gem_address_space *aspace) 354 { 355 struct msm_gem_object *msm_obj = to_msm_bo(obj); 356 struct msm_gem_vma *vma; 357 358 msm_gem_assert_locked(obj); 359 360 vma = msm_gem_vma_new(aspace); 361 if (!vma) 362 return ERR_PTR(-ENOMEM); 363 364 list_add_tail(&vma->list, &msm_obj->vmas); 365 366 return vma; 367 } 368 369 static struct msm_gem_vma *lookup_vma(struct drm_gem_object *obj, 370 struct msm_gem_address_space *aspace) 371 { 372 struct msm_gem_object *msm_obj = to_msm_bo(obj); 373 struct msm_gem_vma *vma; 374 375 msm_gem_assert_locked(obj); 376 377 list_for_each_entry(vma, &msm_obj->vmas, list) { 378 if (vma->aspace == aspace) 379 return vma; 380 } 381 382 return NULL; 383 } 384 385 static void del_vma(struct msm_gem_vma *vma) 386 { 387 if (!vma) 388 return; 389 390 list_del(&vma->list); 391 kfree(vma); 392 } 393 394 /* 395 * If close is true, this also closes the VMA (releasing the allocated 396 * iova range) in addition to removing the iommu mapping. In the eviction 397 * case (!close), we keep the iova allocated, but only remove the iommu 398 * mapping. 399 */ 400 static void 401 put_iova_spaces(struct drm_gem_object *obj, bool close) 402 { 403 struct msm_gem_object *msm_obj = to_msm_bo(obj); 404 struct msm_gem_vma *vma; 405 406 msm_gem_assert_locked(obj); 407 408 list_for_each_entry(vma, &msm_obj->vmas, list) { 409 if (vma->aspace) { 410 msm_gem_vma_purge(vma); 411 if (close) 412 msm_gem_vma_close(vma); 413 } 414 } 415 } 416 417 /* Called with msm_obj locked */ 418 static void 419 put_iova_vmas(struct drm_gem_object *obj) 420 { 421 struct msm_gem_object *msm_obj = to_msm_bo(obj); 422 struct msm_gem_vma *vma, *tmp; 423 424 msm_gem_assert_locked(obj); 425 426 list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) { 427 del_vma(vma); 428 } 429 } 430 431 static struct msm_gem_vma *get_vma_locked(struct drm_gem_object *obj, 432 struct msm_gem_address_space *aspace, 433 u64 range_start, u64 range_end) 434 { 435 struct msm_gem_vma *vma; 436 437 msm_gem_assert_locked(obj); 438 439 vma = lookup_vma(obj, aspace); 440 441 if (!vma) { 442 int ret; 443 444 vma = add_vma(obj, aspace); 445 if (IS_ERR(vma)) 446 return vma; 447 448 ret = msm_gem_vma_init(vma, obj->size, 449 range_start, range_end); 450 if (ret) { 451 del_vma(vma); 452 return ERR_PTR(ret); 453 } 454 } else { 455 GEM_WARN_ON(vma->iova < range_start); 456 GEM_WARN_ON((vma->iova + obj->size) > range_end); 457 } 458 459 return vma; 460 } 461 462 int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma) 463 { 464 struct msm_gem_object *msm_obj = to_msm_bo(obj); 465 struct page **pages; 466 int ret, prot = IOMMU_READ; 467 468 if (!(msm_obj->flags & MSM_BO_GPU_READONLY)) 469 prot |= IOMMU_WRITE; 470 471 if (msm_obj->flags & MSM_BO_MAP_PRIV) 472 prot |= IOMMU_PRIV; 473 474 if (msm_obj->flags & MSM_BO_CACHED_COHERENT) 475 prot |= IOMMU_CACHE; 476 477 msm_gem_assert_locked(obj); 478 479 pages = msm_gem_pin_pages_locked(obj, MSM_MADV_WILLNEED); 480 if (IS_ERR(pages)) 481 return PTR_ERR(pages); 482 483 ret = msm_gem_vma_map(vma, prot, msm_obj->sgt, obj->size); 484 if (ret) 485 msm_gem_unpin_locked(obj); 486 487 return ret; 488 } 489 490 void msm_gem_unpin_locked(struct drm_gem_object *obj) 491 { 492 struct msm_drm_private *priv = obj->dev->dev_private; 493 struct msm_gem_object *msm_obj = to_msm_bo(obj); 494 495 msm_gem_assert_locked(obj); 496 497 mutex_lock(&priv->lru.lock); 498 msm_obj->pin_count--; 499 GEM_WARN_ON(msm_obj->pin_count < 0); 500 update_lru_locked(obj); 501 mutex_unlock(&priv->lru.lock); 502 } 503 504 /* Special unpin path for use in fence-signaling path, avoiding the need 505 * to hold the obj lock by only depending on things that a protected by 506 * the LRU lock. In particular we know that that we already have backing 507 * and and that the object's dma_resv has the fence for the current 508 * submit/job which will prevent us racing against page eviction. 509 */ 510 void msm_gem_unpin_active(struct drm_gem_object *obj) 511 { 512 struct msm_gem_object *msm_obj = to_msm_bo(obj); 513 514 msm_obj->pin_count--; 515 GEM_WARN_ON(msm_obj->pin_count < 0); 516 update_lru_active(obj); 517 } 518 519 struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj, 520 struct msm_gem_address_space *aspace) 521 { 522 return get_vma_locked(obj, aspace, 0, U64_MAX); 523 } 524 525 static int get_and_pin_iova_range_locked(struct drm_gem_object *obj, 526 struct msm_gem_address_space *aspace, uint64_t *iova, 527 u64 range_start, u64 range_end) 528 { 529 struct msm_gem_vma *vma; 530 int ret; 531 532 msm_gem_assert_locked(obj); 533 534 vma = get_vma_locked(obj, aspace, range_start, range_end); 535 if (IS_ERR(vma)) 536 return PTR_ERR(vma); 537 538 ret = msm_gem_pin_vma_locked(obj, vma); 539 if (!ret) 540 *iova = vma->iova; 541 542 return ret; 543 } 544 545 /* 546 * get iova and pin it. Should have a matching put 547 * limits iova to specified range (in pages) 548 */ 549 int msm_gem_get_and_pin_iova_range(struct drm_gem_object *obj, 550 struct msm_gem_address_space *aspace, uint64_t *iova, 551 u64 range_start, u64 range_end) 552 { 553 int ret; 554 555 msm_gem_lock(obj); 556 ret = get_and_pin_iova_range_locked(obj, aspace, iova, range_start, range_end); 557 msm_gem_unlock(obj); 558 559 return ret; 560 } 561 562 /* get iova and pin it. Should have a matching put */ 563 int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, 564 struct msm_gem_address_space *aspace, uint64_t *iova) 565 { 566 return msm_gem_get_and_pin_iova_range(obj, aspace, iova, 0, U64_MAX); 567 } 568 569 /* 570 * Get an iova but don't pin it. Doesn't need a put because iovas are currently 571 * valid for the life of the object 572 */ 573 int msm_gem_get_iova(struct drm_gem_object *obj, 574 struct msm_gem_address_space *aspace, uint64_t *iova) 575 { 576 struct msm_gem_vma *vma; 577 int ret = 0; 578 579 msm_gem_lock(obj); 580 vma = get_vma_locked(obj, aspace, 0, U64_MAX); 581 if (IS_ERR(vma)) { 582 ret = PTR_ERR(vma); 583 } else { 584 *iova = vma->iova; 585 } 586 msm_gem_unlock(obj); 587 588 return ret; 589 } 590 591 static int clear_iova(struct drm_gem_object *obj, 592 struct msm_gem_address_space *aspace) 593 { 594 struct msm_gem_vma *vma = lookup_vma(obj, aspace); 595 596 if (!vma) 597 return 0; 598 599 if (msm_gem_vma_inuse(vma)) 600 return -EBUSY; 601 602 msm_gem_vma_purge(vma); 603 msm_gem_vma_close(vma); 604 del_vma(vma); 605 606 return 0; 607 } 608 609 /* 610 * Get the requested iova but don't pin it. Fails if the requested iova is 611 * not available. Doesn't need a put because iovas are currently valid for 612 * the life of the object. 613 * 614 * Setting an iova of zero will clear the vma. 615 */ 616 int msm_gem_set_iova(struct drm_gem_object *obj, 617 struct msm_gem_address_space *aspace, uint64_t iova) 618 { 619 int ret = 0; 620 621 msm_gem_lock(obj); 622 if (!iova) { 623 ret = clear_iova(obj, aspace); 624 } else { 625 struct msm_gem_vma *vma; 626 vma = get_vma_locked(obj, aspace, iova, iova + obj->size); 627 if (IS_ERR(vma)) { 628 ret = PTR_ERR(vma); 629 } else if (GEM_WARN_ON(vma->iova != iova)) { 630 clear_iova(obj, aspace); 631 ret = -EBUSY; 632 } 633 } 634 msm_gem_unlock(obj); 635 636 return ret; 637 } 638 639 /* 640 * Unpin a iova by updating the reference counts. The memory isn't actually 641 * purged until something else (shrinker, mm_notifier, destroy, etc) decides 642 * to get rid of it 643 */ 644 void msm_gem_unpin_iova(struct drm_gem_object *obj, 645 struct msm_gem_address_space *aspace) 646 { 647 struct msm_gem_vma *vma; 648 649 msm_gem_lock(obj); 650 vma = lookup_vma(obj, aspace); 651 if (!GEM_WARN_ON(!vma)) { 652 msm_gem_vma_unpin(vma); 653 msm_gem_unpin_locked(obj); 654 } 655 msm_gem_unlock(obj); 656 } 657 658 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 659 struct drm_mode_create_dumb *args) 660 { 661 args->pitch = align_pitch(args->width, args->bpp); 662 args->size = PAGE_ALIGN(args->pitch * args->height); 663 return msm_gem_new_handle(dev, file, args->size, 664 MSM_BO_SCANOUT | MSM_BO_WC, &args->handle, "dumb"); 665 } 666 667 int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, 668 uint32_t handle, uint64_t *offset) 669 { 670 struct drm_gem_object *obj; 671 int ret = 0; 672 673 /* GEM does all our handle to object mapping */ 674 obj = drm_gem_object_lookup(file, handle); 675 if (obj == NULL) { 676 ret = -ENOENT; 677 goto fail; 678 } 679 680 *offset = msm_gem_mmap_offset(obj); 681 682 drm_gem_object_put(obj); 683 684 fail: 685 return ret; 686 } 687 688 static void *get_vaddr(struct drm_gem_object *obj, unsigned madv) 689 { 690 struct msm_gem_object *msm_obj = to_msm_bo(obj); 691 struct page **pages; 692 int ret = 0; 693 694 msm_gem_assert_locked(obj); 695 696 if (obj->import_attach) 697 return ERR_PTR(-ENODEV); 698 699 pages = msm_gem_pin_pages_locked(obj, madv); 700 if (IS_ERR(pages)) 701 return ERR_CAST(pages); 702 703 /* increment vmap_count *before* vmap() call, so shrinker can 704 * check vmap_count (is_vunmapable()) outside of msm_obj lock. 705 * This guarantees that we won't try to msm_gem_vunmap() this 706 * same object from within the vmap() call (while we already 707 * hold msm_obj lock) 708 */ 709 msm_obj->vmap_count++; 710 711 if (!msm_obj->vaddr) { 712 msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT, 713 VM_MAP, msm_gem_pgprot(msm_obj, PAGE_KERNEL)); 714 if (msm_obj->vaddr == NULL) { 715 ret = -ENOMEM; 716 goto fail; 717 } 718 } 719 720 return msm_obj->vaddr; 721 722 fail: 723 msm_obj->vmap_count--; 724 msm_gem_unpin_locked(obj); 725 return ERR_PTR(ret); 726 } 727 728 void *msm_gem_get_vaddr_locked(struct drm_gem_object *obj) 729 { 730 return get_vaddr(obj, MSM_MADV_WILLNEED); 731 } 732 733 void *msm_gem_get_vaddr(struct drm_gem_object *obj) 734 { 735 void *ret; 736 737 msm_gem_lock(obj); 738 ret = msm_gem_get_vaddr_locked(obj); 739 msm_gem_unlock(obj); 740 741 return ret; 742 } 743 744 /* 745 * Don't use this! It is for the very special case of dumping 746 * submits from GPU hangs or faults, were the bo may already 747 * be MSM_MADV_DONTNEED, but we know the buffer is still on the 748 * active list. 749 */ 750 void *msm_gem_get_vaddr_active(struct drm_gem_object *obj) 751 { 752 return get_vaddr(obj, __MSM_MADV_PURGED); 753 } 754 755 void msm_gem_put_vaddr_locked(struct drm_gem_object *obj) 756 { 757 struct msm_gem_object *msm_obj = to_msm_bo(obj); 758 759 msm_gem_assert_locked(obj); 760 GEM_WARN_ON(msm_obj->vmap_count < 1); 761 762 msm_obj->vmap_count--; 763 msm_gem_unpin_locked(obj); 764 } 765 766 void msm_gem_put_vaddr(struct drm_gem_object *obj) 767 { 768 msm_gem_lock(obj); 769 msm_gem_put_vaddr_locked(obj); 770 msm_gem_unlock(obj); 771 } 772 773 /* Update madvise status, returns true if not purged, else 774 * false or -errno. 775 */ 776 int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv) 777 { 778 struct msm_drm_private *priv = obj->dev->dev_private; 779 struct msm_gem_object *msm_obj = to_msm_bo(obj); 780 781 msm_gem_lock(obj); 782 783 mutex_lock(&priv->lru.lock); 784 785 if (msm_obj->madv != __MSM_MADV_PURGED) 786 msm_obj->madv = madv; 787 788 madv = msm_obj->madv; 789 790 /* If the obj is inactive, we might need to move it 791 * between inactive lists 792 */ 793 update_lru_locked(obj); 794 795 mutex_unlock(&priv->lru.lock); 796 797 msm_gem_unlock(obj); 798 799 return (madv != __MSM_MADV_PURGED); 800 } 801 802 void msm_gem_purge(struct drm_gem_object *obj) 803 { 804 struct drm_device *dev = obj->dev; 805 struct msm_drm_private *priv = obj->dev->dev_private; 806 struct msm_gem_object *msm_obj = to_msm_bo(obj); 807 808 msm_gem_assert_locked(obj); 809 GEM_WARN_ON(!is_purgeable(msm_obj)); 810 811 /* Get rid of any iommu mapping(s): */ 812 put_iova_spaces(obj, true); 813 814 msm_gem_vunmap(obj); 815 816 drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); 817 818 put_pages(obj); 819 820 put_iova_vmas(obj); 821 822 mutex_lock(&priv->lru.lock); 823 /* A one-way transition: */ 824 msm_obj->madv = __MSM_MADV_PURGED; 825 mutex_unlock(&priv->lru.lock); 826 827 drm_gem_free_mmap_offset(obj); 828 829 /* Our goal here is to return as much of the memory as 830 * is possible back to the system as we are called from OOM. 831 * To do this we must instruct the shmfs to drop all of its 832 * backing pages, *now*. 833 */ 834 shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1); 835 836 invalidate_mapping_pages(file_inode(obj->filp)->i_mapping, 837 0, (loff_t)-1); 838 } 839 840 /* 841 * Unpin the backing pages and make them available to be swapped out. 842 */ 843 void msm_gem_evict(struct drm_gem_object *obj) 844 { 845 struct drm_device *dev = obj->dev; 846 struct msm_gem_object *msm_obj = to_msm_bo(obj); 847 848 msm_gem_assert_locked(obj); 849 GEM_WARN_ON(is_unevictable(msm_obj)); 850 851 /* Get rid of any iommu mapping(s): */ 852 put_iova_spaces(obj, false); 853 854 drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); 855 856 put_pages(obj); 857 } 858 859 void msm_gem_vunmap(struct drm_gem_object *obj) 860 { 861 struct msm_gem_object *msm_obj = to_msm_bo(obj); 862 863 msm_gem_assert_locked(obj); 864 865 if (!msm_obj->vaddr || GEM_WARN_ON(!is_vunmapable(msm_obj))) 866 return; 867 868 vunmap(msm_obj->vaddr); 869 msm_obj->vaddr = NULL; 870 } 871 872 bool msm_gem_active(struct drm_gem_object *obj) 873 { 874 msm_gem_assert_locked(obj); 875 876 if (to_msm_bo(obj)->pin_count) 877 return true; 878 879 return !dma_resv_test_signaled(obj->resv, dma_resv_usage_rw(true)); 880 } 881 882 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout) 883 { 884 bool write = !!(op & MSM_PREP_WRITE); 885 unsigned long remain = 886 op & MSM_PREP_NOSYNC ? 0 : timeout_to_jiffies(timeout); 887 long ret; 888 889 if (op & MSM_PREP_BOOST) { 890 dma_resv_set_deadline(obj->resv, dma_resv_usage_rw(write), 891 ktime_get()); 892 } 893 894 ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(write), 895 true, remain); 896 if (ret == 0) 897 return remain == 0 ? -EBUSY : -ETIMEDOUT; 898 else if (ret < 0) 899 return ret; 900 901 /* TODO cache maintenance */ 902 903 return 0; 904 } 905 906 int msm_gem_cpu_fini(struct drm_gem_object *obj) 907 { 908 /* TODO cache maintenance */ 909 return 0; 910 } 911 912 #ifdef CONFIG_DEBUG_FS 913 void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, 914 struct msm_gem_stats *stats) 915 { 916 struct msm_gem_object *msm_obj = to_msm_bo(obj); 917 struct dma_resv *robj = obj->resv; 918 struct msm_gem_vma *vma; 919 uint64_t off = drm_vma_node_start(&obj->vma_node); 920 const char *madv; 921 922 msm_gem_lock(obj); 923 924 stats->all.count++; 925 stats->all.size += obj->size; 926 927 if (msm_gem_active(obj)) { 928 stats->active.count++; 929 stats->active.size += obj->size; 930 } 931 932 if (msm_obj->pages) { 933 stats->resident.count++; 934 stats->resident.size += obj->size; 935 } 936 937 switch (msm_obj->madv) { 938 case __MSM_MADV_PURGED: 939 stats->purged.count++; 940 stats->purged.size += obj->size; 941 madv = " purged"; 942 break; 943 case MSM_MADV_DONTNEED: 944 stats->purgeable.count++; 945 stats->purgeable.size += obj->size; 946 madv = " purgeable"; 947 break; 948 case MSM_MADV_WILLNEED: 949 default: 950 madv = ""; 951 break; 952 } 953 954 seq_printf(m, "%08x: %c %2d (%2d) %08llx %p", 955 msm_obj->flags, msm_gem_active(obj) ? 'A' : 'I', 956 obj->name, kref_read(&obj->refcount), 957 off, msm_obj->vaddr); 958 959 seq_printf(m, " %08zu %9s %-32s\n", obj->size, madv, msm_obj->name); 960 961 if (!list_empty(&msm_obj->vmas)) { 962 963 seq_puts(m, " vmas:"); 964 965 list_for_each_entry(vma, &msm_obj->vmas, list) { 966 const char *name, *comm; 967 if (vma->aspace) { 968 struct msm_gem_address_space *aspace = vma->aspace; 969 struct task_struct *task = 970 get_pid_task(aspace->pid, PIDTYPE_PID); 971 if (task) { 972 comm = kstrdup(task->comm, GFP_KERNEL); 973 put_task_struct(task); 974 } else { 975 comm = NULL; 976 } 977 name = aspace->name; 978 } else { 979 name = comm = NULL; 980 } 981 seq_printf(m, " [%s%s%s: aspace=%p, %08llx,%s,inuse=%d]", 982 name, comm ? ":" : "", comm ? comm : "", 983 vma->aspace, vma->iova, 984 vma->mapped ? "mapped" : "unmapped", 985 msm_gem_vma_inuse(vma)); 986 kfree(comm); 987 } 988 989 seq_puts(m, "\n"); 990 } 991 992 dma_resv_describe(robj, m); 993 msm_gem_unlock(obj); 994 } 995 996 void msm_gem_describe_objects(struct list_head *list, struct seq_file *m) 997 { 998 struct msm_gem_stats stats = {}; 999 struct msm_gem_object *msm_obj; 1000 1001 seq_puts(m, " flags id ref offset kaddr size madv name\n"); 1002 list_for_each_entry(msm_obj, list, node) { 1003 struct drm_gem_object *obj = &msm_obj->base; 1004 seq_puts(m, " "); 1005 msm_gem_describe(obj, m, &stats); 1006 } 1007 1008 seq_printf(m, "Total: %4d objects, %9zu bytes\n", 1009 stats.all.count, stats.all.size); 1010 seq_printf(m, "Active: %4d objects, %9zu bytes\n", 1011 stats.active.count, stats.active.size); 1012 seq_printf(m, "Resident: %4d objects, %9zu bytes\n", 1013 stats.resident.count, stats.resident.size); 1014 seq_printf(m, "Purgeable: %4d objects, %9zu bytes\n", 1015 stats.purgeable.count, stats.purgeable.size); 1016 seq_printf(m, "Purged: %4d objects, %9zu bytes\n", 1017 stats.purged.count, stats.purged.size); 1018 } 1019 #endif 1020 1021 /* don't call directly! Use drm_gem_object_put() */ 1022 static void msm_gem_free_object(struct drm_gem_object *obj) 1023 { 1024 struct msm_gem_object *msm_obj = to_msm_bo(obj); 1025 struct drm_device *dev = obj->dev; 1026 struct msm_drm_private *priv = dev->dev_private; 1027 1028 mutex_lock(&priv->obj_lock); 1029 list_del(&msm_obj->node); 1030 mutex_unlock(&priv->obj_lock); 1031 1032 put_iova_spaces(obj, true); 1033 1034 if (obj->import_attach) { 1035 GEM_WARN_ON(msm_obj->vaddr); 1036 1037 /* Don't drop the pages for imported dmabuf, as they are not 1038 * ours, just free the array we allocated: 1039 */ 1040 kvfree(msm_obj->pages); 1041 1042 put_iova_vmas(obj); 1043 1044 drm_prime_gem_destroy(obj, msm_obj->sgt); 1045 } else { 1046 msm_gem_vunmap(obj); 1047 put_pages(obj); 1048 put_iova_vmas(obj); 1049 } 1050 1051 drm_gem_object_release(obj); 1052 1053 kfree(msm_obj); 1054 } 1055 1056 static int msm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 1057 { 1058 struct msm_gem_object *msm_obj = to_msm_bo(obj); 1059 1060 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 1061 vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags)); 1062 1063 return 0; 1064 } 1065 1066 /* convenience method to construct a GEM buffer object, and userspace handle */ 1067 int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file, 1068 uint32_t size, uint32_t flags, uint32_t *handle, 1069 char *name) 1070 { 1071 struct drm_gem_object *obj; 1072 int ret; 1073 1074 obj = msm_gem_new(dev, size, flags); 1075 1076 if (IS_ERR(obj)) 1077 return PTR_ERR(obj); 1078 1079 if (name) 1080 msm_gem_object_set_name(obj, "%s", name); 1081 1082 ret = drm_gem_handle_create(file, obj, handle); 1083 1084 /* drop reference from allocate - handle holds it now */ 1085 drm_gem_object_put(obj); 1086 1087 return ret; 1088 } 1089 1090 static enum drm_gem_object_status msm_gem_status(struct drm_gem_object *obj) 1091 { 1092 struct msm_gem_object *msm_obj = to_msm_bo(obj); 1093 enum drm_gem_object_status status = 0; 1094 1095 if (msm_obj->pages) 1096 status |= DRM_GEM_OBJECT_RESIDENT; 1097 1098 if (msm_obj->madv == MSM_MADV_DONTNEED) 1099 status |= DRM_GEM_OBJECT_PURGEABLE; 1100 1101 return status; 1102 } 1103 1104 static const struct vm_operations_struct vm_ops = { 1105 .fault = msm_gem_fault, 1106 .open = drm_gem_vm_open, 1107 .close = drm_gem_vm_close, 1108 }; 1109 1110 static const struct drm_gem_object_funcs msm_gem_object_funcs = { 1111 .free = msm_gem_free_object, 1112 .pin = msm_gem_prime_pin, 1113 .unpin = msm_gem_prime_unpin, 1114 .get_sg_table = msm_gem_prime_get_sg_table, 1115 .vmap = msm_gem_prime_vmap, 1116 .vunmap = msm_gem_prime_vunmap, 1117 .mmap = msm_gem_object_mmap, 1118 .status = msm_gem_status, 1119 .vm_ops = &vm_ops, 1120 }; 1121 1122 static int msm_gem_new_impl(struct drm_device *dev, 1123 uint32_t size, uint32_t flags, 1124 struct drm_gem_object **obj) 1125 { 1126 struct msm_drm_private *priv = dev->dev_private; 1127 struct msm_gem_object *msm_obj; 1128 1129 switch (flags & MSM_BO_CACHE_MASK) { 1130 case MSM_BO_CACHED: 1131 case MSM_BO_WC: 1132 break; 1133 case MSM_BO_CACHED_COHERENT: 1134 if (priv->has_cached_coherent) 1135 break; 1136 fallthrough; 1137 default: 1138 DRM_DEV_DEBUG(dev->dev, "invalid cache flag: %x\n", 1139 (flags & MSM_BO_CACHE_MASK)); 1140 return -EINVAL; 1141 } 1142 1143 msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL); 1144 if (!msm_obj) 1145 return -ENOMEM; 1146 1147 msm_obj->flags = flags; 1148 msm_obj->madv = MSM_MADV_WILLNEED; 1149 1150 INIT_LIST_HEAD(&msm_obj->node); 1151 INIT_LIST_HEAD(&msm_obj->vmas); 1152 1153 *obj = &msm_obj->base; 1154 (*obj)->funcs = &msm_gem_object_funcs; 1155 1156 return 0; 1157 } 1158 1159 struct drm_gem_object *msm_gem_new(struct drm_device *dev, uint32_t size, uint32_t flags) 1160 { 1161 struct msm_drm_private *priv = dev->dev_private; 1162 struct msm_gem_object *msm_obj; 1163 struct drm_gem_object *obj = NULL; 1164 bool use_vram = false; 1165 int ret; 1166 1167 size = PAGE_ALIGN(size); 1168 1169 if (!msm_use_mmu(dev)) 1170 use_vram = true; 1171 else if ((flags & (MSM_BO_STOLEN | MSM_BO_SCANOUT)) && priv->vram.size) 1172 use_vram = true; 1173 1174 if (GEM_WARN_ON(use_vram && !priv->vram.size)) 1175 return ERR_PTR(-EINVAL); 1176 1177 /* Disallow zero sized objects as they make the underlying 1178 * infrastructure grumpy 1179 */ 1180 if (size == 0) 1181 return ERR_PTR(-EINVAL); 1182 1183 ret = msm_gem_new_impl(dev, size, flags, &obj); 1184 if (ret) 1185 return ERR_PTR(ret); 1186 1187 msm_obj = to_msm_bo(obj); 1188 1189 if (use_vram) { 1190 struct msm_gem_vma *vma; 1191 struct page **pages; 1192 1193 drm_gem_private_object_init(dev, obj, size); 1194 1195 msm_gem_lock(obj); 1196 1197 vma = add_vma(obj, NULL); 1198 msm_gem_unlock(obj); 1199 if (IS_ERR(vma)) { 1200 ret = PTR_ERR(vma); 1201 goto fail; 1202 } 1203 1204 to_msm_bo(obj)->vram_node = &vma->node; 1205 1206 msm_gem_lock(obj); 1207 pages = get_pages(obj); 1208 msm_gem_unlock(obj); 1209 if (IS_ERR(pages)) { 1210 ret = PTR_ERR(pages); 1211 goto fail; 1212 } 1213 1214 vma->iova = physaddr(obj); 1215 } else { 1216 ret = drm_gem_object_init(dev, obj, size); 1217 if (ret) 1218 goto fail; 1219 /* 1220 * Our buffers are kept pinned, so allocating them from the 1221 * MOVABLE zone is a really bad idea, and conflicts with CMA. 1222 * See comments above new_inode() why this is required _and_ 1223 * expected if you're going to pin these pages. 1224 */ 1225 mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER); 1226 } 1227 1228 drm_gem_lru_move_tail(&priv->lru.unbacked, obj); 1229 1230 mutex_lock(&priv->obj_lock); 1231 list_add_tail(&msm_obj->node, &priv->objects); 1232 mutex_unlock(&priv->obj_lock); 1233 1234 return obj; 1235 1236 fail: 1237 drm_gem_object_put(obj); 1238 return ERR_PTR(ret); 1239 } 1240 1241 struct drm_gem_object *msm_gem_import(struct drm_device *dev, 1242 struct dma_buf *dmabuf, struct sg_table *sgt) 1243 { 1244 struct msm_drm_private *priv = dev->dev_private; 1245 struct msm_gem_object *msm_obj; 1246 struct drm_gem_object *obj; 1247 uint32_t size; 1248 int ret, npages; 1249 1250 /* if we don't have IOMMU, don't bother pretending we can import: */ 1251 if (!msm_use_mmu(dev)) { 1252 DRM_DEV_ERROR(dev->dev, "cannot import without IOMMU\n"); 1253 return ERR_PTR(-EINVAL); 1254 } 1255 1256 size = PAGE_ALIGN(dmabuf->size); 1257 1258 ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj); 1259 if (ret) 1260 return ERR_PTR(ret); 1261 1262 drm_gem_private_object_init(dev, obj, size); 1263 1264 npages = size / PAGE_SIZE; 1265 1266 msm_obj = to_msm_bo(obj); 1267 msm_gem_lock(obj); 1268 msm_obj->sgt = sgt; 1269 msm_obj->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 1270 if (!msm_obj->pages) { 1271 msm_gem_unlock(obj); 1272 ret = -ENOMEM; 1273 goto fail; 1274 } 1275 1276 ret = drm_prime_sg_to_page_array(sgt, msm_obj->pages, npages); 1277 if (ret) { 1278 msm_gem_unlock(obj); 1279 goto fail; 1280 } 1281 1282 msm_gem_unlock(obj); 1283 1284 drm_gem_lru_move_tail(&priv->lru.pinned, obj); 1285 1286 mutex_lock(&priv->obj_lock); 1287 list_add_tail(&msm_obj->node, &priv->objects); 1288 mutex_unlock(&priv->obj_lock); 1289 1290 return obj; 1291 1292 fail: 1293 drm_gem_object_put(obj); 1294 return ERR_PTR(ret); 1295 } 1296 1297 void *msm_gem_kernel_new(struct drm_device *dev, uint32_t size, 1298 uint32_t flags, struct msm_gem_address_space *aspace, 1299 struct drm_gem_object **bo, uint64_t *iova) 1300 { 1301 void *vaddr; 1302 struct drm_gem_object *obj = msm_gem_new(dev, size, flags); 1303 int ret; 1304 1305 if (IS_ERR(obj)) 1306 return ERR_CAST(obj); 1307 1308 if (iova) { 1309 ret = msm_gem_get_and_pin_iova(obj, aspace, iova); 1310 if (ret) 1311 goto err; 1312 } 1313 1314 vaddr = msm_gem_get_vaddr(obj); 1315 if (IS_ERR(vaddr)) { 1316 msm_gem_unpin_iova(obj, aspace); 1317 ret = PTR_ERR(vaddr); 1318 goto err; 1319 } 1320 1321 if (bo) 1322 *bo = obj; 1323 1324 return vaddr; 1325 err: 1326 drm_gem_object_put(obj); 1327 1328 return ERR_PTR(ret); 1329 1330 } 1331 1332 void msm_gem_kernel_put(struct drm_gem_object *bo, 1333 struct msm_gem_address_space *aspace) 1334 { 1335 if (IS_ERR_OR_NULL(bo)) 1336 return; 1337 1338 msm_gem_put_vaddr(bo); 1339 msm_gem_unpin_iova(bo, aspace); 1340 drm_gem_object_put(bo); 1341 } 1342 1343 void msm_gem_object_set_name(struct drm_gem_object *bo, const char *fmt, ...) 1344 { 1345 struct msm_gem_object *msm_obj = to_msm_bo(bo); 1346 va_list ap; 1347 1348 if (!fmt) 1349 return; 1350 1351 va_start(ap, fmt); 1352 vsnprintf(msm_obj->name, sizeof(msm_obj->name), fmt, ap); 1353 va_end(ap); 1354 } 1355