1 /* 2 * NVIDIA Tegra DRM GEM helper functions 3 * 4 * Copyright (C) 2012 Sascha Hauer, Pengutronix 5 * Copyright (C) 2013 NVIDIA CORPORATION, All rights reserved. 6 * 7 * Based on the GEM/CMA helpers 8 * 9 * Copyright (c) 2011 Samsung Electronics Co., Ltd. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License version 2 as 13 * published by the Free Software Foundation. 14 */ 15 16 #include <linux/dma-buf.h> 17 #include <linux/iommu.h> 18 #include <drm/tegra_drm.h> 19 20 #include "drm.h" 21 #include "gem.h" 22 23 static inline struct tegra_bo *host1x_to_tegra_bo(struct host1x_bo *bo) 24 { 25 return container_of(bo, struct tegra_bo, base); 26 } 27 28 static void tegra_bo_put(struct host1x_bo *bo) 29 { 30 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 31 32 drm_gem_object_unreference_unlocked(&obj->gem); 33 } 34 35 static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt) 36 { 37 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 38 39 return obj->paddr; 40 } 41 42 static void tegra_bo_unpin(struct host1x_bo *bo, struct sg_table *sgt) 43 { 44 } 45 46 static void *tegra_bo_mmap(struct host1x_bo *bo) 47 { 48 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 49 50 return obj->vaddr; 51 } 52 53 static void tegra_bo_munmap(struct host1x_bo *bo, void *addr) 54 { 55 } 56 57 static void *tegra_bo_kmap(struct host1x_bo *bo, unsigned int page) 58 { 59 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 60 61 return obj->vaddr + page * PAGE_SIZE; 62 } 63 64 static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page, 65 void *addr) 66 { 67 } 68 69 static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo) 70 { 71 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 72 73 drm_gem_object_reference(&obj->gem); 74 75 return bo; 76 } 77 78 static const struct host1x_bo_ops tegra_bo_ops = { 79 .get = tegra_bo_get, 80 .put = tegra_bo_put, 81 .pin = tegra_bo_pin, 82 .unpin = tegra_bo_unpin, 83 .mmap = tegra_bo_mmap, 84 .munmap = tegra_bo_munmap, 85 .kmap = tegra_bo_kmap, 86 .kunmap = tegra_bo_kunmap, 87 }; 88 89 static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo) 90 { 91 int prot = IOMMU_READ | IOMMU_WRITE; 92 ssize_t err; 93 94 if (bo->mm) 95 return -EBUSY; 96 97 bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL); 98 if (!bo->mm) 99 return -ENOMEM; 100 101 err = drm_mm_insert_node_generic(&tegra->mm, bo->mm, bo->gem.size, 102 PAGE_SIZE, 0, 0, 0); 103 if (err < 0) { 104 dev_err(tegra->drm->dev, "out of I/O virtual memory: %zd\n", 105 err); 106 goto free; 107 } 108 109 bo->paddr = bo->mm->start; 110 111 err = iommu_map_sg(tegra->domain, bo->paddr, bo->sgt->sgl, 112 bo->sgt->nents, prot); 113 if (err < 0) { 114 dev_err(tegra->drm->dev, "failed to map buffer: %zd\n", err); 115 goto remove; 116 } 117 118 bo->size = err; 119 120 return 0; 121 122 remove: 123 drm_mm_remove_node(bo->mm); 124 free: 125 kfree(bo->mm); 126 return err; 127 } 128 129 static int tegra_bo_iommu_unmap(struct tegra_drm *tegra, struct tegra_bo *bo) 130 { 131 if (!bo->mm) 132 return 0; 133 134 iommu_unmap(tegra->domain, bo->paddr, bo->size); 135 drm_mm_remove_node(bo->mm); 136 kfree(bo->mm); 137 138 return 0; 139 } 140 141 static struct tegra_bo *tegra_bo_alloc_object(struct drm_device *drm, 142 size_t size) 143 { 144 struct tegra_bo *bo; 145 int err; 146 147 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 148 if (!bo) 149 return ERR_PTR(-ENOMEM); 150 151 host1x_bo_init(&bo->base, &tegra_bo_ops); 152 size = round_up(size, PAGE_SIZE); 153 154 err = drm_gem_object_init(drm, &bo->gem, size); 155 if (err < 0) 156 goto free; 157 158 err = drm_gem_create_mmap_offset(&bo->gem); 159 if (err < 0) 160 goto release; 161 162 return bo; 163 164 release: 165 drm_gem_object_release(&bo->gem); 166 free: 167 kfree(bo); 168 return ERR_PTR(err); 169 } 170 171 static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo) 172 { 173 if (bo->pages) { 174 drm_gem_put_pages(&bo->gem, bo->pages, true, true); 175 sg_free_table(bo->sgt); 176 kfree(bo->sgt); 177 } else if (bo->vaddr) { 178 dma_free_wc(drm->dev, bo->gem.size, bo->vaddr, bo->paddr); 179 } 180 } 181 182 static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo) 183 { 184 struct scatterlist *s; 185 unsigned int i; 186 187 bo->pages = drm_gem_get_pages(&bo->gem); 188 if (IS_ERR(bo->pages)) 189 return PTR_ERR(bo->pages); 190 191 bo->num_pages = bo->gem.size >> PAGE_SHIFT; 192 193 bo->sgt = drm_prime_pages_to_sg(bo->pages, bo->num_pages); 194 if (IS_ERR(bo->sgt)) 195 goto put_pages; 196 197 /* 198 * Fake up the SG table so that dma_sync_sg_for_device() can be used 199 * to flush the pages associated with it. 200 * 201 * TODO: Replace this by drm_clflash_sg() once it can be implemented 202 * without relying on symbols that are not exported. 203 */ 204 for_each_sg(bo->sgt->sgl, s, bo->sgt->nents, i) 205 sg_dma_address(s) = sg_phys(s); 206 207 dma_sync_sg_for_device(drm->dev, bo->sgt->sgl, bo->sgt->nents, 208 DMA_TO_DEVICE); 209 210 return 0; 211 212 put_pages: 213 drm_gem_put_pages(&bo->gem, bo->pages, false, false); 214 return PTR_ERR(bo->sgt); 215 } 216 217 static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo) 218 { 219 struct tegra_drm *tegra = drm->dev_private; 220 int err; 221 222 if (tegra->domain) { 223 err = tegra_bo_get_pages(drm, bo); 224 if (err < 0) 225 return err; 226 227 err = tegra_bo_iommu_map(tegra, bo); 228 if (err < 0) { 229 tegra_bo_free(drm, bo); 230 return err; 231 } 232 } else { 233 size_t size = bo->gem.size; 234 235 bo->vaddr = dma_alloc_wc(drm->dev, size, &bo->paddr, 236 GFP_KERNEL | __GFP_NOWARN); 237 if (!bo->vaddr) { 238 dev_err(drm->dev, 239 "failed to allocate buffer of size %zu\n", 240 size); 241 return -ENOMEM; 242 } 243 } 244 245 return 0; 246 } 247 248 struct tegra_bo *tegra_bo_create(struct drm_device *drm, size_t size, 249 unsigned long flags) 250 { 251 struct tegra_bo *bo; 252 int err; 253 254 bo = tegra_bo_alloc_object(drm, size); 255 if (IS_ERR(bo)) 256 return bo; 257 258 err = tegra_bo_alloc(drm, bo); 259 if (err < 0) 260 goto release; 261 262 if (flags & DRM_TEGRA_GEM_CREATE_TILED) 263 bo->tiling.mode = TEGRA_BO_TILING_MODE_TILED; 264 265 if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP) 266 bo->flags |= TEGRA_BO_BOTTOM_UP; 267 268 return bo; 269 270 release: 271 drm_gem_object_release(&bo->gem); 272 kfree(bo); 273 return ERR_PTR(err); 274 } 275 276 struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file, 277 struct drm_device *drm, 278 size_t size, 279 unsigned long flags, 280 u32 *handle) 281 { 282 struct tegra_bo *bo; 283 int err; 284 285 bo = tegra_bo_create(drm, size, flags); 286 if (IS_ERR(bo)) 287 return bo; 288 289 err = drm_gem_handle_create(file, &bo->gem, handle); 290 if (err) { 291 tegra_bo_free_object(&bo->gem); 292 return ERR_PTR(err); 293 } 294 295 drm_gem_object_unreference_unlocked(&bo->gem); 296 297 return bo; 298 } 299 300 static struct tegra_bo *tegra_bo_import(struct drm_device *drm, 301 struct dma_buf *buf) 302 { 303 struct tegra_drm *tegra = drm->dev_private; 304 struct dma_buf_attachment *attach; 305 struct tegra_bo *bo; 306 int err; 307 308 bo = tegra_bo_alloc_object(drm, buf->size); 309 if (IS_ERR(bo)) 310 return bo; 311 312 attach = dma_buf_attach(buf, drm->dev); 313 if (IS_ERR(attach)) { 314 err = PTR_ERR(attach); 315 goto free; 316 } 317 318 get_dma_buf(buf); 319 320 bo->sgt = dma_buf_map_attachment(attach, DMA_TO_DEVICE); 321 if (!bo->sgt) { 322 err = -ENOMEM; 323 goto detach; 324 } 325 326 if (IS_ERR(bo->sgt)) { 327 err = PTR_ERR(bo->sgt); 328 goto detach; 329 } 330 331 if (tegra->domain) { 332 err = tegra_bo_iommu_map(tegra, bo); 333 if (err < 0) 334 goto detach; 335 } else { 336 if (bo->sgt->nents > 1) { 337 err = -EINVAL; 338 goto detach; 339 } 340 341 bo->paddr = sg_dma_address(bo->sgt->sgl); 342 } 343 344 bo->gem.import_attach = attach; 345 346 return bo; 347 348 detach: 349 if (!IS_ERR_OR_NULL(bo->sgt)) 350 dma_buf_unmap_attachment(attach, bo->sgt, DMA_TO_DEVICE); 351 352 dma_buf_detach(buf, attach); 353 dma_buf_put(buf); 354 free: 355 drm_gem_object_release(&bo->gem); 356 kfree(bo); 357 return ERR_PTR(err); 358 } 359 360 void tegra_bo_free_object(struct drm_gem_object *gem) 361 { 362 struct tegra_drm *tegra = gem->dev->dev_private; 363 struct tegra_bo *bo = to_tegra_bo(gem); 364 365 if (tegra->domain) 366 tegra_bo_iommu_unmap(tegra, bo); 367 368 if (gem->import_attach) { 369 dma_buf_unmap_attachment(gem->import_attach, bo->sgt, 370 DMA_TO_DEVICE); 371 drm_prime_gem_destroy(gem, NULL); 372 } else { 373 tegra_bo_free(gem->dev, bo); 374 } 375 376 drm_gem_object_release(gem); 377 kfree(bo); 378 } 379 380 int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm, 381 struct drm_mode_create_dumb *args) 382 { 383 unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8); 384 struct tegra_drm *tegra = drm->dev_private; 385 struct tegra_bo *bo; 386 387 args->pitch = round_up(min_pitch, tegra->pitch_align); 388 args->size = args->pitch * args->height; 389 390 bo = tegra_bo_create_with_handle(file, drm, args->size, 0, 391 &args->handle); 392 if (IS_ERR(bo)) 393 return PTR_ERR(bo); 394 395 return 0; 396 } 397 398 int tegra_bo_dumb_map_offset(struct drm_file *file, struct drm_device *drm, 399 u32 handle, u64 *offset) 400 { 401 struct drm_gem_object *gem; 402 struct tegra_bo *bo; 403 404 gem = drm_gem_object_lookup(drm, file, handle); 405 if (!gem) { 406 dev_err(drm->dev, "failed to lookup GEM object\n"); 407 return -EINVAL; 408 } 409 410 bo = to_tegra_bo(gem); 411 412 *offset = drm_vma_node_offset_addr(&bo->gem.vma_node); 413 414 drm_gem_object_unreference_unlocked(gem); 415 416 return 0; 417 } 418 419 static int tegra_bo_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 420 { 421 struct drm_gem_object *gem = vma->vm_private_data; 422 struct tegra_bo *bo = to_tegra_bo(gem); 423 struct page *page; 424 pgoff_t offset; 425 int err; 426 427 if (!bo->pages) 428 return VM_FAULT_SIGBUS; 429 430 offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >> PAGE_SHIFT; 431 page = bo->pages[offset]; 432 433 err = vm_insert_page(vma, (unsigned long)vmf->virtual_address, page); 434 switch (err) { 435 case -EAGAIN: 436 case 0: 437 case -ERESTARTSYS: 438 case -EINTR: 439 case -EBUSY: 440 return VM_FAULT_NOPAGE; 441 442 case -ENOMEM: 443 return VM_FAULT_OOM; 444 } 445 446 return VM_FAULT_SIGBUS; 447 } 448 449 const struct vm_operations_struct tegra_bo_vm_ops = { 450 .fault = tegra_bo_fault, 451 .open = drm_gem_vm_open, 452 .close = drm_gem_vm_close, 453 }; 454 455 int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma) 456 { 457 struct drm_gem_object *gem; 458 struct tegra_bo *bo; 459 int ret; 460 461 ret = drm_gem_mmap(file, vma); 462 if (ret) 463 return ret; 464 465 gem = vma->vm_private_data; 466 bo = to_tegra_bo(gem); 467 468 if (!bo->pages) { 469 unsigned long vm_pgoff = vma->vm_pgoff; 470 471 vma->vm_flags &= ~VM_PFNMAP; 472 vma->vm_pgoff = 0; 473 474 ret = dma_mmap_wc(gem->dev->dev, vma, bo->vaddr, bo->paddr, 475 gem->size); 476 if (ret) { 477 drm_gem_vm_close(vma); 478 return ret; 479 } 480 481 vma->vm_pgoff = vm_pgoff; 482 } else { 483 pgprot_t prot = vm_get_page_prot(vma->vm_flags); 484 485 vma->vm_flags |= VM_MIXEDMAP; 486 vma->vm_flags &= ~VM_PFNMAP; 487 488 vma->vm_page_prot = pgprot_writecombine(prot); 489 } 490 491 return 0; 492 } 493 494 static struct sg_table * 495 tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach, 496 enum dma_data_direction dir) 497 { 498 struct drm_gem_object *gem = attach->dmabuf->priv; 499 struct tegra_bo *bo = to_tegra_bo(gem); 500 struct sg_table *sgt; 501 502 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); 503 if (!sgt) 504 return NULL; 505 506 if (bo->pages) { 507 struct scatterlist *sg; 508 unsigned int i; 509 510 if (sg_alloc_table(sgt, bo->num_pages, GFP_KERNEL)) 511 goto free; 512 513 for_each_sg(sgt->sgl, sg, bo->num_pages, i) 514 sg_set_page(sg, bo->pages[i], PAGE_SIZE, 0); 515 516 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) 517 goto free; 518 } else { 519 if (sg_alloc_table(sgt, 1, GFP_KERNEL)) 520 goto free; 521 522 sg_dma_address(sgt->sgl) = bo->paddr; 523 sg_dma_len(sgt->sgl) = gem->size; 524 } 525 526 return sgt; 527 528 free: 529 sg_free_table(sgt); 530 kfree(sgt); 531 return NULL; 532 } 533 534 static void tegra_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach, 535 struct sg_table *sgt, 536 enum dma_data_direction dir) 537 { 538 struct drm_gem_object *gem = attach->dmabuf->priv; 539 struct tegra_bo *bo = to_tegra_bo(gem); 540 541 if (bo->pages) 542 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); 543 544 sg_free_table(sgt); 545 kfree(sgt); 546 } 547 548 static void tegra_gem_prime_release(struct dma_buf *buf) 549 { 550 drm_gem_dmabuf_release(buf); 551 } 552 553 static void *tegra_gem_prime_kmap_atomic(struct dma_buf *buf, 554 unsigned long page) 555 { 556 return NULL; 557 } 558 559 static void tegra_gem_prime_kunmap_atomic(struct dma_buf *buf, 560 unsigned long page, 561 void *addr) 562 { 563 } 564 565 static void *tegra_gem_prime_kmap(struct dma_buf *buf, unsigned long page) 566 { 567 return NULL; 568 } 569 570 static void tegra_gem_prime_kunmap(struct dma_buf *buf, unsigned long page, 571 void *addr) 572 { 573 } 574 575 static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma) 576 { 577 return -EINVAL; 578 } 579 580 static void *tegra_gem_prime_vmap(struct dma_buf *buf) 581 { 582 struct drm_gem_object *gem = buf->priv; 583 struct tegra_bo *bo = to_tegra_bo(gem); 584 585 return bo->vaddr; 586 } 587 588 static void tegra_gem_prime_vunmap(struct dma_buf *buf, void *vaddr) 589 { 590 } 591 592 static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = { 593 .map_dma_buf = tegra_gem_prime_map_dma_buf, 594 .unmap_dma_buf = tegra_gem_prime_unmap_dma_buf, 595 .release = tegra_gem_prime_release, 596 .kmap_atomic = tegra_gem_prime_kmap_atomic, 597 .kunmap_atomic = tegra_gem_prime_kunmap_atomic, 598 .kmap = tegra_gem_prime_kmap, 599 .kunmap = tegra_gem_prime_kunmap, 600 .mmap = tegra_gem_prime_mmap, 601 .vmap = tegra_gem_prime_vmap, 602 .vunmap = tegra_gem_prime_vunmap, 603 }; 604 605 struct dma_buf *tegra_gem_prime_export(struct drm_device *drm, 606 struct drm_gem_object *gem, 607 int flags) 608 { 609 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 610 611 exp_info.ops = &tegra_gem_prime_dmabuf_ops; 612 exp_info.size = gem->size; 613 exp_info.flags = flags; 614 exp_info.priv = gem; 615 616 return dma_buf_export(&exp_info); 617 } 618 619 struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm, 620 struct dma_buf *buf) 621 { 622 struct tegra_bo *bo; 623 624 if (buf->ops == &tegra_gem_prime_dmabuf_ops) { 625 struct drm_gem_object *gem = buf->priv; 626 627 if (gem->dev == drm) { 628 drm_gem_object_reference(gem); 629 return gem; 630 } 631 } 632 633 bo = tegra_bo_import(drm, buf); 634 if (IS_ERR(bo)) 635 return ERR_CAST(bo); 636 637 return &bo->gem; 638 } 639